From 5b18f3212322c5ede907f05e0cc6924afbafc4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 3 Mar 2015 19:59:20 +0000 Subject: [PATCH 01/78] tell horizon where is ca cert --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 0ea9551efc..19acfa7c4b 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -400,6 +400,7 @@ - shell: openstack-service restart - lineinfile: dest=/etc/openstack-dashboard/local_settings regexp="^OPENSTACK_KEYSTONE_URL " line="OPENSTACK_KEYSTONE_URL = 'https://{{controller_hostname}}:5000/v2.0'" + - lineinfile: dest=/etc/openstack-dashboard/local_settings regexp="OPENSTACK_SSL_CACERT " line="OPENSTACK_SSL_CACERT = '/etc/pki/tls/certs/fed-cloud09-keystone.pem'" - service: name=httpd state=restarted From 8af53bd5593fd22835e93f768c01dbff7653ee53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 4 Mar 2015 12:38:37 +0000 Subject: [PATCH 02/78] move neutron to port 8696 and set haproxy as stunell --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 19acfa7c4b..e47247685c 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -125,6 +125,7 @@ - ansible-openstack-modules - openstack-keystone - openstack-neutron + - haproxy - yum: name=* state=latest - name: add ssl cert @@ -378,6 +379,7 @@ - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=ssl_cert_file value=/etc/pki/tls/certs/fed-cloud09-neutron.pem - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=ssl_key_file value=/etc/pki/tls/private/fed-cloud09-neutron.key - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=ssl_ca_file value=/etc/pki/tls/certs/fed-cloud09-neutron.pem + - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=bind_port value=8696 - ini_file: dest=/etc/neutron/api-paste.conf section="filter:authtoken" option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/neutron/api-paste.conf section="filter:authtoken" option=auth_protocol value=https @@ -397,6 +399,12 @@ - ini_file: dest=/etc/ceilometer/ceilometer.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - ini_file: dest=/etc/ceilometer/ceilometer.conf section=service_credentials option=os_auth_url value=https://{{ controller_hostname }}:35357/v2.0 + # enable stunell to neutron + - shell: cat /etc/pki/tls/certs/fed-cloud09-keystone.pem /etc/pki/tls/private/fed-cloud09.key > /etc/haproxy/fed-cloud09.combined + - file: path=/etc/haproxy/fed-cloud09.combined user=haproxy mode=644 + - copy: src={{ files }}/files/fedora-cloud/haproxy.cfg dest=/etc/haproxy/haproxy.cfg mode=644 owner=root group=root + - service: name=haproxy state=started enabled=yes + - shell: openstack-service restart - lineinfile: dest=/etc/openstack-dashboard/local_settings regexp="^OPENSTACK_KEYSTONE_URL " line="OPENSTACK_KEYSTONE_URL = 'https://{{controller_hostname}}:5000/v2.0'" From 178fc3859cf3d76ce09cd563b5a83b21ec3767b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 4 Mar 2015 12:57:19 +0000 Subject: [PATCH 03/78] typo --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index e47247685c..76a5675fc1 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -401,7 +401,7 @@ # enable stunell to neutron - shell: cat /etc/pki/tls/certs/fed-cloud09-keystone.pem /etc/pki/tls/private/fed-cloud09.key > /etc/haproxy/fed-cloud09.combined - - file: path=/etc/haproxy/fed-cloud09.combined user=haproxy mode=644 + - file: path=/etc/haproxy/fed-cloud09.combined owner=haproxy mode=644 - copy: src={{ files }}/files/fedora-cloud/haproxy.cfg dest=/etc/haproxy/haproxy.cfg mode=644 owner=root group=root - service: name=haproxy state=started enabled=yes From f55752b2c3e67116c96bdec4e28bf256cf2d80ee Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 4 Mar 2015 15:40:53 +0000 Subject: [PATCH 04/78] Rebuild mirrorlist-osuosl over on osuosl02 --- inventory/group_vars/mirrorlist | 2 +- inventory/host_vars/mirrorlist-osuosl.fedoraproject.org | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inventory/group_vars/mirrorlist b/inventory/group_vars/mirrorlist index 0884e9ed94..c3ee6b10d3 100644 --- a/inventory/group_vars/mirrorlist +++ b/inventory/group_vars/mirrorlist @@ -1,7 +1,7 @@ --- lvm_size: 20000 mem_size: 8192 -num_cpus: 4 +num_cpus: 6 # for systems that do not match the above - specify the same parameter in # the host_vars/$hostname file diff --git a/inventory/host_vars/mirrorlist-osuosl.fedoraproject.org b/inventory/host_vars/mirrorlist-osuosl.fedoraproject.org index fbea4c3dd5..31edce8e2d 100644 --- a/inventory/host_vars/mirrorlist-osuosl.fedoraproject.org +++ b/inventory/host_vars/mirrorlist-osuosl.fedoraproject.org @@ -4,7 +4,7 @@ gw: 140.211.169.193 dns: 140.211.166.130 ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-6 ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL6-x86_64/ -volgroup: /dev/vg_guests +volgroup: /dev/vg_server eth0_ip: 140.211.169.228 -vmhost: osuosl01.fedoraproject.org +vmhost: osuosl02.fedoraproject.org datacenter: osuosl From 1552cde45671e53f39fe639314cf48e4644b0d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 4 Mar 2015 15:42:12 +0000 Subject: [PATCH 05/78] add haproxy.conf which I missed in 8af53bd --- files/fedora-cloud/haproxy.cfg | 75 ++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 files/fedora-cloud/haproxy.cfg diff --git a/files/fedora-cloud/haproxy.cfg b/files/fedora-cloud/haproxy.cfg new file mode 100644 index 0000000000..45b7446c31 --- /dev/null +++ b/files/fedora-cloud/haproxy.cfg @@ -0,0 +1,75 @@ +#--------------------------------------------------------------------- +# Example configuration for a possible web application. See the +# full configuration options online. +# +# http://haproxy.1wt.eu/download/1.4/doc/configuration.txt +# +#--------------------------------------------------------------------- + +#--------------------------------------------------------------------- +# Global settings +#--------------------------------------------------------------------- +global + # to have these messages end up in /var/log/haproxy.log you will + # need to: + # + # 1) configure syslog to accept network log events. This is done + # by adding the '-r' option to the SYSLOGD_OPTIONS in + # /etc/sysconfig/syslog + # + # 2) configure local2 events to go to the /var/log/haproxy.log + # file. A line like the following can be added to + # /etc/sysconfig/syslog + # + # local2.* /var/log/haproxy.log + # + log 127.0.0.1 local2 + + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + daemon + + # turn on stats unix socket + stats socket /var/lib/haproxy/stats + +#--------------------------------------------------------------------- +# common defaults that all the 'listen' and 'backend' sections will +# use if not designated in their block +#--------------------------------------------------------------------- +defaults + mode http + log global + option httplog + option dontlognull + option http-server-close + option forwardfor except 127.0.0.0/8 + option redispatch + retries 3 + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s + maxconn 3000 + +#frontend keystone_public *:5000 +# default_backend keystone_public +#frontend keystone_admin *:35357 +# default_backend keystone_admin +frontend neutron + bind 0.0.0.0:9696 ssl crt /etc/haproxy/fed-cloud09.combined + default_backend neutron + +backend neutron + server neutron 127.0.0.1:8696 check + +backend keystone_public + server keystone_public 127.0.0.1:5000 check + +backend keystone_admin + server keystone_admin 127.0.0.1:35357 check From d3d2ec9b362fd9ba0cac8b3b915199bd858ad4ad Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 4 Mar 2015 15:46:36 +0000 Subject: [PATCH 06/78] Fix up this url --- inventory/host_vars/mirrorlist-osuosl.fedoraproject.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/mirrorlist-osuosl.fedoraproject.org b/inventory/host_vars/mirrorlist-osuosl.fedoraproject.org index 31edce8e2d..320c909e4c 100644 --- a/inventory/host_vars/mirrorlist-osuosl.fedoraproject.org +++ b/inventory/host_vars/mirrorlist-osuosl.fedoraproject.org @@ -2,8 +2,8 @@ nm: 255.255.255.192 gw: 140.211.169.193 dns: 140.211.166.130 -ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-6 -ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL6-x86_64/ +ks_url: http://209.132.181.6/repo/rhel/ks/kvm-rhel-6 +ks_repo: http://209.132.181.6/repo/rhel/RHEL6-x86_64/ volgroup: /dev/vg_server eth0_ip: 140.211.169.228 vmhost: osuosl02.fedoraproject.org From 4dc92de6d9be1ae076630a098bfc22f063c8759f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 4 Mar 2015 15:48:51 +0000 Subject: [PATCH 07/78] typo --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 76a5675fc1..0f70221815 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -402,7 +402,7 @@ # enable stunell to neutron - shell: cat /etc/pki/tls/certs/fed-cloud09-keystone.pem /etc/pki/tls/private/fed-cloud09.key > /etc/haproxy/fed-cloud09.combined - file: path=/etc/haproxy/fed-cloud09.combined owner=haproxy mode=644 - - copy: src={{ files }}/files/fedora-cloud/haproxy.cfg dest=/etc/haproxy/haproxy.cfg mode=644 owner=root group=root + - copy: src={{ files }}/fedora-cloud/haproxy.cfg dest=/etc/haproxy/haproxy.cfg mode=644 owner=root group=root - service: name=haproxy state=started enabled=yes - shell: openstack-service restart From 101a8fb00fb27fa039c5c8a8f1cd73ca4a989c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 4 Mar 2015 16:12:50 +0000 Subject: [PATCH 08/78] neutron do not use ssl, haproxy do that --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 0f70221815..8414117ee8 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -375,7 +375,7 @@ - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=nova_url value=https://{{ controller_hostname }}:8774/v2 - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=nova_admin_auth_url value=https://{{ controller_hostname }}:35357/v2.0 - - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=use_ssl value=True + - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=use_ssl value=False - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=ssl_cert_file value=/etc/pki/tls/certs/fed-cloud09-neutron.pem - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=ssl_key_file value=/etc/pki/tls/private/fed-cloud09-neutron.key - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=ssl_ca_file value=/etc/pki/tls/certs/fed-cloud09-neutron.pem From 621c373b1714f76b933b5b41253941586ea9136d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 4 Mar 2015 21:31:18 +0000 Subject: [PATCH 09/78] Fix varnish PURGE requests These are used by the wiki to purge updated pages Signed-off-by: Patrick Uiterwijk --- roles/varnish/files/proxy.vcl | 40 +++++++++++++---------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/roles/varnish/files/proxy.vcl b/roles/varnish/files/proxy.vcl index 549d0a1cc0..14e8846128 100644 --- a/roles/varnish/files/proxy.vcl +++ b/roles/varnish/files/proxy.vcl @@ -124,33 +124,23 @@ backend mirrormanager2 { } -#acl purge { -# "192.168.1.3"; -# "192.168.1.4"; -# "192.168.1.5"; -# "192.168.1.6"; -# "192.168.1.13"; -# "192.168.1.24"; -# "192.168.1.23"; -# "192.168.1.41"; -# "10.5.126.31"; -# "10.5.126.32"; -# "10.5.126.33"; -# "10.5.126.34"; -# "10.5.126.37"; -# "10.5.126.38"; -#} +acl purge { + "192.168.1.129"; // wiki01.vpn + "192.168.1.130"; // wiki02.vpn + "10.5.126.60"; // wiki01.stg + "10.5.126.63"; // wiki01 + "10.5.126.73"; // wiki02 + "10.5.126.23"; // lockbox01 + "192.168.1.58"; //lockbox01.vpn +} sub vcl_recv { -# if (req.request == "PURGE") { -# if (!client.ip ~ purge) { -# error 405 "Not allowed."; -# } -# if (req.url ~ "^http://") { -# set req.url = regsub(req.url, "http://localhost:6081",""); -# } -# purge_url(req.url); -# } + if (req.method == "PURGE") { + if (!client.ip ~ purge) { + return (synth(405, "Not allowed")); + } + return(purge); + } if (req.url ~ "^/wiki/") { set req.backend_hint = wiki; From 59e09864a06af368c016e7ba7411eb22c27a58f2 Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Thu, 5 Mar 2015 02:47:29 +0000 Subject: [PATCH 10/78] HRF has been deprecated for many months now. Nuke it. Signed-off-by: Ricky Elrod --- .../host_vars/hrf.cloud.fedoraproject.org | 10 ---- .../hosts/hrf.cloud.fedoraproject.org.yml | 59 ------------------- 2 files changed, 69 deletions(-) delete mode 100644 inventory/host_vars/hrf.cloud.fedoraproject.org delete mode 100644 playbooks/hosts/hrf.cloud.fedoraproject.org.yml diff --git a/inventory/host_vars/hrf.cloud.fedoraproject.org b/inventory/host_vars/hrf.cloud.fedoraproject.org deleted file mode 100644 index a76c48354f..0000000000 --- a/inventory/host_vars/hrf.cloud.fedoraproject.org +++ /dev/null @@ -1,10 +0,0 @@ ---- -instance_type: m1.small -image: "{{ el6_qcow_id }}" -keypair: fedora-admin-20130801 -security_group: webserver -zone: nova -hostbase: hrf- -public_ip: 209.132.184.156 -root_auth_users: codeblock -description: "hrf instance (https://github.com/fedora-infra/hrf)" diff --git a/playbooks/hosts/hrf.cloud.fedoraproject.org.yml b/playbooks/hosts/hrf.cloud.fedoraproject.org.yml deleted file mode 100644 index b3eafe2a8b..0000000000 --- a/playbooks/hosts/hrf.cloud.fedoraproject.org.yml +++ /dev/null @@ -1,59 +0,0 @@ -- name: check/create instance - hosts: hrf.cloud.fedoraproject.org - user: root - gather_facts: False - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - tasks: - - include: "{{ tasks }}/persistent_cloud.yml" - -- name: provision instance - hosts: hrf.cloud.fedoraproject.org - user: root - gather_facts: True - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - tasks: - - include: "{{ tasks }}/cloud_setup_basic.yml" - - handlers: - - include: "{{ handlers }}/restart_services.yml" - -- name: deploy hrf - hosts: hrf.cloud.fedoraproject.org - user: root - gather_facts: True - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - tasks: - - name: install deps - yum: state=present name={{ item }} - with_items: - - httpd - - python-flask - - python-fedmsg-meta-fedora-infrastructure - - fedmsg - - mod_wsgi - - htop # not a dep, but handy - - git - - fail2ban - - - name: enable fail2ban and start it - shell: chkconfig fail2ban on && service fail2ban start - - - name: clone the flask repo - git: repo=git://github.com/fedora-infra/hrf.git dest=/srv/www/hrf accept_hostkey=true - - - name: enable port 80 - command: lokkit -p '80:tcp' From ec23aaf08a05eb26e78a4e0294b603c188eda11d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 5 Mar 2015 00:41:37 +0000 Subject: [PATCH 11/78] Allow direct varnish access for internal hosts This allows internal that are in the purge acl to issue purge requests. Apache won't forward purge, since it doesn't know what that is. --- inventory/group_vars/proxies | 6 +++++- inventory/group_vars/proxies-stg | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/proxies b/inventory/group_vars/proxies index 3953b71e5a..c86440a74d 100644 --- a/inventory/group_vars/proxies +++ b/inventory/group_vars/proxies @@ -34,10 +34,14 @@ custom_rules: [ '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 209.132.181.102 --dport 873 -j ACCEPT', - # only allow varnish from localhost + # allow varnish from localhost '-A INPUT -p tcp -m tcp -s 127.0.0.1 --dport 6081 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 127.0.0.1 --dport 6082 -j ACCEPT', + # also allow varnish from internal for purge requests + '-A INPUT -p tcp -m tcp -s 192.168.1.0/24 --dport 6081 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 10.5.126.0/24 --dport 6081 -j ACCEPT', + # Allow koschei.cloud to talk to the inbound fedmsg relay. '-A INPUT -p tcp -m tcp --dport 9941 -s 209.132.184.151 -j ACCEPT', # Allow jenkins.cloud to talk to the inbound fedmsg relay. diff --git a/inventory/group_vars/proxies-stg b/inventory/group_vars/proxies-stg index 1b8fef2de4..2520ff1d48 100644 --- a/inventory/group_vars/proxies-stg +++ b/inventory/group_vars/proxies-stg @@ -33,10 +33,14 @@ custom_rules: [ '-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT', - # only allow varnish from localhost + # allow varnish from localhost '-A INPUT -p tcp -m tcp -s 127.0.0.1 --dport 6081 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 127.0.0.1 --dport 6082 -j ACCEPT', + # also allow varnish from internal for purge requests + '-A INPUT -p tcp -m tcp -s 192.168.1.0/24 --dport 6081 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 10.5.126.0/24 --dport 6081 -j ACCEPT', + # Allow koschei.cloud to talk to the inbound fedmsg relay. '-A INPUT -p tcp -m tcp --dport 9941 -s 209.132.184.151 -j ACCEPT', # Allow jenkins.cloud to talk to the inbound fedmsg relay. From cf5605fa8bccb9aba0287c01a65b3b44768a1396 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 5 Mar 2015 00:56:10 +0000 Subject: [PATCH 12/78] Fix mediawiki to determine proxies and send correct PURGE requests As commented: wgSquidServers is the set it sends a PURGE request to --- .../templates/LocalSettings.php.fp.j2 | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/roles/mediawiki/templates/LocalSettings.php.fp.j2 b/roles/mediawiki/templates/LocalSettings.php.fp.j2 index a8e0142598..2c46482f7a 100644 --- a/roles/mediawiki/templates/LocalSettings.php.fp.j2 +++ b/roles/mediawiki/templates/LocalSettings.php.fp.j2 @@ -322,9 +322,10 @@ $wgSkipSkins = array("chick", "cologneblue", "monobook", "myskin", "nostalgia", $wgSVGConverter = 'rsvg'; -#We use apache, but apparently it's the same difference +# This series of settings is used for reverse proxies $wgUseSquid = true; -$wgSquidServers = array( +# The SquidNoPurge setting is used to determine reverse proxies +$wgSquidServersNoPurge = array( {% if environment == "staging" %} # proxy01.stg "10.5.126.88", @@ -368,7 +369,32 @@ $wgSquidServers = array( "192.168.1.17", {% endif %} ); -$wgSquidServersNoPurge = array('127.0.0.1'); +# This setting is used to send PURGE requests to varnish on reverse proxies upon page changes +$wgSquidServers = array( +{% if environment == "staging" %} + # proxy01.stg + "10.5.126.88:6081", +{% else %} + # proxy01 + "10.5.126.52:6081", + # proxy02 + "192.168.1.12:6081", + # proxy03 + "192.168.1.7:6081", + # proxy04 + "192.168.1.14:6081", + # proxy06 + "192.168.1.63:6081", + # proxy07 + "192.168.1.52:6081", + # proxy08 + "192.168.1.78:6081", + # proxy09 + "192.168.1.15:6081", + # proxy10 + "10.5.126.51:6081", +{% endif %} +); $wgSquidMaxage = 432000; # Don't add rel="nofollow" From 35ee8445ec916a965271d398e9a222eabce631c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Thu, 5 Mar 2015 11:24:50 +0000 Subject: [PATCH 13/78] move cinder to ssl --- files/fedora-cloud/haproxy.cfg | 7 +++++++ playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/files/fedora-cloud/haproxy.cfg b/files/fedora-cloud/haproxy.cfg index 45b7446c31..e894669093 100644 --- a/files/fedora-cloud/haproxy.cfg +++ b/files/fedora-cloud/haproxy.cfg @@ -65,9 +65,16 @@ frontend neutron bind 0.0.0.0:9696 ssl crt /etc/haproxy/fed-cloud09.combined default_backend neutron +frontend cinder + bind 0.0.0.0:8776 ssl crt /etc/haproxy/fed-cloud09.combined + default_backend cinder + backend neutron server neutron 127.0.0.1:8696 check +backend cinder + server cinder 127.0.0.1:8776 check + backend keystone_public server keystone_public 127.0.0.1:5000 check diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 8414117ee8..5e5524d0b4 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -274,13 +274,13 @@ register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:8776/v1/%(tenant_id)s' --adminurl 'http://{{ controller_hostname }}:8776/v1/%(tenant_id)s' --internalurl 'http://{{ controller_hostname }}:8776/v1/%(tenant_id)s' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8776/v1/%(tenant_id)s' --adminurl 'https://{{ controller_hostname }}:8776/v1/%(tenant_id)s' --internalurl 'https://{{ controller_hostname }}:8776/v1/%(tenant_id)s' ) || true # cinderv2 - shell: source /root/keystonerc_admin && keystone service-list | grep 'cinderv2' | awk '{print $2}' register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:8776/v2/%(tenant_id)s' --adminurl 'http://{{ controller_hostname }}:8776/v2/%(tenant_id)s' --internalurl 'http://{{ controller_hostname }}:8776/v2/%(tenant_id)s' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8776/v2/%(tenant_id)s' --adminurl 'https://{{ controller_hostname }}:8776/v2/%(tenant_id)s' --internalurl 'https://{{ controller_hostname }}:8776/v2/%(tenant_id)s' ) || true # glance - shell: source /root/keystonerc_admin && keystone service-list | grep 'glance' | awk '{print $2}' register: SERVICE_ID @@ -364,6 +364,7 @@ - ini_file: dest=/etc/cinder/cinder.conf section=keystone_authtoken option=auth_protocol value=https - ini_file: dest=/etc/cinder/cinder.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - ini_file: dest=/etc/cinder/cinder.conf section=DEFAULT option=backup_swift_url value=https://{{ controller_hostname }}:8080/v1/AUTH_ + - ini_file: dest=/etc/cinder/cinder.conf section=DEFAULT option=osapi_volume_listen_port value=6776 - ini_file: dest=/etc/cinder/api-paste.conf section="filter:authtoken" option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/cinder/api-paste.conf section="filter:authtoken" option=auth_protocol value=https - ini_file: dest=/etc/cinder/api-paste.conf section="filter:authtoken" option=service_protocol value=https From dc156003afc79a7f5d0a83fead8b9d152cded5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Thu, 5 Mar 2015 14:24:08 +0000 Subject: [PATCH 14/78] move all openstack services to SSL --- files/fedora-cloud/haproxy.cfg | 37 +++++++++++++++++- .../fed-cloud09.cloud.fedoraproject.org.yml | 38 +++++++++++-------- roles/cloud_compute/tasks/main.yml | 16 ++++---- 3 files changed, 67 insertions(+), 24 deletions(-) diff --git a/files/fedora-cloud/haproxy.cfg b/files/fedora-cloud/haproxy.cfg index e894669093..e4011ae8d3 100644 --- a/files/fedora-cloud/haproxy.cfg +++ b/files/fedora-cloud/haproxy.cfg @@ -69,11 +69,46 @@ frontend cinder bind 0.0.0.0:8776 ssl crt /etc/haproxy/fed-cloud09.combined default_backend cinder +frontend swift + bind 0.0.0.0:8000 ssl crt /etc/haproxy/fed-cloud09.combined + default_backend swift + +frontend nova + bind 0.0.0.0:8774 ssl crt /etc/haproxy/fed-cloud09.combined + default_backend nova + +frontend ceilometer + bind 0.0.0.0:8777 ssl crt /etc/haproxy/fed-cloud09.combined + default_backend ceilometer + +frontend ec2 + bind 0.0.0.0:8773 ssl crt /etc/haproxy/fed-cloud09.combined + default_backend ec2 + +frontend glance + bind 0.0.0.0:9292 ssl crt /etc/haproxy/fed-cloud09.combined + default_backend glance + backend neutron server neutron 127.0.0.1:8696 check backend cinder - server cinder 127.0.0.1:8776 check + server cinder 127.0.0.1:6776 check + +backend swift + server swift 127.0.0.1:6000 check + +backend nova + server nova 127.0.0.1:6774 check + +backend ceilometer + server ceilometer 127.0.0.1:6777 check + +backend ec2 + server ec2 127.0.0.1:6773 check + +backend glance + server glance 127.0.0.1:7292 check backend keystone_public server keystone_public 127.0.0.1:5000 check diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 5e5524d0b4..7cee941546 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -268,7 +268,7 @@ register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:8777' --adminurl 'http://{{ controller_hostname }}:8777' --internalurl 'http://{{ controller_hostname }}:8777' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8777' --adminurl 'https://{{ controller_hostname }}:8777' --internalurl 'https://{{ controller_hostname }}:8777' ) || true # cinder - shell: source /root/keystonerc_admin && keystone service-list | grep 'cinder ' | awk '{print $2}' register: SERVICE_ID @@ -286,7 +286,7 @@ register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:9292' --adminurl 'http://{{ controller_hostname }}:9292' --internalurl 'http://{{ controller_hostname }}:9292' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:9292' --adminurl 'https://{{ controller_hostname }}:9292' --internalurl 'https://{{ controller_hostname }}:9292' ) || true # keystone --- !!!!! we need to use ADMIN_TOKEN here - shell: source /root/keystonerc_admin && keystone service-list | grep 'keystone' | awk '{print $2}' register: SERVICE_ID @@ -294,7 +294,7 @@ register: ENDPOINT_ID - ini_file: dest=/etc/keystone/keystone.conf section=ssl option=certfile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - ini_file: dest=/etc/keystone/keystone.conf section=ssl option=keyfile value=/etc/pki/tls/private/fed-cloud09-keystone.key - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone --os-token '{{ADMIN_TOKEN}}' --os-endpoint 'http://{{ controller_hostname }}:35357/v2.0' endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:5000/v2.0' --adminurl 'https://{{ controller_hostname }}:35357/v2.0' --internalurl 'https://{{ controller_hostname }}:5000/v2.0' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone --os-token '{{ADMIN_TOKEN}}' --os-endpoint 'https://{{ controller_hostname }}:35357/v2.0' endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:5000/v2.0' --adminurl 'https://{{ controller_hostname }}:35357/v2.0' --internalurl 'https://{{ controller_hostname }}:5000/v2.0' ) || true - ini_file: dest=/etc/keystone/keystone.conf section=ssl option=enable value=True - service: name=openstack-keystone state=restarted - lineinfile: dest=/root/keystonerc_admin regexp="^export OS_AUTH_URL" line="export OS_AUTH_URL=https://{{ controller_hostname }}:5000/v2.0/" @@ -304,37 +304,37 @@ register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:9696/' --adminurl 'http://{{ controller_hostname }}:9696/' --internalurl 'http://{{ controller_hostname }}:9696/' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:9696/' --adminurl 'https://{{ controller_hostname }}:9696/' --internalurl 'https://{{ controller_hostname }}:9696/' ) || true # nova - shell: source /root/keystonerc_admin && keystone service-list | grep 'nova ' | awk '{print $2}' register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:8774/v2/%(tenant_id)s' --adminurl 'http://{{ controller_hostname }}:8774/v2/%(tenant_id)s' --internalurl 'http://{{ controller_hostname }}:8774/v2/%(tenant_id)s' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8774/v2/%(tenant_id)s' --adminurl 'https://{{ controller_hostname }}:8774/v2/%(tenant_id)s' --internalurl 'https://{{ controller_hostname }}:8774/v2/%(tenant_id)s' ) || true # nova_ec2 - shell: source /root/keystonerc_admin && keystone service-list | grep 'nova_ec2' | awk '{print $2}' register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:8773/services/Cloud' --adminurl 'http://{{ controller_hostname }}:8773/services/Admin' --internalurl 'http://{{ controller_hostname }}:8773/services/Cloud' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8773/services/Cloud' --adminurl 'https://{{ controller_hostname }}:8773/services/Admin' --internalurl 'https://{{ controller_hostname }}:8773/services/Cloud' ) || true # novav3 - shell: source /root/keystonerc_admin && keystone service-list | grep 'novav3' | awk '{print $2}' register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:8774/v3' --adminurl 'http://{{ controller_hostname }}:8774/v3' --internalurl 'http://{{ controller_hostname }}:8774/v3' ) || true - # swift - it actually only listen on public port! - #- shell: source /root/keystonerc_admin && keystone service-list | grep 'swift ' | awk '{print $2}' - # register: SERVICE_ID - #- shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' - # register: ENDPOINT_ID - #- shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_private_ip }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{controller_hostname}}:8080/v1/AUTH_%(tenant_id)s' --adminurl 'http://{{controller_private_ip}}:8080' --internalurl 'http://{{controller_private_ip}}:8080/v1/AUTH_%(tenant_id)s' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8774/v3' --adminurl 'https://{{ controller_hostname }}:8774/v3' --internalurl 'https://{{ controller_hostname }}:8774/v3' ) || true + # swift + - shell: source /root/keystonerc_admin && keystone service-list | grep 'swift ' | awk '{print $2}' + register: SERVICE_ID + - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' + register: ENDPOINT_ID + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{controller_hostname}}:8080/v1/AUTH_%(tenant_id)s' --adminurl 'https://{{controller_private_ip}}:8080' --internalurl 'https://{{controller_private_ip}}:8080/v1/AUTH_%(tenant_id)s' ) || true # swift_s3 - shell: source /root/keystonerc_admin && keystone service-list | grep 'swift_s3' | awk '{print $2}' register: SERVICE_ID - shell: source /root/keystonerc_admin && keystone endpoint-list | grep {{SERVICE_ID.stdout}} | awk '{print $2}' register: ENDPOINT_ID - - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'http://{{ controller_hostname }}:8080' --adminurl 'http://{{ controller_hostname }}:8080' --internalurl 'http://{{ controller_hostname }}:8080' ) || true + - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8080' --adminurl 'https://{{ controller_hostname }}:8080' --internalurl 'https://{{ controller_hostname }}:8080' ) || true - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=https://{{ controller_hostname }}:6080/vnc_auto.html @@ -345,11 +345,14 @@ - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_auth_url value=https://{{ controller_hostname }}:35357/v2.0 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_url value=https://{{ controller_hostname }}:9696 + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=osapi_compute_listen_port value=6774 + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=ec2_listen_port value=6773 - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_protocol value=https - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_host value={{ controller_hostname }} - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem + - ini_file: dest=/etc/glance/glance-api.conf section=DEFAULT option=bind_port value=7292 - ini_file: dest=/etc/glance/glance-registry.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/glance/glance-registry.conf section=keystone_authtoken option=auth_host value={{ controller_hostname }} @@ -366,10 +369,11 @@ - ini_file: dest=/etc/cinder/cinder.conf section=DEFAULT option=backup_swift_url value=https://{{ controller_hostname }}:8080/v1/AUTH_ - ini_file: dest=/etc/cinder/cinder.conf section=DEFAULT option=osapi_volume_listen_port value=6776 - ini_file: dest=/etc/cinder/api-paste.conf section="filter:authtoken" option=auth_uri value=https://{{ controller_hostname }}:5000 + - ini_file: dest=/etc/cinder/api-paste.conf section="filter:authtoken" option=auth_host value={{ controller_hostname }} - ini_file: dest=/etc/cinder/api-paste.conf section="filter:authtoken" option=auth_protocol value=https - ini_file: dest=/etc/cinder/api-paste.conf section="filter:authtoken" option=service_protocol value=https - ini_file: dest=/etc/cinder/api-paste.conf section="filter:authtoken" option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - + - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_protocol value=https - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_host value={{ controller_hostname }} @@ -383,6 +387,7 @@ - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=bind_port value=8696 - ini_file: dest=/etc/neutron/api-paste.conf section="filter:authtoken" option=auth_uri value=https://{{ controller_hostname }}:5000 + - ini_file: dest=/etc/neutron/api-paste.conf section="filter:authtoken" option=auth_host value={{ controller_hostname }} - ini_file: dest=/etc/neutron/api-paste.conf section="filter:authtoken" option=auth_protocol value=https - ini_file: dest=/etc/neutron/api-paste.conf section="filter:authtoken" option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem @@ -393,12 +398,15 @@ - ini_file: dest=/etc/swift/proxy-server.conf section="filter:authtoken" option=auth_protocol value=https - ini_file: dest=/etc/swift/proxy-server.conf section="filter:authtoken" option=auth_host value={{ controller_hostname }} - ini_file: dest=/etc/swift/proxy-server.conf section="filter:authtoken" option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem + - ini_file: dest=/etc/swift/proxy-server.conf section=DEFAULT option=bind_port value=6080 + - ini_file: dest=/etc/swift/proxy-server.conf section=DEFAULT option=bind_ip value=127.0.0.1 - ini_file: dest=/etc/ceilometer/ceilometer.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/ceilometer/ceilometer.conf section=keystone_authtoken option=auth_protocol value=https - ini_file: dest=/etc/ceilometer/ceilometer.conf section=keystone_authtoken option=auth_host value={{ controller_hostname }} - ini_file: dest=/etc/ceilometer/ceilometer.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - ini_file: dest=/etc/ceilometer/ceilometer.conf section=service_credentials option=os_auth_url value=https://{{ controller_hostname }}:35357/v2.0 + - ini_file: dest=/etc/ceilometer/ceilometer.conf section=api value=6777 # enable stunell to neutron - shell: cat /etc/pki/tls/certs/fed-cloud09-keystone.pem /etc/pki/tls/private/fed-cloud09.key > /etc/haproxy/fed-cloud09.combined diff --git a/roles/cloud_compute/tasks/main.yml b/roles/cloud_compute/tasks/main.yml index 82646f5b19..6a94de18f3 100644 --- a/roles/cloud_compute/tasks/main.yml +++ b/roles/cloud_compute/tasks/main.yml @@ -32,8 +32,8 @@ - name: Set up db connection to controller ini_file: dest=/etc/nova/nova.conf section=database option=connection value=mysql://nova:{{NOVA_DBPASS}}@{{controller_private_ip}}/nova - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=auth_strategy value=keystone -- ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_uri value=https://{{controller_private_ip}}:5000 -- ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_host value={{controller_private_ip}} +- ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_uri value=https://{{controller_hostname}}:5000 +- ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_host value={{controller_hostname}} - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_protocol value=https - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_port value=35357 - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem @@ -55,9 +55,9 @@ - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vnc_enabled value=True - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vncserver_listen value=0.0.0.0 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vncserver_proxyclient_address value={{compute_private_ip}} -- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=http://{{controller_private_ip}}:6080/vnc_auto.html +- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=http://{{controller_hostname}}:6080/vnc_auto.html -- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_host value={{controller_private_ip}} +- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_host value={{controller_hostname}} - service: name=libvirtd state=started enabled=yes - service: name=messagebus state=started @@ -74,8 +74,8 @@ - openstack-neutron-openvswitch - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=auth_strategy value=keystone -- ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_uri value=https://{{controller_private_ip}}:5000 -- ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_host value={{controller_private_ip}} +- ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_uri value=https://{{controller_hostname}}:5000 +- ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_host value={{controller_hostname}} - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_protocol value=https - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_port value=35357 - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem @@ -115,13 +115,13 @@ - command: ovs-vsctl --may-exist add-br br-int - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=network_api_class value=nova.network.neutronv2.api.API -- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_url value=http://{{controller_private_ip}}:9696 +- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_url value=https://{{controller_hostname}}:9696 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_auth_strategy value=keystone - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_tenant_name value=services - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_username value=neutron - name: set neutron_admin_password ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_password value={{NEUTRON_PASS}} -- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_auth_url value=https://{{controller_private_ip}}:35357/v2.0 +- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_auth_url value=https://{{controller_hostname}}:35357/v2.0 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=linuxnet_interface_driver value=nova.network.linux_net.LinuxOVSInterfaceDriver - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=firewall_driver value=nova.virt.firewall.NoopFirewallDriver - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=security_group_api value=neutron From 46f83e9541857d4dfc393e4deb1ec772fe217053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Thu, 5 Mar 2015 15:20:26 +0000 Subject: [PATCH 15/78] recognize controller cert on compute nodes as valid --- roles/cloud_compute/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/cloud_compute/tasks/main.yml b/roles/cloud_compute/tasks/main.yml index 6a94de18f3..dd3d176252 100644 --- a/roles/cloud_compute/tasks/main.yml +++ b/roles/cloud_compute/tasks/main.yml @@ -17,6 +17,10 @@ notify: - restart network +- name: add cert to ca-bundle.crt so plain curl works + copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/ca-trust/source/anchors/ mode=600 owner=root group=root +- command: /usr/bin/update-ca-trust + - yum: state=present name=https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/rdo-release-icehouse-4.noarch.rpm # http://docs.openstack.org/icehouse/install-guide/install/yum/content/nova-compute.html From ca44b35c4e992028eece04e6af70b9608fa3b2d9 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 5 Mar 2015 16:51:13 +0000 Subject: [PATCH 16/78] taskotron: change buildslave's umask --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- .../taskotron/buildslave-configure/templates/buildbot.tac.j2 | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 6dda96ee2f..937cffddb8 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -182,7 +182,7 @@ factory.addStep(FileUpload(slavesrc="/var/log/taskotron/taskotron.log", masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'))) # change permissions for uuid dir on master to be accessible via http -factory.addStep(MasterShellCommand(command=["chmod", '-R', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/')])) +#factory.addStep(MasterShellCommand(command=["chmod", '-R', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/')])) {% else %} # capture the taskotron log diff --git a/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 b/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 index 7901989ab0..430336976e 100644 --- a/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 +++ b/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 @@ -52,7 +52,11 @@ passwd = '{{ qadevel_stg_buildslave_password }}' keepalive = 600 usepty = 0 +{% if deployment_type == 'dev' %} +umask = 022 +{% else %} umask = None +{% endif %} maxdelay = 300 s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, From db74cc47c69526a6b68ea79d35d5aa465fed58d8 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 5 Mar 2015 19:36:27 +0000 Subject: [PATCH 17/78] Increase nagios nrpe values for larger proxies --- inventory/host_vars/proxy01.phx2.fedoraproject.org | 4 ++-- inventory/host_vars/proxy02.fedoraproject.org | 4 ++-- inventory/host_vars/proxy06.fedoraproject.org | 4 ++-- inventory/host_vars/proxy10.phx2.fedoraproject.org | 3 +++ 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/inventory/host_vars/proxy01.phx2.fedoraproject.org b/inventory/host_vars/proxy01.phx2.fedoraproject.org index 8f965b0401..c9fbdcbd85 100644 --- a/inventory/host_vars/proxy01.phx2.fedoraproject.org +++ b/inventory/host_vars/proxy01.phx2.fedoraproject.org @@ -14,5 +14,5 @@ eth0_ip: 10.5.126.52 # This is consumed by the roles/fedora-web/main role sponsor: redhat -nrpe_procs_warn: 900 -nrpe_procs_crit: 1000 +nrpe_procs_warn: 1200 +nrpe_procs_crit: 1400 diff --git a/inventory/host_vars/proxy02.fedoraproject.org b/inventory/host_vars/proxy02.fedoraproject.org index 4d4625e146..21249f11fd 100644 --- a/inventory/host_vars/proxy02.fedoraproject.org +++ b/inventory/host_vars/proxy02.fedoraproject.org @@ -16,5 +16,5 @@ sponsor: internetx datacenter: internetx postfix_group: vpn -nrpe_procs_warn: 900 -nrpe_procs_crit: 1000 +nrpe_procs_warn: 1200 +nrpe_procs_crit: 1400 diff --git a/inventory/host_vars/proxy06.fedoraproject.org b/inventory/host_vars/proxy06.fedoraproject.org index e807acbd2d..055a56d143 100644 --- a/inventory/host_vars/proxy06.fedoraproject.org +++ b/inventory/host_vars/proxy06.fedoraproject.org @@ -16,5 +16,5 @@ sponsor: osuosl datacenter: osuosl postfix_group: vpn -nrpe_procs_warn: 900 -nrpe_procs_crit: 1000 +nrpe_procs_warn: 1200 +nrpe_procs_crit: 1400 diff --git a/inventory/host_vars/proxy10.phx2.fedoraproject.org b/inventory/host_vars/proxy10.phx2.fedoraproject.org index 2cdcead7a6..c16be55537 100644 --- a/inventory/host_vars/proxy10.phx2.fedoraproject.org +++ b/inventory/host_vars/proxy10.phx2.fedoraproject.org @@ -13,3 +13,6 @@ eth0_ip: 10.5.126.51 # This is consumed by the roles/fedora-web/main role sponsor: redhat + +nrpe_procs_warn: 1200 +nrpe_procs_crit: 1400 From 400b94bd5dedd8a8d6c7b67916f05fb9a5873dd5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 6 Mar 2015 09:22:22 +0100 Subject: [PATCH 18/78] Add current version of the pkgdb-sync-bugzilla script --- roles/pkgdb2/files/pkgdb-sync-bugzilla | 373 +++++++++++++++++++++++++ 1 file changed, 373 insertions(+) create mode 100755 roles/pkgdb2/files/pkgdb-sync-bugzilla diff --git a/roles/pkgdb2/files/pkgdb-sync-bugzilla b/roles/pkgdb2/files/pkgdb-sync-bugzilla new file mode 100755 index 0000000000..408019a7d5 --- /dev/null +++ b/roles/pkgdb2/files/pkgdb-sync-bugzilla @@ -0,0 +1,373 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# +# Copyright © 2013-2014 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, modify, +# copy, or redistribute it subject to the terms and conditions of the GNU +# General Public License v.2, or (at your option) any later version. This +# program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the GNU +# General Public License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public License and +# may only be used or replicated with the express permission of Red Hat, Inc. +# +# Red Hat Author(s): Toshio Kuratomi +# Author(s): Mike Watters +# Author(s): Pierre-Yves Chibon +# +''' +sync information from the packagedb into bugzilla + +This short script takes information about package onwership and imports it +into bugzilla. +''' + +## These two lines are needed to run on EL6 +__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4'] +import pkg_resources + +import argparse +import sys +import os +import itertools +import xmlrpclib +import codecs +import smtplib +import bugzilla +import requests +from email.Message import Message +from fedora.client.fas2 import AccountSystem + + +if 'PKGDB2_CONFIG' not in os.environ \ + and os.path.exists('/etc/pkgdb2/pkgdb2.cfg'): + print 'Using configuration file `/etc/pkgdb2/pkgdb2.cfg`' + os.environ['PKGDB2_CONFIG'] = '/etc/pkgdb2/pkgdb2.cfg' + + +try: + import pkgdb2 +except ImportError: + sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) + import pkgdb2 + + +BZSERVER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_URL') +BZUSER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_USER') +BZPASS = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_PASSWORD') +BZCOMPAPI = pkgdb2.APP.config.get('BUGZILLA_COMPONENT_API') +FASURL = pkgdb2.APP.config.get('PKGDB2_FAS_URL') +FASUSER = pkgdb2.APP.config.get('PKGDB2_FAS_USER') +FASPASS = pkgdb2.APP.config.get('PKGDB2_FAS_PASSWORD') +FASINSECURE = pkgdb2.APP.config.get('PKGDB2_FAS_INSECURE') +NOTIFYEMAIL = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_EMAIL') +PKGDBSERVER = pkgdb2.APP.config.get('SITE_URL') +DRY_RUN = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_DRY_RUN', False) + +# When querying for current info, take segments of 1000 packages a time +BZ_PKG_SEGMENT = 1000 + +class DataChangedError(Exception): + '''Raised when data we are manipulating changes while we're modifying it.''' + pass + +def segment(iterable, chunk, fill=None): + '''Collect data into `chunk` sized block''' + args = [iter(iterable)] * chunk + return itertools.izip_longest(*args, fillvalue=fill) + +class ProductCache(dict): + def __init__(self, bz, acls): + self.bz = bz + self.acls = acls + + # Ask bugzilla for a section of the pkglist. + # Save the information from the section that we want. + def __getitem__(self, key): + try: + return super(ProductCache, self).__getitem__(key) + except KeyError: + # We can only cache products we have pkgdb information for + if key not in self.acls: + raise + + if BZCOMPAPI == 'getcomponentsdetails': + # Old API -- in python-bugzilla. But with current server, this + # gives ProxyError + products = self.server.getcomponentsdetails(key) + elif BZCOMPAPI == 'component.get': + # Way that's undocumented in the partner-bugzilla api but works + # currently + pkglist = acls[key].keys() + products = {} + for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT): + # Format that bugzilla will understand. Strip None's that segment() pads + # out the final data segment() with + query = [dict(product=key, component=p) for p in pkg_segment if p is not None] + raw_data = self.bz._proxy.Component.get(dict(names=query)) + for package in raw_data['components']: + # Reformat data to be the same as what's returned from + # getcomponentsdetails + product = dict(initialowner=package['default_assignee'], + description=package['description'], + initialqacontact=package['default_qa_contact'], + initialcclist=package['default_cc']) + products[package['name'].lower()] = product + self[key] = products + + return super(ProductCache, self).__getitem__(key) + + +class Bugzilla(object): + + def __init__(self, bzServer, username, password, acls): + self.bzXmlRpcServer = bzServer + self.username = username + self.password = password + + self.server = bugzilla.Bugzilla( + url=self.bzXmlRpcServer, + user=self.username, + password=self.password) + self.productCache = ProductCache(self.server, acls) + + # Connect to the fedora account system + self.fas = AccountSystem( + base_url=FASURL, + username=FASUSER, + password=FASPASS) + self.userCache = self.fas.people_by_key( + key='username', + fields=['bugzilla_email']) + + def _get_bugzilla_email(self, username): + '''Return the bugzilla email address for a user. + + First looks in a cache for a username => bugzilla email. If not found, + reloads the cache from fas and tries again. + ''' + try: + return self.userCache[username]['bugzilla_email'].lower() + except KeyError: + if username.startswith('@'): + group = self.fas.group_by_name(username[1:]) + self.userCache[username] = { + 'bugzilla_email': group.mailing_list} + else: + person = self.fas.person_by_username(username) + bz_email = person.get('bugzilla_email', None) + if bz_email is None: + print '%s has no bugzilla email, valid account?' % username + else: + self.userCache[username] = {'bugzilla_email': bz_email} + return self.userCache[username]['bugzilla_email'].lower() + + def add_edit_component(self, package, collection, owner, description, + qacontact=None, cclist=None): + '''Add or update a component to have the values specified. + ''' + # Turn the cclist into something usable by bugzilla + if not cclist or 'people' not in cclist: + initialCCList = list() + else: + initialCCList = [ + self._get_bugzilla_email(cc) for cc in cclist['people']] + if 'groups' in cclist: + group_cc = [ + self._get_bugzilla_email(cc) for cc in cclist['groups']] + initialCCList.extend(group_cc) + + # Add owner to the cclist so comaintainers taking over a bug don't + # have to do this manually + owner = self._get_bugzilla_email(owner) + if owner not in initialCCList: + initialCCList.append(owner) + + # Lookup product + try: + product = self.productCache[collection] + except xmlrpclib.Fault as e: + # Output something useful in args + e.args = (e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError as e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + + pkgKey = package.lower() + if pkgKey in product: + # edit the package information + data = {} + + # Grab bugzilla email for things changable via xmlrpc + if qacontact: + qacontact = self._get_bugzilla_email(qacontact) + else: + qacontact = 'extras-qa@fedoraproject.org' + + # Check for changes to the owner, qacontact, or description + if product[pkgKey]['initialowner'] != owner: + data['initialowner'] = owner + + if product[pkgKey]['description'] != description: + data['description'] = description + if product[pkgKey]['initialqacontact'] != qacontact and ( + qacontact or product[pkgKey]['initialqacontact']): + data['initialqacontact'] = qacontact + + if len(product[pkgKey]['initialcclist']) != len(initialCCList): + data['initialcclist'] = initialCCList + else: + for ccMember in product[pkgKey]['initialcclist']: + if ccMember not in initialCCList: + data['initialcclist'] = initialCCList + break + + if data: + ### FIXME: initialowner has been made mandatory for some + # reason. Asking dkl why. + data['initialowner'] = owner + + # Changes occurred. Submit a request to change via xmlrpc + data['product'] = collection + data['component'] = package + if DRY_RUN: + print '[EDITCOMP] Changing via editComponent(' \ + '%s, %s, "xxxxx")' % (data, self.username) + print '[EDITCOMP] Former values: %s|%s|%s|%s' % ( + product[pkgKey]['initialowner'], + product[pkgKey]['description'], + product[pkgKey]['initialqacontact'], + product[pkgKey]['initialcclist']) + else: + try: + self.server.editcomponent(data) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError, e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + else: + # Add component + if qacontact: + qacontact = self._get_bugzilla_email(qacontact) + else: + qacontact = 'extras-qa@fedoraproject.org' + + data = { + 'product': collection, + 'component': package, + 'description': description, + 'initialowner': owner, + 'initialqacontact': qacontact + } + if initialCCList: + data['initialcclist'] = initialCCList + + if DRY_RUN: + print '[ADDCOMP] Adding new component AddComponent:(' \ + '%s, %s, "xxxxx")' % (data, self.username) + else: + try: + self.server.addcomponent(data) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + + +def send_email(fromAddress, toAddress, subject, message): + '''Send an email if there's an error. + + This will be replaced by sending messages to a log later. + ''' + msg = Message() + msg.add_header('To', ','.join(toAddress)) + msg.add_header('From', fromAddress) + msg.add_header('Subject', subject) + msg.set_payload(message) + smtp = smtplib.SMTP('bastion') + smtp.sendmail(fromAddress, toAddress, msg.as_string()) + smtp.quit() + + +if __name__ == '__main__': + sys.stdout = codecs.getwriter('utf-8')(sys.stdout) + + + parser = argparse.ArgumentParser( + description='Script syncing information between pkgdb and bugzilla' + ) + parser.add_argument( + '--debug', dest='debug', action='store_true', default=False, + help='Print the changes instead of making them in bugzilla') + + args = parser.parse_args() + + if args.debug: + DRY_RUN = True + + # Non-fatal errors to alert people about + errors = [] + + # Get bugzilla information from the package database + req = requests.get('%s/api/bugzilla/?format=json' % PKGDBSERVER) + acls = req.json()['bugzillaAcls'] + + # Initialize the connection to bugzilla + bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS, acls) + + for product in acls.keys(): + if product not in ('Fedora', 'Fedora EPEL'): + continue + for pkg in acls[product]: + if DRY_RUN: + print pkg + pkgInfo = acls[product][pkg] + try: + bugzilla.add_edit_component( + pkg, + product, + pkgInfo['owner'], + pkgInfo['summary'], + pkgInfo['qacontact'], + pkgInfo['cclist']) + except ValueError, e: + # A username didn't have a bugzilla address + errors.append(str(e.args)) + except DataChangedError, e: + # A Package or Collection was returned via xmlrpc but wasn't + # present when we tried to change it + errors.append(str(e.args)) + except xmlrpclib.ProtocolError, e: + # Unrecoverable and likely means that nothing is going to + # succeed. + errors.append(str(e.args)) + break + except xmlrpclib.Error, e: + # An error occurred in the xmlrpc call. Shouldn't happen but + # we better see what it is + errors.append(str(e.args)) + + # Send notification of errors + if errors: + #print '[DEBUG]', '\n'.join(errors) + send_email('accounts@fedoraproject.org', + NOTIFYEMAIL, + 'Errors while syncing bugzilla with the PackageDB', +''' +The following errors were encountered while updating bugzilla with information +from the Package Database. Please have the problems taken care of: + +%s +''' % ('\n'.join(errors),)) + + sys.exit(0) From 3cba755812d8fe1a3f4e57a35caaba735b3f5d9e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 6 Mar 2015 09:29:33 +0100 Subject: [PATCH 19/78] Install the hotfix of pkgdb-sync-bugzilla --- roles/pkgdb2/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/pkgdb2/tasks/main.yml b/roles/pkgdb2/tasks/main.yml index 4f882c2526..507b9357d0 100644 --- a/roles/pkgdb2/tasks/main.yml +++ b/roles/pkgdb2/tasks/main.yml @@ -52,6 +52,12 @@ notify: - restart apache +- name: HOTFIX pkgdb-sync-bugzilla script to notify the users + when: inventory_hostname.startswith('pkgdb02') + copy: src=pkgdb-sync-bugzilla dest=/usr/bin/pkgdb-sync-bugzilla mode=755 + tags: + - config + - name: Install the pkgdb cron jobs - sync bugzilla, update pkg info when: inventory_hostname.startswith('pkgdb02') template: src={{ item.file }} From 2198e5709cf3b0954da2cc36e3061a7870b8456c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 6 Mar 2015 09:30:58 +0100 Subject: [PATCH 20/78] Hotfix pkgdb-sync-bugzilla to automatically send notifications --- roles/pkgdb2/files/pkgdb-sync-bugzilla | 81 ++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/roles/pkgdb2/files/pkgdb-sync-bugzilla b/roles/pkgdb2/files/pkgdb-sync-bugzilla index 408019a7d5..03091d2a23 100755 --- a/roles/pkgdb2/files/pkgdb-sync-bugzilla +++ b/roles/pkgdb2/files/pkgdb-sync-bugzilla @@ -32,9 +32,12 @@ __requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4'] import pkg_resources import argparse +import datetime +import time import sys import os import itertools +import json import xmlrpclib import codecs import smtplib @@ -70,18 +73,24 @@ NOTIFYEMAIL = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_EMAIL') PKGDBSERVER = pkgdb2.APP.config.get('SITE_URL') DRY_RUN = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_DRY_RUN', False) +EMAIL_FROM = 'accounts@fedoraproject.org' +DATA_CACHE = '/var/tmp/pkgdb_sync_bz.json' + # When querying for current info, take segments of 1000 packages a time BZ_PKG_SEGMENT = 1000 + class DataChangedError(Exception): '''Raised when data we are manipulating changes while we're modifying it.''' pass + def segment(iterable, chunk, fill=None): '''Collect data into `chunk` sized block''' args = [iter(iterable)] * chunk return itertools.izip_longest(*args, fillvalue=fill) + class ProductCache(dict): def __init__(self, bz, acls): self.bz = bz @@ -284,7 +293,7 @@ class Bugzilla(object): raise -def send_email(fromAddress, toAddress, subject, message): +def send_email(fromAddress, toAddress, subject, message, ccAddress=None): '''Send an email if there's an error. This will be replaced by sending messages to a log later. @@ -293,12 +302,70 @@ def send_email(fromAddress, toAddress, subject, message): msg.add_header('To', ','.join(toAddress)) msg.add_header('From', fromAddress) msg.add_header('Subject', subject) + if ccAddress is not None: + msg.add_header('Cc', ','.join(ccAddress)) msg.set_payload(message) smtp = smtplib.SMTP('bastion') smtp.sendmail(fromAddress, toAddress, msg.as_string()) smtp.quit() +def notify_users(errors): + ''' Browse the list of errors and when we can retrieve the email + address, use it to notify the user about the issue. + ''' + tmpl_email = pkgdb2.APP.config.get('PKGDB_SYNC_BUGZILLA_EMAIL', None) + if not tmpl_email: + print 'No template email configured in the configuration file, '\ + 'no notification sent to the users' + return + + data = {} + if os.path.exists(DATA_CACHE): + try: + with open(DATA_CACHE) as stream: + data = json.load(stream) + except Exception as err: + print 'Could not read the json file at %s: \nError: %s' % ( + DATA_CACHE, err) + + new_data = {} + for error in errors: + notify_user = False + if 'The name ' in error and ' is not a valid username' in error: + user_email = error.split(' is not a valid username')[0].split( + 'The name ')[1].strip() + now = datetime.datetime.utcnow() + + # See if we already know about this user + if user_email in data and data[user_email]['last_update']: + last_update = datetime.datetime.fromtimestamp( + int(data[user_email]['last_update'])) + # Only notify users once per hour + if (now - last_update).seconds >= 3600: + notify_user = True + else: + new_data[user_email] = data[user_email] + elif not data or user_email not in data: + notify_user = True + + if notify_user: + send_email( + EMAIL_FROM, + [user_email], + subject='Please fix your bugzilla.redhat.com account', + message=tmpl_email, + ccAddress=NOTIFYEMAIL, + ) + + new_data[user_email] = { + 'last_update': time.mktime(now.timetuple()) + } + + with open(DATA_CACHE, 'w') as stream: + json.dump(new_data, stream) + + if __name__ == '__main__': sys.stdout = codecs.getwriter('utf-8')(sys.stdout) @@ -328,7 +395,7 @@ if __name__ == '__main__': for product in acls.keys(): if product not in ('Fedora', 'Fedora EPEL'): continue - for pkg in acls[product]: + for pkg in sorted(acls[product]): if DRY_RUN: print pkg pkgInfo = acls[product][pkg] @@ -355,12 +422,16 @@ if __name__ == '__main__': except xmlrpclib.Error, e: # An error occurred in the xmlrpc call. Shouldn't happen but # we better see what it is - errors.append(str(e.args)) + errors.append('%s -- %s' % (pkg, e.args[-1])) # Send notification of errors if errors: - #print '[DEBUG]', '\n'.join(errors) - send_email('accounts@fedoraproject.org', + if DRY_RUN: + print '[DEBUG]', '\n'.join(errors) + else: + notify_users(errors) + send_email( + EMAIL_FROM, NOTIFYEMAIL, 'Errors while syncing bugzilla with the PackageDB', ''' From 773ddbcaf2fc1c8ae08f79e651a963ad93aa24bc Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 6 Mar 2015 09:33:03 +0100 Subject: [PATCH 21/78] Adjust the pkgdb2 configuration to include the email sent to the users --- roles/pkgdb2/templates/pkgdb2.cfg | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/roles/pkgdb2/templates/pkgdb2.cfg b/roles/pkgdb2/templates/pkgdb2.cfg index c6be330d19..ad11f23712 100644 --- a/roles/pkgdb2/templates/pkgdb2.cfg +++ b/roles/pkgdb2/templates/pkgdb2.cfg @@ -115,3 +115,38 @@ SESSION_COOKIE_SECURE = True # Used by SESSION_COOKIE_PATH APPLICATION_ROOT = '/pkgdb/' + +# PkgDB sync bugzilla email +PKGDB_SYNC_BUGZILLA_EMAIL = """Greetings. + +You are receiving this email because there's a problem with your +bugzilla.redhat.com account. + +If you recently changed the email address associated with your +Fedora account in the Fedora Account System, it is now out of sync +with your bugzilla.redhat.com account. This leads to problems +with Fedora packages you own or are CC'ed on bug reports for. + +Please take one of the following actions: + +a) login to your old bugzilla.redhat.com account and change the email +address to match your current email in the Fedora account system. +https://bugzilla.redhat.com login, click preferences, account +information and enter new email address. + +b) Create a new account in bugzilla.redhat.com to match your +email listed in your Fedora account system account. +https://bugzilla.redhat.com/ click 'new account' and enter email +address. + +c) Change your Fedora Account System email to match your existing +bugzilla.redhat.com account. +https://admin.fedoraproject.org/accounts login, click on 'my account', +then 'edit' and change your email address. + +If you have questions or concerns, please let us know. + +Your prompt attention in this matter is appreciated. + +The Fedora admins. +""" From 9cfcff43f6a985c63ae52507d0dec20ab9f84410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 08:45:13 +0000 Subject: [PATCH 22/78] tell nova that glance use https --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 1 + roles/cloud_compute/tasks/main.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 7cee941546..aeb6a8504a 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -347,6 +347,7 @@ - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_url value=https://{{ controller_hostname }}:9696 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=osapi_compute_listen_port value=6774 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=ec2_listen_port value=6773 + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_api_servers value=https://{{ controller_hostname }}:9292 - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_protocol value=https diff --git a/roles/cloud_compute/tasks/main.yml b/roles/cloud_compute/tasks/main.yml index dd3d176252..5216f7af05 100644 --- a/roles/cloud_compute/tasks/main.yml +++ b/roles/cloud_compute/tasks/main.yml @@ -62,6 +62,8 @@ - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=http://{{controller_hostname}}:6080/vnc_auto.html - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_host value={{controller_hostname}} +- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_protocol value=https +- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_api_servers value=https://{{ controller_hostname }}:9292 - service: name=libvirtd state=started enabled=yes - service: name=messagebus state=started From d8abe657bedb9cd8b4488dc7710aca819836f4f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 09:41:04 +0000 Subject: [PATCH 23/78] provide ssl cert for novncproxy --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index aeb6a8504a..174c64369f 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -125,6 +125,7 @@ - ansible-openstack-modules - openstack-keystone - openstack-neutron + - openstack-nova-common - haproxy - yum: name=* state=latest @@ -144,7 +145,10 @@ copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/tls/certs/fed-cloud09-neutron.pem mode=600 owner=neutron group=root - name: add ssl key for neutron copy: src={{ private }}/files/openstack/fed-cloud09.key dest=/etc/pki/tls/private/fed-cloud09-neutron.key mode=600 owner=neutron group=root - + - name: add ssl cert for nova + copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/tls/certs/fed-cloud09-nova.pem mode=600 owner=nova group=root + - name: add ssl key for nova + copy: src={{ private }}/files/openstack/fed-cloud09.key dest=/etc/pki/tls/private/fed-cloud09-nova.key mode=600 owner=nova group=root # http://docs.openstack.org/trunk/install-guide/install/yum/content/basics-database-controller.html - name: install mysql packages @@ -348,6 +352,8 @@ - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=osapi_compute_listen_port value=6774 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=ec2_listen_port value=6773 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_api_servers value=https://{{ controller_hostname }}:9292 + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=cert value/etc/pki/tls/certs/fed-cloud09-nova.pem + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=key value=/etc/pki/tls/private/fed-cloud09-nova.key - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_protocol value=https From 8694e6f29b197082caf4404a9d91d1e124e5c833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 09:53:03 +0000 Subject: [PATCH 24/78] typo --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 174c64369f..19f5d084fe 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -352,7 +352,7 @@ - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=osapi_compute_listen_port value=6774 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=ec2_listen_port value=6773 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_api_servers value=https://{{ controller_hostname }}:9292 - - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=cert value/etc/pki/tls/certs/fed-cloud09-nova.pem + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=cert value=/etc/pki/tls/certs/fed-cloud09-nova.pem - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=key value=/etc/pki/tls/private/fed-cloud09-nova.key - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 From acc3ccf3908eb30e9ed0167b01126fafd541b3f9 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Fri, 6 Mar 2015 09:58:08 +0000 Subject: [PATCH 25/78] taskotron: change buildslave's umask, properly this time --- .../buildslave-configure/templates/buildbot.tac.j2 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 b/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 index 430336976e..8be2d8d1e8 100644 --- a/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 +++ b/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 @@ -52,11 +52,7 @@ passwd = '{{ qadevel_stg_buildslave_password }}' keepalive = 600 usepty = 0 -{% if deployment_type == 'dev' %} -umask = 022 -{% else %} -umask = None -{% endif %} +umask = 0022 maxdelay = 300 s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, From 348f2b070e1f1df783a8cb3e7e306e142c7fbecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 12:47:21 +0000 Subject: [PATCH 26/78] move non-ssl swift to 7080 so it does not confilect with novncproxy --- files/fedora-cloud/haproxy.cfg | 4 ++-- .../fed-cloud09.cloud.fedoraproject.org.yml | 17 ++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/files/fedora-cloud/haproxy.cfg b/files/fedora-cloud/haproxy.cfg index e4011ae8d3..5489f08186 100644 --- a/files/fedora-cloud/haproxy.cfg +++ b/files/fedora-cloud/haproxy.cfg @@ -70,7 +70,7 @@ frontend cinder default_backend cinder frontend swift - bind 0.0.0.0:8000 ssl crt /etc/haproxy/fed-cloud09.combined + bind 0.0.0.0:8080 ssl crt /etc/haproxy/fed-cloud09.combined default_backend swift frontend nova @@ -96,7 +96,7 @@ backend cinder server cinder 127.0.0.1:6776 check backend swift - server swift 127.0.0.1:6000 check + server swift 127.0.0.1:7080 check backend nova server nova 127.0.0.1:6774 check diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 19f5d084fe..745cccd8e0 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -145,10 +145,10 @@ copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/tls/certs/fed-cloud09-neutron.pem mode=600 owner=neutron group=root - name: add ssl key for neutron copy: src={{ private }}/files/openstack/fed-cloud09.key dest=/etc/pki/tls/private/fed-cloud09-neutron.key mode=600 owner=neutron group=root - - name: add ssl cert for nova - copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/tls/certs/fed-cloud09-nova.pem mode=600 owner=nova group=root - - name: add ssl key for nova - copy: src={{ private }}/files/openstack/fed-cloud09.key dest=/etc/pki/tls/private/fed-cloud09-nova.key mode=600 owner=nova group=root + #- name: add ssl cert for nova + # copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/tls/certs/fed-cloud09-nova.pem mode=600 owner=nova group=root + #- name: add ssl key for nova + # copy: src={{ private }}/files/openstack/fed-cloud09.key dest=/etc/pki/tls/private/fed-cloud09-nova.key mode=600 owner=nova group=root # http://docs.openstack.org/trunk/install-guide/install/yum/content/basics-database-controller.html - name: install mysql packages @@ -352,8 +352,11 @@ - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=osapi_compute_listen_port value=6774 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=ec2_listen_port value=6773 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_api_servers value=https://{{ controller_hostname }}:9292 - - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=cert value=/etc/pki/tls/certs/fed-cloud09-nova.pem - - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=key value=/etc/pki/tls/private/fed-cloud09-nova.key + #- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=cert value=/etc/pki/tls/certs/fed-cloud09-nova.pem + #- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=key value=/etc/pki/tls/private/fed-cloud09-nova.key + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_host value={{ controller_hostname }} + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=ssl_only value=False + - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 - ini_file: dest=/etc/glance/glance-api.conf section=keystone_authtoken option=auth_protocol value=https @@ -405,7 +408,7 @@ - ini_file: dest=/etc/swift/proxy-server.conf section="filter:authtoken" option=auth_protocol value=https - ini_file: dest=/etc/swift/proxy-server.conf section="filter:authtoken" option=auth_host value={{ controller_hostname }} - ini_file: dest=/etc/swift/proxy-server.conf section="filter:authtoken" option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - - ini_file: dest=/etc/swift/proxy-server.conf section=DEFAULT option=bind_port value=6080 + - ini_file: dest=/etc/swift/proxy-server.conf section=DEFAULT option=bind_port value=7080 - ini_file: dest=/etc/swift/proxy-server.conf section=DEFAULT option=bind_ip value=127.0.0.1 - ini_file: dest=/etc/ceilometer/ceilometer.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 From 7f12c2f0695f338b6fcda246c41682e8f76f9dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 13:07:26 +0000 Subject: [PATCH 27/78] do not use https for novnc --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 745cccd8e0..d62d78bc75 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -340,7 +340,7 @@ register: ENDPOINT_ID - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8080' --adminurl 'https://{{ controller_hostname }}:8080' --internalurl 'https://{{ controller_hostname }}:8080' ) || true - - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=https://{{ controller_hostname }}:6080/vnc_auto.html + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=http://{{ controller_hostname }}:6080/vnc_auto.html # set SSL for services - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 From 2f4005d46683c040a702fbf9118bb91305e6f918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 14:12:51 +0000 Subject: [PATCH 28/78] try if this script will work --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index d62d78bc75..87a2125c0d 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -545,7 +545,7 @@ - { name: red, email: 'red@fedoraproject.org', tenant: infrastructure, password: "{{red_password}}" } #- template: src={{ files }}/fedora-cloud/keystonerc_msuchy dest=/root/ owner=root mode=0600 #- shell: source /root/keystonerc_admin && keystone user-password-update --pass 'XXXX' msuchy - + - shell: source /root/keystonerc_admin && F=$(mktemp) && {{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}> "$F" && nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-list | ( grep msuchy || nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-add --pub_key "$F" msuchy ); rm -f "$F" ##### NETWORK #### # http://docs.openstack.org/havana/install-guide/install/apt/content/install-neutron.configure-networks.html From 40c2f245ff39cc4ff17a56e71c08815fc5c3d92e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 15:18:10 +0000 Subject: [PATCH 29/78] try this --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 87a2125c0d..749fd7bee0 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -545,7 +545,10 @@ - { name: red, email: 'red@fedoraproject.org', tenant: infrastructure, password: "{{red_password}}" } #- template: src={{ files }}/fedora-cloud/keystonerc_msuchy dest=/root/ owner=root mode=0600 #- shell: source /root/keystonerc_admin && keystone user-password-update --pass 'XXXX' msuchy - - shell: source /root/keystonerc_admin && F=$(mktemp) && {{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}> "$F" && nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-list | ( grep msuchy || nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-add --pub_key "$F" msuchy ); rm -f "$F" + - nova_keypair: state=present login_username=msuchy + login_password="{{msuchy_password}}" login_tenant_name=copr name=msuchy + public_key={{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }} + #- shell: source /root/keystonerc_admin && F=$(mktemp) && {{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}> "$F" && nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-list | ( grep msuchy || nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-add --pub_key "$F" msuchy ); rm -f "$F" ##### NETWORK #### # http://docs.openstack.org/havana/install-guide/install/apt/content/install-neutron.configure-networks.html From 9ed2df36f55b41cbfb6bf03f62855f2c3e92d145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 15:34:01 +0000 Subject: [PATCH 30/78] public key contains space --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 749fd7bee0..3964e90cca 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -545,9 +545,10 @@ - { name: red, email: 'red@fedoraproject.org', tenant: infrastructure, password: "{{red_password}}" } #- template: src={{ files }}/fedora-cloud/keystonerc_msuchy dest=/root/ owner=root mode=0600 #- shell: source /root/keystonerc_admin && keystone user-password-update --pass 'XXXX' msuchy - - nova_keypair: state=present login_username=msuchy + - nova_keypair: + state=present login_username=msuchy login_password="{{msuchy_password}}" login_tenant_name=copr name=msuchy - public_key={{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }} + public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}" #- shell: source /root/keystonerc_admin && F=$(mktemp) && {{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}> "$F" && nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-list | ( grep msuchy || nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-add --pub_key "$F" msuchy ); rm -f "$F" ##### NETWORK #### From 196ffef022bdfda247758054c71ac301a86b9f9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 6 Mar 2015 15:42:00 +0000 Subject: [PATCH 31/78] add auth_url --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 3964e90cca..265e6bf662 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -546,6 +546,7 @@ #- template: src={{ files }}/fedora-cloud/keystonerc_msuchy dest=/root/ owner=root mode=0600 #- shell: source /root/keystonerc_admin && keystone user-password-update --pass 'XXXX' msuchy - nova_keypair: + auth_url="https://{{controller_hostname}}:35357/v2.0" state=present login_username=msuchy login_password="{{msuchy_password}}" login_tenant_name=copr name=msuchy public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}" From ae9eaa4966ebf50c4359efcb53552531dd9e43eb Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 6 Mar 2015 21:48:13 +0000 Subject: [PATCH 32/78] Try and fix git branches sync to work with git packed refs. --- .../templates/pkgdb_sync_git_branches.py | 154 +++++++++++------- 1 file changed, 91 insertions(+), 63 deletions(-) diff --git a/roles/distgit/templates/pkgdb_sync_git_branches.py b/roles/distgit/templates/pkgdb_sync_git_branches.py index 8643165b71..9055e48a1a 100644 --- a/roles/distgit/templates/pkgdb_sync_git_branches.py +++ b/roles/distgit/templates/pkgdb_sync_git_branches.py @@ -26,8 +26,10 @@ the missing branches (or even the missing repo) """ +import multiprocessing.pool import os import subprocess +import time import requests @@ -56,6 +58,7 @@ GIT_FOLDER = '/srv/git/rpms/' MKBRANCH = '/usr/local/bin/mkbranch' SETUP_PACKAGE = '/usr/local/bin/setup_git_package' +THREADS = 20 VERBOSE = False @@ -67,7 +70,7 @@ class ProcessError(InternalError): pass -def _invoke(program, args): +def _invoke(program, args, cwd=None): '''Run a command and raise an exception if an error occurred. :arg program: The program to invoke @@ -79,63 +82,60 @@ def _invoke(program, args): cmdLine.extend(args) if VERBOSE: print ' '.join(cmdLine) + print ' in', cwd - if VERBOSE: - program = subprocess.Popen( - cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - else: - program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT) + program = subprocess.Popen( + cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) - retCode = program.wait() - if retCode != 0: + stdout, stderr = program.communicate() + + if program.returncode != 0: e = ProcessError() - e.returnCode = retCode + e.returnCode = program.returncode e.cmd = ' '.join(cmdLine) - if VERBOSE: - output = program.stdout.read() - e.message = 'Error, "%s" returned %s: %s' % ( - e.cmd, e.returnCode, output) - print e.message - else: - e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode) + e.cwd = cwd + e.message = 'Error, "%s" (in %r) returned %s\n stdout: %s\n stderr: %s' % ( + e.cmd, e.cwd, e.returnCode, stdout, stderr) + print e.message raise e + return stdout.strip() -def _create_branch(pkgname, branch): + +def _create_branch(pkgname, branch, existing_branches): '''Create a specific branch for a package. :arg pkgname: Name of the package to branch :arg branch: Name of the branch to create + :arg existing_branches: A list of the branches that already exist locally. ''' + branch = branch.replace('*', '').strip() if branch == 'master': print 'ERROR: Proudly refusing to create master branch. Invalid repo?' print 'INFO: Please check %s repo' % pkgname return - branchpath = os.path.join( - GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch) - if not os.path.exists(branchpath): - try: - _invoke(MKBRANCH, [branch, pkgname]) - except ProcessError, e: - if e.returnCode == 255: - # This is a warning, not an error - return - raise - finally: - fedmsg.publish( - topic='branch', - modname='git', - msg=dict( - agent='pkgdb', - name=pkgname, - branch=branch, - ), - ) - elif VERBOSE: - print 'Was asked to create branch %s of package %s, but it '\ - 'already exists' % (pkgname, branch) + if branch in existing_branches: + print 'ERROR: Refusing to create a branch %s that exists' % branch + return + + try: + _invoke(MKBRANCH, [branch, pkgname]) + fedmsg.publish( + topic='branch', + modname='git', + msg=dict( + agent='pkgdb', + name=pkgname, + branch=branch, + ), + ) + except ProcessError, e: + if e.returnCode == 255: + # This is a warning, not an error + return + raise def pkgdb_pkg_branch(): @@ -168,43 +168,48 @@ def get_git_branch(pkg): """ git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg) if not os.path.exists(git_folder): - print 'Could not find %s' % git_folder + if VERBOSE: + print 'Could not find %s' % git_folder return set() - head_folder = os.path.join(git_folder, 'refs', 'heads') - return set(os.listdir(head_folder)) + branches = [ + lclbranch.replace('*', '').strip() + for lclbranch in _invoke('git', ['branch'], cwd=git_folder).split('\n') + ] + return set(branches) -def branch_package(pkgname, branches): +def branch_package(pkgname, requested_branches, existing_branches): '''Create all the branches that are listed in the pkgdb for a package. :arg pkgname: The package to create branches for - :arg branches: The branches to creates + :arg requested_branches: The branches to creates + :arg existing_branches: A list of existing local branches ''' if VERBOSE: - print 'Fixing package %s for branches %s' % (pkgname, branches) + print 'Fixing package %s for branches %s' % (pkgname, requested_branches) # Create the devel branch if necessary - if not os.path.exists( - os.path.join(GIT_FOLDER, '%s.git/refs/heads/master' % pkgname)): + exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname)) + if not exists or 'master' not in existing_branches: _invoke(SETUP_PACKAGE, [pkgname]) - if 'master' in branches: - branches.remove('master') # SETUP_PACKAGE creates master - fedmsg.publish( - topic='branch', - modname='git', - msg=dict( - agent='pkgdb', - name=pkgname, - branch='master', - ), - ) + if 'master' in requested_branches: + requested_branches.remove('master') # SETUP_PACKAGE creates master + fedmsg.publish( + topic='branch', + modname='git', + msg=dict( + agent='pkgdb', + name=pkgname, + branch='master', + ), + ) # Create all the required branches for the package # Use the translated branch name until pkgdb falls inline - for branch in branches: - _create_branch(pkgname, branch) + for branch in requested_branches: + _create_branch(pkgname, branch, existing_branches) def main(): @@ -214,10 +219,14 @@ def main(): local_pkgs = set(os.listdir(GIT_FOLDER)) local_pkgs = set([it.replace('.git', '') for it in local_pkgs]) + if VERBOSE: + print "Found %i local packages" % len(local_pkgs) pkgdb_info = pkgdb_pkg_branch() pkgdb_pkgs = set(pkgdb_info.keys()) + if VERBOSE: + print "Found %i pkgdb packages" % len(pkgdb_pkgs) ## Commented out as we keep the git of retired packages while they won't ## show up in the information retrieved from pkgdb. @@ -230,19 +239,38 @@ def main(): print 'Some packages are present in pkgdb but not locally:' print ', '.join(sorted(pkgdb_pkgs - local_pkgs)) + + if VERBOSE: + print "Finding the lists of local branches for local repos." + start = time.time() + if THREADS == 1: + git_branch_lookup = map(get_git_branch, sorted(pkgdb_info)) + else: + threadpool = multiprocessing.pool.ThreadPool(processes=THREADS) + git_branch_lookup = threadpool.map(get_git_branch, sorted(pkgdb_info)) + + # Zip that list of results up into a lookup dict. + git_branch_lookup = dict(zip(sorted(pkgdb_info), git_branch_lookup)) + + if VERBOSE: + print "Found all local git branches in %0.2fs" % (time.time() - start) + tofix = set() for pkg in sorted(pkgdb_info): pkgdb_branches = pkgdb_info[pkg] - git_branches = get_git_branch(pkg) + git_branches = git_branch_lookup[pkg] diff = (pkgdb_branches - git_branches) if diff: print '%s missing: %s' % (pkg, ','.join(sorted(diff))) tofix.add(pkg) - branch_package(pkg, diff) + branch_package(pkg, diff, git_branches) if tofix: print 'Packages fixed (%s): %s' % ( len(tofix), ', '.join(sorted(tofix))) + else: + if VERBOSE: + print 'Didn\'t find any packages to fix.' if __name__ == '__main__': From ba55206c66c3bc11d1f189f03288fac297f77f9a Mon Sep 17 00:00:00 2001 From: David Gay Date: Fri, 6 Mar 2015 23:17:59 +0000 Subject: [PATCH 33/78] fedimg: vol sizes should be 6GB for now, to account for larger need of Atomic images --- roles/fedimg/vars/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/fedimg/vars/main.yml b/roles/fedimg/vars/main.yml index 387b395d48..1c1a0308cd 100644 --- a/roles/fedimg/vars/main.yml +++ b/roles/fedimg/vars/main.yml @@ -3,8 +3,8 @@ delete_images_on_failure: True aws_util_username: ec2-user aws_test_username: fedora -aws_util_volume_size: 3 -aws_test_volume_size: 3 +aws_util_volume_size: 6 +aws_test_volume_size: 6 # access_id and secret_key are in private vars aws_iam_profile: "arn:aws:iam::013116697141:user/oddshocks" aws_test: "/bin/true" From a424b52e2e59a2823146859ce00a7bdd7ae12553 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sat, 7 Mar 2015 17:30:13 +0000 Subject: [PATCH 34/78] Fix vnc on new cloud --- files/fedora-cloud/openstack-nova-novncproxy | 2 ++ .../fed-cloud09.cloud.fedoraproject.org.yml | 5 ++++- .../iptables/iptables.openstack-compute | 19 +++---------------- roles/cloud_compute/tasks/main.yml | 2 +- 4 files changed, 10 insertions(+), 18 deletions(-) create mode 100644 files/fedora-cloud/openstack-nova-novncproxy diff --git a/files/fedora-cloud/openstack-nova-novncproxy b/files/fedora-cloud/openstack-nova-novncproxy new file mode 100644 index 0000000000..98c73d8b4b --- /dev/null +++ b/files/fedora-cloud/openstack-nova-novncproxy @@ -0,0 +1,2 @@ +# You may specify other parameters to the nova-novncproxy here +OPTIONS="--novncproxy_host 209.132.184.9 --ssl_only" diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 265e6bf662..bb5de645c9 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -340,7 +340,10 @@ register: ENDPOINT_ID - shell: source /root/keystonerc_admin && keystone endpoint-list |grep {{SERVICE_ID.stdout}} |grep -v {{ controller_hostname }} && (keystone endpoint-delete {{ENDPOINT_ID.stdout}} && keystone endpoint-create --region 'RegionOne' --service {{SERVICE_ID.stdout}} --publicurl 'https://{{ controller_hostname }}:8080' --adminurl 'https://{{ controller_hostname }}:8080' --internalurl 'https://{{ controller_hostname }}:8080' ) || true - - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=http://{{ controller_hostname }}:6080/vnc_auto.html + # Setup sysconfig file for novncproxy + - copy: src={{ files }}/fedora-cloud/openstack-nova-novncproxy dest=/etc/sysconfig/openstack-nova-novncproxy mode=644 owner=root group=root + + - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=https://{{ controller_hostname }}:6080/vnc_auto.html # set SSL for services - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_uri value=https://{{ controller_hostname }}:5000 diff --git a/roles/base/templates/iptables/iptables.openstack-compute b/roles/base/templates/iptables/iptables.openstack-compute index 801ac1a4bb..3b7b11bf17 100644 --- a/roles/base/templates/iptables/iptables.openstack-compute +++ b/roles/base/templates/iptables/iptables.openstack-compute @@ -21,6 +21,9 @@ # openstack needs this to handle external ips right -A INPUT -p gre -m comment --comment "001 neutron tunnel port incoming neutron_tunnel" -j ACCEPT +# compute nodes need to allow vnc ports from the controller +-A INPUT -s 172.24.0.9 -p tcp -m tcp --dport 5900:6900 -j ACCEPT + # for nrpe - allow it from nocs -A INPUT -p tcp -m tcp --dport 5666 -s 192.168.1.10 -j ACCEPT # FIXME - this is the global nat-ip and we need the noc01-specific ip @@ -28,22 +31,6 @@ -A INPUT -p tcp -m tcp --dport 5666 -s 209.132.181.35 -j ACCEPT -A INPUT -p tcp -m tcp --dport 5666 -s 10.5.126.41 -j ACCEPT -{% if env != 'staging' and datacenter == 'phx2' and inventory_hostname not in groups['staging-friendly'] %} -# -# In the phx2 datacenter, both production and staging hosts are in the same -# subnet/vlan. We want production hosts to reject connectons from staging group hosts -# to prevent them from interfering with production. There are however a few hosts in -# production we have marked 'staging-friendly' that we do allow staging to talk to for -# mostly read-only data they need. -# -{% for host in groups['staging'] %} -{% if 'eth0_ip' in hostvars[host] %}# {{ host }} --A INPUT -s {{ hostvars[host]['eth0_ip'] }} -j REJECT --reject-with icmp-host-prohibited -{% else %}# {{ host }} has no 'eth0_ip' listed -{% endif %} -{% endfor %} -{% endif %} - # if the host/group defines incoming tcp_ports - allow them {% if tcp_ports is defined %} {% for port in tcp_ports %} diff --git a/roles/cloud_compute/tasks/main.yml b/roles/cloud_compute/tasks/main.yml index 5216f7af05..4470e8df36 100644 --- a/roles/cloud_compute/tasks/main.yml +++ b/roles/cloud_compute/tasks/main.yml @@ -59,7 +59,7 @@ - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vnc_enabled value=True - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vncserver_listen value=0.0.0.0 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vncserver_proxyclient_address value={{compute_private_ip}} -- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=http://{{controller_hostname}}:6080/vnc_auto.html +- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=https://{{controller_hostname}}:6080/vnc_auto.html - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_host value={{controller_hostname}} - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_protocol value=https From 1f6cefeaffc75ff56235a7d6fcbe55960b67e20b Mon Sep 17 00:00:00 2001 From: David Gay Date: Sat, 7 Mar 2015 23:22:30 +0000 Subject: [PATCH 35/78] copy host files from hotness01 for fedimg01, in an attempt to resolve a hosts/connection problem with Fedimg --- .../fedimg01.phx2.fedoraproject.org-hosts | 7 ++++ .../fedimg01.stg.phx2.fedoraproject.org-hosts | 37 ++++++------------- 2 files changed, 19 insertions(+), 25 deletions(-) create mode 100644 roles/hosts/files/fedimg01.phx2.fedoraproject.org-hosts diff --git a/roles/hosts/files/fedimg01.phx2.fedoraproject.org-hosts b/roles/hosts/files/fedimg01.phx2.fedoraproject.org-hosts new file mode 100644 index 0000000000..d251c6de16 --- /dev/null +++ b/roles/hosts/files/fedimg01.phx2.fedoraproject.org-hosts @@ -0,0 +1,7 @@ +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +10.5.126.51 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 fedoraproject.org admin.fedoraproject.org +10.5.126.23 infrastructure.fedoraproject.org +10.5.125.44 pkgs.fedoraproject.org +10.5.125.63 koji.fedoraproject.org +10.5.125.36 kojipkgs.fedoraproject.org diff --git a/roles/hosts/files/fedimg01.stg.phx2.fedoraproject.org-hosts b/roles/hosts/files/fedimg01.stg.phx2.fedoraproject.org-hosts index b55db99ec0..e9c13a6938 100644 --- a/roles/hosts/files/fedimg01.stg.phx2.fedoraproject.org-hosts +++ b/roles/hosts/files/fedimg01.stg.phx2.fedoraproject.org-hosts @@ -1,25 +1,12 @@ -127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 -::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 -10.5.126.23 infrastructure.fedoraproject.org -10.5.126.23 puppet.fedoraproject.org puppet puppet01 puppet01.phx2.fedoraproject.org -10.5.126.51 admin.fedoraproject.org -10.5.126.88 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 fedoraproject.org -10.5.126.83 pkgs.fedoraproject.org pkgs pkgs01 -10.5.126.81 app01.phx2.fedoraproject.org app1 app3 app5 bapp1 app01 app03 app05 bapp01 bapp02 -10.5.126.81 memcached04.phx2.fedoraproject.org memcached04 memcached03 memcached01 memcached02 -10.5.126.91 value03.phx2.fedoraproject.org value3 value03 -10.5.125.119 nfs01.phx2.fedoraproject.org nfs01 nfs1 -10.5.126.92 noc01.phx2.fedoraproject.org noc1 noc01 -10.5.126.82 app02.phx2.fedoraproject.org app2 app4 app02 app04 -10.5.126.85 db02.stg.phx2.fedoraproject.org db05 -10.5.126.204 db01.stg.phx2.fedoraproject.org db-koji01 -10.5.126.23 lockbox01.phx2.fedoraproject.org infrastructure.fedoraproject.org -10.5.125.63 koji.fedoraproject.org koji.stg.fedoraproject.org koji1 koji01 s390.koji.fedoraproject.org sparc.koji.fedoraproject.org arm.koji.fedoraproject.org ppc.koji.fedoraproject.org -10.5.126.27 archives.fedoraproject.org -10.5.126.86 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 -10.5.125.36 kojipkgs.fedoraproject.org -10.5.126.79 ask01.fedoraproject.org ask01 -10.5.126.60 packages01.phx2.fedoraproject.org pacakges01 packages02 -10.5.126.80 ask01.phx2.fedoraproject.org ask ask01 -209.132.183.72 bugzilla.redhat.com -10.5.126.61 paste01.phx2.fedoraproject.org paste01 paste02 +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +10.5.126.88 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 fedoraproject.org admin.fedoraproject.org admin.stg.fedoraproject.org +10.5.126.23 infrastructure.fedoraproject.org +10.5.125.44 pkgs.fedoraproject.org +10.5.126.81 app01.stg.fedoraproject.org bapp02 memcached01 memcached02 memcached03 memcached04 +10.5.126.85 db02.stg.fedoraproject.org db05 db-ask db-tahrir db-elections db-fedocal db-github2fedmsg db-kerneltest db-notifs nuancier_db db-pkgdb2 db-summershum tagger_db +10.5.126.204 db01.stg.phx2.fedoraproject.org db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org +10.5.126.86 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all +10.5.126.87 koji01.stg.phx2.fedoraproject.org koji.stg.fedoraproject.org koji01 kojipkgs kojipkgs.stg.phx2.fedoraproject.org kojipkgs.stg.fedoraproject.org +10.5.125.36 kojipkgs.fedoraproject.org From 77733a4366f709133e3d4262ef8f4a8e760f5bba Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sun, 8 Mar 2015 16:51:34 +0000 Subject: [PATCH 36/78] Add another spam site to be blocked in paste --- roles/paste/templates/config.php | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/paste/templates/config.php b/roles/paste/templates/config.php index 266e9f8bed..f79422121b 100644 --- a/roles/paste/templates/config.php +++ b/roles/paste/templates/config.php @@ -40,6 +40,7 @@ $sg_php_days = 90; $sg_php_score = 50; $sg_php_type = 2; $sg_censor = "vipshare.me +freepremium.info.tm filevis.com terafile.co lafiles.com From 5790f1cfac737f06d1aab88dc3128c08d82a6735 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 9 Mar 2015 07:51:15 +0000 Subject: [PATCH 37/78] buildmaster: fix permissions so artifacts are accessible via http --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 937cffddb8..c423a2f2b7 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -173,13 +173,16 @@ factory.addStep(ShellCommand(command=["runtask", '-i', {% if deployment_type == 'dev' %} +factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/')])) + # copy artifacts to master factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/'), masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/task_output'))) # copy taskotron log to master factory.addStep(FileUpload(slavesrc="/var/log/taskotron/taskotron.log", - masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'))) + masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'), + mode=0644)) # change permissions for uuid dir on master to be accessible via http #factory.addStep(MasterShellCommand(command=["chmod", '-R', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/')])) From 3109826a8eae12e60d52b10e00d42c60468957ff Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 9 Mar 2015 08:39:18 +0000 Subject: [PATCH 38/78] buildmaster: put artifacts into dir according to date --- .../templates/taskotron.master.cfg.j2 | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index c423a2f2b7..6dc1ce2a69 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -173,20 +173,28 @@ factory.addStep(ShellCommand(command=["runtask", '-i', {% if deployment_type == 'dev' %} -factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/')])) + +import datetime +from buildbot.process.properties import renderer + +@renderer +def today(props): + return datetime.datetime.now().strftime("%Y%m%d") + +artifactsdir = Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/%(kw:today)s', today=today) + +# create artifacts dir on master +factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', artifactsdir], + descriptionDone=['Create artifacs dir'])) # copy artifacts to master factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/'), - masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/task_output'))) + masterdest='%s/task_output' % artifactsdir)) # copy taskotron log to master -factory.addStep(FileUpload(slavesrc="/var/log/taskotron/taskotron.log", - masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'), +factory.addStep(FileUpload(slavesrc='/var/log/taskotron/taskotron.log', + masterdest='%s/taskotron.log' % artifactsdir, mode=0644)) - -# change permissions for uuid dir on master to be accessible via http -#factory.addStep(MasterShellCommand(command=["chmod", '-R', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/')])) - {% else %} # capture the taskotron log factory.addStep(ShellCommand(command=["cat", "/var/log/taskotron/taskotron.log"], name="cat_log", From 017d95c14c4073ae772f5c844313b5fce17a986a Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 9 Mar 2015 08:48:30 +0000 Subject: [PATCH 39/78] buildmaster: fix creating artifacts dir --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 6dc1ce2a69..e82144bdce 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -184,7 +184,7 @@ def today(props): artifactsdir = Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/%(kw:today)s', today=today) # create artifacts dir on master -factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', artifactsdir], +factory.addStep(MasterShellCommand(command=["mkdir", '-p', '-m', '0755', artifactsdir], descriptionDone=['Create artifacs dir'])) # copy artifacts to master From 50c29fad0dcaeb161a09524773e997cd0d9184e4 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 9 Mar 2015 09:30:43 +0000 Subject: [PATCH 40/78] buildmaster: fix creating artifacts dir, again --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index e82144bdce..51c1b1fb08 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -181,7 +181,7 @@ from buildbot.process.properties import renderer def today(props): return datetime.datetime.now().strftime("%Y%m%d") -artifactsdir = Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/%(kw:today)s', today=today) +artifactsdir = Interpolate('{{ public_artifacts_dir }}/%(kw:today)s/%(prop:uuid)s', today=today) # create artifacts dir on master factory.addStep(MasterShellCommand(command=["mkdir", '-p', '-m', '0755', artifactsdir], From 23975cd9fb1a1ab76216eac13797bdedb6df2ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 09:50:55 +0000 Subject: [PATCH 41/78] upload ssh keys for users This does work only on initial run. Once user reset his password it will fail. public_key needs to be in items, because variables can not be recursively used e.g.: {{ lookup(... {{item.name}}") }} --- .../fed-cloud09.cloud.fedoraproject.org.yml | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index bb5de645c9..6f17eafa82 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -546,13 +546,34 @@ - { name: codeblock, email: 'codeblock@elrod.me', tenant: infrastructure, password: "{{codeblock_password}}" } - { name: msuchy, email: 'msuchy@redhat.com', tenant: copr, password: "{{msuchy_password}}" } - { name: red, email: 'red@fedoraproject.org', tenant: infrastructure, password: "{{red_password}}" } - #- template: src={{ files }}/fedora-cloud/keystonerc_msuchy dest=/root/ owner=root mode=0600 - #- shell: source /root/keystonerc_admin && keystone user-password-update --pass 'XXXX' msuchy - - nova_keypair: + - name: upload SSH keys for + nova_keypair: auth_url="https://{{controller_hostname}}:35357/v2.0" - state=present login_username=msuchy - login_password="{{msuchy_password}}" login_tenant_name=copr name=msuchy - public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}" + state=present login_username={{ item.name }} + login_password="{{ item.password }}" login_tenant_name={{item.tenant} name={{ item.name }} + public_key="{{ item.public_key }}" + ignore_errors: yes + no_log: True + with_items: + - { name: kevin, email: 'kevin@fedoraproject.org', tenant: infrastructure, password: "{{kevin_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas kevin') }}" } + - { name: laxathom, email: 'laxathom@fedoraproject.org', tenant: infrastructure, password: "{{laxathom_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas laxathom') }}" } + - { name: samkottler, email: 'samkottler@fedoraproject.org', tenant: infrastructure, password: "{{samkottler_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas samkottler') }}" } + - { name: puiterwijk, email: 'puiterwijk@fedoraproject.org', tenant: infrastructure, password: "{{puiterwijk_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas puiterwijk') }}" } + - { name: mattdm, email: 'mattdm@fedoraproject.org', tenant: infrastructure, password: "{{mattdm_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas mattdm') }}" } + - { name: tflink, email: 'tflink@fedoraproject.org', tenant: qa, password: "{{tflink_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas tflink') }}" } + - { name: copr, email: 'admin@fedoraproject.org', tenant: copr, password: "{{copr_password}}", public_key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCeTO0ddXuhDZYM9HyM0a47aeV2yIVWhTpddrQ7/RAIs99XyrsicQLABzmdMBfiZnP0FnHBF/e+2xEkT8hHJpX6bX81jjvs2bb8KP18Nh8vaXI3QospWrRygpu1tjzqZT0Llh4ZVFscum8TrMw4VWXclzdDw6x7csCBjSttqq8F3iTJtQ9XM9/5tCAAOzGBKJrsGKV1CNIrfUo5CSzY+IUVIr8XJ93IB2ZQVASK34T/49egmrWlNB32fqAbDMC+XNmobgn6gO33Yq5Ly7Dk4kqTUx2TEaqDkZfhsVu0YcwV81bmqsltRvpj6bIXrEoMeav7nbuqKcPLTxWEY/2icePF" } +# - { name: twisted, email: 'buildbot@twistedmatrix.com', tenant: pythonbots, password: "{{twisted_password}}", public_key="" } + - { name: ausil, email: 'dennis@ausil.us', tenant: infrastructure, password: "{{ausil_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas ausil') }}" } + - { name: anthomas, email: 'anthomas@redhat.com', tenant: cloudintern, password: "{{anthomas_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas anthomas') }}" } + - { name: jskladan, email: 'jskladan@redhat.com', tenant: qa, password: "{{jskladan_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas jskladan') }}" } + - { name: gholms, email: 'gholms@fedoraproject.org', tenant: cloudintern, password: "{{gholms_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas gholms') }}" } +# - { name: cockpit, email: 'walters@redhat.com', tenant: scratch, password: "{{cockpit_password}}", public_key="" } + - { name: nb, email: 'nb@fedoraproject.org', tenant: infrastructure, password: "{{nb_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas nb') }}" } + - { name: pingou, email: 'pingou@pingoured.fr', tenant: infrastructure, password: "{{pingou_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas pingou') }}" } + - { name: codeblock, email: 'codeblock@elrod.me', tenant: infrastructure, password: "{{codeblock_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas codeblock') }}" } + - { name: msuchy, email: 'msuchy@redhat.com', tenant: copr, password: "{{msuchy_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}" } + - { name: red, email: 'red@fedoraproject.org', tenant: infrastructure, password: "{{red_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas red') }}" } + #- shell: source /root/keystonerc_admin && F=$(mktemp) && {{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}> "$F" && nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-list | ( grep msuchy || nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-add --pub_key "$F" msuchy ); rm -f "$F" ##### NETWORK #### From 7cc7b7a476f82d2487fff869b432fb7145477f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 09:55:27 +0000 Subject: [PATCH 42/78] typo --- .../fed-cloud09.cloud.fedoraproject.org.yml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 6f17eafa82..92844505b9 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -555,24 +555,24 @@ ignore_errors: yes no_log: True with_items: - - { name: kevin, email: 'kevin@fedoraproject.org', tenant: infrastructure, password: "{{kevin_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas kevin') }}" } - - { name: laxathom, email: 'laxathom@fedoraproject.org', tenant: infrastructure, password: "{{laxathom_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas laxathom') }}" } - - { name: samkottler, email: 'samkottler@fedoraproject.org', tenant: infrastructure, password: "{{samkottler_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas samkottler') }}" } - - { name: puiterwijk, email: 'puiterwijk@fedoraproject.org', tenant: infrastructure, password: "{{puiterwijk_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas puiterwijk') }}" } - - { name: mattdm, email: 'mattdm@fedoraproject.org', tenant: infrastructure, password: "{{mattdm_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas mattdm') }}" } - - { name: tflink, email: 'tflink@fedoraproject.org', tenant: qa, password: "{{tflink_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas tflink') }}" } - - { name: copr, email: 'admin@fedoraproject.org', tenant: copr, password: "{{copr_password}}", public_key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCeTO0ddXuhDZYM9HyM0a47aeV2yIVWhTpddrQ7/RAIs99XyrsicQLABzmdMBfiZnP0FnHBF/e+2xEkT8hHJpX6bX81jjvs2bb8KP18Nh8vaXI3QospWrRygpu1tjzqZT0Llh4ZVFscum8TrMw4VWXclzdDw6x7csCBjSttqq8F3iTJtQ9XM9/5tCAAOzGBKJrsGKV1CNIrfUo5CSzY+IUVIr8XJ93IB2ZQVASK34T/49egmrWlNB32fqAbDMC+XNmobgn6gO33Yq5Ly7Dk4kqTUx2TEaqDkZfhsVu0YcwV81bmqsltRvpj6bIXrEoMeav7nbuqKcPLTxWEY/2icePF" } -# - { name: twisted, email: 'buildbot@twistedmatrix.com', tenant: pythonbots, password: "{{twisted_password}}", public_key="" } - - { name: ausil, email: 'dennis@ausil.us', tenant: infrastructure, password: "{{ausil_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas ausil') }}" } - - { name: anthomas, email: 'anthomas@redhat.com', tenant: cloudintern, password: "{{anthomas_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas anthomas') }}" } - - { name: jskladan, email: 'jskladan@redhat.com', tenant: qa, password: "{{jskladan_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas jskladan') }}" } - - { name: gholms, email: 'gholms@fedoraproject.org', tenant: cloudintern, password: "{{gholms_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas gholms') }}" } -# - { name: cockpit, email: 'walters@redhat.com', tenant: scratch, password: "{{cockpit_password}}", public_key="" } - - { name: nb, email: 'nb@fedoraproject.org', tenant: infrastructure, password: "{{nb_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas nb') }}" } - - { name: pingou, email: 'pingou@pingoured.fr', tenant: infrastructure, password: "{{pingou_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas pingou') }}" } - - { name: codeblock, email: 'codeblock@elrod.me', tenant: infrastructure, password: "{{codeblock_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas codeblock') }}" } - - { name: msuchy, email: 'msuchy@redhat.com', tenant: copr, password: "{{msuchy_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}" } - - { name: red, email: 'red@fedoraproject.org', tenant: infrastructure, password: "{{red_password}}", public_key="{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas red') }}" } + - { name: kevin, email: 'kevin@fedoraproject.org', tenant: infrastructure, password: "{{kevin_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas kevin') }}" } + - { name: laxathom, email: 'laxathom@fedoraproject.org', tenant: infrastructure, password: "{{laxathom_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas laxathom') }}" } + - { name: samkottler, email: 'samkottler@fedoraproject.org', tenant: infrastructure, password: "{{samkottler_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas samkottler') }}" } + - { name: puiterwijk, email: 'puiterwijk@fedoraproject.org', tenant: infrastructure, password: "{{puiterwijk_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas puiterwijk') }}" } + - { name: mattdm, email: 'mattdm@fedoraproject.org', tenant: infrastructure, password: "{{mattdm_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas mattdm') }}" } + - { name: tflink, email: 'tflink@fedoraproject.org', tenant: qa, password: "{{tflink_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas tflink') }}" } + - { name: copr, email: 'admin@fedoraproject.org', tenant: copr, password: "{{copr_password}}", public_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCeTO0ddXuhDZYM9HyM0a47aeV2yIVWhTpddrQ7/RAIs99XyrsicQLABzmdMBfiZnP0FnHBF/e+2xEkT8hHJpX6bX81jjvs2bb8KP18Nh8vaXI3QospWrRygpu1tjzqZT0Llh4ZVFscum8TrMw4VWXclzdDw6x7csCBjSttqq8F3iTJtQ9XM9/5tCAAOzGBKJrsGKV1CNIrfUo5CSzY+IUVIr8XJ93IB2ZQVASK34T/49egmrWlNB32fqAbDMC+XNmobgn6gO33Yq5Ly7Dk4kqTUx2TEaqDkZfhsVu0YcwV81bmqsltRvpj6bIXrEoMeav7nbuqKcPLTxWEY/2icePF" } +# - { name: twisted, email: 'buildbot@twistedmatrix.com', tenant: pythonbots, password: "{{twisted_password}}", public_key: "" } + - { name: ausil, email: 'dennis@ausil.us', tenant: infrastructure, password: "{{ausil_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas ausil') }}" } + - { name: anthomas, email: 'anthomas@redhat.com', tenant: cloudintern, password: "{{anthomas_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas anthomas') }}" } + - { name: jskladan, email: 'jskladan@redhat.com', tenant: qa, password: "{{jskladan_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas jskladan') }}" } + - { name: gholms, email: 'gholms@fedoraproject.org', tenant: cloudintern, password: "{{gholms_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas gholms') }}" } +# - { name: cockpit, email: 'walters@redhat.com', tenant: scratch, password: "{{cockpit_password}}", public_key: "" } + - { name: nb, email: 'nb@fedoraproject.org', tenant: infrastructure, password: "{{nb_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas nb') }}" } + - { name: pingou, email: 'pingou@pingoured.fr', tenant: infrastructure, password: "{{pingou_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas pingou') }}" } + - { name: codeblock, email: 'codeblock@elrod.me', tenant: infrastructure, password: "{{codeblock_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas codeblock') }}" } + - { name: msuchy, email: 'msuchy@redhat.com', tenant: copr, password: "{{msuchy_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}" } + - { name: red, email: 'red@fedoraproject.org', tenant: infrastructure, password: "{{red_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas red') }}" } #- shell: source /root/keystonerc_admin && F=$(mktemp) && {{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas msuchy') }}> "$F" && nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-list | ( grep msuchy || nova --os-username msuchy --os-password {{msuchy_password}} --os-tenant-name copr keypair-add --pub_key "$F" msuchy ); rm -f "$F" From ef33eec10970dcf3bde174b1f37fe61ae2831867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 09:58:54 +0000 Subject: [PATCH 43/78] add missing } --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 92844505b9..32689e8a0b 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -549,8 +549,8 @@ - name: upload SSH keys for nova_keypair: auth_url="https://{{controller_hostname}}:35357/v2.0" - state=present login_username={{ item.name }} - login_password="{{ item.password }}" login_tenant_name={{item.tenant} name={{ item.name }} + login_username="{{ item.name }}" + login_password="{{ item.password }}" login_tenant_name="{{item.tenant}}" name="{{ item.name }}" public_key="{{ item.public_key }}" ignore_errors: yes no_log: True From f83bd3fdca375700f9331b7abe67ef0d21796a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 10:06:43 +0000 Subject: [PATCH 44/78] correct name --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 32689e8a0b..ada8fa9417 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -546,7 +546,7 @@ - { name: codeblock, email: 'codeblock@elrod.me', tenant: infrastructure, password: "{{codeblock_password}}" } - { name: msuchy, email: 'msuchy@redhat.com', tenant: copr, password: "{{msuchy_password}}" } - { name: red, email: 'red@fedoraproject.org', tenant: infrastructure, password: "{{red_password}}" } - - name: upload SSH keys for + - name: upload SSH keys for users nova_keypair: auth_url="https://{{controller_hostname}}:35357/v2.0" login_username="{{ item.name }}" From 18967b31a34dc373b50eba3f02a438ed9ff8e197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 10:10:52 +0000 Subject: [PATCH 45/78] fix login of sam kottler --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index ada8fa9417..bf9e0bb6f1 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -557,7 +557,7 @@ with_items: - { name: kevin, email: 'kevin@fedoraproject.org', tenant: infrastructure, password: "{{kevin_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas kevin') }}" } - { name: laxathom, email: 'laxathom@fedoraproject.org', tenant: infrastructure, password: "{{laxathom_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas laxathom') }}" } - - { name: samkottler, email: 'samkottler@fedoraproject.org', tenant: infrastructure, password: "{{samkottler_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas samkottler') }}" } + - { name: samkottler, email: 'samkottler@fedoraproject.org', tenant: infrastructure, password: "{{samkottler_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas skottler') }}" } - { name: puiterwijk, email: 'puiterwijk@fedoraproject.org', tenant: infrastructure, password: "{{puiterwijk_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas puiterwijk') }}" } - { name: mattdm, email: 'mattdm@fedoraproject.org', tenant: infrastructure, password: "{{mattdm_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas mattdm') }}" } - { name: tflink, email: 'tflink@fedoraproject.org', tenant: qa, password: "{{tflink_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas tflink') }}" } From 22cf62300484ffaed2a8888700f33c5de3a6b23a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 10:54:58 +0000 Subject: [PATCH 46/78] define quota for Copr --- .../hosts/fed-cloud09.cloud.fedoraproject.org.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index bf9e0bb6f1..a11bfc7108 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -887,3 +887,15 @@ ethertype: "IPv4" protocol: "tcp" remote_ip_prefix: "172.30.0.1/12" + + + # Update quota for Copr + # SEE: + # nova quota-defaults + # nova quota-show --tenant $TENANT_ID + # default is 10 instances, 20 cores, 51200 RAM, 10 floating IPs + - shell: source /root/keystonerc_admin && keystone tenant-list | grep 'copr' | awk '{print $2}' + register: TENANT_ID + - shell: source /root/keystonerc_admin && nova quota-update --instances 40 --cores 80 --ram 512000 --floating_ips 40 {{ TENANT_ID.stdout }} + + From 00f246d9676e286304e2ef3afb74e362477c169b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 11:58:11 +0000 Subject: [PATCH 47/78] underscore do not always work --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index a11bfc7108..ef03ee552c 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -896,6 +896,6 @@ # default is 10 instances, 20 cores, 51200 RAM, 10 floating IPs - shell: source /root/keystonerc_admin && keystone tenant-list | grep 'copr' | awk '{print $2}' register: TENANT_ID - - shell: source /root/keystonerc_admin && nova quota-update --instances 40 --cores 80 --ram 512000 --floating_ips 40 {{ TENANT_ID.stdout }} + - shell: source /root/keystonerc_admin && nova quota-update --instances 40 --cores 80 --ram 512000 --floating-ips 40 {{ TENANT_ID.stdout }} From 9be513ab9958ac80d4f1e2fcea83d57c8881512e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 15:01:17 +0000 Subject: [PATCH 48/78] try this --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index ef03ee552c..72999676bb 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -681,7 +681,7 @@ name=copr-subnet cidr=172.30.0.1/12 gateway_ip=172.30.0.1 - dns_nameservers=66.35.62.163,140.211.169.201 + dns_nameservers="66.35.62.163,140.211.169.201" register: COPR_SUBNET_ID - name: Connect router's interface to the copr-subnet neutron_router_interface: @@ -798,7 +798,7 @@ name=transient-subnet cidr=172.28.0.1/12 gateway_ip=172.28.0.1 - dns_nameservers=66.35.62.163,140.211.169.201 + dns_nameservers="66.35.62.163,140.211.169.201" register: TRANSIENT_SUBNET_ID - name: Connect router's interface to the transient-subnet neutron_router_interface: From 4846e15dcd7065638187aa07fc8b4323bfd87f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 9 Mar 2015 15:15:25 +0000 Subject: [PATCH 49/78] use quotation marks as much as possible it needs to be used for dns_nameservers, but lets use them everywhere else --- .../fed-cloud09.cloud.fedoraproject.org.yml | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 72999676bb..c712d97cc8 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -618,8 +618,8 @@ neutron_router: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=cloudintern - name=ext-to-cloudintern + tenant_name="cloudintern" + name="ext-to-cloudintern" register: ROUTER_ID - name: Connect router's gateway to the external network neutron_router_gateway: @@ -631,24 +631,24 @@ neutron_network: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=cloudintern - name=cloudintern-net + tenant_name="cloudintern" + name="cloudintern-net" - name: Create a subnet in the cloudintern-net neutron_subnet: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=cloudintern - network_name=cloudintern-net - name=cloudintern-subnet - cidr=172.25.0.1/12 - gateway_ip=172.25.0.1 - dns_nameservers=66.35.62.163,140.211.169.201 + tenant_name="cloudintern" + network_name="cloudintern-net" + name="cloudintern-subnet" + cidr="172.25.0.1/12" + gateway_ip="172.25.0.1" + dns_nameservers="66.35.62.163,140.211.169.201" register: CLOUDINTERN_SUBNET_ID - name: Connect router's interface to the cloudintern-subnet neutron_router_interface: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=cloudintern + tenant_name="cloudintern" router_name="ext-to-cloudintern" subnet_name="cloudintern-subnet" @@ -657,8 +657,8 @@ neutron_router: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=copr - name=ext-to-copr + tenant_name="copr" + name="ext-to-copr" register: ROUTER_ID - name: Connect router's gateway to the external network neutron_router_gateway: @@ -670,8 +670,8 @@ neutron_network: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=copr - name=copr-net + tenant_name="copr" + name="copr-net" - name: Create a subnet in the copr-net neutron_subnet: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" @@ -679,15 +679,15 @@ tenant_name=copr network_name=copr-net name=copr-subnet - cidr=172.30.0.1/12 - gateway_ip=172.30.0.1 + cidr="172.30.0.1/12" + gateway_ip="172.30.0.1" dns_nameservers="66.35.62.163,140.211.169.201" register: COPR_SUBNET_ID - name: Connect router's interface to the copr-subnet neutron_router_interface: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=copr + tenant_name="copr" router_name="ext-to-copr" subnet_name="copr-subnet" @@ -696,8 +696,8 @@ neutron_router: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=infrastructure - name=ext-to-infrastructure + tenant_name="infrastructure" + name="ext-to-infrastructure" register: ROUTER_ID - name: Connect router's gateway to the external network neutron_router_gateway: @@ -709,18 +709,18 @@ neutron_network: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=infrastructure - name=infrastructure-net + tenant_name="infrastructure" + name="infrastructure-net" - name: Create a subnet in the infrastructure-net neutron_subnet: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=infrastructure - network_name=infrastructure-net - name=infrastructure-subnet - cidr=172.26.0.1/12 - gateway_ip=172.26.0.1 - dns_nameservers=66.35.62.163,140.211.169.201 + tenant_name="infrastructure" + network_name="infrastructure-net" + name="infrastructure-subnet" + cidr="172.26.0.1/12" + gateway_ip="172.26.0.1" + dns_nameservers="66.35.62.163,140.211.169.201" register: INFRASTRUCTURE_SUBNET_ID - name: Connect router's interface to the infrastructure-subnet neutron_router_interface: @@ -735,8 +735,8 @@ neutron_router: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=persistent - name=ext-to-persistent + tenant_name="persistent" + name="ext-to-persistent" register: ROUTER_ID - name: Connect router's gateway to the external network neutron_router_gateway: @@ -748,8 +748,8 @@ neutron_network: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=persistent - name=persistent-net + tenant_name="persistent" + name="persistent-net" - name: Create a subnet in the persistent-net neutron_subnet: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" @@ -757,15 +757,15 @@ tenant_name=persistent network_name=persistent-net name=persistent-subnet - cidr=172.27.0.1/12 - gateway_ip=172.27.0.1 - dns_nameservers=66.35.62.163,140.211.169.201 + cidr="172.27.0.1/12" + gateway_ip="172.27.0.1" + dns_nameservers="66.35.62.163,140.211.169.201" register: PERSISTENT_SUBNET_ID - name: Connect router's interface to the persistent-subnet neutron_router_interface: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=persistent + tenant_name="persistent" router_name="ext-to-persistent" subnet_name="persistent-subnet" @@ -775,7 +775,7 @@ login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" tenant_name=transient - name=ext-to-transient + name="ext-to-transient" register: ROUTER_ID - name: Connect router's gateway to the external network neutron_router_gateway: @@ -835,9 +835,9 @@ tenant_name=scratch network_name=scratch-net name=scratch-subnet - cidr=172.29.0.1/12 - gateway_ip=172.29.0.1 - dns_nameservers=66.35.62.163,140.211.169.201 + cidr="172.29.0.1/12" + gateway_ip="172.29.0.1" + dns_nameservers="66.35.62.163,140.211.169.201" register: SCRATCH_SUBNET_ID - name: Connect router's interface to the scratch-subnet neutron_router_interface: From 26d1c425833249949734b7e513fedb341a260f04 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 9 Mar 2015 15:55:39 +0000 Subject: [PATCH 50/78] Update info about external cloud ips and update ranges. --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 8 ++++++++ vars/fedora-cloud.yml | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index c712d97cc8..559fe0b6b6 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -578,6 +578,14 @@ ##### NETWORK #### # http://docs.openstack.org/havana/install-guide/install/apt/content/install-neutron.configure-networks.html + # + # external network is a class C: 209.132.184.0/24 + # 209.132.184.1 to .25 - reserved for hardware. + # 209.132.184.26 to .30 - reserver for test cloud external ips + # 209.132.184.31 to .69 - icehouse cloud + # 209.132.184.70 to .89 - reserved for arm03 SOCs + # 209.132.184.90 to .251 - folsom cloud + # - name: Create en external network neutron_network: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" diff --git a/vars/fedora-cloud.yml b/vars/fedora-cloud.yml index 5986e9515b..31c61c4e84 100644 --- a/vars/fedora-cloud.yml +++ b/vars/fedora-cloud.yml @@ -4,8 +4,8 @@ internal_interface_cidr: 172.24.0.1/24 public_gateway_ip: 209.132.184.254 public_dns: 66.35.62.163 -public_floating_start: 209.132.184.33 -public_floating_end: 209.132.184.46 +public_floating_start: 209.132.184.31 +public_floating_end: 209.132.184.69 controller_public_ip: 209.132.184.9 controller_private_ip: 172.24.0.9 From 745e14a333c3ffc6ece106e095bfa3ff53fba866 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 9 Mar 2015 18:19:24 +0000 Subject: [PATCH 51/78] (fedimg) Crank up the number of fedmsg endpoints. --- roles/fedmsg/base/templates/endpoints-fedimg.py.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/fedmsg/base/templates/endpoints-fedimg.py.j2 b/roles/fedmsg/base/templates/endpoints-fedimg.py.j2 index b13f3a78a2..5a4fb9d4af 100644 --- a/roles/fedmsg/base/templates/endpoints-fedimg.py.j2 +++ b/roles/fedmsg/base/templates/endpoints-fedimg.py.j2 @@ -4,11 +4,15 @@ suffix = 'stg.phx2.fedoraproject.org' suffix = 'phx2.fedoraproject.org' {% endif %} +primary_threads = 4 +atomic_threads = 2 +NUM_FEDIMG_PORTS = 2 * ((primary_threads + atomic_threads) + 1) + config = dict( endpoints={ "fedimg.fedimg01": [ "tcp://fedimg01.%s:30%0.2i" % (suffix, i) - for i in range(4) + for i in range(NUM_FEDIMG_PORTS) ], }, ) From 5b51683547ae84504d29c1dec07d000a797c084a Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 9 Mar 2015 18:24:48 +0000 Subject: [PATCH 52/78] Nuke some other references to hrf.cloud.fedoraproject.org --- inventory/inventory | 2 -- master.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/inventory/inventory b/inventory/inventory index 26164ecbb1..6953c3fe84 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -730,8 +730,6 @@ copr-be.cloud.fedoraproject.org # copr dev instances copr-be-dev.cloud.fedoraproject.org copr-fe-dev.cloud.fedoraproject.org -#hrf -hrf.cloud.fedoraproject.org #shogun-ca.cloud.fedoraproject.org 209.132.184.157 # bodhi.dev.fedoraproject.org diff --git a/master.yml b/master.yml index 8d76d3f615..32a0761b31 100644 --- a/master.yml +++ b/master.yml @@ -99,7 +99,6 @@ - include: /srv/web/infra/ansible/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml - include: /srv/web/infra/ansible/playbooks/hosts/elections-dev.cloud.fedoraproject.org.yml - include: /srv/web/infra/ansible/playbooks/hosts/fedocal.dev.fedoraproject.org.yml -- include: /srv/web/infra/ansible/playbooks/hosts/hrf.cloud.fedoraproject.org.yml - include: /srv/web/infra/ansible/playbooks/hosts/koschei.cloud.fedoraproject.org.yml - include: /srv/web/infra/ansible/playbooks/hosts/lists-dev.cloud.fedoraproject.org.yml - include: /srv/web/infra/ansible/playbooks/hosts/logserver.yml From 1f95cc2e1ba3bce20991dd79763933541dd4d2f7 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 9 Mar 2015 18:47:31 +0000 Subject: [PATCH 53/78] Guess what? there's no epel7 i386. ;) --- roles/packages/web/files/packages-yum.conf | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/roles/packages/web/files/packages-yum.conf b/roles/packages/web/files/packages-yum.conf index a98956ada6..1a39f55872 100644 --- a/roles/packages/web/files/packages-yum.conf +++ b/roles/packages/web/files/packages-yum.conf @@ -222,19 +222,3 @@ baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/7/x86_64/ #mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel7&arch=x86_64 enabled=0 gpgcheck=0 - -[epel-7-i686] -name=EPEL 7 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/7/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=i386 -enabled=0 -gpgcheck=0 - -[epel-7-testing-i686] -name=EPEL 7 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/7/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel7&arch=i386 -enabled=0 -gpgcheck=0 From 603222e7ffe330389493bbe27d2aca6ee4236029 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 10 Mar 2015 07:20:00 +0000 Subject: [PATCH 54/78] buildmaster: revert using date directory for artifacts --- .../templates/taskotron.master.cfg.j2 | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 51c1b1fb08..954033cc59 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -173,27 +173,17 @@ factory.addStep(ShellCommand(command=["runtask", '-i', {% if deployment_type == 'dev' %} - -import datetime -from buildbot.process.properties import renderer - -@renderer -def today(props): - return datetime.datetime.now().strftime("%Y%m%d") - -artifactsdir = Interpolate('{{ public_artifacts_dir }}/%(kw:today)s/%(prop:uuid)s', today=today) - # create artifacts dir on master -factory.addStep(MasterShellCommand(command=["mkdir", '-p', '-m', '0755', artifactsdir], +factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s')], descriptionDone=['Create artifacs dir'])) # copy artifacts to master factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/'), - masterdest='%s/task_output' % artifactsdir)) + masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/task_output'))) # copy taskotron log to master factory.addStep(FileUpload(slavesrc='/var/log/taskotron/taskotron.log', - masterdest='%s/taskotron.log' % artifactsdir, + masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'), mode=0644)) {% else %} # capture the taskotron log From a0a310af2ae00910c8cbd54d33b75b2f478a731e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 08:05:48 +0000 Subject: [PATCH 55/78] alter internal ip ranges /12 is too big, we use /20 which gives us 4096 ip in subnet and plenty of subnets --- .../fed-cloud09.cloud.fedoraproject.org.yml | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 559fe0b6b6..dde6bc2b7a 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -610,16 +610,18 @@ - shell: source /root/keystonerc_admin && nova floating-ip-create external when: packstack_sucessfully_finished.stat.exists == False - # 172.16.0.1/12 -- 172.21.0.1/12 - Free to take - # 172.23.0.1/12 - free (but used by old cloud) - # 172.24.0.1/12 - RESERVED it is used internally for OS - # 172.25.0.1/12 - Cloudintern - # 172.26.0.1/12 - infrastructure - # 172.27.0.1/12 - persistent - # 172.28.0.1/12 - transient - # 172.29.0.1/12 - scratch - # 172.30.0.1/12 - copr - # 172.31.0.1/12 - Free to take + # 172.16.0.1/16 -- 172.22.0.1/16 - free (can be split to /20) + # 172.23.0.1/16 - free (but used by old cloud) + # 172.24.0.1/24 - RESERVED it is used internally for OS + # 172.24.1.0/24 -- 172.24.255.0/24 - likely free (?) + # 172.25.0.1/20 - Cloudintern (172.25.0.1 - 172.25.15.254) + # 172.25.16.1/20 - infrastructure (172.25.16.1 - 172.25.31.254) + # 172.25.32.1/20 - persistent (172.25.32.1 - 172.25.47.254) + # 172.25.48.1/20 - transient (172.25.48.1 - 172.25.63.254) + # 172.25.64.1/20 - scratch (172.25.64.1 - 172.25.79.254) + # 172.25.80.1/20 - copr (172.25.80.1 - 172.25.95.254) + # 172.25.96.1/20 -- 172.25.240.1/20 - free + # 172.26.0.1/16 -- 172.31.0.1/16 - free (can be split to /20) # Cloudintern network - name: Create a router for Cloudintern @@ -648,7 +650,7 @@ tenant_name="cloudintern" network_name="cloudintern-net" name="cloudintern-subnet" - cidr="172.25.0.1/12" + cidr="172.25.0.1/20" gateway_ip="172.25.0.1" dns_nameservers="66.35.62.163,140.211.169.201" register: CLOUDINTERN_SUBNET_ID @@ -687,8 +689,8 @@ tenant_name=copr network_name=copr-net name=copr-subnet - cidr="172.30.0.1/12" - gateway_ip="172.30.0.1" + cidr="172.25.80.1/20" + gateway_ip="172.25.80.1" dns_nameservers="66.35.62.163,140.211.169.201" register: COPR_SUBNET_ID - name: Connect router's interface to the copr-subnet @@ -726,8 +728,8 @@ tenant_name="infrastructure" network_name="infrastructure-net" name="infrastructure-subnet" - cidr="172.26.0.1/12" - gateway_ip="172.26.0.1" + cidr="172.25.16.1/20" + gateway_ip="172.25.16.1" dns_nameservers="66.35.62.163,140.211.169.201" register: INFRASTRUCTURE_SUBNET_ID - name: Connect router's interface to the infrastructure-subnet @@ -765,8 +767,8 @@ tenant_name=persistent network_name=persistent-net name=persistent-subnet - cidr="172.27.0.1/12" - gateway_ip="172.27.0.1" + cidr="172.25.32.1/20" + gateway_ip="172.25.32.1" dns_nameservers="66.35.62.163,140.211.169.201" register: PERSISTENT_SUBNET_ID - name: Connect router's interface to the persistent-subnet @@ -804,8 +806,8 @@ tenant_name=transient network_name=transient-net name=transient-subnet - cidr=172.28.0.1/12 - gateway_ip=172.28.0.1 + cidr=172.25.48.1/20 + gateway_ip=172.25.48.1 dns_nameservers="66.35.62.163,140.211.169.201" register: TRANSIENT_SUBNET_ID - name: Connect router's interface to the transient-subnet @@ -843,8 +845,8 @@ tenant_name=scratch network_name=scratch-net name=scratch-subnet - cidr="172.29.0.1/12" - gateway_ip="172.29.0.1" + cidr="172.25.64.1/20" + gateway_ip="172.25.64.1" dns_nameservers="66.35.62.163,140.211.169.201" register: SCRATCH_SUBNET_ID - name: Connect router's interface to the scratch-subnet @@ -894,7 +896,7 @@ port_range_max: "22" ethertype: "IPv4" protocol: "tcp" - remote_ip_prefix: "172.30.0.1/12" + remote_ip_prefix: "172.25.80.1/20" # Update quota for Copr From 9972ee128db7f0f269bff50768d4431ce73ab5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 08:24:40 +0000 Subject: [PATCH 56/78] implement sec-groups web-80-anywhere, web-443-anywhere and wide-open --- .../fed-cloud09.cloud.fedoraproject.org.yml | 121 ++++++++++++++++-- 1 file changed, 112 insertions(+), 9 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index dde6bc2b7a..598effb02b 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -859,10 +859,10 @@ -################ -# Copr -# ############## - - name: Copr - Create 'ssh-anywhere' security group + ################# + # Security Groups + ################ + - name: Create 'ssh-anywhere' security group neutron_sec_group: login_username: "admin" login_password: "{{ ADMIN_PASS }}" @@ -871,7 +871,7 @@ state: "present" name: 'ssh-anywhere' description: "allow ssh from anywhere" - tenant_name: "copr" + tenant_name: "{{item}}" rules: - direction: "ingress" port_range_min: "22" @@ -879,8 +879,18 @@ ethertype: "IPv4" protocol: "tcp" remote_ip_prefix: "0.0.0.0/0" + with_items: + - cloudintern + - cloudsig + - copr + - infrastructure + - persistent + - pythonbots + - qa + - scratch + - transient - - name: Copr - Create 'ssh-internal' security group + - name: Create 'ssh-internal' security group neutron_sec_group: login_username: "admin" login_password: "{{ ADMIN_PASS }}" @@ -888,16 +898,109 @@ auth_url: "https://{{controller_hostname}}:35357/v2.0" state: "present" name: 'ssh-internal' - description: "allow ssh from copr-network" - tenant_name: "copr" + description: "allow ssh from {{item.name}}-network" + tenant_name: "{{ item.name }}" rules: - direction: "ingress" port_range_min: "22" port_range_max: "22" ethertype: "IPv4" protocol: "tcp" - remote_ip_prefix: "172.25.80.1/20" + remote_ip_prefix: "{{ item.prefix }}" + with_items: + - { name: cloudintern, prefix: '172.25.0.1/20' } + - { name: cloudsig, prefix: '' } + - { name: copr, prefix: '172.25.80.1/20' } + - { name: infrastructure, prefix: "172.25.16.1/20" } + - { name: persistent, prefix: "172.25.32.1/20" } + - { name: pythonbots, prefix: '' } + - { name: qa, prefix: "" } + - { name: scratch, prefix: '172.25.64.1/20' } + - { name: transient, prefix: '172.25.48.1/20' } + - name: Create 'web-80-anywhere' security group + neutron_sec_group: + login_username: "admin" + login_password: "{{ ADMIN_PASS }}" + login_tenant_name: "admin" + auth_url: "https://{{controller_hostname}}:35357/v2.0" + state: "present" + name: 'web-80-anywhere' + description: "allow web-80 from anywhere" + tenant_name: "{{item}}" + rules: + - direction: "ingress" + port_range_min: "80" + port_range_max: "80" + ethertype: "IPv4" + protocol: "tcp" + remote_ip_prefix: "0.0.0.0/0" + with_items: + - cloudintern + - cloudsig + - copr + - infrastructure + - persistent + - pythonbots + - qa + - scratch + - transient + + - name: Create 'web-443-anywhere' security group + neutron_sec_group: + login_username: "admin" + login_password: "{{ ADMIN_PASS }}" + login_tenant_name: "admin" + auth_url: "https://{{controller_hostname}}:35357/v2.0" + state: "present" + name: 'web-443-anywhere' + description: "allow web-443 from anywhere" + tenant_name: "{{item}}" + rules: + - direction: "ingress" + port_range_min: "443" + port_range_max: "443" + ethertype: "IPv4" + protocol: "tcp" + remote_ip_prefix: "0.0.0.0/0" + with_items: + - cloudintern + - cloudsig + - copr + - infrastructure + - persistent + - pythonbots + - qa + - scratch + - transient + + - name: Create 'wide-open' security group + neutron_sec_group: + login_username: "admin" + login_password: "{{ ADMIN_PASS }}" + login_tenant_name: "admin" + auth_url: "https://{{controller_hostname}}:35357/v2.0" + state: "present" + name: 'wide-open' + description: "allow anything from anywhere" + tenant_name: "{{item}}" + rules: + - direction: "ingress" + port_range_min: "0" + port_range_max: "65535" + ethertype: "IPv4" + protocol: "tcp" + remote_ip_prefix: "0.0.0.0/0" + with_items: + - cloudintern + - cloudsig + - copr + - infrastructure + - persistent + - pythonbots + - qa + - scratch + - transient # Update quota for Copr # SEE: From dc67fc57de284282a149b1b3b902e6699974ff73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 08:47:45 +0000 Subject: [PATCH 57/78] add cloudsig subnet --- .../fed-cloud09.cloud.fedoraproject.org.yml | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 598effb02b..536111c38d 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -620,7 +620,8 @@ # 172.25.48.1/20 - transient (172.25.48.1 - 172.25.63.254) # 172.25.64.1/20 - scratch (172.25.64.1 - 172.25.79.254) # 172.25.80.1/20 - copr (172.25.80.1 - 172.25.95.254) - # 172.25.96.1/20 -- 172.25.240.1/20 - free + # 172.25.96.1/20 - cloudsig (172.25.96.1 - 172.25.111.254) + # 172.25.112.1/20 -- 172.25.240.1/20 - free # 172.26.0.1/16 -- 172.31.0.1/16 - free (can be split to /20) # Cloudintern network @@ -857,7 +858,44 @@ router_name="ext-to-scratch" subnet_name="scratch-subnet" - + # cloudsig network + - name: "Create a router for cloudsig" + neutron_router: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + tenant_name=cloudsig + name=ext-to-cloudsig + register: ROUTER_ID + - name: "Connect router's gateway to the external network" + neutron_router_gateway: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + router_name="ext-to-cloudsig" + network_name="external" + - name: Create a private network for cloudsig + neutron_network: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + tenant_name=cloudsig + name=cloudsig-net + - name: Create a subnet in the cloudisg-net + neutron_subnet: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + tenant_name=cloudsig + network_name=cloudsig-net + name=cloudsig-subnet + cidr="172.25.96.1/20" + gateway_ip="172.25.96.1" + dns_nameservers="66.35.62.163,140.211.169.201" + register: CLOUDSIG_SUBNET_ID + - name: "Connect router's interface to the cloudsig-subnet" + neutron_router_interface: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + tenant_name=cloudsig + router_name="ext-to-cloudisg" + subnet_name="cloudsig-subnet" ################# # Security Groups @@ -909,7 +947,7 @@ remote_ip_prefix: "{{ item.prefix }}" with_items: - { name: cloudintern, prefix: '172.25.0.1/20' } - - { name: cloudsig, prefix: '' } + - { name: cloudsig, prefix: '172.25.96.1/20' } - { name: copr, prefix: '172.25.80.1/20' } - { name: infrastructure, prefix: "172.25.16.1/20" } - { name: persistent, prefix: "172.25.32.1/20" } From 544eac7c4ee2325f7a47ed5e78750e1dd40b29fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 08:54:01 +0000 Subject: [PATCH 58/78] add qa network --- .../fed-cloud09.cloud.fedoraproject.org.yml | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 536111c38d..d316a0c349 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -621,7 +621,8 @@ # 172.25.64.1/20 - scratch (172.25.64.1 - 172.25.79.254) # 172.25.80.1/20 - copr (172.25.80.1 - 172.25.95.254) # 172.25.96.1/20 - cloudsig (172.25.96.1 - 172.25.111.254) - # 172.25.112.1/20 -- 172.25.240.1/20 - free + # 172.25.112.1/20 - qa (172.25.112.1 - 172.25.127.254) + # 172.25.128.1/20 -- 172.25.240.1/20 - free # 172.26.0.1/16 -- 172.31.0.1/16 - free (can be split to /20) # Cloudintern network @@ -878,7 +879,7 @@ auth_url="https://{{controller_hostname}}:35357/v2.0" tenant_name=cloudsig name=cloudsig-net - - name: Create a subnet in the cloudisg-net + - name: Create a subnet in the cloudsig-net neutron_subnet: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" @@ -897,6 +898,45 @@ router_name="ext-to-cloudisg" subnet_name="cloudsig-subnet" + # qa network + - name: "Create a router for QA" + neutron_router: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + tenant_name=qa + name=ext-to-qa + register: ROUTER_ID + - name: "Connect router's gateway to the external network" + neutron_router_gateway: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + router_name="ext-to-qa" + network_name="qa" + - name: Create a private network for QA + neutron_network: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + tenant_name=qa + name=qa-net + - name: Create a subnet in the qa-net + neutron_subnet: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + tenant_name=qa + network_name=qa-net + name=qa-subnet + cidr="172.25.112.1/20" + gateway_ip="172.25.112.1" + dns_nameservers="66.35.62.163,140.211.169.201" + register: QA_SUBNET_ID + - name: "Connect router's interface to the qa-subnet" + neutron_router_interface: + login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" + auth_url="https://{{controller_hostname}}:35357/v2.0" + tenant_name=qa + router_name="ext-to-qa" + subnet_name="qa-subnet" + ################# # Security Groups ################ @@ -952,7 +992,7 @@ - { name: infrastructure, prefix: "172.25.16.1/20" } - { name: persistent, prefix: "172.25.32.1/20" } - { name: pythonbots, prefix: '' } - - { name: qa, prefix: "" } + - { name: qa, prefix: "172.25.112.1/20" } - { name: scratch, prefix: '172.25.64.1/20' } - { name: transient, prefix: '172.25.48.1/20' } From 066c544140dc3e81dae778e27efed16932bb7b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 09:06:01 +0000 Subject: [PATCH 59/78] remove spagetti code --- .../fed-cloud09.cloud.fedoraproject.org.yml | 359 ++++-------------- 1 file changed, 67 insertions(+), 292 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index d316a0c349..0dc0415807 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -626,316 +626,91 @@ # 172.26.0.1/16 -- 172.31.0.1/16 - free (can be split to /20) # Cloudintern network - - name: Create a router for Cloudintern + - name: Create a router for all tenants neutron_router: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="cloudintern" - name="ext-to-cloudintern" - register: ROUTER_ID - - name: Connect router's gateway to the external network - neutron_router_gateway: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - router_name="ext-to-cloudintern" - network_name="external" - - name: Create a private network for cloudintern - neutron_network: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="cloudintern" - name="cloudintern-net" - - name: Create a subnet in the cloudintern-net - neutron_subnet: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="cloudintern" - network_name="cloudintern-net" - name="cloudintern-subnet" - cidr="172.25.0.1/20" - gateway_ip="172.25.0.1" - dns_nameservers="66.35.62.163,140.211.169.201" - register: CLOUDINTERN_SUBNET_ID - - name: Connect router's interface to the cloudintern-subnet - neutron_router_interface: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="cloudintern" - router_name="ext-to-cloudintern" - subnet_name="cloudintern-subnet" - - # Copr network - - name: Create a router for copr - neutron_router: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="copr" - name="ext-to-copr" - register: ROUTER_ID - - name: Connect router's gateway to the external network - neutron_router_gateway: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - router_name="ext-to-copr" - network_name="external" - - name: Create a private network for copr - neutron_network: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="copr" - name="copr-net" - - name: Create a subnet in the copr-net - neutron_subnet: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=copr - network_name=copr-net - name=copr-subnet - cidr="172.25.80.1/20" - gateway_ip="172.25.80.1" - dns_nameservers="66.35.62.163,140.211.169.201" - register: COPR_SUBNET_ID - - name: Connect router's interface to the copr-subnet - neutron_router_interface: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="copr" - router_name="ext-to-copr" - subnet_name="copr-subnet" - - # infrastructure network - - name: Create a router for infrastructure - neutron_router: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="infrastructure" - name="ext-to-infrastructure" - register: ROUTER_ID - - name: Connect router's gateway to the external network - neutron_router_gateway: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - router_name="ext-to-infrastructure" - network_name="external" - - name: Create a private network for infrastructure - neutron_network: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="infrastructure" - name="infrastructure-net" - - name: Create a subnet in the infrastructure-net - neutron_subnet: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="infrastructure" - network_name="infrastructure-net" - name="infrastructure-subnet" - cidr="172.25.16.1/20" - gateway_ip="172.25.16.1" - dns_nameservers="66.35.62.163,140.211.169.201" - register: INFRASTRUCTURE_SUBNET_ID - - name: Connect router's interface to the infrastructure-subnet - neutron_router_interface: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=infrastructure - router_name="ext-to-infrastructure" - subnet_name="infrastructure-subnet" - - # persistent network - - name: Create a router for persistent - neutron_router: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="persistent" - name="ext-to-persistent" - register: ROUTER_ID - - name: Connect router's gateway to the external network - neutron_router_gateway: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - router_name="ext-to-persistent" - network_name="external" - - name: Create a private network for persistent - neutron_network: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="persistent" - name="persistent-net" - - name: Create a subnet in the persistent-net - neutron_subnet: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=persistent - network_name=persistent-net - name=persistent-subnet - cidr="172.25.32.1/20" - gateway_ip="172.25.32.1" - dns_nameservers="66.35.62.163,140.211.169.201" - register: PERSISTENT_SUBNET_ID - - name: Connect router's interface to the persistent-subnet - neutron_router_interface: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name="persistent" - router_name="ext-to-persistent" - subnet_name="persistent-subnet" - - # transient network - - name: Create a router for transient - neutron_router: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=transient - name="ext-to-transient" - register: ROUTER_ID - - name: Connect router's gateway to the external network - neutron_router_gateway: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - router_name="ext-to-transient" - network_name="external" - - name: Create a private network for transient - neutron_network: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=transient - name=transient-net - - name: Create a subnet in the transient-net - neutron_subnet: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=transient - network_name=transient-net - name=transient-subnet - cidr=172.25.48.1/20 - gateway_ip=172.25.48.1 - dns_nameservers="66.35.62.163,140.211.169.201" - register: TRANSIENT_SUBNET_ID - - name: Connect router's interface to the transient-subnet - neutron_router_interface: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=transient - router_name="ext-to-transient" - subnet_name="transient-subnet" - - # scratch network - - name: Create a router for scratch - neutron_router: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=scratch - name=ext-to-scratch - register: ROUTER_ID - - name: Connect router's gateway to the external network - neutron_router_gateway: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - router_name="ext-to-scratch" - network_name="external" - - name: Create a private network for scratch - neutron_network: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=scratch - name=scratch-net - - name: Create a subnet in the scratch-net - neutron_subnet: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=scratch - network_name=scratch-net - name=scratch-subnet - cidr="172.25.64.1/20" - gateway_ip="172.25.64.1" - dns_nameservers="66.35.62.163,140.211.169.201" - register: SCRATCH_SUBNET_ID - - name: Connect router's interface to the scratch-subnet - neutron_router_interface: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=scratch - router_name="ext-to-scratch" - subnet_name="scratch-subnet" - - # cloudsig network - - name: "Create a router for cloudsig" - neutron_router: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=cloudsig - name=ext-to-cloudsig - register: ROUTER_ID + tenant_name="{{ item }}" + name="ext-to-{{ item }}" + with_items: + - cloudintern + - cloudsig + - copr + - infrastructure + - persistent + - pythonbots + - qa + - scratch + - transient - name: "Connect router's gateway to the external network" neutron_router_gateway: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - router_name="ext-to-cloudsig" + router_name="ext-to-{{ item }}" network_name="external" - - name: Create a private network for cloudsig + with_items: + - cloudintern + - cloudsig + - copr + - infrastructure + - persistent + - pythonbots + - qa + - scratch + - transient + - name: Create a private network for all tenants neutron_network: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=cloudsig - name=cloudsig-net - - name: Create a subnet in the cloudsig-net + tenant_name="{{ item }}" + name="{{ item }}-net" + with_items: + - cloudintern + - cloudsig + - copr + - infrastructure + - persistent + - pythonbots + - qa + - scratch + - transient + - name: Create a subnet for all tenants neutron_subnet: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=cloudsig - network_name=cloudsig-net - name=cloudsig-subnet - cidr="172.25.96.1/20" - gateway_ip="172.25.96.1" + tenant_name="{{ item.name }}" + network_name="{{ item.name }}-net" + name="{{ item.name }}-subnet" + cidr="{{ item.cidr }}" + gateway_ip="{{ item.gateway }}" dns_nameservers="66.35.62.163,140.211.169.201" - register: CLOUDSIG_SUBNET_ID - - name: "Connect router's interface to the cloudsig-subnet" + with_items: + - { name: cloudintern, cidr: '172.25.0.1/20', gateway: '172.25.0.1' } + - { name: cloudsig, cidr: '172.25.96.1/20', gateway: '172.25.96.1' } + - { name: copr, cidr: '172.25.80.1/20', gateway: '172.25.80.1' } + - { name: infrastructure, cidr: '172.25.16.1/20', gateway: '172.25.16.1' } + - { name: persistent, cidr: '172.25.32.1/20', gateway: '172.25.32.1' } + - { name: pythonbots, cidr: '', gateway: '' } + - { name: qa, cidr: '172.25.112.1/20', gateway: '172.25.112.1' } + - { name: scratch, cidr: '172.25.64.1/20', gateway: '172.25.64.1' } + - { name: transient, cidr: '172.25.48.1/20', gateway: '172.25.48.1' } + - name: "Connect router's interface to the TENANT-subnet" neutron_router_interface: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=cloudsig - router_name="ext-to-cloudisg" - subnet_name="cloudsig-subnet" - - # qa network - - name: "Create a router for QA" - neutron_router: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=qa - name=ext-to-qa - register: ROUTER_ID - - name: "Connect router's gateway to the external network" - neutron_router_gateway: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - router_name="ext-to-qa" - network_name="qa" - - name: Create a private network for QA - neutron_network: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=qa - name=qa-net - - name: Create a subnet in the qa-net - neutron_subnet: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=qa - network_name=qa-net - name=qa-subnet - cidr="172.25.112.1/20" - gateway_ip="172.25.112.1" - dns_nameservers="66.35.62.163,140.211.169.201" - register: QA_SUBNET_ID - - name: "Connect router's interface to the qa-subnet" - neutron_router_interface: - login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" - auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name=qa - router_name="ext-to-qa" - subnet_name="qa-subnet" + tenant_name={{ item }}" + router_name="ext-to-{{ item }}" + subnet_name={{ item }}-subnet" + with_items: + - cloudintern + - cloudsig + - copr + - infrastructure + - persistent + - pythonbots + - qa + - scratch + - transient ################# # Security Groups From 479267be08d7e5945c38981f4560c26b16855333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 09:08:34 +0000 Subject: [PATCH 60/78] add pythonbots network --- .../hosts/fed-cloud09.cloud.fedoraproject.org.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 0dc0415807..cf22b661c1 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -622,7 +622,8 @@ # 172.25.80.1/20 - copr (172.25.80.1 - 172.25.95.254) # 172.25.96.1/20 - cloudsig (172.25.96.1 - 172.25.111.254) # 172.25.112.1/20 - qa (172.25.112.1 - 172.25.127.254) - # 172.25.128.1/20 -- 172.25.240.1/20 - free + # 172.25.128.1/20 - pythonbots (172.25.128.1 - 172.25.143.254) + # 172.25.143.1/20 -- 172.25.240.1/20 - free # 172.26.0.1/16 -- 172.31.0.1/16 - free (can be split to /20) # Cloudintern network @@ -690,7 +691,7 @@ - { name: copr, cidr: '172.25.80.1/20', gateway: '172.25.80.1' } - { name: infrastructure, cidr: '172.25.16.1/20', gateway: '172.25.16.1' } - { name: persistent, cidr: '172.25.32.1/20', gateway: '172.25.32.1' } - - { name: pythonbots, cidr: '', gateway: '' } + - { name: pythonbots, cidr: '172.25.128.1/20', gateway: '172.25.128.1' } - { name: qa, cidr: '172.25.112.1/20', gateway: '172.25.112.1' } - { name: scratch, cidr: '172.25.64.1/20', gateway: '172.25.64.1' } - { name: transient, cidr: '172.25.48.1/20', gateway: '172.25.48.1' } @@ -698,9 +699,9 @@ neutron_router_interface: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" auth_url="https://{{controller_hostname}}:35357/v2.0" - tenant_name={{ item }}" + tenant_name="{{ item }}" router_name="ext-to-{{ item }}" - subnet_name={{ item }}-subnet" + subnet_name="{{ item }}-subnet" with_items: - cloudintern - cloudsig @@ -766,7 +767,7 @@ - { name: copr, prefix: '172.25.80.1/20' } - { name: infrastructure, prefix: "172.25.16.1/20" } - { name: persistent, prefix: "172.25.32.1/20" } - - { name: pythonbots, prefix: '' } + - { name: pythonbots, prefix: '172.25.128.1/20' } - { name: qa, prefix: "172.25.112.1/20" } - { name: scratch, prefix: '172.25.64.1/20' } - { name: transient, prefix: '172.25.48.1/20' } From c9a9ebaf08449d1f8f25d4349c653f32699afa4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 09:11:25 +0000 Subject: [PATCH 61/78] make vim formatting happy --- .../hosts/fed-cloud09.cloud.fedoraproject.org.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index cf22b661c1..fff2e2bf5c 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -53,7 +53,7 @@ - command: vgrename vg_guests cinder-volumes ignore_errors: yes - - lvg: vg=cinder-volumes pvs=/dev/md127 pesize=32 vg_options='' + - lvg: vg=cinder-volumes pvs=/dev/md127 pesize=32 vg_options="" - template: src={{ files }}/fedora-cloud/hosts dest=/etc/hosts owner=root mode=0644 @@ -716,7 +716,7 @@ ################# # Security Groups ################ - - name: Create 'ssh-anywhere' security group + - name: "Create 'ssh-anywhere' security group" neutron_sec_group: login_username: "admin" login_password: "{{ ADMIN_PASS }}" @@ -744,7 +744,7 @@ - scratch - transient - - name: Create 'ssh-internal' security group + - name: "Create 'ssh-internal' security group" neutron_sec_group: login_username: "admin" login_password: "{{ ADMIN_PASS }}" @@ -772,7 +772,7 @@ - { name: scratch, prefix: '172.25.64.1/20' } - { name: transient, prefix: '172.25.48.1/20' } - - name: Create 'web-80-anywhere' security group + - name: "Create 'web-80-anywhere' security group" neutron_sec_group: login_username: "admin" login_password: "{{ ADMIN_PASS }}" @@ -800,7 +800,7 @@ - scratch - transient - - name: Create 'web-443-anywhere' security group + - name: "Create 'web-443-anywhere' security group" neutron_sec_group: login_username: "admin" login_password: "{{ ADMIN_PASS }}" @@ -828,7 +828,7 @@ - scratch - transient - - name: Create 'wide-open' security group + - name: "Create 'wide-open' security group" neutron_sec_group: login_username: "admin" login_password: "{{ ADMIN_PASS }}" From 890389d15173e52ce21764760e375d6deddd652a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 09:13:04 +0000 Subject: [PATCH 62/78] remove unused code --- .../fed-cloud09.cloud.fedoraproject.org.yml | 58 ------------------- 1 file changed, 58 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index fff2e2bf5c..33bad13d34 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -145,10 +145,6 @@ copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/tls/certs/fed-cloud09-neutron.pem mode=600 owner=neutron group=root - name: add ssl key for neutron copy: src={{ private }}/files/openstack/fed-cloud09.key dest=/etc/pki/tls/private/fed-cloud09-neutron.key mode=600 owner=neutron group=root - #- name: add ssl cert for nova - # copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/tls/certs/fed-cloud09-nova.pem mode=600 owner=nova group=root - #- name: add ssl key for nova - # copy: src={{ private }}/files/openstack/fed-cloud09.key dest=/etc/pki/tls/private/fed-cloud09-nova.key mode=600 owner=nova group=root # http://docs.openstack.org/trunk/install-guide/install/yum/content/basics-database-controller.html - name: install mysql packages @@ -194,31 +190,6 @@ regexp="RABBITMQ_NODE_PORT" line=" 'RABBITMQ_NODE_PORTTTTT' => $port," backup=yes - #- lineinfile: - # dest=/usr/share/openstack-puppet/modules/rabbitmq/templates/rabbitmq.config.erb - # regexp="cacertfile" - # line=" {ssl_options, [{cacertfile,\"<%= @ssl_cert %>\"}," - # backup=yes - #- lineinfile: - # dest=/usr/share/openstack-puppet/modules/neutron/manifests/init.pp - # regexp="rabbit_use_ssl = " - # line=" $rabbit_use_ssl = true," - # backup=yes - #- lineinfile: - # dest=/usr/share/openstack-puppet/modules/nova/manifests/init.pp - # regexp="rabbit_use_ssl = " - # line=" $rabbit_use_ssl = true," - # backup=yes - #- lineinfile: - # dest=/usr/share/openstack-puppet/modules/glance/manifests/notify/rabbitmq.pp - # regexp="rabbit_use_ssl = " - # line=" $rabbit_use_ssl = true," - # backup=yes - #- lineinfile: - # dest=/usr/share/openstack-puppet/modules/ceilometer/manifests/init.pp - # regexp="rabbit_use_ssl = " - # line=" $rabbit_use_ssl = true," - # backup=yes - lineinfile: dest=/usr/lib/python2.7/site-packages/packstack/puppet/templates/mongodb.pp regexp="pidfilepath" @@ -239,33 +210,6 @@ - lineinfile: dest=/etc/rabbitmq/rabbitmq-env.conf regexp="^RABBITMQ_NODE_PORT=" state="absent" - service: name=rabbitmq-server state=started - # WORKAROUND again - #- ini_file: dest=/etc/keystone/keystone.conf section="DEFAULT" option="rabbit_use_ssl" value="true" - #- service: name=rabbitmq-server state=restarted - #- ini_file: dest=/etc/nova/nova.conf section="DEFAULT" option="rabbit_use_ssl" value="true" - #- ini_file: dest=/etc/cinder/cinder.conf section="DEFAULT" option="rabbit_use_ssl" value="true" - #- ini_file: dest=/etc/ceilometer/ceilometer.conf section="DEFAULT" option="rabbit_use_ssl" value="true" - #- service: name="{{item}}" state=restarted - # with_items: - # - openstack-ceilometer-alarm-evaluator - # - openstack-ceilometer-alarm-notifier - # - openstack-ceilometer-api - # - openstack-ceilometer-central - # - openstack-ceilometer-collector - # - openstack-ceilometer-compute - # - openstack-ceilometer-notification - # - openstack-cinder-api - # - openstack-cinder-backup - # - openstack-cinder-scheduler - # - openstack-cinder-volume - # - openstack-nova-api - # - openstack-nova-cert - # - openstack-nova-compute - # - openstack-nova-conductor - # - openstack-nova-consoleauth - # - openstack-nova-novncproxy - # - openstack-nova-scheduler - # flip endpoints internalurl to internal IP # ceilometer - shell: source /root/keystonerc_admin && keystone service-list | grep ceilometer | awk '{print $2}' @@ -355,8 +299,6 @@ - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=osapi_compute_listen_port value=6774 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=ec2_listen_port value=6773 - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_api_servers value=https://{{ controller_hostname }}:9292 - #- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=cert value=/etc/pki/tls/certs/fed-cloud09-nova.pem - #- ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=key value=/etc/pki/tls/private/fed-cloud09-nova.key - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_host value={{ controller_hostname }} - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=ssl_only value=False From df50e454487b61cd74711e829de01974ec71606c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 10 Mar 2015 13:03:21 +0000 Subject: [PATCH 63/78] F22 stuff for fedora-packages. --- roles/packages/web/files/distmappings.py | 4 ++ roles/packages/web/files/packages-yum.conf | 48 ++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/roles/packages/web/files/distmappings.py b/roles/packages/web/files/distmappings.py index f7355ae2c3..66a5e75703 100644 --- a/roles/packages/web/files/distmappings.py +++ b/roles/packages/web/files/distmappings.py @@ -1,6 +1,10 @@ # Global list of koji tags we care about tags = ({'name': 'Rawhide', 'tag': 'f22'}, + {'name': 'Fedora 22', 'tag': 'f22-updates'}, + {'name': 'Fedora 22', 'tag': 'f22'}, + {'name': 'Fedora 22 Testing', 'tag': 'f22-updates-testing'}, + {'name': 'Fedora 21', 'tag': 'f21-updates'}, {'name': 'Fedora 21', 'tag': 'f21'}, {'name': 'Fedora 21 Testing', 'tag': 'f21-updates-testing'}, diff --git a/roles/packages/web/files/packages-yum.conf b/roles/packages/web/files/packages-yum.conf index 1a39f55872..70f54729a5 100644 --- a/roles/packages/web/files/packages-yum.conf +++ b/roles/packages/web/files/packages-yum.conf @@ -45,6 +45,54 @@ enabled=0 gpgcheck=0 +[fedora-22-x86_64] +name=Fedora 22 +failovermethod=priority +baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/22/Everything/x86_64/os/ +#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-22&arch=x86_64 +enabled=0 +gpgcheck=0 + +[fedora-22-updates-x86_64] +name=Fedora 22 - Updates +failovermethod=priority +baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/22/x86_64/ +#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f22&arch=x86_64 +enabled=0 +gpgcheck=0 + +[fedora-22-testing-x86_64] +name=Fedora 22 - Testing +failovermethod=priority +baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/22/x86_64/ +#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f22&arch=x86_64 +enabled=0 +gpgcheck=0 + +[fedora-22-i686] +name=Fedora 22 +failovermethod=priority +baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/22/Everything/i386/os/ +#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-22&arch=i386 +enabled=0 +gpgcheck=0 + +[fedora-22-updates-i686] +name=Fedora 22 - Updates +failovermethod=priority +baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/22/i386/ +#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f22&arch=i386 +enabled=0 +gpgcheck=0 + +[fedora-22-testing-i686] +name=Fedora 22 - Testing +failovermethod=priority +baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/22/i386/ +#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f22&arch=i386 +enabled=0 + + [fedora-21-x86_64] name=Fedora 21 From 7df4b622efb6977ccd7ba4e269ff3ef5583f8e9c Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 10 Mar 2015 13:17:34 +0000 Subject: [PATCH 64/78] buildmaster: another try on date directory --- .../templates/taskotron.master.cfg.j2 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 954033cc59..64eeadf07c 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -185,6 +185,17 @@ factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron/artifac factory.addStep(FileUpload(slavesrc='/var/log/taskotron/taskotron.log', masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'), mode=0644)) + +import datetime +from buildbot.process.properties import renderer + +@renderer +def today(props): + return datetime.datetime.now().strftime("%Y%m%d%H%M%S") + +# move artifacts dir +factory.addStep(MasterShellCommand(command=["mkdir", '-p', '-m', '0755', Interpolate('{{ public_artifacts_dir }}/%(kw:today)s', today=today), '&&', 'mv', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/'), Interpolate('{{ public_artifacts_dir }}/%(kw:today)s/', today=today)], + descriptionDone=['Move artifacs dir'])) {% else %} # capture the taskotron log factory.addStep(ShellCommand(command=["cat", "/var/log/taskotron/taskotron.log"], name="cat_log", From a981011747b149abecfeb1ae9f485a200826fdd4 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 10 Mar 2015 13:23:49 +0000 Subject: [PATCH 65/78] buildmaster: do not include time in dir name --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 64eeadf07c..063e2817c0 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -191,7 +191,7 @@ from buildbot.process.properties import renderer @renderer def today(props): - return datetime.datetime.now().strftime("%Y%m%d%H%M%S") + return datetime.datetime.now().strftime("%Y%m%d") # move artifacts dir factory.addStep(MasterShellCommand(command=["mkdir", '-p', '-m', '0755', Interpolate('{{ public_artifacts_dir }}/%(kw:today)s', today=today), '&&', 'mv', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/'), Interpolate('{{ public_artifacts_dir }}/%(kw:today)s/', today=today)], From 1a6a0aa35dc849f099981a5f4fba92219709f749 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 10 Mar 2015 13:28:38 +0000 Subject: [PATCH 66/78] buildmaster: disable date directory for a bit --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 063e2817c0..d433f96554 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -185,7 +185,7 @@ factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron/artifac factory.addStep(FileUpload(slavesrc='/var/log/taskotron/taskotron.log', masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'), mode=0644)) - +''' import datetime from buildbot.process.properties import renderer @@ -196,6 +196,7 @@ def today(props): # move artifacts dir factory.addStep(MasterShellCommand(command=["mkdir", '-p', '-m', '0755', Interpolate('{{ public_artifacts_dir }}/%(kw:today)s', today=today), '&&', 'mv', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/'), Interpolate('{{ public_artifacts_dir }}/%(kw:today)s/', today=today)], descriptionDone=['Move artifacs dir'])) +''' {% else %} # capture the taskotron log factory.addStep(ShellCommand(command=["cat", "/var/log/taskotron/taskotron.log"], name="cat_log", From 47d2af583ba06e724261c53afc507a88fbb6c910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 15:09:14 +0000 Subject: [PATCH 67/78] compute node: restart services using handlers --- roles/cloud_compute/handlers/main.yml | 8 ++ roles/cloud_compute/tasks/main.yml | 142 +++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 roles/cloud_compute/handlers/main.yml diff --git a/roles/cloud_compute/handlers/main.yml b/roles/cloud_compute/handlers/main.yml new file mode 100644 index 0000000000..62468cf42a --- /dev/null +++ b/roles/cloud_compute/handlers/main.yml @@ -0,0 +1,8 @@ +- name: update-ca-trust + command: /usr/bin/update-ca-trust + +- name: restart neutron-openvswitch-agent + service: name=neutron-openvswitch-agent state=restarted + +- name: restart openstack-nova-compute + service: name=openstack-nova-compute state=restarted diff --git a/roles/cloud_compute/tasks/main.yml b/roles/cloud_compute/tasks/main.yml index 4470e8df36..2a631edaf6 100644 --- a/roles/cloud_compute/tasks/main.yml +++ b/roles/cloud_compute/tasks/main.yml @@ -1,5 +1,7 @@ --- # Configure another compute node for Fedora Cloud + handlers: + - include: "{{ handlers }}/restart_services.yml" - authorized_key: user=root key="{{ lookup('file', files + '/fedora-cloud/fed09-ssh-key.pub') }}" - template: src={{ files }}/fedora-cloud/hosts dest=/etc/hosts owner=root mode=0644 @@ -19,7 +21,9 @@ - name: add cert to ca-bundle.crt so plain curl works copy: src={{ private }}/files/openstack/fed-cloud09.pem dest=/etc/pki/ca-trust/source/anchors/ mode=600 owner=root group=root -- command: /usr/bin/update-ca-trust + notify: + - update ca-trust +- meta: flush_handlers - yum: state=present name=https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/rdo-release-icehouse-4.noarch.rpm @@ -35,35 +39,84 @@ - name: Set up db connection to controller ini_file: dest=/etc/nova/nova.conf section=database option=connection value=mysql://nova:{{NOVA_DBPASS}}@{{controller_private_ip}}/nova + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=auth_strategy value=keystone + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_uri value=https://{{controller_hostname}}:5000 + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_host value={{controller_hostname}} + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_protocol value=https + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=auth_port value=35357 + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem - + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=admin_user value=nova + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=admin_tenant_name value=services + notify: + - restart openstack-nova-compute - name: set admin_password ini_file: dest=/etc/nova/nova.conf section=keystone_authtoken option=admin_password value={{NOVA_PASS}} + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=rpc_backend value=nova.openstack.common.rpc.impl_kombu + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=rabbit_host value={{controller_private_ip}} + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=rabbit_hosts value={{controller_private_ip}}:5672 + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=rabbit_userid value=amqp_user + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=rabbit_password value={{ CONFIG_AMQP_AUTH_PASSWORD }} + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=rabbit_port value=5672 + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=rabbit_use_ssl value=False + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=my_ip value={{compute_private_ip}} + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vnc_enabled value=True + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vncserver_listen value=0.0.0.0 + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=vncserver_proxyclient_address value={{compute_private_ip}} + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=novncproxy_base_url value=https://{{controller_hostname}}:6080/vnc_auto.html + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_host value={{controller_hostname}} + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_protocol value=https + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=glance_api_servers value=https://{{ controller_hostname }}:9292 + notify: + - restart openstack-nova-compute - service: name=libvirtd state=started enabled=yes - service: name=messagebus state=started @@ -80,59 +133,138 @@ - openstack-neutron-openvswitch - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=auth_strategy value=keystone + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_uri value=https://{{controller_hostname}}:5000 + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_host value={{controller_hostname}} + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_protocol value=https + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=auth_port value=35357 + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=cafile value=/etc/pki/tls/certs/fed-cloud09-keystone.pem + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=admin_user value=neutron + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=admin_tenant_name value=services + notify: + - restart neutron-openvswitch-agent - name: set admin_password ini_file: dest=/etc/neutron/neutron.conf section=keystone_authtoken option=admin_password value={{NEUTRON_PASS}} + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=rpc_backend value=neutron.openstack.common.rpc.impl_kombu + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=rabbit_host value={{controller_private_ip}} + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=rabbit_hosts value={{controller_private_ip}}:5672 + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=rabbit_userid value=amqp_user + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=rabbit_password value={{ CONFIG_AMQP_AUTH_PASSWORD }} + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=rabbit_port value=5672 + notify: + - restart neutron-openvswitch-agent # uncomment if you want to debug compute instance #- ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=verbose value=True +# notify: +# - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=core_plugin value=neutron.plugins.ml2.plugin.Ml2Plugin + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/neutron.conf section=DEFAULT option=service_plugins value=neutron.services.l3_router.l3_router_plugin.L3RouterPlugin + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=ml2 option=type_drivers value=local,flat,gre + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=ml2 option=tenant_network_types value=gre + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=ml2 option=mechanism_drivers value=openvswitch + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=ml2_type_gre option=tunnel_id_ranges value=1:1000 + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=ovs option=local_ip value={{compute_private_ip}} + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=ovs option=tunnel_type value=gre + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=ovs option=tunnel_types value=gre + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=agent option=tunnel_types value=gre + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=ovs option=enable_tunneling value=True + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=securitygroup option=firewall_driver value=neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver + notify: + - restart neutron-openvswitch-agent - ini_file: dest=/etc/neutron/plugins/ml2/ml2_conf.ini section=securitygroup option=enable_security_group value=True + notify: + - restart neutron-openvswitch-agent # WORKAROUND https://ask.openstack.org/en/question/28734/instance-failed-to-spawn-you-must-call-aug-init-first-to-initialize-augeas/ - ini_file: dest=/usr/lib/systemd/system/neutron-openvswitch-agent.service section=Service option=ExecStart value="/usr/bin/neutron-openvswitch-agent --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini --log-file /var/log/neutron/openvswitch-agent.log" + notify: + - restart neutron-openvswitch-agent - service: name=openvswitch state=started enabled=yes - command: ovs-vsctl --may-exist add-br br-int - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=network_api_class value=nova.network.neutronv2.api.API + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_url value=https://{{controller_hostname}}:9696 + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_auth_strategy value=keystone + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_tenant_name value=services + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_username value=neutron + notify: + - restart openstack-nova-compute - name: set neutron_admin_password ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_password value={{NEUTRON_PASS}} + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=neutron_admin_auth_url value=https://{{controller_hostname}}:35357/v2.0 + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=linuxnet_interface_driver value=nova.network.linux_net.LinuxOVSInterfaceDriver + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=firewall_driver value=nova.virt.firewall.NoopFirewallDriver + notify: + - restart openstack-nova-compute - ini_file: dest=/etc/nova/nova.conf section=DEFAULT option=security_group_api value=neutron + notify: + - restart openstack-nova-compute - file: src=/etc/neutron/plugins/ml2/ml2_conf.ini dest=/etc/neutron/plugin.ini state=link - -- service: name=neutron-openvswitch-agent state=restarted enabled=yes -- service: name=openstack-nova-compute state=restarted enabled=yes + notify: + - restart openstack-nova-compute From 3933b10054e851d5daf3488f0b417a6e35dd6602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 15:30:18 +0000 Subject: [PATCH 68/78] immediately restart network when network config was changed --- .../hosts/fed-cloud09.cloud.fedoraproject.org.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index 33bad13d34..b8824c792a 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -93,20 +93,31 @@ when: packstack_sucessfully_finished.stat.exists == False ignore_errors: yes - lineinfile: dest=/etc/sysconfig/network-scripts/ifcfg-eth1 regexp="^ONBOOT=" line="ONBOOT=yes" + notify: + - restart network # only for first run - lineinfile: dest=/etc/sysconfig/network-scripts/ifcfg-eth1 regexp="^NETMASK=" line="NETMASK=255.255.255.0" when: packstack_sucessfully_finished.stat.exists == False + notify: + - restart network - lineinfile: dest=/etc/sysconfig/network-scripts/ifcfg-eth1 regexp="^IPADDR=" line="IPADDR={{controller_private_ip}}" when: packstack_sucessfully_finished.stat.exists == False + notify: + - restart network - lineinfile: dest=/etc/sysconfig/network-scripts/ifcfg-eth1 regexp="BOOTPROTO=" line="BOOTPROTO=none" + notify: + - restart network - template: src={{files}}/fedora-cloud/ifcfg-br-ex dest=/etc/sysconfig/network-scripts/ifcfg-br-ex owner=root mode=0644 when: packstack_sucessfully_finished.stat.exists == False + notify: + - restart network - template: src={{files}}/fedora-cloud/ifcfg-eth0 dest=/etc/sysconfig/network-scripts/ifcfg-eth0 owner=root mode=0644 when: packstack_sucessfully_finished.stat.exists == False + notify: + - restart network - command: ifup eth1 when: packstack_sucessfully_finished.stat.exists == False - - # FIXME notify network service restart, eth1 must be up and configured + - meta: flush_handlers # http://docs.openstack.org/trunk/install-guide/install/yum/content/basics-ntp.html - service: name=ntpd state=started enabled=yes From 6916d104ac69c179676d7fed984028d0d88f6547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 15:34:30 +0000 Subject: [PATCH 69/78] typo --- roles/cloud_compute/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cloud_compute/handlers/main.yml b/roles/cloud_compute/handlers/main.yml index 62468cf42a..574ac98963 100644 --- a/roles/cloud_compute/handlers/main.yml +++ b/roles/cloud_compute/handlers/main.yml @@ -1,4 +1,4 @@ -- name: update-ca-trust +- name: update ca-trust command: /usr/bin/update-ca-trust - name: restart neutron-openvswitch-agent From ab3dbaf4df0bc2cfa56182b9b20d7d2111018df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 15:35:21 +0000 Subject: [PATCH 70/78] try this --- roles/cloud_compute/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cloud_compute/handlers/main.yml b/roles/cloud_compute/handlers/main.yml index 574ac98963..5fd3101e46 100644 --- a/roles/cloud_compute/handlers/main.yml +++ b/roles/cloud_compute/handlers/main.yml @@ -1,4 +1,4 @@ -- name: update ca-trust +- name: "update ca-trust" command: /usr/bin/update-ca-trust - name: restart neutron-openvswitch-agent From 2764fefc722e3b3595c7106467266908787c91de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 15:36:30 +0000 Subject: [PATCH 71/78] try this --- roles/cloud_compute/handlers/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/cloud_compute/handlers/main.yml b/roles/cloud_compute/handlers/main.yml index 5fd3101e46..676e798a85 100644 --- a/roles/cloud_compute/handlers/main.yml +++ b/roles/cloud_compute/handlers/main.yml @@ -1,8 +1,8 @@ - name: "update ca-trust" command: /usr/bin/update-ca-trust -- name: restart neutron-openvswitch-agent +- name: "restart neutron-openvswitch-agent" service: name=neutron-openvswitch-agent state=restarted -- name: restart openstack-nova-compute +- name: "restart openstack-nova-compute" service: name=openstack-nova-compute state=restarted From 58abe3e92231745d91bbe6e90536ee3b2c737fb4 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 10 Mar 2015 15:48:58 +0000 Subject: [PATCH 72/78] Simplify this line --- roles/cloud_compute/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cloud_compute/tasks/main.yml b/roles/cloud_compute/tasks/main.yml index 2a631edaf6..dd9fb00c4a 100644 --- a/roles/cloud_compute/tasks/main.yml +++ b/roles/cloud_compute/tasks/main.yml @@ -3,7 +3,7 @@ handlers: - include: "{{ handlers }}/restart_services.yml" -- authorized_key: user=root key="{{ lookup('file', files + '/fedora-cloud/fed09-ssh-key.pub') }}" +- authorized_key: user=root key="{{ files }}/fedora-cloud/fed09-ssh-key.pub" - template: src={{ files }}/fedora-cloud/hosts dest=/etc/hosts owner=root mode=0644 - lineinfile: dest=/etc/sysconfig/network-scripts/ifcfg-eth1 regexp="^ONBOOT=" line="ONBOOT=yes" From af46c8bb427c07a8dacf7d7c7533ae26adbfbc55 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 10 Mar 2015 15:53:02 +0000 Subject: [PATCH 73/78] Drop handlers here. --- roles/cloud_compute/tasks/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/cloud_compute/tasks/main.yml b/roles/cloud_compute/tasks/main.yml index dd9fb00c4a..99f5507fdd 100644 --- a/roles/cloud_compute/tasks/main.yml +++ b/roles/cloud_compute/tasks/main.yml @@ -1,8 +1,5 @@ --- # Configure another compute node for Fedora Cloud - handlers: - - include: "{{ handlers }}/restart_services.yml" - - authorized_key: user=root key="{{ files }}/fedora-cloud/fed09-ssh-key.pub" - template: src={{ files }}/fedora-cloud/hosts dest=/etc/hosts owner=root mode=0644 From 5bfbe1ed227ace106aad7fc563c6c9114e4db923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Tue, 10 Mar 2015 16:00:35 +0000 Subject: [PATCH 74/78] Revert "Simplify this line" This reverts commit 58abe3e92231745d91bbe6e90536ee3b2c737fb4. Conflicts: roles/cloud_compute/tasks/main.yml --- roles/cloud_compute/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/cloud_compute/tasks/main.yml b/roles/cloud_compute/tasks/main.yml index 99f5507fdd..70da22f617 100644 --- a/roles/cloud_compute/tasks/main.yml +++ b/roles/cloud_compute/tasks/main.yml @@ -1,6 +1,7 @@ --- # Configure another compute node for Fedora Cloud -- authorized_key: user=root key="{{ files }}/fedora-cloud/fed09-ssh-key.pub" + +- authorized_key: user=root key="{{ lookup('file', files + '/fedora-cloud/fed09-ssh-key.pub') }}" - template: src={{ files }}/fedora-cloud/hosts dest=/etc/hosts owner=root mode=0644 - lineinfile: dest=/etc/sysconfig/network-scripts/ifcfg-eth1 regexp="^ONBOOT=" line="ONBOOT=yes" From 2f2ca62aa67762acdc517b59a1201e4dee2aa7c4 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 10 Mar 2015 15:33:37 +0000 Subject: [PATCH 75/78] (fedimg) keys should be readable, not executable. --- roles/fedimg/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index b18f54fd0f..fb8f1029d7 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -63,7 +63,7 @@ - name: copy keys into pki directory for production copy: src={{private}}/files/fedimg/{{item}} dest=/etc/pki/fedimg/{{item}} - owner=fedmsg group=fedmsg mode=0100 + owner=fedmsg group=fedmsg mode=0400 with_items: - fedimg-prod - fedimg-prod.pub From c8028bf0bbbb5348b5268a89bbff48292b486ad5 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 10 Mar 2015 19:15:16 +0000 Subject: [PATCH 76/78] Throw up an icon for fedimg. --- roles/apps-fp-o/files/img/icons/fedimg.png | Bin 0 -> 1554 bytes roles/apps-fp-o/tasks/main.yml | 5 +++++ 2 files changed, 5 insertions(+) create mode 100644 roles/apps-fp-o/files/img/icons/fedimg.png diff --git a/roles/apps-fp-o/files/img/icons/fedimg.png b/roles/apps-fp-o/files/img/icons/fedimg.png new file mode 100644 index 0000000000000000000000000000000000000000..2cc02f0f738f2a5767e5ad8e948648fefc3d0a7d GIT binary patch literal 1554 zcmV+t2JQKYP)KYK~!jg<(g}3R7DiWe>3-PUvx_!JeopcdBjLa%OhaaAXq4B zG$BS4P=S^rf#4?-H1ep)CKffuM~p_rL}+<5vBAWINCFf#5kw;|L4#5#0RvdD-Ii`? zY4_ece%S49AG>$A+mkv6lX$94~DM^`+AE56fId4FhqEIe{H6cM68FQ6Sy~QU)wBHaZmQ zeRTc3GFQ>vSR7(XV3*51)ilf+VAi1I#m;FEOIKAas*h$Eb>3E%A6a4=<|*!-c!1fU zPE~F`^lUUk+XO1N9DW_F?|_UUlNLLpkz!wEdF=)}qliJe?ECdkfB zCp$MCsYPC`lTvkZ@|H;t-Qh@Z#6z07!o$>jZ z09dBttot&*{cshfRM0MOY7X2^M=`lD*G>v3)>OOPV}c1t zEvW6w1_%yE(Jvc~)Oe(6~$wxetpBS2c0VXTDE6j~n5!|opuKnwJT z8~tv&`=pJH|5~v;e#A=va<|Jpk$bz(87k)bxAvy_M52|%i{{f7NCk5fEhKKF^{~WZ zTB3EtjS>$@U`nEO41y^VL1uKj5lgmY=nm$Iz$qC-X=MbUB;ZZ7ib3F&DB>z8jS!k8 zg1SVjh#NuO{ha{nq7)X#6D=cdERIRxFx`NPYCQ6=42S8KC9bD#0VfizW}pc8d5!C7 zPxr=qXEX=Pw7*GapkG;$?JU#sf7;jpScVl%4wewRAaJ0u%&r=s!_J|(A@bwG*J;q! z?05Ir*m?eL1HkL)z}IO;wzG*5pgmcWdm)g(>UW>{M+v^L^Qb@DK+}!zflnQ*#|(_x z=|lCHzWyU<`Yg`W`isq`5&GVj#_O$IK6lezzTmT$T}6$-1OTKZJ_7DU@==z;U44#g zbxpxGqs!#vfs53gxE^e4ZEWZ8?z4D3p&RP;bWpqNS6Z9egKdBQa*b05E@AjWcjDTG zM%>lsY^NdMRhss3D3=}tD#0Jvc>_4JQb|hHX>DqcI2D8xWQ`kz0$Og~=~XxCi6dDj zBPW&4j?gr;L%?gL6~&))|J|#!mZp0?Z8x$@fI_<<-sTQyw!e_F6rP5*sDgZ*M#QNZ zhBJ3>W%jC=_d5PBbuH~Olvn~ZG4g1Xx_WNiS95$l8~fES)@)sLRVh&h823zCbPR9={&1bPhncFkYF8-5cEDk0H;i Date: Tue, 10 Mar 2015 19:38:47 +0000 Subject: [PATCH 77/78] Typofix. --- roles/apps-fp-o/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/apps-fp-o/tasks/main.yml b/roles/apps-fp-o/tasks/main.yml index e4a5d5b62b..fd7b11153b 100644 --- a/roles/apps-fp-o/tasks/main.yml +++ b/roles/apps-fp-o/tasks/main.yml @@ -30,6 +30,6 @@ - apps-fp-o - name: Copy over any extra icons we carry - sync: src=img/icons dest=/srv/web/apps-fp-o/img/icons + synchronize: src=img/icons dest=/srv/web/apps-fp-o/img/icons tags: - apps-fp-o From cacbbe6ec2211c20254b25cfa3ef5572588d53d2 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 10 Mar 2015 19:40:31 +0000 Subject: [PATCH 78/78] My goodness. Do trailing slashes matter here? --- roles/apps-fp-o/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/apps-fp-o/tasks/main.yml b/roles/apps-fp-o/tasks/main.yml index fd7b11153b..9b1644750b 100644 --- a/roles/apps-fp-o/tasks/main.yml +++ b/roles/apps-fp-o/tasks/main.yml @@ -30,6 +30,6 @@ - apps-fp-o - name: Copy over any extra icons we carry - synchronize: src=img/icons dest=/srv/web/apps-fp-o/img/icons + synchronize: src=img/icons/ dest=/srv/web/apps-fp-o/img/icons/ tags: - apps-fp-o