First cut at migrating openqa-stg to fedmsg-3

This includes some tweaks to the core fedmsg roles to allow a
'generic' way of indicating that a box should use fedmsg-hub-3
not fedmsg-hub, and make the restart notification work for that.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

handlers/restart_services.yml

@@ -18,7 +18,10 @@
   command: /usr/local/bin/conditional-restart.sh fedmsg-gateway
 
 - name: restart fedmsg-hub
-  command: /usr/local/bin/conditional-restart.sh fedmsg-hub
+  command: /usr/local/bin/conditional-restart.sh {{ item }}
+  with_items:
+  - fedmsg-hub
+  - fedmsg-hub-3
 
 - name: restart fedmsg-hub-3
   command: /usr/local/bin/conditional-restart.sh fedmsg-hub-3

inventory/inventory

@@ -1033,6 +1033,7 @@ mailman
 mailman-stg
 mdapi
 mdapi-stg
+openqa-stg
 
 ## END fedmsg services
 

roles/autocloudreporter/tasks/main.yml

@@ -11,25 +11,58 @@
 ##              EVER BE TRUE ON ONE SYSTEM IN THE WORLD**
 ##       default - False
 
-- name: Install required packages
+- name: Install required packages (Python 2)
   dnf:
     name: ['python2-fedfind', 'python2-fedmsg-consumers', 'python2-resultsdb_api',
            'python2-resultsdb_conventions-fedora', 'python2-setuptools']
     state: present
+  when: "'python34-fedmsg' not in group_names"
   tags:
   - packages
 
+- name: Install required packages (Python 3)
+  dnf:
+    name: ['python3-fedfind', 'python3-fedmsg-consumers', 'python3-resultsdb_api',
+           'python3-resultsdb_conventions-fedora', 'python3-setuptools']
+    state: present
+  when: "'python34-fedmsg' in group_names"
+  tags:
+  - packages
+
+- name: Check if Python 2 autocloudreporter is installed
+  find:
+    paths: /usr/lib/python2.7/site-packages
+    patterns: "autocloudreporter*"
+    register: py2acrinstalled
+  changed_when: "1 != 1"
+  failed_when: "1 != 1"
+  check_mode: no
+
 - name: Check out autocloudreporter
   git:
     repo: https://pagure.io/fedora-qa/autocloudreporter.git
     dest: /root/autocloudreporter
   register: gitacr
 
-- name: Install autocloudreporter
-  command: "python setup.py install"
+- name: Install autocloudreporter (Python 2)
+  command: "python2 setup.py install"
   args:
     chdir: /root/autocloudreporter
-  when: "gitacr is changed"
+  when: "gitacr is changed and 'python34-fedmsg' not in group_names"
+  notify:
+  - restart fedmsg-hub
+
+- name: Remove autocloudreporter (Python 2)
+  command: "pip2 --disable-pip-version-check uninstall autocloudreporter"
+  when: "py2acrinstalled is defined and py2acrinstalled|length>0 and 'python34-fedmsg' in group_names"
+  notify:
+  - restart fedmsg-hub
+
+- name: Install autocloudreporter (Python 3)
+  command: "python3 setup.py install"
+  args:
+    chdir: /root/autocloudreporter
+  when: "(gitcc is changed or (py2acrinstalled is defined and py2acrinstalled|length>0)) and 'python34-fedmsg' in group_names"
   notify:
   - restart fedmsg-hub
 
@@ -39,3 +72,12 @@
   - restart fedmsg-hub
   tags:
   - config
+
+- name: Remove Python 2 packages
+  dnf:
+    name: ['python2-fedfind', 'python2-fedmsg-consumers', 'python2-resultsdb_api',
+           'python2-resultsdb_conventions-fedora', 'python2-setuptools']
+    state: absent
+  when: "'python34-fedmsg' in group_names"
+  tags:
+  - packages

roles/check-compose/tasks/main.yml

@@ -40,25 +40,58 @@
 #  tags:
 #  - packages
 
-- name: Install required packages
+- name: Install required packages (Python 2)
   dnf:
     name: ['python2-fedfind', 'python2-fedmsg-consumers', 'python2-openqa_client',
            'python2-setuptools', 'python2-six']
     state: present
+  when: "'python34-fedmsg' not in group_names"
   tags:
   - packages
 
+- name: Install required packages (Python 3)
+  dnf:
+    name: ['python2-pip', 'python3-fedfind', 'python3-fedmsg-consumers', 'python3-openqa_client',
+           'python3-setuptools', 'python3-six']
+    state: present
+  when: "'python34-fedmsg' in group_names"
+  tags:
+  - packages
+
+- name: Check if Python 2 check-compose is installed
+  find:
+    paths: /usr/lib/python2.7/site-packages
+    patterns: "check_compose*"
+    register: py2ccinstalled
+  changed_when: "1 != 1"
+  failed_when: "1 != 1"
+  check_mode: no
+
 - name: Check out check-compose
   git:
     repo: https://pagure.io/fedora-qa/check-compose.git
     dest: /root/check-compose
   register: gitcc
 
-- name: Install check-compose
-  command: "python setup.py install"
+- name: Install check-compose (Python 2)
+  command: "python2 setup.py install"
   args:
     chdir: /root/check-compose
-  when: "gitcc is changed"
+  when: "gitcc is changed and 'python34-fedmsg' not in group_names"
+  notify:
+  - restart fedmsg-hub
+
+- name: Remove check-compose (Python 2)
+  command: "pip2 --disable-pip-version-check uninstall check-compose"
+  when: "py2ccinstalled is defined and py2ccinstalled|length>0 and 'python34-fedmsg' in group_names"
+  notify:
+  - restart fedmsg-hub
+
+- name: Install check-compose (Python 3)
+  command: "python3 setup.py install"
+  args:
+    chdir: /root/check-compose
+  when: "(gitcc is changed or (py2ccinstalled is defined and py2ccinstalled|length>0)) and 'python34-fedmsg' in group_names"
   notify:
   - restart fedmsg-hub
 
@@ -69,6 +102,15 @@
   tags:
   - config
 
+- name: Remove Python 2 packages
+  dnf:
+    name: ['python2-fedfind', 'python2-fedmsg-consumers', 'python2-openqa_client',
+           'python2-setuptools', 'python2-six']
+    state: absent
+  when: "'python34-fedmsg' in group_names"
+  tags:
+  - packages
+
 - name: Install config file
   template: src=check-compose.conf.j2 dest=/etc/check-compose.conf mode=0644
   tags:

roles/fedmsg/base/tasks/main.yml

@@ -29,7 +29,7 @@
   tags: fedmsg/base
 
 - name: install the python3 fedmsg package (dnf)
-  dnf: pkg=python3-fedmsg-core state=present
+  dnf: pkg=python3-fedmsg state=present
   when: "'python34-fedmsg' in group_names and ansible_distribution_major_version|int > 21"
   tags: fedmsg/base
 

roles/fedmsg/hub/tasks/main.yml

@@ -5,16 +5,31 @@
   package: name=fedmsg-hub state=present
   tags:
   - packages
-  when: ansible_distribution_major_version|int < 22
+  when: "'python34-fedmsg' not in group_names and ansible_distribution_major_version|int < 22"
 
 - name: install needed packages
   dnf: pkg=fedmsg-hub state=present
   tags:
   - packages
-  when: ansible_distribution_major_version|int > 21
+  when: "'python34-fedmsg' not in group_names and ansible_distribution_major_version|int > 21"
+
+- name: install the python34 fedmsg package (yum)
+  package: name=python34-fedmsg-core state=present
+  when: "'python34-fedmsg' in group_names and ansible_distribution_major_version|int < 22"
+  tags: fedmsg/base
+
+- name: install the python3 fedmsg package (dnf)
+  dnf: pkg=python3-fedmsg state=present
+  when: "'python34-fedmsg' in group_names and ansible_distribution_major_version|int > 21"
+  tags: fedmsg/base
 
 - name: fedmsg-hub service
   service: name=fedmsg-hub state=started enabled=yes
+  when: "'python34-fedmsg' not in group_names"
+
+- name: fedmsg-hub-3 service
+  service: name=fedmsg-hub-3 state=started enabled=yes
+  when: "'python34-fedmsg' in group_names"
 
 - name: enable the websocket server if we should
   copy: src=websockets.py dest=/etc/fedmsg.d/websockets.py

roles/openqa/dispatcher/tasks/main.yml

@@ -49,20 +49,39 @@
 #  tags:
 #  - packages
 
-- name: Install required packages
+- name: Install required packages (Python 2)
   dnf:
     name: ['python2-fedfind', 'python2-wikitcms', 'python2-fedmsg-consumers',
            'python2-openqa_client', 'python2-requests', 'python2-resultsdb_api',
            'python2-resultsdb_conventions-fedora', 'python2-setuptools', 'python-six']
     state: present
+  when: "'python34-fedmsg' not in group_names"
   tags:
   - packages
 
-- name: Install required packages (wiki oidc auth)
+- name: Install required packages (wiki oidc auth) (Python 2)
   dnf:
     name: python2-openidc-client
     state: present
-  when: "wikitcms_token is defined"
+  when: "wikitcms_token is defined and 'python34-fedmsg' not in group_names"
+  tags:
+  - packages
+
+- name: Install required packages (Python 3)
+  dnf:
+    name: ['python2-pip', 'python3-fedfind', 'python3-wikitcms', 'python3-fedmsg-consumers',
+           'python3-openqa_client', 'python3-requests', 'python3-resultsdb_api',
+           'python3-resultsdb_conventions-fedora', 'python3-setuptools', 'python3-six']
+    state: present
+  when: "'python34-fedmsg' in group_names"
+  tags:
+  - packages
+
+- name: Install required packages (wiki oidc auth) (Python 3)
+  dnf:
+    name: python3-openidc-client
+    state: present
+  when: "wikitcms_token is defined and 'python34-fedmsg' in group_names"
   tags:
   - packages
 
@@ -95,14 +114,47 @@
   failed_when: "1 != 1"
   check_mode: no
 
-- name: Install fedora_openqa
-  command: "python setup.py install"
+- name: Check if Python 2 fedora_openqa is installed
+  find:
+    paths: /usr/lib/python2.7/site-packages
+    patterns: "fedora_openqa*"
+    register: py2oqainstalled
+  changed_when: "1 != 1"
+  failed_when: "1 != 1"
+  check_mode: no
+
+- name: Install fedora_openqa (Python 2)
+  command: "python2 setup.py install"
   args:
     chdir: /root/fedora_openqa
-  when: "gittools is changed or not insttools.stat.exists"
+  when: "(gittools is changed or not insttools.stat.exists) and 'python34-fedmsg' not in group_names"
   notify:
   - restart fedmsg-hub
 
+- name: Remove fedora_openqa (Python 2)
+  command: "pip2 --disable-pip-version-check uninstall fedora-openqa"
+  when: "py2oqainstalled is defined and py2oqainstalled|length>0 and 'python34-fedmsg' in group_names"
+  notify:
+  - restart fedmsg-hub
+
+- name: Install fedora_openqa (Python 3)
+  command: "python3 setup.py install"
+  args:
+    chdir: /root/fedora_openqa
+  when: "(gittools is changed or not insttools.stat.exists or (py2oqainstalled is defined and py2oqainstalled|length>0)) and 'python34-fedmsg' in group_names"
+  notify:
+  - restart fedmsg-hub
+
+- name: Remove Python 2 packages
+  dnf:
+    name: ['python2-fedfind', 'python2-wikitcms', 'python2-fedmsg', 'python2-fedmsg-consumers',
+           'python2-openqa_client', 'python2-requests', 'python2-resultsdb_api',
+           'python2-resultsdb_conventions-fedora', 'python2-setuptools', 'python-six']
+    state: absent
+  when: "'python34-fedmsg' in group_names"
+  tags:
+  - packages
+
 # For now, we want ppc64 jobs generated on staging but not prod (as we
 # have ppc64 workers for stg but not prod). So we have a WANTED file
 # for staging.

roles/openqa/server/tasks/main.yml

@@ -67,12 +67,23 @@
 
 # this is separate from the step below so we can easily flip it between
 # stable and testing
-- name: Install openQA packages
+- name: Install openQA packages (Python 2)
   dnf:
     name: ['openqa', 'openqa-httpd', 'openqa-plugin-fedmsg', 'openqa-plugin-fedoraupdaterestart',
            'python2-fedfind', 'python3-fedfind']
     state: present
     enablerepo: "updates-testing"
+  when: "'python34-fedmsg' not in group_names"
+  tags:
+  - packages
+
+- name: Install openQA packages (Python 3)
+  dnf:
+    name: ['openqa', 'openqa-httpd', 'openqa-plugin-fedmsg', 'openqa-plugin-fedoraupdaterestart',
+           'python3-fedfind']
+    state: present
+    enablerepo: "updates-testing"
+  when: "'python34-fedmsg' in group_names"
   tags:
   - packages
 

roles/relvalconsumer/tasks/main.yml

@@ -23,20 +23,45 @@
 #  tags:
 #  - packages
 
-- name: Install required packages
+- name: Install required packages (Python 2)
   dnf:
     name: ['python2-fedfind', 'python2-wikitcms', 'python2-fedmsg-consumers', 'python2-mwclient',
            'python2-setuptools', 'relval']
     state: present
+  when: "'python34-fedmsg' not in group_names"
   tags:
   - packages
 
-- name: Install required packages (wiki oidc auth)
+- name: Install required packages (wiki oidc auth) (Python 2)
   dnf: name=python2-openidc-client state=present enablerepo="updates"
-  when: "wikitcms_token is defined"
+  when: "wikitcms_token is defined and 'python34-fedmsg' not in group_names"
   tags:
   - packages
 
+- name: Install required packages (Python 3)
+  dnf:
+    name: ['python3-fedfind', 'python3-wikitcms', 'python3-fedmsg-consumers', 'python3-mwclient',
+           'python3-setuptools', 'relval']
+    state: present
+  when: "'python34-fedmsg' in group_names"
+  tags:
+  - packages
+
+- name: Install required packages (wiki oidc auth) (Python 3)
+  dnf: name=python3-openidc-client state=present enablerepo="updates"
+  when: "wikitcms_token is defined and 'python34-fedmsg' in group_names"
+  tags:
+  - packages
+
+- name: Check if Python 2 relvalconsumer is installed
+  find:
+    paths: /usr/lib/python2.7/site-packages
+    patterns: "relvalconsumer*"
+    register: py2rvinstalled
+  changed_when: "1 != 1"
+  failed_when: "1 != 1"
+  check_mode: no
+
 # next two duplicated with openqa/dispatcher, but I kinda don't want to
 # move them up somewhere shared, I like the roles to be somewhat usable
 # outside of Fedora infra...
@@ -78,11 +103,25 @@
     dest: /root/relvalconsumer
   register: gitrvc
 
-- name: Install relvalconsumer
-  command: "python setup.py install"
+- name: Install relvalconsumer (Python 2)
+  command: "python2 setup.py install"
   args:
     chdir: /root/relvalconsumer
-  when: "gitrvc is changed"
+  when: "gitrvc is changed and 'python34-fedmsg' not in group_names"
+  notify:
+  - restart fedmsg-hub
+
+- name: Remove relvalconsumer (Python 2)
+  command: "pip2 uninstall relvalconsumer"
+  when: "py2rvinstalled is defined and py2rvinstalled|length>0 and 'python34-fedmsg' in group_names"
+  notify:
+  - restart fedmsg-hub
+
+- name: Install relvalconsumer (Python 3)
+  command: "python3 setup.py install"
+  args:
+    chdir: /root/relvalconsumer
+  when: "(gitrvc is changed or (py2rvinstalled is defined and py2rvinstalled|length>0)) and 'python34-fedmsg' in group_names"
   notify:
   - restart fedmsg-hub
 
@@ -93,3 +132,12 @@
   - restart fedmsg-hub
   tags:
   - config
+
+- name: Remove Python 2 packages
+  dnf:
+    name: ['python2-fedfind', 'python2-wikitcms', 'python2-fedmsg-consumers', 'python2-mwclient',
+           'python2-setuptools']
+    state: absent
+  when: "'python34-fedmsg' in group_names"
+  tags:
+  - packages