Merge branch 'master' of /git/ansible

files/hotfix/fedimg/consumers.py

@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+# This file is part of fedimg.
+# Copyright (C) 2014-2017 Red Hat, Inc.
+#
+# fedimg is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# fedimg is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with fedimg; if not, see http://www.gnu.org/licenses,
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Authors:  David Gay <dgay@redhat.com>
+#           Sayan Chowdhury <sayanchowdhury@fedoraproject.org>
+"""
+This is the `fedmsg consumer`_ that subscribes to the topic emitted after the
+completion of the nightly and production compose. The consumer on receving the
+message uploads the image using the API of the cloud providers.
+"""
+
+import logging
+import multiprocessing.pool
+
+import fedmsg.consumers
+import fedmsg.encoding
+import fedfind.release
+
+import fedimg.uploader
+
+from fedimg.config import PROCESS_COUNT, STATUS_FILTER
+from fedimg.utils import get_rawxz_urls, get_value_from_dict
+
+LOG = logging.getLogger(__name__)
+
+
+class FedimgConsumer(fedmsg.consumers.FedmsgConsumer):
+    """
+    A `fedmsg consumer`_ that listens to the pungi compose topics and kicks
+    of the process to upload the images to various cloud providers.
+
+    Attributes:
+        topic (str): The topics this consumer is subscribed to. Set to
+            ``org.fedoraproject.prod.pungi.compose.status.change``.
+        config_key (str): The key to set to ``True`` in the fedmsg config to
+            enable this consumer. The key is ``fedimgconsumer.prod.enabled``.
+    """
+    topic = ['org.fedoraproject.prod.pungi.compose.status.change']
+    config_key = "fedimgconsumer.prod.enabled"
+
+    def __init__(self, *args, **kwargs):
+        LOG.info("FedimgConsumer initializing")
+        super(FedimgConsumer, self).__init__(*args, **kwargs)
+
+        # Threadpool for upload jobs
+        LOG.info("Creating thread pool of %s process", PROCESS_COUNT)
+        self.upload_pool = multiprocessing.pool.ThreadPool(
+            processes=PROCESS_COUNT
+        )
+        LOG.info("FedimgConsumer initialized")
+
+    def consume(self, msg):
+        """
+        This is called when we receive a message matching our topics.
+
+        Args:
+            msg (dict): The raw message from fedmsg.
+        """
+        LOG.info('Received %r %r', msg['topic'], msg['body']['msg_id'])
+
+        msg_info = msg['body']['msg']
+        if msg_info['status'] not in STATUS_FILTER:
+            return
+
+        location = msg_info['location']
+        compose_id = msg_info['compose_id']
+        compose_metadata = fedfind.release.get_release(cid=compose_id).metadata
+
+        # Till F27, both cloud-base and atomic images were available
+        # under variant CloudImages. With F28 and onward releases,
+        # cloud-base image compose moved to cloud variant and atomic images
+        # moved under atomic variant.
+        prev_rel = ['26', '27']
+        if msg_info['release_version'] in prev_rel:
+            images_meta = get_value_from_dict(
+                compose_metadata, 'images', 'payload', 'images', 'CloudImages',
+                'x86_64')
+        else:
+            images_meta = get_value_from_dict(
+                compose_metadata, 'images', 'payload', 'images',
+                'Cloud', 'x86_64')
+            images_meta.extend(get_value_from_dict(
+                compose_metadata, 'images', 'payload',
+                'images', 'AtomicHost', 'x86_64'))
+
+        if images_meta is None:
+            LOG.debug('No compatible image found to process')
+            return
+
+        upload_urls = get_rawxz_urls(location, images_meta)
+        if len(upload_urls) > 0:
+            LOG.info("Start processing compose id: %s", compose_id)
+            fedimg.uploader.upload(
+                pool=self.upload_pool,
+                urls=upload_urls,
+                compose_id=compose_id
+            )
+
+
+class FedimgStagingConsumer(FedimgConsumer):
+    """
+    A `fedmsg consumer`_ that listens to the staging pungi compose topics and
+    kicks of the process to upload the images to various cloud providers.
+
+    Attributes:
+        topic (str): The topics this consumer is subscribed to. Set to
+            ``org.fedoraproject.stg.pungi.compose.status.change``.
+        config_key (str): The key to set to ``True`` in the fedmsg config to
+            enable this consumer. The key is ``fedimgconsumer.stg.enabled``.
+    """
+    topic = ['org.fedoraproject.stg.pungi.compose.status.change']
+    config_key = "fedimgconsumer.stg.enabled"
+
+
+class FedimgDevConsumer(FedimgConsumer):
+    """
+    A `fedmsg consumer`_ that listens to the dev pungi compose topics and
+    kicks of the process to upload the images to various cloud providers.
+
+    Attributes:
+        topic (str): The topics this consumer is subscribed to. Set to
+            ``org.fedoraproject.dev.pungi.compose.status.change``.
+        config_key (str): The key to set to ``True`` in the fedmsg config to
+            enable this consumer. The key is ``fedimgconsumer.dev.enabled``.
+    """
+    topic = ['org.fedoraproject.dev.pungi.compose.status.change']
+    config_key = "fedimgconsumer.dev.enabled"
+

inventory/group_vars/pdc-backend

@@ -23,18 +23,18 @@ csi_relationship: |
  fedmsg-hub daemon that loads the pdc-updater consumer plugin.  However, the
  pdc-updater plugin is configured to do different things in each place.
 
- On pdc-updater01, the compose handler is enabled which listens for new pungi
+ On pdc-backend01, the compose handler is enabled which listens for new pungi
  composes, and stores them in PDC.  Fedora QE uses this data.  The consumer
  has only a single thread enabled to avoid OOMing itself with more than one
  compose at a time.
 
- On pdc-updater02, the modularity handlers are enabled which listen for MBS
- activity, and store that in PDC.  pdc-updater02 also hosts the retirement
+ On pdc-backend02, the modularity handlers are enabled which listen for MBS
+ activity, and store that in PDC.  pdc-backend02 also hosts the retirement
  handler which listens to dist-git for new dead.package files, and propagates
  the retirement to PDC (by prematurely EOLing the branch).  Multiple threads are
  enabled so that it can work more efficiently on these smaller tasks.
 
- On pdc-updater03, the dep chain handlers are enabled which listen for koji
+ On pdc-backend03, the dep chain handlers are enabled which listen for koji
  messages and store dep chain information in PDC, like what rpms depend on what
  other rpms at build time, and what containers depend on what rpms, etc..
  Multiple threads are enabled so that it can work more efficiently on these

inventory/group_vars/pdc-backend-stg

@@ -23,11 +23,11 @@ csi_relationship: |
  a fedmsg-hub daemon that loads the pdc-updater consumer plugin.  However, the
  pdc-updater plugin is configured to do different things in each place.
 
- On pdc-updater01, the compose handler is enabled which listens for new pungi
+ On pdc-backend01, the compose handler is enabled which listens for new pungi
  composes, and stores them in PDC.  Fedora QE uses this data.  The consumer
  has only a single thread enabled to avoid OOMing itself with more than one
  compose at a time.
 
- On pdc-updater02, the dep chain and modularity handlers are enabled which
+ On pdc-backend02, the dep chain and modularity handlers are enabled which
  listen for koji and MBS activity, and store that in PDC.  Multiple threads
  are enabled so that it can work more efficiently on these smaller tasks.

inventory/group_vars/resultsdb-prod

@@ -76,3 +76,4 @@ fedmsg_certs:
   group: apache
   can_send:
   - taskotron.result.new
+  - resultsdb.result.new

inventory/group_vars/resultsdb-stg

@@ -73,3 +73,4 @@ fedmsg_certs:
   group: apache
   can_send:
   - taskotron.result.new
+  - resultsdb.result.new

playbooks/groups/osbs-cluster.yml

@@ -305,7 +305,8 @@
         state: restarted
 
   tasks:
-
+    - name: Ensures /etc/dnsmasq.d/ dir exists
+      file: path="/etc/dnsmasq.d/" state=directory
     - name: install fedora dnsmasq specific config
       copy:
         src: "{{files}}/osbs/fedora-dnsmasq.conf.{{env}}"
@@ -324,12 +325,6 @@
     - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
 
   tasks:
-    - name: set policy for koji builder in openshift for osbs
-      shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_stg_username }} && touch /etc/origin/koji-builder-policy-added"
-      args:
-        creates: "/etc/origin/koji-builder-policy-added"
-      when: env == "staging"
-
     - name: set policy for koji builder in openshift for osbs
       shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_prod_username }} && touch /etc/origin/koji-builder-policy-added"
       args:
@@ -340,6 +335,7 @@
       shell: "oadm policy add-role-to-user -n default edit system:serviceaccount:default:builder && touch /etc/origin/atomic-reactor-policy-added"
       args:
         creates: "/etc/origin/atomic-reactor-policy-added"
+      when: env == "production"
 
 - name: Deploy OSBS on top of OpenShift
   hosts: osbs-masters-stg[0]:osbs-masters[0]
@@ -399,6 +395,16 @@
   tags:
     - osbs-worker-namespace
   user: root
+  vars_files:
+    - /srv/web/infra/ansible/vars/global.yml
+    - "/srv/private/ansible/vars.yml"
+    - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  vars:
+    osbs_kubeconfig_path: /etc/origin/master/admin.kubeconfig
+    osbs_environment:
+      KUBECONFIG: "{{ osbs_kubeconfig_path }}"
+
   roles:
     - role: osbs-namespace
       osbs_namespace: "{{ osbs_worker_namespace }}"

roles/fedimg/tasks/main.yml

@@ -134,3 +134,11 @@
   tags:
   - cron
   - fedimg
+
+- name: hotfix - copy the consumers.py over to the site-packages
+  copy: src="{{ files }}/hotfix/fedimg/consumers.py" dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py
+  notify:
+  - restart fedmsg-hub
+  tags:
+  - fedimg
+  - hotfix

roles/openshift-apps/release-monitoring/files/deploymentconfig.yml

@@ -22,6 +22,13 @@ items:
         maxUnavailable: 25%
         timeoutSeconds: 600
         updatePeriodSeconds: 1
+        pre:
+          failurePolicy: Abort
+          execNewPod:
+            containerName: release-monitoring-web
+            command: [ /bin/sh, -i, -c, "alembic -c /etc/anitya/alembic.ini upgrade head" ]
+            volumes:
+            - config-volume
       type: Rolling
     template:
       metadata:

roles/openshift-apps/release-monitoring/templates/configmap.yml

@@ -70,3 +70,38 @@ data:
         [anitya_log_config.root]
             level = "ERROR"
             handlers = ["console"]
+  alembic.ini: |-
+    [alembic]
+    script_location = anitya:db/migrations
+    sourceless = false
+{% if env == 'staging' %}
+    sqlalchemy.url = "postgresql://{{ anitya_stg_db_user }}:{{ anitya_stg_db_pass }}@{{ anitya_stg_db_host }}/{{ anitya_stg_db_name }}"
+{% else %}
+    sqlalchemy.url = "postgresql://{{ anitya_db_user }}:{{ anitya_db_pass }}@{{ anitya_db_host }}/{{ anitya_db_name }}"
+{% endif %}
+    [loggers]
+    keys = root,sqlalchemy,alembic
+    [handlers]
+    keys = console
+    [formatters]
+    keys = generic
+    [logger_root]
+    level = WARN
+    handlers = console
+    qualname =
+    [logger_sqlalchemy]
+    level = WARN
+    handlers =
+    qualname = sqlalchemy.engine
+    [logger_alembic]
+    level = INFO
+    handlers =
+    qualname = alembic
+    [handler_console]
+    class = StreamHandler
+    args = (sys.stderr,)
+    level = NOTSET
+    formatter = generic
+    [formatter_generic]
+    format = %(levelname)-5.5s [%(name)s] %(message)s
+    datefmt = %H:%M:%S

roles/rkhunter/templates/rkhunter.conf.j2

@@ -643,3 +643,7 @@ SCRIPTWHITELIST=/usr/bin/groups
 SCRIPTWHITELIST=/usr/bin/GET
 SCRIPTWHITELIST=/sbin/ifup
 SCRIPTWHITELIST=/sbin/ifdown
+{% if inventory_hostname.startswith(('db','pagure','retrace','anitya','upstream')) %}
+# Set this size very large on postgres running servers.
+IPC_SEG_SIZE=100000000000
+{% endif %}