From d596a7c84d6cb6bafa339aff3699ae7da4e9ee87 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Mon, 16 Apr 2018 23:13:09 +0530 Subject: [PATCH] fedimg: Update the changes for release 1.2.0 - Remove the cron to kill the EC2 nodes - Update the trigger_upload - Remove the hotfixes Signed-off-by: Sayan Chowdhury --- files/hotfix/fedimg/consumers.py | 144 ---------- files/hotfix/fedimg/ec2imgpublisher.py | 296 -------------------- roles/fedimg/files/cron/kill_ec2_nodes.cron | 2 - roles/fedimg/files/cron/kill_ec2_nodes.py | 39 --- roles/fedimg/files/trigger_upload.py | 4 +- roles/fedimg/tasks/main.yml | 40 --- 6 files changed, 2 insertions(+), 523 deletions(-) delete mode 100644 files/hotfix/fedimg/consumers.py delete mode 100644 files/hotfix/fedimg/ec2imgpublisher.py delete mode 100644 roles/fedimg/files/cron/kill_ec2_nodes.cron delete mode 100644 roles/fedimg/files/cron/kill_ec2_nodes.py diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py deleted file mode 100644 index 410b64190d..0000000000 --- a/files/hotfix/fedimg/consumers.py +++ /dev/null @@ -1,144 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of fedimg. -# Copyright (C) 2014-2017 Red Hat, Inc. -# -# fedimg is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# fedimg is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with fedimg; if not, see http://www.gnu.org/licenses, -# or write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: David Gay -# Sayan Chowdhury -""" -This is the `fedmsg consumer`_ that subscribes to the topic emitted after the -completion of the nightly and production compose. The consumer on receving the -message uploads the image using the API of the cloud providers. -""" - -import logging -import multiprocessing.pool - -import fedmsg.consumers -import fedmsg.encoding -import fedfind.release - -import fedimg.uploader - -from fedimg.config import PROCESS_COUNT, STATUS_FILTER -from fedimg.utils import get_rawxz_urls, get_value_from_dict - -LOG = logging.getLogger(__name__) - - -class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): - """ - A `fedmsg consumer`_ that listens to the pungi compose topics and kicks - of the process to upload the images to various cloud providers. - - Attributes: - topic (str): The topics this consumer is subscribed to. Set to - ``org.fedoraproject.prod.pungi.compose.status.change``. - config_key (str): The key to set to ``True`` in the fedmsg config to - enable this consumer. The key is ``fedimgconsumer.prod.enabled``. - """ - topic = ['org.fedoraproject.prod.pungi.compose.status.change'] - config_key = "fedimgconsumer.prod.enabled" - - def __init__(self, *args, **kwargs): - LOG.info("FedimgConsumer initializing") - super(FedimgConsumer, self).__init__(*args, **kwargs) - - # Threadpool for upload jobs - LOG.info("Creating thread pool of %s process", PROCESS_COUNT) - self.upload_pool = multiprocessing.pool.ThreadPool( - processes=PROCESS_COUNT - ) - LOG.info("FedimgConsumer initialized") - - def consume(self, msg): - """ - This is called when we receive a message matching our topics. - - Args: - msg (dict): The raw message from fedmsg. - """ - LOG.info('Received %r %r', msg['topic'], msg['body']['msg_id']) - - msg_info = msg['body']['msg'] - if msg_info['status'] not in STATUS_FILTER: - return - - location = msg_info['location'] - compose_id = msg_info['compose_id'] - compose_metadata = fedfind.release.get_release(cid=compose_id).metadata - - # Till F27, both cloud-base and atomic images were available - # under variant CloudImages. With F28 and onward releases, - # cloud-base image compose moved to cloud variant and atomic images - # moved under atomic variant. - prev_rel = ['26', '27'] - if msg_info['release_version'] in prev_rel: - images_meta = get_value_from_dict( - compose_metadata, 'images', 'payload', 'images', 'CloudImages', - 'x86_64') - else: - images_meta = get_value_from_dict( - compose_metadata, 'images', 'payload', 'images', - 'Cloud', 'x86_64') - images_meta.extend(get_value_from_dict( - compose_metadata, 'images', 'payload', - 'images', 'AtomicHost', 'x86_64')) - - if images_meta is None: - LOG.debug('No compatible image found to process') - return - - upload_urls = get_rawxz_urls(location, images_meta) - if len(upload_urls) > 0: - LOG.info("Start processing compose id: %s", compose_id) - fedimg.uploader.upload( - pool=self.upload_pool, - urls=upload_urls, - compose_id=compose_id - ) - - -class FedimgStagingConsumer(FedimgConsumer): - """ - A `fedmsg consumer`_ that listens to the staging pungi compose topics and - kicks of the process to upload the images to various cloud providers. - - Attributes: - topic (str): The topics this consumer is subscribed to. Set to - ``org.fedoraproject.stg.pungi.compose.status.change``. - config_key (str): The key to set to ``True`` in the fedmsg config to - enable this consumer. The key is ``fedimgconsumer.stg.enabled``. - """ - topic = ['org.fedoraproject.stg.pungi.compose.status.change'] - config_key = "fedimgconsumer.stg.enabled" - - -class FedimgDevConsumer(FedimgConsumer): - """ - A `fedmsg consumer`_ that listens to the dev pungi compose topics and - kicks of the process to upload the images to various cloud providers. - - Attributes: - topic (str): The topics this consumer is subscribed to. Set to - ``org.fedoraproject.dev.pungi.compose.status.change``. - config_key (str): The key to set to ``True`` in the fedmsg config to - enable this consumer. The key is ``fedimgconsumer.dev.enabled``. - """ - topic = ['org.fedoraproject.dev.pungi.compose.status.change'] - config_key = "fedimgconsumer.dev.enabled" - diff --git a/files/hotfix/fedimg/ec2imgpublisher.py b/files/hotfix/fedimg/ec2imgpublisher.py deleted file mode 100644 index f9053a0db5..0000000000 --- a/files/hotfix/fedimg/ec2imgpublisher.py +++ /dev/null @@ -1,296 +0,0 @@ -# This file is part of fedimg. -# Copyright (C) 2014-2017 Red Hat, Inc. -# -# fedimg is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# fedimg is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with fedimg; if not, see http://www.gnu.org/licenses, -# or write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: Sayan Chowdhury -# - -import logging -log = logging.getLogger("fedmsg") - -import re - -from time import sleep - -import fedimg.messenger - -from fedimg.utils import external_run_command, get_item_from_regex -from fedimg.utils import get_image_name_from_ami_name -from fedimg.services.ec2.ec2base import EC2Base - - -class EC2ImagePublisher(EC2Base): - """ Comment goes here """ - - def __init__(self, **kwargs): - defaults = { - 'access_key': None, - 'compose_id': None, - 'image_id': None, - 'image_name': 'Fedora-AMI', - 'image_description': 'Fedora AMI Description', - 'service': 'EC2', - 'region': None, - 'secret_key': None, - 'visibility': 'all', - 'push_notifications': False, - } - - for (prop, default) in defaults.iteritems(): - setattr(self, prop, kwargs.get(prop, default)) - - def _retry_till_image_is_public(self, image): - """ Comment goes here """ - - driver = self._connect() - - is_image_public = False - while True: - try: - is_image_public = driver.ex_modify_image_attribute( - image, - {'LaunchPermission.Add.1.Group': 'all'}) - except Exception as e: - if 'InvalidAMIID.Unavailable' in str(e): - # The copy isn't completed yet, so wait for 20 seconds - # more. - sleep(20) - continue - break - - return is_image_public - - def _retry_till_snapshot_is_public(self, snapshot): - - driver = self._connect() - - while True: - is_snapshot_public = driver.ex_modify_snapshot_attribute( - snapshot, - {'CreateVolumePermission.Add.1.Group': 'all'}) - - if is_snapshot_public: - break - - return is_snapshot_public - - - def _retry_till_snapshot_is_available(self, image): - - driver = self._connect() - while True: - image = driver.get_image(image.id) - snapshot_id = image.extra['block_device_mapping'][0]['ebs']['snapshot_id'] - - if snapshot_id: - break - - return snapshot_id - - def _generate_dummy_snapshot_object(self, snapshot_id): - - driver = self._connect() - - snapshot_obj = type('', (), {})() - snapshot_obj.id = snapshot_id - snapshot = driver.list_snapshots(snapshot=snapshot_obj) - - return snapshot - - def _retry_till_blk_mapping_is_available(self, image): - - while True: - image = self._connect().get_image(image_id=image.id) - blk_mapping = image.extra['block_device_mapping'] - - if blk_mapping: - return blk_mapping - - def get_snapshot_from_image(self, image): - """ Comment goes here """ - if isinstance(image, str): - image_id = image - image = self._connect().get_image(image_id) - - blk_mapping = image.extra['block_device_mapping'] - if not blk_mapping: - blk_mapping = self._retry_till_blk_mapping_is_available(image) - - snapshot_id = blk_mapping[0]['ebs']['snapshot_id'] - if snapshot_id is None: - snapshot_id = self._retry_till_snapshot_is_available(image) - - snapshot = self._generate_dummy_snapshot_object(snapshot_id)[0] - - return snapshot - - def get_volume_type_from_image(self, image): - if isinstance(image, str): - image_id = image - image = self._connect().get_image(image_id) - - blk_mapping = image.extra['block_device_mapping'] - if not blk_mapping: - blk_mapping = self._retry_till_blk_mapping_is_available(image) - - return blk_mapping[0]['ebs']['volume_type'] - - def get_virt_type_from_image(self, image): - return 'hvm' - - def publish_images(self, region_image_mapping=None): - """ Comment goes here """ - - published_images = [] - if region_image_mapping is None: - return published_images - - for region, image_id in region_image_mapping: - self.set_region(region) - - log.info('Publish image (%s) in %s started' % (image_id, region)) - image = self._connect().get_image(image_id=image_id) - is_image_public = self._retry_till_image_is_public(image) - log.info('Publish image (%s) in %s completed' % (image_id, region)) - - log.info('Publish snaphsot for image (%s) in %s started' % (image_id, region)) - snapshot = self.get_snapshot_from_image(image) - log.info('Fetched snapshot for image (%s): %s' % (image_id, snapshot.id)) - is_snapshot_public = self._retry_till_snapshot_is_public(snapshot) - log.info('Publish snaphsot for image (%s) in %s completed' % (image_id, region)) - - volume_type = self.get_volume_type_from_image(image) - virt_type = self.get_virt_type_from_image(image) - - if self.push_notifications: - fedimg.messenger.notify( - topic='image.publish', - msg=dict( - image_name=image.name, - destination=self.region, - service=self.service, - compose=self.compose_id, - extra=dict( - id=image.id, - virt_type=virt_type, - vol_type=volume_type - ) - ) - ) - - fedimg.messenger.notify( - topic='image.upload', - msg=dict( - image_name=image.name, - destination=self.region, - service=self.service, - status='completed', - compose=self.compose_id, - extra=dict( - id=image.id, - virt_type=virt_type, - vol_type=volume_type - ) - ) - ) - - published_images.append({ - 'image_id': image.id, - 'is_image_public': is_image_public, - 'snapshot_id': snapshot.id, - 'is_snapshot_public': is_snapshot_public, - 'regions': self.region - }) - - return published_images - - def copy_images_to_regions(self, image_id=None, base_region=None, regions=None): - """ Comment goes here """ - - if (image_id is None) or (regions is None) or (base_region is None): - return - - counter = 0 - copied_images = [] - - self.set_region(base_region) - image = self._connect().get_image(image_id=image_id) - if not image: - return [] - - for region in regions: - log.info('Copy %s to %s started' % (image_id, region)) - self.set_region(region) - self.image_name = get_image_name_from_ami_name(image.name, region) - - while True: - if counter > 0: - self.image_name = re.sub( - '\d(?!\d)', - lambda x: str(int(x.group(0))+1), - self.image_name - ) - try: - copied_image = self._connect().copy_image( - source_region=base_region, - image=image, - name=self.image_name, - description=self.image_description) - - virt_type = image.extra['virtualization_type'] - volume_type = image.extra['block_device_mapping'][0]['ebs']['volume_type'] - - if self.push_notifications: - fedimg.messenger.notify( - topic='image.copy', - msg=dict( - image_name=copied_image.name, - destination=self.region, - service=self.service, - compose_id=self.compose_id, - extra=dict( - id=copied_image.id, - virt_type=virt_type, - vol_type=volume_type, - source_image_id=image.id - ) - ) - ) - - log.info('Copy %s to %s is completed.' % (image_id, region)) - copied_images.append({ - 'region': region, - 'copied_image_id': copied_image.id - }) - break - - except Exception as e: - log.info('Could not register ' - 'with name: %r' % self.image_name) - if 'InvalidAMIName.Duplicate' in str(e): - counter = counter + 1 - else: - log.info('Failed') - break - - return copied_images - - def deprecate_images(self, image_ids=None, snapshot_perm='all'): - raise NotImplementedError - - def delete_images(self, image_ids=None, snapshot_perm='all'): - raise NotImplementedError diff --git a/roles/fedimg/files/cron/kill_ec2_nodes.cron b/roles/fedimg/files/cron/kill_ec2_nodes.cron deleted file mode 100644 index bda9440a34..0000000000 --- a/roles/fedimg/files/cron/kill_ec2_nodes.cron +++ /dev/null @@ -1,2 +0,0 @@ -MAILTO=sysadmin-fedimg-members@fedoraproject.org -0 * * * * fedmsg /usr/share/fedimg/cronjobs/kill_ec2_nodes.py 2>&1 | grep -v 'Cannot find qpid python' diff --git a/roles/fedimg/files/cron/kill_ec2_nodes.py b/roles/fedimg/files/cron/kill_ec2_nodes.py deleted file mode 100644 index c325e698a8..0000000000 --- a/roles/fedimg/files/cron/kill_ec2_nodes.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/env python -# -*- coding: utf8 -*- - -# NOTE this is taken from the github repo -# https://github.com/fedora-infra/fedimg/blob/develop/bin/kill_ec2_nodes.py - -import logging -logging.basicConfig() - -from libcloud.compute.types import Provider -from libcloud.compute.providers import get_driver -import datetime -import fedimg - -EC2_ACCESS_ID = fedimg.AWS_ACCESS_ID -EC2_SECRET_KEY = fedimg.AWS_SECRET_KEY - -def kill_all_instances(region): - """ - Kills all the instances which are running for 2 hours or more. - - :param region: AWS region - """ - cls = get_driver(Provider.EC2) - driver = cls(EC2_ACCESS_ID, EC2_SECRET_KEY, region=region) - nodes = driver.list_nodes() - for n in nodes: - d1 = datetime.datetime.strptime(n.extra['launch_time'], '%Y-%m-%dT%H:%M:%S.000Z') - d2 = datetime.datetime.utcnow() - delta = d2 - d1 - if delta.total_seconds() > 7200: # If more than 2 hours of up time. - n.destroy() - - -if __name__ == '__main__': - regions = set([metadata.split('|')[0] - for metadata in fedimg.AWS_AMIS.split('\n')]) - for region in regions: - kill_all_instances(region) diff --git a/roles/fedimg/files/trigger_upload.py b/roles/fedimg/files/trigger_upload.py index f7484ec4bd..831e1a3e4e 100644 --- a/roles/fedimg/files/trigger_upload.py +++ b/roles/fedimg/files/trigger_upload.py @@ -16,8 +16,8 @@ log = logging.getLogger('fedmsg') def trigger_upload(url, compose_id, push_notifications): upload_pool = multiprocessing.pool.ThreadPool(processes=4) - compose_meta = {'compose_id': compose_id} - fedimg.uploader.upload(upload_pool, [url], compose_meta=compose_meta, + fedimg.uploader.upload(upload_pool, [url], + compose_id=compose_id, push_notifications=push_notifications) diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index fceb5d77a9..94c3904013 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -108,43 +108,3 @@ tags: - cron - fedimg - -- name: fedimg cron scripts - copy: > - src=cron/{{ item }} - dest=/usr/share/fedimg/cronjobs/{{ item }} - owner=fedmsg - mode=744 - with_items: - - kill_ec2_nodes.py - when: env != "staging" - tags: - - cron - - fedimg - -- name: fedimg cron definitions - copy: > - src=cron/{{ item }} - dest=/etc/cron.d/{{ item }} - owner=root - mode=644 - with_items: - - kill_ec2_nodes.cron - when: env != "staging" - tags: - - cron - - fedimg - -- name: hotfix - copy the consumers.py over to the site-packages - copy: src="{{ files }}/hotfix/fedimg/consumers.py" dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py - notify: - - restart fedmsg-hub - tags: - - fedimg - - hotfix - -- name: hotfix - copy the ec2imgpublisher over to the site-packages - copy: src="{{ files }}/hotfix/fedimg/ec2imgpublisher.py" dest=/usr/lib/python2.7/site-packages/fedimg/services/ec2/ec2imgpublisher.py - tags: - - fedimg - - hotfix