diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py deleted file mode 100644 index 8195321299..0000000000 --- a/files/hotfix/fedimg/consumers.py +++ /dev/null @@ -1,151 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of fedimg. -# Copyright (C) 2014-2017 Red Hat, Inc. -# -# fedimg is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# fedimg is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with fedimg; if not, see http://www.gnu.org/licenses, -# or write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: David Gay -# Sayan Chowdhury -""" -This is the `fedmsg consumer`_ that subscribes to the topic emitted after the -completion of the nightly and production compose. The consumer on receving the -message uploads the image using the API of the cloud providers. -""" - -import logging -import multiprocessing.pool - -import fedmsg.consumers -import fedmsg.encoding -import fedfind.release -import fedfind.exceptions - -import fedimg.uploader - -from fedimg.config import PROCESS_COUNT, STATUS_FILTER -from fedimg.utils import get_rawxz_urls, get_value_from_dict - -_log = logging.getLogger(__name__) - - -class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): - """ - A `fedmsg consumer`_ that listens to the pungi compose topics and kicks - of the process to upload the images to various cloud providers. - - Attributes: - topic (str): The topics this consumer is subscribed to. Set to - ``org.fedoraproject.prod.pungi.compose.status.change``. - config_key (str): The key to set to ``True`` in the fedmsg config to - enable this consumer. The key is ``fedimgconsumer.prod.enabled``. - """ - topic = ['org.fedoraproject.prod.pungi.compose.status.change'] - config_key = "fedimgconsumer.prod.enabled" - - def __init__(self, *args, **kwargs): - _log.info("FedimgConsumer initializing") - super(FedimgConsumer, self).__init__(*args, **kwargs) - - # Threadpool for upload jobs - _log.info("Creating thread pool of %s process", PROCESS_COUNT) - self.upload_pool = multiprocessing.pool.ThreadPool( - processes=PROCESS_COUNT - ) - _log.info("FedimgConsumer initialized") - - def consume(self, msg): - """ - This is called when we receive a message matching our topics. - - Args: - msg (dict): The raw message from fedmsg. - """ - _log.info('Received %r %r', msg['topic'], msg['body']['msg_id']) - - msg_info = msg['body']['msg'] - if msg_info['status'] not in STATUS_FILTER: - _log.debug('%s is not valid status' % msg_info['status']) - return - - location = msg_info['location'] - compose_id = msg_info['compose_id'] - try: - compose_metadata = fedfind.release.get_release(cid=compose_id).metadata - except fedfind.exceptions.UnsupportedComposeError: - _log.debug("%r is unsupported compose" % compose_id) - return - - - # Till F27, both cloud-base and atomic images were available - # under variant CloudImages. With F28 and onward releases, - # cloud-base image compose moved to cloud variant and atomic images - # moved under atomic variant. - prev_rel = ['26', '27'] - if msg_info['release_version'] in prev_rel: - images_meta = get_value_from_dict( - compose_metadata, 'images', 'payload', 'images', 'CloudImages', - 'x86_64') - else: - images_meta = get_value_from_dict( - compose_metadata, 'images', 'payload', 'images', - 'Cloud', 'x86_64') - images_meta.extend(get_value_from_dict( - compose_metadata, 'images', 'payload', - 'images', 'AtomicHost', 'x86_64')) - - if images_meta is None: - _log.debug('No compatible image found to process') - return - - upload_urls = get_rawxz_urls(location, images_meta) - if len(upload_urls) > 0: - _log.info("Start processing compose id: %s", compose_id) - fedimg.uploader.upload( - pool=self.upload_pool, - urls=upload_urls, - compose_id=compose_id, - push_notifications=True - ) - - -class FedimgStagingConsumer(FedimgConsumer): - """ - A `fedmsg consumer`_ that listens to the staging pungi compose topics and - kicks of the process to upload the images to various cloud providers. - - Attributes: - topic (str): The topics this consumer is subscribed to. Set to - ``org.fedoraproject.stg.pungi.compose.status.change``. - config_key (str): The key to set to ``True`` in the fedmsg config to - enable this consumer. The key is ``fedimgconsumer.stg.enabled``. - """ - topic = ['org.fedoraproject.stg.pungi.compose.status.change'] - config_key = "fedimgconsumer.stg.enabled" - - -class FedimgDevConsumer(FedimgConsumer): - """ - A `fedmsg consumer`_ that listens to the dev pungi compose topics and - kicks of the process to upload the images to various cloud providers. - - Attributes: - topic (str): The topics this consumer is subscribed to. Set to - ``org.fedoraproject.dev.pungi.compose.status.change``. - config_key (str): The key to set to ``True`` in the fedmsg config to - enable this consumer. The key is ``fedimgconsumer.dev.enabled``. - """ - topic = ['org.fedoraproject.dev.pungi.compose.status.change'] - config_key = "fedimgconsumer.dev.enabled" diff --git a/files/hotfix/fedimg/ec2imgpublisher.py b/files/hotfix/fedimg/ec2imgpublisher.py deleted file mode 100644 index b1eab4858d..0000000000 --- a/files/hotfix/fedimg/ec2imgpublisher.py +++ /dev/null @@ -1,318 +0,0 @@ -# This file is part of fedimg. -# Copyright (C) 2014-2017 Red Hat, Inc. -# -# fedimg is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# fedimg is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with fedimg; if not, see http://www.gnu.org/licenses, -# or write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: Sayan Chowdhury -# - -import logging -_log = logging.getLogger(__name__) - -import re - -from time import sleep - -import fedimg.messenger - -from fedimg.utils import external_run_command, get_item_from_regex -from fedimg.utils import get_image_name_from_ami_name -from fedimg.utils import get_image_name_from_ami_name_for_fedmsg -from fedimg.services.ec2.ec2base import EC2Base - - -class EC2ImagePublisher(EC2Base): - """ Comment goes here """ - - def __init__(self, **kwargs): - defaults = { - 'access_key': None, - 'compose_id': None, - 'image_id': None, - 'image_name': 'Fedora-AMI', - 'image_url': None, - 'image_description': 'Fedora AMI Description', - 'service': 'EC2', - 'region': None, - 'secret_key': None, - 'visibility': 'all', - 'push_notifications': False, - } - - for (prop, default) in defaults.iteritems(): - setattr(self, prop, kwargs.get(prop, default)) - - def _retry_till_image_is_public(self, image): - """ Comment goes here """ - - driver = self._connect() - - is_image_public = False - while True: - try: - is_image_public = driver.ex_modify_image_attribute( - image, - {'LaunchPermission.Add.1.Group': 'all'}) - except Exception as e: - if 'InvalidAMIID.Unavailable' in str(e): - # The copy isn't completed yet, so wait for 20 seconds - # more. - sleep(20) - continue - break - - return is_image_public - - def _retry_till_snapshot_is_public(self, snapshot): - - driver = self._connect() - - while True: - is_snapshot_public = driver.ex_modify_snapshot_attribute( - snapshot, - {'CreateVolumePermission.Add.1.Group': 'all'}) - - if is_snapshot_public: - break - - return is_snapshot_public - - def _retry_till_image_is_available(self, image_id): - driver = self._connect() - - while True: - try: - image = driver.get_image(image_id) - image_name = image.name - if image_name is None: - continue - return image - except Exception as e: - if 'InvalidAMIID.Unavailable' in str(e): - # The copy isn't completed yet, so wait for 20 seconds - # more. - sleep(20) - continue - - def _retry_till_snapshot_is_available(self, image): - - driver = self._connect() - while True: - image = driver.get_image(image.id) - snapshot_id = image.extra['block_device_mapping'][0]['ebs']['snapshot_id'] - - if snapshot_id: - break - - return snapshot_id - - def _generate_dummy_snapshot_object(self, snapshot_id): - - driver = self._connect() - - snapshot_obj = type('', (), {})() - snapshot_obj.id = snapshot_id - snapshot = driver.list_snapshots(snapshot=snapshot_obj) - - return snapshot - - def _retry_till_blk_mapping_is_available(self, image): - - while True: - image = self._connect().get_image(image_id=image.id) - blk_mapping = image.extra['block_device_mapping'] - - if blk_mapping: - return blk_mapping - - def get_snapshot_from_image(self, image): - """ Comment goes here """ - if isinstance(image, str): - image_id = image - image = self._connect().get_image(image_id) - - blk_mapping = image.extra['block_device_mapping'] - if not blk_mapping: - blk_mapping = self._retry_till_blk_mapping_is_available(image) - - snapshot_id = blk_mapping[0]['ebs']['snapshot_id'] - if snapshot_id is None: - snapshot_id = self._retry_till_snapshot_is_available(image) - - snapshot = self._generate_dummy_snapshot_object(snapshot_id)[0] - - return snapshot - - def get_volume_type_from_image(self, image): - if isinstance(image, str): - image_id = image - image = self._connect().get_image(image_id) - - blk_mapping = image.extra['block_device_mapping'] - if not blk_mapping: - blk_mapping = self._retry_till_blk_mapping_is_available(image) - - return blk_mapping[0]['ebs']['volume_type'] - - def get_virt_type_from_image(self, image): - return 'hvm' - - def publish_images(self, region_image_mapping=None): - """ Comment goes here """ - - published_images = [] - if region_image_mapping is None: - return published_images - - for region, image_id in region_image_mapping: - self.set_region(region) - - _log.info('Publish image (%s) in %s started' % (image_id, region)) - image = self._connect().get_image(image_id=image_id) - is_image_public = self._retry_till_image_is_public(image) - _log.info('Publish image (%s) in %s completed' % (image_id, region)) - - _log.info('Publish snaphsot for image (%s) in %s started' % (image_id, region)) - snapshot = self.get_snapshot_from_image(image) - _log.info('Fetched snapshot for image (%s): %s' % (image_id, snapshot.id)) - is_snapshot_public = self._retry_till_snapshot_is_public(snapshot) - _log.info('Publish snaphsot for image (%s) in %s completed' % (image_id, region)) - - volume_type = self.get_volume_type_from_image(image) - virt_type = self.get_virt_type_from_image(image) - - if self.push_notifications: - fedimg.messenger.notify( - topic='image.publish', - msg=dict( - image_name=get_image_name_from_ami_name_for_fedmsg(image.name), - image_url=self.image_url, - destination=self.region, - service=self.service, - compose=self.compose_id, - extra=dict( - id=image.id, - virt_type=virt_type, - vol_type=volume_type - ) - ) - ) - - fedimg.messenger.notify( - topic='image.upload', - msg=dict( - image_name=get_image_name_from_ami_name_for_fedmsg(image.name), - image_url=self.image_url, - destination=self.region, - service=self.service, - status='completed', - compose=self.compose_id, - extra=dict( - id=image.id, - virt_type=virt_type, - vol_type=volume_type - ) - ) - ) - - published_images.append({ - 'image_id': image.id, - 'is_image_public': is_image_public, - 'snapshot_id': snapshot.id, - 'is_snapshot_public': is_snapshot_public, - 'regions': self.region - }) - - return published_images - - def copy_images_to_regions(self, image_id=None, base_region=None, regions=None): - """ Comment goes here """ - - if (image_id is None) or (regions is None) or (base_region is None): - return - - counter = 0 - copied_images = [] - - self.set_region(base_region) - image = self._connect().get_image(image_id=image_id) - if not image: - return [] - - for region in regions: - _log.info('Copy %s to %s started' % (image_id, region)) - self.set_region(region) - self.image_name = get_image_name_from_ami_name(image.name, region) - - while True: - if counter > 0: - self.image_name = re.sub( - '\d(?!\d)', - lambda x: str(int(x.group(0))+1), - self.image_name - ) - try: - copied_image = self._connect().copy_image( - source_region=base_region, - image=image, - name=self.image_name, - description=self.image_description) - - copied_image = self._retry_till_image_is_available(copied_image.id) - - virt_type = image.extra['virtualization_type'] - volume_type = image.extra['block_device_mapping'][0]['ebs']['volume_type'] - - if self.push_notifications: - fedimg.messenger.notify( - topic='image.copy', - msg=dict( - image_name=get_image_name_from_ami_name_for_fedmsg(copied_image.name), - destination=self.region, - service=self.service, - compose_id=self.compose_id, - extra=dict( - id=copied_image.id, - virt_type=virt_type, - vol_type=volume_type, - source_image_id=image.id - ) - ) - ) - - _log.info('Copy %s to %s is completed.' % (image_id, region)) - copied_images.append({ - 'region': region, - 'copied_image_id': copied_image.id - }) - break - - except Exception as e: - _log.info('Could not register ' - 'with name: %r' % self.image_name) - if 'InvalidAMIName.Duplicate' in str(e): - counter = counter + 1 - else: - _log.info('Failed') - break - - return copied_images - - def deprecate_images(self, image_ids=None, snapshot_perm='all'): - raise NotImplementedError - - def delete_images(self, image_ids=None, snapshot_perm='all'): - raise NotImplementedError diff --git a/files/hotfix/fedimg/utils.py b/files/hotfix/fedimg/utils.py deleted file mode 100644 index c64c0360b1..0000000000 --- a/files/hotfix/fedimg/utils.py +++ /dev/null @@ -1,183 +0,0 @@ -# This file is part of fedimg. -# Copyright (C) 2014-2017 Red Hat, Inc. -# -# fedimg is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# fedimg is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with fedimg; if not, see http://www.gnu.org/licenses, -# or write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: David Gay -# Ralph Bean -# Sayan Chowdhury - -""" -Utility functions for fedimg. -""" -import logging -_log = logging.getLogger(__name__) - -import functools -import os -import re -import socket -import subprocess -import tempfile - -import paramiko -from libcloud.compute.types import Provider -from libcloud.compute.providers import get_driver - - -def get_file_arch(file_name): - """ Takes a file name (probably of a .raw.xz image file) and returns - the suspected architecture of the contained image. If it doesn't look - like a 32-bit or 64-bit image, None is returned. """ - if file_name.find('x86_64') != -1: - return 'x86_64' - else: - return None - - -def get_rawxz_urls(location, images): - """ Iterates through all the images metadata and returns the url of .raw.xz - files. - """ - rawxz_list = [f['path'] for f in images if f['path'].endswith('.raw.xz')] - if not rawxz_list: - return [] - - return map((lambda path: '{}/{}'.format(location, path)), rawxz_list) - - -def get_virt_types_from_url(url): - """ Takes a URL to a .raw.xz image file) and returns the suspected - virtualization type that the image file should be registered as. """ - return ['hvm'] - - -def region_to_driver(region): - """ Takes a region name (ex. 'eu-west-1') and returns - the appropriate libcloud provider value. """ - cls = get_driver(Provider.EC2) - return functools.partial(cls, region=region) - - -def ssh_connection_works(username, ip, keypath): - """ Returns True if an SSH connection can me made to `username`@`ip`. """ - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - works = False - try: - ssh.connect(ip, username=username, - key_filename=keypath) - works = True - except (paramiko.BadHostKeyException, - paramiko.AuthenticationException, - paramiko.SSHException, socket.error): - pass - ssh.close() - return works - - -def get_value_from_dict(_dict, *keys): - for key in keys: - try: - _dict = _dict[key] - except KeyError: - return None - return _dict - - -def external_run_command(command): - _log.debug("Starting the command: %r" % command) - ret = subprocess.Popen(' '.join(command), stdin=subprocess.PIPE, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True) - out, err = ret.communicate() - _log.debug("Finished executing the command: %r" % command) - retcode = ret.returncode - return out, err, retcode - - -def get_item_from_regex(output, regex): - match = re.search(regex, output) - if match is None: - return '' - else: - return match.group(1) - - -def get_file_name_image(image_url): - return image_url.split('/')[-1] - - -def get_source_from_image(image_url): - tmpdir = tempfile.mkdtemp() - file_name = get_file_name_image(image_url) - file_path = os.path.join(tmpdir, file_name) - - _log.info("[PREP] Preparing temporary directory for download: %r" % tmpdir) - output, error, retcode = external_run_command([ - 'wget', - image_url, - '-P', - tmpdir - ]) - - if retcode != 0: - return '' - - output, error, retcode = external_run_command([ - 'xzcat', - file_path, - '>', - file_path.rstrip('.xz') - ]) - - if retcode != 0: - return '' - - return file_path.rstrip('.xz') - - -def get_volume_type_from_image(image, region): - return image.extra['block_device_mapping'][0]['ebs']['volume_type'] - - -def get_virt_type_from_image(image): - return 'hvm' - - -def get_image_name_from_image(image_url, virt_type='', region='', respin='0', - volume_type=''): - - file_name = get_file_name_image(image_url) - build_name = file_name.replace('.raw.xz', '') - - return '-'.join( - [x for x in [build_name, virt_type, region, volume_type, respin] if x]) - - -def get_image_name_from_ami_name(image_name, region): - name_vt_region, volume_type, respin = image_name.rsplit('-', 2) - name_vt = name_vt_region.rsplit('-', 3)[:-3][0] - - return '-'.join( - [x for x in [name_vt, region, volume_type, respin] if x]) - - -def get_image_name_from_ami_name_for_fedmsg(image_name): - name_vt_region, volume_type, respin = image_name.rsplit('-', 2) - image_name = name_vt_region.rsplit('-', 4)[:-4][0] - - return image_name diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index f3081e3f70..b1538c43a3 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -114,27 +114,3 @@ tags: - cron - fedimg - -- name: hotfix - copy the utils.py over to the site-packages - copy: src="{{ files }}/hotfix/fedimg/utils.py" dest=/usr/lib/python2.7/site-packages/fedimg/utils.py - notify: - - restart fedmsg-hub - tags: - - fedimg - - hotfix - -- name: hotfix - copy the ec2imgpublisher.py over to the site-packages - copy: src="{{ files }}/hotfix/fedimg/ec2imgpublisher.py" dest=/usr/lib/python2.7/site-packages/fedimg/services/ec2/ec2imgpublisher.py - notify: - - restart fedmsg-hub - tags: - - fedimg - - hotfix - -- name: hotfix - copy the consumers.py over to the site-packages - copy: src="{{ files }}/hotfix/fedimg/consumers.py" dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py - notify: - - restart fedmsg-hub - tags: - - fedimg - - hotfix