infrastructure/ansible
2
0
Fork 0
Code Pull requests 7 Activity Actions

fedimg: Remove the earlier three pushed hotfix

Browse source 
Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012@gmail.com>
This commit is contained in:
Sayan Chowdhury 2018-04-26 01:00:07 +05:30 • committed by Sayan Chowdhury
parent 637dcfcd86
commit 8f56ce4b93
4 changed files with 0 additions and 676 deletions
Download patch file Download diff file Expand all files Collapse all files

151
files/hotfix/fedimg/consumers.py
View file

@ -1,151 +0,0 @@
# -*- coding: utf-8 -*-
# This file is part of fedimg.
# Copyright (C) 2014-2017 Red Hat, Inc.
#
# fedimg is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# fedimg is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with fedimg; if not, see http://www.gnu.org/licenses,
# or write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors: David Gay <dgay@redhat.com>
# Sayan Chowdhury <sayanchowdhury@fedoraproject.org>
"""
This is the `fedmsg consumer`_ that subscribes to the topic emitted after the
completion of the nightly and production compose. The consumer on receving the
message uploads the image using the API of the cloud providers.
"""
import logging
import multiprocessing.pool
import fedmsg.consumers
import fedmsg.encoding
import fedfind.release
import fedfind.exceptions
import fedimg.uploader
from fedimg.config import PROCESS_COUNT, STATUS_FILTER
from fedimg.utils import get_rawxz_urls, get_value_from_dict
_log = logging.getLogger(__name__)
class FedimgConsumer(fedmsg.consumers.FedmsgConsumer):
"""
A `fedmsg consumer`_ that listens to the pungi compose topics and kicks
of the process to upload the images to various cloud providers.
Attributes:
topic (str): The topics this consumer is subscribed to. Set to
``org.fedoraproject.prod.pungi.compose.status.change``.
config_key (str): The key to set to ``True`` in the fedmsg config to
enable this consumer. The key is ``fedimgconsumer.prod.enabled``.
"""
topic = ['org.fedoraproject.prod.pungi.compose.status.change']
config_key = "fedimgconsumer.prod.enabled"
def __init__(self, *args, **kwargs):
_log.info("FedimgConsumer initializing")
super(FedimgConsumer, self).__init__(*args, **kwargs)
# Threadpool for upload jobs
_log.info("Creating thread pool of %s process", PROCESS_COUNT)
self.upload_pool = multiprocessing.pool.ThreadPool(
processes=PROCESS_COUNT
)
_log.info("FedimgConsumer initialized")
def consume(self, msg):
"""
This is called when we receive a message matching our topics.
Args:
msg (dict): The raw message from fedmsg.
"""
_log.info('Received %r %r', msg['topic'], msg['body']['msg_id'])
msg_info = msg['body']['msg']
if msg_info['status'] not in STATUS_FILTER:
_log.debug('%s is not valid status' % msg_info['status'])
return
location = msg_info['location']
compose_id = msg_info['compose_id']
try:
compose_metadata = fedfind.release.get_release(cid=compose_id).metadata
except fedfind.exceptions.UnsupportedComposeError:
_log.debug("%r is unsupported compose" % compose_id)
return
# Till F27, both cloud-base and atomic images were available
# under variant CloudImages. With F28 and onward releases,
# cloud-base image compose moved to cloud variant and atomic images
# moved under atomic variant.
prev_rel = ['26', '27']
if msg_info['release_version'] in prev_rel:
images_meta = get_value_from_dict(
compose_metadata, 'images', 'payload', 'images', 'CloudImages',
'x86_64')
else:
images_meta = get_value_from_dict(
compose_metadata, 'images', 'payload', 'images',
'Cloud', 'x86_64')
images_meta.extend(get_value_from_dict(
compose_metadata, 'images', 'payload',
'images', 'AtomicHost', 'x86_64'))
if images_meta is None:
_log.debug('No compatible image found to process')
return
upload_urls = get_rawxz_urls(location, images_meta)
if len(upload_urls) > 0:
_log.info("Start processing compose id: %s", compose_id)
fedimg.uploader.upload(
pool=self.upload_pool,
urls=upload_urls,
compose_id=compose_id,
push_notifications=True
)
class FedimgStagingConsumer(FedimgConsumer):
"""
A `fedmsg consumer`_ that listens to the staging pungi compose topics and
kicks of the process to upload the images to various cloud providers.
Attributes:
topic (str): The topics this consumer is subscribed to. Set to
``org.fedoraproject.stg.pungi.compose.status.change``.
config_key (str): The key to set to ``True`` in the fedmsg config to
enable this consumer. The key is ``fedimgconsumer.stg.enabled``.
"""
topic = ['org.fedoraproject.stg.pungi.compose.status.change']
config_key = "fedimgconsumer.stg.enabled"
class FedimgDevConsumer(FedimgConsumer):
"""
A `fedmsg consumer`_ that listens to the dev pungi compose topics and
kicks of the process to upload the images to various cloud providers.
Attributes:
topic (str): The topics this consumer is subscribed to. Set to
``org.fedoraproject.dev.pungi.compose.status.change``.
config_key (str): The key to set to ``True`` in the fedmsg config to
enable this consumer. The key is ``fedimgconsumer.dev.enabled``.
"""
topic = ['org.fedoraproject.dev.pungi.compose.status.change']
config_key = "fedimgconsumer.dev.enabled"

318
files/hotfix/fedimg/ec2imgpublisher.py
View file

@ -1,318 +0,0 @@
# This file is part of fedimg.
# Copyright (C) 2014-2017 Red Hat, Inc.
#
# fedimg is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# fedimg is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with fedimg; if not, see http://www.gnu.org/licenses,
# or write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors: Sayan Chowdhury <sayanchowdhury@fedoraproject.org>
#
import logging
_log = logging.getLogger(__name__)
import re
from time import sleep
import fedimg.messenger
from fedimg.utils import external_run_command, get_item_from_regex
from fedimg.utils import get_image_name_from_ami_name
from fedimg.utils import get_image_name_from_ami_name_for_fedmsg
from fedimg.services.ec2.ec2base import EC2Base
class EC2ImagePublisher(EC2Base):
""" Comment goes here """
def __init__(self, **kwargs):
defaults = {
'access_key': None,
'compose_id': None,
'image_id': None,
'image_name': 'Fedora-AMI',
'image_url': None,
'image_description': 'Fedora AMI Description',
'service': 'EC2',
'region': None,
'secret_key': None,
'visibility': 'all',
'push_notifications': False,
}
for (prop, default) in defaults.iteritems():
setattr(self, prop, kwargs.get(prop, default))
def _retry_till_image_is_public(self, image):
""" Comment goes here """
driver = self._connect()
is_image_public = False
while True:
try:
is_image_public = driver.ex_modify_image_attribute(
image,
{'LaunchPermission.Add.1.Group': 'all'})
except Exception as e:
if 'InvalidAMIID.Unavailable' in str(e):
# The copy isn't completed yet, so wait for 20 seconds
# more.
sleep(20)
continue
break
return is_image_public
def _retry_till_snapshot_is_public(self, snapshot):
driver = self._connect()
while True:
is_snapshot_public = driver.ex_modify_snapshot_attribute(
snapshot,
{'CreateVolumePermission.Add.1.Group': 'all'})
if is_snapshot_public:
break
return is_snapshot_public
def _retry_till_image_is_available(self, image_id):
driver = self._connect()
while True:
try:
image = driver.get_image(image_id)
image_name = image.name
if image_name is None:
continue
return image
except Exception as e:
if 'InvalidAMIID.Unavailable' in str(e):
# The copy isn't completed yet, so wait for 20 seconds
# more.
sleep(20)
continue
def _retry_till_snapshot_is_available(self, image):
driver = self._connect()
while True:
image = driver.get_image(image.id)
snapshot_id = image.extra['block_device_mapping'][0]['ebs']['snapshot_id']
if snapshot_id:
break
return snapshot_id
def _generate_dummy_snapshot_object(self, snapshot_id):
driver = self._connect()
snapshot_obj = type('', (), {})()
snapshot_obj.id = snapshot_id
snapshot = driver.list_snapshots(snapshot=snapshot_obj)
return snapshot
def _retry_till_blk_mapping_is_available(self, image):
while True:
image = self._connect().get_image(image_id=image.id)
blk_mapping = image.extra['block_device_mapping']
if blk_mapping:
return blk_mapping
def get_snapshot_from_image(self, image):
""" Comment goes here """
if isinstance(image, str):
image_id = image
image = self._connect().get_image(image_id)
blk_mapping = image.extra['block_device_mapping']
if not blk_mapping:
blk_mapping = self._retry_till_blk_mapping_is_available(image)
snapshot_id = blk_mapping[0]['ebs']['snapshot_id']
if snapshot_id is None:
snapshot_id = self._retry_till_snapshot_is_available(image)
snapshot = self._generate_dummy_snapshot_object(snapshot_id)[0]
return snapshot
def get_volume_type_from_image(self, image):
if isinstance(image, str):
image_id = image
image = self._connect().get_image(image_id)
blk_mapping = image.extra['block_device_mapping']
if not blk_mapping:
blk_mapping = self._retry_till_blk_mapping_is_available(image)
return blk_mapping[0]['ebs']['volume_type']
def get_virt_type_from_image(self, image):
return 'hvm'
def publish_images(self, region_image_mapping=None):
""" Comment goes here """
published_images = []
if region_image_mapping is None:
return published_images
for region, image_id in region_image_mapping:
self.set_region(region)
_log.info('Publish image (%s) in %s started' % (image_id, region))
image = self._connect().get_image(image_id=image_id)
is_image_public = self._retry_till_image_is_public(image)
_log.info('Publish image (%s) in %s completed' % (image_id, region))
_log.info('Publish snaphsot for image (%s) in %s started' % (image_id, region))
snapshot = self.get_snapshot_from_image(image)
_log.info('Fetched snapshot for image (%s): %s' % (image_id, snapshot.id))
is_snapshot_public = self._retry_till_snapshot_is_public(snapshot)
_log.info('Publish snaphsot for image (%s) in %s completed' % (image_id, region))
volume_type = self.get_volume_type_from_image(image)
virt_type = self.get_virt_type_from_image(image)
if self.push_notifications:
fedimg.messenger.notify(
topic='image.publish',
msg=dict(
image_name=get_image_name_from_ami_name_for_fedmsg(image.name),
image_url=self.image_url,
destination=self.region,
service=self.service,
compose=self.compose_id,
extra=dict(
id=image.id,
virt_type=virt_type,
vol_type=volume_type
)
)
)
fedimg.messenger.notify(
topic='image.upload',
msg=dict(
image_name=get_image_name_from_ami_name_for_fedmsg(image.name),
image_url=self.image_url,
destination=self.region,
service=self.service,
status='completed',
compose=self.compose_id,
extra=dict(
id=image.id,
virt_type=virt_type,
vol_type=volume_type
)
)
)
published_images.append({
'image_id': image.id,
'is_image_public': is_image_public,
'snapshot_id': snapshot.id,
'is_snapshot_public': is_snapshot_public,
'regions': self.region
})
return published_images
def copy_images_to_regions(self, image_id=None, base_region=None, regions=None):
""" Comment goes here """
if (image_id is None) or (regions is None) or (base_region is None):
return
counter = 0
copied_images = []
self.set_region(base_region)
image = self._connect().get_image(image_id=image_id)
if not image:
return []
for region in regions:
_log.info('Copy %s to %s started' % (image_id, region))
self.set_region(region)
self.image_name = get_image_name_from_ami_name(image.name, region)
while True:
if counter > 0:
self.image_name = re.sub(
'\d(?!\d)',
lambda x: str(int(x.group(0))+1),
self.image_name
)
try:
copied_image = self._connect().copy_image(
source_region=base_region,
image=image,
name=self.image_name,
description=self.image_description)
copied_image = self._retry_till_image_is_available(copied_image.id)
virt_type = image.extra['virtualization_type']
volume_type = image.extra['block_device_mapping'][0]['ebs']['volume_type']
if self.push_notifications:
fedimg.messenger.notify(
topic='image.copy',
msg=dict(
image_name=get_image_name_from_ami_name_for_fedmsg(copied_image.name),
destination=self.region,
service=self.service,
compose_id=self.compose_id,
extra=dict(
id=copied_image.id,
virt_type=virt_type,
vol_type=volume_type,
source_image_id=image.id
)
)
)
_log.info('Copy %s to %s is completed.' % (image_id, region))
copied_images.append({
'region': region,
'copied_image_id': copied_image.id
})
break
except Exception as e:
_log.info('Could not register '
'with name: %r' % self.image_name)
if 'InvalidAMIName.Duplicate' in str(e):
counter = counter + 1
else:
_log.info('Failed')
break
return copied_images
def deprecate_images(self, image_ids=None, snapshot_perm='all'):
raise NotImplementedError
def delete_images(self, image_ids=None, snapshot_perm='all'):
raise NotImplementedError

183
files/hotfix/fedimg/utils.py
View file

@ -1,183 +0,0 @@
# This file is part of fedimg.
# Copyright (C) 2014-2017 Red Hat, Inc.
#
# fedimg is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# fedimg is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with fedimg; if not, see http://www.gnu.org/licenses,
# or write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors: David Gay <dgay@redhat.com>
# Ralph Bean <rbean@redhat.com>
# Sayan Chowdhury <sayanchowdhury@fedoraproject.org>
"""
Utility functions for fedimg.
"""
import logging
_log = logging.getLogger(__name__)
import functools
import os
import re
import socket
import subprocess
import tempfile
import paramiko
from libcloud.compute.types import Provider
from libcloud.compute.providers import get_driver
def get_file_arch(file_name):
""" Takes a file name (probably of a .raw.xz image file) and returns
the suspected architecture of the contained image. If it doesn't look
like a 32-bit or 64-bit image, None is returned. """
if file_name.find('x86_64') != -1:
return 'x86_64'
else:
return None
def get_rawxz_urls(location, images):
""" Iterates through all the images metadata and returns the url of .raw.xz
files.
"""
rawxz_list = [f['path'] for f in images if f['path'].endswith('.raw.xz')]
if not rawxz_list:
return []
return map((lambda path: '{}/{}'.format(location, path)), rawxz_list)
def get_virt_types_from_url(url):
""" Takes a URL to a .raw.xz image file) and returns the suspected
virtualization type that the image file should be registered as. """
return ['hvm']
def region_to_driver(region):
""" Takes a region name (ex. 'eu-west-1') and returns
the appropriate libcloud provider value. """
cls = get_driver(Provider.EC2)
return functools.partial(cls, region=region)
def ssh_connection_works(username, ip, keypath):
""" Returns True if an SSH connection can me made to `username`@`ip`. """
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
works = False
try:
ssh.connect(ip, username=username,
key_filename=keypath)
works = True
except (paramiko.BadHostKeyException,
paramiko.AuthenticationException,
paramiko.SSHException, socket.error):
pass
ssh.close()
return works
def get_value_from_dict(_dict, *keys):
for key in keys:
try:
_dict = _dict[key]
except KeyError:
return None
return _dict
def external_run_command(command):
_log.debug("Starting the command: %r" % command)
ret = subprocess.Popen(' '.join(command), stdin=subprocess.PIPE, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
close_fds=True)
out, err = ret.communicate()
_log.debug("Finished executing the command: %r" % command)
retcode = ret.returncode
return out, err, retcode
def get_item_from_regex(output, regex):
match = re.search(regex, output)
if match is None:
return ''
else:
return match.group(1)
def get_file_name_image(image_url):
return image_url.split('/')[-1]
def get_source_from_image(image_url):
tmpdir = tempfile.mkdtemp()
file_name = get_file_name_image(image_url)
file_path = os.path.join(tmpdir, file_name)
_log.info("[PREP] Preparing temporary directory for download: %r" % tmpdir)
output, error, retcode = external_run_command([
'wget',
image_url,
'-P',
tmpdir
])
if retcode != 0:
return ''
output, error, retcode = external_run_command([
'xzcat',
file_path,
'>',
file_path.rstrip('.xz')
])
if retcode != 0:
return ''
return file_path.rstrip('.xz')
def get_volume_type_from_image(image, region):
return image.extra['block_device_mapping'][0]['ebs']['volume_type']
def get_virt_type_from_image(image):
return 'hvm'
def get_image_name_from_image(image_url, virt_type='', region='', respin='0',
volume_type=''):
file_name = get_file_name_image(image_url)
build_name = file_name.replace('.raw.xz', '')
return '-'.join(
[x for x in [build_name, virt_type, region, volume_type, respin] if x])
def get_image_name_from_ami_name(image_name, region):
name_vt_region, volume_type, respin = image_name.rsplit('-', 2)
name_vt = name_vt_region.rsplit('-', 3)[:-3][0]
return '-'.join(
[x for x in [name_vt, region, volume_type, respin] if x])
def get_image_name_from_ami_name_for_fedmsg(image_name):
name_vt_region, volume_type, respin = image_name.rsplit('-', 2)
image_name = name_vt_region.rsplit('-', 4)[:-4][0]
return image_name