From 426a07abff5ee6f1d052369f69c26d2c7c77a749 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Wed, 31 May 2017 16:32:10 +0530 Subject: [PATCH] Revert "Add the original models and __init__.py files for hotfix" This reverts commit 44d0d33361602b49664a406386077bf34b4a9689. --- files/hotfix/autocloud/__init__.py | 89 ---------- files/hotfix/autocloud/consumer.py | 226 ++++++++++++++---------- files/hotfix/autocloud/models.py | 121 ------------- roles/autocloud/backend/tasks/main.yml | 16 -- roles/autocloud/frontend/tasks/main.yml | 15 -- 5 files changed, 130 insertions(+), 337 deletions(-) delete mode 100644 files/hotfix/autocloud/__init__.py delete mode 100644 files/hotfix/autocloud/models.py diff --git a/files/hotfix/autocloud/__init__.py b/files/hotfix/autocloud/__init__.py deleted file mode 100644 index 84d100516f..0000000000 --- a/files/hotfix/autocloud/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -from retask.task import Task -from retask.queue import Queue - -import autocloud -from autocloud.models import init_model, ComposeJobDetails -from autocloud.producer import publish_to_fedmsg - -import datetime - -import logging -log = logging.getLogger("fedmsg") - - -def produce_jobs(infox): - """ Queue the jobs into jobqueue - :args infox: list of dictionaries contains the image url and the buildid - """ - jobqueue = Queue('jobqueue') - jobqueue.connect() - - family_mapping = { - 'Cloud_Base': 'b', - 'Atomic': 'a' - } - - session = init_model() - timestamp = datetime.datetime.now() - for info in infox: - image_name = info['path'].split('.x86_64')[0].split('/')[-1] - jd = ComposeJobDetails( - arch=info['arch'], - compose_id=info['compose']['id'], - created_on=timestamp, - family=family_mapping[info['subvariant']], - image_url=info['absolute_path'], - last_updated=timestamp, - release=info['compose']['release'], - status='q', - subvariant=info['subvariant'], - user='admin', - image_format=info['format'], - image_type=info['type'], - image_name=image_name, - ) - session.add(jd) - session.commit() - - job_details_id = jd.id - log.info('Save {jd_id} to database'.format(jd_id=job_details_id)) - - info.update({'job_id': jd.id}) - task = Task(info) - jobqueue.enqueue(task) - log.info('Enqueue {jd_id} to redis'.format(jd_id=job_details_id)) - - publish_to_fedmsg(topic='image.queued', - compose_url=info['absolute_path'], - compose_id=info['compose']['id'], - image_name=image_name, - status='queued', - job_id=info['job_id'], - release=info['compose']['release'], - family=jd.family.value, - type=info['type']) - - session.close() - - -def is_valid_image(image_url): - if autocloud.VIRTUALBOX: - supported_image_ext = ('.vagrant-virtualbox.box',) - else: - supported_image_ext = ('.qcow2', '.vagrant-libvirt.box') - - if image_url.endswith(supported_image_ext): - return True - - return False - - -def get_image_name(image_name): - if 'vagrant' in image_name.lower(): - if autocloud.VIRTUALBOX: - image_name = '{image_name}-Virtualbox'.format( - image_name=image_name) - else: - image_name = '{image_name}-Libvirt'.format(image_name=image_name) - return image_name diff --git a/files/hotfix/autocloud/consumer.py b/files/hotfix/autocloud/consumer.py index 1147b769c4..c70cde9841 100644 --- a/files/hotfix/autocloud/consumer.py +++ b/files/hotfix/autocloud/consumer.py @@ -1,18 +1,11 @@ # -*- coding: utf-8 -*- -from datetime import datetime -import requests import fedmsg.consumers -import fedfind.release - -from sqlalchemy import exc +import koji +from autocloud.utils import get_image_url, produce_jobs, get_image_name import autocloud -from autocloud.models import init_model, ComposeDetails -from autocloud.producer import publish_to_fedmsg -from autocloud.utils import is_valid_image, produce_jobs - import logging log = logging.getLogger("fedmsg") @@ -20,110 +13,151 @@ DEBUG = autocloud.DEBUG class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): - """ - Fedmsg consumer for Autocloud - """ if DEBUG: topic = [ - 'org.fedoraproject.dev.__main__.pungi.compose.status.change' + 'org.fedoraproject.dev.__main__.buildsys.build.state.change', + 'org.fedoraproject.dev.__main__.buildsys.task.state.change', ] else: topic = [ - 'org.fedoraproject.prod.pungi.compose.status.change' + 'org.fedoraproject.prod.buildsys.build.state.change', + 'org.fedoraproject.prod.buildsys.task.state.change', ] config_key = 'autocloud.consumer.enabled' def __init__(self, *args, **kwargs): - log.info("Autocloud Consumer is ready for action.") super(AutoCloudConsumer, self).__init__(*args, **kwargs) + def _get_tasks(self, builds): + """ Takes a list of koji createImage task IDs and returns dictionary of + build ids and image url corresponding to that build ids""" + + if autocloud.VIRTUALBOX: + _supported_images = ('Fedora-Cloud-Base-Vagrant', + 'Fedora-Cloud-Atomic-Vagrant',) + else: + _supported_images = ('Fedora-Cloud-Base-Vagrant', + 'Fedora-Cloud-Atomic-Vagrant', + 'Fedora-Cloud-Atomic', 'Fedora-Cloud-Base',) + + for build in builds: + log.info('Got Koji build {0}'.format(build)) + + # Create a Koji connection to the Fedora Koji instance + koji_session = koji.ClientSession(autocloud.KOJI_SERVER_URL) + + image_files = [] # list of full URLs of files + + if len(builds) == 1: + task_result = koji_session.getTaskResult(builds[0]) + name = task_result.get('name') + #TODO: Change to get the release information from PDC instead + # of koji once it is set up + release = task_result.get('version') + if name in _supported_images: + task_relpath = koji.pathinfo.taskrelpath(int(builds[0])) + url = get_image_url(task_result.get('files'), task_relpath) + if url: + name = get_image_name(image_name=name) + data = { + 'buildid': builds[0], + 'image_url': url, + 'name': name, + 'release': release, + } + image_files.append(data) + elif len(builds) >= 2: + koji_session.multicall = True + for build in builds: + koji_session.getTaskResult(build) + results = koji_session.multiCall() + for result in results: + + if not result: + continue + + name = result[0].get('name') + if name not in _supported_images: + continue + + #TODO: Change to get the release information from PDC instead + # of koji once it is set up + release = result[0].get('version') + task_relpath = koji.pathinfo.taskrelpath( + int(result[0].get('task_id'))) + url = get_image_url(result[0].get('files'), task_relpath) + if url: + name = get_image_name(image_name=name) + data = { + 'buildid': result[0]['task_id'], + 'image_url': url, + 'name': name, + 'release': release, + } + image_files.append(data) + + return image_files + def consume(self, msg): """ This is called when we receive a message matching the topic. """ + if msg['topic'].endswith('.buildsys.task.state.change'): + # Do the thing you've always done... this will go away soon. + # releng is transitioning away from it. + self._consume_scratch_task(msg) + elif msg['topic'].endswith('.buildsys.build.state.change'): + # Do the new thing we need to do. handle a 'real build' from koji, + # not just a scratch task. + self._consume_real_build(msg) + else: + raise NotImplementedError("Should be impossible to get here...") + + def _consume_real_build(self, msg): + builds = list() # These will be the Koji task IDs to upload, if any. + + msg = msg['body']['msg'] + if msg['owner'] != 'releng': + log.debug("Dropping message. Owned by %r" % msg['owner']) + return + + if msg['instance'] != 'primary': + log.info("Dropping message. From %r instance." % msg['instance']) + return + + # Don't upload *any* images if one of them fails. + if msg['new'] != 1: + log.info("Dropping message. State is %r" % msg['new']) + return + + koji_session = koji.ClientSession(autocloud.KOJI_SERVER_URL) + children = koji_session.getTaskChildren(msg['task_id']) + for child in children: + if child["method"] == "createImage": + builds.append(child["id"]) + + if len(builds) > 0: + produce_jobs(self._get_tasks(builds)) + + def _consume_scratch_task(self, msg): + builds = list() # These will be the Koji build IDs to upload, if any. + + msg_info = msg["body"]["msg"]["info"] + log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) - STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) - VARIANTS_F = ('CloudImages',) + # If the build method is "image", we check to see if the child + # task's method is "createImage". + if msg_info["method"] == "image": + if isinstance(msg_info["children"], list): + for child in msg_info["children"]: + if child["method"] == "createImage": + # We only care about the image if the build + # completed successfully (with state code 2). + if child["state"] == 2: + builds.append(child["id"]) - images = [] - compose_db_update = False - msg_body = msg['body'] - status = msg_body['msg']['status'] - compose_images_json = None - - if status in STATUS_F: - location = msg_body['msg']['location'] - json_metadata = '{}/metadata/images.json'.format(location) - resp = requests.get(json_metadata) - compose_images_json = getattr(resp, 'json', False) - - if compose_images_json is not None: - compose_images_json = compose_images_json() - compose_images = compose_images_json['payload']['images'] - compose_details = compose_images_json['payload']['compose'] - compose_images = dict((variant, compose_images[variant]) - for variant in VARIANTS_F - if variant in compose_images) - compose_id = compose_details['id'] - rel = fedfind.release.get_release(cid=compose_id) - release = rel.release - compose_details.update({'release': release}) - - compose_images_variants = [variant for variant in VARIANTS_F - if variant in compose_images] - - for variant in compose_images_variants: - compose_image = compose_images[variant] - for arch, payload in compose_image.iteritems(): - for item in payload: - relative_path = item['path'] - if not is_valid_image(relative_path): - continue - absolute_path = '{}/{}'.format(location, relative_path) - item.update({ - 'compose': compose_details, - 'absolute_path': absolute_path, - }) - images.append(item) - compose_db_update = True - - if compose_db_update: - session = init_model() - compose_date = datetime.strptime(compose_details['date'], '%Y%m%d') - try: - cd = ComposeDetails( - date=compose_date, - compose_id=compose_details['id'], - respin=compose_details['respin'], - type=compose_details['type'], - status=u'q', - location=location, - ) - - session.add(cd) - session.commit() - - compose_details.update({ - 'status': 'queued', - 'compose_job_id': cd.id, - }) - publish_to_fedmsg(topic='compose.queued', - **compose_details) - except exc.IntegrityError: - session.rollback() - cd = session.query(ComposeDetails).filter_by( - compose_id=compose_details['id']).first() - log.info('Compose already exists %s: %s' % ( - compose_details['id'], - cd.id - )) - session.close() - - num_images = len(images) - for pos, image in enumerate(images): - image.update({'pos': (pos+1, num_images)}) - - produce_jobs(images) + if len(builds) > 0: + produce_jobs(self._get_tasks(builds)) diff --git a/files/hotfix/autocloud/models.py b/files/hotfix/autocloud/models.py deleted file mode 100644 index 4a6f35f5a5..0000000000 --- a/files/hotfix/autocloud/models.py +++ /dev/null @@ -1,121 +0,0 @@ -# -*- coding: utf-8 -*- - -import datetime - -from sqlalchemy import Column, Integer, String, DateTime, Text -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.orm import scoped_session -from sqlalchemy_utils import ChoiceType - -import autocloud - -Base = declarative_base() - - -class JobDetails(Base): - __tablename__ = 'job_details' - - STATUS_TYPES = ( - ('s', 'Success'), - ('f', 'Failed'), - ('a', 'Aborted'), - ('r', 'Running'), - ('q', 'Queued') - ) - - IMAGE_FAMILY_TYPES = ( - ('b', 'Base'), - ('a', 'Atomic') - ) - - ARCH_TYPES = ( - ('i386', 'i386'), - ('x86_64', 'x86_64') - ) - - id = Column(Integer, primary_key=True) - taskid = Column(String(255), nullable=False) - status = Column(ChoiceType(STATUS_TYPES)) - family = Column(ChoiceType(IMAGE_FAMILY_TYPES)) - arch = Column(ChoiceType(ARCH_TYPES)) - release = Column(String(255)) - output = Column(Text, nullable=False, default='') - created_on = Column(DateTime, default=datetime.datetime.utcnow) - last_updated = Column(DateTime, default=datetime.datetime.utcnow) - user = Column(String(255), nullable=False) - - -class ComposeDetails(Base): - __tablename__ = 'compose_details' - - STATUS_TYPES = ( - ('c', 'Complete'), - ('q', 'Queued'), - ('r', 'Running'), - ) - id = Column(Integer, primary_key=True) - date = Column(DateTime, nullable=False) - compose_id = Column(String(255), nullable=False, unique=True) - respin = Column(Integer, nullable=False) - type = Column(String(255), nullable=False) - passed = Column(Integer, nullable=True, default=0) - failed = Column(Integer, nullable=True, default=0) - status = Column(ChoiceType(STATUS_TYPES)) - created_on = Column(DateTime, default=datetime.datetime.utcnow) - last_updated = Column(DateTime, default=datetime.datetime.utcnow) - location = Column(String(255), nullable=False) - - -class ComposeJobDetails(Base): - __tablename__ = 'compose_job_details' - - STATUS_TYPES = ( - ('s', 'Success'), - ('f', 'Failed'), - ('a', 'Aborted'), - ('r', 'Running'), - ('q', 'Queued') - ) - - IMAGE_FAMILY_TYPES = ( - ('b', u'Base'), - ('a', u'Atomic') - ) - - ARCH_TYPES = ( - ('i386', 'i386'), - ('x86_64', 'x86_64') - ) - - id = Column(Integer, primary_key=True) - arch = Column(ChoiceType(ARCH_TYPES)) - compose_id = Column(String(255), nullable=False) - created_on = Column(DateTime, default=datetime.datetime.utcnow) - family = Column(ChoiceType(IMAGE_FAMILY_TYPES)) - image_url = Column(String(255), nullable=False) - last_updated = Column(DateTime, default=datetime.datetime.utcnow) - output = Column(Text, nullable=False, default='') - release = Column(String(255)) - status = Column(ChoiceType(STATUS_TYPES)) - subvariant = Column(String(255), nullable=False) - user = Column(String(255), nullable=False) - image_format = Column(String(255), nullable=False) - image_type = Column(String(255), nullable=False) - image_name = Column(String(255), nullable=False) - - -def create_tables(): - # Create an engine that stores data in the local directory - engine = create_engine(autocloud.SQLALCHEMY_URI) - - # Create all tables in the engine. This is equivalent to "Create Table" - # statements in raw SQL. - Base.metadata.create_all(engine) - - -def init_model(): - engine = create_engine(autocloud.SQLALCHEMY_URI) - scopedsession = scoped_session(sessionmaker(bind=engine)) - return scopedsession diff --git a/roles/autocloud/backend/tasks/main.yml b/roles/autocloud/backend/tasks/main.yml index 5e61d09479..a09c40cfd1 100644 --- a/roles/autocloud/backend/tasks/main.yml +++ b/roles/autocloud/backend/tasks/main.yml @@ -138,19 +138,3 @@ tags: - autocloud - autocloud/backend - -# -# Install hotfix to add the architecture to aarch64 -# See PR - https://github.com/kushaldas/autocloud/pull/56/ -# -- name: hotfix - copy over models.py to autocloud/models.py - copy: src='{{ files }}/{{ item.src }}' dest={{ item.dest }} - with_items: - - { src: 'hotfix/autocloud/models.py', dest: '/usr/lib/python2.7/site-packages/autocloud' } - - { src: 'hotfix/autocloud/consumer.py', dest: '/usr/lib/python2.7/site-packages/autocloud' } - - { src: 'hotfix/autocloud/__init__.py', dest: '/usr/lib/python2.7/site-packages/autocloud/utils' } - notify: - - restart fedmsg-hub - tags: - - autocloud - - hotfix diff --git a/roles/autocloud/frontend/tasks/main.yml b/roles/autocloud/frontend/tasks/main.yml index 3312fb52e8..1084b4c705 100644 --- a/roles/autocloud/frontend/tasks/main.yml +++ b/roles/autocloud/frontend/tasks/main.yml @@ -66,18 +66,3 @@ - autocloud - autocloud/frontend - selinux - -# -# Install hotfix to add the architecture to aarch64 -# See PR - https://github.com/kushaldas/autocloud/pull/56/ -# -- name: hotfix - copy over models.py to autocloud/models.py - copy: src='{{ files }}/{{ item.src }}' dest={{ item.dest }} - with_items: - - { src: 'hotfix/autocloud/models.py', dest: '/usr/lib/python2.7/site-packages/autocloud' } - - { src: 'hotfix/autocloud/__init__.py', dest: '/usr/lib/python2.7/site-packages/autocloud/utils' } - notify: - - restart fedmsg-hub - tags: - - autocloud - - hotfix