Drop this atomic-composer role for now to avoid confusion.
This commit is contained in:
Kevin Fenzi
parent
b552bd3e2a
commit
c59a958c8a
6 changed files with 0 additions and 364 deletions
|
|
@ -42,12 +42,3 @@
|
|||
|
||||
handlers:
|
||||
- include: "{{ handlers }}/restart_services.yml"
|
||||
|
||||
- name: Install the fedmsg-atomic-composer
|
||||
# Staging only, for now.
|
||||
hosts: releng-stg
|
||||
user: root
|
||||
gather_facts: True
|
||||
|
||||
roles:
|
||||
- atomic-composer
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
This role is for the (currently single) server that performs
|
||||
rpm-ostree tree composes. It takes as input two primary data sets:
|
||||
|
||||
1) The Fedora package set for a particular branch (f21, rawhide)
|
||||
2) A git repository, https://git.fedorahosted.org/cgit/fedora-atomic.git
|
||||
|
||||
This output gets written into an OSTree repository which is pushed to:
|
||||
http://dl.fedoraproject.org/pub/fedora/linux/atomic/
|
||||
|
||||
This is only for Fedora 21. rawhide trees are handled via
|
||||
https://git.fedorahosted.org/cgit/releng/tree/scripts/run-pungi
|
||||
|
||||
It uses lmacken's
|
||||
|
||||
https://github.com/fedora-infra/fedmsg-atomic-composer
|
||||
|
||||
which has the ability to listen for fedmsg notifications of package
|
||||
set changes, but is not actually doing so at the moment due to
|
||||
concerns about fedmsg reliability (?).
|
||||
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import copy
|
||||
import json
|
||||
import glob
|
||||
import time
|
||||
import shutil
|
||||
import logging
|
||||
import tempfile
|
||||
import traceback
|
||||
import subprocess
|
||||
import pkg_resources
|
||||
|
||||
from datetime import datetime
|
||||
from mako.template import Template
|
||||
|
||||
|
||||
class AtomicComposer(object):
|
||||
"""An atomic ostree composer"""
|
||||
|
||||
def compose(self, release):
|
||||
release = copy.deepcopy(release)
|
||||
# We need to use /var/tmp because systemd-nspawn will mount
|
||||
# a tmpfs on /tmp in the container.
|
||||
release['tmp_dir'] = tempfile.mkdtemp(dir='/var/tmp')
|
||||
release['timestamp'] = time.strftime('%y%m%d.%H%M')
|
||||
try:
|
||||
self.setup_logger(release)
|
||||
self.log.debug(release)
|
||||
self.update_configs(release)
|
||||
self.generate_mock_config(release)
|
||||
self.init_mock(release)
|
||||
self.sync_in(release)
|
||||
self.ostree_init(release)
|
||||
self.generate_repo_files(release)
|
||||
self.ostree_compose(release)
|
||||
self.update_ostree_summary(release)
|
||||
self.sync_out(release)
|
||||
release['result'] = 'success'
|
||||
self.cleanup(release)
|
||||
except:
|
||||
if hasattr(self, 'log'):
|
||||
self.log.exception('Compose failed')
|
||||
else:
|
||||
traceback.print_exc()
|
||||
release['result'] = 'failed'
|
||||
return release
|
||||
|
||||
def setup_logger(self, release):
|
||||
name = '{name}-{timestamp}'.format(**release)
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
log_dir = release['log_dir']
|
||||
log_file = os.path.join(log_dir, name)
|
||||
release['log_file'] = log_file
|
||||
if not os.path.isdir(log_dir):
|
||||
os.makedirs(log_dir)
|
||||
stdout = logging.StreamHandler()
|
||||
handler = logging.FileHandler(log_file)
|
||||
log_format = ('%(asctime)s - %(levelname)s - %(filename)s:'
|
||||
'%(lineno)d - %(message)s')
|
||||
formatter = logging.Formatter(log_format)
|
||||
handler.setFormatter(formatter)
|
||||
handler.setLevel(logging.DEBUG)
|
||||
stdout.setFormatter(formatter)
|
||||
stdout.setLevel(logging.DEBUG)
|
||||
logger.addHandler(handler)
|
||||
logger.addHandler(stdout)
|
||||
self.log = logger
|
||||
|
||||
def cleanup(self, release):
|
||||
"""Cleanup any temporary files after the compose"""
|
||||
shutil.rmtree(release['tmp_dir'])
|
||||
|
||||
def update_configs(self, release):
|
||||
""" Update the fedora-atomic.git repositories for a given release """
|
||||
git_repo = release['git_repo']
|
||||
git_cache = release['git_cache']
|
||||
if not os.path.isdir(git_cache):
|
||||
self.call(['git', 'clone', '--mirror', git_repo, git_cache])
|
||||
else:
|
||||
self.call(['git', 'fetch', '--all', '--prune'], cwd=git_cache)
|
||||
git_dir = release['git_dir'] = os.path.join(release['tmp_dir'],
|
||||
os.path.basename(git_repo))
|
||||
self.call(['git', 'clone', '-b', release['git_branch'],
|
||||
git_cache, git_dir])
|
||||
|
||||
if release['delete_repo_files']:
|
||||
for repo_file in glob.glob(os.path.join(git_dir, '*.repo')):
|
||||
self.log.info('Deleting %s' % repo_file)
|
||||
os.unlink(repo_file)
|
||||
|
||||
def mock_cmd(self, release, *cmd, **kwargs):
|
||||
"""Run a mock command in the chroot for a given release"""
|
||||
fmt = '{mock_cmd}'
|
||||
if kwargs.get('new_chroot') is True:
|
||||
fmt +=' --new-chroot'
|
||||
fmt += ' --configdir={mock_dir}'
|
||||
self.call(fmt.format(**release).split()
|
||||
+ list(cmd))
|
||||
|
||||
def init_mock(self, release):
|
||||
"""Initialize/update our mock chroot"""
|
||||
root = '/var/lib/mock/%s' % release['mock']
|
||||
if not os.path.isdir(root):
|
||||
self.mock_cmd(release, '--init')
|
||||
self.log.info('mock chroot initialized')
|
||||
else:
|
||||
if release.get('mock_clean'):
|
||||
self.mock_cmd(release, '--clean')
|
||||
self.mock_cmd(release, '--init')
|
||||
self.log.info('mock chroot cleaned & initialized')
|
||||
else:
|
||||
self.mock_cmd(release, '--update')
|
||||
self.log.info('mock chroot updated')
|
||||
|
||||
def generate_mock_config(self, release):
|
||||
"""Dynamically generate our mock configuration"""
|
||||
mock_tmpl = pkg_resources.resource_string(__name__, 'templates/mock.mako')
|
||||
mock_dir = release['mock_dir'] = os.path.join(release['tmp_dir'], 'mock')
|
||||
mock_cfg = os.path.join(release['mock_dir'], release['mock'] + '.cfg')
|
||||
os.mkdir(mock_dir)
|
||||
for cfg in ('site-defaults.cfg', 'logging.ini'):
|
||||
os.symlink('/etc/mock/%s' % cfg, os.path.join(mock_dir, cfg))
|
||||
with file(mock_cfg, 'w') as cfg:
|
||||
mock_out = Template(mock_tmpl).render(**release)
|
||||
self.log.debug('Writing %s:\n%s', mock_cfg, mock_out)
|
||||
cfg.write(mock_out)
|
||||
|
||||
def mock_chroot(self, release, cmd, **kwargs):
|
||||
"""Run a commend in the mock container for a release"""
|
||||
self.mock_cmd(release, '--chroot', cmd, **kwargs)
|
||||
|
||||
def generate_repo_files(self, release):
|
||||
"""Dynamically generate our yum repo configuration"""
|
||||
repo_tmpl = pkg_resources.resource_string(__name__, 'templates/repo.mako')
|
||||
repo_file = os.path.join(release['git_dir'], '%s.repo' % release['repo'])
|
||||
with file(repo_file, 'w') as repo:
|
||||
repo_out = Template(repo_tmpl).render(**release)
|
||||
self.log.debug('Writing repo file %s:\n%s', repo_file, repo_out)
|
||||
repo.write(repo_out)
|
||||
self.log.info('Wrote repo configuration to %s', repo_file)
|
||||
|
||||
def ostree_init(self, release):
|
||||
"""Initialize the OSTree for a release"""
|
||||
out = release['output_dir'].rstrip('/')
|
||||
base = os.path.dirname(out)
|
||||
if not os.path.isdir(base):
|
||||
self.log.info('Creating %s', base)
|
||||
os.makedirs(base, mode=0755)
|
||||
if not os.path.isdir(out):
|
||||
self.mock_chroot(release, release['ostree_init'])
|
||||
|
||||
def ostree_compose(self, release):
|
||||
"""Compose the OSTree in the mock container"""
|
||||
start = datetime.utcnow()
|
||||
treefile = os.path.join(release['git_dir'], 'treefile.json')
|
||||
cmd = release['ostree_compose'] % treefile
|
||||
with file(treefile, 'w') as tree:
|
||||
json.dump(release['treefile'], tree)
|
||||
# Only use new_chroot for the invocation, as --clean and --new-chroot are buggy together right now
|
||||
self.mock_chroot(release, cmd, new_chroot=True)
|
||||
self.log.info('rpm-ostree compose complete (%s)',
|
||||
datetime.utcnow() - start)
|
||||
|
||||
def update_ostree_summary(self, release):
|
||||
"""Update the ostree summary file and return a path to it"""
|
||||
self.log.info('Updating the ostree summary for %s', release['name'])
|
||||
self.mock_chroot(release, release['ostree_summary'])
|
||||
return os.path.join(release['output_dir'], 'summary')
|
||||
|
||||
def sync_in(self, release):
|
||||
"""Sync the canonical repo to our local working directory"""
|
||||
tree = release['canonical_dir']
|
||||
if os.path.exists(tree) and release.get('rsync_in_objs'):
|
||||
out = release['output_dir']
|
||||
if not os.path.isdir(out):
|
||||
self.log.info('Creating %s', out)
|
||||
os.makedirs(out)
|
||||
self.call(release['rsync_in_objs'])
|
||||
self.call(release['rsync_in_rest'])
|
||||
|
||||
def sync_out(self, release):
|
||||
"""Sync our tree to the canonical location"""
|
||||
if release.get('rsync_out_objs'):
|
||||
tree = release['canonical_dir']
|
||||
if not os.path.isdir(tree):
|
||||
self.log.info('Creating %s', tree)
|
||||
os.makedirs(tree)
|
||||
self.call(release['rsync_out_objs'])
|
||||
self.call(release['rsync_out_rest'])
|
||||
|
||||
def call(self, cmd, **kwargs):
|
||||
"""A simple subprocess wrapper"""
|
||||
if isinstance(cmd, basestring):
|
||||
cmd = cmd.split()
|
||||
self.log.info('Running %s', cmd)
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, **kwargs)
|
||||
out, err = p.communicate()
|
||||
if out:
|
||||
self.log.info(out)
|
||||
if err:
|
||||
self.log.error(err)
|
||||
if p.returncode != 0:
|
||||
self.log.error('returncode = %d' % p.returncode)
|
||||
raise Exception
|
||||
return out, err, p.returncode
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
# Check if we're running on RHEL6 and disable
|
||||
# `mock --new-chroot` and `rpm-ostree --workdir-tmpfs`
|
||||
import platform
|
||||
dist = platform.dist()
|
||||
rhel6 = dist[0] == 'redhat' and int(float(dist[1])) == 6
|
||||
|
||||
config = dict(
|
||||
releases={
|
||||
'f21-updates': {
|
||||
'name': 'f21-updates',
|
||||
'repo': 'updates',
|
||||
'version': '21',
|
||||
'arch': 'x86_64',
|
||||
|
||||
# OSTree treefile configuration
|
||||
# https://github.com/projectatomic/rpm-ostree/blob/master/doc/treefile.md
|
||||
'tree': 'docker-host',
|
||||
'treefile': {
|
||||
'include': 'fedora-atomic-docker-host.json',
|
||||
'ref': 'fedora-atomic/f21/x86_64/updates/docker-host',
|
||||
'repos': ['fedora-21', 'updates'],
|
||||
},
|
||||
|
||||
# The name of the mock container to build and maintain
|
||||
'mock': 'fedora-21-updates-x86_64',
|
||||
|
||||
# The git branch to use in the `git_repo` for the parent
|
||||
# treefile & repo configurations
|
||||
'git_branch': 'f21',
|
||||
|
||||
# Add or overwrite yum repository name:urls. This lets you
|
||||
# compose trees against your own repositories.
|
||||
'repos': {},
|
||||
},
|
||||
|
||||
'f21-updates-testing': {
|
||||
'name': 'f21-updates-testing',
|
||||
'repo': 'updates-testing',
|
||||
'version': '21',
|
||||
'arch': 'x86_64',
|
||||
'tree': 'docker-host',
|
||||
'treefile': {
|
||||
'include': 'fedora-atomic-docker-host.json',
|
||||
'ref': 'fedora-atomic/f21/x86_64/updates-testing/docker-host',
|
||||
'repos': ['fedora-21', 'updates', 'updates-testing'],
|
||||
},
|
||||
'git_branch': 'f21',
|
||||
'mock': 'fedora-21-updates-testing-x86_64',
|
||||
'repos': {},
|
||||
},
|
||||
},
|
||||
|
||||
# Package repositories to use in the mock container and ostree compose
|
||||
repos={
|
||||
'updates': 'https://dl.fedoraproject.org/pub/fedora/linux/updates/{version}/{arch}/',
|
||||
'updates-testing': 'https://dl.fedoraproject.org/pub/fedora/linux/updates/testing/{version}/{arch}/',
|
||||
},
|
||||
|
||||
# Output directories
|
||||
work_dir='/srv/fedora-atomic',
|
||||
prod_dir='{work_dir}/production',
|
||||
canonical_dir='{prod_dir}/{version}/{arch}/{repo}/{tree}',
|
||||
output_dir='{work_dir}/{version}/{arch}/{repo}/{tree}',
|
||||
log_dir='{work_dir}/logs/{version}/{arch}/{repo}/{tree}',
|
||||
|
||||
# The git repo containing our parent treefiles and yum repos
|
||||
git_repo='https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
git_cache='{work_dir}/fedora-atomic.git',
|
||||
|
||||
# Mock command
|
||||
mock_cmd='/usr/bin/mock%s-r {mock}' % (rhel6 and ' ' or ' --new-chroot '),
|
||||
|
||||
# OSTree commands
|
||||
ostree_init='/usr/bin/ostree --repo={output_dir} init --mode=archive-z2',
|
||||
ostree_compose='/usr/bin/rpm-ostree compose tree' +
|
||||
(rhel6 and ' ' or ' --workdir-tmpfs ') + '--repo={output_dir} %s',
|
||||
ostree_summary='/usr/bin/ostree --repo={output_dir} summary --update',
|
||||
|
||||
# rsync commands
|
||||
rsync_in_objs='/usr/bin/rsync -rvp --ignore-existing {canonical_dir}/objects/ {output_dir}/objects/',
|
||||
rsync_in_rest='/usr/bin/rsync -rvp --exclude=objects/ {canonical_dir}/ {output_dir}/',
|
||||
rsync_out_objs='/usr/bin/rsync -rvp --ignore-existing {output_dir}/objects/ {canonical_dir}/objects/',
|
||||
rsync_out_rest='/usr/bin/rsync -rvp --exclude=objects/ {output_dir}/ {canonical_dir}/',
|
||||
|
||||
map_to_release=('work_dir', 'prod_dir', 'output_dir', 'log_dir',
|
||||
'git_repo', 'git_cache', 'mock_cmd', 'ostree_init',
|
||||
'ostree_compose', 'ostree_summary', 'canonical_dir',
|
||||
'repos', 'rsync_in_1', 'rsync_in_2', 'rsync_out_1',
|
||||
'rsync_out_2'),
|
||||
)
|
||||
|
||||
# Map and expand certain variables to each release
|
||||
for key in config.get('map_to_release', []):
|
||||
for name, release in config['releases'].items():
|
||||
if isinstance(config[key], dict):
|
||||
release[key] = {}
|
||||
for k, v in config[key].items():
|
||||
release[key][k] = v.format(**release)
|
||||
else:
|
||||
release[key] = config[key].format(**release)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
---
|
||||
# This role sets up the fedmsg-driven atomic ostree composer
|
||||
|
||||
- yum: name=python2-fedmsg-atomic-composer state=present
|
||||
|
||||
- user: name=masher
|
||||
shell=/bin/bash
|
||||
groups="mock,ftpsync,fedmsg"
|
||||
register: user
|
||||
|
||||
- file: path=/srv/fedora-atomic/
|
||||
owner={{ user['name'] }} group={{ user['name'] }} mode=755 state=directory
|
||||
|
||||
- seboolean: name=rsync_client state=true persistent=true
|
||||
|
|
@ -1 +0,0 @@
|
|||
trees: ['rawhide', 'f21']
|
||||
Loading…
Add table
Add a link
Reference in a new issue