Remove koschei roles

This commit is contained in:
Mikolaj Izdebski 2019-09-26 06:41:31 +02:00 committed by Pierre-Yves Chibon
parent 12d3da3ebd
commit 3f8a28c085
28 changed files with 0 additions and 694 deletions

View file

@ -22,7 +22,6 @@
- builder_repo
- collectd/base
- { role: sudo, sudoers: "{{ private }}/files/sudo/koschei01-sudoers" }
- koschei/backend
- role: keytab/service
owner_user: koschei
owner_group: koschei

View file

@ -24,7 +24,6 @@
- { role: sudo, sudoers: "{{ private }}/files/sudo/koschei01-sudoers" }
- { role: openvpn/client, when: env != "staging" }
- { role: mod_wsgi, when: env != "staging" }
- koschei/frontend
tasks:
- import_tasks: "{{ tasks_path }}/2fa_client.yml"

View file

@ -1,9 +0,0 @@
#!/bin/bash
if [ "$#" -lt 2 ]; then
echo "Usage: refresh-group <group-name> <distgit-group-name> [...]"
exit 1
fi
set -e
group_name="${1}"; shift
distgit_group_name="${1}"; shift
curl -s "https://src.fedoraproject.org/api/0/group/${distgit_group_name}?projects=1" | jq -r '.projects[]|select(.namespace=="rpms").name' | koschei-admin edit-group "${group_name}" --content-from-file - "${@}"

View file

@ -1,12 +0,0 @@
#!/bin/bash
if [ "$#" -lt 2 ]; then
echo "Usage: refresh-group <group-name> <repoquery-args>..."
exit 1
fi
set -e
repodata=`find /var/cache/koschei/repodata/ -path '*/primary-f*' -name repomd.xml | sort -nr | head -n1 | xargs dirname | xargs dirname`
repoid=`grep -o 'primary-[^/]*' <<< "$repodata"`
group=$1
shift
dnf repoquery -q --available --repofrompath "$repoid,$repodata" --disablerepo \* --enablerepo "$repoid" --qf '%{sourcerpm}' "$@" | sed 's|-[^-]*-[^-]*$||' \
| koschei-admin edit-group "$group" --content-from-file - --append

View file

@ -1,31 +0,0 @@
#!/usr/libexec/koschei/koschei-admin
# Refresh group based on module contents
# Usage: koschei-refresh-module <group-name> <module-name> <module-stream>
import re
import sys
import os
from pdc_client import PDCClient
from koschei import data, backend, plugin
from koschei.config import load_config
import koschei.models as m
# Parse args
group_namespace=None
(_, group_name, module_name, module_stream) = sys.argv
if '/' in group_name:
(group_namespace, group_name) = group_name.split('/')
# Get latest module from PDC
pdc_url = 'https://{0}/rest_api/v1/'.format(os.environ['PDC'])
pdc = PDCClient(pdc_url, develop=True, ssl_verify=True)
modules = pdc['unreleasedvariants'](variant_type='module', variant_name=module_name, variant_version=module_stream, active=True, page_size=-1)
latest_module = sorted(modules, key=lambda mod: mod['variant_release'])[-1]
group_content = [re.match(r'^(.+)-[^-]+-[^-]+\.src.rpm$', rpm).group(1) for rpm in latest_module['rpms'] if rpm.endswith('.src.rpm')]
# Update contents of Koschei group
load_config(['/usr/share/koschei/config.cfg', '/etc/koschei/config-backend.cfg'])
session = backend.KoscheiBackendSession()
group = session.db.query(m.PackageGroup).filter_by(name=group_name, namespace=group_namespace).one()
data.set_group_content(session, group, group_content)
session.db.commit()

View file

@ -1,8 +0,0 @@
#!/bin/sh
# Mark all packages in global group as tracked
# Usage: koschei-track-group <group-name>
set -e
test -n "$1"
exec koschei-admin psql <<<"UPDATE package SET tracked = TRUE WHERE id IN (SELECT p.id FROM package p JOIN package_group_relation pgr ON p.base_id = pgr.base_id JOIN package_group g ON g.id = pgr.group_id WHERE NOT p.tracked AND g.name = '$1' AND g.namespace IS NULL)"

View file

@ -1,7 +0,0 @@
import socket
hostname = socket.gethostname().split('.', 1)[0]
config = {
"name": "koschei.%s" % hostname,
}

View file

@ -1,2 +0,0 @@
[Service]
MemoryLimit=2G

View file

@ -1,21 +0,0 @@
- name: reload systemd
command: systemctl daemon-reload
- name: restart koschei-polling
action: service name=koschei-polling state=restarted
- name: restart koschei-build-resolver
action: service name=koschei-build-resolver state=restarted
- name: restart koschei-repo-resolver
action: service name=koschei-repo-resolver state=restarted
- name: restart koschei-scheduler
action: service name=koschei-scheduler state=restarted
- name: restart koschei-watcher
action: service name=koschei-watcher state=restarted
- name: restart koschei backend services
service: name={{ item }} state=restarted
with_items: "{{ koschei_backend_services }}"

View file

@ -1,3 +0,0 @@
---
dependencies:
- koschei/common

View file

@ -1,139 +0,0 @@
- name: install packages
package: name={{ item }} state=present
with_items:
- koschei-admin
- koschei-backend
- koschei-backend-fedora
- koschei-backend-copr
- postgresql
- jq
tags:
- koschei
- packages
- name: create systemd config directories
file: >
path="/etc/systemd/system/{{ item }}.service.d"
state=directory
with_items: "{{ koschei_backend_services }}"
notify:
- reload systemd
tags:
- koschei
- config
- service
- name: install systemd config files (memory)
copy: >
src=systemd-memory-limit.conf
dest=/etc/systemd/system/{{ item }}.service.d/memory-limit.conf
with_items:
- koschei-build-resolver
- koschei-repo-resolver
notify:
- reload systemd
- restart koschei-build-resolver
- restart koschei-repo-resolver
tags:
- koschei
- config
- service
- name: install systemd config files (restart)
template: >
src=systemd-restart-limit.conf.j2
dest=/etc/systemd/system/{{ item }}.service.d/restart.conf
with_items: "{{ koschei_backend_services }}"
notify:
- reload systemd
tags:
- koschei
- config
- service
- name: install systemd config files (environment)
template: >
src=systemd-environment.conf.j2
dest=/etc/systemd/system/{{ item }}.service.d/environment.conf
with_items: "{{ koschei_backend_services }}"
notify:
- reload systemd
- restart koschei backend services
tags:
- koschei
- config
- service
- name: install config files
template: >
src="{{ item }}.j2"
dest="/etc/koschei/{{ item }}"
owner=koschei
group=koschei
mode=0400
with_items:
- config-admin.cfg
- config-backend.cfg
- copr-config
notify:
- restart koschei backend services
tags:
- koschei
- config
- name: install koschei fedmsg config
copy: src=koschei.py dest=/etc/fedmsg.d/koschei.py
notify:
- restart koschei backend services
tags:
- koschei
- config
- fedmsgdconfig
- name: install koji client config file
template: >
src="koji.conf.j2"
dest="/etc/koji.conf"
tags:
- koschei
- config
- name: enable services to start on boot
service: name={{ item }} state=started enabled=true
with_items: "{{ koschei_backend_services }}"
tags:
- koschei
- service
- name: Copy the cleanup cronjob
template: src=cron-db-cleanup.j2 dest=/etc/cron.d/cron-db-cleanup
tags:
- koschei
- config
- name: Copy refresh-group script
copy: >
src={{ item }}
dest=/usr/local/bin/{{ item }}
mode=755
with_items:
- koschei-refresh-group
- koschei-refresh-distgit-group
- koschei-refresh-module
- koschei-track-group
tags:
- koschei
- config
- name: Copy the group refresh cronjob
template: src=cron-refresh-groups.j2 dest=/etc/cron.d/cron-refresh-groups
tags:
- koschei
- config
- name: access to postgres for scripts
template: src=pgpass.j2 dest=/var/cache/koschei/.pgpass
owner=koschei group=koschei mode=0600
tags:
- koschei
- config

View file

@ -1,15 +0,0 @@
# Config for operations that require koscheiadmin credentials.
config = {
"database_config": {
"host": "{{ koschei_pgsql_hostname }}",
"database": "koschei",
"username": "koscheiadmin",
"password": "{{ koscheiadmin_db_password }}",
},
"unpriv_db_username": "koscheiuser",
}
# Local Variables:
# mode: Python
# End:
# vi: ft=python

View file

@ -1,123 +0,0 @@
# This is a config file for Koschei that can override values in default
# configuration in /usr/share/koschei/config.cfg. It is a python file expecting
# assignment to config dictionary which will be recursively merged with the
# default one.
config = {
"database_config": {
"host": "{{ koschei_pgsql_hostname }}",
"username": "koscheiuser",
"password": "{{ koscheiuser_db_password }}",
"database": "koschei"
},
"koji_config": {
"server": "https://{{ koschei_koji_hub }}/kojihub",
"weburl": "https://{{ koschei_koji_web }}/koji",
"topurl": "https://{{ koschei_kojipkgs }}",
"login_method": "gssapi_login",
{% if env == 'staging' %}
"max_builds": 16,
"build_arches": ['x86_64'],
"load_threshold": 1,
"task_priority": 30,
{% else %}
"max_builds": 60,
"build_arches": ['x86_64', 'aarch64', 'ppc64', 'ppc64le'],
"load_threshold": 0.75,
"task_priority": 30,
{% endif %}
},
"dependency": {
"build_group": "build",
"for_arch": "x86_64",
"repo_cache_items": 10,
"arches": ['x86_64', 'i386'],
"persist_chunk_size": 300,
"resolver_queue_size": 100,
"dependency_fetch_chunk_size": 150,
"dependency_cache_capacity": 50000,
"cache_l2_capacity": 100,
"ignore_weak_deps": True,
},
"services": {
"polling": {
{% if env == 'staging' %}
"interval": 1, # in seconds
{% else %}
"interval": 20*60, # in seconds
{% endif %}
},
"build_resolver": {
"memory_limit": 1024**2, # kilobytes
},
"repo_resolver": {
"memory_limit": 1024**2, # kilobytes
},
"watcher": {
"memory_limit": 256*1024, # kilobytes
},
},
"priorities": {
"calculation_interval": 3600, # seconds
},
"logging": {
"loggers": {
"": {
# "handlers": ["stderr", "email"],
"handlers": ["stderr"],
},
"fedmsg": {
"level": "ERROR",
},
},
"handlers": {
"email": {
"class": "logging.handlers.SMTPHandler",
"level": "WARN",
"mailhost": "localhost",
"fromaddr": "koschei@fedoraproject.org",
"toaddrs": ['sysadmin-koschei-members@fedoraproject.org'],
"subject": "Koschei backend {{ env }} warning",
},
},
},
{% if env == 'staging' %}
"plugins": ['fedmsg', 'copr'],
{% else %}
"plugins": ['fedmsg'],
{% endif %}
"fedmsg": {
{% if env == 'staging' %}
"topic": "org.fedoraproject.stg.buildsys",
{% else %}
"topic": "org.fedoraproject.prod.buildsys",
{% endif %}
},
"fedmsg-publisher": {
"enabled": True,
"modname": "koschei",
},
"caching": {
"rpm_requires": {
"backend": "dogpile.cache.memcached",
"expiration_time": None,
"arguments": {
"url": "memcached01",
},
},
},
"copr": {
"require_admin": True,
"copr_owner": "mizdebsk",
"config_path": "/etc/koschei/copr-config",
"chroot_name": "custom-1-x86_64",
"max_builds": 15,
"default_schedule_count": 8,
"fedmsg_topic": "org.fedoraproject.prod.copr.build.end",
},
}
# Local Variables:
# mode: Python
# End:
# vi: ft=python

View file

@ -1,5 +0,0 @@
[copr-cli]
login = {{ koschei_copr_login }}
username = koschei
token = {{ koschei_copr_token }}
copr_url = {{ koschei_copr_url }}

View file

@ -1,3 +0,0 @@
SHELL=/bin/bash
MAILTO=sysadmin-koschei-members@fedoraproject.org
0 3 * * * koschei koschei-admin cleanup --older-than 6 >/dev/null

View file

@ -1,15 +0,0 @@
SHELL=/bin/bash
MAILTO=sysadmin-koschei-members@fedoraproject.org
0 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-group php 'php*'
{% if env != 'stg' %}
5 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-distgit-group rust-sig rust-sig --append
7 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-distgit-group go-sig go-sig --append
9 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-distgit-group stewardship-sig stewardship-sig
15 0-23/3 * * * koschei /usr/local/bin/koschei-track-group rust-sig >/dev/null
17 0-23/3 * * * koschei /usr/local/bin/koschei-track-group go-sig >/dev/null
19 0-23/3 * * * koschei /usr/local/bin/koschei-track-group stewardship-sig >/dev/null
{%- endif %}
# I'd use dnf clean, but it leaves stuff behind
5 1 * * * koschei rm -rf /var/tmp/dnf-koschei-*

View file

@ -1,7 +0,0 @@
[koji]
server = https://{{ koschei_koji_hub }}/kojihub
topurl = https://{{ koschei_kojipkgs }}
weburl = https://{{ koschei_koji_web }}/koji
topdir = /mnt/koji
authtype = kerberos
krbservice = host

View file

@ -1,2 +0,0 @@
{{ koschei_pgsql_hostname }}:*:koschei:koscheiuser:{{ koscheiuser_db_password }}
{{ koschei_pgsql_hostname }}:*:koschei:koscheiadmin:{{ koscheiadmin_db_password }}

View file

@ -1,2 +0,0 @@
[Service]
Environment="KRB5_CLIENT_KTNAME=FILE:/etc/krb5.koschei_{{ inventory_hostname }}.keytab"

View file

@ -1,6 +0,0 @@
[Service]
{% if env == 'staging' %}
RestartSec=600
{% else %}
RestartSec=10
{% endif %}

View file

@ -1,17 +0,0 @@
---
- name: Add koschei copr dev repo on stg
when: env == "staging"
template: src=copr.repo.j2 dest=/etc/yum.repos.d/copr.repo
tags:
- koschei
- config
- packages
- yumrepos
- name: Install common packages
package: name={{ item }} state=present
with_items:
- python3-memcached
tags:
- koschei
- packages

View file

@ -1,8 +0,0 @@
[msimacek-koschei]
name=Copr repo for koschei owned by msimacek
baseurl=https://copr-be.cloud.fedoraproject.org/results/msimacek/koschei/fedora-$releasever-$basearch/
skip_if_unavailable=True
gpgcheck=1
gpgkey=https://copr-be.cloud.fedoraproject.org/results/msimacek/koschei/pubkey.gpg
enabled=0
enabled_metadata=1

View file

@ -1,44 +0,0 @@
# Copyright (C) 2018 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Author: Mikolaj Izdebski <mizdebsk@redhat.com>
import requests
from koschei.config import get_config
from koschei.plugin import listen_event
def get_packages_per_user(session):
session.log.debug("Requesting pagure_owner_alias.json")
req = requests.get(get_config('hotfix_pagure_extras.owner_alias_url'))
if not req.ok:
session.log.info("Failed to get pagure_owner_alias.json, status=%d",
req.status_code)
return {}
pkgs_per_user = {}
for pkg, users in req.json()['rpms'].items():
for user in users:
pkgs_per_user.setdefault(user, []).append(pkg)
return pkgs_per_user
@listen_event('get_user_packages')
def get_user_packages(session, username):
def create():
return get_packages_per_user(session)
pkg_map = session.cache('hotfix_pagure_extras').get_or_create('packages_per_user', create)
return pkg_map.get(str(username))

View file

@ -1,2 +0,0 @@
- name: reload httpd
action: service name=httpd state=reloaded

View file

@ -1,4 +0,0 @@
---
dependencies:
- koschei/common
- apache

View file

@ -1,68 +0,0 @@
- name: install packages
package: name={{ item }} state=present
with_items:
- koschei-frontend
- koschei-frontend-fedora
- koschei-frontend-copr
- mod_auth_openidc
tags:
- koschei
- packages
- name: install config files
template: >
src="{{ item }}.j2"
dest="/etc/koschei/{{ item }}"
owner=koschei
group=koschei
mode=0400
with_items:
- config-frontend.cfg
notify:
- reload httpd
tags:
- koschei
- config
- name: install httpd config file
template: >
src="httpd.conf.j2"
dest="/etc/httpd/conf.d/koschei.conf"
notify:
- reload httpd
tags:
- koschei
- config
- name: set httpd selinux booleans
seboolean: name={{ item }} state=yes persistent=yes
with_items:
- httpd_can_network_connect
- httpd_can_network_connect_db
- httpd_can_network_memcache
- httpd_execmem
notify:
- reload httpd
tags:
- koschei
- selinux
- name: enable httpd to start on boot
service: name=httpd state=started enabled=true
tags:
- koschei
- service
- name: create directory for hotfix_pagure_extras plugin
file: name=/usr/lib/python3.7/site-packages/koschei/plugins/hotfix_pagure_extras_plugin state=directory
tags:
- koschei
- hotfix
- name: install hotfix_pagure_extras plugin
copy: src=hotfix_pagure_extras_plugin.py dest=/usr/lib/python3.7/site-packages/koschei/plugins/hotfix_pagure_extras_plugin/frontend.py
notify:
- reload httpd
tags:
- koschei
- hotfix

View file

@ -1,105 +0,0 @@
# This is a config file for Koschei that can override values in default
# configuration in /usr/share/koschei/config.cfg. It is a python file expecting
# assignment to config dictionary which will be recursively merged with the
# default one.
config = {
"database_config": {
"host": "{{ koschei_pgsql_hostname }}",
"username": "koscheiuser",
"password": "{{ koscheiuser_db_password }}",
"database": "koschei"
},
"koji_config": {
"topurl": "https://{{ koschei_kojipkgs }}",
"weburl": "https://{{ koschei_koji_web }}/koji",
},
"flask": {
"SECRET_KEY": "{{ koschei_flask_secret_key }}",
},
"logging": {
"loggers": {
"": {
"level": "INFO",
# "handlers": ["stderr", "email"],
"handlers": ["stderr"],
},
"requests": {
"level": "WARN",
},
},
"handlers": {
"email": {
"class": "logging.handlers.SMTPHandler",
"level": "WARN",
"mailhost": "localhost",
"fromaddr": "koschei@fedoraproject.org",
"toaddrs": ['sysadmin-koschei-members@fedoraproject.org'],
"subject": "Koschei frontend {{ env }} warning",
},
},
},
{% if env == 'staging' %}
"plugins": ['hotfix_pagure_extras', 'copr'],
{% else %}
"plugins": ['hotfix_pagure_extras'],
{% endif %}
"caching": {
"pagure": {
"users": {
"backend": "dogpile.cache.memcached",
"expiration_time": 21600, # 6 hours
"arguments": {
"url": "memcached01:11211",
},
},
},
"hotfix_pagure_extras": {
"backend": "dogpile.cache.memcached",
"expiration_time": 3600, # 1 hour
"arguments": {
"url": "memcached01:11211",
},
},
},
"pagure": {
"api_url": "https://src{{ env_suffix }}.fedoraproject.org/api/0",
},
"hotfix_pagure_extras": {
"owner_alias_url": "https://src{{ env_suffix }}.fedoraproject.org/extras/pagure_owner_alias.json",
},
"frontend": {
"builds_per_page": 8,
"auth": {
"user_re": "(.+)",
"user_env": "OIDC_CLAIM_nickname",
},
"fedora_assets_url": "/global",
"fedmenu_url": "/fedmenu",
"fedmenu_data_url": "/js/data.js",
},
"links": [
{"name": "Packages",
"url": "https://apps{{ env_suffix }}.fedoraproject.org/packages/{package.name}"},
{"name": "Bodhi",
"url": "https://bodhi{{ env_suffix }}.fedoraproject.org/updates?packages={package.name}"},
{"name": "Dist-git",
"url": "https://src{{ env_suffix }}.fedoraproject.org/rpms/{package.name}"},
{"name": "Bugzilla",
"url": "https://{{ koschei_bugzilla }}/buglist.cgi?product={package.collection.bugzilla_product}&component={package.name}"},
{"name": "Koji",
"url": "https://{{ koschei_koji_web }}/koji/packageinfo?packageID={package.name}"},
],
"bugreport": {
"url": "https://{{ koschei_bugzilla }}/enter_bug.cgi?{query}",
},
"copr": {
"require_admin": True,
"copr_owner": "mizdebsk",
"default_schedule_count": 8,
},
}
# Local Variables:
# mode: Python
# End:
# vi: ft=python

View file

@ -1,34 +0,0 @@
<VirtualHost *>
ServerName {{ inventory_hostname }}
WSGIDaemonProcess koschei user=koschei group=koschei threads=5 home=/usr/share/koschei
WSGIScriptAlias /koschei /usr/share/koschei/koschei.wsgi
Alias /koschei/static /usr/share/koschei/static/
RewriteEngine on
RewriteRule ^/koschei/documentation/?$ https://fedoraproject.org/wiki/Koschei [R=301,L]
<Directory /usr/share/koschei>
WSGIProcessGroup koschei
WSGIApplicationGroup %{GLOBAL}
Options All
AllowOverride All
Require all granted
</Directory>
OIDCRedirectURI "{{ koschei_topurl }}/login/redirect_uri"
OIDCProviderMetadataURL "https://{{ koschei_oidc_provider }}/openidc/wellknown_openid_configuration"
OIDCClientID "koschei"
OIDCClientSecret "{{ koschei_oidc_client_secret }}"
OIDCCryptoPassphrase "{{ koschei_oidc_crypto_secret }}"
OIDCSSLValidateServer On
OIDCResponseType "code"
OIDCSessionType client-cookie
OIDCScope "openid profile"
<Location /koschei/login>
AuthType openid-connect
Require valid-user
</Location>
</VirtualHost>