Add Koschei stg/prod hosts

- koschei-specific setup tasks are moved to a new koschei role
- cloud dev instance is made to use koschei role
- stg and prod instances are added to inventory
- koschei host group is introduced

files/koschei/koschei.repo

@@ -1,13 +0,0 @@
-[koschei-mizdebsk]
-name=Koschei repo
-baseurl=https://mizdebsk.fedorapeople.org/koschei/repo/
-enabled=1
-gpgcheck=0
-metadata_expire=60
-
-[koschei-msimacek]
-name=Koschei repo
-baseurl=https://msimacek.fedorapeople.org/koschei/repo/
-enabled=1
-gpgcheck=0
-metadata_expire=60

handlers/restart_services.yml

@@ -47,6 +47,18 @@
 - name: restart kojid
   action: service name=kojid state=restarted
 
+- name: restart koschei-polling
+  action: service name=koschei-polling state=restarted
+
+- name: restart koschei-resolver
+  action: service name=koschei-resolver state=restarted
+
+- name: restart koschei-scheduler
+  action: service name=koschei-scheduler state=restarted
+
+- name: restart koschei-watcher
+  action: service name=koschei-watcher state=restarted
+
 - name: restart libvirtd
   action: service name=libvirtd state=restarted
 

inventory/group_vars/kojipkgs

@@ -29,6 +29,7 @@ csi_relationship: |
 
     - Things that rely on this host:
       - all koji builders/buildsystem
+      - koschei
       - external users downloading packages from koji. 
 
 # Need a eth0/eth1 install here. 

inventory/group_vars/koschei

@@ -0,0 +1,40 @@
+---
+# Define resources for this group of hosts here.
+lvm_size: 20000
+mem_size: 4096
+num_cpus: 4
+
+# for systems that do not match the above - specify the same parameter in
+# the host_vars/$hostname file
+
+tcp_ports: [ 80, 443 ]
+
+custom_rules: [
+    # Need for rsync from log01 for logs.
+    '-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT',
+    '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT',
+ ]
+
+fas_client_groups: sysadmin-koschei,fi-apprentice
+
+freezes: false
+
+# These are consumed by a task in roles/fedmsg/base/main.yml
+fedmsg_certs:
+- service: shell
+  owner: root
+  group: root
+- service: koschei
+  owner: root
+  group: koschei
+
+# For the MOTD
+csi_security_category: Low
+csi_primary_contact: Fedora admins - admin@fedoraproject.org
+csi_purpose: Koschei continuous integration system
+csi_relationship: |
+    This machine depends on:
+    - PostgreSQL DB server
+    - Koji hub and kojipkgs
+    - fedmsg hub and relay
+    - bastion (for mail relay)

inventory/host_vars/koschei01.stg.phx2.fedoraproject.org

@@ -0,0 +1,21 @@
+---
+nm: 255.255.255.0
+gw: ???
+dns: ???
+
+ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-7
+ks_repo: http://10.5.126.23/repo/rhel/RHEL7-x86_64/
+
+volgroup: ???
+eth0_ip: ???
+vmhost: ???
+datacenter: phx2
+
+# stg needs less resources than prod
+mem_size: 2048
+num_cpus: 2
+
+koschei_pgsql_hostname: db01.stg.phx2.fedoraproject.org
+koschei_koji_hub: koji01.stg.phx2.fedoraproject.org
+koschei_kojipkgs: koji01.stg.phx2.fedoraproject.org
+koschei_koji_tag: f23

inventory/inventory

@@ -292,6 +292,12 @@ koji01.stg.phx2.fedoraproject.org
 [kojipkgs]
 kojipkgs01.phx2.fedoraproject.org
 
+[koschei]
+#koschei01.phx2.fedoraproject.org
+
+[koschei-stg]
+koschei01.stg.phx2.fedoraproject.org
+
 [infracore]
 lockbox01.phx2.fedoraproject.org
 log01.phx2.fedoraproject.org

master.yml

@@ -51,6 +51,7 @@
 - include: /srv/web/infra/ansible/playbooks/groups/keyserver.yml
 - include: /srv/web/infra/ansible/playbooks/groups/koji-hub.yml
 - include: /srv/web/infra/ansible/playbooks/groups/kojipkgs.yml
+- include: /srv/web/infra/ansible/playbooks/groups/koschei.yml
 - include: /srv/web/infra/ansible/playbooks/groups/lockbox.yml
 - include: /srv/web/infra/ansible/playbooks/groups/mailman.yml
 - include: /srv/web/infra/ansible/playbooks/groups/mirrorlist2.yml

playbooks/groups/koschei.yml

@@ -0,0 +1,50 @@
+- name: make koschei
+  hosts: koschei:koschei-stg
+  user: root
+  gather_facts: False
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  tasks:
+  - include: "{{ tasks }}/virt_instance_create.yml"
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+- name: install koschei
+  hosts: koschei:koschei-stg
+  user: root
+  gather_facts: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  pre_tasks:
+  - include: "{{ tasks }}/apache.yml"
+
+  roles:
+  - base
+  - rkhunter
+  - nagios_client
+  - hosts
+  - fas_client
+  - builder_repo
+  - collectd/base
+  - koschei
+  - fedmsg/base
+  - sudo
+
+  tasks:
+  - include: "{{ tasks }}/yumrepos.yml"
+  - include: "{{ tasks }}/2fa_client.yml"
+  - include: "{{ tasks }}/motd.yml"
+  - include: "{{ tasks }}/apache.yml"
+  - include: "{{ tasks }}/mod_wsgi.yml"
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"

playbooks/hosts/koschei.cloud.fedoraproject.org.yml

@@ -22,94 +22,17 @@
    - "/srv/private/ansible/vars.yml"
    - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
 
-  vars:
+  roles:
-    packages:
+  - koschei
-    - koschei
-    services:
-    - koschei-polling
-    - koschei-resolver
-    - koschei-scheduler
-    - koschei-watcher
-    # httpd is here temporarly only, it will be removed once koschei
-    # implements "base" role
-    - httpd
-    # flag controlling whether koji PEM private key and certificate
-    # should be deployed by playbook
-    cert: false
 
   tasks:
-
   - include: "{{ tasks }}/growroot_cloud.yml"
   - include: "{{ tasks }}/cloud_setup_basic.yml"
   - include: "{{ tasks }}/postfix_basic.yml"
 
-  # Temporary yum repo hosted on fedorapeople, it will be replaced by
-  # Fedora infra repo once Koschei completes RFR.  Copr can't be used
-  # because of limitations of Fedora cloud routing -- machines in
-  # different networks can't access each other, even through public IP
-  - name: add koschei yum repo
-    action: copy src="{{ files }}/koschei/koschei.repo" dest="/etc/yum.repos.d/koschei.repo"
-
-  - name: yum update koschei package
-    yum: name={{item}} state=latest
-    with_items: "{{packages}}"
-    register: yumupdate
-    # TODO: restart httpd
-    tags:
-    - packages
-
-  - name: stop koschei
-    action: service name={{item}} state=stopped
-    with_items: "{{services}}"
-    when: yumupdate.changed
-
-  - name: install /etc/koschei/config.cfg file
-    template: src="{{ files }}/koschei/config.cfg.j2" dest="/etc/koschei/config.cfg"
-    notify:
-    - restart koschei
-    # TODO: restart httpd
-    tags:
-    - config
-
-  - name: install koschei.pem koji key and cert
-    copy: >
-      src="{{ private }}/files/koschei/koschei.pem"
-      dest="/etc/koschei/koschei.pem"
-      owner=koschei
-      group=koschei
-      mode=0400
-    when: cert
-    tags:
-    - config
-
-  - name: install koji ca cert
-    copy: >
-      src="{{ puppet_private }}/fedora-ca.cert"
-      dest="/etc/koschei/fedora-ca.cert"
-      owner=root
-      group=root
-      mode=0644
-    tags:
-    - config
-
-  - name: run koschei migration
-    command: alembic -c /usr/share/koschei/alembic.ini upgrade head
-    sudo_user: koschei
-    when: yumupdate.changed
-
-  - name: enable koschei to start
-    action: service name={{item}} state=running enabled=true
-    with_items: "{{services}}"
-    tags:
-    - service
-
   handlers:
   - include: "{{ handlers }}/restart_services.yml"
 
-  - name: restart koschei
-    action: service name={{item}} state=restarted
-    with_items: "{{services}}"
-
 - name: setup fedmsg
   hosts: koschei.cloud.fedoraproject.org
   user: root

roles/koschei/tasks/main.yml

@@ -0,0 +1,62 @@
+- name: install packages
+  yum: name={{ item }} state=installed
+  with_items:
+  - koschei
+  tags:
+  - koschei
+  - packages
+
+- name: install config.cfg
+  template: >
+    src="config.cfg.j2"
+    dest="/etc/koschei/config.cfg"
+  notify:
+  - restart koschei-polling
+  - restart koschei-resolver
+  - restart koschei-scheduler
+  - restart koschei-watcher
+  - restart httpd
+  tags:
+  - koschei
+  - config
+
+- name: install koji key and cert
+  copy: >
+    src="{{ private }}/files/koschei/koschei.pem"
+    dest="/etc/koschei/koschei.pem"
+    owner=koschei
+    group=koschei
+    mode=0400
+  notify:
+  - restart koschei-scheduler
+  tags:
+  - koschei
+  - config
+
+- name: install koji ca cert
+  copy: >
+    src="{{ puppet_private }}/fedora-ca.cert"
+    dest="/etc/koschei/fedora-ca.cert"
+    owner=root
+    group=root
+    mode=0644
+  notify:
+  - restart koschei-polling
+  - restart koschei-resolver
+  - restart koschei-scheduler
+  - restart koschei-watcher
+  - restart httpd
+  tags:
+  - koschei
+  - config
+
+- name: enable services to start on boot
+  action: service name={{ item }} state=running enabled=true
+  with_items:
+  - koschei-polling
+  - koschei-resolver
+  - koschei-scheduler
+  - koschei-watcher
+  tags:
+  - koschei
+  - service

roles/koschei/templates/config.cfg.j2

@@ -4,14 +4,45 @@
 # default one.
 config = {
     "database_config": {
-        "username": "koschei",
+        "hostname": "{{ koschei_pgsql_hostname }}",
+        "username": "koscheiuser",
         "password": "{{ koschei_pgsql_password }}",
         "database": "koschei"
     },
     "koji_config": {
+        "server": "http://{{ koschei_koji_hub }}/kojihub",
+        "topurl": "http://{{ koschei_kojipkgs }}",
+        "weburl": "http://{{ koschei_koji_hub }}/koji",
         "cert": "/etc/koschei/koschei.pem",
         "ca": "/etc/koschei/fedora-ca.cert",
         "server_ca": "/etc/koschei/fedora-ca.cert",
+        "source_tag": "{{ koschei_koji_tag }}",
+        "target_tag": "{{ koschei_koji_tag }}",
+        "build_tag": "{{ koschei_koji_tag }}-build",
+        "max_builds": 30,
+        "build_arches": ['i386', 'x86_64', 'armhfp'],
+        "build_opts": {
+        },
+        "load_threshold": 0.5,
+        "task_priority": 30,
+    },
+    "dependency": {
+        "build_group": "build",
+        "for_arch": "x86_64",
+        "repo_cache_items": 10,
+        "keep_build_deps_for": 5,
+        "repos": {
+            "x86_64": "http://{{ koschei_kojipkgs }}/repos/{{ koschei_koji_tag }}-build/{repo_id}/x86_64",
+            "i386": "http://{{ koschei_kojipkgs }}/repos/{{ koschei_koji_tag }}-build/{repo_id}/i386",
+        },
+    },
+    "services": {
+        "watcher": {
+            "watchdog_interval": 600, # in seconds
+        },
+        "polling": {
+            "interval": 20 * 60,
+        },
     },
     "flask": {
         "SECRET_KEY": "{{ koschei_flask_secret_key }}",
@@ -29,7 +60,7 @@ config = {
                 "level": "WARN",
                 "mailhost": "localhost",
                 "fromaddr": "koschei@fedoraproject.org",
-                "toaddrs": ['msimacek@redhat.com', 'mizdebsk@redhat.com'],
+                "toaddrs": ['sysadmin-koschei-members@fedoraproject.org'],
                 "subject": "Koschei warning",
             },
         },
@@ -38,18 +69,6 @@ config = {
         "enabled": True,
         "modname": "koschei",
     },
-#    "services": {
-#        "polling": {
-#            "interval": 60,
-#        },
-#    },
-    "dependency": {
-        "repo_chache_items": 5,
-        "keep_build_deps_for": 2
-    },
-    "koji_config": {
-        "max_builds": 30
-    },
 }
 
 # Local Variables: