riscv-koji secondary hub

Here's a pull request to setup a secondary riscv-koji hub.

There are still outstanding issues, but things should be
good enough to merge and deploy the initial instance and
interate from there. Also I plan to run a --check --diff to make sure
there's no changes on primary hubs/etc.

Signed-off-by: Kevin Fenzi <kevin@scrye.com>

inventory/group_vars/koji_riscv

@@ -0,0 +1,21 @@
+---
+# Define resources for this group of hosts here.
+custom_rules: [
+docker_registry: "candidate-registry.fedoraproject.org"
+ipa_client_shell_groups:
+  - sysadmin-riscv
+ipa_client_sudo_groups:
+  - sysadmin-riscv
+ipa_host_group: kojihub_riscv
+ipa_host_group_desc: riscv Koji Hub hosts
+koji_hub: "riscv-koji.fedoraproject.org/kojihub"
+koji_root: "riscv-koji.fedoraproject.org/koji"
+lvm_size: 100000
+mem_size: 32768
+max_mem_size: 65536
+nfs_mount_opts: "rw,hard,bg,intr,noatime,nodev,nosuid,sec=sys,nfsvers=4"
+num_cpus: 8
+primary_auth_source: ipa
+source_registry: "registry.fedoraproject.org"
+tcp_ports: [80, 443, 111, 2049]
+udp_ports: [111, 2049]

inventory/host_vars/koji01.iad2.fedoraproject.org

@@ -2,7 +2,7 @@
 datacenter: iad2
 eth0_ipv4_gw: 10.3.169.254
 eth0_ipv4_ip: 10.3.169.104
-fedmsg_koji_instance: primary
+koji_instance: primary
 koji_server_url: "https://koji.fedoraproject.org/kojihub"
 koji_topurl: "https://kojipkgs.fedoraproject.org/"
 koji_weburl: "https://koji.fedoraproject.org/koji"

inventory/host_vars/koji01.stg.iad2.fedoraproject.org

@@ -2,7 +2,7 @@
 datacenter: iad2
 eth0_ipv4_gw: 10.3.167.254
 eth0_ipv4_ip: 10.3.167.64
-fedmsg_koji_instance: primary
+koji_instance: primary
 koji_server_url: "https://koji.stg.fedoraproject.org/kojihub"
 koji_topurl: "https://kojipkgs.stg.fedoraproject.org/"
 koji_weburl: "https://koji.stg.fedoraproject.org/koji"

inventory/host_vars/koji02.iad2.fedoraproject.org

@@ -2,7 +2,7 @@
 datacenter: iad2
 eth0_ipv4_gw: 10.3.169.254
 eth0_ipv4_ip: 10.3.169.105
-fedmsg_koji_instance: primary
+koji_instance: primary
 koji_server_url: "https://koji.fedoraproject.org/kojihub"
 koji_topurl: "https://kojipkgs.fedoraproject.org/"
 koji_weburl: "https://koji.fedoraproject.org/koji"

inventory/host_vars/riscv-koji01.iad2.fedoraproject.org

@@ -0,0 +1,14 @@
+---
+datacenter: iad2
+eth0_ipv4_gw: 10.3.172.254
+eth0_ipv4_ip: 10.3.172.21
+koji_instance: secondary
+koji_server_url: "https://riscv-koji.fedoraproject.org/kojihub"
+koji_topurl: "https://riscv-koji.fedoraproject.org/"
+koji_weburl: "https://riscv-koji.fedoraproject.org/koji"
+ks_repo: https://infrastructure.fedoraproject.org/pub/fedora/linux/releases/41/Server/x86_64/os/
+ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-fedora
+nrpe_procs_crit: 1000
+nrpe_procs_warn: 900
+vmhost: bvmhost-x86-riscv01.iad2.fedoraproject.org
+volgroup: /dev/vg_guests

inventory/inventory

@@ -199,6 +199,9 @@ kernel02.iad2.fedoraproject.org
 koji01.iad2.fedoraproject.org
 koji02.iad2.fedoraproject.org
 
+[koji_riscv]
+riscv-koji01.iad2.fedoraproject.org
+
 [koji_stg]
 koji01.stg.iad2.fedoraproject.org
 

playbooks/groups/koji-hub.yml

@@ -36,15 +36,32 @@
   - role: keytab/service
     service: kojira
     host: "koji{{env_suffix}}.fedoraproject.org"
+    when: env != 'secondary'
+  - role: keytab/service
+    service: kojira
+    host: "riscv-koji{{env_suffix}}.fedoraproject.org"
+    when: env == 'secondary'
   - role: keytab/service
     service: koji-gc
     owner_user: apache
     host: "koji{{env_suffix}}.fedoraproject.org"
+    when: env != 'secondary'
+  - role: keytab/service
+    service: koji-gc
+    owner_user: apache
+    host: "riscv-koji{{env_suffix}}.fedoraproject.org"
+    when: env == 'secondary'
   - koji_hub
   - role: keytab/service
     service: HTTP
     owner_user: apache
     host: "koji{{env_suffix}}.fedoraproject.org"
+    when: env != 'secondary'
+  - role: keytab/service
+    service: HTTP
+    owner_user: apache
+    host: "riscv-koji{{env_suffix}}.fedoraproject.org"
+    when: env == 'secondary'
   - {role: nfs/server, when: env == "staging"}
 
     # production nfs mounts from netapp
@@ -92,13 +109,13 @@
     mnt_dir: '/mnt/koji/ostree'
     nfs_src_dir: 'fedora_ostree_content/ostree'
     mount_stg: true
-    when: env != 'staging'
+    when: env == 'production' and inventory_hostname.startswith('koji')
 
   - role: nfs/client
     mnt_dir: '/mnt/koji/compose/ostree'
     mount_stg: true
     nfs_src_dir: 'fedora_ostree_content/compose/ostree'
-    when: env != 'staging'
+    when: env == 'production' and inventory_hostname.startswith('koji')
 
     # In staging, we mount fedora_koji as read only (see nfs_mount_opts)
   - role: nfs/client
@@ -106,6 +123,12 @@
     nfs_src_dir: 'fedora_koji'
     when: env == 'staging' and inventory_hostname.startswith('koji')
 
+  - role: nfs/client
+    mnt_dir: '/mnt/fedora_koji/'
+    mount_stg: true
+    nfs_src_dir: 'fedora_riscv_koji'
+    when: inventory_hostname.startswith('riscv')
+
   - role: nfs/client
     mnt_dir: '/mnt/koji/ostree'
     nfs_src_dir: 'fedora_ostree_content/ostree'
@@ -122,6 +145,12 @@
   - role: rabbit/user
     user_name: "koji{{ env_suffix }}"
     user_sent_topics: ^org\.fedoraproject\.{{ env_short }}\.buildsys\..*
+    when: koji_instance != 'secondary'
+
+  - role: rabbit/user
+    user_name: "riscv-koji{{ env_suffix }}"
+    user_sent_topics: ^org\.fedoraproject\.{{ env_short }}\.buildsys\..*
+    when: koji_instance == 'secondary'
 
   tasks:
   - import_tasks: "{{ tasks_path }}/motd.yml"

playbooks/include/proxies-reverseproxy.yml

@@ -664,6 +664,16 @@
     http_not_https_yes_this_is_insecure_and_i_feel_bad: true
     when: env == "staging"
 
+  - role: httpd/reverseproxy
+    website: riscv-koji.fedoraproject.org
+    destname: koji
+    keephost: true
+    balancer_name: riscv-koji
+    balancer_members:
+    - "riscv-koji01.{{ datacenter }}.fedoraproject.org"
+    http_not_https_yes_this_is_insecure_and_i_feel_bad: true
+    when: koji_instance == "secondary"
+
   - role: httpd/reverseproxy
     website: kojipkgs.fedoraproject.org
     destname: kojipkgs

playbooks/include/proxies-websites.yml

@@ -1221,3 +1221,8 @@
     cert_name: "{{wildcard_cert_name}}"
     tags:
     - bugs
+
+  - role: httpd/website
+    site_name: riscv-koji.fedoraproject.org
+    sslonly: true
+    cert_name: "{{wildcard_cert_name}}"

roles/koji_hub/tasks/main.yml

@@ -124,8 +124,8 @@
   - koji_hub
   - fedora-messaging
 
-- name: Deploy koji/rabbitmq certificate
+- name: deploy koji/rabbitmq certificate (primary)
-  ansible.builtin.copy: src={{ item.src }}
+  copy: src={{ item.src }}
         dest=/etc/pki/rabbitmq/kojicert/{{ item.dest }}
         owner={{ item.owner }} group=root mode={{ item.mode }}
   with_items:
@@ -141,6 +141,30 @@
       dest: koji.ca
       owner: apache
       mode: "0644"
+  when: inventory_hostname.startswith('koji')
+  tags:
+  - config
+  - koji_hub
+  - fedora-messaging
+
+- name: deploy koji/rabbitmq certificate (secondary)
+  copy: src={{ item.src }}
+        dest=/etc/pki/rabbitmq/kojicert/{{ item.dest }}
+        owner={{ item.owner }} group=root mode={{ item.mode }}
+  with_items:
+    - src: "{{private}}/files/rabbitmq/{{env}}/pki/issued/riscv-koji{{ env_suffix }}.crt"
+      dest: koji.crt
+      owner: apache
+      mode: "0644"
+    - src: "{{private}}/files/rabbitmq/{{env}}/pki/private/riscv-koji{{ env_suffix }}.key"
+      dest: koji.key
+      owner: apache
+      mode: "600"
+    - src: "{{private}}/files/rabbitmq/{{env}}/pki/ca.crt"
+      dest: koji.ca
+      owner: apache
+      mode: "0644"
+  when: inventory_hostname.startswith('riscv-koji')
   tags:
   - config
   - koji_hub
@@ -203,8 +227,8 @@
 # install keytabs
 #
 
-- name: Install koji-hub keytab
+- name: install koji-hub keytab
-  ansible.builtin.copy: src={{ private }}/files/keytabs/{{ env }}/koji-hub-{{ fedmsg_koji_instance }} dest=/etc/koji-hub/koji-hub.keytab
+  copy: src={{ private }}/files/keytabs/{{ env }}/koji-hub-{{ koji_instance }} dest=/etc/koji-hub/koji-hub.keytab
         owner=apache group=apache mode=0600
   notify:
   - reload httpd
@@ -325,13 +349,19 @@
   - koji_hub
   when: env == "staging" and ansible_hostname.startswith('koji')
 
-- name: Make mnt/koji directory
+- name: make mnt/koji directory (primary)
   ansible.builtin.file: state=link src=/mnt/fedora_koji/koji dest=/mnt/koji owner=root group=root
   tags:
   - koji_hub
   when: ansible_hostname.startswith('koji')
 
-- name: Check selinux default context for /mnt/fedora_koji in staging
+- name: make mnt/koji directory (secondary)
+  file: state=link src=/mnt/fedora_koji/koji dest=/mnt/koji owner=root group=root
+  tags:
+  - koji_hub
+  when: ansible_hostname.startswith('riscv-koji')
+
+- name: check selinux default context for /mnt/fedora_koji in staging
   ansible.builtin.command: matchpathcon /mnt/fedora_koji
   register: mnt_fedora_koji_context
   when: env == "staging"
@@ -368,7 +398,7 @@
   - koji_hub
   when: ansible_distribution == "RedHat" and ansible_distribution_major_version|int == 7
 
-- name: Make httpd override directory
+- name: make httpd override directory
   ansible.builtin.file:
     state: directory
     path: /etc/systemd/system/httpd.service.d
@@ -421,6 +451,7 @@
   - koji-gc
   - koji-prune-signed-copies
   - koji-sidetag-cleanup
+  when: ansible_hostname.startswith('koji01')
   tags:
   - files
   - koji_hub

roles/koji_hub/templates/fedora-messaging.toml

@@ -13,8 +13,13 @@ topic_prefix = "org.fedoraproject.prod"
 
 [tls]
 ca_cert = "/etc/pki/rabbitmq/kojicert/koji.ca"
+{% if koji_instance == "secondary" %}
+keyfile = "/etc/pki/rabbitmq/kojicert/riscv-koji.key"
+certfile = "/etc/pki/rabbitmq/kojicert/riscv-koji.crt"
+{% else %}
 keyfile = "/etc/pki/rabbitmq/kojicert/koji.key"
 certfile = "/etc/pki/rabbitmq/kojicert/koji.crt"
+{% endif %}
 
 [client_properties]
 app = "Koji"

roles/koji_hub/templates/hub.conf.j2

@@ -8,15 +8,22 @@ DBHost = db-koji01
 LogLevel = koji._koji_plugin__koji-fedoramessaging:DEBUG
 LogFormat = %(asctime)s [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s %(name)s: %(message)s
 DBPass = {{ kojiStgPassword }}
-{% else %}
+AuthPrincipal = host/koji{{env_suffix}}.fedoraproject.org
+{% elif koji_instance = "primary" %}
 DBHost = db-koji01
 DBPass = {{ kojiPassword }}
-{% endif %}
 AuthPrincipal = host/koji{{env_suffix}}.fedoraproject.org
-{% if env == "staging" %}
-ProxyPrincipals = modularity@STG.FEDORAPROJECT.ORG,HTTP/koji.stg.fedoraproject.org@STG.FEDORAPROJECT.ORG,sigul/sign-bridge01.stg.iad2.fedoraproject.org@STG.FEDORAPROJECT.ORG
 {% else %}
+DBHost = db-riscv-koji01
+DBPass = {{ riscvkojiPassword }}
+AuthPrincipal = host/riscv-koji{{env_suffix}}.fedoraproject.org
+{% endif %}
+{% if env == "staging" %}
+ProxyPrincipals = HTTP/koji.stg.fedoraproject.org@STG.FEDORAPROJECT.ORG,sigul/sign-bridge01.stg.iad2.fedoraproject.org@STG.FEDORAPROJECT.ORG
+{% elif koji_instance = "primary" %}
 ProxyPrincipals = HTTP/koji.fedoraproject.org@FEDORAPROJECT.ORG,sigul/sign-bridge01.iad2.fedoraproject.org@FEDORAPROJECT.ORG
+{% else %}
+ProxyPrincipals = HTTP/riscv-koji.fedoraproject.org@FEDORAPROJECT.ORG,sigul/sign-bridge01.iad2.fedoraproject.org@FEDORAPROJECT.ORG
 {% endif %}
 KojiDir = /mnt/koji
 MemoryWarnThreshold = 10000
@@ -31,8 +38,10 @@ SeparateSourceTags = f{{ FedoraRawhideNumber }}-build eln-build
 # Kerb auth
 {% if env == "staging" %}
 HostPrincipalFormat = compile/%s@STG.FEDORAPROJECT.ORG
-{% else %}
+{% elif koji_instance = "primary" %}
 HostPrincipalFormat = compile/%s@FEDORAPROJECT.ORG
+{% else %}
+HostPrincipalFormat = compile-riscv/%s@FEDORAPROJECT.ORG
 {% endif %}
 AuthKeytab = /etc/koji-hub/koji-hub.keytab
 
@@ -50,7 +59,13 @@ ProxyDNs = emailAddress=buildsys@fedoraproject.org,CN=kojiweb,OU=Fedora Builders
 
 ##  Other options  ##
 LoginCreatesUser = On
-KojiWebURL = http://koji.fedoraproject.org/koji
+{% if env == "staging" %}
+KojiWebURL = https://koji.stg.fedoraproject.org/koji
+{% elif koji_instance = "primary" %}
+KojiWebURL = https://koji.fedoraproject.org/koji
+{% else %}
+KojiWebURL = https://riscv-koji.fedoraproject.org/koji
+{% endif %}
 # The domain name that will be appended to Koji usernames
 # when creating email notifications
 EmailDomain = fedoraproject.org
@@ -93,6 +108,8 @@ MissingPolicyOk = False
 #Plugins = darkserver-plugin
 {% if env == "staging" %}
 Plugins = osbuild koji-fedoramessaging runroot_hub hub_containerbuild tag2distrepo sidetag_hub save_failed_tree flatpak kiwi
+{% elif koji_instance = "primary" %}
+Plugins = osbuild koji-fedoramessaging runroot_hub hub_containerbuild tag2distrepo sidetag_hub save_failed_tree flatpak kiwi
 {% else %}
 Plugins = osbuild koji-fedoramessaging runroot_hub hub_containerbuild tag2distrepo sidetag_hub save_failed_tree flatpak kiwi
 {% endif %}
@@ -138,10 +155,6 @@ channel =
        method build chainbuild !! req
        has_perm customchannel :: req
     }
-{% if env == 'staging' %}
-    # kojid-cloud-scheduler tag setup for stg env only
-    tag buildaws && fromtag buildaws && method build :: use buildaws
-{% endif %}
 
 #we want pesign-test-app to always go to the secure-boot channel even for scratch builds
     source */pesign-test-app* && has_perm secure-boot :: use secure-boot

roles/koji_hub/templates/koji-gc.conf.j2

@@ -4,9 +4,15 @@
 [main]
 ; For Kerberos authentication
 ; the principal to connect with
+{% if koji_instance == "secondary" %}
+principal=koji-gc/riscv-koji{{env_suffix}}.fedoraproject.org@{{ipa_realm}}
+; The location of the keytab for the principal above
+keytab=/etc/krb5.koji-gc_riscv-koji{{env_suffix}}.fedoraproject.org.keytab
+{% else %}
 principal=koji-gc/koji{{env_suffix}}.fedoraproject.org@{{ipa_realm}}
 ; The location of the keytab for the principal above
 keytab=/etc/krb5.koji-gc_koji{{env_suffix}}.fedoraproject.org.keytab
+{% endif %}
 krb_rdns = False
 smtp_host = bastion.iad2.fedoraproject.org
 

roles/koji_hub/templates/kojira.conf.j2

@@ -5,9 +5,15 @@
 
 ; For Kerberos authentication
 ; the principal to connect with
+{% if koji_instance == "secondary" %}
+principal=kojira/riscv-koji{{env_suffix}}.fedoraproject.org@{{ipa_realm}}
+; The location of the keytab for the principal above
+keytab=/etc/krb5.kojira_riscv-koji{{env_suffix}}.fedoraproject.org.keytab
+{% else %}
 principal=kojira/koji{{env_suffix}}.fedoraproject.org@{{ipa_realm}}
 ; The location of the keytab for the principal above
 keytab=/etc/krb5.kojira_koji{{env_suffix}}.fedoraproject.org.keytab
+{% endif %}
 
 ; The URL for the koji hub server
 server={{ koji_server_url }}

roles/koji_hub/templates/kojiweb.conf.j2

@@ -35,7 +35,11 @@ WSGIDaemonProcess koji lang=C.UTF-8
     AuthType GSSAPI
     GssapiSSLonly Off
     AuthName "GSSAPI Single Sign On Login"
+{% if koji_instance == "secondary" %}
+    GssapiCredStore keytab:/etc/krb5.HTTP_riscv-koji{{env_suffix}}.fedoraproject.org.keytab
+{% else %}
     GssapiCredStore keytab:/etc/krb5.HTTP_koji{{env_suffix}}.fedoraproject.org.keytab
+{% endif %}
     Require valid-user
 </Location>
 

roles/koji_hub/templates/web.conf.j2

@@ -6,9 +6,12 @@ SiteName = koji
 {% if env == 'staging' %}
 KojiHubURL = https://koji.stg.fedoraproject.org/kojihub
 KojiFilesURL = https://kojipkgs.stg.fedoraproject.org/
-{% else %}
+{% elif env == 'production'  %}
 KojiHubURL = https://koji.fedoraproject.org/kojihub
 KojiFilesURL = https://kojipkgs.fedoraproject.org/
+{% else %}
+KojiHubURL = https://riscv-koji.fedoraproject.org/kojihub
+KojiFilesURL = https://riscv-kojipkgs.fedoraproject.org/
 {% endif %}
 
 # SSL authentication options
@@ -21,8 +24,10 @@ LoginTimeout = 72
 # This must be changed and uncommented before deployment
 {% if env == 'staging' %}
 Secret = {{ kojiSecret }}
-{% else %}
+{% elif env == 'production'  %}
 Secret = {{ kojiStgSecret }}
+{% else %}
+Secret = {{ riscvkojiSecret }}
 {% endif %}
 
 LibPath = /usr/share/koji-web/lib
@@ -34,6 +39,9 @@ KojiHubCA = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
 {% if env == 'staging' %}
 Tasks = buildContainer,createContainer,flatpakBuild,flatpakBuildArch,runroot,osbuildImage,createKiwiImage,kiwiBuild
 ParentTasks = buildContainer,flatpakBuild,osbuildImage,kiwiBuild
+{% elif env == 'production'  %}
+Tasks = buildContainer,createContainer,flatpakBuild,flatpakBuildArch,runroot,osbuildImage,createKiwiImage,kiwiBuild
+ParentTasks = buildContainer,flatpakBuild,osbuildImage,kiwiBuild
 {% else %}
 Tasks = buildContainer,createContainer,flatpakBuild,flatpakBuildArch,runroot,osbuildImage,createKiwiImage,kiwiBuild
 ParentTasks = buildContainer,flatpakBuild,osbuildImage,kiwiBuild