diff --git a/inventory/group_vars/osbs-nodes b/inventory/group_vars/osbs-nodes new file mode 100644 index 0000000000..d337069253 --- /dev/null +++ b/inventory/group_vars/osbs-nodes @@ -0,0 +1,21 @@ +--- +# Define resources for this group of hosts here. +lvm_size: 60000 +mem_size: 8192 +num_cpus: 2 + +tcp_ports: [ 80, 443, 8443] + +fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran +sudoers: "{{ private }}/files/sudo/00releng-sudoers" + +docker_cert_dir: "/etc/docker/certs.d/candidate-registry.fedoraproject.org" +docker_registry: "candidate-registry.fedoraproject.org" +source_registry: "registry.fedoraproject.org" + +osbs_url: "osbs.fedoraproject.org" +osbs_koji_username: "kojibuilder" + +koji_url: "koji.fedoraproject.org" + +osbs_client_conf_path: /etc/osbs.conf diff --git a/inventory/group_vars/osbs-nodes-stg b/inventory/group_vars/osbs-nodes-stg new file mode 100644 index 0000000000..6bcadb7bce --- /dev/null +++ b/inventory/group_vars/osbs-nodes-stg @@ -0,0 +1,21 @@ +--- +# Define resources for this group of hosts here. +lvm_size: 60000 +mem_size: 8192 +num_cpus: 2 + +tcp_ports: [ 80, 443, 8443] + +fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran +sudoers: "{{ private }}/files/sudo/00releng-sudoers" + +docker_cert_dir: "/etc/docker/certs.d/candidate-registry.stg.fedoraproject.org" +source_registry: "registry.stg.fedoraproject.org" +docker_registry: "candidate-registry.stg.fedoraproject.org" + +osbs_url: "osbs.stg.fedoraproject.org" +osbs_koji_username: "kojibuilder_stg" + +koji_url: "koji.stg.fedoraproject.org" + +osbs_client_conf_path: /etc/osbs.conf diff --git a/inventory/host_vars/osbs-node01.phx2.fedoraproject.org b/inventory/host_vars/osbs-node01.phx2.fedoraproject.org new file mode 100644 index 0000000000..ee1fb0c6c3 --- /dev/null +++ b/inventory/host_vars/osbs-node01.phx2.fedoraproject.org @@ -0,0 +1,18 @@ +--- +nm: 255.255.255.0 +gw: 10.5.125.254 +dns: 10.5.126.21 +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-24-osbs +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/24/Server/x86_64/os/ +volgroup: /dev/vg_guests +eth0_ip: 10.5.125.53 +vmhost: bvirthost01.phx2.fedoraproject.org +datacenter: phx2 + +nrpe_procs_warn: 900 +nrpe_procs_crit: 1000 + +lvm_size: 120g +mem_size: 16384 +max_mem_size: 16384 +num_cpus: 4 diff --git a/inventory/host_vars/osbs-node01.stg.phx2.fedoraproject.org b/inventory/host_vars/osbs-node01.stg.phx2.fedoraproject.org new file mode 100644 index 0000000000..1292301bcd --- /dev/null +++ b/inventory/host_vars/osbs-node01.stg.phx2.fedoraproject.org @@ -0,0 +1,19 @@ +--- +nm: 255.255.255.0 +gw: 10.5.126.254 +dns: 10.5.126.21 +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-24-osbs +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/24/Server/x86_64/os/ +volgroup: /dev/vg_guests +eth0_ip: 10.5.126.219 +vmhost: virthost20.phx2.fedoraproject.org +datacenter: phx2 +host_group: osbs-nodes-stg + +nrpe_procs_warn: 900 +nrpe_procs_crit: 1000 + +lvm_size: 120g +mem_size: 8192 +max_mem_size: 16384 +num_cpus: 4 diff --git a/inventory/host_vars/osbs-node02.phx2.fedoraproject.org b/inventory/host_vars/osbs-node02.phx2.fedoraproject.org new file mode 100644 index 0000000000..5487458bf8 --- /dev/null +++ b/inventory/host_vars/osbs-node02.phx2.fedoraproject.org @@ -0,0 +1,18 @@ +--- +nm: 255.255.255.0 +gw: 10.5.125.254 +dns: 10.5.126.21 +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-24-osbs +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/24/Server/x86_64/os/ +volgroup: /dev/vg_guests +eth0_ip: 10.5.125.54 +vmhost: bvirthost01.phx2.fedoraproject.org +datacenter: phx2 + +nrpe_procs_warn: 900 +nrpe_procs_crit: 1000 + +lvm_size: 120g +mem_size: 16384 +max_mem_size: 16384 +num_cpus: 4 diff --git a/inventory/inventory b/inventory/inventory index 3aecf220e1..2a0687aac1 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -662,6 +662,7 @@ mm-crawler01.stg.phx2.fedoraproject.org beaker-stg01.qa.fedoraproject.org zanata2fedmsg01.stg.phx2.fedoraproject.org osbs-master01.stg.phx2.fedoraproject.org +osbs-node01.stg.phx2.fedoraproject.org docker-registry01.stg.phx2.fedoraproject.org docker-candidate-registry01.stg.phx2.fedoraproject.org @@ -1168,9 +1169,17 @@ taskotron01.qa.fedoraproject.org [osbs] osbs-master01.phx2.fedoraproject.org +[osbs-nodes] +osbs-node01.phx2.fedoraproject.org +osbs-node02.phx2.fedoraproject.org + [osbs-stg] osbs-master01.stg.phx2.fedoraproject.org + +[osbs-nodes-stg] +osbs-node01.stg.phx2.fedoraproject.org + [docker-registry] docker-registry01.phx2.fedoraproject.org docker-candidate-registry01.phx2.fedoraproject.org diff --git a/playbooks/groups/osbs-node.yml b/playbooks/groups/osbs-node.yml new file mode 100644 index 0000000000..59639bc8ca --- /dev/null +++ b/playbooks/groups/osbs-node.yml @@ -0,0 +1,423 @@ +# create an osbs server +- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=osbs-nodes:osbs-nodes-stg" + +- name: make the box be real + hosts: osbs-nodes:osbs-nodes-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - nagios/client + - hosts + - fas_client + - collectd/base + - rsyncd + - sudo + - { role: openvpn/client, + when: env != "staging" } + + tasks: + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +# - name: pre-install osbs tasks +# hosts: osbs-nodes:osbs-nodes-stg +# vars_files: +# - /srv/web/infra/ansible/vars/global.yml +# - /srv/private/ansible/vars.yml +# - /srv/private/ansible/files/openstack/passwords.yml +# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + +# tasks: +# - name: copy docker-storage-setup config +# copy: +# src: "{{files}}/osbs/docker-storage-setup" +# dest: "/etc/sysconfig/docker-storage-setup" + +# - name: create cert dir for openshift public facing REST API SSL +# file: +# path: "/etc/origin/master/named_certificates" +# state: "directory" + +# - name: install cert for openshift public facing REST API SSL +# copy: +# src: "{{private}}/files/osbs/{{env}}/osbs-internal.pem" +# dest: "/etc/origin/master/named_certificates/{{osbs_url}}.pem" + +# - name: install key for openshift public facing REST API SSL +# copy: +# src: "{{private}}/files/osbs/{{env}}/osbs-internal.key" +# dest: "/etc/origin/master/named_certificates/{{osbs_url}}.key" + +# - name: ensure origin conf dir exists +# file: +# path: "/etc/origin" +# state: "directory" + +# - name: place htpasswd file +# copy: +# src: "{{private}}/files/httpd/osbs-{{env}}.htpasswd" +# dest: /etc/origin/htpasswd + +# roles: +# - { +# role: push-docker, +# docker_cert_name: "containerbuild", +# docker_cert_dir: "/etc/docker/certs.d/candidate-registry.stg.fedoraproject.org", +# when: env == "staging" +# } +# - { +# role: push-docker, +# docker_cert_name: "containerbuild", +# docker_cert_dir: "/etc/docker/certs.d/candidate-registry.fedoraproject.org", +# when: env == "production" +# } + +# - name: setup osbs +# hosts: osbs-nodes:osbs-nodes-stg +# vars_files: +# - /srv/web/infra/ansible/vars/global.yml +# - /srv/private/ansible/vars.yml +# - /srv/private/ansible/files/openstack/passwords.yml +# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + +# roles: +# - osbs-atomic-reactor +# - { +# role: osbs-common, +# osbs_manage_firewalld: false, +# } +# - osbs-install-openshift +# - { +# role: osbs-master, +# osbs_openshift_loglevel: 2, +# osbs_master_export_port: true, +# osbs_manage_firewalld: false, +# osbs_proxy_cert_file: '/etc/origin/proxy_selfsigned.crt', +# osbs_proxy_key_file: '/etc/origin/proxy_selfsigned.key', +# osbs_proxy_certkey_file: '/etc/origin/proxy_certkey.crt', +# osbs_proxy_ca_file: '/etc/origin/proxy_selfsigned.crt', +# osbs_readonly_users: [], +# osbs_readonly_groups: [], +# osbs_readwrite_users: [ "{{ osbs_koji_stg_username }}" ], +# osbs_readwrite_groups: [], +# osbs_admin_users: [], +# osbs_admin_groups: [], +# osbs_master_max_pods: 3, +# osbs_update_packages: false, +# osbs_image_gc_high_threshold: 90, +# osbs_image_gc_low_threshold: 80, +# osbs_identity_provider: "htpasswd_provider", +# osbs_identity_htpasswd: { +# name: htpasswd_provider, +# challenge: true, +# login: true, +# provider_file: "/etc/origin/htpasswd" +# }, +# osbs_named_certificates: { +# enabled: true, +# cert_file: "named_certificates/{{osbs_url}}.pem", +# key_file: "named_certificates/{{osbs_url}}.key", +# names: [ "{{osbs_url}}" ], +# }, +# osbs_public_api_url: "{{osbs_url}}", +# when: env == "staging" +# } +# - { +# role: osbs-master, +# osbs_openshift_loglevel: 2, +# osbs_master_export_port: true, +# osbs_manage_firewalld: false, +# osbs_proxy_cert_file: '/etc/origin/proxy_selfsigned.crt', +# osbs_proxy_key_file: '/etc/origin/proxy_selfsigned.key', +# osbs_proxy_certkey_file: '/etc/origin/proxy_certkey.crt', +# osbs_proxy_ca_file: '/etc/origin/proxy_selfsigned.crt', +# osbs_readonly_users: [], +# osbs_readonly_groups: [], +# osbs_readwrite_users: [ "{{ osbs_koji_prod_username }}" ], +# osbs_readwrite_groups: [], +# osbs_admin_users: [], +# osbs_admin_groups: [], +# osbs_master_max_pods: 3, +# osbs_update_packages: false, +# osbs_image_gc_high_threshold: 90, +# osbs_image_gc_low_threshold: 80, +# osbs_identity_provider: "htpasswd_provider", +# osbs_identity_htpasswd: { +# name: htpasswd_provider, +# challenge: true, +# login: true, +# provider_file: "/etc/origin/htpasswd" +# }, +# osbs_named_certificates: { +# enabled: true, +# cert_file: "named_certificates/{{osbs_url}}.pem", +# key_file: "named_certificates/{{osbs_url}}.key", +# names: [ "{{osbs_url}}" ], +# }, +# osbs_public_api_url: "{{osbs_url}}", +# when: env == "production" +# } + +# - { +# role: osbs-client, +# general: { +# verbose: 0, +# build_json_dir: '/usr/share/osbs/', +# openshift_required_version: 1.1.0, +# }, +# default: { +# username: "{{ osbs_koji_stg_username }}", +# password: "{{ osbs_koji_stg_password }}", +# koji_certs_secret: "koji", +# openshift_url: 'https://{{osbs_url}}/', +# registry_uri: 'https://{{docker_registry}}/v2', +# source_registry_uri: 'https://{{source_registry}}/v2', +# build_host: '{{osbs_url}}', +# koji_root: 'https://{{koji_url}}/koji', +# koji_hub: 'https://{{koji_url}}/kojihub', +# sources_command: 'fedpkg sources', +# build_type: 'prod', +# authoritative_registry: 'registry.example.com', +# vendor: 'Fedora Project', +# verify_ssl: true, +# use_auth: true, +# builder_use_auth: true, +# distribution_scope: 'private', +# registry_api_versions: 'v2', +# builder_openshift_url: 'https://172.17.0.1:8443/' +# }, +# when: env == "staging" +# } +# - { +# role: osbs-client, +# general: { +# verbose: 0, +# build_json_dir: '/usr/share/osbs/', +# openshift_required_version: 1.1.0, +# }, +# default: { +# username: "{{ osbs_koji_prod_username }}", +# password: "{{ osbs_koji_prod_password }}", +# koji_certs_secret: "koji", +# openshift_url: 'https://{{osbs_url}}/', +# registry_uri: 'https://{{docker_registry}}/v2', +# source_registry_uri: 'https://{{source_registry}}/v2', +# build_host: '{{osbs_url}}', +# koji_root: 'https://{{koji_url}}/koji', +# koji_hub: 'https://{{koji_url}}/kojihub', +# sources_command: 'fedpkg sources', +# build_type: 'prod', +# authoritative_registry: 'registry.example.com', +# vendor: 'Fedora Project', +# verify_ssl: true, +# use_auth: true, +# builder_use_auth: true, +# distribution_scope: 'private', +# registry_api_versions: 'v2', +# builder_openshift_url: 'https://172.17.0.1:8443/' +# }, +# when: env == "production" +# } + +# - name: post-install osbs tasks +# hosts: osbs-nodes:osbs-nodes-stg +# vars_files: +# - /srv/web/infra/ansible/vars/global.yml +# - /srv/private/ansible/vars.yml +# - /srv/private/ansible/files/openstack/passwords.yml +# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml +# vars: +# osbs_kubeconfig_path: /etc/origin/master/admin.kubeconfig +# osbs_environment: +# KUBECONFIG: "{{ osbs_kubeconfig_path }}" +# koji_pki_dir: /etc/pki/koji +# koji_ca_cert_path: "{{koji_pki_dir}}/fedora-server-ca.cert" +# koji_cert_path: "{{koji_pki_dir}}/fedora-builder.pem" +# koji_builder_user: dockerbuilder +# osbs_builder_user: builder + + +# handlers: +# - name: buildroot container +# shell: 'docker build --no-cache --rm -t buildroot /etc/osbs/buildroot/' + +# - name: oc secrets new +# shell: "oc secrets new koji cert={{ koji_cert_path }} ca={{ koji_ca_cert_path }} serverca={{ koji_ca_cert_path }}" +# environment: "{{ osbs_environment }}" +# notify: oc secrets add + +# - name: oc secrets add +# shell: "oc secrets add serviceaccount/{{ osbs_builder_user }} secrets/koji --for=mount" +# environment: "{{ osbs_environment }}" + + +# tasks: +# - name: set nrpe read access for osbs.conf for nagios monitoring +# acl: name={{ osbs_client_conf_path }} entity=nrpe etype=user permissions=r state=present + +# - name: pull fedora required docker images +# shell: "docker pull {{item}}" +# with_items: "{{fedora_required_images}}" +# delegate_to: compose-x86-01.phx2.fedoraproject.org +# register: docker_pull_fedora_delegated +# changed_when: "'Downloaded newer image' in docker_pull_fedora_delegated.stdout" + +# - name: tag fedora required docker images for our registry +# shell: "docker tag {{item}} {{docker_registry}}/{{item}}" +# with_items: "{{fedora_required_images}}" +# delegate_to: compose-x86-01.phx2.fedoraproject.org +# when: docker_pull_fedora_delegated|changed + +# - name: push fedora required docker images to our registry +# shell: "docker push {{docker_registry}}/{{item}}" +# with_items: "{{fedora_required_images}}" +# delegate_to: compose-x86-01.phx2.fedoraproject.org +# when: docker_pull_fedora_delegated|changed + +# - name: register origin_version_out rpm query +# shell: "rpm -q origin --qf '%{Version}'" +# register: origin_version_out +# always_run: true +# changed_when: False + +# - set_fact: +# origin_version: "{{origin_version_out.stdout}}" + +# - name: pull openshift required docker images +# shell: "docker pull {{item}}:v{{origin_version}}" +# with_items: "{{openshift_required_images}}" +# delegate_to: compose-x86-01.phx2.fedoraproject.org +# register: docker_pull_openshift_delegated +# changed_when: "'Downloaded newer image' in docker_pull_openshift_delegated.stdout" + +# - name: tag openshift required docker images for our registry +# shell: "docker tag {{item}}:v{{origin_version}} {{docker_registry}}/{{item}}:v{{origin_version}}" +# with_items: "{{openshift_required_images}}" +# delegate_to: compose-x86-01.phx2.fedoraproject.org +# when: docker_pull_openshift_delegated|changed + +# - name: push openshift required docker images to our registry +# shell: "docker push {{docker_registry}}/{{item}}:v{{origin_version}}" +# with_items: "{{openshift_required_images}}" +# delegate_to: compose-x86-01.phx2.fedoraproject.org +# when: docker_pull_openshift_delegated|changed + +# - name: Ensure koji dockerbuilder cert path exists +# file: +# path: "{{ koji_pki_dir }}" +# state: "directory" +# mode: 0400 + +# - name: Add koji dockerbuilder cert for Content Generator import +# copy: +# src: "{{private}}/files/koji/containerbuild.pem" +# dest: "{{ koji_cert_path }}" +# notify: oc secrets new + +# - name: Add koji dockerbuilder ca cert for Content Generator import +# copy: +# src: "{{private}}/files/koji/buildercerts/fedora-ca.cert" +# dest: "{{ koji_ca_cert_path }}" +# notify: oc secrets new + +# - name: create fedora image stream for OpenShift +# shell: "echo '{ \"apiVersion\": \"v1\", \"kind\": \"ImageStream\", \"metadata\": { \"name\": \"fedora\" }, \"spec\": { \"dockerImageRepository\": \"{{docker_registry}}/fedora\" } }' | oc create -f - && touch /etc/origin/fedoraimagestreamcreated" +# environment: "{{ osbs_environment }}" +# args: +# creates: /etc/origin/fedoraimagestreamcreated + +# - name: set policy for koji builder in openshift for osbs +# shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_stg_username }} && touch /etc/origin/koji-builder-policy-added" +# args: +# creates: "/etc/origin/koji-builder-policy-added" +# when: env == "staging" + +# - name: set policy for koji builder in openshift for osbs +# shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_prod_username }} && touch /etc/origin/koji-builder-policy-added" +# args: +# creates: "/etc/origin/koji-builder-policy-added" +# when: env == "production" + +# - name: set policy for koji builder in openshift for atomic-reactor +# shell: "oadm policy add-role-to-user -n default edit system:serviceaccount:default:builder && touch /etc/origin/atomic-reactor-policy-added" +# args: +# creates: "/etc/origin/atomic-reactor-policy-added" + +# - name: Create buildroot container conf directory +# file: +# path: "/etc/osbs/buildroot/" +# state: directory + +# - name: Upload Dockerfile for buildroot container +# copy: +# src: "{{ files }}/osbs/buildroot-Dockerfile-{{env}}" +# dest: "/etc/osbs/buildroot/Dockerfile" +# mode: 0400 +# notify: +# - buildroot container + +# - name: Upload internal CA for buildroot +# copy: +# src: "{{private}}/files/osbs/{{env}}/osbs-internal.pem" +# dest: "/etc/osbs/buildroot/ca.crt" +# mode: 0400 +# notify: +# - buildroot container + +# - name: stat /usr/share/atomic-reactor/atomic-reactor.tar.gz +# stat: +# path: /usr/share/atomic-reactor/atomic-reactor.tar.gz +# register: usr_ar_stat + +# - name: stat /etc/osbs/buildroot/atomic-reactor.tar.gz +# stat: +# path: /etc/osbs/buildroot/atomic-reactor.tar.gz +# register: etc_ar_stat + +# - name: remove old hardlink to /etc/osbs/buildroot/atomic-reactor.tar.gz +# file: +# path: /etc/osbs/buildroot/atomic-reactor.tar.gz +# state: absent +# when: etc_ar_stat.stat.exists and usr_ar_stat.stat.checksum != etc_ar_stat.stat.checksum + +# - name: Hardlink atomic-reactor source for buildroot container (because Docker) +# file: +# src: /usr/share/atomic-reactor/atomic-reactor.tar.gz +# dest: /etc/osbs/buildroot/atomic-reactor.tar.gz +# state: hard +# notify: +# - buildroot container +# when: etc_ar_stat.stat.exists and usr_ar_stat.stat.checksum != etc_ar_stat.stat.checksum + +# - name: pull fedora required docker images +# shell: "docker pull {{docker_registry}}/{{item}}" +# with_items: "{{fedora_required_images}}" +# register: docker_pull_fedora +# changed_when: "'Downloaded newer image' in docker_pull_fedora.stdout" + +# - name: pull openshift required docker images +# shell: "docker pull {{docker_registry}}/{{item}}:v{{origin_version}}" +# with_items: "{{openshift_required_images}}" +# register: docker_pull_openshift +# changed_when: "'Downloaded newer image' in docker_pull_openshift.stdout" + +# - name: tag openshift required docker images locally +# shell: "docker tag {{docker_registry}}/{{item}}:v{{origin_version}} {{item}}:v{{origin_version}}" +# with_items: "{{openshift_required_images}}" +# when: docker_pull_openshift|changed + +# - name: refresh fedora image streams +# shell: "oc import-image fedora --all" +# when: docker_pull_fedora|changed