ok lets try and make some boxes for maxamillion

inventory/group_vars/osbs-nodes

@@ -0,0 +1,21 @@
+---
+# Define resources for this group of hosts here.
+lvm_size: 60000
+mem_size: 8192
+num_cpus: 2
+
+tcp_ports: [ 80, 443, 8443]
+
+fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran
+sudoers: "{{ private }}/files/sudo/00releng-sudoers"
+
+docker_cert_dir: "/etc/docker/certs.d/candidate-registry.fedoraproject.org"
+docker_registry: "candidate-registry.fedoraproject.org"
+source_registry: "registry.fedoraproject.org"
+
+osbs_url: "osbs.fedoraproject.org"
+osbs_koji_username: "kojibuilder"
+
+koji_url: "koji.fedoraproject.org"
+
+osbs_client_conf_path: /etc/osbs.conf

inventory/group_vars/osbs-nodes-stg

@@ -0,0 +1,21 @@
+---
+# Define resources for this group of hosts here.
+lvm_size: 60000
+mem_size: 8192
+num_cpus: 2
+
+tcp_ports: [ 80, 443, 8443]
+
+fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran
+sudoers: "{{ private }}/files/sudo/00releng-sudoers"
+
+docker_cert_dir: "/etc/docker/certs.d/candidate-registry.stg.fedoraproject.org"
+source_registry: "registry.stg.fedoraproject.org"
+docker_registry: "candidate-registry.stg.fedoraproject.org"
+
+osbs_url: "osbs.stg.fedoraproject.org"
+osbs_koji_username: "kojibuilder_stg"
+
+koji_url: "koji.stg.fedoraproject.org"
+
+osbs_client_conf_path: /etc/osbs.conf

inventory/host_vars/osbs-node01.phx2.fedoraproject.org

@@ -0,0 +1,18 @@
+---
+nm: 255.255.255.0
+gw: 10.5.125.254
+dns: 10.5.126.21
+ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-24-osbs
+ks_repo: http://10.5.126.23/pub/fedora/linux/releases/24/Server/x86_64/os/
+volgroup: /dev/vg_guests
+eth0_ip: 10.5.125.53
+vmhost: bvirthost01.phx2.fedoraproject.org
+datacenter: phx2
+
+nrpe_procs_warn: 900
+nrpe_procs_crit: 1000
+
+lvm_size: 120g
+mem_size: 16384
+max_mem_size: 16384
+num_cpus: 4

inventory/host_vars/osbs-node01.stg.phx2.fedoraproject.org

@@ -0,0 +1,19 @@
+---
+nm: 255.255.255.0
+gw: 10.5.126.254
+dns: 10.5.126.21
+ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-24-osbs
+ks_repo: http://10.5.126.23/pub/fedora/linux/releases/24/Server/x86_64/os/
+volgroup: /dev/vg_guests
+eth0_ip: 10.5.126.219
+vmhost: virthost20.phx2.fedoraproject.org
+datacenter: phx2
+host_group: osbs-nodes-stg
+
+nrpe_procs_warn: 900
+nrpe_procs_crit: 1000
+
+lvm_size: 120g
+mem_size: 8192
+max_mem_size: 16384
+num_cpus: 4

inventory/host_vars/osbs-node02.phx2.fedoraproject.org

@@ -0,0 +1,18 @@
+---
+nm: 255.255.255.0
+gw: 10.5.125.254
+dns: 10.5.126.21
+ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-24-osbs
+ks_repo: http://10.5.126.23/pub/fedora/linux/releases/24/Server/x86_64/os/
+volgroup: /dev/vg_guests
+eth0_ip: 10.5.125.54
+vmhost: bvirthost01.phx2.fedoraproject.org
+datacenter: phx2
+
+nrpe_procs_warn: 900
+nrpe_procs_crit: 1000
+
+lvm_size: 120g
+mem_size: 16384
+max_mem_size: 16384
+num_cpus: 4

inventory/inventory

@@ -662,6 +662,7 @@ mm-crawler01.stg.phx2.fedoraproject.org
 beaker-stg01.qa.fedoraproject.org
 zanata2fedmsg01.stg.phx2.fedoraproject.org
 osbs-master01.stg.phx2.fedoraproject.org
+osbs-node01.stg.phx2.fedoraproject.org
 docker-registry01.stg.phx2.fedoraproject.org
 docker-candidate-registry01.stg.phx2.fedoraproject.org
 
@@ -1168,9 +1169,17 @@ taskotron01.qa.fedoraproject.org
 [osbs]
 osbs-master01.phx2.fedoraproject.org
 
+[osbs-nodes]
+osbs-node01.phx2.fedoraproject.org
+osbs-node02.phx2.fedoraproject.org
+
 [osbs-stg]
 osbs-master01.stg.phx2.fedoraproject.org
 
+
+[osbs-nodes-stg]
+osbs-node01.stg.phx2.fedoraproject.org
+
 [docker-registry]
 docker-registry01.phx2.fedoraproject.org
 docker-candidate-registry01.phx2.fedoraproject.org

playbooks/groups/osbs-node.yml

@@ -0,0 +1,423 @@
+# create an osbs server
+- include:  "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=osbs-nodes:osbs-nodes-stg"
+
+- name: make the box be real
+  hosts: osbs-nodes:osbs-nodes-stg
+  user: root
+  gather_facts: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  roles:
+  - base
+  - rkhunter
+  - nagios/client
+  - hosts
+  - fas_client
+  - collectd/base
+  - rsyncd
+  - sudo
+  - { role: openvpn/client,
+      when: env != "staging" }
+
+  tasks:
+  - include: "{{ tasks }}/yumrepos.yml"
+  - include: "{{ tasks }}/2fa_client.yml"
+  - include: "{{ tasks }}/motd.yml"
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+# - name: pre-install osbs tasks
+#   hosts: osbs-nodes:osbs-nodes-stg
+#   vars_files:
+#     - /srv/web/infra/ansible/vars/global.yml
+#     - /srv/private/ansible/vars.yml
+#     - /srv/private/ansible/files/openstack/passwords.yml
+#     - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+#   tasks:
+#     - name: copy docker-storage-setup config
+#       copy:
+#         src: "{{files}}/osbs/docker-storage-setup"
+#         dest:  "/etc/sysconfig/docker-storage-setup"
+
+#     - name: create cert dir for openshift public facing REST API SSL
+#       file:
+#         path: "/etc/origin/master/named_certificates"
+#         state: "directory"
+
+#     - name: install cert for openshift public facing REST API SSL
+#       copy:
+#         src: "{{private}}/files/osbs/{{env}}/osbs-internal.pem"
+#         dest: "/etc/origin/master/named_certificates/{{osbs_url}}.pem"
+
+#     - name: install key for openshift public facing REST API SSL
+#       copy:
+#         src: "{{private}}/files/osbs/{{env}}/osbs-internal.key"
+#         dest: "/etc/origin/master/named_certificates/{{osbs_url}}.key"
+
+#     - name: ensure origin conf dir exists
+#       file:
+#         path: "/etc/origin"
+#         state: "directory"
+
+#     - name: place htpasswd file
+#       copy:
+#         src: "{{private}}/files/httpd/osbs-{{env}}.htpasswd"
+#         dest: /etc/origin/htpasswd
+
+#   roles:
+#     - {
+#       role: push-docker,
+#         docker_cert_name: "containerbuild",
+#         docker_cert_dir: "/etc/docker/certs.d/candidate-registry.stg.fedoraproject.org",
+#       when: env == "staging"
+#     }
+#     - {
+#       role: push-docker,
+#         docker_cert_name: "containerbuild",
+#         docker_cert_dir: "/etc/docker/certs.d/candidate-registry.fedoraproject.org",
+#       when: env == "production"
+#     }
+
+# - name: setup osbs
+#   hosts: osbs-nodes:osbs-nodes-stg
+#   vars_files:
+#     - /srv/web/infra/ansible/vars/global.yml
+#     - /srv/private/ansible/vars.yml
+#     - /srv/private/ansible/files/openstack/passwords.yml
+#     - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+#   roles:
+#     - osbs-atomic-reactor
+#     - {
+#       role: osbs-common,
+#         osbs_manage_firewalld: false,
+#     }
+#     - osbs-install-openshift
+#     - {
+#       role: osbs-master,
+#         osbs_openshift_loglevel: 2,
+#         osbs_master_export_port: true,
+#         osbs_manage_firewalld: false,
+#         osbs_proxy_cert_file: '/etc/origin/proxy_selfsigned.crt',
+#         osbs_proxy_key_file: '/etc/origin/proxy_selfsigned.key',
+#         osbs_proxy_certkey_file: '/etc/origin/proxy_certkey.crt',
+#         osbs_proxy_ca_file: '/etc/origin/proxy_selfsigned.crt',
+#         osbs_readonly_users: [],
+#         osbs_readonly_groups: [],
+#         osbs_readwrite_users: [ "{{ osbs_koji_stg_username }}" ],
+#         osbs_readwrite_groups: [],
+#         osbs_admin_users: [],
+#         osbs_admin_groups: [],
+#         osbs_master_max_pods: 3,
+#         osbs_update_packages: false,
+#         osbs_image_gc_high_threshold: 90,
+#         osbs_image_gc_low_threshold: 80,
+#         osbs_identity_provider: "htpasswd_provider",
+#         osbs_identity_htpasswd: {
+#           name: htpasswd_provider,
+#           challenge: true,
+#           login: true,
+#           provider_file: "/etc/origin/htpasswd"
+#         },
+#         osbs_named_certificates: {
+#           enabled: true,
+#           cert_file: "named_certificates/{{osbs_url}}.pem",
+#           key_file: "named_certificates/{{osbs_url}}.key",
+#           names: [ "{{osbs_url}}" ],
+#         },
+#         osbs_public_api_url: "{{osbs_url}}",
+#         when: env == "staging"
+#       }
+#     - {
+#       role: osbs-master,
+#         osbs_openshift_loglevel: 2,
+#         osbs_master_export_port: true,
+#         osbs_manage_firewalld: false,
+#         osbs_proxy_cert_file: '/etc/origin/proxy_selfsigned.crt',
+#         osbs_proxy_key_file: '/etc/origin/proxy_selfsigned.key',
+#         osbs_proxy_certkey_file: '/etc/origin/proxy_certkey.crt',
+#         osbs_proxy_ca_file: '/etc/origin/proxy_selfsigned.crt',
+#         osbs_readonly_users: [],
+#         osbs_readonly_groups: [],
+#         osbs_readwrite_users: [ "{{ osbs_koji_prod_username }}" ],
+#         osbs_readwrite_groups: [],
+#         osbs_admin_users: [],
+#         osbs_admin_groups: [],
+#         osbs_master_max_pods: 3,
+#         osbs_update_packages: false,
+#         osbs_image_gc_high_threshold: 90,
+#         osbs_image_gc_low_threshold: 80,
+#         osbs_identity_provider: "htpasswd_provider",
+#         osbs_identity_htpasswd: {
+#           name: htpasswd_provider,
+#           challenge: true,
+#           login: true,
+#           provider_file: "/etc/origin/htpasswd"
+#         },
+#         osbs_named_certificates: {
+#           enabled: true,
+#           cert_file: "named_certificates/{{osbs_url}}.pem",
+#           key_file: "named_certificates/{{osbs_url}}.key",
+#           names: [ "{{osbs_url}}" ],
+#         },
+#         osbs_public_api_url: "{{osbs_url}}",
+#         when: env == "production"
+#       }
+
+#     - {
+#       role: osbs-client,
+#         general: {
+#           verbose: 0,
+#           build_json_dir: '/usr/share/osbs/',
+#           openshift_required_version: 1.1.0,
+#         },
+#         default: {
+#           username: "{{ osbs_koji_stg_username }}",
+#           password: "{{ osbs_koji_stg_password }}",
+#           koji_certs_secret: "koji",
+#           openshift_url: 'https://{{osbs_url}}/',
+#           registry_uri: 'https://{{docker_registry}}/v2',
+#           source_registry_uri: 'https://{{source_registry}}/v2',
+#           build_host: '{{osbs_url}}',
+#           koji_root: 'https://{{koji_url}}/koji',
+#           koji_hub: 'https://{{koji_url}}/kojihub',
+#           sources_command: 'fedpkg sources',
+#           build_type: 'prod',
+#           authoritative_registry: 'registry.example.com',
+#           vendor: 'Fedora Project',
+#           verify_ssl: true,
+#           use_auth: true,
+#           builder_use_auth: true,
+#           distribution_scope: 'private',
+#           registry_api_versions: 'v2',
+#           builder_openshift_url: 'https://172.17.0.1:8443/'
+#         },
+#       when: env == "staging"
+#       }
+#     - {
+#       role: osbs-client,
+#         general: {
+#           verbose: 0,
+#           build_json_dir: '/usr/share/osbs/',
+#           openshift_required_version: 1.1.0,
+#         },
+#         default: {
+#           username: "{{ osbs_koji_prod_username }}",
+#           password: "{{ osbs_koji_prod_password }}",
+#           koji_certs_secret: "koji",
+#           openshift_url: 'https://{{osbs_url}}/',
+#           registry_uri: 'https://{{docker_registry}}/v2',
+#           source_registry_uri: 'https://{{source_registry}}/v2',
+#           build_host: '{{osbs_url}}',
+#           koji_root: 'https://{{koji_url}}/koji',
+#           koji_hub: 'https://{{koji_url}}/kojihub',
+#           sources_command: 'fedpkg sources',
+#           build_type: 'prod',
+#           authoritative_registry: 'registry.example.com',
+#           vendor: 'Fedora Project',
+#           verify_ssl: true,
+#           use_auth: true,
+#           builder_use_auth: true,
+#           distribution_scope: 'private',
+#           registry_api_versions: 'v2',
+#           builder_openshift_url: 'https://172.17.0.1:8443/'
+#         },
+#       when: env == "production"
+#       }
+
+# - name: post-install osbs tasks
+#   hosts: osbs-nodes:osbs-nodes-stg
+#   vars_files:
+#     - /srv/web/infra/ansible/vars/global.yml
+#     - /srv/private/ansible/vars.yml
+#     - /srv/private/ansible/files/openstack/passwords.yml
+#     - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+#   vars:
+#     osbs_kubeconfig_path: /etc/origin/master/admin.kubeconfig
+#     osbs_environment:
+#       KUBECONFIG: "{{ osbs_kubeconfig_path }}"
+#     koji_pki_dir: /etc/pki/koji
+#     koji_ca_cert_path: "{{koji_pki_dir}}/fedora-server-ca.cert"
+#     koji_cert_path: "{{koji_pki_dir}}/fedora-builder.pem"
+#     koji_builder_user: dockerbuilder
+#     osbs_builder_user: builder
+
+
+#   handlers:
+#     - name: buildroot container
+#       shell: 'docker build --no-cache --rm -t buildroot /etc/osbs/buildroot/'
+
+#     - name: oc secrets new
+#       shell: "oc secrets new koji cert={{ koji_cert_path }} ca={{ koji_ca_cert_path }} serverca={{ koji_ca_cert_path }}"
+#       environment: "{{ osbs_environment }}"
+#       notify: oc secrets add
+
+#     - name: oc secrets add
+#       shell: "oc secrets add serviceaccount/{{ osbs_builder_user }} secrets/koji --for=mount"
+#       environment: "{{ osbs_environment }}"
+
+
+#   tasks:
+#     - name: set nrpe read access for osbs.conf for nagios monitoring
+#       acl: name={{ osbs_client_conf_path }} entity=nrpe etype=user permissions=r state=present
+
+#     - name: pull fedora required docker images
+#       shell: "docker pull {{item}}"
+#       with_items: "{{fedora_required_images}}"
+#       delegate_to: compose-x86-01.phx2.fedoraproject.org
+#       register: docker_pull_fedora_delegated
+#       changed_when: "'Downloaded newer image' in docker_pull_fedora_delegated.stdout"
+
+#     - name: tag fedora required docker images for our registry
+#       shell: "docker tag {{item}} {{docker_registry}}/{{item}}"
+#       with_items: "{{fedora_required_images}}"
+#       delegate_to: compose-x86-01.phx2.fedoraproject.org
+#       when: docker_pull_fedora_delegated|changed
+
+#     - name: push fedora required docker images to our registry
+#       shell: "docker push {{docker_registry}}/{{item}}"
+#       with_items: "{{fedora_required_images}}"
+#       delegate_to: compose-x86-01.phx2.fedoraproject.org
+#       when: docker_pull_fedora_delegated|changed
+
+#     - name: register origin_version_out rpm query
+#       shell: "rpm -q origin --qf '%{Version}'"
+#       register: origin_version_out
+#       always_run: true
+#       changed_when: False
+
+#     - set_fact:
+#         origin_version: "{{origin_version_out.stdout}}"
+
+#     - name: pull openshift required docker images
+#       shell: "docker pull {{item}}:v{{origin_version}}"
+#       with_items: "{{openshift_required_images}}"
+#       delegate_to: compose-x86-01.phx2.fedoraproject.org
+#       register: docker_pull_openshift_delegated
+#       changed_when: "'Downloaded newer image' in docker_pull_openshift_delegated.stdout"
+
+#     - name: tag openshift required docker images for our registry
+#       shell: "docker tag {{item}}:v{{origin_version}} {{docker_registry}}/{{item}}:v{{origin_version}}"
+#       with_items: "{{openshift_required_images}}"
+#       delegate_to: compose-x86-01.phx2.fedoraproject.org
+#       when: docker_pull_openshift_delegated|changed
+
+#     - name: push openshift required docker images to our registry
+#       shell: "docker push {{docker_registry}}/{{item}}:v{{origin_version}}"
+#       with_items: "{{openshift_required_images}}"
+#       delegate_to: compose-x86-01.phx2.fedoraproject.org
+#       when: docker_pull_openshift_delegated|changed
+
+#     - name: Ensure koji dockerbuilder cert path exists
+#       file:
+#         path: "{{ koji_pki_dir }}"
+#         state: "directory"
+#         mode: 0400
+
+#     - name: Add koji dockerbuilder cert for Content Generator import
+#       copy:
+#         src: "{{private}}/files/koji/containerbuild.pem"
+#         dest: "{{ koji_cert_path }}"
+#       notify: oc secrets new
+
+#     - name: Add koji dockerbuilder ca cert for Content Generator import
+#       copy:
+#         src: "{{private}}/files/koji/buildercerts/fedora-ca.cert"
+#         dest: "{{ koji_ca_cert_path }}"
+#       notify: oc secrets new
+
+#     - name: create fedora image stream for OpenShift
+#       shell: "echo '{ \"apiVersion\": \"v1\", \"kind\": \"ImageStream\", \"metadata\": { \"name\": \"fedora\" }, \"spec\": { \"dockerImageRepository\": \"{{docker_registry}}/fedora\" } }' | oc create -f - && touch /etc/origin/fedoraimagestreamcreated"
+#       environment: "{{ osbs_environment }}"
+#       args:
+#         creates: /etc/origin/fedoraimagestreamcreated
+
+#     - name: set policy for koji builder in openshift for osbs
+#       shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_stg_username }} && touch /etc/origin/koji-builder-policy-added"
+#       args:
+#         creates: "/etc/origin/koji-builder-policy-added"
+#       when: env == "staging"
+
+#     - name: set policy for koji builder in openshift for osbs
+#       shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_prod_username }} && touch /etc/origin/koji-builder-policy-added"
+#       args:
+#         creates: "/etc/origin/koji-builder-policy-added"
+#       when: env == "production"
+
+#     - name: set policy for koji builder in openshift for atomic-reactor
+#       shell: "oadm policy add-role-to-user -n default edit system:serviceaccount:default:builder && touch /etc/origin/atomic-reactor-policy-added"
+#       args:
+#         creates: "/etc/origin/atomic-reactor-policy-added"
+
+#     - name: Create buildroot container conf directory
+#       file:
+#         path: "/etc/osbs/buildroot/"
+#         state: directory
+
+#     - name: Upload Dockerfile for buildroot container
+#       copy:
+#         src: "{{ files }}/osbs/buildroot-Dockerfile-{{env}}"
+#         dest: "/etc/osbs/buildroot/Dockerfile"
+#         mode: 0400
+#       notify:
+#         - buildroot container
+
+#     - name: Upload internal CA for buildroot
+#       copy:
+#         src: "{{private}}/files/osbs/{{env}}/osbs-internal.pem"
+#         dest: "/etc/osbs/buildroot/ca.crt"
+#         mode: 0400
+#       notify:
+#         - buildroot container
+
+#     - name: stat /usr/share/atomic-reactor/atomic-reactor.tar.gz
+#       stat:
+#         path: /usr/share/atomic-reactor/atomic-reactor.tar.gz
+#       register: usr_ar_stat
+
+#     - name: stat /etc/osbs/buildroot/atomic-reactor.tar.gz
+#       stat:
+#         path: /etc/osbs/buildroot/atomic-reactor.tar.gz
+#       register: etc_ar_stat
+
+#     - name: remove old hardlink to /etc/osbs/buildroot/atomic-reactor.tar.gz
+#       file:
+#         path: /etc/osbs/buildroot/atomic-reactor.tar.gz
+#         state: absent
+#       when: etc_ar_stat.stat.exists and usr_ar_stat.stat.checksum != etc_ar_stat.stat.checksum
+
+#     - name: Hardlink atomic-reactor source for buildroot container (because Docker)
+#       file:
+#         src: /usr/share/atomic-reactor/atomic-reactor.tar.gz
+#         dest: /etc/osbs/buildroot/atomic-reactor.tar.gz
+#         state: hard
+#       notify:
+#         - buildroot container
+#       when: etc_ar_stat.stat.exists and usr_ar_stat.stat.checksum != etc_ar_stat.stat.checksum
+
+#     - name: pull fedora required docker images
+#       shell: "docker pull {{docker_registry}}/{{item}}"
+#       with_items: "{{fedora_required_images}}"
+#       register: docker_pull_fedora
+#       changed_when: "'Downloaded newer image' in docker_pull_fedora.stdout"
+
+#     - name: pull openshift required docker images
+#       shell: "docker pull {{docker_registry}}/{{item}}:v{{origin_version}}"
+#       with_items: "{{openshift_required_images}}"
+#       register: docker_pull_openshift
+#       changed_when: "'Downloaded newer image' in docker_pull_openshift.stdout"
+
+#     - name: tag openshift required docker images locally
+#       shell: "docker tag {{docker_registry}}/{{item}}:v{{origin_version}} {{item}}:v{{origin_version}}"
+#       with_items: "{{openshift_required_images}}"
+#       when: docker_pull_openshift|changed
+
+#     - name: refresh fedora image streams
+#       shell: "oc import-image fedora --all"
+#       when: docker_pull_fedora|changed