diff --git a/playbooks/groups/osbs/configure-osbs.yml b/playbooks/groups/osbs/configure-osbs.yml new file mode 100644 index 0000000000..c9a11bcf1d --- /dev/null +++ b/playbooks/groups/osbs/configure-osbs.yml @@ -0,0 +1,3 @@ +- import_playbook: "/srv/web/infra/ansible/playbooks/groups/osbs/setup-orchestrator-namespace.yml" +- import_playbook: "/srv/web/infra/ansible/playbooks/groups/osbs/setup-worker-namespace.yml" +- import_playbook: "/srv/web/infra/ansible/playbooks/groups/osbs/osbs-post-install.yml" diff --git a/playbooks/groups/osbs/osbs-post-install.yml b/playbooks/groups/osbs/osbs-post-install.yml new file mode 100644 index 0000000000..d290460bc1 --- /dev/null +++ b/playbooks/groups/osbs/osbs-post-install.yml @@ -0,0 +1,196 @@ +- name: post-install master host osbs tasks + hosts: osbs_masters_stg:osbs_masters:osbs_aarch64_masters_stg[0]:osbs_aarch64_masters[0] + tags: + - osbs-post-install + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - /srv/private/ansible/vars.yml + - /srv/private/ansible/files/openstack/passwords.yml + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + vars: + osbs_kubeconfig_path: /etc/origin/master/admin.kubeconfig + osbs_environment: + KUBECONFIG: "{{ osbs_kubeconfig_path }}" + + tasks: + - name: cron entry to clean up old builds + copy: + src: "{{files}}/osbs/cleanup-old-osbs-builds" + dest: "/etc/cron.d/cleanup-old-osbs-builds" + +- name: post-install osbs control tasks + hosts: osbs_control + tags: osbs-post-install + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - /srv/private/ansible/vars.yml + - /srv/private/ansible/files/openstack/passwords.yml + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + tasks: + - name: enable nrpe for monitoring (noc01) + iptables: action=insert chain=INPUT destination_port=5666 protocol=tcp source=10.5.126.41 state=present jump=ACCEPT + tags: + - iptables + + +- name: post-install node host osbs tasks + hosts: osbs_nodes_stg:osbs_nodes:osbs_aarch64_nodes_stg:osbs_aarch64_nodes + tags: + - osbs-post-install + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - /srv/private/ansible/vars.yml + - /srv/private/ansible/files/openstack/passwords.yml + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + vars: + osbs_kubeconfig_path: /etc/origin/master/admin.kubeconfig + osbs_environment: + KUBECONFIG: "{{ osbs_kubeconfig_path }}" + + + handlers: + - name: Remove the previous buildroot image + docker_image: + state: absent + name: buildroot + + - name: Build the new buildroot container + docker_image: + path: /etc/osbs/buildroot/ + name: buildroot + nocache: yes + + - name: restart and reload docker service + systemd: + name: docker + state: restarted + daemon_reload: yes + + tasks: + - name: enable nrpe for monitoring (noc01) + iptables: action=insert chain=INPUT destination_port=5666 protocol=tcp source=10.5.126.41 state=present jump=ACCEPT + tags: + - iptables + + - name: copy docker iptables script + copy: + src: "{{files}}/osbs/fix-docker-iptables.{{ env }}" + dest: /usr/local/bin/fix-docker-iptables + mode: 0755 + tags: + - iptables + notify: + - restart and reload docker service + + - name: copy docker custom service config + copy: + src: "{{files}}/osbs/docker.firewall.service" + dest: /etc/systemd/system/docker.service.d/firewall.conf + tags: + - docker + notify: + - restart and reload docker service + + - name: copy the osbs customization file + copy: + src: "{{item}}" + dest: "/etc/osbs/buildroot/" + owner: root + mode: 0600 + with_items: + - "{{files}}/osbs/worker_customize.json" + - "{{files}}/osbs/orchestrator_customize.json" + + - name: Create buildroot container conf directory + file: + path: "/etc/osbs/buildroot/" + state: directory + + - name: Upload Dockerfile for buildroot container + template: + src: "{{ files }}/osbs/buildroot-Dockerfile-{{env}}.j2" + dest: "/etc/osbs/buildroot/Dockerfile" + mode: 0400 + notify: + - Remove the previous buildroot image + - Build the new buildroot container + + - name: Upload krb5.conf for buildroot container + template: + src: "{{ roles_path }}/base/templates/krb5.conf.j2" + dest: "/etc/osbs/buildroot/krb5.conf" + mode: 0644 + notify: + - Remove the previous buildroot image + - Build the new buildroot container + + - name: Upload internal CA for buildroot + copy: + src: "{{private}}/files/osbs/{{env}}/osbs-internal.pem" + dest: "/etc/osbs/buildroot/ca.crt" + mode: 0400 + notify: + - Remove the previous buildroot image + - Build the new buildroot container + + - name: stat infra repofile + stat: + path: "/etc/yum.repos.d/infra-tags.repo" + register: infra_repo_stat + + - name: stat /etc/osbs/buildroot/ infra repofile + stat: + path: "/etc/osbs/buildroot/infra-tags.repo" + register: etcosbs_infra_repo_stat + + - name: remove old /etc/osbs/buildroot/ infra repofile + file: + path: "/etc/osbs/buildroot/infra-tags.repo" + state: absent + when: etcosbs_infra_repo_stat.stat.exists and infra_repo_stat.stat.checksum != etcosbs_infra_repo_stat.stat.checksum + + - name: Copy repofile for buildroot container (because Docker) + copy: + src: "/etc/yum.repos.d/infra-tags.repo" + dest: "/etc/osbs/buildroot/infra-tags.repo" + remote_src: true + notify: + - Remove the previous buildroot image + - Build the new buildroot container + when: etcosbs_infra_repo_stat.stat.exists == false + + - name: stat /etc/ keytab + stat: + path: "/etc/krb5.osbs_{{osbs_url}}.keytab" + register: etc_kt_stat + + - name: stat /etc/osbs/buildroot/ keytab + stat: + path: "/etc/osbs/buildroot/krb5.osbs_{{osbs_url}}.keytab" + register: etcosbs_kt_stat + + - name: remove old hardlink to /etc/osbs/buildroot/ keytab + file: + path: "/etc/osbs/buildroot/krb5.osbs_{{osbs_url}}.keytab" + state: absent + when: etcosbs_kt_stat.stat.exists and etc_kt_stat.stat.checksum != etcosbs_kt_stat.stat.checksum + + - name: Hardlink keytab for buildroot container (because Docker) + file: + src: "/etc/krb5.osbs_{{osbs_url}}.keytab" + dest: "/etc/osbs/buildroot/krb5.osbs_{{osbs_url}}.keytab" + state: hard + notify: + - Remove the previous buildroot image + - Build the new buildroot container + + when: etcosbs_kt_stat.stat.exists == false + + - name: pull fedora required docker images + command: "docker pull {{source_registry}}/{{item}}" + with_items: "{{fedora_required_images}}" + register: docker_pull_fedora + changed_when: "'Downloaded newer image' in docker_pull_fedora.stdout" + + - name: enable nrpe for monitoring (noc01) + iptables: action=insert chain=INPUT destination_port=5666 protocol=tcp source=10.5.126.41 state=present jump=ACCEPT diff --git a/playbooks/groups/osbs/setup-orchestrator-namespace.yml b/playbooks/groups/osbs/setup-orchestrator-namespace.yml new file mode 100644 index 0000000000..b68704a970 --- /dev/null +++ b/playbooks/groups/osbs/setup-orchestrator-namespace.yml @@ -0,0 +1,183 @@ +- name: Create orchestrator namespace + hosts: osbs_masters_stg[0]:osbs_masters[0] + roles: + - role: osbs-namespace + osbs_orchestrator: true + osbs_worker_clusters: "{{ osbs_conf_worker_clusters }}" + osbs_cpu_limitrange: "{{ osbs_orchestrator_cpu_limitrange }}" + osbs_nodeselector: "{{ osbs_orchestrator_default_nodeselector|default('') }}" + osbs_sources_command: "{{ osbs_conf_sources_command }}" + osbs_readwrite_users: "{{ osbs_conf_readwrite_users }}" + osbs_service_accounts: "{{ osbs_conf_service_accounts }}" + koji_use_kerberos: true + koji_kerberos_keytab: "FILE:/etc/krb5.osbs_{{ osbs_url }}.keytab" + koji_kerberos_principal: "osbs/{{osbs_url}}@{{ ipa_realm }}" + tags: + - osbs-orchestrator-namespace + +- name: setup reactor config secret in orchestrator namespace + hosts: osbs_masters_stg[0]:osbs_masters[0] + roles: + - role: osbs-secret + osbs_secret_name: reactor-config-secret + osbs_secret_files: + - source: "/tmp/{{ osbs_namespace }}-{{ env }}-reactor-config-secret.yml" + dest: config.yaml + tags: + - osbs-orchestrator-namespace + +- name: setup client config secret in orchestrator namespace + hosts: osbs_masters_stg[0]:osbs_masters[0] + roles: + - role: osbs-secret + osbs_secret_name: client-config-secret + osbs_secret_files: + - source: "/tmp/{{ osbs_namespace }}-{{ env }}-client-config-secret.conf" + dest: osbs.conf + tags: + - osbs-orchestrator-namespace + +- name: setup ODCS secret in orchestrator namespace + hosts: osbs_masters_stg[0]:osbs_masters[0] + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + roles: + - role: osbs-secret + osbs_secret_name: odcs-oidc-secret + osbs_secret_files: + - source: "{{ private }}/files/osbs/{{ env }}/odcs-oidc-token" + dest: token + tags: + - osbs-orchestrator-namespace + +- name: Save orchestrator token x86_64 + hosts: osbs_masters_stg[0]:osbs_masters[0] + tasks: + - name: get orchestrator service account token + command: "oc -n {{ osbs_worker_namespace }} sa get-token orchestrator" + register: orchestator_token_x86_64 + - name: save the token locally + local_action: > + copy + content="{{ orchestator_token_x86_64.stdout }}" + dest=/tmp/.orchestator-token-x86_64 + mode=0400 + tags: + - osbs-orchestrator-namespace + +- name: setup orchestrator token for x86_64-osbs + hosts: osbs_masters_stg[0]:osbs_masters[0] + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + roles: + - role: osbs-secret + osbs_secret_name: x86-64-orchestrator + osbs_secret_files: + - source: "/tmp/.orchestator-token-x86_64" + dest: token + + post_tasks: + - name: Delete the temporary secret file + local_action: > + file + state=absent + path="/tmp/.orchestator-token-x86_64" + tags: + - osbs-orchestrator-namespace + +- name: Save orchestrator token aarch64 + hosts: osbs_aarch64_masters_stg[0]:osbs_aarch64_masters[0] + tasks: + - name: get orchestrator service account token + command: "oc -n {{ osbs_worker_namespace }} sa get-token orchestrator" + register: orchestator_token_aarch64 + - name: save the token locally + local_action: > + copy + content="{{ orchestator_token_aarch64.stdout }}" + dest=/tmp/.orchestator-token-aarch64 + mode=0400 + tags: + - osbs-orchestrator-namespace + +- name: setup orchestrator token for aarch64-osbs + hosts: osbs_masters_stg[0]:osbs_masters[0] + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + roles: + - role: osbs-secret + osbs_secret_can_fail: true + osbs_secret_name: aarch64-orchestrator + osbs_secret_files: + - source: "/tmp/.orchestator-token-aarch64" + dest: token + + post_tasks: + - name: Delete the temporary secret file + local_action: > + file + state=absent + path="/tmp/.orchestator-token-aarch64" + + tags: + - osbs-orchestrator-namespace + +- name: Add dockercfg secret to allow registry push orchestrator + hosts: osbs_masters_stg[0]:osbs_masters[0] + tags: + - osbs-dockercfg-secret + user: root + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + pre_tasks: + - name: Create the username:password string needed by the template + set_fact: + auth_info_prod: "{{candidate_registry_osbs_prod_username}}:{{candidate_registry_osbs_prod_password}}" + auth_info_stg: "{{candidate_registry_osbs_stg_username}}:{{candidate_registry_osbs_stg_password}}" + + - name: Create the dockercfg secret file + local_action: > + template + src="{{ files }}/osbs/dockercfg-{{env}}-secret.j2" + dest="/tmp/.dockercfg" + mode=0400 + + roles: + - role: osbs-secret + osbs_secret_name: "v2-registry-dockercfg" + osbs_secret_type: kubernetes.io/dockercfg + osbs_secret_files: + - source: "/tmp/.dockercfg" + dest: .dockercfg + + post_tasks: + - name: Delete the temporary secret file + local_action: > + file + state=absent + path="/tmp/.dockercfg" + +- name: Add the orchestrator labels to the nodes + hosts: osbs_masters_stg[0]:osbs_masters[0] + tags: + - osbs-labels-nodes + tasks: + - name: Add the orchestrator labels to the nodes + command: "oc -n {{ osbs_namespace }} label nodes {{ item }} orchestrator=true --overwrite" + loop: "{{ groups['osbs_nodes_stg'] }}" + when: env == "staging" + + - name: Add the orchestrator labels to the nodes + command: "oc -n {{ osbs_namespace }} label nodes {{ item }} orchestrator=true --overwrite" + loop: "{{ groups['osbs_nodes'] }}" + when: env == "production" diff --git a/playbooks/groups/osbs/setup-worker-namespace.yml b/playbooks/groups/osbs/setup-worker-namespace.yml new file mode 100644 index 0000000000..149b7b15b3 --- /dev/null +++ b/playbooks/groups/osbs/setup-worker-namespace.yml @@ -0,0 +1,106 @@ +- name: Create worker namespace + hosts: osbs_masters_stg[0]:osbs_masters[0]:osbs_aarch64_masters_stg[0]:osbs_aarch64_masters[0] + tags: + - osbs-worker-namespace + user: root + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + vars: + osbs_kubeconfig_path: /etc/origin/master/admin.kubeconfig + osbs_environment: + KUBECONFIG: "{{ osbs_kubeconfig_path }}" + + roles: + - role: osbs-namespace + osbs_namespace: "{{ osbs_worker_namespace }}" + osbs_service_accounts: "{{ osbs_worker_service_accounts }}" + osbs_nodeselector: "{{ osbs_worker_default_nodeselector|default('') }}" + osbs_sources_command: "{{ osbs_conf_sources_command }}" + +- name: setup ODCS secret in worker namespace + hosts: osbs_masters_stg[0]:osbs_masters[0]:osbs_aarch64_masters_stg[0]:osbs_aarch64_masters[0] + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + roles: + - role: osbs-secret + osbs_namespace: "{{ osbs_worker_namespace }}" + osbs_secret_name: odcs-oidc-secret + osbs_secret_files: + - source: "{{ private }}/files/osbs/{{ env }}/odcs-oidc-token" + dest: token + tags: + - osbs-worker-namespace + +- name: Add dockercfg secret to allow registry push worker + hosts: osbs_masters_stg[0]:osbs_masters[0]:osbs_aarch64_masters_stg[0]:osbs_aarch64_masters[0] + tags: + - osbs-dockercfg-secret + - osbs-worker-namespace + user: root + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + pre_tasks: + - name: Create the username:password string needed by the template + set_fact: + auth_info_prod: "{{candidate_registry_osbs_prod_username}}:{{candidate_registry_osbs_prod_password}}" + auth_info_stg: "{{candidate_registry_osbs_stg_username}}:{{candidate_registry_osbs_stg_password}}" + + - name: Create the dockercfg secret file + local_action: > + template + src="{{ files }}/osbs/dockercfg-{{env}}-secret.j2" + dest="/tmp/.dockercfg" + mode=0400 + + roles: + - role: osbs-secret + osbs_namespace: "{{ osbs_worker_namespace }}" + osbs_secret_name: "v2-registry-dockercfg" + osbs_secret_type: kubernetes.io/dockercfg + osbs_secret_files: + - source: "/tmp/.dockercfg" + dest: .dockercfg + + post_tasks: + - name: Delete the temporary secret file + local_action: > + file + state=absent + path="/tmp/.dockercfg" + +- name: Add the worker labels to the nodes x86_64 + hosts: osbs_masters_stg[0]:osbs_masters[0] + tags: + - osbs-labels-nodes + tasks: + - name: Add the worker label + command: "oc -n {{ osbs_worker_namespace }} label nodes {{ item }} worker=true --overwrite" + loop: "{{ groups['osbs_nodes_stg'] }}" + when: env == "staging" + - name: Add the worker label + command: "oc -n {{ osbs_worker_namespace }} label nodes {{ item }} worker=true --overwrite" + loop: "{{ groups['osbs_nodes'] }}" + when: env == "production" + +- name: Add the worker labels to the nodes aarch64 + hosts: osbs_aarch64_masters_stg[0]:osbs_aarch64_masters[0] + tags: + - osbs-labels-nodes + tasks: + - name: Add the worker label + command: "oc -n {{ osbs_worker_namespace }} label nodes {{ item }} worker=true --overwrite" + loop: "{{ groups['osbs-aarch64-nodes-stg'] }}" + when: env == "staging" + - name: Add the worker label + command: "oc -n {{ osbs_worker_namespace }} label nodes {{ item }} worker=true --overwrite" + loop: "{{ groups['osbs-aarch64-nodes'] }}" + when: env == "production"