Unify copr prod and dev playbooks

files/copr/forward-dev

@@ -1,2 +1,3 @@
 msuchy+coprmachine@redhat.com
 asamalik@redhat.com
+vgologuz@redhat.com

inventory/group_vars/copr

@@ -0,0 +1,3 @@
+---
+devel: false
+_forward-src: "{{ files }}/copr/forward"

inventory/group_vars/copr-back

@@ -0,0 +1,2 @@
+---
+

inventory/group_vars/copr-back-stg

@@ -0,0 +1,2 @@
+---
+

inventory/group_vars/copr-front

@@ -0,0 +1,2 @@
+---
+copr-hostname: "copr-fe.cloud.fedoraproject.org"

inventory/group_vars/copr-front-stg

@@ -0,0 +1,2 @@
+---
+copr-hostname: "copr-fe-dev.cloud.fedoraproject.org"

inventory/group_vars/copr-stg

@@ -0,0 +1,3 @@
+---
+devel: true
+_forward-src: "{{ files }}/copr/forward-dev"

inventory/host_vars/209.132.184.142

@@ -23,3 +23,5 @@ fedmsg_certs:
   owner: root
   group: copr
 
+# Copr vars
+copr-hostname: copr-be.cloud.fedoraproject.org

inventory/host_vars/copr-be-dev.cloud.fedoraproject.org

@@ -9,3 +9,6 @@ public_ip: 209.132.184.179
 root_auth_users:  bkabrda msuchy tradej pingou
 description: copr dispatcher and repo server - dev instance
 tcp_ports: ['22', '80', '443']
+
+# Copr vars
+copr-hostname: copr-be-dev.cloud.fedoraproject.org

inventory/inventory

@@ -698,3 +698,22 @@ virthost
 bvirthost
 buildvmhost
 
+[copr-front-stg]
+copr-fe-dev.cloud.fedoraproject.org
+
+[copr-back-stg]
+copr-be-dev.cloud.fedoraproject.org
+
+[copr-front]
+209.132.184.144
+
+[copr-back]
+209.132.184.142
+
+[copr]
+copr-front
+copr-back
+
+[copr-stg]
+copr-front-dev
+copr-back-dev

playbooks/groups/copr-back.yml

@@ -0,0 +1,234 @@
+- name: check/create instance
+  hosts: copr-back:copr-back-stg
+  user: root
+  gather_facts: False
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+
+  tasks:
+  - include: "{{ tasks }}/persistent_cloud.yml"
+  - include: "{{ tasks }}/growroot_cloud.yml"
+
+- name: provision instance
+  hosts: copr-back:copr-back-stg
+  user: root
+  gather_facts: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  # Roles are run first, before tasks, regardless of where you place them here.
+  roles:
+  - fedmsg/base
+
+  tasks:
+  - include: "{{ tasks }}/cloud_setup_basic.yml"
+  - include: "{{ tasks }}/iptables.yml"
+  - include: "{{ tasks }}/common_scripts.yml"
+
+  - name: prepare mount point
+    file: state=directory path=/var/lib/copr/public_html
+    when: not devel
+
+  - name: mount up disk of copr repo
+    mount: name=/var/lib/copr/public_html src='LABEL=copr-repo' fstype=ext4 state=mounted
+    when: not devel
+
+  - name: mount /tmp/
+    mount: name=/tmp src='tmpfs' fstype=tmpfs state=mounted
+    when: not devel
+
+  # We cannot use repo pointing to our self :(
+  #- name: copy copr.repo
+  #  copy: src=$files/copr/fe/yum/copr.repo dest=/etc/yum.repos.d/copr.repo
+
+  - name: set the hostname
+    shell: hostname "{{ copr-hostname }}"
+
+  - name: copy .forward file
+    copy: src="{{ _forward_src }}" dest=/root/.forward owner=root group=root
+
+  - name: deploy /etc/hosts file
+    copy: src="{{ files }}/copr/hosts" dest=/etc/hosts owner=root group=root mode=644
+
+  # packages needed
+  - name: add packages for copr-be
+    yum: state=present name={{ item }}
+    with_items:
+    - copr-selinux
+    - copr-backend
+    - git
+    - screen
+    - python-novaclient
+
+  - name: add packages for copr-be, production only
+    yum: state=present name={{ item }}
+    when: not devel
+    with_items:
+    - fail2ban
+    - system-config-firewall-base
+
+  - name: make copr dirs
+    file: state=directory path={{ item }}
+    with_items:
+    - /var/lib/copr/jobs
+    - /var/lib/copr/public_html/results
+
+  - name: Umask results
+    command: /usr/bin/umask 0000 chdir=/var/lib/copr/public_html/results
+
+  - name: setup dirs there
+    file: state=directory path="/home/copr/{{ item }}" owner=copr group=copr mode=0700
+    with_items:
+    - cloud
+    - .ssh
+
+  - name: add copr-buildsys keys to copr user path
+    copy: src={{ item }} dest=/home/copr/cloud/ owner=copr group=copr mode=0600
+    with_fileglob:
+     - "{{ private }}/files/openstack/copr-copr/*"
+
+  - name: setup privkey for copr user
+    copy: src="{{ private }}/files/copr/buildsys.priv" dest=/home/copr/.ssh/id_rsa owner=copr group=copr mode=600
+
+  - name: setup copr user ssh config file
+    copy: src="{{ files }}/copr/ssh_config" dest=/home/copr/.ssh/config  owner=copr group=copr mode=600
+
+  - name: create empty known_hosts
+    copy: src=/dev/null dest=/home/copr/.ssh/known_hosts owner=copr group=copr mode=600
+
+  - name: replace bashrc for copr user
+    copy: src="{{ files }}/copr/copr_bashrc" dest=/home/copr/.bashrc owner=copr group=copr mode=600
+
+  - name: auth_key so we can login to localhost as the copr user from the copr user
+    authorized_key: user=copr key="{{ item }}"
+    with_file:
+    - "{{ files }}/copr/provision/files/buildsys.pub"
+
+  - name: copy keystonerc
+    template: src="{{ files }}/copr/keystonerc" dest=/root/ owner=root group=root mode=600
+    when: not devel
+
+  - name: copy .boto file
+    copy: src="{{ files }}/copr/boto" dest=/home/copr/.boto owner=copr group=copr
+
+  # setup webserver
+  - name: add config for copr-repo path
+    copy: src="{{ files }}/copr/lighttpd/lighttpd.conf" dest=/etc/lighttpd/lighttpd.conf owner=root group=root mode=0644
+    notify:
+    - restart lighttpd
+
+  - name: copy httpd ssl certificates
+    copy: src="{{ puppet_private }}/httpd/{{ item }}" dest="/etc/lighttpd/{{ item }}" owner=root group=root mode=0600
+    when: not devel
+    with_items:
+    - copr-be.fedoraproject.org.key
+    - copr-be.fedoraproject.org.crt
+    notify:
+    - concate ssl certs
+    tags:
+    - config
+
+  - name: copy httpd ssl certificates (CAcert)
+    copy: src="{{ files }}/copr/DigiCertCA.crt" dest="/etc/lighttpd/" owner=root group=root mode=0600
+    when: not devel
+    tags:
+    - config
+    notify:
+    - restart lighttpd
+
+  # mime default to text/plain and enable dirlisting for indexes
+  - name: update lighttpd configs
+    copy: src="{{ files }}/copr/lighttpd/{{ item }}" dest="/etc/lighttpd/conf.d/{{ item }}" owner=root group=root mode=0644
+    with_items:
+    - dirlisting.conf
+    - mime.conf
+    notify:
+    - restart lighttpd
+
+  - name: start webserver
+    service: state=running enabled=yes name=lighttpd
+
+  - name: start fail2ban
+    service: state=running enabled=yes name=fail2ban
+    when: not devel
+
+  # setup dirs for the ansible execution off of provisioning
+  - name: dirs from provision
+    file: state=directory path="/home/copr/provision/{{ item }}" owner=copr group=copr
+    with_items:
+    - action_plugins
+    - library
+    - files
+    - files/mock
+    tags:
+    - provision_config
+
+  - name: put ansible.cfg for all this into /etc/ansible/ on the system
+    copy: src="{{ files }}/copr/provision/ansible.cfg" dest=/etc/ansible/ansible.cfg
+    tags:
+    - provision_config
+
+
+  - name: put some files into the provision subdir
+    template: src="{{ files }}/copr/provision/{{ item }}" dest="/home/copr/provision/{{ item }}"
+    with_items:
+    - inventory
+    - terminatepb.yml
+    tags:
+    - provision_config
+
+  - name: put some files into the provision subdir, devel only
+    copy: src="{{ files }}/copr/provision/{{ item }}" dest="/home/copr/provision/{{ item }}"
+    when: devel
+    with_items:
+    - builderpb.yml
+    tags:
+    - provision_config
+
+
+  - name: put files into the files subdir off of provisioning
+    copy: src={{ item }} dest=/home/copr/provision/files/
+    with_fileglob:
+     - "{{ files }}/copr/provision/files/*"
+    tags:
+    - provision_config
+
+  # ansible lacks a recurse - so we need this until then
+  - name: put files into the files/mock subdir off of provisioning
+    copy: src={{ item }} dest=/home/copr/provision/files/mock
+    with_fileglob:
+     - "{{ files }}/copr/provision/files/mock/*"
+    tags:
+    - provision_config
+
+  - name: copy copr-be.conf
+    template: src="{{ files }}/copr/copr-be.conf" dest=/etc/copr/copr-be.conf  owner=root group=copr mode=640
+    notify:
+    - restart copr-backend
+    tags:
+    - config
+
+  - name: copy delete-forgotten-instances.pl
+    copy: src="{{ files }}/copr/delete-forgotten-instances.pl" dest=/home/copr/delete-forgotten-instances.pl mode=755
+
+  - name: copy delete-forgotten-instances.cron
+    copy: src="{{ files }}/copr/delete-forgotten-instances.cron" dest=/etc/cron.daily/delete-forgotten-instances owner=root group=root mode=755
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+  - name: concate ssl certs
+    shell: "cat /etc/lighttpd/copr-be.fedoraproject.org.key /etc/lighttpd/copr-be.fedoraproject.org.crt > /etc/lighttpd/copr-be.fedoraproject.org.pem"
+    notify:
+    - chmod_key
+    - restart lighttpd
+  - name: chmod_key
+    file: path=/etc/lighttpd/copr-be.fedoraproject.org.pem owner=root group=root mode=0600
+  - name: restart copr-backend
+    service: name=copr-backend state=restarted
+  - name: restart lighttpd
+    service: name=lighttpd state=restarted

playbooks/groups/copr-front.yml

@@ -0,0 +1,118 @@
+- name: check/create instance
+  hosts: copr-front:copr-front-stg
+  user: root
+  gather_facts: False
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+
+  tasks:
+  - include: "{{ tasks }}/persistent_cloud.yml"
+  - include: "{{ tasks }}/growroot_cloud.yml"
+
+- name: provision instance
+  hosts: copr-front:copr-front-stg
+  user: root
+  gather_facts: True
+
+  vars_files: 
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  tasks:
+  - include: "{{ tasks }}/cloud_setup_basic.yml"
+  - include: "{{ tasks }}/postfix_basic.yml"
+
+  - include: "{{ tasks }}/iptables.yml"
+
+  - name: mount up disk of copr fe
+    mount: name=/srv/copr-fe src='LABEL=copr-fe' fstype=ext4 state=mounted
+    when: not devel
+
+  - name: mount up bind mount for postgres
+    mount: src=/srv/copr-fe/pgsqldb name=/var/lib/pgsql fstype=auto opts=bind state=mounted
+    when: not devel
+
+  - name: copy copr.repo
+    copy: src="{{ files }}/copr/fe/yum/copr.repo" dest=/etc/yum.repos.d/copr.repo
+
+  - name: set the hostname
+    shell: hostname "{{ copr-hostname }}"
+
+  - name: copy .forward file
+    copy: src="{{ _forward_src }}" dest=/root/.forward owner=root group=root
+
+  - name: deploy /etc/hosts file
+    copy: src="{{ files }}/copr/hosts" dest=/etc/hosts owner=root group=root mode=644
+
+  - name: install copr-fe pkgs
+    yum: state=latest pkg={{ item }}
+    with_items:
+    - copr-frontend
+    - copr-selinux
+    - postgresql-server
+    - bash-completion
+    - fail2ban
+    - mod_ssl
+    - system-config-firewall-base
+    tags:
+    - packages
+
+  - name: install copr configs
+    template: src="{{ files }}/copr/fe/copr.conf" dest=/etc/copr/copr.conf mode=600
+    notify:
+    - restart httpd
+    tags:
+    - config
+
+  - name: copy apache files to conf.d
+    copy: src="{{ files }}/copr/fe/httpd/{{ item }}" dest="/etc/httpd/conf.d/{{ item }}"
+    with_items:
+    - coprs.conf
+    - welcome.conf
+
+  - name: copy pg_hba.conf
+    copy: src="{{ files }}/copr/fe/pg/pg_hba.conf" dest=/var/lib/pgsql/data/pg_hba.conf owner=postgres group=postgres mode=0600
+
+  - name: copy httpd ssl certificates (crt)
+    copy: src="{{ puppet_private }}/httpd/copr-fe.fedoraproject.org.crt" dest="/etc/pki/tls/certs/" owner=root group=root mode=0600
+    tags:
+    - config
+
+  - name: copy httpd ssl certificates (key)
+    copy: src="{{ puppet_private }}/httpd/copr-fe.fedoraproject.org.key" dest="/etc/pki/tls/private/" owner=root group=root mode=0600
+    tags:
+    - config
+
+  - name: copy httpd ssl certificates (CAcert)
+    copy: src="{{ files }}/copr/DigiCertCA.crt" dest="/etc/pki/tls/certs/" owner=root group=root mode=0600
+    when: not devel
+    tags:
+    - config
+
+  - lineinfile: dest=/etc/httpd/conf.d/coprs.conf regexp="SSLCertificateFile " insertafter="^#SSLCertificateFile " line="SSLCertificateFile /etc/pki/tls/certs/copr-fe.fedoraproject.org.crt"
+    notify:
+    - restart httpd
+
+  - lineinfile: dest=/etc/httpd/conf.d/coprs.conf regexp="SSLCertificateKeyFile " insertafter="^#SSLCertificateKeyFile " line="SSLCertificateKeyFile /etc/pki/tls/private/copr-fe.fedoraproject.org.key"
+    notify:
+    - restart httpd
+
+  - lineinfile: dest=/etc/httpd/conf.d/coprs.conf regexp="SSLCertificateChainFile " insertafter="SSLCertificateKeyFile " line="SSLCertificateChainFile /etc/pki/tls/certs/DigiCertCA.crt"
+    when: not devel
+    notify:
+    - restart httpd
+ 
+  - name: enable services
+    service: state=running enabled=yes name={{ item }}
+    with_items:
+    - httpd
+    - postgresql
+    - fail2ban
+
+  handlers:
+  - name: restart httpd
+    service: name=httpd state=restarted
+  - include: "{{ handlers }}/restart_services.yml"