From d903e76839183b0d68b245d4b2b8eba1130f8112 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 19 Feb 2014 17:25:50 +0000 Subject: [PATCH] A stab at summershum deployment. --- .../summershum01.phx2.fedoraproject.org-hosts | 5 ++ ...mershum01.stg.phx2.fedoraproject.org-hosts | 11 +++ inventory/group_vars/summershum | 21 ++++++ inventory/group_vars/summershum-stg | 21 ++++++ .../summershum01.phx2.fedoraproject.org | 14 ++++ .../summershum01.stg.phx2.fedoraproject.org | 14 ++++ inventory/inventory | 7 ++ playbooks/groups/summershum.yml | 69 +++++++++++++++++++ roles/fedmsg_base/tasks/main.yml | 2 + .../templates/endpoints-summershum.py.j2 | 13 ++++ roles/fedmsg_base/templates/ssl.py.j2 | 3 + roles/summershum/files/patched-fedmsg-hub | 16 +++++ roles/summershum/tasks/main.yml | 37 ++++++++++ roles/summershum/templates/summershum.py | 17 +++++ 14 files changed, 250 insertions(+) create mode 100644 files/hosts/summershum01.phx2.fedoraproject.org-hosts create mode 100644 files/hosts/summershum01.stg.phx2.fedoraproject.org-hosts create mode 100644 inventory/group_vars/summershum create mode 100644 inventory/group_vars/summershum-stg create mode 100644 inventory/host_vars/summershum01.phx2.fedoraproject.org create mode 100644 inventory/host_vars/summershum01.stg.phx2.fedoraproject.org create mode 100644 playbooks/groups/summershum.yml create mode 100644 roles/fedmsg_base/templates/endpoints-summershum.py.j2 create mode 100644 roles/summershum/files/patched-fedmsg-hub create mode 100644 roles/summershum/tasks/main.yml create mode 100644 roles/summershum/templates/summershum.py diff --git a/files/hosts/summershum01.phx2.fedoraproject.org-hosts b/files/hosts/summershum01.phx2.fedoraproject.org-hosts new file mode 100644 index 0000000000..fb61f1cdab --- /dev/null +++ b/files/hosts/summershum01.phx2.fedoraproject.org-hosts @@ -0,0 +1,5 @@ +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +10.5.126.52 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 fedoraproject.org +10.5.126.23 infrastructure.fedoraproject.org +10.5.126.71 db-summershum db-summershum diff --git a/files/hosts/summershum01.stg.phx2.fedoraproject.org-hosts b/files/hosts/summershum01.stg.phx2.fedoraproject.org-hosts new file mode 100644 index 0000000000..6c26f3e8c7 --- /dev/null +++ b/files/hosts/summershum01.stg.phx2.fedoraproject.org-hosts @@ -0,0 +1,11 @@ +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +10.5.126.89 admin.fedoraproject.org +10.5.126.88 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 fedoraproject.org +10.5.126.86 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all +10.5.126.23 infrastructure.fedoraproject.org + +10.5.126.81 memcached03 memcached03.stg app01 app01.stg + +10.5.126.85 db-summershum db-summershum diff --git a/inventory/group_vars/summershum b/inventory/group_vars/summershum new file mode 100644 index 0000000000..b6a60f2fd7 --- /dev/null +++ b/inventory/group_vars/summershum @@ -0,0 +1,21 @@ +--- +# Define resources for this group of hosts here. +lvm_size: 20000 +mem_size: 1024 +num_cpus: 2 + +# for systems that do not match the above - specify the same parameter in +# the host_vars/$hostname file + +tcp_ports: [ 3000 ] + +fas_client_groups: sysadmin-noc,sysadmin-badges + +# These are consumed by a task in roles/fedmsg_base/main.yml +fedmsg_certs: +- service: shell + owner: root + group: sysadmin +- service: summershum + owner: root + group: fedmsg diff --git a/inventory/group_vars/summershum-stg b/inventory/group_vars/summershum-stg new file mode 100644 index 0000000000..b6a60f2fd7 --- /dev/null +++ b/inventory/group_vars/summershum-stg @@ -0,0 +1,21 @@ +--- +# Define resources for this group of hosts here. +lvm_size: 20000 +mem_size: 1024 +num_cpus: 2 + +# for systems that do not match the above - specify the same parameter in +# the host_vars/$hostname file + +tcp_ports: [ 3000 ] + +fas_client_groups: sysadmin-noc,sysadmin-badges + +# These are consumed by a task in roles/fedmsg_base/main.yml +fedmsg_certs: +- service: shell + owner: root + group: sysadmin +- service: summershum + owner: root + group: fedmsg diff --git a/inventory/host_vars/summershum01.phx2.fedoraproject.org b/inventory/host_vars/summershum01.phx2.fedoraproject.org new file mode 100644 index 0000000000..bda4da4161 --- /dev/null +++ b/inventory/host_vars/summershum01.phx2.fedoraproject.org @@ -0,0 +1,14 @@ +--- +nm: 255.255.255.0 +gw: 10.5.126.254 +dns: 10.5.126.21 + +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-6 +ks_repo: http://10.5.126.23/repo/rhel/RHEL6-x86_64/ + +eth0_ip: 10.5.126.205 + +volgroup: /dev/vg_virthost01 +vmhost: virthost01.phx2.fedoraproject.org + +datacenter: phx2 diff --git a/inventory/host_vars/summershum01.stg.phx2.fedoraproject.org b/inventory/host_vars/summershum01.stg.phx2.fedoraproject.org new file mode 100644 index 0000000000..1f7f645022 --- /dev/null +++ b/inventory/host_vars/summershum01.stg.phx2.fedoraproject.org @@ -0,0 +1,14 @@ +--- +nm: 255.255.255.0 +gw: 10.5.126.254 +dns: 10.5.126.21 + +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-6 +ks_repo: http://10.5.126.23/repo/rhel/RHEL6-x86_64/ + +eth0_ip: 10.5.126.209 + +volgroup: /dev/vg_virthost10 +vmhost: virthost10.phx2.fedoraproject.org + +datacenter: phx2 diff --git a/inventory/inventory b/inventory/inventory index 09b4585d76..abe0a2ff11 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -379,8 +379,15 @@ paste01.stg.phx2.fedoraproject.org pkgs01.stg.phx2.fedoraproject.org proxy01.stg.phx2.fedoraproject.org releng01.stg.phx2.fedoraproject.org +summershum01.stg.phx2.fedoraproject.org value01.stg.phx2.fedoraproject.org +[summershum] +summershum01.phx2.fedoraproject.org + +[summershum-stg] +summershum01.stg.phx2.fedoraproject.org + [taskotron] taskotron-dev01.qa.fedoraproject.org diff --git a/playbooks/groups/summershum.yml b/playbooks/groups/summershum.yml new file mode 100644 index 0000000000..f4f2f8a193 --- /dev/null +++ b/playbooks/groups/summershum.yml @@ -0,0 +1,69 @@ +# create a new summershum server +# NOTE: should be used with --limit most of the time +# NOTE: make sure there is room/space for this server on the vmhost +# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars + +- name: make summershum server + hosts: summershum;summershum-stg + user: root + gather_facts: False + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + tasks: + - include: "{{ tasks }}/virt_instance_create.yml" + - include: "{{ tasks }}/accelerate_prep.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: dole out the generic configuration + hosts: summershum;summershum-stg + user: root + gather_facts: True + accelerate: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - denyhosts + - nagios_client + - fas_client + - fedmsg_base + + tasks: + - include: "{{ tasks }}/hosts.yml" + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + - include: "{{ tasks }}/sudo.yml" + # The proxies don't actually need to talk to these hosts so we won't bother + # putting them on the vpn. + #- include: "{{ tasks }}/openvpn_client.yml" + # when: env != "staging" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: dole out the service-specific config + hosts: summershum;summershum-stg + user: root + gather_facts: True + accelerate: True + + roles: + - fedmsg-hub + - summershum + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml diff --git a/roles/fedmsg_base/tasks/main.yml b/roles/fedmsg_base/tasks/main.yml index 11afc58fc7..35a9965168 100644 --- a/roles/fedmsg_base/tasks/main.yml +++ b/roles/fedmsg_base/tasks/main.yml @@ -6,6 +6,7 @@ yum: pkg={{ item }} state=installed with_items: - fedmsg + - libsemanage-python tags: - packages @@ -23,6 +24,7 @@ - endpoints-fedbadges.py - endpoints-nuancier.py - endpoints-mailman.py + - endpoints-summershum.py - relay.py - pkgdb.py - logging.py diff --git a/roles/fedmsg_base/templates/endpoints-summershum.py.j2 b/roles/fedmsg_base/templates/endpoints-summershum.py.j2 new file mode 100644 index 0000000000..10fb80fcda --- /dev/null +++ b/roles/fedmsg_base/templates/endpoints-summershum.py.j2 @@ -0,0 +1,13 @@ +{% if env == 'staging' %} +suffix = 'stg.phx2.fedoraproject.org' +{% else %} +suffix = 'phx2.fedoraproject.org' +{% endif %} + +config = dict( + endpoints={ + "summershum.summershum01": [ + "tcp://summershum01.%s:3000" % suffix, + ], + }, +) diff --git a/roles/fedmsg_base/templates/ssl.py.j2 b/roles/fedmsg_base/templates/ssl.py.j2 index f3455dc9c1..1ddd08a2d9 100644 --- a/roles/fedmsg_base/templates/ssl.py.j2 +++ b/roles/fedmsg_base/templates/ssl.py.j2 @@ -119,6 +119,9 @@ config = dict( ("fedbadges.badges-backend01", "fedbadges-badges-backend01.%s" % suffix), ("shell.badges-backend01", "shell-badges-backend01.%s" % suffix), + ("summershum.summershum01", "summershum-summershum01.%s" % suffix), + ("shell.summershum01", "shell-summershum01.%s" % suffix), + ("tahrir.badges-web01", "tahrir-badges-web01.%s" % suffix), ("shell.badges-web01", "shell-badges-web01.%s" % suffix), ("tahrir.badges-web02", "tahrir-badges-web02.%s" % suffix), diff --git a/roles/summershum/files/patched-fedmsg-hub b/roles/summershum/files/patched-fedmsg-hub new file mode 100644 index 0000000000..e9e305bbab --- /dev/null +++ b/roles/summershum/files/patched-fedmsg-hub @@ -0,0 +1,16 @@ +#!/usr/bin/python +# This file is managed by ansible. +# +# Its like "permanent hotfix" so that the fedmsg-hub loads +# the forward-compat sqlalchemy. +# + +__requires__ = ['fedmsg', "sqlalchemy>=0.8"] +import sys +from pkg_resources import load_entry_point + +if __name__ == '__main__': + sys.exit( + load_entry_point('fedmsg', 'console_scripts', 'fedmsg-hub')() + ) + diff --git a/roles/summershum/tasks/main.yml b/roles/summershum/tasks/main.yml new file mode 100644 index 0000000000..8e1c3fe40b --- /dev/null +++ b/roles/summershum/tasks/main.yml @@ -0,0 +1,37 @@ +--- +# Configuration for the summershum consumer + +- name: install needed packages + yum: pkg={{ item }} state=installed + with_items: + - python-summershum + - python-psycopg2 + +- name: ensure summershum has a place to unpack files + file: > + dest=/var/cache/summershum + state=directory + mode=0755 + owner=fedmsg + group=fedmsg + +- name: copy database configuration + template: > + src={{ item }} dest=/etc/fedmsg.d/{{ item }} + owner=fedmsg group=fedmsg mode=0600 + with_items: + - summershum.py + notify: + - restart fedmsg-hub + +# Here I'm "permanently" hotfixing the fedmsg-hub +# It needs to load the forward-compat sqlalchemy0.7 package from the get-go. +- name: copy over patched fedmsg-hub + copy: > + src=patched-fedmsg-hub dest=/usr/bin/fedmsg-hub + owner=root group=root mode=0755 + tags: + - patches + - hotfix + notify: + - restart fedmsg-hub diff --git a/roles/summershum/templates/summershum.py b/roles/summershum/templates/summershum.py new file mode 100644 index 0000000000..716e10b965 --- /dev/null +++ b/roles/summershum/templates/summershum.py @@ -0,0 +1,17 @@ +config = { + # This is just a key to tell the fedmsg-hub to initialize us. + 'summershum.enabled': True, + 'summershum.sqlalchemy.url': 'postgresql://{{summershum_db_user}}:{{summershum_db_password}}@db-summershum/summershum', + 'summershum.lookaside': 'http://pkgs.fedoraproject.org/lookaside/pkgs/', + 'summershum.datagrepper': 'https://apps.fedoraproject.org/datagrepper/', + + 'logging': { + 'loggers': { + 'summershum': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False + }, + }, + }, +}