diff --git a/inventory/group_vars/ipsilon-stg b/inventory/group_vars/ipsilon-stg new file mode 100644 index 0000000000..828c0859ff --- /dev/null +++ b/inventory/group_vars/ipsilon-stg @@ -0,0 +1,15 @@ +--- +# Define resources for this group of hosts here. +lvm_size: 20000 +mem_size: 1024 +num_cpus: 2 + +# for systems that do not match the above - specify the same parameter in +# the host_vars/$hostname file + +tcp_ports: [ 80, 443 ] + +# Neeed for rsync from log01 for logs. +custom_rules: [ '-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT' ] + +fas_client_groups: sysadmin-main,sysadmin-accounts diff --git a/inventory/host_vars/ipsilon01.stg.phx2.fedoraproject.org b/inventory/host_vars/ipsilon01.stg.phx2.fedoraproject.org new file mode 100644 index 0000000000..713c8d9c25 --- /dev/null +++ b/inventory/host_vars/ipsilon01.stg.phx2.fedoraproject.org @@ -0,0 +1,12 @@ +--- +nm: 255.255.255.0 +gw: 10.5.126.254 +dns: 10.5.126.21 + +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-7 +ks_repo: http://10.5.126.23/repo/rhel/RHEL7-x86_64/ + +volgroup: /dev/vg_guests +eth0_ip: 10.5.126.35 +vmhost: virthost11.phx2.fedoraproject.org +datacenter: phx2 diff --git a/inventory/inventory b/inventory/inventory index 06f19dc649..b2b9343f8e 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -293,6 +293,9 @@ log01.phx2.fedoraproject.org noc01.phx2.fedoraproject.org noc02.fedoraproject.org +[ipsilon-stg] +ipsilon01.stg.phx2.fedoraproject.org + [dhcp] dhcp01.phx2.fedoraproject.org @@ -466,6 +469,7 @@ hotness01.stg.phx2.fedoraproject.org kerneltest01.stg.phx2.fedoraproject.org koji01.stg.phx2.fedoraproject.org mailman01.stg.phx2.fedoraproject.org +ipsilon01.stg.phx2.fedoraproject.org notifs-backend01.stg.phx2.fedoraproject.org notifs-web01.stg.phx2.fedoraproject.org notifs-web02.stg.phx2.fedoraproject.org @@ -501,6 +505,8 @@ log01.phx2.fedoraproject.org kojipkgs01.phx2.fedoraproject.org ns03.phx2.fedoraproject.org ns04.phx2.fedoraproject.org +db-qa01.qa.fedoraproject.org +proxy10.phx2.fedoraproject.org [summershum] summershum01.phx2.fedoraproject.org diff --git a/playbooks/groups/ipsilon.yml b/playbooks/groups/ipsilon.yml new file mode 100644 index 0000000000..11690c23aa --- /dev/null +++ b/playbooks/groups/ipsilon.yml @@ -0,0 +1,68 @@ +# create a new FedOAuth server +# NOTE: should be used with --limit most of the time +# NOTE: make sure there is room/space for this server on the vmhost +# NOTE: most of these vars_path come from group_vars/ipsilon* or from hostvars + +- name: make ipsilon + hosts: ipsilon-stg + user: root + gather_facts: False + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + tasks: + - include: "{{ tasks }}/virt_instance_create.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: make the box be real + hosts: ipsilon-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - denyhosts + - nagios_client + - hosts + - fas_client + - rsyncd + - sudo + - { role: openvpn/client, + when: env != "staging" } + + tasks: + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + - include: "{{ tasks }}/apache.yml" + - include: "{{ tasks }}/mod_wsgi.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: deploy ipsilon itself + hosts: ipsilon-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - "{{ vars_path }}/{{ ansible_distribution }}.yml" + + roles: + - ipsilon + + handlers: + - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/anitya/backend/files/backup-database b/roles/anitya/backend/files/backup-database index 872f359e41..3f6e7d8fb1 100644 --- a/roles/anitya/backend/files/backup-database +++ b/roles/anitya/backend/files/backup-database @@ -3,4 +3,8 @@ DB=$1 +# Make our latest backup /usr/bin/pg_dump -C $DB | /usr/bin/xz > /backups/$DB-$(date +%F).dump.xz + +# Also, delete the backup from a few days ago. +rm -f /backups/$DB-$(date --date="3 days ago" +%F).dump.xz diff --git a/roles/anitya/backend/tasks/main.yml b/roles/anitya/backend/tasks/main.yml index ca3d3076de..a414202b82 100644 --- a/roles/anitya/backend/tasks/main.yml +++ b/roles/anitya/backend/tasks/main.yml @@ -45,6 +45,16 @@ tags: - anitya_backend +- name: Set up some cronjobs to backup databases as configured + template: > + src=cron-backup-database + dest=/etc/cron.d/cron-backup-database-{{ item }} + with_items: + - anitya + tags: + - cron + - postgresql + - name: Add our postgres config file. copy: > src={{ item }} diff --git a/roles/fas_server/files/export-bugzilla b/roles/fas_server/files/export-bugzilla new file mode 100644 index 0000000000..5bec117915 --- /dev/null +++ b/roles/fas_server/files/export-bugzilla @@ -0,0 +1,153 @@ +#!/usr/bin/python -t +__requires__ = 'TurboGears' +import pkg_resources +pkg_resources.require('CherryPy >= 2.0, < 3.0alpha') + +import logging +logging.basicConfig() + +import os +import sys +import getopt +import xmlrpclib +import smtplib +from email.Message import Message +import warnings + +# Ignore DeprecationWarnings. This allows us to stop getting email +# from the cron job. We'll see the same warnings from the server starting up +warnings.simplefilter('ignore', DeprecationWarning) + +import turbogears +import bugzilla +from turbogears import config +cfgfile = '/etc/export-bugzilla.cfg' +if os.access('./export-bugzilla.cfg', os.R_OK): + cfgfile = './export-bugzilla.cfg' +turbogears.update_config(configfile=cfgfile) +from turbogears.database import session +from fas.model import BugzillaQueue + +BZSERVER = config.get('bugzilla.url', 'https://bugdev.devel.redhat.com/bugzilla-cvs/xmlrpc.cgi') +BZUSER = config.get('bugzilla.username') +BZPASS = config.get('bugzilla.password') +MAILSERVER = config.get('mail.server', 'localhost') +ADMINEMAIL = config.get('mail.admin_email', 'admin@fedoraproject.org') +NOTIFYEMAIL = config.get('mail.notify_email', ['admin@fedoraproject.org']) + +if __name__ == '__main__': + opts, args = getopt.getopt(sys.argv[1:], '', ('usage', 'help')) + if len(args) != 2 or ('--usage','') in opts or ('--help','') in opts: + print """ + Usage: export-bugzilla.py GROUP BUGZILLA_GROUP + """ + sys.exit(1) + ourGroup = args[0] + bzGroup = args[1] + + server = bugzilla.Bugzilla(url=BZSERVER, user=BZUSER, password=BZPASS, + cookiefile=None, tokenfile=None) + bugzilla_queue = BugzillaQueue.query.join('group').filter_by( + name=ourGroup) + + no_bz_account = [] + for entry in bugzilla_queue: + # Make sure we have a record for this user in bugzilla + if entry.action == 'r': + # Remove the user's bugzilla group + try: + server.updateperms(entry.email, 'rem', bzGroup) + except xmlrpclib.Fault, e: + if e.faultCode == 51: + # It's okay, not having this user is equivalent to setting + # them to not have this group. + pass + else: + raise + + elif entry.action == 'a': + # Make sure the user exists + try: + server.getuser(entry.email) + except xmlrpclib.Fault, e: + if e.faultCode == 51: + # This user doesn't have a bugzilla account yet + # add them to a list and we'll let them know. + no_bz_account.append(entry) + continue + else: + print 'Error:', e, entry.email, entry.person.human_name + raise + server.updateperms(entry.email, 'add', bzGroup) + else: + print 'Unrecognized action code: %s %s %s %s %s' % (entry.action, + entry.email, entry.person.human_name, entry.person.username, entry.group.name) + continue + + # Remove them from the queue + session.delete(entry) + session.flush() + +# Mail the people without bugzilla accounts + if '$USER' in NOTIFYEMAIL: + for person in no_bz_account: + smtplib.SMTP(MAILSERVER) + msg = Message() + message = '''Hello %(name)s, + + As a Fedora packager, we grant you permissions to make changes to bugs in + bugzilla to all Fedora bugs. This lets you work together with other Fedora + developers in an easier fashion. However, to enable this functionality, we + need to have your bugzilla email address stored in the Fedora Account System. + At the moment you have: + + %(email)s + + which bugzilla is telling us is not an account in bugzilla. If you could + please set up an account in bugzilla with this address or change your email + address on your Fedora Account to match an existing bugzilla account this would + let us go forward. + + Note: this message is being generated by an automated script. You'll continue + getting this message until the problem is resolved. Sorry for the + inconvenience. + + Thank you, + The Fedora Account System + %(admin_email)s + ''' % {'name': person.person.human_name, 'email': person.email, + 'admin_email': ADMINEMAIL} + + msg.add_header('To', person.email) + msg.add_header('From', ADMINEMAIL) + msg.add_header('Subject', 'Fedora Account System and Bugzilla Mismatch') + msg.set_payload(message) + smtp = smtplib.SMTP(MAILSERVER) + smtp.sendmail(ADMINEMAIL, [person.email], msg.as_string()) + smtp.quit() + recipients = [e for e in NOTIFYEMAIL if e != '$USER'] + if recipients and no_bz_account: + smtplib.SMTP(MAILSERVER) + msg = Message() + people = [] + for person in no_bz_account: + if person.person.status == 'Active': + people.append(' %(user)s -- %(name)s -- %(email)s' % + {'name': person.person.human_name, 'email': person.email, + 'user': person.person.username}) + if people: + people = '\n'.join(people) + message = ''' +The following people are in the packager group but do not have email addresses +that are valid in bugzilla: +%s + +''' % people + + msg.add_header('From', ADMINEMAIL) + msg.add_header('To', ', '.join(recipients)) + msg.add_header('Subject', 'Fedora Account System and Bugzilla Mismatch') + msg.set_payload(message) + smtp = smtplib.SMTP(MAILSERVER) + smtp.sendmail(ADMINEMAIL, recipients, msg.as_string()) + smtp.quit() diff --git a/roles/fas_server/tasks/main.yml b/roles/fas_server/tasks/main.yml index d26ea73e52..eecc8584d4 100644 --- a/roles/fas_server/tasks/main.yml +++ b/roles/fas_server/tasks/main.yml @@ -288,6 +288,19 @@ tags: - config + +- name: HOTFIX fix the export-bugzilla cron to not store bugzilla token + copy: > + src="export-bugzilla" + dest="/usr/sbin/export-bugzilla" + owner=root + group=root + mode=0755 + tags: + - config + - hotfix + + - name: run export-bugzilla program cron: > name="export-bugzilla" diff --git a/roles/hosts/files/taskotron-stg01.qa.fedoraproject.org-hosts b/roles/hosts/files/taskotron-stg01.qa.fedoraproject.org-hosts new file mode 100644 index 0000000000..3ef49c5af5 --- /dev/null +++ b/roles/hosts/files/taskotron-stg01.qa.fedoraproject.org-hosts @@ -0,0 +1,22 @@ +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +10.5.125.63 koji.fedoraproject.org +10.5.125.36 kojipkgs.fedoraproject.org +10.5.126.23 infrastructure.fedoraproject.org +10.5.125.44 pkgs.fedoraproject.org pkgs +# +# This is proxy01.phx2.fedoraproject.org +# +10.5.126.51 mirrors.fedoraproject.org +10.5.126.51 admin.fedoraproject.org +# there are some firewall issues ATM that make this IP for hub not work +# changing to a public IP as a workaround until firewall rules are changed +# 10.5.126.51 hub.fedoraproject.org +152.19.134.142 hub.fedoraproject.org + +# for VPN +10.5.126.12 gateway.phx2.fedoraproject.org gateway bastion01.phx2.fedoraproject.org bastion01 +10.5.126.11 bastion02.phx2.fedoraproject.org bastion02 + +# these are needed for the beaker-project repos +74.207.232.43 beaker-project.org diff --git a/roles/ipsilon/files/templates/index.html b/roles/ipsilon/files/templates/index.html new file mode 100644 index 0000000000..d217b7a6c9 --- /dev/null +++ b/roles/ipsilon/files/templates/index.html @@ -0,0 +1,7 @@ +{% extends "layout.html" %} +{% block toptext %} +This is the Federated Open Authentication provider homepage. +{% endblock %} +{% block main %} +You will be redirected to this application whenever another application requires you to authenticate. +{% endblock %} diff --git a/roles/ipsilon/files/templates/internalerror.html b/roles/ipsilon/files/templates/internalerror.html new file mode 100644 index 0000000000..055e903062 --- /dev/null +++ b/roles/ipsilon/files/templates/internalerror.html @@ -0,0 +1,15 @@ +{% extends "layout.html" %} +{% block toptext %} +500 - Internal Server Error +{% endblock %} +{% block main %} + {% if message: %} +
{{ message }}
+ {% else %} +Ipsilon encountered an unexpected internal error while trying to + fulfill your request.
+ {% endif %} +Please retry again.
+If the error persists, contact the server administrator to resolve + the problem.
+{% endblock %} diff --git a/roles/ipsilon/files/templates/layout.html b/roles/ipsilon/files/templates/layout.html new file mode 100644 index 0000000000..cdb8e06bdb --- /dev/null +++ b/roles/ipsilon/files/templates/layout.html @@ -0,0 +1,35 @@ + + + + +{% block toptext %}{% endblock %}
+Something prevented a successful logout
+You are still logged in as {{ user.fullname }}
+{% else %} +Successfully logged out.
+Return to Home page
+{% endif %} +{% endblock %} diff --git a/roles/ipsilon/files/templates/notfound.html b/roles/ipsilon/files/templates/notfound.html new file mode 100644 index 0000000000..38203cb0ed --- /dev/null +++ b/roles/ipsilon/files/templates/notfound.html @@ -0,0 +1,13 @@ +{% extends "layout.html" %} +{% block toptext %} +404 - Not Found +{% endblock %} +{% block main %} + {% if message: %} +{{ message }}
+ {% else %} +This page does not exist.
+ {% endif %} +If you think this is an error, contact the server administrator to resolve + the problem.
+{% endblock %} diff --git a/roles/ipsilon/files/templates/openid/consent_form.html b/roles/ipsilon/files/templates/openid/consent_form.html new file mode 100644 index 0000000000..102d7e4913 --- /dev/null +++ b/roles/ipsilon/files/templates/openid/consent_form.html @@ -0,0 +1,36 @@ +{% extends "layout.html" %} +{% block toptext %} +{{trustroot}} is asking to authenticate via OpenID using FedOAuth +{% endblock %} +{% block main %} + Review the authorization details +{{ message }}
+ {% else %} +Authentication was not succesful
+ {% endif %} + +{% endblock %} diff --git a/roles/ipsilon/tasks/main.yml b/roles/ipsilon/tasks/main.yml new file mode 100644 index 0000000000..c80d42f0b2 --- /dev/null +++ b/roles/ipsilon/tasks/main.yml @@ -0,0 +1,56 @@ +--- +# Configuration for the ipsilon webapp + +- name: clean yum metadata + command: yum clean all + tags: + - packages + +- name: install needed packages + yum: pkg={{ item }} state=present + with_items: + - ipsilon + - ipsilon-authfas + - ipsilon-openid + - ipsilon-persona + - python-psycopg2 + - libsemanage-python + tags: + - packages + +- name: copy ipsilon templates + copy: src=templates + dest=/usr/share/ipsilon/templates-fedora + owner=ipsilon group=ipsilon mode=0666 + +- name: copy ipsilon configuration + template: src={{ item }}.cfg + dest=/etc/ipsilon/{{ item }}.cfg + owner=ipsilon group=ipsilon mode=0600 + with_items: + - ipsilon + - configuration + tags: + - config + notify: + - restart apache + +- name: copy persona private key + copy: src={{ private }}/files/ipsilon/persona.key dest=/etc/ipsilon/persona.key + owner=ipsilon group=ipsilon mode=0600 + when: env != "staging" + +- name: copy persona STG private key + copy: src={{ private }}/files/ipsilon/persona.stg.key dest=/etc/ipsilon/persona.stg.key + owner=ipsilon group=ipsilon mode=0600 + when: env == "staging" + +- name: set sebooleans so ipsilon can talk to the db + action: seboolean name=httpd_can_network_connect_db + state=true + persistent=true + +- name: apply selinux type to the wsgi file + file: > + dest=/usr/sbin/ipsilon + setype=httpd_sys_content_t diff --git a/roles/ipsilon/templates/configuration.cfg b/roles/ipsilon/templates/configuration.cfg new file mode 100644 index 0000000000..cae974a9e1 --- /dev/null +++ b/roles/ipsilon/templates/configuration.cfg @@ -0,0 +1,40 @@ +[login_config] +global enabled=fas +{% if env == 'staging' %} +fas FAS url=https://admin.stg.fedoraproject.org/accounts/ +{% else %} +fas FAS url=https://admin.fedoraproject.org/accounts/ +{% endif %} +fas FAS Proxy client user Agent=Fedora Ipsilon +fas FAS Insecure Auth=False + + + + +[provider_config] +global enabled=persona,openid + +{% if env == 'staging' %} +persona allowed domains=stg.fedoraproject.org +persona issuer domain=id.stg.fedoraproject.org +persona idp key file=/etc/ipsilon/persona.stg.key +{% else %} +persona allowed domains=fedoraproject.org +persona issuer domain=id.fedoraproject.org +persona idp key file=/etc/ipsilon/persona.key +{% endif %} + +{% if env == 'staging' %} +openid database url="postgresql://{{ ipsilon_db_user }}:{{ ipsilon_db_pass }}@{{ ipsilon_db_host }}.stg/{{ ipsilon_db_name }}" +openid endpoint url=https://id.stg.fedoraproject.org/openid/ +openid identity url template=http://localhost/openid/id/%(username)s/ +openid trusted roots= +{% else %} +openid database url="postgresql://{{ ipsilon_db_user }}:{{ ipsilon_db_pass }}@{{ ipsilon_db_host }}/{{ ipsilon_db_name }}" +openid endpoint url=https://id.fedoraproject.org/openid/ +openid identity url template=http://%(username)s.id.fedoraproject.org/ +openid trusted roots=http://jenkins.cloud.fedoraproject.org/securityRealm/finishLogin,https://ask.fedoraproject.org/,https://fedorahosted.org/,https://badges.fedoraproject.org,https://apps.fedoraproject.org/tagger/,https://apps.fedoraproject.org/nuancier/,https://apps.fedoraproject.org/datagrepper/,https://apps.fedoraproject.org/calendar/,http://apps.fedoraproject.org/notifications/,http://copr.fedoraproject.org/,http://copr-fe.cloud.fedoraproject.org/,https://admin.fedoraproject.org/pkgdb/,https://admin.fedoraproject.org/voting/,https://apps.fedoraproject.org/github2fedmsg,https://admin.fedoraproject.org,https://apps.fedoraproject.org/,https://release-monitoring.org/ +{% endif %} +openid untrusted roots= +openid enabled extensions=Teams,Attribute Exchange,CLAs,Simple Registration + diff --git a/roles/ipsilon/templates/ipsilon.cfg b/roles/ipsilon/templates/ipsilon.cfg new file mode 100644 index 0000000000..f2ad40818a --- /dev/null +++ b/roles/ipsilon/templates/ipsilon.cfg @@ -0,0 +1,22 @@ +[global] +debug = False +tools.log_request_response.on = False +template_dir = "/srv/ipsilon/templates" + +log.screen = False +base.mount = "" +base.dir = "/usr/share/ipsilon" +admin.config.db = "configfile:///etc/ipsilon/configuration.cfg" +user.prefs.db = "configfile:///etc/ipsilon/configuration.cfg" +{% if env == 'staging' %} +transactions.db = "postgresql://{{ ipsilon_db_user }}:{{ ipsilon_db_pass }}@{{ ipsilon_db_host }}.stg/{{ ipsilon_db_name }}" +{% else %} +transactions.db = "postgresql://{{ ipsilon_db_user }}:{{ ipsilon_db_pass }}@{{ ipsilon_db_host }}/{{ ipsilon_db_name }}" +{% endif %} + +tools.sessions.on = True +tools.sessions.name = "fedora_ipsilon_session_id" +tools.sessions.storage_type = "Sql" +tools.sessions.timeout = 60 +tools.sessions.httponly = True +tools.sessions.secure = True diff --git a/scripts/public-db-copy b/scripts/public-db-copy index 2c96cf1cb0..bae78592be 100755 --- a/scripts/public-db-copy +++ b/scripts/public-db-copy @@ -8,3 +8,4 @@ scp db-datanommer02:/backups/datanommer-$(date +%F).dump.xz /srv/web/infra/db-du scp db-koji01:/backups/koji-$(date +%F).dump.xz /srv/web/infra/db-dumps/koji.dump.xz scp db01:/backups/pkgdb2-$(date +%F).dump.xz /srv/web/infra/db-dumps/pkgdb2.dump.xz scp db01:/backups/fedoratagger-$(date +%F).dump.xz /srv/web/infra/db-dumps/fedoratagger.dump.xz +scp anitya-backend01:/backups/anitya-$(date +%F).dump.xz /srv/web/infra/db-dumps/anitya.dump.xz