adding ufmonitor role for upstreamfirst.fic.o

inventory/host_vars/upstreamfirst.fedorainfracloud.org

@@ -30,6 +30,8 @@ cloud_networks:
 ############################################################
 
 tcp_ports: [ 22, 25, 80, 443, 9418,
+    # this is used for postgres access from docker
+    5432,
     # Used for the eventsource server
     8088,
     # This is for the pagure public fedmsg relay

playbooks/hosts/upstreamfirst.fedorainfracloud.org.yml

@@ -71,3 +71,19 @@
 
   handlers:
   - include: "{{ handlers_path }}/restart_services.yml"
+
+- name: deploy ufmonitor
+  hosts: upstreamfirst.fedorainfracloud.org
+  user: root
+  gather_facts: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - "{{ vars_path }}/{{ ansible_distribution }}.yml"
+
+  roles:
+      - { role: ufmonitor, tags: ['ufmonitor'] }
+
+  handlers:
+  - include: "{{ handlers_path }}/restart_services.yml"

roles/ufmonitor/defaults/main.yml

@@ -0,0 +1,7 @@
+apiserver_container_image: "docker.io/fedoraqa/ufmonitor-apiserver:latest"
+updater_container_image: "docker.io/fedoraqa/ufmonitor-updater:latest"
+ufmonitor_db_name: "ufmonitor"
+ufmonitor_db_user: "ufmonitor"
+ufmonitor_db_password: "terribleinsecurepassword"
+ufmonitor_config_path: "/etc/sysconfig/ufmonitor"
+ufmonitor_home: "/var/www/ufmonitor"

roles/ufmonitor/files/media/css/c3.css

@@ -0,0 +1,167 @@
+/*-- Chart --*/
+.c3 svg {
+  font: 10px sans-serif;
+  -webkit-tap-highlight-color: transparent; }
+
+.c3 path, .c3 line {
+  fill: none;
+  stroke: #000; }
+
+.c3 text {
+  -webkit-user-select: none;
+  -moz-user-select: none;
+  user-select: none; }
+
+.c3-legend-item-tile,
+.c3-xgrid-focus,
+.c3-ygrid,
+.c3-event-rect,
+.c3-bars path {
+  shape-rendering: crispEdges; }
+
+.c3-chart-arc path {
+  stroke: #fff; }
+
+.c3-chart-arc text {
+  fill: #fff;
+  font-size: 13px; }
+
+/*-- Axis --*/
+/*-- Grid --*/
+.c3-grid line {
+  stroke: #aaa; }
+
+.c3-grid text {
+  fill: #aaa; }
+
+.c3-xgrid, .c3-ygrid {
+  stroke-dasharray: 3 3; }
+
+/*-- Text on Chart --*/
+.c3-text.c3-empty {
+  fill: #808080;
+  font-size: 2em; }
+
+/*-- Line --*/
+.c3-line {
+  stroke-width: 1px; }
+
+/*-- Point --*/
+.c3-circle._expanded_ {
+  stroke-width: 1px;
+  stroke: white; }
+
+.c3-selected-circle {
+  fill: white;
+  stroke-width: 2px; }
+
+/*-- Bar --*/
+.c3-bar {
+  stroke-width: 0; }
+
+.c3-bar._expanded_ {
+  fill-opacity: 0.75; }
+
+/*-- Focus --*/
+.c3-target.c3-focused {
+  opacity: 1; }
+
+.c3-target.c3-focused path.c3-line, .c3-target.c3-focused path.c3-step {
+  stroke-width: 2px; }
+
+.c3-target.c3-defocused {
+  opacity: 0.3 !important; }
+
+/*-- Region --*/
+.c3-region {
+  fill: steelblue;
+  fill-opacity: .1; }
+
+/*-- Brush --*/
+.c3-brush .extent {
+  fill-opacity: .1; }
+
+/*-- Select - Drag --*/
+/*-- Legend --*/
+.c3-legend-item {
+  font-size: 12px; }
+
+.c3-legend-item-hidden {
+  opacity: 0.15; }
+
+.c3-legend-background {
+  opacity: 0.75;
+  fill: white;
+  stroke: lightgray;
+  stroke-width: 1; }
+
+/*-- Title --*/
+.c3-title {
+  font: 14px sans-serif; }
+
+/*-- Tooltip --*/
+.c3-tooltip-container {
+  z-index: 10; }
+
+.c3-tooltip {
+  border-collapse: collapse;
+  border-spacing: 0;
+  background-color: #fff;
+  empty-cells: show;
+  -webkit-box-shadow: 7px 7px 12px -9px #777777;
+  -moz-box-shadow: 7px 7px 12px -9px #777777;
+  box-shadow: 7px 7px 12px -9px #777777;
+  opacity: 0.9; }
+
+.c3-tooltip tr {
+  border: 1px solid #CCC; }
+
+.c3-tooltip th {
+  background-color: #aaa;
+  font-size: 14px;
+  padding: 2px 5px;
+  text-align: left;
+  color: #FFF; }
+
+.c3-tooltip td {
+  font-size: 13px;
+  padding: 3px 6px;
+  background-color: #fff;
+  border-left: 1px dotted #999; }
+
+.c3-tooltip td > span {
+  display: inline-block;
+  width: 10px;
+  height: 10px;
+  margin-right: 6px; }
+
+.c3-tooltip td.value {
+  text-align: right; }
+
+/*-- Area --*/
+.c3-area {
+  stroke-width: 0;
+  opacity: 0.2; }
+
+/*-- Arc --*/
+.c3-chart-arcs-title {
+  dominant-baseline: middle;
+  font-size: 1.3em; }
+
+.c3-chart-arcs .c3-chart-arcs-background {
+  fill: #e0e0e0;
+  stroke: none; }
+
+.c3-chart-arcs .c3-chart-arcs-gauge-unit {
+  fill: #000;
+  font-size: 16px; }
+
+.c3-chart-arcs .c3-chart-arcs-gauge-max {
+  fill: #777; }
+
+.c3-chart-arcs .c3-chart-arcs-gauge-min {
+  fill: #777; }
+
+.c3-chart-arc .c3-gauge-value {
+  fill: #000;
+  /*  font-size: 28px !important;*/ }

roles/ufmonitor/tasks/main.yml

@@ -0,0 +1,64 @@
+---
+# tasklist for setting up upstreamfirst-monitor (uf-monitor) on an el7 host
+
+
+- name: Install docker
+  package: name={{item}} state=present
+  with_items:
+  - docker
+  - python-docker-py
+
+- name: Run docker
+  service: name=docker state=started enabled=yes
+
+- name: Pull apiserver image
+  docker_image:
+    name: "{{ apiserver_container_image }}"
+    tls_verify: true
+
+- name: Pull updater image
+  docker_image:
+    name: "{{ apiserver_container_image }}"
+    tls_verify: true
+
+- name: ensure ufmonitor database is created
+  become: true
+  become_user: postgres
+  postgresql_db: db={{ ufmonitor_db_name }}
+
+- name: ensure ufmonitor db user has access to dev database
+  become: true
+  become_user: postgres
+  postgresql_user: db={{ ufmonitor_db_name }} user={{ ufmonitor_db_user }} password={{ ufmonitor_db_password }} role_attr_flags=NOSUPERUSER
+
+- name: generate environment file for apiserver
+  template: src=ufmonitor-envvars.j2 dest=/etc/sysconfig/ufmonitor
+
+- name: Deploy service file for apiserver
+  template: src=ufmonitor-apiserver.service.j2 dest=/etc/systemd/system/ufmonitor-apiserver.service
+  notify:
+  - reload systemd
+
+- name: Enable apiserver
+  service: name=ufmonitor-apiserver enabled=yes
+
+- name: install script to update ufmonitor database
+  template: src=update-ufmonitor.j2 dest=/usr/local/bin/update-ufmonitor mode=0755
+
+- name: Setup cron to update ufmonitor database
+  cron: name="update-ufmonitor-database" minute="15" user="root"
+        job="/usr/local/bin/update-ufmonitor"
+        cron_file=update-ufmonitor-database
+
+- name: ensure ufmonitor document root exists
+  file: path="{{ ufmonitor_home }}" state=directory owner=apache group=apache mode=0775
+
+- name: copy over index.html
+  template: src=index.html.j2 dest="{{ ufmonitor_home }}/index.html }}" owner=apache group=apache mode=0775
+
+- name: copy over static support files
+  copy: src=media dest="{{ ufmonitor_home }}/media }}" directory_mode=yes owner=apache group=apache mode=0775
+
+- name: generate httpd config
+  template: src=ufmonitor.conf.j2 dest=/etc/httpd/conf.d/ufmonitor.conf mode=0755
+

roles/ufmonitor/templates/index.html.j2

@@ -0,0 +1,189 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+    <title>Upstream First - Monitoring</title>
+
+    <!-- Load c3.css -->
+    <link href="media/css/c3.css" rel="stylesheet">
+
+    <!-- Load d3.js (v4.9.1) and c3.js (v0.4.12) -->
+    <script src="media/js/d3.min.js" charset="utf-8"></script>
+    <script src="media/js/c3.min.js"></script>
+
+    <script src="https://code.jquery.com/jquery-3.2.1.min.js"></script>
+    <!-- Latest compiled and minified CSS -->
+
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
+
+    <!-- Optional theme -->
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha384-rHyoN1iRsVXV4nD0JutlnGaslCJuC7uwjduW9SVrLvRYooPp2bWYgmgJQIXwl/Sp" crossorigin="anonymous">
+
+    <!-- Latest compiled and minified JavaScript -->
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
+    <!-- Latest compiled and minified CSS -->
+    <link rel="stylesheet" href="http://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.11.1/bootstrap-table.min.css">
+
+    <!-- Latest compiled and minified JavaScript -->
+    <script src="http://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.11.1/bootstrap-table.min.js"></script>
+
+
+
+  </head>
+
+  <body>
+
+    <h1>Background</h1>
+
+    <p>The purpose of this page is to be a quick reference on how far along the "<a href="https://fedoraproject.org/wiki/UpstreamFirstTesting/HowToMoveTests" target="new">Upstream First"</a> initiative is. There are currently 410 packages that Red Hat has internal tests for, which are all being moved upstream to their respective packages. Packages that have a repo on our <a href="https://upstreamfirst.fedorainfracloud.org/">forge</a> are counted as "moved," packages with open issues are counted as "working on." If there is no repo on the forge for a package, it's counted in the "untouched" category.</p>
+
+    <h4>Raw Numbers:</h4>
+    <p>
+      <ul>
+    <li id="total-tests">Total Tests: Loading...</li>
+    <li id="migrated-tests">Migrated: Loading...</li>
+    <li id="working-on">Working on: Loading...</li>
+      </ul>
+    </p>
+
+    <h3>Current Status</h3>
+
+    <div id="chart"></div>
+
+    <div id="tables">
+
+      <h3>Package List</h3>
+      <table id='table' data-search='true'
+         data-show-refresh="true"
+         data-sort-name='name'
+         data-sort-desc='desc'
+         data-show-toggle="true"                                                                                 data-show-columns="true"></table>
+
+    </div> <!-- End of tables div -->
+
+    <p> <sub>(Updated every hour)</sub></p>
+
+    <script type="text/javascript">
+
+      apiRequest = new XMLHttpRequest();
+
+      apiRequest.onreadystatechange = getString;
+      apiRequest.open("GET", "https://status.{{ external_hostname }}/api/all", false);
+      apiRequest.send();
+
+      function getString() {
+          if (apiRequest.readyState === XMLHttpRequest.DONE) {
+              if (apiRequest.status === 200) {
+                 return apiRequest.responseText;
+              }
+          }
+      }
+
+      results = apiRequest.responseText;
+      results = JSON.parse(results);
+
+      $('#table').bootstrapTable({
+      data: results,
+      columns:[{
+        field: 'id',
+        title: 'id'
+      }, {
+        field: 'name',
+        title: 'Name'
+      }, {
+        field: 'status',
+        title: 'Status'
+      }, {
+        field: 'contact',
+        title: 'Contact'
+      }, {
+        field: 'pagure_link',
+        title: 'Forge Link'
+      }]
+      });
+
+
+    </script>
+
+    <script type="text/javascript">
+$(document).ready(function () {
+      var moved = "Moved to Forge";
+      var untouched = "Untouched";
+      var working_on = "Working On";
+
+      <!-- Write out our AJAX calls to update the page counts. -->
+      apiRequest = new XMLHttpRequest();
+
+      apiRequest.onreadystatechange = getString;
+      apiRequest.open("GET", "https://status.{{ external_hostname }}/api/counts", false);
+      apiRequest.send();
+
+      function getString() {
+          if (apiRequest.readyState === XMLHttpRequest.DONE) {
+              if (apiRequest.status === 200) {
+                 return apiRequest.responseText;
+              }
+          }
+      }
+
+      results = apiRequest.responseText;
+      results = JSON.parse(results);
+
+      var moved_number = results.total_moved.valueOf();
+      var working_on_number = results.WORKING.valueOf();
+      var untouched_number = results.UNKNOWN.valueOf();
+
+      // Fill out the actual values in the table
+      document.getElementById("total-tests").innerText = "Total Tests: " + results.total;
+      document.getElementById("migrated-tests").innerText = "Migrated: " + moved_number;
+      document.getElementById("working-on").innerText = "Working-on: " + working_on_number;
+
+
+
+      var chart = c3.generate({
+        data: {
+          columns: [
+              [moved, moved_number],
+              [working_on, working_on_number ],
+              [untouched, untouched_number],
+          ],
+          type : 'donut',
+          onclick: function (d, i) { console.log("onclick", d, i); },
+          onmouseover: function (d, i) { console.log("onmouseover", d, i); },
+          onmouseout: function (d, i) { console.log("onmouseout", d, i); }
+        },
+        donut: {
+          title: "Migration Status"
+        },
+    color: {
+      pattern: ['#0000ff', '#00ff00', '#ff0000']
+    }
+      });
+
+      setTimeout(function () {
+        chart.load({
+          columns: [
+            [moved, moved_number],
+            [untouched, untouched_number],
+          ]
+        });
+      }, 1500);
+
+      setTimeout(function () {
+        chart.unload({
+          ids: 'data1'
+        });
+        chart.unload({
+          ids: 'data2'
+        });
+      }, 2500);
+});
+    </script>
+
+  </body>
+
+</html>

roles/ufmonitor/templates/ufmonitor-apiserver.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=ufmonitor apiserver
+After=docker.service
+Requires=docker.service
+
+[Service]
+TimeoutStartSec=0
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=-{{ ufmonitor_config_path }}
+ExecStart=/usr/bin/docker run --detach --log-driver none --name %n -p 8000:8000 --add-host=postgres:{{ public_ip }} {{ apiserver_container_image }} -l /var/log/ufmonitor/%n.log
+ExecStop=/usr/bin/docker rm --force %n
+TimeoutStopSec=180
+
+[Install]
+WantedBy=multi-user.target

roles/ufmonitor/templates/ufmonitor-envvars.j2

@@ -0,0 +1 @@
+POSTGRES_DB_PASSWORD='{{ ufmonitor_db_password }}'

roles/ufmonitor/templates/ufmonitor.conf.j2

@@ -0,0 +1,39 @@
+## Redirects http -> https
+<VirtualHost *:80>
+  RewriteEngine on
+  RewriteRule ^/\.well-known/(.*) /srv/web/acme-challenge/.well-known/$1 [L]
+  ServerName {{ external_hostname }}
+  Redirect permanent / https://{{ external_hostname }}/
+</VirtualHost>
+
+## End of redirects http -> https
+
+<VirtualHost *:443>
+  ServerName status.{{ external_hostname }}
+
+  Alias "/robots.txt" "/var/www/html/robots.txt"
+
+  ServerAdmin admin@fedoraproject.org
+
+  SSLEngine on
+  SSLProtocol {{ ssl_protocols }}
+  SSLCipherSuite {{ ssl_ciphers }}
+  # Use secure TLSv1.1 and TLSv1.2 ciphers
+  Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload"
+
+  SSLCertificateFile    /etc/letsencrypt/live/{{ external_hostname }}/cert.pem
+  SSLCertificateKeyFile /etc/letsencrypt/live/{{ external_hostname }}/privkey.pem
+  SSLCertificateChainFile /etc/letsencrypt/live/{{ external_hostname }}/fullchain.pem
+  SSLHonorCipherOrder On
+  SSLCipherSuite RC4-SHA:AES128-SHA:ALL:!ADH:!EXP:!LOW:!MD5:!SSLV2:!NULL
+  SSLProtocol ALL -SSLv2
+
+  DocumentRoot "{{ ufmonitor_home }}"
+
+  # proxy the docker containers running the actual api process
+  <Location /api/ >
+    ProxyPass http://127.0.0.1:8000/
+    ProxyPassReverse http://127.0.0.1:8000/
+  </Location>
+
+</VirtualHost>

roles/ufmonitor/templates/update-ufmonitor.j2

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+#
+# This job handles updating the ufmonitor database with new data from a pagure instance
+#
+
+source {{ ufmonitor_config_path }}
+docker run --add-host=postgres:{{ public_ip }} {{ updater_container_image }} -l /var/log/ufmonitor/updater.log

roles/ufmonitor/vars/main.yml

@@ -0,0 +1,4 @@
+mirrormanager_uid: 441
+mirrormanager_gid: 441
+mirrors_gid: 263
+mirrors2_gid: 529