From b40421a85c294dbdef98bf8ed6545a02686bdf3c Mon Sep 17 00:00:00 2001
From: Valentin Gologuzov <vgologuz@redhat.com>
Date: Tue, 31 Mar 2015 17:15:09 +0200
Subject: [PATCH] [copr] install logstash config and ensure running logstash
 service

---
 inventory/group_vars/copr-back                |  1 +
 inventory/group_vars/copr-back-stg            |  1 +
 roles/copr/backend/handlers/main.yml          |  3 +
 roles/copr/backend/tasks/main.yml             | 16 ++++--
 .../templates/logstash.d/copr_backend.conf    | 55 +++++++++++++++++++
 5 files changed, 71 insertions(+), 5 deletions(-)
 create mode 100644 roles/copr/backend/templates/logstash.d/copr_backend.conf

diff --git a/inventory/group_vars/copr-back b/inventory/group_vars/copr-back
index d34c297e1d..0fc8b77711 100644
--- a/inventory/group_vars/copr-back
+++ b/inventory/group_vars/copr-back
@@ -5,3 +5,4 @@ _copr_be_conf: "copr-be.conf"
 do_sign: "true"
 
 spawn_in_advance: "true"
+frontend_base_url = "http://copr-fe.cloud.fedoraproject.org"
diff --git a/inventory/group_vars/copr-back-stg b/inventory/group_vars/copr-back-stg
index a2274dff2b..d037b5ac2b 100644
--- a/inventory/group_vars/copr-back-stg
+++ b/inventory/group_vars/copr-back-stg
@@ -5,3 +5,4 @@ _copr_be_conf: "copr-be.conf-dev"
 do_sign: "true"
 
 spawn_in_advance: "true"
+frontend_base_url = "http://copr-fe-dev.cloud.fedoraproject.org"
diff --git a/roles/copr/backend/handlers/main.yml b/roles/copr/backend/handlers/main.yml
index 39bd0b0250..9e3ac40699 100644
--- a/roles/copr/backend/handlers/main.yml
+++ b/roles/copr/backend/handlers/main.yml
@@ -3,3 +3,6 @@
 
 - name: restart copr-backend
   service: name=copr-backend state=restarted
+
+- name: restart logstash
+  service: name=logstash state=restarted
diff --git a/roles/copr/backend/tasks/main.yml b/roles/copr/backend/tasks/main.yml
index 16c165735a..e3905c4c18 100644
--- a/roles/copr/backend/tasks/main.yml
+++ b/roles/copr/backend/tasks/main.yml
@@ -11,9 +11,6 @@
   - obs-signd # TODO: remove, since it will be installed as dependency by copr-backend
   - redis # we need this to start the service in next step
 
-- name: enable and start redis  # TODO: .service in copr-backend should depend on redis
-  service: name=redis enabled=yes state=started
-
 - name: make copr dirs
   file: state=directory path={{ item }}
   with_items:
@@ -133,8 +130,17 @@
   tags:
   - config
 
-- name: enable and run copr-backend
-  service: name="copr-backend" enabled=yes state=running
+- name: copy logstash config
+  template: src=logstash.d/copr_backend.conf dest=/etc/logstash.d/copr_backend.conf
+  notify:
+  - restart logstash
+
+- name: enable and run copr-backend services
+  service: name="{{ item }}" enabled=yes state=running
+  with_items:
+  - redis       # TODO: .service in copr-backend should depend on redis
+  - logstash
+  - copr-backend
 
 - name: copy delete-forgotten-instances.pl
   copy: src="delete-forgotten-instances.pl" dest=/home/copr/delete-forgotten-instances.pl mode=755
diff --git a/roles/copr/backend/templates/logstash.d/copr_backend.conf b/roles/copr/backend/templates/logstash.d/copr_backend.conf
new file mode 100644
index 0000000000..532dad34ba
--- /dev/null
+++ b/roles/copr/backend/templates/logstash.d/copr_backend.conf
@@ -0,0 +1,55 @@
+input {
+  #file {
+  #  path => "/var/log/copr/backend.log"
+  #  type => "copr.backend.main"
+  #}
+  file {
+    path => "/var/log/lighttpd/access.log"
+    type => "lighttpd-access"
+  }
+}
+
+filter {
+  mutate {
+    add_tag => [ "backend" ]
+  }
+  if [type] == 'lighttpd-access' {
+    grok {
+      patterns_dir => "/usr/share/logstash/patterns"
+      pattern => "%{LIGHTTPD}"
+    }
+    date {
+       match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+    }
+    if "repodata/repomd.xml" in [request] and "devel/repodata/repomd.xml" not in [request]  {
+      mutate { add_tag => "repomdxml" }
+      mutate { add_tag => "publish_stat" }
+      grok {
+        match => ["request", "/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/repodata/repomd.xml"]
+      }
+    }
+    if [request] =~ "rpm$" {
+      mutate { add_tag => "rpm" }
+      mutate { add_tag => "publish_stat" }
+      grok {
+        match => ["request", "/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/%{USERNAME:copr_build_dir}/%{USERNAME:copr_rpm}"]
+      }
+    }
+  }
+}
+
+output {
+  if "publish_stat" in [tags] {
+      http {
+        url => "{{ frontend_base_url }}/stats_rcv/from_logstash"
+        format => "json"
+        http_method => "post"
+      }
+
+      file {
+        path => "/var/log/logstash/published.log"
+        codec => "rubydebug"
+      }
+  }
+
+}