[copr] install logstash config and ensure running logstash service

inventory/group_vars/copr-back

@@ -5,3 +5,4 @@ _copr_be_conf: "copr-be.conf"
 do_sign: "true"
 
 spawn_in_advance: "true"
+frontend_base_url = "http://copr-fe.cloud.fedoraproject.org"

inventory/group_vars/copr-back-stg

@@ -5,3 +5,4 @@ _copr_be_conf: "copr-be.conf-dev"
 do_sign: "true"
 
 spawn_in_advance: "true"
+frontend_base_url = "http://copr-fe-dev.cloud.fedoraproject.org"

roles/copr/backend/handlers/main.yml

@@ -3,3 +3,6 @@
 
 - name: restart copr-backend
   service: name=copr-backend state=restarted
+
+- name: restart logstash
+  service: name=logstash state=restarted

roles/copr/backend/tasks/main.yml

@@ -11,9 +11,6 @@
   - obs-signd # TODO: remove, since it will be installed as dependency by copr-backend
   - redis # we need this to start the service in next step
 
-- name: enable and start redis  # TODO: .service in copr-backend should depend on redis
-  service: name=redis enabled=yes state=started
-
 - name: make copr dirs
   file: state=directory path={{ item }}
   with_items:
@@ -133,8 +130,17 @@
   tags:
   - config
 
-- name: enable and run copr-backend
-  service: name="copr-backend" enabled=yes state=running
+- name: copy logstash config
+  template: src=logstash.d/copr_backend.conf dest=/etc/logstash.d/copr_backend.conf
+  notify:
+  - restart logstash
+
+- name: enable and run copr-backend services
+  service: name="{{ item }}" enabled=yes state=running
+  with_items:
+  - redis       # TODO: .service in copr-backend should depend on redis
+  - logstash
+  - copr-backend
 
 - name: copy delete-forgotten-instances.pl
   copy: src="delete-forgotten-instances.pl" dest=/home/copr/delete-forgotten-instances.pl mode=755

roles/copr/backend/templates/logstash.d/copr_backend.conf

@@ -0,0 +1,55 @@
+input {
+  #file {
+  #  path => "/var/log/copr/backend.log"
+  #  type => "copr.backend.main"
+  #}
+  file {
+    path => "/var/log/lighttpd/access.log"
+    type => "lighttpd-access"
+  }
+}
+
+filter {
+  mutate {
+    add_tag => [ "backend" ]
+  }
+  if [type] == 'lighttpd-access' {
+    grok {
+      patterns_dir => "/usr/share/logstash/patterns"
+      pattern => "%{LIGHTTPD}"
+    }
+    date {
+       match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+    }
+    if "repodata/repomd.xml" in [request] and "devel/repodata/repomd.xml" not in [request]  {
+      mutate { add_tag => "repomdxml" }
+      mutate { add_tag => "publish_stat" }
+      grok {
+        match => ["request", "/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/repodata/repomd.xml"]
+      }
+    }
+    if [request] =~ "rpm$" {
+      mutate { add_tag => "rpm" }
+      mutate { add_tag => "publish_stat" }
+      grok {
+        match => ["request", "/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/%{USERNAME:copr_build_dir}/%{USERNAME:copr_rpm}"]
+      }
+    }
+  }
+}
+
+output {
+  if "publish_stat" in [tags] {
+      http {
+        url => "{{ frontend_base_url }}/stats_rcv/from_logstash"
+        format => "json"
+        http_method => "post"
+      }
+
+      file {
+        path => "/var/log/logstash/published.log"
+        codec => "rubydebug"
+      }
+  }
+
+}