First rough cut at a logserver setup. Will need lots more tweaking.

inventory/host_vars/log01.phx2.fedoraproject.org

@@ -3,8 +3,8 @@ nm: 255.255.255.0
 gw: 10.5.126.254
 dns: 10.5.126.21,10.5.126.22
 
-ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-6-dl
+ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-7
-ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL6-x86_64/
+ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL7-x86_64/
 volgroup: /dev/vg_virthost
 eth0_ip: 10.5.126.13
 vmhost: virthost18.phx2.fedoraproject.org

inventory/inventory

@@ -307,6 +307,7 @@ koji01.stg.phx2.fedoraproject.org
 
 [infracore]
 lockbox01.phx2.fedoraproject.org
+log01.phx2.fedoraproject.org
 log02.phx2.fedoraproject.org
 noc01.phx2.fedoraproject.org
 noc02.fedoraproject.org

playbooks/hosts/logserver.yml

@@ -0,0 +1,48 @@
+- name: make logs server
+  hosts: log01.phx2.fedoraproject.org
+  user: root
+  gather_facts: False
+  accelerate: "{{ accelerated }}"
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  tasks:
+  - include: "{{ tasks }}/virt_instance_create.yml"
+  - include: "{{ tasks }}/accelerate_prep.yml"
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+- name: make the box be real
+  hosts: log01.phx2.fedoraproject.org
+  user: root
+  gather_facts: True
+  accelerate: "{{ accelerated }}"
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  roles:
+  - base
+  - rkhunter
+  - nagios_client
+  - hosts
+  - fas_client
+  - collectd/base
+  - collectd/server
+  - sudo
+
+  tasks:
+  - include: "{{ tasks }}/yumrepos.yml"
+  - include: "{{ tasks }}/apache.yml"
+  - include: "{{ tasks }}/2fa_client.yml"
+  - include: "{{ tasks }}/motd.yml"
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+  - include: "{{ handlers }}/semanage.yml"

roles/base/files/rsyslog/rsyslog.conf.log01.phx2.fedoraproject.org

@@ -0,0 +1,212 @@
+################################################################################
+# RSYSLOG.CONF - central logging server                                        #
+################################################################################
+#                                    N O T E                                   #
+#------------------------------------------------------------------------------#
+# This rsyslog configuration is suitable for a central log host. This is not   #
+# intended for a normal server.                                                #
+#                                                                              #
+# This config was built and tested for rsyslog version rsyslog-2.0.0.          #
+################################################################################
+
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+#                                M O D U L E S                                 #
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+$ModLoad immark.so # mark messages
+$MarkMessagePeriod 1200 # every 20 minutes
+$ModLoad imudp.so #udp sockets
+$ModLoad imtcp.so #tcp socks
+$ModLoad imuxsock # local syslog() listener
+$ModLoad imklog # local klog logging
+
+
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+#                        G L O B A L   S E T T I N G S                         #
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+$umask 0000
+$CreateDirs on
+$DirCreateMode 0755
+$FileCreateMode 0644
+$FileOwner root
+$FileGroup root
+$RepeatedMsgReduction on
+$EscapeControlCharactersOnReceive off
+$UDPServerRun 514
+$InputTCPMaxSessions 2000
+$InputTCPServerRun 514
+$InputTCPServerRun 5000	
+$IMUXSockRateLimitInterval 0
+$SystemLogRateLimitInterval 0
+
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+#                              T E M P L A T E S                               #
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+
+$template CustomFormat,"%TIMESTAMP% %FROMHOST% %HOSTNAME% %syslogtag%%msg%\n"
+$template TraditionalFormat,"%timegenerated% %HOSTNAME% %syslogtag%%msg%\n"
+$template MergeFormat,"%timegenerated% %FROMHOST% %syslogtag%%msg%\n"
+$template RawMessage,"%msg:2:2048%\n"
+
+
+#------------------------------------------------------------------------------#
+# templates for merged remote logging                                                 #
+#------------------------------------------------------------------------------#
+$template m_messages, "/var/log/merged/messages.log"
+$template m_secure, "/var/log/merged/secure.log"
+$template m_mail, "/var/log/merged/mail.log"
+$template m_cron, "/var/log/merged/cron.log"
+$template m_spooler, "/var/log/merged/spooler.log"
+$template m_boot, "/var/log/merged/boot.log"
+$template m_kern, "/var/log/merged/kernel.log"
+$template m_audit, "/var/log/merged/audit.log"
+$template m_http_error, "/var/log/merged/http_error.log"
+$template m_http_apps, "/var/log/merged/apps.log"
+
+#------------------------------------------------------------------------------#
+# templates for remote logging                                                 #
+#------------------------------------------------------------------------------#
+$template r_messages, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/messages.log"
+$template r_secure, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/secure.log"
+$template r_mail, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/mail.log"
+$template r_cron, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/cron.log"
+$template r_spooler, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/spooler.log"
+$template r_boot, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/boot.log"
+$template r_kern, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/kernel.log"
+$template r_audit, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/audit.log"
+$template r_http_error, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/http_error.log"
+$template r_http_apps, "/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/apps.log"
+
+#------------------------------------------------------------------------------#
+# templates for http logging                                                   #
+#------------------------------------------------------------------------------#
+$template HttpAccessTemplate,"/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/http/%APP-NAME%"
+$template HttpErrorTemplate,"/var/log/hosts/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/http/%APP-NAME%"
+
+#------------------------------------------------------------------------------#
+# templates for local logging                                                  #
+#------------------------------------------------------------------------------#
+$template l_messages, "/var/log/messages"
+$template l_secure, "/var/log/secure"
+$template l_mail, "/var/log/maillog"
+$template l_cron, "/var/log/cron"
+$template l_spooler, "/var/log/spooler"
+$template l_boot, "/var/log/boot.log"
+$template l_idgaud, "/var/log/idgaudit.log"
+$template l_idgsys, "/var/log/idgsystem.log"
+
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+#                         R E M O T E   L O G G I N G                          #
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+
+#------------------------------------------------------------------------------#
+# remote messages log                                                          #
+#------------------------------------------------------------------------------#
+*.info;mail.none;authpriv.none;cron.none;local6.none;local5.none;local4.none	?r_messages;TraditionalFormat
+
+#------------------------------------------------------------------------------#
+# remote auth/secure log                                                       #
+#------------------------------------------------------------------------------#
+authpriv.*					-?r_secure;TraditionalFormat
+
+#------------------------------------------------------------------------------#
+# remote mail log                                                              #
+#------------------------------------------------------------------------------#
+mail.*						-?r_mail;TraditionalFormat
+
+#------------------------------------------------------------------------------#
+# remote cron log                                                              #
+#------------------------------------------------------------------------------#
+cron.*						?r_cron;TraditionalFormat
+
+#------------------------------------------------------------------------------#
+# remote spool log                                                             #
+#------------------------------------------------------------------------------#
+uucp,news.crit					?r_spooler;TraditionalFormat
+
+#------------------------------------------------------------------------------#
+# remote audit log                                                              #
+#------------------------------------------------------------------------------#
+local6.*					?r_audit;TraditionalFormat
+
+
+#------------------------------------------------------------------------------#
+# remote http_error log                                                              #
+#------------------------------------------------------------------------------#
+local5.*					?r_http_error;TraditionalFormat
+
+#------------------------------------------------------------------------------#
+# remote http app log                                                              #
+#------------------------------------------------------------------------------#
+local4.*					?r_http_apps;TraditionalFormat
+
+
+#------------------------------------------------------------------------------#
+# remote boot log                                                              #
+#------------------------------------------------------------------------------#
+local7.*					?r_boot;TraditionalFormat
+
+#------------------------------------------------------------------------------#
+# remote kernel log                                                            #
+#------------------------------------------------------------------------------#
+kern.*						?r_kern;TraditionalFormat
+
+
+
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+#                     MERGED  R E M O T E   L O G G I N G                          #
+#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#
+
+#------------------------------------------------------------------------------#
+# remote messages log                                                          #
+#------------------------------------------------------------------------------#
+*.info;mail.none;authpriv.none;cron.none;local6.none;local5.none;local4.none	?m_messages;MergeFormat
+
+#------------------------------------------------------------------------------#
+# remote auth/secure log                                                       #
+#------------------------------------------------------------------------------#
+authpriv.*					-?m_secure;MergeFormat
+
+#------------------------------------------------------------------------------#
+# remote mail log                                                              #
+#------------------------------------------------------------------------------#
+mail.*						-?m_mail;MergeFormat
+
+#------------------------------------------------------------------------------#
+# remote cron log                                                              #
+#------------------------------------------------------------------------------#
+cron.*						?m_cron;MergeFormat
+
+#------------------------------------------------------------------------------#
+# remote spool log                                                             #
+#------------------------------------------------------------------------------#
+uucp,news.crit					?m_spooler;MergeFormat
+
+
+#------------------------------------------------------------------------------#
+# remote boot log                                                              #
+#------------------------------------------------------------------------------#
+local7.*					?m_boot;MergeFormat
+
+#------------------------------------------------------------------------------#
+# remote kernel log                                                            #
+#------------------------------------------------------------------------------#
+kern.*						?m_kern;MergeFormat
+
+#------------------------------------------------------------------------------#
+# remote http_error
+#------------------------------------------------------------------------------#
+local5.*					?m_http_error;MergeFormat
+
+#------------------------------------------------------------------------------#
+# remote http_apps
+#------------------------------------------------------------------------------#
+local4.*					?m_http_apps;MergeFormat
+
+
+#------------------------------------------------------------------------------#
+# remote audit log                                                              #
+#------------------------------------------------------------------------------#
+#this means: if log message does not contain 'type=AVC' then drop it
+# this is only for the merged logs from auditd
+:msg, !contains, "type=AVC" ~
+local6.*					?m_audit;MergeFormat

roles/base/tasks/main.yml

@@ -121,6 +121,7 @@
 - name: rsyslog.conf
   copy: src={{ item }} dest=/etc/rsyslog.conf mode=644
   with_first_found:
+    - rsyslog/rsyslog.conf.{{ aansible_fqdn }}
     - rsyslog/rsyslog.conf.{{ dist_tag }}
     - rsyslog/rsyslog.conf.default
   notify:

roles/collectd/base/files/network-server.conf

@@ -0,0 +1,5 @@
+LoadPlugin network
+
+<Plugin "network">
+        Listen "0.0.0.0"
+</Plugin>

roles/collectd/base/tasks/main.yml

@@ -20,6 +20,14 @@
   copy: src=network-client.conf dest=/etc/collectd.d/network.conf
   notify:
   - restart collectd
+  when: not inventory_hostname.startswith('log')
+
+# install collectd-network config
+- name: /etc/collectd.d/network.conf
+  copy: src=network-server.conf dest=/etc/collectd.d/network.conf
+  notify:
+  - restart collectd
+  when: inventory_hostname.startswith('log')
 
 # apache - localhost only - pretty much any apache server
 - name: install collectd-apache

roles/collectd/server/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+# collectd server setup 
+
+# install pkg
+- name: install collectd server packages
+  yum: name={{ item }} state=installed
+  with_items: 
+  - collectd-rrdtool
+  - collectd-ping
+  - collectd-web