From 90cc6ce6a54c13de658c929fedfbc2cbdc5f32b0 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 15 Jul 2013 21:06:54 +0000 Subject: [PATCH] First stab at moving bacula over to ansible. --- files/bacula/bacula-dir.conf.j2 | 1167 +++++++++++++++++++++ files/bacula/bacula-fd.conf.j2 | 46 + files/bacula/bacula-sd.conf.j2 | 104 ++ files/bacula/bconsole.conf.j2 | 10 + files/bacula/fedora_delete_catalog_backup | 5 + files/bacula/fedora_make_catalog_backup | 3 + files/mysql/my.cnf | 74 ++ playbooks/groups/backup-server.yml | 32 + tasks/bacula_server.yml | 58 + tasks/mysql_server.yml | 18 + 10 files changed, 1517 insertions(+) create mode 100644 files/bacula/bacula-dir.conf.j2 create mode 100644 files/bacula/bacula-fd.conf.j2 create mode 100644 files/bacula/bacula-sd.conf.j2 create mode 100644 files/bacula/bconsole.conf.j2 create mode 100755 files/bacula/fedora_delete_catalog_backup create mode 100755 files/bacula/fedora_make_catalog_backup create mode 100644 files/mysql/my.cnf create mode 100644 playbooks/groups/backup-server.yml create mode 100644 tasks/bacula_server.yml create mode 100644 tasks/mysql_server.yml diff --git a/files/bacula/bacula-dir.conf.j2 b/files/bacula/bacula-dir.conf.j2 new file mode 100644 index 0000000000..8c6f453a83 --- /dev/null +++ b/files/bacula/bacula-dir.conf.j2 @@ -0,0 +1,1167 @@ + +# Default Bacula Director Configuration file +# +# The only thing that MUST be changed is to add one or more +# file or directory names in the Include directive of the +# FileSet resource. +# +# For Bacula release 2.0.3 (06 March 2007) -- redhat (Zod) +# +# You might also want to change the default email address +# from root to your address. See the "mail" and "operator" +# directives in the Messages resource. +# + +Director { # define myself + Name = bacula-dir + DIRport = 9101 # where we listen for UA connections + QueryFile = "/etc/bacula/query.sql" + WorkingDirectory = "/var/spool/bacula" + PidDirectory = "/var/run" + Maximum Concurrent Jobs = 10 + Password = "{{ bacula5PasswordCon }}" # Console password + Messages = Daemon +} + +# Should catch most common files (basically pulls /) +JobDefs { + Name = "LightJob" + Type = Backup + Level = Incremental + Client = bacula-fd + FileSet = "Light Set" + Schedule = "WeeklyCycle" + Storage = TapeDrive-1 + Messages = Standard + Pool = Full-Pool + SpoolData = yes + Priority = 10 + Maximum Concurrent Jobs = 10 +} + +JobDefs { + Name = "PeopleJob" + Type = Backup + Level = Incremental + Client = bacula-fd + FileSet = "People Set" + Schedule = "WeeklyCycle" + Storage = TapeDrive-1 + Messages = Standard + Pool = Full-Pool + SpoolData = yes + Priority = 10 + Maximum Concurrent Jobs = 10 +} + + +JobDefs { + Name = "PuppetJob" + Type = Backup + Level = Incremental + Client = bacula-fd + FileSet = "Puppet Set" + Schedule = "WeeklyCycle" + Storage = TapeDrive-1 + Messages = Standard + Pool = Full-Pool + SpoolData = yes + Priority = 10 + Maximum Concurrent Jobs = 10 +} + +# Should catch most common files (basically pulls /) +JobDefs { + Name = "DefaultJob" + Type = Backup + Level = Incremental + Client = bacula-fd + FileSet = "Full Set" + Schedule = "WeeklyCycle" + Storage = TapeDrive-1 + Messages = Standard + Pool = Full-Pool + SpoolData = yes + Priority = 10 + Maximum Concurrent Jobs = 10 +} +# job for log servers. +JobDefs { + Name = "DefaultLogJob" + Type = Backup + Level = Incremental + Client = bacula-fd + FileSet = "Full Log Set" + Schedule = "WeeklyCycle" + Storage = TapeDrive-1 + Messages = Standard + Pool = Full-Pool + SpoolData = yes + Priority = 10 + Maximum Concurrent Jobs = 10 +} + +# For the CVS/GIT box +JobDefs { + Name = "SCMJob" + Type = Backup + Level = Incremental + Client = bacula-fd + FileSet = "SCM Set" + Schedule = "WeeklyCycle" + Storage = TapeDrive-1 + Messages = Standard + Pool = Full-Pool + SpoolData = yes +# Full Backup Pool = Full-Pool +# Incremental Backup Pool = Inc-Pool +# Differential Backup Pool = Diff-Pool + Priority = 10 + Maximum Concurrent Jobs = 10 +} + + +# Should catch most common files (basically pulls /) +JobDefs { + Name = "CatalogJob" + Type = Backup + Level = Incremental + Client = bacula-fd + FileSet = "Full Set" + Schedule = "WeeklyCycle" + Storage = TapeDrive-1 + Messages = Standard + Pool = Full-Pool + SpoolData = yes +# Full Backup Pool = Full-Pool-Catalog + Priority = 10 + Maximum Concurrent Jobs = 10 +} + +# For the koji mount + +#Job { +# Name = "fedorapeople.org" +# Client = "fedorapeople.org" +# JobDefs = "DefaultJob" +# Write Bootstrap = "/var/spool/bacula/fedorapeople.org.bsr" +#} + +Job { + Name = "db05" + Client = "db05" + JobDefs = "DefaultJob" + Write Bootstrap = "/var/spool/bacula/db05.bsr" +} + +Job { + Name = "db01" + Client = "db01" + JobDefs = "DefaultJob" + Write Bootstrap = "/var/spool/bacula/db02.bsr" +} + +Job { + Name = "db-fas01" + Client = "db-fas01" + JobDefs = "DefaultJob" + Write Bootstrap = "/var/spool/bacula/db-fas01.bsr" +} + +#Job { +# Name = "db02" +# Client = "db02" +# JobDefs = "DefaultJob" +# Write Bootstrap = "/var/spool/bacula/db02.bsr" +#} + +Job { + Name = "db04" + Client = "db04" + JobDefs = "DefaultJob" + Write Bootstrap = "/var/spool/bacula/db04.bsr" +} + +Job { + Name = "fas01" + Client = "fas01" + JobDefs = "DefaultJob" + Write Bootstrap = "/var/spool/bacula/fas01.bsr" +} + +#Job { +# Name = "cvs1" +# Client = "cvs1" +# JobDefs = "SCMJob" +# Write Bootstrap = "/var/spool/bacula/cvs1.bsr" +#} + +Job { + Name = "pkgs01" + Client = "pkgs01" + JobDefs = "SCMJob" + Write Bootstrap = "/var/spool/bacula/pkgs01.bsr" +} + +Job { + Name = "collab04" + Client = "collab04" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/collab04.bsr" +} + + +Job { + Name = "hosted-lists01" + Client = "hosted-lists01" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/hosted-lists01.bsr" +} + +Job { + Name = "hosted04" + Client = "hosted04" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/hosted04.bsr" +} + +#Job { +# Name = "hosted04" +# Client = "hosted04" +# JobDefs = "LightJob" +# Write Bootstrap = "/var/spool/bacula/hosted04.bsr" +#} + +Job { + Name = "noc01" + Client = "noc01" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/noc01.bsr" +} + +Job { + Name = "lockbox01" + Client = "lockbox01" + JobDefs = "PuppetJob" + Write Bootstrap = "/var/spool/bacula/lockbox01.bsr" +} + +Job { + Name = "ask01" + Client = "ask01" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/ask01.bsr" +} + +Job { + Name = "bastion01" + Client = "bastion01" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/bastion01.bsr" +} + +Job { + Name = "bastion02" + Client = "bastion02" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/bastion02.bsr" +} + + +Job { + Name = "proxy01" + Client = "proxy01" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/proxy01.bsr" +} + + +Job { + Name = "proxy02" + Client = "proxy02" + JobDefs = "LightJob" + Write Bootstrap = "/var/spool/bacula/proxy02.bsr" +} + +#Job { +# Name = "people02" +# Client = "people02" +# JobDefs = "PeopleJob" +# Write Bootstrap = "/var/spool/bacula/people02.bsr" +#} + +Job { + Name = "people03" + Client = "people03" + JobDefs = "PeopleJob" + Write Bootstrap = "/var/spool/bacula/people03.bsr" +} + +Job { + Name = "releng03" + Client = "releng03" + JobDefs = "DefaultJob" + Write Bootstrap = "/var/spool/bacula/releng03.bsr" +} + +Job { + Name = "releng04" + Client = "releng04" + JobDefs = "DefaultJob" + Write Bootstrap = "/var/spool/bacula/releng04.bsr" +} + +Job { + Name = "relepel01" + Client = "relepel01" + JobDefs = "DefaultJob" + Write Bootstrap = "/var/spool/bacula/relepel01.bsr" +} + +#Job { +# Name = "koji" +# Client = "nfs01" +# JobDefs = "KojiJob" +# Write Bootstrap = "/var/spool/bacula/koji.bsr" +#} + +Job { + Name = "log02" + Client = "log02" + JobDefs = "DefaultLogJob" + Write Bootstrap = "/var/spool/bacula/log02.bsr" +} + +#Job { +# Name = "backup03" +# Client = "backup03" +# JobDefs = "LightJob" +# Write Bootstrap = "/var/spool/bacula/backup03.bsr" +#} + +# Backup the catalog database (after the nightly save) +Job { + Name = "BackupCatalog" + JobDefs = "CatalogJob" + Level = Full + FileSet="Catalog" + Schedule = "WeeklyCycleAfterBackup" + # This creates an ASCII copy of the catalog + RunBeforeJob = "/usr/local/bin/fedora_make_catalog_backup" + # This deletes the copy of the catalog + RunAfterJob = "/usr/local/bin/fedora_delete_catalog_backup" + Write Bootstrap = "/var/spool/bacula/BackupCatalog.bsr" + Priority = 11 # run after main backup +} + +# +# Standard Restore template, to be changed by Console program +# Only one such job is needed for all Jobs/Clients/Storage ... +# +Job { + Name = "RestoreFiles" + Type = Restore + Client=bacula-fd + FileSet="Full Set" + Storage = TapeDrive-1 + Pool = Default +# Full Backup Pool = Full-Pool +# Incremental Backup Pool = Inc-Pool +# Differential Backup Pool = Diff-Pool + Messages = Standard + Where = /tmp/bacula-restores +} + + + +FileSet { + Name = "Light Set" + Include { + Options { + signature = SHA1 + Compression = GZIP9 + } + File = /etc + File = /root + File = /home + File = /var + File = /srv + } + Exclude { + File = /proc + File = /tmp + File = /.journal + File = /.fsck + File = /dev + File = /sys + File = /.swap + File = /srv/torrent/btholding + File = /srv/web/docroot/epel + File = /srv/rpmbuild/epel + File = /srv/gitweb-cache + File = /var/tmp + File = /glusterfs + } +} + +FileSet { + Name = "People Set" + Include { + Options { + signature = SHA1 + Compression = GZIP9 + } + File = /etc + File = /root + File = /var + File = /srv + } + Exclude { + File = /proc + File = /tmp + File = /.journal + File = /.fsck + File = /dev + File = /sys + File = /.swap + File = /srv/torrent/btholding + File = /var/tmp + } +} + +FileSet { + Name = "Puppet Set" + Include { + Options { + signature = SHA1 + Compression = GZIP9 + WildDir=.snapshot + Exclude=yes + } + File = /etc + File = /root + File = /home + File = /var + File = /srv + File = /git + File = /mnt/fedora/app + File = /mnt/fedora/cvs + } + Exclude { + File = /proc + File = /tmp + File = /.journal + File = /.fsck + File = /dev + File = /sys + File = /.swap + File = /mnt/fedora/app/fi-repo/rhel + File = /var/tmp + } +} + +FileSet { + Name = "SCM Set" + Include { + Options { + signature = SHA1 + Compression = GZIP9 + } + File = / + File = /srv + File = /srv/cache/lookaside/ + } + Exclude { + File = /proc + File = /tmp + File = /.journal + File = /.fsck + File = /dev + File = /sys + File = /.swap + File = /var/tmp + File = /srv/gitweb-cache + } +} + + +# List of files to be backed up +FileSet { + Name = "Full Set" + Include { + Options { + Compression = GZIP9 + signature = SHA1 + } + +# +# Put your list of files here, preceded by 'File =', one per line +# or include an external list with: +# +# File = /bacula/bacula.sql diff --git a/files/mysql/my.cnf b/files/mysql/my.cnf new file mode 100644 index 0000000000..fb4a7de97d --- /dev/null +++ b/files/mysql/my.cnf @@ -0,0 +1,74 @@ +[mysqld] +datadir=/var/lib/mysql +socket=/var/lib/mysql/mysql.sock +user=mysql +# Default to using old password format for compatibility with mysql 3.x +# clients (those using the mysqlclient10 compatibility package). +old_passwords=1 +max_connections=900 +query_cache_size=64M +query_cache_limit=2M +ft_min_word_len=3 + +log-slow-queries=/var/log/mysqld/slow-queries.log +long_query_time = 2 +general_log = 1 +general_log_file = /var/log/mysqld/mysql-transfer.log + +skip-locking +key_buffer = 384M +key_buffer_size=64M +max_allowed_packet = 16M +table_cache = 2048 +sort_buffer_size = 8M +join_buffer_size = 8M +read_buffer_size = 2M +read_rnd_buffer_size = 16M +bulk_insert_buffer_size = 64M +myisam_sort_buffer_size = 128M +myisam_max_sort_file_size=15G +myisam_max_extra_sort_file_size = 10G +thread_cache_size = 8 +# Try number of CPU's*2 for thread_concurrency +thread_concurrency = 16 +thread_stack = 192K + +transaction_isolation = REPEATABLE-READ + +back_log = 50 +binlog_cache_size = 1M +max_heap_table_size = 128M + +tmp_table_size = 128M + +innodb_additional_mem_pool_size = 16M +innodb_buffer_pool_size = 4G +innodb_file_io_threads = 4 +innodb_thread_concurrency = 16 +innodb_flush_log_at_trx_commit = 1 +innodb_log_buffer_size = 8M +#innodb_log_file_size = 2G +#innodb_log_files_in_group = 3 +innodb_max_dirty_pages_pct = 90 + + +[mysqld_safe] +log-error=/var/log/mysqld.log +pid-file=/var/run/mysqld/mysqld.pid +open-files-limit = 8192 + +[isamchk] +key_buffer = 512M +sort_buffer_size = 512M +read_buffer = 8M +write_buffer = 8M + +[myisamchk] +key_buffer = 512M +sort_buffer_size = 512M +read_buffer = 8M +write_buffer = 8M + +[mysqlhotcopy] +interactive-timeout + diff --git a/playbooks/groups/backup-server.yml b/playbooks/groups/backup-server.yml new file mode 100644 index 0000000000..a807bddbd4 --- /dev/null +++ b/playbooks/groups/backup-server.yml @@ -0,0 +1,32 @@ +# create a new backup server system +# NOTE: should be used with --limit most of the time +# NOTE: make sure there is room/space for this instance on the buildvmhost +# NOTE: most of these vars come from group_vars/backup_server or from hostvars + +- name: make backup server system + hosts: backup-server + user: root + gather_facts: False + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - ${private}/vars.yml + - ${vars}/${ansible_distribution}.yml + + tasks: + tasks: + - include: $tasks/hosts.yml + - include: $tasks/yumrepos.yml + - include: $tasks/base.yml + - include: $tasks/fas_client.yml + - include: $tasks/2fa_client.yml + - include: $tasks/motd.yml + - include: $tasks/sudo.yml + - include: $tasks/rkhunter.yml + - include: $tasks/denyhosts.yml + - include: $tasks/nagios_client.yml + - include: $tasks/mysql_server.yml + - include: $tasks/bacula_server.yml + + handlers: + - include: $handlers/restart_services.yml diff --git a/tasks/bacula_server.yml b/tasks/bacula_server.yml new file mode 100644 index 0000000000..30eb41c4cb --- /dev/null +++ b/tasks/bacula_server.yml @@ -0,0 +1,58 @@ +--- +# tasklist for setting up the backup server. +- name: install bacula + yum: pkg=$item state=installed + with_items: + - bacula-director-mysql + - bacula-sd + - bacula-storage-mysql + - bacula-console + - bacula-client + - mysql-server + - mtx + tags: + - packages + +# install bacula config files. +- name: install bacula storage director configs + template: src=$files/bacula/bacula-sd.conf.j2 dest=/etc/bacula/bacula-sd.conf mode=640 + notify: + - restart bacula-sd + tags: + - config + +- name: install bacula director configs + template: src=$files/bacula/bacula-dir.conf.j2 dest=/etc/bacula/bacula-dir.conf mode=640 + notify: + - restart bacula-dir + tags: + - config + +- name: install bacula file configs + template: src=$files/bacula/bacula-fd.conf.j2 dest=/etc/bacula/bacula-fd.conf mode=640 + notify: + - restart bacula-fd + tags: + - config + +- name: install bacula console config + template: src=$files/bacula/bconsole.conf.j2 dest=/etc/bacula/bconsole.conf mode=640 + tags: + - config + +- name: install bacula scripts + copy: src=$files/bacula/$item dest=/usr/local/bin/$item mode=0755 + with_items: + - fedora_make_catalog_backup + - fedora_delete_catalog_backup + tags: + - config + +- name: enable bacula-sd service + service: name=bacula-sd state=started enabled=yes + +- name: enable bacula-dir service + service: name=bacula-dir state=started enabled=yes + +- name: enable bacula-fd service + service: name=bacula-fd state=started enabled=yes diff --git a/tasks/mysql_server.yml b/tasks/mysql_server.yml new file mode 100644 index 0000000000..0ff3c2375a --- /dev/null +++ b/tasks/mysql_server.yml @@ -0,0 +1,18 @@ +--- +# +# Setup mysql server. +# +- name: install mysql server packages + yum: name=$item state=installed + with_items: + - mysql-server + tags: + - packages + +- name: install our my.cnf + copy: src=$files/mysql/my.cnf dest=/etc/my.cnf owner=root group=root mode=0644 + +- name: Set mysql-server to run + service: name=mysqld enabled=yes state=running + tags: + - service