defaults/main.yml

@@ -3,20 +3,67 @@ cloud_apps: /opt
 cloud_storage: /opt/storage
 cloud_stage: prod
 cloud_update: false
+ssh_port: 1802
+
+# clients push only daily backup
+# server (!= client) will then pull daily backups
+backup_client: true
+
+# Server should pull 1 hour later the backups.
+backup_hour_shift: 1
+
+backup_times_weekday: "*"
+backup_times_hour:
+  - 6
+  - 10
+  - 14
+  - 18
+  - 22
+backup_times_minute: 0
+
+backup_keep_days: 7
 
 backup_owner: backup
+backup_owner_ssh_key: "ssh-rsa THEREISSUCHRSAKEYYET"
 backup_group: "{{ backup_owner }}"
-backup_owner_ssh_private_key: "ssh-rsa THEREISNOSUCHRSAKEYYET"
+backup_permission: 0770
-backup_owner_ssh_public_key: "ssh-rsa THEREISNOSUCHRSAKEYYET"
 
-backup_location: "/borg"
+backup_hosts:
+  - name: server1-name
+    domain: host1.my-domain.tld
+    ip: 001.112.223.123
+  - name: server2-name
+    domain: host2.my-domain.tld
 
-backup_root: "{{ cloud_storage }}/backup"
+backup_folder: "{{ cloud_apps }}/backup"
-backup_run_folder: "{{ cloud_storage }}"
 
-backup_storage_key: "MyStorageKey"
+#backup_client_scripts:
+#  - /path/to/script -param
+backup_client_scripts: []
+#backup_client_script_envs:
+#  - var: VAR_NAME
+#    path: /some/path
+#  - var: VAR_NAME
+#    value: some_value
+backup_client_script_envs: []
 
-backup_client: true
+backup_lock: "{{ backup_folder }}/backup.lock"
-backup_host: "example"
+
-backup_cron_specialtime: "daily"
+#backup_targets:
-backup_cron_owner: "{{ backup_owner }}"
+#  db:
+#    - type: mariadb
+#      name: db_name_mdb
+#    - type: pgsql
+#      name: db_name_pg
+#  mail:
+#    - type: file
+#      path: /path/to/some/file
+#    - type: mariadb
+#      name: db_name_mdb_mail
+#  file:
+#    - /path/to/some/file
+#    - /path/to/some/dir
+backup_targets:
+  db: []
+  mail: []
+  file: []

tasks/client.yml

@@ -1,46 +1,38 @@
 ---
-- name: Setup backup storage password
+- name: generate backup ssh user
-  copy:
+  user:
-    content: "{{ backup_storage_key }}"
+    name: "{{ backup_owner }}"
-    dest: "{{ backup_home }}/.borg.key"
-    owner: "{{ backup_owner }}"
     group: "{{ backup_group }}"
-    mode: "0600"
+    comment: Backup user
+    shell: /bin/sh
 
-- name: Setup ssh folder
+- name: Setup User ssh_key
-  file:
+  authorized_key: 
-    state: directory
+    user: "{{ backup_owner }}"
-    path: "{{ backup_home }}/.ssh"
+    key: "{{ backup_owner_ssh_key }}"
-    owner: "{{ backup_owner }}"
-    group: "{{ backup_group }}"
-    mode: "0700"
 
-- name: Setup ssh keys
+- name: generate config lists
-  copy:
+  template:
-    content: "{{ item.content }}"
+    src: backup/conf/{{ conf }}.list.j2
-    dest: "{{ backup_home }}/.ssh/{{ item.file }}"
+    dest: "{{ backup_conf }}/{{ conf }}.list"
-    owner: "{{ backup_owner }}"
-    group: "{{ backup_group }}"
-    mode: "0600"
   loop:
-    - content: "{{ backup_owner_ssh_private_key }}"
+    - filedir
-      file: "id_rsa"
+    - db
-    - content: "{{ backup_owner_ssh_public_key }}"
+    - mail
-      file: "id_rsa.pub"
   loop_control:
-    label: "{{ item.file }}"
+    loop_var: conf
+    label: "{{ conf }}.list"
 
 - name: setup backup script
   template:
-    src: "./usr/local/bin/{{ backup_host }}-backup.sh.j2"
+    src: backup/backup_on_client.sh.j2
-    dest: "/usr/local/bin/backup.sh"
+    dest: "{{ backup_folder }}/backup.sh"
-    owner: "{{ backup_owner }}"
+    mode: 0740
-    group: "{{ backup_group }}"
-    mode: "0750"
 
-- name: setup cron backup job
+- name: setup cronjob for client backup generation
   cron:
-    name: "{{ backup_host }} backup"
+    name: "backup"
-    user: "{{ backup_cron_owner }}"
+    weekday: "{{ backup_times_weekday }}"
-    job: "/usr/local/bin/backup.sh"
+    hour: "{{ backup_times_hour | join(',') }}"
-    special_time: "{{ backup_cron_specialtime }}"
+    minute: "{{ backup_times_minute }}"
+    job: "{{ backup_folder }}/backup.sh"

tasks/main.yml

@@ -1,43 +1,34 @@
 ---
+# Basics
 - name: install requirements for backup system
   apt:
     update_cache: yes
     state: "{% if cloud_update | bool %}latest{% else %}present{% endif %}"
     install_recommends: yes
     pkg:
-      - borgbackup
+      - rsync
 
-- name: setup backup location
+- name: setup storage folders for backup
   file:
     state: directory
-    path: "{{ backup_root }}"
+    path: "{{ dir.value.path }}"
-    owner: "root"
-    group: "root"
-    mode: "0755"
-
-- name: Setup required groups of users
-  group:
-    name: "{{ backup_group }}"
-    state: present
-
-- name: generate backup user
-  user:
-    name: "{{ backup_owner }}"
-    group: "{{ backup_group }}"
-    comment: backup user
-    shell: /sbin/nologin
-    home: "{{ backup_home }}"
-
-- name: setup backup storage
-  file:
-    state: directory
-    path: "{{ backup_storage }}"
     owner: "{{ backup_owner }}"
     group: "{{ backup_group }}"
-    mode: "0777"
+    mode: "{{ backup_permission }}"
+  with_dict: "{{ backup_storage }}"
+  loop_control:
+    loop_var: dir
+    label: "{{ dir.key }}"
 
-- include_tasks: master.yml
+- name: setup conf folder
-  when: not backup_client
+  file:
+    state: directory
+    path: "{{ backup_folder }}/conf"
 
-- include_tasks: client.yml
+- name: setup client backup structure
+  include_tasks: client.yml
   when: backup_client
+
+- name: setup server backup structure
+  include_tasks: server.yml
+  when: not backup_client

tasks/master.yml

@@ -1 +0,0 @@
----

tasks/server.yml

@@ -0,0 +1,27 @@
+---
+- name: analyse backup hours for pulling server
+  set_fact:
+    backup_times_server_hour: "{{ backup_times_server_hour | default([]) + [item + backup_hour_shift] }}"
+  loop: "{{ backup_times_hour }}"
+
+- name: generate backup user ssh key
+  user:
+    name: "{{ backup_owner }}"
+    generate_ssh_key: yes
+    ssh_key_type: rsa
+    ssh_key_bits: 4096
+    home: "{{ backup_folder }}"
+
+- name: template main server backup pull script
+  template:
+    src: backup/backup_from_clients.sh.j2
+    dest: "{{ backup_folder }}/pull-backup.sh"
+    mode: 0740
+
+- name: setup cronjob for server backup pull
+  cron:
+    name: "pull backups"
+    weekday: "{{ backup_times_weekday }}"
+    hour: "{{ backup_times_server_hour | join(',') }}"
+    minute: "{{ backup_times_minute }}"
+    job: "{{ backup_folder }}/pull-backup.sh"

templates/backup/backup_from_clients.sh.j2

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+cd {{ backup_folder }}
+if [ -f {{ backup_lock }} ]; then
+    echo "Backup lock! Cannot procceed as lock '{{ backup_lock }}' exist."
+    echo "Associated PID is $(cat {{ backup_lock }})"
+    exit 1
+fi
+
+echo "$$" > {{ backup_lock }}
+BKPTIMESTAMP=$(date +"%y-%m-%d.%H")
+BKP=$BKPTIMESTAMP"_backup"
+
+
+# generate rolling folder
+{% for target in backup_targets %}
+{% for host in backup_hosts %}
+{% if 'domain' in host %}
+mkdir -p {{ target }}/{{ host.name + '_' + host.domain }}/$BKP
+{% elif 'ip' in host %}
+mkdir -p {{ target }}/{{ host.name + '_' + host.ip }}/$BKP
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+
+# pull backups
+{% for target in backup_targets %}
+{% for host in backup_hosts %}
+{% if 'domain' in host %}
+rsync --rsh='ssh -p{{ ssh_port }} -i ~{{ backup_owner }}/.ssh/id_rsa'  -chav --stats --ignore-missing-args {{ backup_owner }}@{{ host.domain }}:{{ backup_folder }}/{{ target }}.tgz {{ target }}/{{ host.name + '_' + host.domain }}/$BKP
+{% elif 'ip' in host %}
+rsync --rsh='ssh -p{{ ssh_port }} -i ~{{ backup_owner }}/.ssh/id_rsa' -chav --stats --ignore-missing-args {{ backup_owner }}@{{ host.ip }}:{{ backup_folder }}/{{ target }}.tgz {{ target }}/{{ host.name + '_' + host.ip }}/$BKP
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+
+# link to latest
+{% for target in backup_targets %}
+{% for host in backup_hosts %}
+{% if 'domain' in host %}
+ln -sf {{ backup_folder }}/{{ target }}/{{ host.name + '_' + host.domain }}/$BKP {{ backup_folder }}/{{ target }}/{{ host.name + '_' + host.domain }}/LATEST
+{% elif 'ip' in host %}
+ln -sf {{ backup_folder }}/{{ target }}/{{ host.name + '_' + host.ip }}/$BKP {{ backup_folder }}/{{ target }}/{{ host.name + '_' + host.ip }}/LATEST
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+
+# keep only last {{ backup_keep_days * ( backup_times_hour | length ) }}
+{% for target in backup_targets %}
+{% for host in backup_hosts %}
+{% if 'domain' in host %}
+DELETE_LIST=$(ls -1dtr {{ target }}/{{ host.name + '_' + host.domain }}/* | head -n -{{ backup_keep_days * ( backup_times_hour | length ) }})
+{% elif 'ip' in host %}
+DELETE_LIST=$(ls -1dtr {{ target }}/{{ host.name + '_' + host.ip }}/* | head -n -{{ backup_keep_days * ( backup_times_hour | length ) }})
+{% endif %}
+if (( $( echo $DELETE_LIST | wc -w ) > 0 )); then
+    rm -rf $DELETE_LIST
+fi
+{% endfor %}
+{% endfor %}
+
+
+rm {{ backup_lock }}

templates/backup/backup_on_client.sh.j2

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+if [ -f {{ backup_lock }} ]; then
+    echo "Backup lock! Cannot procceed as lock '{{ backup_lock }}' exist."
+    echo "Associated PID is $(cat {{ backup_lock }})"
+    exit 1
+fi
+echo "$$" > {{ backup_lock }}
+
+cd {{ backup_folder }}
+
+# Load script envs
+echo "Load script environment variables"
+{% for scr_env in backup_client_script_envs %}
+{% if 'path' in scr_env %}
+export {{ scr_env.var }}={{ scr_env.path }}:${{ scr_env.var }}
+{% else %}
+export {{ scr_env.var }}={{ scr_env.value }}
+{% endif %}
+{% endfor %}
+
+
+# Perform client pre backup scripts for
+echo "Perform pre-backup scripts"
+{% for cmd in backup_client_scripts %}
+{{ cmd }};
+{% endfor %}
+
+
+# Backup files and directories
+echo "Copy files and directories"
+while read DIRFILE; do
+    echo " - Copy $DIRFILE"
+    BACKUP_SUBDIR=$(dirname $DIRFILE)
+    mkdir -p "{{ backup_storage.file.path }}/$BACKUP_SUBDIR"
+    cp -r -u $DIRFILE "{{ backup_storage.file.path }}/$DIRFILE"
+done < {{ backup_conf }}/filedir.list
+rm -f file.tgz #remove old backup
+tar czfv file.tgz file
+rm -rf file/*
+
+# Backup databases
+echo "Generate DB dumps"
+while read DBS; do
+    DBTYPE=$(echo $DBS | cut -f 1 -d "{{ backup_conf_db_sep }}" );
+    DBNAME=$(echo $DBS | cut -f 2 -d "{{ backup_conf_db_sep }}" );
+    if [ $DBTYPE == "mariadb" ] || [ $DBTYPE == "mysql" ]; then
+        echo "- dump to {{ backup_storage.db.path }}/$DBTYPE/$DBNAME.sql"
+        mkdir -p {{ backup_storage.db.path }}/$DBTYPE;
+        mysqldump $DBNAME | gzip -c > {{ backup_storage.db.path }}/$DBTYPE/$DBNAME.sql.gz;
+    elif [ $DBTYPE == "pgsql" ]; then
+        echo "- dump to {{ backup_storage.db.path }}/$DBTYPE/$DBNAME.sql"
+        mkdir -p {{ backup_storage.db.path }}/$DBTYPE;
+        sudo -u postgres bash -c "pg_dump $DBNAME | gzip -c > /tmp/$DBNAME.sql.gz";
+        mv /tmp/$DBNAME.sql.gz /backup/db/$DBTYPE/$DBNAME.sql.gz;
+    else
+        echo "- ERROR! DB type '$DBTYPE' for '$DBNAME' is unknown to backup."
+    fi
+done < {{ backup_conf }}/db.list
+rm -f db.tgz #remove old backup
+tar czfv db.tgz db
+rm -rf db/*
+
+#Mail
+rm -f mail.tgz #remove old backup
+tar czfv mail.tgz mail
+rm -rf mail/*
+
+rm {{ backup_lock }}
+chown {{ backup_owner }}:{{ backup_group }} {{ backup_folder }}/*.tgz

templates/backup/conf/db.list.j2

@@ -0,0 +1,5 @@
+{% if 'db' in backup_targets %}
+{% for db in backup_targets.db %}
+{{ db.type }}{{ backup_conf_db_sep }}{{ db.name }}
+{% endfor %}
+{% endif %}

templates/backup/conf/filedir.list.j2

@@ -0,0 +1,5 @@
+{% if 'file' in backup_targets %}
+{% for filedir in backup_targets.file %}
+{{ filedir }}
+{% endfor %}
+{% endif %}

templates/backup/conf/mail.list.j2

@@ -0,0 +1,5 @@
+{% if 'mail' in backup_targets %}
+{% for mail in backup_targets.mail %}
+{{ mail }}
+{% endfor %}
+{% endif %}

templates/usr/local/bin/email-backup.sh.j2

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-BORGUSER="{{ backup_owner }}";
-RUNFOLDER="{{ backup_run_folder }}";
-TARGETFOLDER="{{ backup_storage }}";
-REPOLOCATION="{{ backup_location }}";
-ARCHIVENAME="mailcow-$(date '+%s')";
-
-cd $RUNFOLDER;
-MAILCOW_BACKUP_LOCATION="$TARGETFOLDER/" ./helper-scripts/backup_and_restore.sh backup all;
-LATESTBACKUP="$(ls -t $TARGETFOLDER | head -n 1)";
-
-if [[ "$LATESTBACKUP" != *"mailcow"* ]];
-then
-    echo "NOT MAILCOW! ABORT!";
-    exit 1;
-fi
-
-chown -R $BORGUSER: $TARGETFOLDER/$LATESTBACKUP;
-sudo -H -u $BORGUSER bash -c '
-TARGETFOLDER='$TARGETFOLDER';
-REPOLOCATION='$REPOLOCATION';
-ARCHIVENAME='$ARCHIVENAME';
-export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
-borg create -C lzma $REPOLOCATION::$ARCHIVENAME $TARGETFOLDER/$LATESTBACKUP';
-rm -rf $TARGETFOLDER/$LATESTBACKUP;

templates/usr/local/bin/idp-backup.sh.j2

@@ -1 +0,0 @@
-#!/bin/bash

templates/usr/local/bin/vault-backup.sh.j2

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-BORGUSER="{{ backup_owner }}";
-RUNFOLDER="{{ backup_run_folder }}";
-TARGETFOLDER="{{ backup_storage }}";
-REPOLOCATION="{{ backup_location }}";
-ARCHIVENAME="vault-$(date '+%s')";
-
-cd $RUNFOLDER;
-
-cp -r $RUNFOLDER/home $TARGETFOLDER/$ARCHIVENAME;
-chown -R $BORGUSER: $TARGETFOLDER/$ARCHIVENAME;
-
-sudo -H -u $BORGUSER bash -c '
-TARGETFOLDER='$TARGETFOLDER';
-REPOLOCATION='$REPOLOCATION';
-ARCHIVENAME='$ARCHIVENAME';
-export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
-borg create -C lzma $REPOLOCATION::$ARCHIVENAME $TARGETFOLDER/$ARCHIVENAME';
-rm -rf $TARGETFOLDER/$ARCHIVENAME;

vars/main.yml

@@ -1,3 +1,12 @@
 ---
-backup_home: "{{ backup_root }}/home"
+backup_conf: "{{ backup_folder }}/conf"
-backup_storage: "{{ backup_root }}/storage"
+
+backup_conf_db_sep: ":"
+
+backup_storage:
+  mail:
+    path: "{{ backup_folder }}/mail"
+  db:
+    path: "{{ backup_folder }}/db"
+  file:
+    path: "{{ backup_folder }}/file"