Setup backup server configuration for backup replication.

defaults/main.yml

@@ -26,3 +26,19 @@ backup_script:
     echo "This is executed before borg backup. Please collect data for backup in path: {{ backup_storage }}"
   postwork_restore: |
     echo "This is executed after borg restore. Please collect data during restore from path: {{ backup_storage }}"
+
+backup_ssh_port: 22
+backup_host: my-borg-repo.tld
+backup_clients: []
+#  app1:
+#    owner: user1
+#    app: app1
+#    borgkey: BorgRepoKey1
+#    sshpubkey: SshPubKey1
+#    sshprivkey: SshPrivKey1
+#  app2:
+#    owner: user2
+#    app: app2
+#    borgkey: BorgRepoKey2
+#    sshpubkey: SshPubKey2
+#    sshprivkey: SshPrivKey2

files/opt/backup/filter_backups.py

@@ -0,0 +1,117 @@
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+from argparse import ArgumentParser, FileType
+
+def filter_backups_timeslot(offset, backups, timeslot='days', keep=7, mode='max', printy=True):
+    keep_backups = []
+    for timeslot_off in range(1, keep):
+        indices = [
+            idx
+            for idx,value in enumerate(backups)
+            if (
+                value[1] >= (offset - relativedelta(**{timeslot:timeslot_off}))
+            )
+        ]
+        if not indices:
+            continue
+        if mode == "min":
+            #removed not considered backups for this slot!
+            for idx in reversed(indices[1:]):
+                backups.pop(idx)
+            keep_backups.append(backups.pop(indices[0]))
+        elif mode == "max":
+            keep_backups.append(backups.pop(indices[-1]))
+            #removed not considered backups for this slot!
+            for idx in reversed(indices[:-1]):
+                backups.pop(idx)
+        else:
+            for idx in sorted(indices, reverse=True):
+                keep_backups.append(backups.pop(idx))
+    return keep_backups, backups
+
+def filter_backups(backuplist, show_keep=True, keep_hours=24, keep_days=7, keep_weeks=4, keep_months=12, keep_years=10):
+    backups = sorted(
+        [
+            (backup, datetime.fromtimestamp(int(backup.split('-')[1])))
+            for backup in backuplist
+        ], key=lambda v: v[1], reverse=True
+    )
+    offset = backups[0][1]
+    keep_backups = [backups.pop(0)]
+
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='hours',keep=keep_hours,mode='all')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='days',keep=keep_days,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='weeks',keep=keep_weeks,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='months',keep=keep_months,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='years',keep=keep_years,mode='max')
+    keep_backups.extend(part_keep_backups)
+
+    keeps = [backup[0] for backup in keep_backups]
+    if not show_keep:
+        return [backup for backup in backuplist if backup not in keeps]
+    return keeps
+
+def main():
+    parser = ArgumentParser(description="Program to print backups to keep or delete based on time stamps.")
+    parser.add_argument(
+        "-f", "--file", metavar="FILE", type=FileType('r'), required=True,
+        help="Path to file with list of backups to filter. Expects per line one backup name in format <appname>-<timestamp>")
+    parser.add_argument(
+        "-d", "--print-delete", action="store_true", required=False,
+        help="Instead of objects to kept, print out objects to be deleted."
+    )
+    parser.add_argument(
+        "-H", "--hours-keep", metavar="KEEP", type=int, required=False,
+        default=24, help="Number of objects to keep on hourly basis."
+    )
+    parser.add_argument(
+        "-D", "--days-keep", metavar="KEEP", type=int, required=False,
+        default=7, help="Number of objects to keep on daily basis."
+    )
+    parser.add_argument(
+        "-W", "--weeks-keep", metavar="KEEP", type=int, required=False,
+        default=4, help="Number of objects to keep on weekly basis."
+    )
+    parser.add_argument(
+        "-M", "--months-keep", metavar="KEEP", type=int, required=False,
+        default=12, help="Number of objects to keep on monthly basis."
+    )
+    parser.add_argument(
+        "-Y", "--years-keep", metavar="KEEP", type=int, required=False,
+        default=10, help="Number of objects to keep on yearly basis."
+    )
+
+    args = parser.parse_args()
+    show_keep = not args.print_delete
+    keep_hours = args.hours_keep
+    keep_days = args.days_keep
+    keep_weeks = args.weeks_keep
+    keep_months = args.months_keep
+    keep_years = args.years_keep
+
+    backuplist = [
+        line.strip()
+        for line in args.file
+        if len(line.strip()) > 0
+    ]
+    fail = False
+    for idx,backup in enumerate(backuplist):
+        if len(backup) < 10 or not backup[-10:].isnumeric():
+            print(f"WARNING! Line {idx+1} has no timestamp suffix")
+            fail = True
+    if fail:
+        exit(1)
+
+    considered_backups = filter_backups(
+        backuplist, show_keep,
+        keep_hours, keep_days, keep_weeks, keep_months, keep_years
+    )
+    for backup in considered_backups:
+        print(backup)
+
+if __name__ == '__main__':
+    main()

tasks/client.yml

@@ -7,14 +7,6 @@
     group: "{{ backup_group }}"
     mode: "0600"
 
-- name: Setup ssh folder
-  file:
-    state: directory
-    path: "{{ backup_home }}/.ssh"
-    owner: "{{ backup_owner }}"
-    group: "{{ backup_group }}"
-    mode: "0700"
-
 - name: Setup ssh keys
   copy:
     content: "{{ item.content }}"

tasks/main.yml

@@ -25,9 +25,17 @@
     name: "{{ backup_owner }}"
     group: "{{ backup_group }}"
     comment: backup user
-    shell: /sbin/nologin
+    shell: /bin/bash
     home: "{{ backup_home }}"
 
+- name: Setup ssh folder
+  file:
+    state: directory
+    path: "{{ backup_home }}/.ssh"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0700"
+
 - name: setup backup storage
   file:
     state: directory
@@ -36,7 +44,7 @@
     group: "{{ backup_group }}"
     mode: "0777"
 
-- include_tasks: master.yml
+- include_tasks: server.yml
   when: not backup_client
 
 - include_tasks: client.yml

tasks/master.yml

@@ -1 +0,0 @@
----

tasks/server.yml

@@ -0,0 +1,74 @@
+---
+- name: setup backup installation folder
+  file:
+    state: directory
+    path: "{{ backup_inst }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+
+- name: install dateutil library service
+  apt:
+    update_cache: yes
+    state: "{% if cloud_update | bool %}latest{% else %}present{% endif %}"
+    install_recommends: yes
+    pkg: python3-dateutil
+
+- name: setup filter program for backup pruning
+  copy:
+    src: opt/backup/filter_backups.py
+    dest: "{{ backup_inst }}/filter_backups.py"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+
+- name: setup ssh public keys of clients
+  copy:
+    content: "{{ backup_clients[item].sshpubkey }}"
+    dest: "{{ backup_home }}/.ssh/{{ item }}.pub"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_clients.keys() }}"
+
+- name: setup ssh private keys of clients
+  copy:
+    content: "{{ backup_clients[item].sshprivkey }}"
+    dest: "{{ backup_home }}/.ssh/{{ item }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_clients.keys() }}"
+
+- name: setup ssh config of clients
+  template:
+    src: "home/backup/.ssh/config.j2"
+    dest: "{{ backup_home }}/.ssh/config"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+
+- name: setup backup key folder
+  file:
+    state: directory
+    path: "{{ backup_key_folder }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0700"
+
+- name: setup borg repository keys
+  copy:
+    content: "{{ backup_clients[item].borgkey }}"
+    dest: "{{ backup_key_folder }}/{{ backup_clients[item].app }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_clients.keys() }}"
+
+- name: setup borg server script
+  template:
+    src: "opt/backup/inst/cloud-server-backup.j2"
+    dest: "{{ backup_inst }}/cloud-server-backup"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"

templates/home/backup/.ssh/config.j2

@@ -0,0 +1,7 @@
+{% for client in backup_clients %}
+Host {{ client }}
+    HostName {{ backup_host }}
+    Port {{ backup_ssh_port }}
+    User {{ backup_clients[client].owner }}
+    IdentityFile {{ backup_home }}/.ssh/{{ client }}
+{% endfor %}

templates/opt/backup/inst/cloud-server-backup.j2

@@ -0,0 +1,166 @@
+#!/bin/bash
+
+BORGUSER="{{ backup_owner }}";
+RUNFOLDER="{{ backup_storage }}";
+APPLIST="{{ backup_clients | json_query('*.app') | join(' ') }}"
+
+declare -A BORG_CLIENT_MAP
+{% for client in backup_clients %}
+BORG_CLIENT_MAP[{{ backup_clients[client].app }}]="{{ client }}"
+{% endfor %}
+
+declare -A BORG_USER_MAP
+{% for client in backup_clients %}
+BORG_USER_MAP[{{ backup_clients[client].app }}]="{{ backup_clients[client].owner }}"
+{% endfor %}
+
+
+help (){
+    echo "cloud-server-backup - backup and restore script on cloud backup with borg target v1.0 by L.Hahn.
+
+Usage: $0 COMMAND [APPS]
+
+COMMAND:
+    - remote-apps-list              List available apps in remote borg repositories.
+    - remote-list [APPS]            List available archives for apps in remote borg repositories.
+    - remote-prune [APPS]           Prune remote app (or all) borg repositories.
+    - list                          List available archives of remote repositories on local repository.
+    - backup                        Perform backup of remote borg repositories and create a new archive in local borg repository.
+    - prune                         Prune local borg backup of remote repositories.
+    - restore ARCHIVENAME [APPS]    Upload backup to borg repository to restore older file states.
+                                        May turn off your application if still running.
+                                        If latest is provided as ARCHIVENAME, the latest one based on timestamp is taken.
+                                        IF ARCHIVENAME is provided, will try to download it; throws error if not found.
+                                        If APP is provided, only the mentioned app repositories are restored.
+
+
+APPS:                               A single app or comma-separated (sub-)set of below listed apps to be considered.
+{% for client in backup_clients %}
+    - {{ backup_clients[client].app }}
+{% endfor %}
+";
+}
+
+
+remote_list_app () {
+    for APP in $APPLIST;
+    do
+        echo $APP;
+    done
+}
+
+remote_list () {
+    REMOTELIST=$1;
+    for APP in $REMOTELIST;
+    do
+        REPOSITORYCLIENT=${BORG_CLIENT_MAP[$APP]}
+        BORG_REPO_USER=${BORG_USER_MAP[$APP]}
+        REPOLOCATION="ssh://$BORG_REPO_USER@{{ backup_host }}:{{ backup_ssh_port }}/./$APP"
+        BORG_PASSPHRASE=$(cat {{ backup_key_folder }}/$APP)
+        KEYFILE="{{ backup_home }}/.ssh/$REPOSITORYCLIENT"
+        sudo -H -u $BORGUSER bash -c '
+        export BORG_PASSPHRASE='$BORG_PASSPHRASE';
+        KEYFILE='$KEYFILE';
+        REPOLOCATION='$REPOLOCATION';
+        borg list $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"'
+    done
+}
+
+remote_prune () {
+    REMOTEPRUNELIST=$1;
+    for APP in $REMOTEPRUNELIST;
+    do
+        ARCHIVLIST=$(remote_list $APP | cut -f 1 -d ' ');
+        echo "$ARCHIVLIST" > {{ backup_inst }}/backuplist.txt;
+        python3 {{ backup_inst }}/filter_backups.py -f {{ backup_inst }}/backuplist.txt -d;
+    done
+}
+
+
+has_argument () {
+    if [[ -z "$1" ]];
+    then
+        echo "Missing required paramter!";
+        exit 1;
+    fi
+}
+
+validate_apps_or_default () {
+    VALID=1;
+    if [[ -z "$1" ]];
+    then
+        echo $APPLIST;
+    else
+        CHECKLIST=$(echo $1 | tr ',' ' ');
+        for APP in $CHECKLIST;
+        do
+            if [[ ! ${APPLIST[@]} =~ $APP ]]
+            then
+                VALID=0;
+                echo "$APP is not a valid app.";
+            fi
+        done
+        if [[ ! $VALID == 1 ]]
+        then
+            echo "Aborting.";
+            exit 1;
+        else
+            echo $CHECKLIST;
+        fi
+    fi
+}
+
+(
+    flock -n 9 || {
+        echo "BACKUP ALREADY RUNNING! ABORTING.";
+        exit 1;
+    }
+
+    if [ $# = 0 ]; then
+        help;
+        exit 0;
+    fi
+
+    action=$1
+    case $action in
+        "remote-apps-list")
+            remote_list_app;
+            ;;
+
+        "remote-list")
+            USERLIST=$(validate_apps_or_default "$2");
+            remote_list "$USERLIST";
+            ;;
+
+        "remote-prune")
+            USERLIST=$(validate_apps_or_default "$2");
+            remote_prune "$USERLIST";
+            ;;
+
+        "list")
+            #local_list;
+            ;;
+
+        "prune")
+            #local_prune;
+            ;;
+
+        "backup")
+            #local_backup;
+            ;;
+
+        "restore")
+            has_argument $2;
+            USERLIST=$(validate_apps_or_default $3);
+            #local_restore $2;
+            ;;
+        "delete")
+            has_argument $2;
+            #local_delete $2;
+            ;;
+        *)
+            help;
+            exit 0;
+            ;;
+    esac
+) 9>/var/run/lock/cloud-server-backup.lock;

templates/usr/local/bin/cloud_backup.j2

@@ -143,5 +143,4 @@ has_argument () {
             exit 0;
             ;;
     esac
-    echo ""
 ) 9>/var/run/lock/cloud-backup.lock;

vars/main.yml

@@ -1,3 +1,5 @@
 ---
 backup_home: "{{ backup_root }}/home"
-backup_storage: "{{ backup_root }}/storage"
+backup_inst: "{{ backup_root }}/inst"
+backup_storage: "{{ backup_root }}/storage"
+backup_key_folder: "{{ backup_home }}/.keys"