Backup server: skip self-pruning repos in remote-prune

backup_server_remote_skip lists apps whose client prunes the repository itself; remote-prune without arguments leaves them alone (an explicit app list still works).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

defaults/main.yml

@@ -20,9 +20,37 @@ backup_client: true
 backup_app: app
 backup_cron_specialtime: "daily"
 backup_cron_owner: "{{ backup_owner }}"
+backup_server_remote_prune_time: 6
+backup_server_remote_backup_time: 8
+backup_server_local_prune_time: 10
+#server: clients to prune/mirror (defaults to backup_clients), apps not mirrored locally, local compression
+backup_server_clients: "{{ backup_clients }}"
+backup_server_local_skip: []
+#apps whose client prunes its own repository; remote-prune (without arguments) skips them
+backup_server_remote_skip: []
+backup_server_compression: lzma
+#local mirror of the remote repositories (defaults to <backup_root>/storage/remote-backups)
+#backup_server_mirror: /mnt/nas/backup/mirror
+
 
 backup_script:
   prework_backup: |
     echo "This is executed before borg backup. Please collect data for backup in path: {{ backup_storage }}"
   postwork_restore: |
     echo "This is executed after borg restore. Please collect data during restore from path: {{ backup_storage }}"
+
+backup_ssh_port: 22
+backup_host: my-borg-repo.tld
+backup_clients: []
+#  app1:
+#    owner: user1
+#    app: app1
+#    borgkey: BorgRepoKey1
+#    sshpubkey: SshPubKey1
+#    sshprivkey: SshPrivKey1
+#  app2:
+#    owner: user2
+#    app: app2
+#    borgkey: BorgRepoKey2
+#    sshpubkey: SshPubKey2
+#    sshprivkey: SshPrivKey2

files/opt/backup/filter_backups.py

@@ -0,0 +1,117 @@
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+from argparse import ArgumentParser, FileType
+
+def filter_backups_timeslot(offset, backups, timeslot='days', keep=7, mode='max', printy=True):
+    keep_backups = []
+    for timeslot_off in range(1, keep):
+        indices = [
+            idx
+            for idx,value in enumerate(backups)
+            if (
+                value[1] >= (offset - relativedelta(**{timeslot:timeslot_off}))
+            )
+        ]
+        if not indices:
+            continue
+        if mode == "min":
+            #removed not considered backups for this slot!
+            for idx in reversed(indices[1:]):
+                backups.pop(idx)
+            keep_backups.append(backups.pop(indices[0]))
+        elif mode == "max":
+            keep_backups.append(backups.pop(indices[-1]))
+            #removed not considered backups for this slot!
+            for idx in reversed(indices[:-1]):
+                backups.pop(idx)
+        else:
+            for idx in sorted(indices, reverse=True):
+                keep_backups.append(backups.pop(idx))
+    return keep_backups, backups
+
+def filter_backups(backuplist, show_keep=True, keep_hours=24, keep_days=7, keep_weeks=4, keep_months=12, keep_years=10):
+    backups = sorted(
+        [
+            (backup, datetime.fromtimestamp(int(backup.split('-')[1])))
+            for backup in backuplist
+        ], key=lambda v: v[1], reverse=True
+    )
+    offset = backups[0][1]
+    keep_backups = [backups.pop(0)]
+
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='hours',keep=keep_hours,mode='all')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='days',keep=keep_days,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='weeks',keep=keep_weeks,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='months',keep=keep_months,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='years',keep=keep_years,mode='max')
+    keep_backups.extend(part_keep_backups)
+
+    keeps = [backup[0] for backup in keep_backups]
+    if not show_keep:
+        return [backup for backup in backuplist if backup not in keeps]
+    return keeps
+
+def main():
+    parser = ArgumentParser(description="Program to print backups to keep or delete based on time stamps.")
+    parser.add_argument(
+        "-f", "--file", metavar="FILE", type=FileType('r'), required=True,
+        help="Path to file with list of backups to filter. Expects per line one backup name in format <appname>-<timestamp>")
+    parser.add_argument(
+        "-d", "--print-delete", action="store_true", required=False,
+        help="Instead of objects to kept, print out objects to be deleted."
+    )
+    parser.add_argument(
+        "-H", "--hours-keep", metavar="KEEP", type=int, required=False,
+        default=24, help="Number of objects to keep on hourly basis."
+    )
+    parser.add_argument(
+        "-D", "--days-keep", metavar="KEEP", type=int, required=False,
+        default=7, help="Number of objects to keep on daily basis."
+    )
+    parser.add_argument(
+        "-W", "--weeks-keep", metavar="KEEP", type=int, required=False,
+        default=4, help="Number of objects to keep on weekly basis."
+    )
+    parser.add_argument(
+        "-M", "--months-keep", metavar="KEEP", type=int, required=False,
+        default=12, help="Number of objects to keep on monthly basis."
+    )
+    parser.add_argument(
+        "-Y", "--years-keep", metavar="KEEP", type=int, required=False,
+        default=10, help="Number of objects to keep on yearly basis."
+    )
+
+    args = parser.parse_args()
+    show_keep = not args.print_delete
+    keep_hours = args.hours_keep
+    keep_days = args.days_keep
+    keep_weeks = args.weeks_keep
+    keep_months = args.months_keep
+    keep_years = args.years_keep
+
+    backuplist = [
+        line.strip()
+        for line in args.file
+        if len(line.strip()) > 0
+    ]
+    fail = False
+    for idx,backup in enumerate(backuplist):
+        if len(backup) < 10 or not backup[-10:].isnumeric():
+            print(f"WARNING! Line {idx+1} has no timestamp suffix")
+            fail = True
+    if fail:
+        exit(1)
+
+    considered_backups = filter_backups(
+        backuplist, show_keep,
+        keep_hours, keep_days, keep_weeks, keep_months, keep_years
+    )
+    for backup in considered_backups:
+        print(backup)
+
+if __name__ == '__main__':
+    main()

tasks/client.yml

@@ -1,20 +1,4 @@
 ---
-- name: Setup backup storage password
-  copy:
-    content: "{{ backup_storage_key }}"
-    dest: "{{ backup_home }}/.borg.key"
-    owner: "{{ backup_owner }}"
-    group: "{{ backup_group }}"
-    mode: "0600"
-
-- name: Setup ssh folder
-  file:
-    state: directory
-    path: "{{ backup_home }}/.ssh"
-    owner: "{{ backup_owner }}"
-    group: "{{ backup_group }}"
-    mode: "0700"
-
 - name: Setup ssh keys
   copy:
     content: "{{ item.content }}"

tasks/main.yml

@@ -25,9 +25,25 @@
     name: "{{ backup_owner }}"
     group: "{{ backup_group }}"
     comment: backup user
-    shell: /sbin/nologin
+    shell: /bin/bash
     home: "{{ backup_home }}"
 
+- name: Setup backup storage password
+  copy:
+    content: "{{ backup_storage_key }}"
+    dest: "{{ backup_home }}/.borg.key"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+
+- name: Setup ssh folder
+  file:
+    state: directory
+    path: "{{ backup_home }}/.ssh"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0700"
+
 - name: setup backup storage
   file:
     state: directory
@@ -36,7 +52,7 @@
     group: "{{ backup_group }}"
     mode: "0777"
 
-- include_tasks: master.yml
+- include_tasks: server.yml
   when: not backup_client
 
 - include_tasks: client.yml

tasks/master.yml

@@ -1 +0,0 @@
----

tasks/server.yml

@@ -0,0 +1,118 @@
+---
+- name: setup backup installation folder
+  file:
+    state: directory
+    path: "{{ backup_inst }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+
+- name: install dateutil library service
+  apt:
+    update_cache: yes
+    state: "{% if cloud_update | bool %}latest{% else %}present{% endif %}"
+    install_recommends: yes
+    pkg:
+      - python3-dateutil
+      - rsync
+
+- name: setup filter program for backup pruning
+  copy:
+    src: opt/backup/filter_backups.py
+    dest: "{{ backup_inst }}/filter_backups.py"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+
+- name: setup ssh public keys of clients
+  copy:
+    content: "{{ backup_server_clients[item].sshpubkey }}"
+    dest: "{{ backup_home }}/.ssh/{{ item }}.pub"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_server_clients.keys() }}"
+
+- name: setup ssh private keys of clients
+  copy:
+    content: "{{ backup_server_clients[item].sshprivkey }}"
+    dest: "{{ backup_home }}/.ssh/{{ item }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_server_clients.keys() }}"
+
+- name: setup ssh config of clients
+  template:
+    src: "home/backup/.ssh/config.j2"
+    dest: "{{ backup_home }}/.ssh/config"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+
+- name: setup backup key folder
+  file:
+    state: directory
+    path: "{{ backup_key_folder }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0700"
+
+- name: setup borg repository keys
+  copy:
+    content: "{{ backup_server_clients[item].borgkey }}"
+    dest: "{{ backup_key_folder }}/{{ backup_server_clients[item].app }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_server_clients.keys() }}"
+
+- name: setup local backup storage folder
+  file:
+    state: directory
+    path: "{{ item }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+  loop:
+    - "{{ backup_server_mirror | default(backup_remotes_folder) }}"
+    - "{{ backup_location }}/{{ backup_app }}"
+
+- name: setup borg local repository
+  shell:
+    cmd: "BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key) borg init --encryption repokey {{ backup_location }}/{{ backup_app }}"
+    creates: "{{ backup_location }}/{{ backup_app }}/config"
+  become_user: "{{ backup_owner }}"
+  become: yes
+
+- name: setup borg server script
+  template:
+    src: "opt/backup/inst/cloud-server-backup.j2"
+    dest: "{{ backup_inst }}/cloud-server-backup"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+
+- name: setup remote prune cron backup server job
+  cron:
+    name: "cloud server remote prune"
+    user: "{{ backup_cron_owner }}"
+    job: "{{ backup_inst }}/cloud-server-backup remote-prune"
+    hour: "{{ backup_server_remote_prune_time }}"
+    minute: 0
+
+- name: setup remote backup cron backup server job
+  cron:
+    name: "cloud server remote backup"
+    user: "{{ backup_cron_owner }}"
+    job: "{{ backup_inst }}/cloud-server-backup backup"
+    hour: "{{ backup_server_remote_backup_time }}"
+    minute: 0
+
+- name: setup local prune cron backup server job
+  cron:
+    name: "cloud server local prune"
+    user: "{{ backup_cron_owner }}"
+    job: "{{ backup_inst }}/cloud-server-backup prune"
+    hour: "{{ backup_server_local_prune_time }}"
+    minute: 0

templates/home/backup/.ssh/config.j2

@@ -0,0 +1,7 @@
+{% for client in backup_server_clients %}
+Host {{ client }}
+    HostName {{ backup_host }}
+    Port {{ backup_ssh_port }}
+    User {{ backup_server_clients[client].owner }}
+    IdentityFile {{ backup_home }}/.ssh/{{ client }}
+{% endfor %}

templates/opt/backup/inst/cloud-server-backup.j2

@@ -0,0 +1,285 @@
+{% set server_clients = backup_server_clients | default(backup_clients) %}
+#!/bin/bash
+
+BORGUSER="{{ backup_owner }}";
+TARGETFOLDER="{{ backup_remotes_folder }}";
+LOCALREPOLOCATION="{{ backup_location }}/{{ backup_app }}";
+APPLIST="{{ server_clients | json_query('*.app') | join(' ') }}";
+# repos that are only pruned remotely, not mirrored locally
+LOCALAPPLIST="{{ server_clients | json_query('*.app') | difference(backup_server_local_skip) | join(' ') }}";
+# repos pruned by their own client (e.g. a NAS borg plugin), skipped by remote-prune without arguments
+REMOTEPRUNEAPPLIST="{{ server_clients | json_query('*.app') | difference(backup_server_remote_skip) | join(' ') }}";
+MIRRORFOLDER="{{ backup_server_mirror | default(backup_remotes_folder) }}";
+
+declare -A BORG_CLIENT_MAP
+{% for client in server_clients %}
+BORG_CLIENT_MAP[{{ server_clients[client].app }}]="{{ client }}";
+{% endfor %}
+
+declare -A BORG_USER_MAP
+{% for client in server_clients %}
+BORG_USER_MAP[{{ server_clients[client].app }}]="{{ server_clients[client].owner }}";
+{% endfor %}
+
+
+help (){
+    echo "cloud-server-backup - backup and restore script on cloud backup with borg target v1.0 by L.Hahn.
+
+Usage: $0 COMMAND [APPS]
+
+COMMAND:
+    - remote-apps-list              List available apps in remote borg repositories.
+    - remote-list [APPS]            List available archives for apps in remote borg repositories.
+    - remote-prune [APPS]           Prune remote app (or all) borg repositories.
+    - list                          List available archives of remote repositories on local repository.
+    - backup                        Perform backup of remote borg repositories and create a new archive in local borg repository.
+    - prune                         Prune local borg backup of remote repositories.
+    - restore ARCHIVENAME [APPS]    Upload backup to borg repository to restore older file states.
+                                        May turn off your application if still running.
+                                        If latest is provided as ARCHIVENAME, the latest one based on timestamp is taken.
+                                        IF ARCHIVENAME is provided, will try to download it; throws error if not found.
+                                        If APP is provided, only the mentioned app repositories are restored.
+
+
+APPS:                               A single app or comma-separated (sub-)set of below listed apps to be considered.
+{% for client in server_clients %}
+    - {{ server_clients[client].app }}
+{% endfor %}
+";
+}
+
+###############################################################################
+#=== HELP FUNCTIONS ==========================================================#
+###############################################################################
+remote_delete_single_backup () {
+    REPOLOCATION=$1;
+    ARCHIVENAME=$2;
+    BORG_PASSPHRASE=$3;
+    KEYFILE=$4;
+    echo "Remove archive $ARCHIVENAME";
+    sudo -H -u $BORGUSER bash -c '
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE='$BORG_PASSPHRASE';
+    KEYFILE='$KEYFILE';
+    REPOLOCATION='$REPOLOCATION';
+    borg delete $REPOLOCATION::$ARCHIVENAME --rsh "/usr/bin/ssh -i $KEYFILE"';
+}
+
+remote_free_space () {
+    REPOLOCATION=$1;
+    KEYFILE=$2;
+    APP=$3;
+    echo "Free space on repository for app '$APP'.";
+    sudo -H -u $BORGUSER bash -c '
+    KEYFILE='$KEYFILE';
+    REPOLOCATION='$REPOLOCATION';
+    borg compact $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"';
+}
+
+has_argument () {
+    if [[ -z "$1" ]];
+    then
+        echo "Missing required paramter!";
+        exit 1;
+    fi
+}
+
+validate_apps_or_default () {
+    VALID=1;
+    if [[ -z "$1" ]];
+    then
+        echo $APPLIST;
+    else
+        CHECKLIST=$(echo $1 | tr ',' ' ');
+        for APP in $CHECKLIST;
+        do
+            if [[ ! ${APPLIST[@]} =~ $APP ]]
+            then
+                VALID=0;
+                echo "$APP is not a valid app.";
+            fi
+        done
+        if [[ ! $VALID == 1 ]]
+        then
+            echo "Aborting.";
+            exit 1;
+        else
+            echo $CHECKLIST;
+        fi
+    fi
+}
+
+
+###############################################################################
+#=== USECASE FUNCTIONS =======================================================#
+###############################################################################
+remote_list_app () {
+    for APP in $APPLIST;
+    do
+        echo $APP;
+    done
+}
+
+remote_list () {
+    REMOTELIST=$1;
+    for APP in $REMOTELIST;
+    do
+        REPOSITORYCLIENT=${BORG_CLIENT_MAP[$APP]}
+        BORG_REPO_USER=${BORG_USER_MAP[$APP]}
+        REPOLOCATION="ssh://$BORG_REPO_USER@{{ backup_host }}:{{ backup_ssh_port }}/./$APP"
+        BORG_PASSPHRASE=$(cat {{ backup_key_folder }}/$APP)
+        KEYFILE="{{ backup_home }}/.ssh/$REPOSITORYCLIENT"
+        ARCHIVEIDS=$(
+            sudo -H -u $BORGUSER bash -c '
+                export BORG_PASSPHRASE='$BORG_PASSPHRASE';
+                KEYFILE='$KEYFILE';
+                REPOLOCATION='$REPOLOCATION';
+                borg list $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"' | sort -r
+        );
+        echo "$ARCHIVEIDS";
+    done
+}
+
+remote_prune () {
+    REMOTEPRUNELIST=$1;
+    for APP in $REMOTEPRUNELIST;
+    do
+        BORG_REPO_USER=${BORG_USER_MAP[$APP]}
+        BORG_PASSPHRASE=$(cat {{ backup_key_folder }}/$APP)
+        REPOLOCATION="ssh://$BORG_REPO_USER@{{ backup_host }}:{{ backup_ssh_port }}/./$APP"
+        KEYFILE="{{ backup_home }}/.ssh/${BORG_CLIENT_MAP[$APP]}"
+
+        sudo -H -u $BORGUSER bash -c '
+        KEYFILE='$KEYFILE';
+        REPOLOCATION='$REPOLOCATION';
+        export BORG_PASSPHRASE='$BORG_PASSPHRASE';
+        borg prune -H 24 -d 7 -w 4 -m -1 -y 100  $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"';
+
+        remote_free_space $REPOLOCATION $KEYFILE $APP;
+    done
+}
+
+local_list () {
+    ARCHIVEIDS=$(
+        sudo -H -u $BORGUSER bash -c '
+        REPOLOCATION='$LOCALREPOLOCATION';
+        export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+        borg list $REPOLOCATION' | grep {{ backup_app }} | sort -r);
+    echo "$ARCHIVEIDS";
+}
+
+local_prune () {
+    sudo -H -u $BORGUSER bash -c '
+    KEYFILE='$KEYFILE';
+    REPOLOCATION='$LOCALREPOLOCATION';
+    export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+    borg prune -H 24 -d 7 -w 4 -m -1 -y 100  $REPOLOCATION';
+
+    echo "Free space on local repository.";
+    sudo -H -u $BORGUSER bash -c '
+    REPOLOCATION='$LOCALREPOLOCATION';
+    borg compact $REPOLOCATION';
+}
+
+local_backup () {
+    ARCHIVENAME="{{ backup_app }}-$(date '+%s')";
+    mkdir -p $MIRRORFOLDER;
+    chown $BORGUSER: $MIRRORFOLDER;
+    # keep a persistent mirror so rsync only transfers what changed
+    for APP in $LOCALAPPLIST;
+    do
+        sudo -H -u $BORGUSER bash -c '
+        MIRRORFOLDER='$MIRRORFOLDER';
+        REPOLOCATION='${BORG_CLIENT_MAP[$APP]}':'$APP';
+        rsync -ar --delete $REPOLOCATION $MIRRORFOLDER/;';
+    done
+
+    # borg repositories are encrypted already, compression rarely helps
+    sudo -H -u $BORGUSER bash -c '
+    MIRRORFOLDER='$MIRRORFOLDER';
+    REPOLOCATION='$LOCALREPOLOCATION';
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+    borg create -C {{ backup_server_compression }} $REPOLOCATION::$ARCHIVENAME $MIRRORFOLDER;';
+}
+
+local_delete () {
+    ARCHIVENAME=$1;
+    ARCHIVEIDS=$(local_list | cut -f 1 -d ' ');
+    if [[ "$ARCHIVEIDS" != *$ARCHIVENAME* ]];
+    then
+        echo "ERROR! Provided archivename '$ARCHIVENAME' is not part of the available archives! Aborting.";
+        exit 1;
+    fi
+
+    echo "Remove archive $ARCHIVENAME";
+    sudo -H -u $BORGUSER bash -c '
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE='$(cat {{ backup_home }}/.borg.key)';
+    REPOLOCATION='$LOCALREPOLOCATION';
+    borg delete $REPOLOCATION::$ARCHIVENAME';
+}
+
+
+###############################################################################
+#=== MAIN FUNCTIONS ==========================================================#
+###############################################################################
+
+(
+    flock -n 9 || {
+        echo "BACKUP ALREADY RUNNING! ABORTING.";
+        exit 1;
+    }
+
+    if [ $# = 0 ]; then
+        help;
+        exit 0;
+    fi
+
+    action=$1
+    case $action in
+        "remote-apps-list")
+            remote_list_app;
+            ;;
+
+        "remote-list")
+            USERLIST=$(validate_apps_or_default "$2");
+            remote_list "$USERLIST";
+            ;;
+
+        "remote-prune")
+            if [[ -z "$2" ]]; then
+                USERLIST=$REMOTEPRUNEAPPLIST;
+            else
+                USERLIST=$(validate_apps_or_default "$2");
+            fi
+            remote_prune "$USERLIST";
+            ;;
+
+        "list")
+            local_list;
+            ;;
+
+        "prune")
+            local_prune;
+            ;;
+
+        "backup")
+            local_backup;
+            ;;
+
+        "restore")
+            has_argument $2;
+            USERLIST=$(validate_apps_or_default $3);
+            #local_restore $2;
+            ;;
+        "delete")
+            has_argument $2;
+            local_delete $2;
+            ;;
+        *)
+            help;
+            exit 0;
+            ;;
+    esac
+) 9>/var/run/lock/cloud-server-backup.lock;

templates/usr/local/bin/cloud_backup.j2

@@ -143,5 +143,4 @@ has_argument () {
             exit 0;
             ;;
     esac
-    echo ""
 ) 9>/var/run/lock/cloud-backup.lock;

vars/main.yml

@@ -1,3 +1,6 @@
 ---
 backup_home: "{{ backup_root }}/home"
+backup_inst: "{{ backup_root }}/inst"
 backup_storage: "{{ backup_root }}/storage"
+backup_key_folder: "{{ backup_home }}/.keys"
+backup_remotes_folder: "{{ backup_storage }}/remote-backups"