Backup server: skip self-pruning repos in remote-prune

backup_server_remote_skip lists apps whose client prunes the repository itself; remote-prune without arguments leaves them alone (an explicit app list still works).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

defaults/main.yml

@@ -3,67 +3,54 @@ cloud_apps: /opt
 cloud_storage: /opt/storage
 cloud_stage: prod
 cloud_update: false
-ssh_port: 1802
-
-# clients push only daily backup
-# server (!= client) will then pull daily backups
-backup_client: true
-
-# Server should pull 1 hour later the backups.
-backup_hour_shift: 1
-
-backup_times_weekday: "*"
-backup_times_hour:
-  - 6
-  - 10
-  - 14
-  - 18
-  - 22
-backup_times_minute: 0
-
-backup_keep_days: 7
 
 backup_owner: backup
-backup_owner_ssh_key: "ssh-rsa THEREISSUCHRSAKEYYET"
 backup_group: "{{ backup_owner }}"
-backup_permission: 0770
+backup_owner_ssh_private_key: "ssh-rsa THEREISNOSUCHRSAKEYYET"
+backup_owner_ssh_public_key: "ssh-rsa THEREISNOSUCHRSAKEYYET"
 
-backup_hosts:
-  - name: server1-name
-    domain: host1.my-domain.tld
-    ip: 001.112.223.123
-  - name: server2-name
-    domain: host2.my-domain.tld
+backup_location: "/borg"
 
-backup_folder: "{{ cloud_apps }}/backup"
+backup_root: "{{ cloud_storage }}/backup"
+backup_run_folder: "{{ cloud_storage }}"
 
-#backup_client_scripts:
-#  - /path/to/script -param
-backup_client_scripts: []
-#backup_client_script_envs:
-#  - var: VAR_NAME
-#    path: /some/path
-#  - var: VAR_NAME
-#    value: some_value
-backup_client_script_envs: []
+backup_storage_key: "MyStorageKey"
 
-backup_lock: "{{ backup_folder }}/backup.lock"
+backup_client: true
+backup_app: app
+backup_cron_specialtime: "daily"
+backup_cron_owner: "{{ backup_owner }}"
+backup_server_remote_prune_time: 6
+backup_server_remote_backup_time: 8
+backup_server_local_prune_time: 10
+#server: clients to prune/mirror (defaults to backup_clients), apps not mirrored locally, local compression
+backup_server_clients: "{{ backup_clients }}"
+backup_server_local_skip: []
+#apps whose client prunes its own repository; remote-prune (without arguments) skips them
+backup_server_remote_skip: []
+backup_server_compression: lzma
+#local mirror of the remote repositories (defaults to <backup_root>/storage/remote-backups)
+#backup_server_mirror: /mnt/nas/backup/mirror
 
-#backup_targets:
-#  db:
-#    - type: mariadb
-#      name: db_name_mdb
-#    - type: pgsql
-#      name: db_name_pg
-#  mail:
-#    - type: file
-#      path: /path/to/some/file
-#    - type: mariadb
-#      name: db_name_mdb_mail
-#  file:
-#    - /path/to/some/file
-#    - /path/to/some/dir
-backup_targets:
-  db: []
-  mail: []
-  file: []
+
+backup_script:
+  prework_backup: |
+    echo "This is executed before borg backup. Please collect data for backup in path: {{ backup_storage }}"
+  postwork_restore: |
+    echo "This is executed after borg restore. Please collect data during restore from path: {{ backup_storage }}"
+
+backup_ssh_port: 22
+backup_host: my-borg-repo.tld
+backup_clients: []
+#  app1:
+#    owner: user1
+#    app: app1
+#    borgkey: BorgRepoKey1
+#    sshpubkey: SshPubKey1
+#    sshprivkey: SshPrivKey1
+#  app2:
+#    owner: user2
+#    app: app2
+#    borgkey: BorgRepoKey2
+#    sshpubkey: SshPubKey2
+#    sshprivkey: SshPrivKey2

files/opt/backup/filter_backups.py

@@ -0,0 +1,117 @@
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+from argparse import ArgumentParser, FileType
+
+def filter_backups_timeslot(offset, backups, timeslot='days', keep=7, mode='max', printy=True):
+    keep_backups = []
+    for timeslot_off in range(1, keep):
+        indices = [
+            idx
+            for idx,value in enumerate(backups)
+            if (
+                value[1] >= (offset - relativedelta(**{timeslot:timeslot_off}))
+            )
+        ]
+        if not indices:
+            continue
+        if mode == "min":
+            #removed not considered backups for this slot!
+            for idx in reversed(indices[1:]):
+                backups.pop(idx)
+            keep_backups.append(backups.pop(indices[0]))
+        elif mode == "max":
+            keep_backups.append(backups.pop(indices[-1]))
+            #removed not considered backups for this slot!
+            for idx in reversed(indices[:-1]):
+                backups.pop(idx)
+        else:
+            for idx in sorted(indices, reverse=True):
+                keep_backups.append(backups.pop(idx))
+    return keep_backups, backups
+
+def filter_backups(backuplist, show_keep=True, keep_hours=24, keep_days=7, keep_weeks=4, keep_months=12, keep_years=10):
+    backups = sorted(
+        [
+            (backup, datetime.fromtimestamp(int(backup.split('-')[1])))
+            for backup in backuplist
+        ], key=lambda v: v[1], reverse=True
+    )
+    offset = backups[0][1]
+    keep_backups = [backups.pop(0)]
+
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='hours',keep=keep_hours,mode='all')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='days',keep=keep_days,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='weeks',keep=keep_weeks,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='months',keep=keep_months,mode='max')
+    keep_backups.extend(part_keep_backups)
+    part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='years',keep=keep_years,mode='max')
+    keep_backups.extend(part_keep_backups)
+
+    keeps = [backup[0] for backup in keep_backups]
+    if not show_keep:
+        return [backup for backup in backuplist if backup not in keeps]
+    return keeps
+
+def main():
+    parser = ArgumentParser(description="Program to print backups to keep or delete based on time stamps.")
+    parser.add_argument(
+        "-f", "--file", metavar="FILE", type=FileType('r'), required=True,
+        help="Path to file with list of backups to filter. Expects per line one backup name in format <appname>-<timestamp>")
+    parser.add_argument(
+        "-d", "--print-delete", action="store_true", required=False,
+        help="Instead of objects to kept, print out objects to be deleted."
+    )
+    parser.add_argument(
+        "-H", "--hours-keep", metavar="KEEP", type=int, required=False,
+        default=24, help="Number of objects to keep on hourly basis."
+    )
+    parser.add_argument(
+        "-D", "--days-keep", metavar="KEEP", type=int, required=False,
+        default=7, help="Number of objects to keep on daily basis."
+    )
+    parser.add_argument(
+        "-W", "--weeks-keep", metavar="KEEP", type=int, required=False,
+        default=4, help="Number of objects to keep on weekly basis."
+    )
+    parser.add_argument(
+        "-M", "--months-keep", metavar="KEEP", type=int, required=False,
+        default=12, help="Number of objects to keep on monthly basis."
+    )
+    parser.add_argument(
+        "-Y", "--years-keep", metavar="KEEP", type=int, required=False,
+        default=10, help="Number of objects to keep on yearly basis."
+    )
+
+    args = parser.parse_args()
+    show_keep = not args.print_delete
+    keep_hours = args.hours_keep
+    keep_days = args.days_keep
+    keep_weeks = args.weeks_keep
+    keep_months = args.months_keep
+    keep_years = args.years_keep
+
+    backuplist = [
+        line.strip()
+        for line in args.file
+        if len(line.strip()) > 0
+    ]
+    fail = False
+    for idx,backup in enumerate(backuplist):
+        if len(backup) < 10 or not backup[-10:].isnumeric():
+            print(f"WARNING! Line {idx+1} has no timestamp suffix")
+            fail = True
+    if fail:
+        exit(1)
+
+    considered_backups = filter_backups(
+        backuplist, show_keep,
+        keep_hours, keep_days, keep_weeks, keep_months, keep_years
+    )
+    for backup in considered_backups:
+        print(backup)
+
+if __name__ == '__main__':
+    main()

tasks/client.yml

@@ -1,38 +1,30 @@
 ---
-- name: generate backup ssh user
-  user:
-    name: "{{ backup_owner }}"
+- name: Setup ssh keys
+  copy:
+    content: "{{ item.content }}"
+    dest: "{{ backup_home }}/.ssh/{{ item.file }}"
+    owner: "{{ backup_owner }}"
     group: "{{ backup_group }}"
-    comment: Backup user
-    shell: /bin/sh
-
-- name: Setup User ssh_key
-  authorized_key: 
-    user: "{{ backup_owner }}"
-    key: "{{ backup_owner_ssh_key }}"
-
-- name: generate config lists
-  template:
-    src: backup/conf/{{ conf }}.list.j2
-    dest: "{{ backup_conf }}/{{ conf }}.list"
+    mode: "0600"
   loop:
-    - filedir
-    - db
-    - mail
+    - content: "{{ backup_owner_ssh_private_key }}"
+      file: "id_rsa"
+    - content: "{{ backup_owner_ssh_public_key }}"
+      file: "id_rsa.pub"
   loop_control:
-    loop_var: conf
-    label: "{{ conf }}.list"
+    label: "{{ item.file }}"
 
 - name: setup backup script
   template:
-    src: backup/backup_on_client.sh.j2
-    dest: "{{ backup_folder }}/backup.sh"
-    mode: 0740
+    src: "./usr/local/bin/cloud_backup.j2"
+    dest: "/usr/local/bin/cloud_backup"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
 
-- name: setup cronjob for client backup generation
+- name: setup cron backup job
   cron:
-    name: "backup"
-    weekday: "{{ backup_times_weekday }}"
-    hour: "{{ backup_times_hour | join(',') }}"
-    minute: "{{ backup_times_minute }}"
-    job: "{{ backup_folder }}/backup.sh"
+    name: "cloud backup"
+    user: "{{ backup_cron_owner }}"
+    job: "/usr/local/bin/cloud_backup backup"
+    special_time: "{{ backup_cron_specialtime }}"

tasks/main.yml

@@ -1,34 +1,59 @@
 ---
-# Basics
 - name: install requirements for backup system
   apt:
     update_cache: yes
     state: "{% if cloud_update | bool %}latest{% else %}present{% endif %}"
     install_recommends: yes
     pkg:
-      - rsync
+      - borgbackup
 
-- name: setup storage folders for backup
+- name: setup backup location
   file:
     state: directory
-    path: "{{ dir.value.path }}"
+    path: "{{ backup_root }}"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+
+- name: Setup required groups of users
+  group:
+    name: "{{ backup_group }}"
+    state: present
+
+- name: generate backup user
+  user:
+    name: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    comment: backup user
+    shell: /bin/bash
+    home: "{{ backup_home }}"
+
+- name: Setup backup storage password
+  copy:
+    content: "{{ backup_storage_key }}"
+    dest: "{{ backup_home }}/.borg.key"
     owner: "{{ backup_owner }}"
     group: "{{ backup_group }}"
-    mode: "{{ backup_permission }}"
-  with_dict: "{{ backup_storage }}"
-  loop_control:
-    loop_var: dir
-    label: "{{ dir.key }}"
+    mode: "0600"
 
-- name: setup conf folder
+- name: Setup ssh folder
   file:
     state: directory
-    path: "{{ backup_folder }}/conf"
+    path: "{{ backup_home }}/.ssh"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0700"
 
-- name: setup client backup structure
-  include_tasks: client.yml
+- name: setup backup storage
+  file:
+    state: directory
+    path: "{{ backup_storage }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0777"
+
+- include_tasks: server.yml
+  when: not backup_client
+
+- include_tasks: client.yml
   when: backup_client
-
-- name: setup server backup structure
-  include_tasks: server.yml
-  when: not backup_client

tasks/server.yml

@@ -1,27 +1,118 @@
 ---
-- name: analyse backup hours for pulling server
-  set_fact:
-    backup_times_server_hour: "{{ backup_times_server_hour | default([]) + [item + backup_hour_shift] }}"
-  loop: "{{ backup_times_hour }}"
+- name: setup backup installation folder
+  file:
+    state: directory
+    path: "{{ backup_inst }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
 
-- name: generate backup user ssh key
-  user:
-    name: "{{ backup_owner }}"
-    generate_ssh_key: yes
-    ssh_key_type: rsa
-    ssh_key_bits: 4096
-    home: "{{ backup_folder }}"
+- name: install dateutil library service
+  apt:
+    update_cache: yes
+    state: "{% if cloud_update | bool %}latest{% else %}present{% endif %}"
+    install_recommends: yes
+    pkg:
+      - python3-dateutil
+      - rsync
 
-- name: template main server backup pull script
+- name: setup filter program for backup pruning
+  copy:
+    src: opt/backup/filter_backups.py
+    dest: "{{ backup_inst }}/filter_backups.py"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+
+- name: setup ssh public keys of clients
+  copy:
+    content: "{{ backup_server_clients[item].sshpubkey }}"
+    dest: "{{ backup_home }}/.ssh/{{ item }}.pub"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_server_clients.keys() }}"
+
+- name: setup ssh private keys of clients
+  copy:
+    content: "{{ backup_server_clients[item].sshprivkey }}"
+    dest: "{{ backup_home }}/.ssh/{{ item }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_server_clients.keys() }}"
+
+- name: setup ssh config of clients
   template:
-    src: backup/backup_from_clients.sh.j2
-    dest: "{{ backup_folder }}/pull-backup.sh"
-    mode: 0740
+    src: "home/backup/.ssh/config.j2"
+    dest: "{{ backup_home }}/.ssh/config"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
 
-- name: setup cronjob for server backup pull
+- name: setup backup key folder
+  file:
+    state: directory
+    path: "{{ backup_key_folder }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0700"
+
+- name: setup borg repository keys
+  copy:
+    content: "{{ backup_server_clients[item].borgkey }}"
+    dest: "{{ backup_key_folder }}/{{ backup_server_clients[item].app }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0600"
+  loop: "{{ backup_server_clients.keys() }}"
+
+- name: setup local backup storage folder
+  file:
+    state: directory
+    path: "{{ item }}"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+  loop:
+    - "{{ backup_server_mirror | default(backup_remotes_folder) }}"
+    - "{{ backup_location }}/{{ backup_app }}"
+
+- name: setup borg local repository
+  shell:
+    cmd: "BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key) borg init --encryption repokey {{ backup_location }}/{{ backup_app }}"
+    creates: "{{ backup_location }}/{{ backup_app }}/config"
+  become_user: "{{ backup_owner }}"
+  become: yes
+
+- name: setup borg server script
+  template:
+    src: "opt/backup/inst/cloud-server-backup.j2"
+    dest: "{{ backup_inst }}/cloud-server-backup"
+    owner: "{{ backup_owner }}"
+    group: "{{ backup_group }}"
+    mode: "0750"
+
+- name: setup remote prune cron backup server job
   cron:
-    name: "pull backups"
-    weekday: "{{ backup_times_weekday }}"
-    hour: "{{ backup_times_server_hour | join(',') }}"
-    minute: "{{ backup_times_minute }}"
-    job: "{{ backup_folder }}/pull-backup.sh"
+    name: "cloud server remote prune"
+    user: "{{ backup_cron_owner }}"
+    job: "{{ backup_inst }}/cloud-server-backup remote-prune"
+    hour: "{{ backup_server_remote_prune_time }}"
+    minute: 0
+
+- name: setup remote backup cron backup server job
+  cron:
+    name: "cloud server remote backup"
+    user: "{{ backup_cron_owner }}"
+    job: "{{ backup_inst }}/cloud-server-backup backup"
+    hour: "{{ backup_server_remote_backup_time }}"
+    minute: 0
+
+- name: setup local prune cron backup server job
+  cron:
+    name: "cloud server local prune"
+    user: "{{ backup_cron_owner }}"
+    job: "{{ backup_inst }}/cloud-server-backup prune"
+    hour: "{{ backup_server_local_prune_time }}"
+    minute: 0

templates/backup/backup_from_clients.sh.j2

@@ -1,68 +0,0 @@
-#!/bin/bash
-
-set -e -o pipefail
-
-cd {{ backup_folder }}
-if [ -f {{ backup_lock }} ]; then
-    echo "Backup lock! Cannot procceed as lock '{{ backup_lock }}' exist."
-    echo "Associated PID is $(cat {{ backup_lock }})"
-    exit 1
-fi
-
-echo "$$" > {{ backup_lock }}
-BKPTIMESTAMP=$(date +"%y-%m-%d.%H")
-BKP=$BKPTIMESTAMP"_backup"
-
-
-# generate rolling folder
-{% for target in backup_targets %}
-{% for host in backup_hosts %}
-{% if 'domain' in host %}
-mkdir -p {{ target }}/{{ host.name + '_' + host.domain }}/$BKP
-{% elif 'ip' in host %}
-mkdir -p {{ target }}/{{ host.name + '_' + host.ip }}/$BKP
-{% endif %}
-{% endfor %}
-{% endfor %}
-
-
-# pull backups
-{% for target in backup_targets %}
-{% for host in backup_hosts %}
-{% if 'domain' in host %}
-rsync --rsh='ssh -p{{ ssh_port }} -i ~{{ backup_owner }}/.ssh/id_rsa'  -chav --stats --ignore-missing-args {{ backup_owner }}@{{ host.domain }}:{{ backup_folder }}/{{ target }}.tgz {{ target }}/{{ host.name + '_' + host.domain }}/$BKP
-{% elif 'ip' in host %}
-rsync --rsh='ssh -p{{ ssh_port }} -i ~{{ backup_owner }}/.ssh/id_rsa' -chav --stats --ignore-missing-args {{ backup_owner }}@{{ host.ip }}:{{ backup_folder }}/{{ target }}.tgz {{ target }}/{{ host.name + '_' + host.ip }}/$BKP
-{% endif %}
-{% endfor %}
-{% endfor %}
-
-
-# link to latest
-{% for target in backup_targets %}
-{% for host in backup_hosts %}
-{% if 'domain' in host %}
-ln -sf {{ backup_folder }}/{{ target }}/{{ host.name + '_' + host.domain }}/$BKP {{ backup_folder }}/{{ target }}/{{ host.name + '_' + host.domain }}/LATEST
-{% elif 'ip' in host %}
-ln -sf {{ backup_folder }}/{{ target }}/{{ host.name + '_' + host.ip }}/$BKP {{ backup_folder }}/{{ target }}/{{ host.name + '_' + host.ip }}/LATEST
-{% endif %}
-{% endfor %}
-{% endfor %}
-
-
-# keep only last {{ backup_keep_days * ( backup_times_hour | length ) }}
-{% for target in backup_targets %}
-{% for host in backup_hosts %}
-{% if 'domain' in host %}
-DELETE_LIST=$(ls -1dtr {{ target }}/{{ host.name + '_' + host.domain }}/* | head -n -{{ backup_keep_days * ( backup_times_hour | length ) }})
-{% elif 'ip' in host %}
-DELETE_LIST=$(ls -1dtr {{ target }}/{{ host.name + '_' + host.ip }}/* | head -n -{{ backup_keep_days * ( backup_times_hour | length ) }})
-{% endif %}
-if (( $( echo $DELETE_LIST | wc -w ) > 0 )); then
-    rm -rf $DELETE_LIST
-fi
-{% endfor %}
-{% endfor %}
-
-
-rm {{ backup_lock }}

templates/backup/backup_on_client.sh.j2

@@ -1,72 +0,0 @@
-#!/bin/bash
-
-set -e -o pipefail
-
-if [ -f {{ backup_lock }} ]; then
-    echo "Backup lock! Cannot procceed as lock '{{ backup_lock }}' exist."
-    echo "Associated PID is $(cat {{ backup_lock }})"
-    exit 1
-fi
-echo "$$" > {{ backup_lock }}
-
-cd {{ backup_folder }}
-
-# Load script envs
-echo "Load script environment variables"
-{% for scr_env in backup_client_script_envs %}
-{% if 'path' in scr_env %}
-export {{ scr_env.var }}={{ scr_env.path }}:${{ scr_env.var }}
-{% else %}
-export {{ scr_env.var }}={{ scr_env.value }}
-{% endif %}
-{% endfor %}
-
-
-# Perform client pre backup scripts for
-echo "Perform pre-backup scripts"
-{% for cmd in backup_client_scripts %}
-{{ cmd }};
-{% endfor %}
-
-
-# Backup files and directories
-echo "Copy files and directories"
-while read DIRFILE; do
-    echo " - Copy $DIRFILE"
-    BACKUP_SUBDIR=$(dirname $DIRFILE)
-    mkdir -p "{{ backup_storage.file.path }}/$BACKUP_SUBDIR"
-    cp -r -u $DIRFILE "{{ backup_storage.file.path }}/$DIRFILE"
-done < {{ backup_conf }}/filedir.list
-rm -f file.tgz #remove old backup
-tar czfv file.tgz file
-rm -rf file/*
-
-# Backup databases
-echo "Generate DB dumps"
-while read DBS; do
-    DBTYPE=$(echo $DBS | cut -f 1 -d "{{ backup_conf_db_sep }}" );
-    DBNAME=$(echo $DBS | cut -f 2 -d "{{ backup_conf_db_sep }}" );
-    if [ $DBTYPE == "mariadb" ] || [ $DBTYPE == "mysql" ]; then
-        echo "- dump to {{ backup_storage.db.path }}/$DBTYPE/$DBNAME.sql"
-        mkdir -p {{ backup_storage.db.path }}/$DBTYPE;
-        mysqldump $DBNAME | gzip -c > {{ backup_storage.db.path }}/$DBTYPE/$DBNAME.sql.gz;
-    elif [ $DBTYPE == "pgsql" ]; then
-        echo "- dump to {{ backup_storage.db.path }}/$DBTYPE/$DBNAME.sql"
-        mkdir -p {{ backup_storage.db.path }}/$DBTYPE;
-        sudo -u postgres bash -c "pg_dump $DBNAME | gzip -c > /tmp/$DBNAME.sql.gz";
-        mv /tmp/$DBNAME.sql.gz /backup/db/$DBTYPE/$DBNAME.sql.gz;
-    else
-        echo "- ERROR! DB type '$DBTYPE' for '$DBNAME' is unknown to backup."
-    fi
-done < {{ backup_conf }}/db.list
-rm -f db.tgz #remove old backup
-tar czfv db.tgz db
-rm -rf db/*
-
-#Mail
-rm -f mail.tgz #remove old backup
-tar czfv mail.tgz mail
-rm -rf mail/*
-
-rm {{ backup_lock }}
-chown {{ backup_owner }}:{{ backup_group }} {{ backup_folder }}/*.tgz

templates/backup/conf/db.list.j2

@@ -1,5 +0,0 @@
-{% if 'db' in backup_targets %}
-{% for db in backup_targets.db %}
-{{ db.type }}{{ backup_conf_db_sep }}{{ db.name }}
-{% endfor %}
-{% endif %}

templates/backup/conf/filedir.list.j2

@@ -1,5 +0,0 @@
-{% if 'file' in backup_targets %}
-{% for filedir in backup_targets.file %}
-{{ filedir }}
-{% endfor %}
-{% endif %}

templates/backup/conf/mail.list.j2

@@ -1,5 +0,0 @@
-{% if 'mail' in backup_targets %}
-{% for mail in backup_targets.mail %}
-{{ mail }}
-{% endfor %}
-{% endif %}

templates/home/backup/.ssh/config.j2

@@ -0,0 +1,7 @@
+{% for client in backup_server_clients %}
+Host {{ client }}
+    HostName {{ backup_host }}
+    Port {{ backup_ssh_port }}
+    User {{ backup_server_clients[client].owner }}
+    IdentityFile {{ backup_home }}/.ssh/{{ client }}
+{% endfor %}

templates/opt/backup/inst/cloud-server-backup.j2

@@ -0,0 +1,285 @@
+{% set server_clients = backup_server_clients | default(backup_clients) %}
+#!/bin/bash
+
+BORGUSER="{{ backup_owner }}";
+TARGETFOLDER="{{ backup_remotes_folder }}";
+LOCALREPOLOCATION="{{ backup_location }}/{{ backup_app }}";
+APPLIST="{{ server_clients | json_query('*.app') | join(' ') }}";
+# repos that are only pruned remotely, not mirrored locally
+LOCALAPPLIST="{{ server_clients | json_query('*.app') | difference(backup_server_local_skip) | join(' ') }}";
+# repos pruned by their own client (e.g. a NAS borg plugin), skipped by remote-prune without arguments
+REMOTEPRUNEAPPLIST="{{ server_clients | json_query('*.app') | difference(backup_server_remote_skip) | join(' ') }}";
+MIRRORFOLDER="{{ backup_server_mirror | default(backup_remotes_folder) }}";
+
+declare -A BORG_CLIENT_MAP
+{% for client in server_clients %}
+BORG_CLIENT_MAP[{{ server_clients[client].app }}]="{{ client }}";
+{% endfor %}
+
+declare -A BORG_USER_MAP
+{% for client in server_clients %}
+BORG_USER_MAP[{{ server_clients[client].app }}]="{{ server_clients[client].owner }}";
+{% endfor %}
+
+
+help (){
+    echo "cloud-server-backup - backup and restore script on cloud backup with borg target v1.0 by L.Hahn.
+
+Usage: $0 COMMAND [APPS]
+
+COMMAND:
+    - remote-apps-list              List available apps in remote borg repositories.
+    - remote-list [APPS]            List available archives for apps in remote borg repositories.
+    - remote-prune [APPS]           Prune remote app (or all) borg repositories.
+    - list                          List available archives of remote repositories on local repository.
+    - backup                        Perform backup of remote borg repositories and create a new archive in local borg repository.
+    - prune                         Prune local borg backup of remote repositories.
+    - restore ARCHIVENAME [APPS]    Upload backup to borg repository to restore older file states.
+                                        May turn off your application if still running.
+                                        If latest is provided as ARCHIVENAME, the latest one based on timestamp is taken.
+                                        IF ARCHIVENAME is provided, will try to download it; throws error if not found.
+                                        If APP is provided, only the mentioned app repositories are restored.
+
+
+APPS:                               A single app or comma-separated (sub-)set of below listed apps to be considered.
+{% for client in server_clients %}
+    - {{ server_clients[client].app }}
+{% endfor %}
+";
+}
+
+###############################################################################
+#=== HELP FUNCTIONS ==========================================================#
+###############################################################################
+remote_delete_single_backup () {
+    REPOLOCATION=$1;
+    ARCHIVENAME=$2;
+    BORG_PASSPHRASE=$3;
+    KEYFILE=$4;
+    echo "Remove archive $ARCHIVENAME";
+    sudo -H -u $BORGUSER bash -c '
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE='$BORG_PASSPHRASE';
+    KEYFILE='$KEYFILE';
+    REPOLOCATION='$REPOLOCATION';
+    borg delete $REPOLOCATION::$ARCHIVENAME --rsh "/usr/bin/ssh -i $KEYFILE"';
+}
+
+remote_free_space () {
+    REPOLOCATION=$1;
+    KEYFILE=$2;
+    APP=$3;
+    echo "Free space on repository for app '$APP'.";
+    sudo -H -u $BORGUSER bash -c '
+    KEYFILE='$KEYFILE';
+    REPOLOCATION='$REPOLOCATION';
+    borg compact $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"';
+}
+
+has_argument () {
+    if [[ -z "$1" ]];
+    then
+        echo "Missing required paramter!";
+        exit 1;
+    fi
+}
+
+validate_apps_or_default () {
+    VALID=1;
+    if [[ -z "$1" ]];
+    then
+        echo $APPLIST;
+    else
+        CHECKLIST=$(echo $1 | tr ',' ' ');
+        for APP in $CHECKLIST;
+        do
+            if [[ ! ${APPLIST[@]} =~ $APP ]]
+            then
+                VALID=0;
+                echo "$APP is not a valid app.";
+            fi
+        done
+        if [[ ! $VALID == 1 ]]
+        then
+            echo "Aborting.";
+            exit 1;
+        else
+            echo $CHECKLIST;
+        fi
+    fi
+}
+
+
+###############################################################################
+#=== USECASE FUNCTIONS =======================================================#
+###############################################################################
+remote_list_app () {
+    for APP in $APPLIST;
+    do
+        echo $APP;
+    done
+}
+
+remote_list () {
+    REMOTELIST=$1;
+    for APP in $REMOTELIST;
+    do
+        REPOSITORYCLIENT=${BORG_CLIENT_MAP[$APP]}
+        BORG_REPO_USER=${BORG_USER_MAP[$APP]}
+        REPOLOCATION="ssh://$BORG_REPO_USER@{{ backup_host }}:{{ backup_ssh_port }}/./$APP"
+        BORG_PASSPHRASE=$(cat {{ backup_key_folder }}/$APP)
+        KEYFILE="{{ backup_home }}/.ssh/$REPOSITORYCLIENT"
+        ARCHIVEIDS=$(
+            sudo -H -u $BORGUSER bash -c '
+                export BORG_PASSPHRASE='$BORG_PASSPHRASE';
+                KEYFILE='$KEYFILE';
+                REPOLOCATION='$REPOLOCATION';
+                borg list $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"' | sort -r
+        );
+        echo "$ARCHIVEIDS";
+    done
+}
+
+remote_prune () {
+    REMOTEPRUNELIST=$1;
+    for APP in $REMOTEPRUNELIST;
+    do
+        BORG_REPO_USER=${BORG_USER_MAP[$APP]}
+        BORG_PASSPHRASE=$(cat {{ backup_key_folder }}/$APP)
+        REPOLOCATION="ssh://$BORG_REPO_USER@{{ backup_host }}:{{ backup_ssh_port }}/./$APP"
+        KEYFILE="{{ backup_home }}/.ssh/${BORG_CLIENT_MAP[$APP]}"
+
+        sudo -H -u $BORGUSER bash -c '
+        KEYFILE='$KEYFILE';
+        REPOLOCATION='$REPOLOCATION';
+        export BORG_PASSPHRASE='$BORG_PASSPHRASE';
+        borg prune -H 24 -d 7 -w 4 -m -1 -y 100  $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"';
+
+        remote_free_space $REPOLOCATION $KEYFILE $APP;
+    done
+}
+
+local_list () {
+    ARCHIVEIDS=$(
+        sudo -H -u $BORGUSER bash -c '
+        REPOLOCATION='$LOCALREPOLOCATION';
+        export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+        borg list $REPOLOCATION' | grep {{ backup_app }} | sort -r);
+    echo "$ARCHIVEIDS";
+}
+
+local_prune () {
+    sudo -H -u $BORGUSER bash -c '
+    KEYFILE='$KEYFILE';
+    REPOLOCATION='$LOCALREPOLOCATION';
+    export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+    borg prune -H 24 -d 7 -w 4 -m -1 -y 100  $REPOLOCATION';
+
+    echo "Free space on local repository.";
+    sudo -H -u $BORGUSER bash -c '
+    REPOLOCATION='$LOCALREPOLOCATION';
+    borg compact $REPOLOCATION';
+}
+
+local_backup () {
+    ARCHIVENAME="{{ backup_app }}-$(date '+%s')";
+    mkdir -p $MIRRORFOLDER;
+    chown $BORGUSER: $MIRRORFOLDER;
+    # keep a persistent mirror so rsync only transfers what changed
+    for APP in $LOCALAPPLIST;
+    do
+        sudo -H -u $BORGUSER bash -c '
+        MIRRORFOLDER='$MIRRORFOLDER';
+        REPOLOCATION='${BORG_CLIENT_MAP[$APP]}':'$APP';
+        rsync -ar --delete $REPOLOCATION $MIRRORFOLDER/;';
+    done
+
+    # borg repositories are encrypted already, compression rarely helps
+    sudo -H -u $BORGUSER bash -c '
+    MIRRORFOLDER='$MIRRORFOLDER';
+    REPOLOCATION='$LOCALREPOLOCATION';
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+    borg create -C {{ backup_server_compression }} $REPOLOCATION::$ARCHIVENAME $MIRRORFOLDER;';
+}
+
+local_delete () {
+    ARCHIVENAME=$1;
+    ARCHIVEIDS=$(local_list | cut -f 1 -d ' ');
+    if [[ "$ARCHIVEIDS" != *$ARCHIVENAME* ]];
+    then
+        echo "ERROR! Provided archivename '$ARCHIVENAME' is not part of the available archives! Aborting.";
+        exit 1;
+    fi
+
+    echo "Remove archive $ARCHIVENAME";
+    sudo -H -u $BORGUSER bash -c '
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE='$(cat {{ backup_home }}/.borg.key)';
+    REPOLOCATION='$LOCALREPOLOCATION';
+    borg delete $REPOLOCATION::$ARCHIVENAME';
+}
+
+
+###############################################################################
+#=== MAIN FUNCTIONS ==========================================================#
+###############################################################################
+
+(
+    flock -n 9 || {
+        echo "BACKUP ALREADY RUNNING! ABORTING.";
+        exit 1;
+    }
+
+    if [ $# = 0 ]; then
+        help;
+        exit 0;
+    fi
+
+    action=$1
+    case $action in
+        "remote-apps-list")
+            remote_list_app;
+            ;;
+
+        "remote-list")
+            USERLIST=$(validate_apps_or_default "$2");
+            remote_list "$USERLIST";
+            ;;
+
+        "remote-prune")
+            if [[ -z "$2" ]]; then
+                USERLIST=$REMOTEPRUNEAPPLIST;
+            else
+                USERLIST=$(validate_apps_or_default "$2");
+            fi
+            remote_prune "$USERLIST";
+            ;;
+
+        "list")
+            local_list;
+            ;;
+
+        "prune")
+            local_prune;
+            ;;
+
+        "backup")
+            local_backup;
+            ;;
+
+        "restore")
+            has_argument $2;
+            USERLIST=$(validate_apps_or_default $3);
+            #local_restore $2;
+            ;;
+        "delete")
+            has_argument $2;
+            local_delete $2;
+            ;;
+        *)
+            help;
+            exit 0;
+            ;;
+    esac
+) 9>/var/run/lock/cloud-server-backup.lock;

templates/usr/local/bin/cloud_backup.j2

@@ -0,0 +1,146 @@
+#!/bin/bash
+
+BORGUSER="{{ backup_owner }}";
+RUNFOLDER="{{ backup_run_folder }}";
+TARGETFOLDER="{{ backup_storage }}";
+REPOLOCATION="{{ backup_location }}";
+ARCHIVENAME="{{ backup_app }}-$(date '+%s')";
+
+help (){
+    echo "cloud_backup - backup and restore script on cloud with borg target v1.0 by L.Hahn.
+
+Usage: $0 COMMAND [ARCHIVENAME]
+
+COMMAND:
+    - list                      List available archives in remote borg repository for your host.
+    - backup                    Perform backup of your host and create a new archive in borg repository.
+    - restore [ARCHIVENAME]     Download backup from borg repository to your host and restore files.
+                                    May turn off your application if still running.
+                                    If no ARCHIVENAME is provided, the latest one based on timestamp is taken.
+                                    IF ARCHIVENAME is provided, will try to download it; throws error if not found.
+";
+}
+
+
+### INDIVIDUAL TEMPLATE PART ###
+prework_backup () {
+    {{ backup_script.prework_backup | indent( width=4) }}
+}
+
+postwork_restore () {
+    {{ backup_script.postwork_restore | indent( width=4) }}
+}
+
+
+### SHARED TEMPLATE PART ###
+get_archives () {
+    ARCHIVEIDS=$(
+        sudo -H -u $BORGUSER bash -c '
+        REPOLOCATION='$REPOLOCATION';
+        export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+        borg list $REPOLOCATION' | grep {{ backup_app }} | sort -r);
+    echo "$ARCHIVEIDS";
+}
+
+cloud_backup () {
+    mkdir -p $TARGETFOLDER/$ARCHIVENAME;
+    prework_backup;
+    cd /;
+    sudo -H -u $BORGUSER bash -c '
+    TARGETFOLDER='$TARGETFOLDER';
+    REPOLOCATION='$REPOLOCATION';
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+    borg create -C lzma $REPOLOCATION::$ARCHIVENAME $TARGETFOLDER/$ARCHIVENAME';
+
+    rm -rf $TARGETFOLDER/$ARCHIVENAME;
+}
+
+cloud_restore () {
+    ARCHIVENAME=$1;
+    ARCHIVEIDS=$(get_archives | cut -f 1 -d ' ');
+    if [[ "${ARCHIVENAME,,}" == "latest" ]];
+    then
+        ARCHIVENAME=$(echo "$ARCHIVEIDS" | head -n 1);
+    else
+        if [[ "$ARCHIVEIDS" != *$ARCHIVENAME* ]];
+        then
+            echo "ERROR! Provided archivename '$ARCHIVENAME' is not part of the available archives! Aborting.";
+            exit 1;
+        fi
+    fi
+
+    mkdir -p $TARGETFOLDER/$ARCHIVENAME;
+    chown -R $BORGUSER: $TARGETFOLDER
+
+    cd /;
+    sudo -H -u $BORGUSER bash -c '
+    REPOLOCATION='$REPOLOCATION';
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+    borg extract $REPOLOCATION::$ARCHIVENAME --list';
+
+    postwork_restore;
+    rm -rf $TARGETFOLDER/$ARCHIVENAME;
+}
+
+cloud_backup_delete () {
+    ARCHIVENAME=$1;
+    ARCHIVEIDS=$(get_archives | cut -f 1 -d ' ');
+    if [[ "$ARCHIVEIDS" != *$ARCHIVENAME* ]];
+    then
+        echo "ERROR! Provided archivename '$ARCHIVENAME' is not part of the available archives! Aborting.";
+        exit 1;
+    fi
+
+    sudo -H -u $BORGUSER bash -c '
+    REPOLOCATION='$REPOLOCATION';
+    ARCHIVENAME='$ARCHIVENAME';
+    export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key);
+    borg delete $REPOLOCATION::$ARCHIVENAME';
+
+}
+
+has_argument () {
+    if [[ -z "$1" ]];
+    then
+        echo "Missing required paramter!";
+        exit 1;
+    fi
+}
+
+(
+    flock -n 9 || {
+        echo "BACKUP ALREADY RUNNING! ABORTING.";
+        exit 1;
+    }
+
+    if [ $# = 0 ]; then
+        help;
+        exit 0;
+    fi
+
+    action=$1
+    case $action in
+        "list")
+            get_archives;
+            ;;
+
+        "backup")
+            cloud_backup;
+            ;;
+
+        "restore")
+            has_argument $2;
+            cloud_restore $2;
+            ;;
+        "delete")
+            has_argument $2;
+            cloud_backup_delete $2;
+            ;;
+        *)
+            help;
+            exit 0;
+            ;;
+    esac
+) 9>/var/run/lock/cloud-backup.lock;

vars/main.yml

@@ -1,12 +1,6 @@
 ---
-backup_conf: "{{ backup_folder }}/conf"
-
-backup_conf_db_sep: ":"
-
-backup_storage:
-  mail:
-    path: "{{ backup_folder }}/mail"
-  db:
-    path: "{{ backup_folder }}/db"
-  file:
-    path: "{{ backup_folder }}/file"
+backup_home: "{{ backup_root }}/home"
+backup_inst: "{{ backup_root }}/inst"
+backup_storage: "{{ backup_root }}/storage"
+backup_key_folder: "{{ backup_home }}/.keys"
+backup_remotes_folder: "{{ backup_storage }}/remote-backups"