diff --git a/defaults/main.yml b/defaults/main.yml index 8f8d5ed..ead8397 100755 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -20,9 +20,29 @@ backup_client: true backup_app: app backup_cron_specialtime: "daily" backup_cron_owner: "{{ backup_owner }}" +backup_server_remote_prune_time: 6 +backup_server_remote_backup_time: 8 +backup_server_local_prune_time: 10 + backup_script: prework_backup: | echo "This is executed before borg backup. Please collect data for backup in path: {{ backup_storage }}" postwork_restore: | echo "This is executed after borg restore. Please collect data during restore from path: {{ backup_storage }}" + +backup_ssh_port: 22 +backup_host: my-borg-repo.tld +backup_clients: [] +# app1: +# owner: user1 +# app: app1 +# borgkey: BorgRepoKey1 +# sshpubkey: SshPubKey1 +# sshprivkey: SshPrivKey1 +# app2: +# owner: user2 +# app: app2 +# borgkey: BorgRepoKey2 +# sshpubkey: SshPubKey2 +# sshprivkey: SshPrivKey2 \ No newline at end of file diff --git a/files/opt/backup/filter_backups.py b/files/opt/backup/filter_backups.py new file mode 100644 index 0000000..cd32097 --- /dev/null +++ b/files/opt/backup/filter_backups.py @@ -0,0 +1,117 @@ +from datetime import datetime +from dateutil.relativedelta import relativedelta +from argparse import ArgumentParser, FileType + +def filter_backups_timeslot(offset, backups, timeslot='days', keep=7, mode='max', printy=True): + keep_backups = [] + for timeslot_off in range(1, keep): + indices = [ + idx + for idx,value in enumerate(backups) + if ( + value[1] >= (offset - relativedelta(**{timeslot:timeslot_off})) + ) + ] + if not indices: + continue + if mode == "min": + #removed not considered backups for this slot! + for idx in reversed(indices[1:]): + backups.pop(idx) + keep_backups.append(backups.pop(indices[0])) + elif mode == "max": + keep_backups.append(backups.pop(indices[-1])) + #removed not considered backups for this slot! + for idx in reversed(indices[:-1]): + backups.pop(idx) + else: + for idx in sorted(indices, reverse=True): + keep_backups.append(backups.pop(idx)) + return keep_backups, backups + +def filter_backups(backuplist, show_keep=True, keep_hours=24, keep_days=7, keep_weeks=4, keep_months=12, keep_years=10): + backups = sorted( + [ + (backup, datetime.fromtimestamp(int(backup.split('-')[1]))) + for backup in backuplist + ], key=lambda v: v[1], reverse=True + ) + offset = backups[0][1] + keep_backups = [backups.pop(0)] + + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='hours',keep=keep_hours,mode='all') + keep_backups.extend(part_keep_backups) + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='days',keep=keep_days,mode='max') + keep_backups.extend(part_keep_backups) + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='weeks',keep=keep_weeks,mode='max') + keep_backups.extend(part_keep_backups) + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='months',keep=keep_months,mode='max') + keep_backups.extend(part_keep_backups) + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='years',keep=keep_years,mode='max') + keep_backups.extend(part_keep_backups) + + keeps = [backup[0] for backup in keep_backups] + if not show_keep: + return [backup for backup in backuplist if backup not in keeps] + return keeps + +def main(): + parser = ArgumentParser(description="Program to print backups to keep or delete based on time stamps.") + parser.add_argument( + "-f", "--file", metavar="FILE", type=FileType('r'), required=True, + help="Path to file with list of backups to filter. Expects per line one backup name in format -") + parser.add_argument( + "-d", "--print-delete", action="store_true", required=False, + help="Instead of objects to kept, print out objects to be deleted." + ) + parser.add_argument( + "-H", "--hours-keep", metavar="KEEP", type=int, required=False, + default=24, help="Number of objects to keep on hourly basis." + ) + parser.add_argument( + "-D", "--days-keep", metavar="KEEP", type=int, required=False, + default=7, help="Number of objects to keep on daily basis." + ) + parser.add_argument( + "-W", "--weeks-keep", metavar="KEEP", type=int, required=False, + default=4, help="Number of objects to keep on weekly basis." + ) + parser.add_argument( + "-M", "--months-keep", metavar="KEEP", type=int, required=False, + default=12, help="Number of objects to keep on monthly basis." + ) + parser.add_argument( + "-Y", "--years-keep", metavar="KEEP", type=int, required=False, + default=10, help="Number of objects to keep on yearly basis." + ) + + args = parser.parse_args() + show_keep = not args.print_delete + keep_hours = args.hours_keep + keep_days = args.days_keep + keep_weeks = args.weeks_keep + keep_months = args.months_keep + keep_years = args.years_keep + + backuplist = [ + line.strip() + for line in args.file + if len(line.strip()) > 0 + ] + fail = False + for idx,backup in enumerate(backuplist): + if len(backup) < 10 or not backup[-10:].isnumeric(): + print(f"WARNING! Line {idx+1} has no timestamp suffix") + fail = True + if fail: + exit(1) + + considered_backups = filter_backups( + backuplist, show_keep, + keep_hours, keep_days, keep_weeks, keep_months, keep_years + ) + for backup in considered_backups: + print(backup) + +if __name__ == '__main__': + main() diff --git a/tasks/client.yml b/tasks/client.yml index 18852cd..080e7e3 100644 --- a/tasks/client.yml +++ b/tasks/client.yml @@ -1,20 +1,4 @@ --- -- name: Setup backup storage password - copy: - content: "{{ backup_storage_key }}" - dest: "{{ backup_home }}/.borg.key" - owner: "{{ backup_owner }}" - group: "{{ backup_group }}" - mode: "0600" - -- name: Setup ssh folder - file: - state: directory - path: "{{ backup_home }}/.ssh" - owner: "{{ backup_owner }}" - group: "{{ backup_group }}" - mode: "0700" - - name: Setup ssh keys copy: content: "{{ item.content }}" diff --git a/tasks/main.yml b/tasks/main.yml index 2110f30..1b19779 100755 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -25,9 +25,25 @@ name: "{{ backup_owner }}" group: "{{ backup_group }}" comment: backup user - shell: /sbin/nologin + shell: /bin/bash home: "{{ backup_home }}" +- name: Setup backup storage password + copy: + content: "{{ backup_storage_key }}" + dest: "{{ backup_home }}/.borg.key" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + +- name: Setup ssh folder + file: + state: directory + path: "{{ backup_home }}/.ssh" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0700" + - name: setup backup storage file: state: directory @@ -36,7 +52,7 @@ group: "{{ backup_group }}" mode: "0777" -- include_tasks: master.yml +- include_tasks: server.yml when: not backup_client - include_tasks: client.yml diff --git a/tasks/master.yml b/tasks/master.yml deleted file mode 100644 index 73b314f..0000000 --- a/tasks/master.yml +++ /dev/null @@ -1 +0,0 @@ ---- \ No newline at end of file diff --git a/tasks/server.yml b/tasks/server.yml new file mode 100644 index 0000000..a73f42b --- /dev/null +++ b/tasks/server.yml @@ -0,0 +1,118 @@ +--- +- name: setup backup installation folder + file: + state: directory + path: "{{ backup_inst }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0750" + +- name: install dateutil library service + apt: + update_cache: yes + state: "{% if cloud_update | bool %}latest{% else %}present{% endif %}" + install_recommends: yes + pkg: + - python3-dateutil + - rsync + +- name: setup filter program for backup pruning + copy: + src: opt/backup/filter_backups.py + dest: "{{ backup_inst }}/filter_backups.py" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0750" + +- name: setup ssh public keys of clients + copy: + content: "{{ backup_clients[item].sshpubkey }}" + dest: "{{ backup_home }}/.ssh/{{ item }}.pub" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + loop: "{{ backup_clients.keys() }}" + +- name: setup ssh private keys of clients + copy: + content: "{{ backup_clients[item].sshprivkey }}" + dest: "{{ backup_home }}/.ssh/{{ item }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + loop: "{{ backup_clients.keys() }}" + +- name: setup ssh config of clients + template: + src: "home/backup/.ssh/config.j2" + dest: "{{ backup_home }}/.ssh/config" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + +- name: setup backup key folder + file: + state: directory + path: "{{ backup_key_folder }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0700" + +- name: setup borg repository keys + copy: + content: "{{ backup_clients[item].borgkey }}" + dest: "{{ backup_key_folder }}/{{ backup_clients[item].app }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + loop: "{{ backup_clients.keys() }}" + +- name: setup local backup storage folder + file: + state: directory + path: "{{ item }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0750" + loop: + - "{{ backup_remotes_folder }}" + - "{{ backup_location }}/{{ backup_app }}" + +- name: setup borg local repository + shell: + cmd: "BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key) borg init --encryption repokey {{ backup_location }}/{{ backup_app }}" + creates: "{{ backup_location }}/{{ backup_app }}/config" + become_user: "{{ backup_owner }}" + become: yes + +- name: setup borg server script + template: + src: "opt/backup/inst/cloud-server-backup.j2" + dest: "{{ backup_inst }}/cloud-server-backup" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0750" + +- name: setup remote prune cron backup server job + cron: + name: "cloud server remote prune" + user: "{{ backup_cron_owner }}" + job: "{{ backup_inst }}/cloud-server-backup remote-prune" + hour: "{{ backup_server_remote_prune_time }}" + minute: 0 + +- name: setup remote backup cron backup server job + cron: + name: "cloud server remote backup" + user: "{{ backup_cron_owner }}" + job: "{{ backup_inst }}/cloud-server-backup backup" + hour: "{{ backup_server_remote_backup_time }}" + minute: 0 + +- name: setup local prune cron backup server job + cron: + name: "cloud server local prune" + user: "{{ backup_cron_owner }}" + job: "{{ backup_inst }}/cloud-server-backup prune" + hour: "{{ backup_server_local_prune_time }}" + minute: 0 \ No newline at end of file diff --git a/templates/home/backup/.ssh/config.j2 b/templates/home/backup/.ssh/config.j2 new file mode 100644 index 0000000..3bceb68 --- /dev/null +++ b/templates/home/backup/.ssh/config.j2 @@ -0,0 +1,7 @@ +{% for client in backup_clients %} +Host {{ client }} + HostName {{ backup_host }} + Port {{ backup_ssh_port }} + User {{ backup_clients[client].owner }} + IdentityFile {{ backup_home }}/.ssh/{{ client }} +{% endfor %} \ No newline at end of file diff --git a/templates/opt/backup/inst/cloud-server-backup.j2 b/templates/opt/backup/inst/cloud-server-backup.j2 new file mode 100644 index 0000000..37258a1 --- /dev/null +++ b/templates/opt/backup/inst/cloud-server-backup.j2 @@ -0,0 +1,287 @@ +#!/bin/bash + +BORGUSER="{{ backup_owner }}"; +TARGETFOLDER="{{ backup_remotes_folder }}"; +LOCALREPOLOCATION="{{ backup_location }}/{{ backup_app }}"; +APPLIST="{{ backup_clients | json_query('*.app') | join(' ') }}"; + +declare -A BORG_CLIENT_MAP +{% for client in backup_clients %} +BORG_CLIENT_MAP[{{ backup_clients[client].app }}]="{{ client }}"; +{% endfor %} + +declare -A BORG_USER_MAP +{% for client in backup_clients %} +BORG_USER_MAP[{{ backup_clients[client].app }}]="{{ backup_clients[client].owner }}"; +{% endfor %} + + +help (){ + echo "cloud-server-backup - backup and restore script on cloud backup with borg target v1.0 by L.Hahn. + +Usage: $0 COMMAND [APPS] + +COMMAND: + - remote-apps-list List available apps in remote borg repositories. + - remote-list [APPS] List available archives for apps in remote borg repositories. + - remote-prune [APPS] Prune remote app (or all) borg repositories. + - list List available archives of remote repositories on local repository. + - backup Perform backup of remote borg repositories and create a new archive in local borg repository. + - prune Prune local borg backup of remote repositories. + - restore ARCHIVENAME [APPS] Upload backup to borg repository to restore older file states. + May turn off your application if still running. + If latest is provided as ARCHIVENAME, the latest one based on timestamp is taken. + IF ARCHIVENAME is provided, will try to download it; throws error if not found. + If APP is provided, only the mentioned app repositories are restored. + + +APPS: A single app or comma-separated (sub-)set of below listed apps to be considered. +{% for client in backup_clients %} + - {{ backup_clients[client].app }} +{% endfor %} +"; +} + +############################################################################### +#=== HELP FUNCTIONS ==========================================================# +############################################################################### +remote_delete_single_backup () { + REPOLOCATION=$1; + ARCHIVENAME=$2; + BORG_PASSPHRASE=$3; + KEYFILE=$4; + echo "Remove archive $ARCHIVENAME"; + sudo -H -u $BORGUSER bash -c ' + ARCHIVENAME='$ARCHIVENAME'; + export BORG_PASSPHRASE='$BORG_PASSPHRASE'; + KEYFILE='$KEYFILE'; + REPOLOCATION='$REPOLOCATION'; + borg delete $REPOLOCATION::$ARCHIVENAME --rsh "/usr/bin/ssh -i $KEYFILE"'; +} + +remote_free_space () { + REPOLOCATION=$1; + KEYFILE=$2; + APP=$3; + echo "Free space on repository for app '$APP'."; + sudo -H -u $BORGUSER bash -c ' + KEYFILE='$KEYFILE'; + REPOLOCATION='$REPOLOCATION'; + borg compact $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"'; +} + +local_delete_single_backup () { + ARCHIVENAME=$1; + echo "Remove archive $ARCHIVENAME"; + sudo -H -u $BORGUSER bash -c ' + ARCHIVENAME='$ARCHIVENAME'; + export BORG_PASSPHRASE='$(cat {{ backup_home }}/.borg.key)'; + REPOLOCATION='$LOCALREPOLOCATION'; + borg delete $REPOLOCATION::$ARCHIVENAME'; +} + +local_free_space () { + echo "Free space on local repository."; + sudo -H -u $BORGUSER bash -c ' + REPOLOCATION='$LOCALREPOLOCATION'; + borg compact $REPOLOCATION'; +} + +has_argument () { + if [[ -z "$1" ]]; + then + echo "Missing required paramter!"; + exit 1; + fi +} + +validate_apps_or_default () { + VALID=1; + if [[ -z "$1" ]]; + then + echo $APPLIST; + else + CHECKLIST=$(echo $1 | tr ',' ' '); + for APP in $CHECKLIST; + do + if [[ ! ${APPLIST[@]} =~ $APP ]] + then + VALID=0; + echo "$APP is not a valid app."; + fi + done + if [[ ! $VALID == 1 ]] + then + echo "Aborting."; + exit 1; + else + echo $CHECKLIST; + fi + fi +} + + +############################################################################### +#=== USECASE FUNCTIONS =======================================================# +############################################################################### +remote_list_app () { + for APP in $APPLIST; + do + echo $APP; + done +} + +remote_list () { + REMOTELIST=$1; + for APP in $REMOTELIST; + do + REPOSITORYCLIENT=${BORG_CLIENT_MAP[$APP]} + BORG_REPO_USER=${BORG_USER_MAP[$APP]} + REPOLOCATION="ssh://$BORG_REPO_USER@{{ backup_host }}:{{ backup_ssh_port }}/./$APP" + BORG_PASSPHRASE=$(cat {{ backup_key_folder }}/$APP) + KEYFILE="{{ backup_home }}/.ssh/$REPOSITORYCLIENT" + ARCHIVEIDS=$( + sudo -H -u $BORGUSER bash -c ' + export BORG_PASSPHRASE='$BORG_PASSPHRASE'; + KEYFILE='$KEYFILE'; + REPOLOCATION='$REPOLOCATION'; + borg list $REPOLOCATION --rsh "/usr/bin/ssh -i $KEYFILE"' | sort -r + ); + echo "$ARCHIVEIDS"; + done +} + +remote_prune () { + REMOTEPRUNELIST=$1; + for APP in $REMOTEPRUNELIST; + do + ARCHIVELIST=$(remote_list $APP | cut -f 1 -d ' '); + echo "$ARCHIVELIST" > {{ backup_inst }}/backuplist.txt; + DELETELIST=$(python3 {{ backup_inst }}/filter_backups.py -f {{ backup_inst }}/backuplist.txt -d;) + BORG_REPO_USER=${BORG_USER_MAP[$APP]} + BORG_PASSPHRASE=$(cat {{ backup_key_folder }}/$APP) + REPOLOCATION="ssh://$BORG_REPO_USER@{{ backup_host }}:{{ backup_ssh_port }}/./$APP" + KEYFILE="{{ backup_home }}/.ssh/${BORG_CLIENT_MAP[$APP]}" + for DELETEBACKUP in $DELETELIST; + do + remote_delete_single_backup $REPOLOCATION $DELETEBACKUP $BORG_PASSPHRASE $KEYFILE; + done + remote_free_space $REPOLOCATION $KEYFILE $APP; + rm {{ backup_inst }}/backuplist.txt; + done +} + +local_list () { + ARCHIVEIDS=$( + sudo -H -u $BORGUSER bash -c ' + REPOLOCATION='$LOCALREPOLOCATION'; + export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key); + borg list $REPOLOCATION' | grep {{ backup_app }} | sort -r); + echo "$ARCHIVEIDS"; +} + +local_prune () { + ARCHIVELIST=$(local_list | cut -f 1 -d ' '); + echo "$ARCHIVELIST" > {{ backup_inst }}/locallist.txt; + DELETELIST=$(python3 {{ backup_inst }}/filter_backups.py -f {{ backup_inst }}/locallist.txt -d;) + for DELETEBACKUP in $DELETELIST; + do + local_delete_single_backup $DELETEBACKUP; + done + local_free_space; + rm {{ backup_inst }}/locallist.txt; +} + +local_backup () { + ARCHIVENAME="{{ backup_app }}-$(date '+%s')"; + mkdir -p $TARGETFOLDER/$ARCHIVENAME; + chown $BORGUSER: $TARGETFOLDER/$ARCHIVENAME; + for APP in $APPLIST; + do + sudo -H -u $BORGUSER bash -c ' + TARGETFOLDER='$TARGETFOLDER'; + ARCHIVENAME='$ARCHIVENAME'; + REPOLOCATION='${BORG_CLIENT_MAP[$APP]}':'$APP'; + rsync -azr $REPOLOCATION $TARGETFOLDER/$ARCHIVENAME/;'; + done + + sudo -H -u $BORGUSER bash -c ' + TARGETFOLDER='$TARGETFOLDER'; + REPOLOCATION='$LOCALREPOLOCATION'; + ARCHIVENAME='$ARCHIVENAME'; + export BORG_PASSPHRASE=$(cat {{ backup_home }}/.borg.key); + borg create -C lzma $REPOLOCATION::$ARCHIVENAME $TARGETFOLDER/$ARCHIVENAME;'; + + rm -rf $TARGETFOLDER/$ARCHIVENAME; +} + +local_delete () { + ARCHIVENAME=$1; + ARCHIVEIDS=$(local_list | cut -f 1 -d ' '); + if [[ "$ARCHIVEIDS" != *$ARCHIVENAME* ]]; + then + echo "ERROR! Provided archivename '$ARCHIVENAME' is not part of the available archives! Aborting."; + exit 1; + fi + local_delete_single_backup $ARCHIVENAME; +} + + +############################################################################### +#=== MAIN FUNCTIONS ==========================================================# +############################################################################### + +( + flock -n 9 || { + echo "BACKUP ALREADY RUNNING! ABORTING."; + exit 1; + } + + if [ $# = 0 ]; then + help; + exit 0; + fi + + action=$1 + case $action in + "remote-apps-list") + remote_list_app; + ;; + + "remote-list") + USERLIST=$(validate_apps_or_default "$2"); + remote_list "$USERLIST"; + ;; + + "remote-prune") + USERLIST=$(validate_apps_or_default "$2"); + remote_prune "$USERLIST"; + ;; + + "list") + local_list; + ;; + + "prune") + local_prune; + ;; + + "backup") + local_backup; + ;; + + "restore") + has_argument $2; + USERLIST=$(validate_apps_or_default $3); + #local_restore $2; + ;; + "delete") + has_argument $2; + local_delete $2; + ;; + *) + help; + exit 0; + ;; + esac +) 9>/var/run/lock/cloud-server-backup.lock; \ No newline at end of file diff --git a/templates/usr/local/bin/cloud_backup.j2 b/templates/usr/local/bin/cloud_backup.j2 index 65d27cd..1d02594 100644 --- a/templates/usr/local/bin/cloud_backup.j2 +++ b/templates/usr/local/bin/cloud_backup.j2 @@ -143,5 +143,4 @@ has_argument () { exit 0; ;; esac - echo "" ) 9>/var/run/lock/cloud-backup.lock; diff --git a/vars/main.yml b/vars/main.yml index ffd23a9..0731f27 100755 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,3 +1,6 @@ --- backup_home: "{{ backup_root }}/home" -backup_storage: "{{ backup_root }}/storage" \ No newline at end of file +backup_inst: "{{ backup_root }}/inst" +backup_storage: "{{ backup_root }}/storage" +backup_key_folder: "{{ backup_home }}/.keys" +backup_remotes_folder: "{{ backup_storage }}/remote-backups" \ No newline at end of file