From ae923c3743c0d3e279c62a771188128f4b414c91 Mon Sep 17 00:00:00 2001 From: lhahn Date: Sun, 11 Aug 2024 20:57:32 +0200 Subject: [PATCH] Setup backup server configuration for backup replication. --- defaults/main.yml | 16 ++++ files/opt/backup/filter_backups.py | 117 +++++++++++++++++++++++++++ tasks/client.yml | 8 -- tasks/main.yml | 10 ++- tasks/master.yml | 1 - tasks/server.yml | 60 ++++++++++++++ templates/home/backup/.ssh/config.j2 | 7 ++ vars/main.yml | 4 +- 8 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 files/opt/backup/filter_backups.py delete mode 100644 tasks/master.yml create mode 100644 tasks/server.yml create mode 100644 templates/home/backup/.ssh/config.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 8f8d5ed..8cfc64b 100755 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -26,3 +26,19 @@ backup_script: echo "This is executed before borg backup. Please collect data for backup in path: {{ backup_storage }}" postwork_restore: | echo "This is executed after borg restore. Please collect data during restore from path: {{ backup_storage }}" + +backup_ssh_port: 22 +backup_host: my-borg-repo.tld +backup_clients: [] +# app1: +# owner: user1 +# app: app1 +# borgkey: BorgRepoKey1 +# sshpubkey: SshPubKey1 +# sshprivkey: SshPrivKey1 +# app2: +# owner: user2 +# app: app2 +# borgkey: BorgRepoKey2 +# sshpubkey: SshPubKey2 +# sshprivkey: SshPrivKey2 \ No newline at end of file diff --git a/files/opt/backup/filter_backups.py b/files/opt/backup/filter_backups.py new file mode 100644 index 0000000..cd32097 --- /dev/null +++ b/files/opt/backup/filter_backups.py @@ -0,0 +1,117 @@ +from datetime import datetime +from dateutil.relativedelta import relativedelta +from argparse import ArgumentParser, FileType + +def filter_backups_timeslot(offset, backups, timeslot='days', keep=7, mode='max', printy=True): + keep_backups = [] + for timeslot_off in range(1, keep): + indices = [ + idx + for idx,value in enumerate(backups) + if ( + value[1] >= (offset - relativedelta(**{timeslot:timeslot_off})) + ) + ] + if not indices: + continue + if mode == "min": + #removed not considered backups for this slot! + for idx in reversed(indices[1:]): + backups.pop(idx) + keep_backups.append(backups.pop(indices[0])) + elif mode == "max": + keep_backups.append(backups.pop(indices[-1])) + #removed not considered backups for this slot! + for idx in reversed(indices[:-1]): + backups.pop(idx) + else: + for idx in sorted(indices, reverse=True): + keep_backups.append(backups.pop(idx)) + return keep_backups, backups + +def filter_backups(backuplist, show_keep=True, keep_hours=24, keep_days=7, keep_weeks=4, keep_months=12, keep_years=10): + backups = sorted( + [ + (backup, datetime.fromtimestamp(int(backup.split('-')[1]))) + for backup in backuplist + ], key=lambda v: v[1], reverse=True + ) + offset = backups[0][1] + keep_backups = [backups.pop(0)] + + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='hours',keep=keep_hours,mode='all') + keep_backups.extend(part_keep_backups) + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='days',keep=keep_days,mode='max') + keep_backups.extend(part_keep_backups) + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='weeks',keep=keep_weeks,mode='max') + keep_backups.extend(part_keep_backups) + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='months',keep=keep_months,mode='max') + keep_backups.extend(part_keep_backups) + part_keep_backups, backups = filter_backups_timeslot(offset, backups, timeslot='years',keep=keep_years,mode='max') + keep_backups.extend(part_keep_backups) + + keeps = [backup[0] for backup in keep_backups] + if not show_keep: + return [backup for backup in backuplist if backup not in keeps] + return keeps + +def main(): + parser = ArgumentParser(description="Program to print backups to keep or delete based on time stamps.") + parser.add_argument( + "-f", "--file", metavar="FILE", type=FileType('r'), required=True, + help="Path to file with list of backups to filter. Expects per line one backup name in format -") + parser.add_argument( + "-d", "--print-delete", action="store_true", required=False, + help="Instead of objects to kept, print out objects to be deleted." + ) + parser.add_argument( + "-H", "--hours-keep", metavar="KEEP", type=int, required=False, + default=24, help="Number of objects to keep on hourly basis." + ) + parser.add_argument( + "-D", "--days-keep", metavar="KEEP", type=int, required=False, + default=7, help="Number of objects to keep on daily basis." + ) + parser.add_argument( + "-W", "--weeks-keep", metavar="KEEP", type=int, required=False, + default=4, help="Number of objects to keep on weekly basis." + ) + parser.add_argument( + "-M", "--months-keep", metavar="KEEP", type=int, required=False, + default=12, help="Number of objects to keep on monthly basis." + ) + parser.add_argument( + "-Y", "--years-keep", metavar="KEEP", type=int, required=False, + default=10, help="Number of objects to keep on yearly basis." + ) + + args = parser.parse_args() + show_keep = not args.print_delete + keep_hours = args.hours_keep + keep_days = args.days_keep + keep_weeks = args.weeks_keep + keep_months = args.months_keep + keep_years = args.years_keep + + backuplist = [ + line.strip() + for line in args.file + if len(line.strip()) > 0 + ] + fail = False + for idx,backup in enumerate(backuplist): + if len(backup) < 10 or not backup[-10:].isnumeric(): + print(f"WARNING! Line {idx+1} has no timestamp suffix") + fail = True + if fail: + exit(1) + + considered_backups = filter_backups( + backuplist, show_keep, + keep_hours, keep_days, keep_weeks, keep_months, keep_years + ) + for backup in considered_backups: + print(backup) + +if __name__ == '__main__': + main() diff --git a/tasks/client.yml b/tasks/client.yml index 18852cd..6091c2a 100644 --- a/tasks/client.yml +++ b/tasks/client.yml @@ -7,14 +7,6 @@ group: "{{ backup_group }}" mode: "0600" -- name: Setup ssh folder - file: - state: directory - path: "{{ backup_home }}/.ssh" - owner: "{{ backup_owner }}" - group: "{{ backup_group }}" - mode: "0700" - - name: Setup ssh keys copy: content: "{{ item.content }}" diff --git a/tasks/main.yml b/tasks/main.yml index 2110f30..d2ac49a 100755 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -28,6 +28,14 @@ shell: /sbin/nologin home: "{{ backup_home }}" +- name: Setup ssh folder + file: + state: directory + path: "{{ backup_home }}/.ssh" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0700" + - name: setup backup storage file: state: directory @@ -36,7 +44,7 @@ group: "{{ backup_group }}" mode: "0777" -- include_tasks: master.yml +- include_tasks: server.yml when: not backup_client - include_tasks: client.yml diff --git a/tasks/master.yml b/tasks/master.yml deleted file mode 100644 index 73b314f..0000000 --- a/tasks/master.yml +++ /dev/null @@ -1 +0,0 @@ ---- \ No newline at end of file diff --git a/tasks/server.yml b/tasks/server.yml new file mode 100644 index 0000000..baa5e3b --- /dev/null +++ b/tasks/server.yml @@ -0,0 +1,60 @@ +--- +- name: setup backup installation folder + file: + state: directory + path: "{{ backup_inst }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0755" + +- name: setup filter program for backup pruning + copy: + src: opt/backup/filter_backups.py + dest: "{{ backup_inst }}/filter_backups.py" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0750" + +- name: setup ssh public keys of clients + copy: + content: "{{ backup_clients[item].sshpubkey }}" + dest: "{{ backup_home }}/.ssh/{{ item }}.pub" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + loop: "{{ backup_clients.keys() }}" + +- name: setup ssh private keys of clients + copy: + content: "{{ backup_clients[item].sshprivkey }}" + dest: "{{ backup_home }}/.ssh/{{ item }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + loop: "{{ backup_clients.keys() }}" + +- name: setup ssh config of clients + template: + src: "home/backup/.ssh/config.j2" + dest: "{{ backup_home }}/.ssh/config" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + +- name: setup backup key folder + file: + state: directory + path: "{{ backup_key_folder }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0700" + + +- name: setup borg repository keys + copy: + content: "{{ backup_clients[item].borgkey }}" + dest: "{{ backup_key_folder }}/{{ backup_clients[item].app }}" + owner: "{{ backup_owner }}" + group: "{{ backup_group }}" + mode: "0600" + loop: "{{ backup_clients.keys() }}" \ No newline at end of file diff --git a/templates/home/backup/.ssh/config.j2 b/templates/home/backup/.ssh/config.j2 new file mode 100644 index 0000000..3bceb68 --- /dev/null +++ b/templates/home/backup/.ssh/config.j2 @@ -0,0 +1,7 @@ +{% for client in backup_clients %} +Host {{ client }} + HostName {{ backup_host }} + Port {{ backup_ssh_port }} + User {{ backup_clients[client].owner }} + IdentityFile {{ backup_home }}/.ssh/{{ client }} +{% endfor %} \ No newline at end of file diff --git a/vars/main.yml b/vars/main.yml index ffd23a9..68d2564 100755 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,3 +1,5 @@ --- backup_home: "{{ backup_root }}/home" -backup_storage: "{{ backup_root }}/storage" \ No newline at end of file +backup_inst: "{{ backup_root }}/inst" +backup_storage: "{{ backup_root }}/storage" +backup_key_folder: "{{ backup_home }}/.keys" \ No newline at end of file