diff --git a/bin/actions/service.py b/bin/actions/service.py index 12b244a..84b435e 100755 --- a/bin/actions/service.py +++ b/bin/actions/service.py @@ -325,7 +325,7 @@ def cmd_list(args): print() println("System services:", "blue") - systemd_timers = ["borg-backup.timer", "dynamic-dns.timer"] + systemd_timers = ["borg-backup.timer", "borg-local-sync.timer", "dynamic-dns.timer"] for timer in systemd_timers: is_active, is_enabled, next_run = get_systemd_timer_status(timer) diff --git a/bin/actions/timers.py b/bin/actions/timers.py index 8547f3a..20fcf64 100755 --- a/bin/actions/timers.py +++ b/bin/actions/timers.py @@ -80,6 +80,7 @@ def main(): # Show timer statuses timers = [ ("borg-backup.timer", True), + ("borg-local-sync.timer", True), ("dynamic-dns.timer", True) ] diff --git a/config/ansible/handlers/main.yml b/config/ansible/handlers/main.yml index ea98ace..b9c1b9f 100644 --- a/config/ansible/handlers/main.yml +++ b/config/ansible/handlers/main.yml @@ -10,3 +10,21 @@ name: ssh state: restarted enabled: true + +- name: reload systemd + become: true + ansible.builtin.systemd: + daemon_reload: true + +- name: restart borg-local-sync + become: true + ansible.builtin.systemd: + name: borg-local-sync.service + enabled: true + +- name: restart borg-local-sync-timer + become: true + ansible.builtin.systemd: + name: borg-local-sync.timer + state: restarted + enabled: true diff --git a/config/ansible/tasks/servers/borg-local-sync.yml b/config/ansible/tasks/servers/borg-local-sync.yml new file mode 100644 index 0000000..e354bce --- /dev/null +++ b/config/ansible/tasks/servers/borg-local-sync.yml @@ -0,0 +1,95 @@ +--- +- name: Borg Local Sync Installation and Configuration + block: + - name: Set Borg backup facts + ansible.builtin.set_fact: + borg_passphrase: "{{ lookup('community.general.onepassword', 'Borg Backup', vault='Dotfiles', field='password') }}" + borg_config_dir: "{{ ansible_env.HOME }}/.config/borg" + borg_backup_dir: "/mnt/services" + borg_repo_dir: "/mnt/object_storage/borg-repo" + + - name: Create Borg local sync script + template: + src: borg-local-sync.sh.j2 + dest: /usr/local/bin/borg-local-sync.sh + mode: "0755" + owner: root + group: root + become: yes + tags: + - borg-local-sync + + - name: Create Borg local sync systemd service + template: + src: borg-local-sync.service.j2 + dest: /etc/systemd/system/borg-local-sync.service + mode: "0644" + owner: root + group: root + become: yes + notify: + - reload systemd + tags: + - borg-local-sync + + - name: Create Borg local sync systemd timer + template: + src: borg-local-sync.timer.j2 + dest: /etc/systemd/system/borg-local-sync.timer + mode: "0644" + owner: root + group: root + become: yes + notify: + - reload systemd + - restart borg-local-sync-timer + tags: + - borg-local-sync + + - name: Create log file for Borg local sync + file: + path: /var/log/borg-local-sync.log + state: touch + owner: root + group: root + mode: "0644" + become: yes + tags: + - borg-local-sync + + - name: Enable and start Borg local sync timer + systemd: + name: borg-local-sync.timer + enabled: yes + state: started + daemon_reload: yes + become: yes + tags: + - borg-local-sync + + - name: Add logrotate configuration for Borg local sync + copy: + content: | + /var/log/borg-local-sync.log { + daily + rotate 30 + compress + delaycompress + missingok + notifempty + create 644 root root + } + dest: /etc/logrotate.d/borg-local-sync + mode: "0644" + owner: root + group: root + become: yes + tags: + - borg-local-sync + - borg + - backup + + tags: + - borg-local-sync + - borg + - backup diff --git a/config/ansible/tasks/servers/server.yml b/config/ansible/tasks/servers/server.yml index 9bc814b..e287d15 100644 --- a/config/ansible/tasks/servers/server.yml +++ b/config/ansible/tasks/servers/server.yml @@ -35,6 +35,11 @@ tags: - borg-backup + - name: Include Borg Local Sync tasks + ansible.builtin.include_tasks: borg-local-sync.yml + tags: + - borg-local-sync + - name: System performance optimizations ansible.posix.sysctl: name: "{{ item.name }}" diff --git a/config/ansible/templates/borg-backup.sh.j2 b/config/ansible/templates/borg-backup.sh.j2 index 6147ada..3d3bb99 100644 --- a/config/ansible/templates/borg-backup.sh.j2 +++ b/config/ansible/templates/borg-backup.sh.j2 @@ -26,6 +26,7 @@ log() { # Telegram notification function send_telegram() { local message="$1" + local silent="${2:-false}" if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then log "Telegram credentials not configured, skipping notification" @@ -36,7 +37,8 @@ send_telegram() { { "chat_id": "$TELEGRAM_CHAT_ID", "text": "$message", - "parse_mode": "HTML" + "parse_mode": "HTML", + "disable_notification": $silent } EOF ) @@ -129,7 +131,7 @@ if [ $global_exit -eq 0 ]; then 📊 Repository: {{ borg_repo_dir }} 🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S') -All operations completed without errors." +All operations completed without errors." "true" elif [ $global_exit -eq 1 ]; then log "Backup completed with warnings (exit code: $global_exit)" send_telegram "⚠️ Borg Backup Warning diff --git a/config/ansible/templates/borg-local-sync.service.j2 b/config/ansible/templates/borg-local-sync.service.j2 new file mode 100644 index 0000000..f31b5da --- /dev/null +++ b/config/ansible/templates/borg-local-sync.service.j2 @@ -0,0 +1,48 @@ +[Unit] +Description=Borg Local Sync - Copy Borg repository to local storage +Documentation=man:borg(1) +After=network-online.target +Wants=network-online.target +# Ensure this runs after the main backup has completed +After=borg-backup.service + +[Service] +Type=oneshot +User=root +Group=root + +# Set up environment +Environment="PATH=/usr/local/bin:/usr/bin:/bin" +Environment="LANG=en_US.UTF-8" +Environment="LC_ALL=en_US.UTF-8" + +# Security settings +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=/var/log /mnt/borg-backups {{ borg_config_dir }} +PrivateTmp=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectControlGroups=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes + +# Resource limits +MemoryMax=2G +CPUQuota=80% +IOWeight=200 + +# Timeout settings (local sync might take a while for initial copy) +TimeoutStartSec=3600 +TimeoutStopSec=300 + +# Execute the sync script +ExecStart=/usr/local/bin/borg-local-sync.sh + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=borg-local-sync + +[Install] +WantedBy=multi-user.target diff --git a/config/ansible/templates/borg-local-sync.sh.j2 b/config/ansible/templates/borg-local-sync.sh.j2 new file mode 100644 index 0000000..785e171 --- /dev/null +++ b/config/ansible/templates/borg-local-sync.sh.j2 @@ -0,0 +1,227 @@ +#!/bin/bash + +# Borg local sync script for creating local copies of cloud backups +# This script syncs the Borg repository from JuiceFS/S3 to local ZFS storage + +# Set environment variables +export BORG_REPO_SOURCE="{{ borg_repo_dir }}" +export BORG_REPO_LOCAL="/mnt/borg-backups" +export ZFS_POOL="datapool" +export ZFS_DATASET="datapool/borg-backups" +export MOUNT_POINT="/mnt/borg-backups" + +# Telegram notification variables +export TELEGRAM_BOT_TOKEN="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='password') }}" +export TELEGRAM_CHAT_ID="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='chat_id') }}" + +# Log function +log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a /var/log/borg-local-sync.log +} + +# Telegram notification function +send_telegram() { + local message="$1" + local silent="${2:-false}" + + if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then + log "Telegram credentials not configured, skipping notification" + return + fi + + local payload=$(cat < /dev/null 2>&1 + + if [ $? -eq 0 ]; then + log "Telegram notification sent successfully" + else + log "Failed to send Telegram notification" + fi +} + +# Check if ZFS pool is available +check_zfs_pool() { + if ! zpool status "$ZFS_POOL" > /dev/null 2>&1; then + log "ERROR: ZFS pool $ZFS_POOL is not available" + send_telegram "❌ Borg Local Sync Failed + +❌ ZFS pool not available: $ZFS_POOL +🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') + +The 20TB USB drive may not be connected or the ZFS pool is not imported. +Please check the physical connection and run: sudo zpool import $ZFS_POOL" + return 1 + fi + + # Check if the specific ZFS dataset exists + if ! zfs list "$ZFS_DATASET" > /dev/null 2>&1; then + log "ERROR: ZFS dataset $ZFS_DATASET is not available" + send_telegram "❌ Borg Local Sync Failed + +❌ ZFS dataset not available: $ZFS_DATASET +🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') + +The ZFS dataset may not exist or be mounted. +Please check: sudo zfs create $ZFS_DATASET" + return 1 + fi + return 0 +} + +# Check if mount point is available +check_mount_point() { + if ! mountpoint -q "$MOUNT_POINT"; then + log "ERROR: Mount point $MOUNT_POINT is not mounted" + send_telegram "❌ Borg Local Sync Failed + +❌ Mount point not available: $MOUNT_POINT +🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') + +The ZFS dataset may not be mounted. +Please check: sudo zfs mount $ZFS_DATASET" + return 1 + fi + return 0 +} + +# Check if source repository is available +check_source_repo() { + if [ ! -d "$BORG_REPO_SOURCE" ]; then + log "ERROR: Source Borg repository not found: $BORG_REPO_SOURCE" + send_telegram "❌ Borg Local Sync Failed + +❌ Source repository not found: $BORG_REPO_SOURCE +🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') + +JuiceFS may not be mounted or the source repository path is incorrect." + return 1 + fi + return 0 +} + +# Check available space +check_space() { + local source_size=$(sudo du -sb "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1) + local available_space=$(df -B1 "$MOUNT_POINT" | tail -1 | awk '{print $4}') + + if [ -z "$source_size" ]; then + log "WARNING: Could not determine source repository size" + return 0 + fi + + # Add 20% buffer for safety + local required_space=$((source_size * 120 / 100)) + + if [ "$available_space" -lt "$required_space" ]; then + local source_gb=$((source_size / 1024 / 1024 / 1024)) + local available_gb=$((available_space / 1024 / 1024 / 1024)) + local required_gb=$((required_space / 1024 / 1024 / 1024)) + + log "ERROR: Insufficient space. Source: ${source_gb}GB, Available: ${available_gb}GB, Required: ${required_gb}GB" + send_telegram "❌ Borg Local Sync Failed + +❌ Insufficient disk space +📊 Source size: ${source_gb}GB +💾 Available: ${available_gb}GB +⚠️ Required: ${required_gb}GB (with 20% buffer) +🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') + +Please free up space on the local backup drive." + return 1 + fi + + return 0 +} + +# Perform the sync +sync_repository() { + log "Starting rsync of Borg repository" + + # Get initial sizes for reporting + local source_size_before=$(sudo du -sh "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1) + local dest_size_before="0B" + if [ -d "$BORG_REPO_LOCAL" ]; then + dest_size_before=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1) + fi + + # Perform the sync with detailed logging + sudo rsync -avh --delete --progress \ + --exclude="lock.exclusive" \ + --exclude="lock.roster" \ + "$BORG_REPO_SOURCE/" "$BORG_REPO_LOCAL/" 2>&1 | while read line; do + log "rsync: $line" + done + + local rsync_exit=${PIPESTATUS[0]} + + # Get final sizes for reporting + local dest_size_after=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1) + + if [ $rsync_exit -eq 0 ]; then + log "Rsync completed successfully" + send_telegram "🔒 Borg Local Sync Success + +✅ Local backup sync completed successfully +📂 Source: $BORG_REPO_SOURCE (${source_size_before}) +💾 Destination: $BORG_REPO_LOCAL (${dest_size_after}) +🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S') + +Local backup copy is now up to date." "true" + return 0 + else + log "Rsync failed with exit code: $rsync_exit" + send_telegram "❌ Borg Local Sync Failed + +❌ Rsync failed during repository sync +📂 Source: $BORG_REPO_SOURCE +💾 Destination: $BORG_REPO_LOCAL +🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') + +Exit code: $rsync_exit +Check logs: /var/log/borg-local-sync.log" + return 1 + fi +} + +# Main execution +log "Starting Borg local sync process" + +# Run all pre-flight checks +if ! check_zfs_pool; then + exit 1 +fi + +if ! check_mount_point; then + exit 1 +fi + +if ! check_source_repo; then + exit 1 +fi + +if ! check_space; then + exit 1 +fi + +# All checks passed, proceed with sync +log "All pre-flight checks passed, starting sync" + +if sync_repository; then + log "Local sync completed successfully" + exit 0 +else + log "Local sync failed" + exit 1 +fi diff --git a/config/ansible/templates/borg-local-sync.timer.j2 b/config/ansible/templates/borg-local-sync.timer.j2 new file mode 100644 index 0000000..acf54da --- /dev/null +++ b/config/ansible/templates/borg-local-sync.timer.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Run Borg Local Sync daily +Documentation=man:borg(1) +Requires=borg-local-sync.service + +[Timer] +# Run daily at 3:00 AM (1 hour after main backup at 2:00 AM) +OnCalendar=*-*-* 03:00:00 +# Add randomization to prevent conflicts if multiple systems exist +RandomizedDelaySec=300 +# Ensure timer persists across reboots +Persistent=true +# Wake system from suspend if needed +WakeSystem=false + +[Install] +WantedBy=timers.target diff --git a/config/nextcloud.cfg b/config/nextcloud.cfg index 5a3aa7e..a49db9c 100644 --- a/config/nextcloud.cfg +++ b/config/nextcloud.cfg @@ -35,24 +35,24 @@ useNewBigFolderSizeLimit=true 0\Folders\2\version=2 0\Folders\2\virtualFilesMode=off 0\Folders\3\ignoreHiddenFiles=false -0\Folders\3\journalPath=.sync_65289e64a490.db -0\Folders\3\localPath=/home/menno/Documents/ +0\Folders\3\journalPath=.sync_886cca272fe5.db +0\Folders\3\localPath=/home/menno/Pictures/ 0\Folders\3\paused=false -0\Folders\3\targetPath=/Documents +0\Folders\3\targetPath=/Pictures 0\Folders\3\version=2 0\Folders\3\virtualFilesMode=off 0\Folders\4\ignoreHiddenFiles=false -0\Folders\4\journalPath=.sync_886cca272fe5.db -0\Folders\4\localPath=/home/menno/Pictures/ +0\Folders\4\journalPath=.sync_90ea5e3c7a33.db +0\Folders\4\localPath=/home/menno/Videos/ 0\Folders\4\paused=false -0\Folders\4\targetPath=/Pictures +0\Folders\4\targetPath=/Videos 0\Folders\4\version=2 0\Folders\4\virtualFilesMode=off 0\Folders\5\ignoreHiddenFiles=false -0\Folders\5\journalPath=.sync_90ea5e3c7a33.db -0\Folders\5\localPath=/home/menno/Videos/ +0\Folders\5\journalPath=.sync_65289e64a490.db +0\Folders\5\localPath=/home/menno/Documents/ 0\Folders\5\paused=false -0\Folders\5\targetPath=/Videos +0\Folders\5\targetPath=/Documents 0\Folders\5\version=2 0\Folders\5\virtualFilesMode=off 0\Folders\6\ignoreHiddenFiles=false @@ -92,4 +92,4 @@ useDownloadLimit=0 useUploadLimit=0 [Settings] -geometry=@ByteArray(\x1\xd9\xd0\xcb\0\x3\0\0\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a\0\0\0\x1\0\0\0\0\x14\0\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a) +geometry=@ByteArray(\x1\xd9\xd0\xcb\0\x3\0\0\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj\0\0\0\x1\0\0\0\0\x14\0\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj)