Add Borg local sync system service and configuration

bin/actions/service.py

@@ -325,7 +325,7 @@ def cmd_list(args):
     print()
     println("System services:", "blue")
 
-    systemd_timers = ["borg-backup.timer", "dynamic-dns.timer"]
+    systemd_timers = ["borg-backup.timer", "borg-local-sync.timer", "dynamic-dns.timer"]
 
     for timer in systemd_timers:
         is_active, is_enabled, next_run = get_systemd_timer_status(timer)

bin/actions/timers.py

@@ -80,6 +80,7 @@ def main():
     # Show timer statuses
     timers = [
         ("borg-backup.timer", True),
+        ("borg-local-sync.timer", True),
         ("dynamic-dns.timer", True)
     ]
 

config/ansible/handlers/main.yml

@@ -10,3 +10,21 @@
     name: ssh
     state: restarted
     enabled: true
+
+- name: reload systemd
+  become: true
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: restart borg-local-sync
+  become: true
+  ansible.builtin.systemd:
+    name: borg-local-sync.service
+    enabled: true
+
+- name: restart borg-local-sync-timer
+  become: true
+  ansible.builtin.systemd:
+    name: borg-local-sync.timer
+    state: restarted
+    enabled: true

config/ansible/tasks/servers/borg-local-sync.yml

@@ -0,0 +1,95 @@
+---
+- name: Borg Local Sync Installation and Configuration
+  block:
+    - name: Set Borg backup facts
+      ansible.builtin.set_fact:
+        borg_passphrase: "{{ lookup('community.general.onepassword', 'Borg Backup', vault='Dotfiles', field='password') }}"
+        borg_config_dir: "{{ ansible_env.HOME }}/.config/borg"
+        borg_backup_dir: "/mnt/services"
+        borg_repo_dir: "/mnt/object_storage/borg-repo"
+
+    - name: Create Borg local sync script
+      template:
+        src: borg-local-sync.sh.j2
+        dest: /usr/local/bin/borg-local-sync.sh
+        mode: "0755"
+        owner: root
+        group: root
+      become: yes
+      tags:
+        - borg-local-sync
+
+    - name: Create Borg local sync systemd service
+      template:
+        src: borg-local-sync.service.j2
+        dest: /etc/systemd/system/borg-local-sync.service
+        mode: "0644"
+        owner: root
+        group: root
+      become: yes
+      notify:
+        - reload systemd
+      tags:
+        - borg-local-sync
+
+    - name: Create Borg local sync systemd timer
+      template:
+        src: borg-local-sync.timer.j2
+        dest: /etc/systemd/system/borg-local-sync.timer
+        mode: "0644"
+        owner: root
+        group: root
+      become: yes
+      notify:
+        - reload systemd
+        - restart borg-local-sync-timer
+      tags:
+        - borg-local-sync
+
+    - name: Create log file for Borg local sync
+      file:
+        path: /var/log/borg-local-sync.log
+        state: touch
+        owner: root
+        group: root
+        mode: "0644"
+      become: yes
+      tags:
+        - borg-local-sync
+
+    - name: Enable and start Borg local sync timer
+      systemd:
+        name: borg-local-sync.timer
+        enabled: yes
+        state: started
+        daemon_reload: yes
+      become: yes
+      tags:
+        - borg-local-sync
+
+    - name: Add logrotate configuration for Borg local sync
+      copy:
+        content: |
+          /var/log/borg-local-sync.log {
+              daily
+              rotate 30
+              compress
+              delaycompress
+              missingok
+              notifempty
+              create 644 root root
+          }
+        dest: /etc/logrotate.d/borg-local-sync
+        mode: "0644"
+        owner: root
+        group: root
+      become: yes
+      tags:
+        - borg-local-sync
+        - borg
+        - backup
+
+  tags:
+    - borg-local-sync
+    - borg
+    - backup

config/ansible/tasks/servers/server.yml

@@ -35,6 +35,11 @@
       tags:
         - borg-backup
 
+    - name: Include Borg Local Sync tasks
+      ansible.builtin.include_tasks: borg-local-sync.yml
+      tags:
+        - borg-local-sync
+
     - name: System performance optimizations
       ansible.posix.sysctl:
         name: "{{ item.name }}"

config/ansible/templates/borg-backup.sh.j2

@@ -26,6 +26,7 @@ log() {
 # Telegram notification function
 send_telegram() {
     local message="$1"
+    local silent="${2:-false}"
 
     if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then
         log "Telegram credentials not configured, skipping notification"
@@ -36,7 +37,8 @@ send_telegram() {
 {
     "chat_id": "$TELEGRAM_CHAT_ID",
     "text": "$message",
-    "parse_mode": "HTML"
+    "parse_mode": "HTML",
+    "disable_notification": $silent
 }
 EOF
 )
@@ -129,7 +131,7 @@ if [ $global_exit -eq 0 ]; then
 📊 Repository: {{ borg_repo_dir }}
 🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S')
 
-All operations completed without errors."
+All operations completed without errors." "true"
 elif [ $global_exit -eq 1 ]; then
     log "Backup completed with warnings (exit code: $global_exit)"
     send_telegram "⚠️ <b>Borg Backup Warning</b>

config/ansible/templates/borg-local-sync.service.j2

@@ -0,0 +1,48 @@
+[Unit]
+Description=Borg Local Sync - Copy Borg repository to local storage
+Documentation=man:borg(1)
+After=network-online.target
+Wants=network-online.target
+# Ensure this runs after the main backup has completed
+After=borg-backup.service
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+
+# Set up environment
+Environment="PATH=/usr/local/bin:/usr/bin:/bin"
+Environment="LANG=en_US.UTF-8"
+Environment="LC_ALL=en_US.UTF-8"
+
+# Security settings
+ProtectSystem=strict
+ProtectHome=read-only
+ReadWritePaths=/var/log /mnt/borg-backups {{ borg_config_dir }}
+PrivateTmp=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+
+# Resource limits
+MemoryMax=2G
+CPUQuota=80%
+IOWeight=200
+
+# Timeout settings (local sync might take a while for initial copy)
+TimeoutStartSec=3600
+TimeoutStopSec=300
+
+# Execute the sync script
+ExecStart=/usr/local/bin/borg-local-sync.sh
+
+# Logging
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=borg-local-sync
+
+[Install]
+WantedBy=multi-user.target

config/ansible/templates/borg-local-sync.sh.j2

@@ -0,0 +1,227 @@
+#!/bin/bash
+
+# Borg local sync script for creating local copies of cloud backups
+# This script syncs the Borg repository from JuiceFS/S3 to local ZFS storage
+
+# Set environment variables
+export BORG_REPO_SOURCE="{{ borg_repo_dir }}"
+export BORG_REPO_LOCAL="/mnt/borg-backups"
+export ZFS_POOL="datapool"
+export ZFS_DATASET="datapool/borg-backups"
+export MOUNT_POINT="/mnt/borg-backups"
+
+# Telegram notification variables
+export TELEGRAM_BOT_TOKEN="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='password') }}"
+export TELEGRAM_CHAT_ID="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='chat_id') }}"
+
+# Log function
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a /var/log/borg-local-sync.log
+}
+
+# Telegram notification function
+send_telegram() {
+    local message="$1"
+    local silent="${2:-false}"
+
+    if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then
+        log "Telegram credentials not configured, skipping notification"
+        return
+    fi
+
+    local payload=$(cat <<EOF
+{
+    "chat_id": "$TELEGRAM_CHAT_ID",
+    "text": "$message",
+    "parse_mode": "HTML",
+    "disable_notification": $silent
+}
+EOF
+)
+
+    curl -s -X POST \
+        -H "Content-Type: application/json" \
+        -d "$payload" \
+        "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" > /dev/null 2>&1
+
+    if [ $? -eq 0 ]; then
+        log "Telegram notification sent successfully"
+    else
+        log "Failed to send Telegram notification"
+    fi
+}
+
+# Check if ZFS pool is available
+check_zfs_pool() {
+    if ! zpool status "$ZFS_POOL" > /dev/null 2>&1; then
+        log "ERROR: ZFS pool $ZFS_POOL is not available"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ ZFS pool not available: $ZFS_POOL
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The 20TB USB drive may not be connected or the ZFS pool is not imported.
+Please check the physical connection and run: sudo zpool import $ZFS_POOL"
+        return 1
+    fi
+
+    # Check if the specific ZFS dataset exists
+    if ! zfs list "$ZFS_DATASET" > /dev/null 2>&1; then
+        log "ERROR: ZFS dataset $ZFS_DATASET is not available"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ ZFS dataset not available: $ZFS_DATASET
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The ZFS dataset may not exist or be mounted.
+Please check: sudo zfs create $ZFS_DATASET"
+        return 1
+    fi
+    return 0
+}
+
+# Check if mount point is available
+check_mount_point() {
+    if ! mountpoint -q "$MOUNT_POINT"; then
+        log "ERROR: Mount point $MOUNT_POINT is not mounted"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ Mount point not available: $MOUNT_POINT
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The ZFS dataset may not be mounted.
+Please check: sudo zfs mount $ZFS_DATASET"
+        return 1
+    fi
+    return 0
+}
+
+# Check if source repository is available
+check_source_repo() {
+    if [ ! -d "$BORG_REPO_SOURCE" ]; then
+        log "ERROR: Source Borg repository not found: $BORG_REPO_SOURCE"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ Source repository not found: $BORG_REPO_SOURCE
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+JuiceFS may not be mounted or the source repository path is incorrect."
+        return 1
+    fi
+    return 0
+}
+
+# Check available space
+check_space() {
+    local source_size=$(sudo du -sb "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1)
+    local available_space=$(df -B1 "$MOUNT_POINT" | tail -1 | awk '{print $4}')
+
+    if [ -z "$source_size" ]; then
+        log "WARNING: Could not determine source repository size"
+        return 0
+    fi
+
+    # Add 20% buffer for safety
+    local required_space=$((source_size * 120 / 100))
+
+    if [ "$available_space" -lt "$required_space" ]; then
+        local source_gb=$((source_size / 1024 / 1024 / 1024))
+        local available_gb=$((available_space / 1024 / 1024 / 1024))
+        local required_gb=$((required_space / 1024 / 1024 / 1024))
+
+        log "ERROR: Insufficient space. Source: ${source_gb}GB, Available: ${available_gb}GB, Required: ${required_gb}GB"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ Insufficient disk space
+📊 Source size: ${source_gb}GB
+💾 Available: ${available_gb}GB
+⚠️ Required: ${required_gb}GB (with 20% buffer)
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Please free up space on the local backup drive."
+        return 1
+    fi
+
+    return 0
+}
+
+# Perform the sync
+sync_repository() {
+    log "Starting rsync of Borg repository"
+
+    # Get initial sizes for reporting
+    local source_size_before=$(sudo du -sh "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1)
+    local dest_size_before="0B"
+    if [ -d "$BORG_REPO_LOCAL" ]; then
+        dest_size_before=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1)
+    fi
+
+    # Perform the sync with detailed logging
+    sudo rsync -avh --delete --progress \
+        --exclude="lock.exclusive" \
+        --exclude="lock.roster" \
+        "$BORG_REPO_SOURCE/" "$BORG_REPO_LOCAL/" 2>&1 | while read line; do
+        log "rsync: $line"
+    done
+
+    local rsync_exit=${PIPESTATUS[0]}
+
+    # Get final sizes for reporting
+    local dest_size_after=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1)
+
+    if [ $rsync_exit -eq 0 ]; then
+        log "Rsync completed successfully"
+        send_telegram "🔒 <b>Borg Local Sync Success</b>
+
+✅ Local backup sync completed successfully
+📂 Source: $BORG_REPO_SOURCE (${source_size_before})
+💾 Destination: $BORG_REPO_LOCAL (${dest_size_after})
+🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Local backup copy is now up to date." "true"
+        return 0
+    else
+        log "Rsync failed with exit code: $rsync_exit"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ Rsync failed during repository sync
+📂 Source: $BORG_REPO_SOURCE
+💾 Destination: $BORG_REPO_LOCAL
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Exit code: $rsync_exit
+Check logs: /var/log/borg-local-sync.log"
+        return 1
+    fi
+}
+
+# Main execution
+log "Starting Borg local sync process"
+
+# Run all pre-flight checks
+if ! check_zfs_pool; then
+    exit 1
+fi
+
+if ! check_mount_point; then
+    exit 1
+fi
+
+if ! check_source_repo; then
+    exit 1
+fi
+
+if ! check_space; then
+    exit 1
+fi
+
+# All checks passed, proceed with sync
+log "All pre-flight checks passed, starting sync"
+
+if sync_repository; then
+    log "Local sync completed successfully"
+    exit 0
+else
+    log "Local sync failed"
+    exit 1
+fi

config/ansible/templates/borg-local-sync.timer.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Run Borg Local Sync daily
+Documentation=man:borg(1)
+Requires=borg-local-sync.service
+
+[Timer]
+# Run daily at 3:00 AM (1 hour after main backup at 2:00 AM)
+OnCalendar=*-*-* 03:00:00
+# Add randomization to prevent conflicts if multiple systems exist
+RandomizedDelaySec=300
+# Ensure timer persists across reboots
+Persistent=true
+# Wake system from suspend if needed
+WakeSystem=false
+
+[Install]
+WantedBy=timers.target

config/nextcloud.cfg

@@ -35,24 +35,24 @@ useNewBigFolderSizeLimit=true
 0\Folders\2\version=2
 0\Folders\2\virtualFilesMode=off
 0\Folders\3\ignoreHiddenFiles=false
-0\Folders\3\journalPath=.sync_65289e64a490.db
+0\Folders\3\journalPath=.sync_886cca272fe5.db
-0\Folders\3\localPath=/home/menno/Documents/
+0\Folders\3\localPath=/home/menno/Pictures/
 0\Folders\3\paused=false
-0\Folders\3\targetPath=/Documents
+0\Folders\3\targetPath=/Pictures
 0\Folders\3\version=2
 0\Folders\3\virtualFilesMode=off
 0\Folders\4\ignoreHiddenFiles=false
-0\Folders\4\journalPath=.sync_886cca272fe5.db
+0\Folders\4\journalPath=.sync_90ea5e3c7a33.db
-0\Folders\4\localPath=/home/menno/Pictures/
+0\Folders\4\localPath=/home/menno/Videos/
 0\Folders\4\paused=false
-0\Folders\4\targetPath=/Pictures
+0\Folders\4\targetPath=/Videos
 0\Folders\4\version=2
 0\Folders\4\virtualFilesMode=off
 0\Folders\5\ignoreHiddenFiles=false
-0\Folders\5\journalPath=.sync_90ea5e3c7a33.db
+0\Folders\5\journalPath=.sync_65289e64a490.db
-0\Folders\5\localPath=/home/menno/Videos/
+0\Folders\5\localPath=/home/menno/Documents/
 0\Folders\5\paused=false
-0\Folders\5\targetPath=/Videos
+0\Folders\5\targetPath=/Documents
 0\Folders\5\version=2
 0\Folders\5\virtualFilesMode=off
 0\Folders\6\ignoreHiddenFiles=false
@@ -92,4 +92,4 @@ useDownloadLimit=0
 useUploadLimit=0
 
 [Settings]
-geometry=@ByteArray(\x1\xd9\xd0\xcb\0\x3\0\0\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a\0\0\0\x1\0\0\0\0\x14\0\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a)
+geometry=@ByteArray(\x1\xd9\xd0\xcb\0\x3\0\0\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj\0\0\0\x1\0\0\0\0\x14\0\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj)