diff --git a/bin/actions/service.py b/bin/actions/service.py
index 12b244a..84b435e 100755
--- a/bin/actions/service.py
+++ b/bin/actions/service.py
@@ -325,7 +325,7 @@ def cmd_list(args):
print()
println("System services:", "blue")
- systemd_timers = ["borg-backup.timer", "dynamic-dns.timer"]
+ systemd_timers = ["borg-backup.timer", "borg-local-sync.timer", "dynamic-dns.timer"]
for timer in systemd_timers:
is_active, is_enabled, next_run = get_systemd_timer_status(timer)
diff --git a/bin/actions/timers.py b/bin/actions/timers.py
index 8547f3a..20fcf64 100755
--- a/bin/actions/timers.py
+++ b/bin/actions/timers.py
@@ -80,6 +80,7 @@ def main():
# Show timer statuses
timers = [
("borg-backup.timer", True),
+ ("borg-local-sync.timer", True),
("dynamic-dns.timer", True)
]
diff --git a/config/ansible/handlers/main.yml b/config/ansible/handlers/main.yml
index ea98ace..b9c1b9f 100644
--- a/config/ansible/handlers/main.yml
+++ b/config/ansible/handlers/main.yml
@@ -10,3 +10,21 @@
name: ssh
state: restarted
enabled: true
+
+- name: reload systemd
+ become: true
+ ansible.builtin.systemd:
+ daemon_reload: true
+
+- name: restart borg-local-sync
+ become: true
+ ansible.builtin.systemd:
+ name: borg-local-sync.service
+ enabled: true
+
+- name: restart borg-local-sync-timer
+ become: true
+ ansible.builtin.systemd:
+ name: borg-local-sync.timer
+ state: restarted
+ enabled: true
diff --git a/config/ansible/tasks/servers/borg-local-sync.yml b/config/ansible/tasks/servers/borg-local-sync.yml
new file mode 100644
index 0000000..e354bce
--- /dev/null
+++ b/config/ansible/tasks/servers/borg-local-sync.yml
@@ -0,0 +1,95 @@
+---
+- name: Borg Local Sync Installation and Configuration
+ block:
+ - name: Set Borg backup facts
+ ansible.builtin.set_fact:
+ borg_passphrase: "{{ lookup('community.general.onepassword', 'Borg Backup', vault='Dotfiles', field='password') }}"
+ borg_config_dir: "{{ ansible_env.HOME }}/.config/borg"
+ borg_backup_dir: "/mnt/services"
+ borg_repo_dir: "/mnt/object_storage/borg-repo"
+
+ - name: Create Borg local sync script
+ template:
+ src: borg-local-sync.sh.j2
+ dest: /usr/local/bin/borg-local-sync.sh
+ mode: "0755"
+ owner: root
+ group: root
+ become: yes
+ tags:
+ - borg-local-sync
+
+ - name: Create Borg local sync systemd service
+ template:
+ src: borg-local-sync.service.j2
+ dest: /etc/systemd/system/borg-local-sync.service
+ mode: "0644"
+ owner: root
+ group: root
+ become: yes
+ notify:
+ - reload systemd
+ tags:
+ - borg-local-sync
+
+ - name: Create Borg local sync systemd timer
+ template:
+ src: borg-local-sync.timer.j2
+ dest: /etc/systemd/system/borg-local-sync.timer
+ mode: "0644"
+ owner: root
+ group: root
+ become: yes
+ notify:
+ - reload systemd
+ - restart borg-local-sync-timer
+ tags:
+ - borg-local-sync
+
+ - name: Create log file for Borg local sync
+ file:
+ path: /var/log/borg-local-sync.log
+ state: touch
+ owner: root
+ group: root
+ mode: "0644"
+ become: yes
+ tags:
+ - borg-local-sync
+
+ - name: Enable and start Borg local sync timer
+ systemd:
+ name: borg-local-sync.timer
+ enabled: yes
+ state: started
+ daemon_reload: yes
+ become: yes
+ tags:
+ - borg-local-sync
+
+ - name: Add logrotate configuration for Borg local sync
+ copy:
+ content: |
+ /var/log/borg-local-sync.log {
+ daily
+ rotate 30
+ compress
+ delaycompress
+ missingok
+ notifempty
+ create 644 root root
+ }
+ dest: /etc/logrotate.d/borg-local-sync
+ mode: "0644"
+ owner: root
+ group: root
+ become: yes
+ tags:
+ - borg-local-sync
+ - borg
+ - backup
+
+ tags:
+ - borg-local-sync
+ - borg
+ - backup
diff --git a/config/ansible/tasks/servers/server.yml b/config/ansible/tasks/servers/server.yml
index 9bc814b..e287d15 100644
--- a/config/ansible/tasks/servers/server.yml
+++ b/config/ansible/tasks/servers/server.yml
@@ -35,6 +35,11 @@
tags:
- borg-backup
+ - name: Include Borg Local Sync tasks
+ ansible.builtin.include_tasks: borg-local-sync.yml
+ tags:
+ - borg-local-sync
+
- name: System performance optimizations
ansible.posix.sysctl:
name: "{{ item.name }}"
diff --git a/config/ansible/templates/borg-backup.sh.j2 b/config/ansible/templates/borg-backup.sh.j2
index 6147ada..3d3bb99 100644
--- a/config/ansible/templates/borg-backup.sh.j2
+++ b/config/ansible/templates/borg-backup.sh.j2
@@ -26,6 +26,7 @@ log() {
# Telegram notification function
send_telegram() {
local message="$1"
+ local silent="${2:-false}"
if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then
log "Telegram credentials not configured, skipping notification"
@@ -36,7 +37,8 @@ send_telegram() {
{
"chat_id": "$TELEGRAM_CHAT_ID",
"text": "$message",
- "parse_mode": "HTML"
+ "parse_mode": "HTML",
+ "disable_notification": $silent
}
EOF
)
@@ -129,7 +131,7 @@ if [ $global_exit -eq 0 ]; then
📊 Repository: {{ borg_repo_dir }}
🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S')
-All operations completed without errors."
+All operations completed without errors." "true"
elif [ $global_exit -eq 1 ]; then
log "Backup completed with warnings (exit code: $global_exit)"
send_telegram "⚠️ Borg Backup Warning
diff --git a/config/ansible/templates/borg-local-sync.service.j2 b/config/ansible/templates/borg-local-sync.service.j2
new file mode 100644
index 0000000..f31b5da
--- /dev/null
+++ b/config/ansible/templates/borg-local-sync.service.j2
@@ -0,0 +1,48 @@
+[Unit]
+Description=Borg Local Sync - Copy Borg repository to local storage
+Documentation=man:borg(1)
+After=network-online.target
+Wants=network-online.target
+# Ensure this runs after the main backup has completed
+After=borg-backup.service
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+
+# Set up environment
+Environment="PATH=/usr/local/bin:/usr/bin:/bin"
+Environment="LANG=en_US.UTF-8"
+Environment="LC_ALL=en_US.UTF-8"
+
+# Security settings
+ProtectSystem=strict
+ProtectHome=read-only
+ReadWritePaths=/var/log /mnt/borg-backups {{ borg_config_dir }}
+PrivateTmp=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+
+# Resource limits
+MemoryMax=2G
+CPUQuota=80%
+IOWeight=200
+
+# Timeout settings (local sync might take a while for initial copy)
+TimeoutStartSec=3600
+TimeoutStopSec=300
+
+# Execute the sync script
+ExecStart=/usr/local/bin/borg-local-sync.sh
+
+# Logging
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=borg-local-sync
+
+[Install]
+WantedBy=multi-user.target
diff --git a/config/ansible/templates/borg-local-sync.sh.j2 b/config/ansible/templates/borg-local-sync.sh.j2
new file mode 100644
index 0000000..785e171
--- /dev/null
+++ b/config/ansible/templates/borg-local-sync.sh.j2
@@ -0,0 +1,227 @@
+#!/bin/bash
+
+# Borg local sync script for creating local copies of cloud backups
+# This script syncs the Borg repository from JuiceFS/S3 to local ZFS storage
+
+# Set environment variables
+export BORG_REPO_SOURCE="{{ borg_repo_dir }}"
+export BORG_REPO_LOCAL="/mnt/borg-backups"
+export ZFS_POOL="datapool"
+export ZFS_DATASET="datapool/borg-backups"
+export MOUNT_POINT="/mnt/borg-backups"
+
+# Telegram notification variables
+export TELEGRAM_BOT_TOKEN="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='password') }}"
+export TELEGRAM_CHAT_ID="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='chat_id') }}"
+
+# Log function
+log() {
+ echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a /var/log/borg-local-sync.log
+}
+
+# Telegram notification function
+send_telegram() {
+ local message="$1"
+ local silent="${2:-false}"
+
+ if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then
+ log "Telegram credentials not configured, skipping notification"
+ return
+ fi
+
+ local payload=$(cat < /dev/null 2>&1
+
+ if [ $? -eq 0 ]; then
+ log "Telegram notification sent successfully"
+ else
+ log "Failed to send Telegram notification"
+ fi
+}
+
+# Check if ZFS pool is available
+check_zfs_pool() {
+ if ! zpool status "$ZFS_POOL" > /dev/null 2>&1; then
+ log "ERROR: ZFS pool $ZFS_POOL is not available"
+ send_telegram "❌ Borg Local Sync Failed
+
+❌ ZFS pool not available: $ZFS_POOL
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The 20TB USB drive may not be connected or the ZFS pool is not imported.
+Please check the physical connection and run: sudo zpool import $ZFS_POOL"
+ return 1
+ fi
+
+ # Check if the specific ZFS dataset exists
+ if ! zfs list "$ZFS_DATASET" > /dev/null 2>&1; then
+ log "ERROR: ZFS dataset $ZFS_DATASET is not available"
+ send_telegram "❌ Borg Local Sync Failed
+
+❌ ZFS dataset not available: $ZFS_DATASET
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The ZFS dataset may not exist or be mounted.
+Please check: sudo zfs create $ZFS_DATASET"
+ return 1
+ fi
+ return 0
+}
+
+# Check if mount point is available
+check_mount_point() {
+ if ! mountpoint -q "$MOUNT_POINT"; then
+ log "ERROR: Mount point $MOUNT_POINT is not mounted"
+ send_telegram "❌ Borg Local Sync Failed
+
+❌ Mount point not available: $MOUNT_POINT
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The ZFS dataset may not be mounted.
+Please check: sudo zfs mount $ZFS_DATASET"
+ return 1
+ fi
+ return 0
+}
+
+# Check if source repository is available
+check_source_repo() {
+ if [ ! -d "$BORG_REPO_SOURCE" ]; then
+ log "ERROR: Source Borg repository not found: $BORG_REPO_SOURCE"
+ send_telegram "❌ Borg Local Sync Failed
+
+❌ Source repository not found: $BORG_REPO_SOURCE
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+JuiceFS may not be mounted or the source repository path is incorrect."
+ return 1
+ fi
+ return 0
+}
+
+# Check available space
+check_space() {
+ local source_size=$(sudo du -sb "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1)
+ local available_space=$(df -B1 "$MOUNT_POINT" | tail -1 | awk '{print $4}')
+
+ if [ -z "$source_size" ]; then
+ log "WARNING: Could not determine source repository size"
+ return 0
+ fi
+
+ # Add 20% buffer for safety
+ local required_space=$((source_size * 120 / 100))
+
+ if [ "$available_space" -lt "$required_space" ]; then
+ local source_gb=$((source_size / 1024 / 1024 / 1024))
+ local available_gb=$((available_space / 1024 / 1024 / 1024))
+ local required_gb=$((required_space / 1024 / 1024 / 1024))
+
+ log "ERROR: Insufficient space. Source: ${source_gb}GB, Available: ${available_gb}GB, Required: ${required_gb}GB"
+ send_telegram "❌ Borg Local Sync Failed
+
+❌ Insufficient disk space
+📊 Source size: ${source_gb}GB
+💾 Available: ${available_gb}GB
+⚠️ Required: ${required_gb}GB (with 20% buffer)
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Please free up space on the local backup drive."
+ return 1
+ fi
+
+ return 0
+}
+
+# Perform the sync
+sync_repository() {
+ log "Starting rsync of Borg repository"
+
+ # Get initial sizes for reporting
+ local source_size_before=$(sudo du -sh "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1)
+ local dest_size_before="0B"
+ if [ -d "$BORG_REPO_LOCAL" ]; then
+ dest_size_before=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1)
+ fi
+
+ # Perform the sync with detailed logging
+ sudo rsync -avh --delete --progress \
+ --exclude="lock.exclusive" \
+ --exclude="lock.roster" \
+ "$BORG_REPO_SOURCE/" "$BORG_REPO_LOCAL/" 2>&1 | while read line; do
+ log "rsync: $line"
+ done
+
+ local rsync_exit=${PIPESTATUS[0]}
+
+ # Get final sizes for reporting
+ local dest_size_after=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1)
+
+ if [ $rsync_exit -eq 0 ]; then
+ log "Rsync completed successfully"
+ send_telegram "🔒 Borg Local Sync Success
+
+✅ Local backup sync completed successfully
+📂 Source: $BORG_REPO_SOURCE (${source_size_before})
+💾 Destination: $BORG_REPO_LOCAL (${dest_size_after})
+🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Local backup copy is now up to date." "true"
+ return 0
+ else
+ log "Rsync failed with exit code: $rsync_exit"
+ send_telegram "❌ Borg Local Sync Failed
+
+❌ Rsync failed during repository sync
+📂 Source: $BORG_REPO_SOURCE
+💾 Destination: $BORG_REPO_LOCAL
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Exit code: $rsync_exit
+Check logs: /var/log/borg-local-sync.log"
+ return 1
+ fi
+}
+
+# Main execution
+log "Starting Borg local sync process"
+
+# Run all pre-flight checks
+if ! check_zfs_pool; then
+ exit 1
+fi
+
+if ! check_mount_point; then
+ exit 1
+fi
+
+if ! check_source_repo; then
+ exit 1
+fi
+
+if ! check_space; then
+ exit 1
+fi
+
+# All checks passed, proceed with sync
+log "All pre-flight checks passed, starting sync"
+
+if sync_repository; then
+ log "Local sync completed successfully"
+ exit 0
+else
+ log "Local sync failed"
+ exit 1
+fi
diff --git a/config/ansible/templates/borg-local-sync.timer.j2 b/config/ansible/templates/borg-local-sync.timer.j2
new file mode 100644
index 0000000..acf54da
--- /dev/null
+++ b/config/ansible/templates/borg-local-sync.timer.j2
@@ -0,0 +1,17 @@
+[Unit]
+Description=Run Borg Local Sync daily
+Documentation=man:borg(1)
+Requires=borg-local-sync.service
+
+[Timer]
+# Run daily at 3:00 AM (1 hour after main backup at 2:00 AM)
+OnCalendar=*-*-* 03:00:00
+# Add randomization to prevent conflicts if multiple systems exist
+RandomizedDelaySec=300
+# Ensure timer persists across reboots
+Persistent=true
+# Wake system from suspend if needed
+WakeSystem=false
+
+[Install]
+WantedBy=timers.target
diff --git a/config/nextcloud.cfg b/config/nextcloud.cfg
index 5a3aa7e..a49db9c 100644
--- a/config/nextcloud.cfg
+++ b/config/nextcloud.cfg
@@ -35,24 +35,24 @@ useNewBigFolderSizeLimit=true
0\Folders\2\version=2
0\Folders\2\virtualFilesMode=off
0\Folders\3\ignoreHiddenFiles=false
-0\Folders\3\journalPath=.sync_65289e64a490.db
-0\Folders\3\localPath=/home/menno/Documents/
+0\Folders\3\journalPath=.sync_886cca272fe5.db
+0\Folders\3\localPath=/home/menno/Pictures/
0\Folders\3\paused=false
-0\Folders\3\targetPath=/Documents
+0\Folders\3\targetPath=/Pictures
0\Folders\3\version=2
0\Folders\3\virtualFilesMode=off
0\Folders\4\ignoreHiddenFiles=false
-0\Folders\4\journalPath=.sync_886cca272fe5.db
-0\Folders\4\localPath=/home/menno/Pictures/
+0\Folders\4\journalPath=.sync_90ea5e3c7a33.db
+0\Folders\4\localPath=/home/menno/Videos/
0\Folders\4\paused=false
-0\Folders\4\targetPath=/Pictures
+0\Folders\4\targetPath=/Videos
0\Folders\4\version=2
0\Folders\4\virtualFilesMode=off
0\Folders\5\ignoreHiddenFiles=false
-0\Folders\5\journalPath=.sync_90ea5e3c7a33.db
-0\Folders\5\localPath=/home/menno/Videos/
+0\Folders\5\journalPath=.sync_65289e64a490.db
+0\Folders\5\localPath=/home/menno/Documents/
0\Folders\5\paused=false
-0\Folders\5\targetPath=/Videos
+0\Folders\5\targetPath=/Documents
0\Folders\5\version=2
0\Folders\5\virtualFilesMode=off
0\Folders\6\ignoreHiddenFiles=false
@@ -92,4 +92,4 @@ useDownloadLimit=0
useUploadLimit=0
[Settings]
-geometry=@ByteArray(\x1\xd9\xd0\xcb\0\x3\0\0\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a\0\0\0\x1\0\0\0\0\x14\0\0\0\n\0\0\0\0\0\0\0\f7\0\0\x2\x8a)
+geometry=@ByteArray(\x1\xd9\xd0\xcb\0\x3\0\0\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj\0\0\0\x1\0\0\0\0\x14\0\0\0\0\0\0\0\x4\xe0\0\0\x2\x37\0\0\aj)