refactor

2025-09-23 13:19:48 +00:00
parent a04a4abef6
commit dd3753fab4
170 changed files with 907 additions and 1715 deletions
--- a/ansible/templates/borg-backup.service.j2
+++ b/ansible/templates/borg-backup.service.j2
@@ -0,0 +1,31 @@
+[Unit]
+Description=Borg Backup Service
+After=network.target
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+ExecStart={{ borg_config_dir }}/backup.sh
+StandardOutput=journal
+StandardError=journal
+Environment="BORG_PASSPHRASE={{ borg_passphrase }}"
+Environment="BORG_REPO={{ borg_repo_dir }}"
+Environment="BORG_CACHE_DIR={{ borg_config_dir }}/cache"
+Environment="BORG_CONFIG_DIR={{ borg_config_dir }}/config"
+Environment="BORG_SECURITY_DIR={{ borg_config_dir }}/security"
+Environment="BORG_KEYS_DIR={{ borg_config_dir }}/keys"
+
+# Security settings
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ReadWritePaths=/mnt/services /mnt/object_storage /var/log {{ borg_config_dir }}
+ProtectHome=read-only
+ProtectControlGroups=true
+RestrictRealtime=true
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+
+[Install]
+WantedBy=multi-user.target
--- a/ansible/templates/borg-backup.sh.j2
+++ b/ansible/templates/borg-backup.sh.j2
@@ -0,0 +1,157 @@
+#!/bin/bash
+
+# Borg backup script for /mnt/services
+# This script creates incremental backups of the services directory
+
+# Set environment variables
+export BORG_REPO="{{ borg_repo_dir }}"
+export BORG_PASSPHRASE="{{ borg_passphrase }}"
+export BORG_CACHE_DIR="{{ borg_config_dir }}/cache"
+export BORG_CONFIG_DIR="{{ borg_config_dir }}/config"
+export BORG_SECURITY_DIR="{{ borg_config_dir }}/security"
+export BORG_KEYS_DIR="{{ borg_config_dir }}/keys"
+
+# Telegram notification variables
+export TELEGRAM_BOT_TOKEN="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='password') }}"
+export TELEGRAM_CHAT_ID="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='chat_id') }}"
+
+# Backup name with timestamp
+BACKUP_NAME="services-$(date +%Y%m%d-%H%M%S)"
+
+# Log function
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a /var/log/borg-backup.log
+}
+
+# Telegram notification function
+send_telegram() {
+    local message="$1"
+    local silent="${2:-false}"
+
+    if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then
+        log "Telegram credentials not configured, skipping notification"
+        return
+    fi
+
+    local payload=$(cat <<EOF
+{
+    "chat_id": "$TELEGRAM_CHAT_ID",
+    "text": "$message",
+    "parse_mode": "HTML",
+    "disable_notification": $silent
+}
+EOF
+)
+
+    curl -s -X POST \
+        -H "Content-Type: application/json" \
+        -d "$payload" \
+        "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" > /dev/null 2>&1
+
+    if [ $? -eq 0 ]; then
+        log "Telegram notification sent successfully"
+    else
+        log "Failed to send Telegram notification"
+    fi
+}
+
+# Ensure all Borg directories exist
+mkdir -p "$BORG_CACHE_DIR"
+mkdir -p "$BORG_CONFIG_DIR"
+mkdir -p "$BORG_SECURITY_DIR"
+mkdir -p "$BORG_KEYS_DIR"
+
+# Start backup
+log "Starting Borg backup: $BACKUP_NAME"
+
+# Create backup
+borg create \
+    --verbose \
+    --filter AME \
+    --list \
+    --stats \
+    --show-rc \
+    --compression lz4 \
+    --exclude-caches \
+    --exclude '*.tmp' \
+    --exclude '*.temp' \
+    --exclude '*.log' \
+    --exclude '*/.cache' \
+    --exclude '*/cache' \
+    --exclude '*/logs' \
+    --exclude '*/tmp' \
+    --exclude '*/node_modules' \
+    --exclude '*/__pycache__' \
+    "::$BACKUP_NAME" \
+    {{ borg_backup_dir }}
+
+backup_exit=$?
+
+log "Backup finished with exit code: $backup_exit"
+
+# Prune old backups (keep last 7 daily, 4 weekly, 6 monthly)
+log "Pruning old backups"
+
+# Check if there are any archives to prune first
+archive_count=$(borg list --short --prefix 'services-' 2>/dev/null | wc -l)
+
+if [ "$archive_count" -gt 1 ]; then
+    borg prune \
+        --list \
+        --prefix 'services-' \
+        --show-rc \
+        --keep-daily 7 \
+        --keep-weekly 4 \
+        --keep-monthly 6
+    prune_exit=$?
+else
+    log "Only one or no archives found, skipping prune"
+    prune_exit=0
+fi
+
+log "Prune finished with exit code: $prune_exit"
+
+# Compact repository
+log "Compacting repository"
+borg compact
+
+compact_exit=$?
+
+log "Compact finished with exit code: $compact_exit"
+
+# Global exit status
+global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit ))
+global_exit=$(( compact_exit > global_exit ? compact_exit : global_exit ))
+
+if [ $global_exit -eq 0 ]; then
+    log "Backup completed successfully"
+    send_telegram "🔒 <b>Borg Backup Success</b>
+
+✅ Backup: $BACKUP_NAME completed successfully
+📊 Repository: {{ borg_repo_dir }}
+🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S')
+
+All operations completed without errors." "true"
+elif [ $global_exit -eq 1 ]; then
+    log "Backup completed with warnings (exit code: $global_exit)"
+    send_telegram "⚠️ <b>Borg Backup Warning</b>
+
+⚠️ Backup: $BACKUP_NAME completed with warnings
+📊 Repository: {{ borg_repo_dir }}
+🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Exit code: $global_exit
+Check logs for details: /var/log/borg-backup.log"
+else
+    log "Backup completed with warnings or errors (exit code: $global_exit)"
+    send_telegram "❌ <b>Borg Backup Failed</b>
+
+❌ Backup: $BACKUP_NAME failed
+📊 Repository: {{ borg_repo_dir }}
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Exit code: $global_exit
+Check logs immediately: /var/log/borg-backup.log"
+fi
+
+exit $global_exit
--- a/ansible/templates/borg-backup.timer.j2
+++ b/ansible/templates/borg-backup.timer.j2
@@ -0,0 +1,12 @@
+[Unit]
+Description=Run Borg Backup Daily
+Requires=borg-backup.service
+
+[Timer]
+# Run daily at 2 AM
+OnCalendar=daily
+Persistent=true
+RandomizedDelaySec=1800
+
+[Install]
+WantedBy=timers.target
--- a/ansible/templates/borg-local-sync.service.j2
+++ b/ansible/templates/borg-local-sync.service.j2
@@ -0,0 +1,48 @@
+[Unit]
+Description=Borg Local Sync - Copy Borg repository to local storage
+Documentation=man:borg(1)
+After=network-online.target
+Wants=network-online.target
+# Ensure this runs after the main backup has completed
+After=borg-backup.service
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+
+# Set up environment
+Environment="PATH=/usr/local/bin:/usr/bin:/bin"
+Environment="LANG=en_US.UTF-8"
+Environment="LC_ALL=en_US.UTF-8"
+
+# Security settings
+ProtectSystem=strict
+ProtectHome=read-only
+ReadWritePaths=/var/log /mnt/borg-backups {{ borg_config_dir }}
+PrivateTmp=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+
+# Resource limits
+MemoryMax=2G
+CPUQuota=80%
+IOWeight=200
+
+# Timeout settings (local sync might take a while for initial copy)
+TimeoutStartSec=3600
+TimeoutStopSec=300
+
+# Execute the sync script
+ExecStart=/usr/local/bin/borg-local-sync.sh
+
+# Logging
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=borg-local-sync
+
+[Install]
+WantedBy=multi-user.target
--- a/ansible/templates/borg-local-sync.sh.j2
+++ b/ansible/templates/borg-local-sync.sh.j2
@@ -0,0 +1,227 @@
+#!/bin/bash
+
+# Borg local sync script for creating local copies of cloud backups
+# This script syncs the Borg repository from JuiceFS/S3 to local ZFS storage
+
+# Set environment variables
+export BORG_REPO_SOURCE="{{ borg_repo_dir }}"
+export BORG_REPO_LOCAL="/mnt/borg-backups"
+export ZFS_POOL="datapool"
+export ZFS_DATASET="datapool/borg-backups"
+export MOUNT_POINT="/mnt/borg-backups"
+
+# Telegram notification variables
+export TELEGRAM_BOT_TOKEN="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='password') }}"
+export TELEGRAM_CHAT_ID="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='chat_id') }}"
+
+# Log function
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a /var/log/borg-local-sync.log
+}
+
+# Telegram notification function
+send_telegram() {
+    local message="$1"
+    local silent="${2:-false}"
+
+    if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then
+        log "Telegram credentials not configured, skipping notification"
+        return
+    fi
+
+    local payload=$(cat <<EOF
+{
+    "chat_id": "$TELEGRAM_CHAT_ID",
+    "text": "$message",
+    "parse_mode": "HTML",
+    "disable_notification": $silent
+}
+EOF
+)
+
+    curl -s -X POST \
+        -H "Content-Type: application/json" \
+        -d "$payload" \
+        "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" > /dev/null 2>&1
+
+    if [ $? -eq 0 ]; then
+        log "Telegram notification sent successfully"
+    else
+        log "Failed to send Telegram notification"
+    fi
+}
+
+# Check if ZFS pool is available
+check_zfs_pool() {
+    if ! zpool status "$ZFS_POOL" > /dev/null 2>&1; then
+        log "ERROR: ZFS pool $ZFS_POOL is not available"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ ZFS pool not available: $ZFS_POOL
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The 20TB USB drive may not be connected or the ZFS pool is not imported.
+Please check the physical connection and run: sudo zpool import $ZFS_POOL"
+        return 1
+    fi
+
+    # Check if the specific ZFS dataset exists
+    if ! zfs list "$ZFS_DATASET" > /dev/null 2>&1; then
+        log "ERROR: ZFS dataset $ZFS_DATASET is not available"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ ZFS dataset not available: $ZFS_DATASET
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The ZFS dataset may not exist or be mounted.
+Please check: sudo zfs create $ZFS_DATASET"
+        return 1
+    fi
+    return 0
+}
+
+# Check if mount point is available
+check_mount_point() {
+    if ! mountpoint -q "$MOUNT_POINT"; then
+        log "ERROR: Mount point $MOUNT_POINT is not mounted"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ Mount point not available: $MOUNT_POINT
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+The ZFS dataset may not be mounted.
+Please check: sudo zfs mount $ZFS_DATASET"
+        return 1
+    fi
+    return 0
+}
+
+# Check if source repository is available
+check_source_repo() {
+    if [ ! -d "$BORG_REPO_SOURCE" ]; then
+        log "ERROR: Source Borg repository not found: $BORG_REPO_SOURCE"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ Source repository not found: $BORG_REPO_SOURCE
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+JuiceFS may not be mounted or the source repository path is incorrect."
+        return 1
+    fi
+    return 0
+}
+
+# Check available space
+check_space() {
+    local source_size=$(sudo du -sb "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1)
+    local available_space=$(df -B1 "$MOUNT_POINT" | tail -1 | awk '{print $4}')
+
+    if [ -z "$source_size" ]; then
+        log "WARNING: Could not determine source repository size"
+        return 0
+    fi
+
+    # Add 20% buffer for safety
+    local required_space=$((source_size * 120 / 100))
+
+    if [ "$available_space" -lt "$required_space" ]; then
+        local source_gb=$((source_size / 1024 / 1024 / 1024))
+        local available_gb=$((available_space / 1024 / 1024 / 1024))
+        local required_gb=$((required_space / 1024 / 1024 / 1024))
+
+        log "ERROR: Insufficient space. Source: ${source_gb}GB, Available: ${available_gb}GB, Required: ${required_gb}GB"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ Insufficient disk space
+📊 Source size: ${source_gb}GB
+💾 Available: ${available_gb}GB
+⚠️ Required: ${required_gb}GB (with 20% buffer)
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Please free up space on the local backup drive."
+        return 1
+    fi
+
+    return 0
+}
+
+# Perform the sync
+sync_repository() {
+    log "Starting rsync of Borg repository"
+
+    # Get initial sizes for reporting
+    local source_size_before=$(sudo du -sh "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1)
+    local dest_size_before="0B"
+    if [ -d "$BORG_REPO_LOCAL" ]; then
+        dest_size_before=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1)
+    fi
+
+    # Perform the sync with detailed logging
+    sudo rsync -avh --delete --progress \
+        --exclude="lock.exclusive" \
+        --exclude="lock.roster" \
+        "$BORG_REPO_SOURCE/" "$BORG_REPO_LOCAL/" 2>&1 | while read line; do
+        log "rsync: $line"
+    done
+
+    local rsync_exit=${PIPESTATUS[0]}
+
+    # Get final sizes for reporting
+    local dest_size_after=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1)
+
+    if [ $rsync_exit -eq 0 ]; then
+        log "Rsync completed successfully"
+        send_telegram "🔒 <b>Borg Local Sync Success</b>
+
+✅ Local backup sync completed successfully
+📂 Source: $BORG_REPO_SOURCE (${source_size_before})
+💾 Destination: $BORG_REPO_LOCAL (${dest_size_after})
+🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Local backup copy is now up to date." "true"
+        return 0
+    else
+        log "Rsync failed with exit code: $rsync_exit"
+        send_telegram "❌ <b>Borg Local Sync Failed</b>
+
+❌ Rsync failed during repository sync
+📂 Source: $BORG_REPO_SOURCE
+💾 Destination: $BORG_REPO_LOCAL
+🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
+
+Exit code: $rsync_exit
+Check logs: /var/log/borg-local-sync.log"
+        return 1
+    fi
+}
+
+# Main execution
+log "Starting Borg local sync process"
+
+# Run all pre-flight checks
+if ! check_zfs_pool; then
+    exit 1
+fi
+
+if ! check_mount_point; then
+    exit 1
+fi
+
+if ! check_source_repo; then
+    exit 1
+fi
+
+if ! check_space; then
+    exit 1
+fi
+
+# All checks passed, proceed with sync
+log "All pre-flight checks passed, starting sync"
+
+if sync_repository; then
+    log "Local sync completed successfully"
+    exit 0
+else
+    log "Local sync failed"
+    exit 1
+fi
--- a/ansible/templates/borg-local-sync.timer.j2
+++ b/ansible/templates/borg-local-sync.timer.j2
@@ -0,0 +1,17 @@
+[Unit]
+Description=Run Borg Local Sync daily
+Documentation=man:borg(1)
+Requires=borg-local-sync.service
+
+[Timer]
+# Run daily at 3:00 AM (1 hour after main backup at 2:00 AM)
+OnCalendar=*-*-* 03:00:00
+# Add randomization to prevent conflicts if multiple systems exist
+RandomizedDelaySec=300
+# Ensure timer persists across reboots
+Persistent=true
+# Wake system from suspend if needed
+WakeSystem=false
+
+[Install]
+WantedBy=timers.target
--- a/ansible/templates/dynamic-dns-systemd.env.j2
+++ b/ansible/templates/dynamic-dns-systemd.env.j2
@@ -0,0 +1,12 @@
+# Dynamic DNS Environment Configuration for SystemD
+# This file contains sensitive credentials and should be kept secure
+# Credentials are automatically retrieved from OnePassword
+
+# CloudFlare API Token (required)
+# Retrieved from OnePassword: CloudFlare API Token
+CLOUDFLARE_API_TOKEN={{ lookup('community.general.onepassword', 'CloudFlare API Token', vault='Dotfiles', field='password') }}
+
+# Telegram Bot Credentials (for notifications when IP changes)
+# Retrieved from OnePassword: Telegram DynDNS Bot
+TELEGRAM_BOT_TOKEN={{ lookup('community.general.onepassword', 'Telegram DynDNS Bot', vault='Dotfiles', field='password') }}
+TELEGRAM_CHAT_ID={{ lookup('community.general.onepassword', 'Telegram DynDNS Bot', vault='Dotfiles', field='chat_id') }}
--- a/ansible/templates/juicefs.service.j2
+++ b/ansible/templates/juicefs.service.j2
@@ -0,0 +1,24 @@
+[Unit]
+Description=JuiceFS
+After=network.target
+Before=docker.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/juicefs mount redis://:{{ redis_password }}@mennos-desktop:6379/0 /mnt/object_storage \
+  --cache-dir=/var/jfsCache \
+  --buffer-size=4096 \
+  --prefetch=16 \
+  --cache-size=131072 \
+  --attr-cache=60 \
+  --entry-cache=60 \
+  --open-cache=60 \
+  -o writeback_cache \
+  --max-uploads=80 \
+  --max-deletes=80 \
+  --writeback \
+  --upload-delay=30s
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
--- a/ansible/templates/sshd_config.j2
+++ b/ansible/templates/sshd_config.j2
@@ -0,0 +1,130 @@
+# This is the sshd server system-wide configuration file.  See
+# sshd_config(5) for more information.
+
+# This sshd was compiled with PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games
+
+# The strategy used for options in the default sshd_config shipped with
+# OpenSSH is to specify options with their default value where
+# possible, but leave them commented.  Uncommented options override the
+# default value.
+
+Include /etc/ssh/sshd_config.d/*.conf
+
+# When systemd socket activation is used (the default), the socket
+# configuration must be re-generated after changing Port, AddressFamily, or
+# ListenAddress.
+#
+# For changes to take effect, run:
+#
+#   systemctl daemon-reload
+#   systemctl restart ssh.socket
+#
+Port 400
+AddressFamily any
+#ListenAddress 0.0.0.0
+#ListenAddress ::
+
+#HostKey /etc/ssh/ssh_host_rsa_key
+#HostKey /etc/ssh/ssh_host_ecdsa_key
+#HostKey /etc/ssh/ssh_host_ed25519_key
+
+# Ciphers and keying
+#RekeyLimit default none
+
+# Logging
+#SyslogFacility AUTH
+LogLevel INFO
+
+# Authentication:
+
+#LoginGraceTime 2m
+PermitRootLogin no
+#StrictModes yes
+#MaxAuthTries 6
+#MaxSessions 10
+
+PubkeyAuthentication yes
+
+# Expect .ssh/authorized_keys2 to be disregarded by default in future.
+#AuthorizedKeysFile	.ssh/authorized_keys .ssh/authorized_keys2
+
+#AuthorizedPrincipalsFile none
+
+#AuthorizedKeysCommand none
+#AuthorizedKeysCommandUser nobody
+
+# For this to work you will also need host keys in /etc/ssh/ssh_known_hosts
+#HostbasedAuthentication no
+# Change to yes if you don't trust ~/.ssh/known_hosts for
+# HostbasedAuthentication
+#IgnoreUserKnownHosts no
+# Don't read the user's ~/.rhosts and ~/.shosts files
+#IgnoreRhosts yes
+
+# To disable tunneled clear text passwords, change to no here!
+PasswordAuthentication no
+#PermitEmptyPasswords no
+
+# Change to yes to enable challenge-response passwords (beware issues with
+# some PAM modules and threads)
+KbdInteractiveAuthentication no
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+#KerberosGetAFSToken no
+
+# GSSAPI options
+#GSSAPIAuthentication no
+#GSSAPICleanupCredentials yes
+#GSSAPIStrictAcceptorCheck yes
+#GSSAPIKeyExchange no
+
+# Set this to 'yes' to enable PAM authentication, account processing,
+# and session processing. If this is enabled, PAM authentication will
+# be allowed through the KbdInteractiveAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via KbdInteractiveAuthentication may bypass
+# the setting of "PermitRootLogin prohibit-password".
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and KbdInteractiveAuthentication to 'no'.
+UsePAM yes
+
+#AllowAgentForwarding yes
+#AllowTcpForwarding yes
+#GatewayPorts no
+X11Forwarding yes
+#X11DisplayOffset 10
+#X11UseLocalhost yes
+#PermitTTY yes
+PrintMotd no
+#PrintLastLog yes
+#TCPKeepAlive yes
+#PermitUserEnvironment no
+#Compression delayed
+ClientAliveInterval 300
+ClientAliveCountMax 2
+#UseDNS no
+#PidFile /run/sshd.pid
+#MaxStartups 10:30:100
+#PermitTunnel no
+#ChrootDirectory none
+#VersionAddendum none
+
+# no default banner path
+#Banner none
+
+# Allow client to pass locale environment variables
+AcceptEnv LANG LC_*
+
+# override default of no subsystems
+Subsystem	sftp	/usr/lib/openssh/sftp-server
+
+# Example of overriding settings on a per-user basis
+#Match User anoncvs
+X11Forwarding yes
+AllowTcpForwarding yes
+#	PermitTTY no
+#	ForceCommand cvs server