From a987adb86b612422f036072df8ae532f2b3183d1 Mon Sep 17 00:00:00 2001 From: Menno van Leeuwen Date: Thu, 11 Dec 2025 11:45:54 +0100 Subject: [PATCH] Move Borg repo to local backups and remove JuiceFS Use /mnt/borg-backups in place of /mnt/object_storage for Borg. Remove JuiceFS and Redis artifacts (tasks, templates, service configs) and delete borg-local-sync tooling. Update borg-backup service ReadWritePaths, remove Plex slow tvshows mount, add system sysctl performance tunings, and apply minor code and flake.lock updates. --- README.md | 26 -- ansible/tasks/servers/borg-backup.yml | 4 +- ansible/tasks/servers/borg-local-sync.yml | 95 ------ ansible/tasks/servers/juicefs.yml | 94 ------ ansible/tasks/servers/server.yml | 299 +++++++++--------- .../services/plex/docker-compose.yml.j2 | 1 - .../services/redis/docker-compose.yml.j2 | 26 -- .../tasks/servers/services/redis/redis.yml | 82 ----- ansible/templates/borg-backup.service.j2 | 2 +- ansible/templates/borg-local-sync.service.j2 | 48 --- ansible/templates/borg-local-sync.sh.j2 | 227 ------------- ansible/templates/borg-local-sync.timer.j2 | 17 - ansible/templates/juicefs.service.j2 | 24 -- bin/actions/service.py | 8 +- bin/actions/source.py | 4 +- flake.lock | 18 +- 16 files changed, 165 insertions(+), 810 deletions(-) delete mode 100644 ansible/tasks/servers/borg-local-sync.yml delete mode 100644 ansible/tasks/servers/juicefs.yml delete mode 100644 ansible/tasks/servers/services/redis/docker-compose.yml.j2 delete mode 100644 ansible/tasks/servers/services/redis/redis.yml delete mode 100644 ansible/templates/borg-local-sync.service.j2 delete mode 100644 ansible/templates/borg-local-sync.sh.j2 delete mode 100644 ansible/templates/borg-local-sync.timer.j2 delete mode 100644 ansible/templates/juicefs.service.j2 diff --git a/README.md b/README.md index 9f9d2f8..9e05b2c 100755 --- a/README.md +++ b/README.md @@ -57,37 +57,11 @@ If you add a new system you should add the relevant files to these paths. ### Server reboots -In case you reboot a server, it's likely that this runs JuiceFS. -To be sure that every service is properly accessing JuiceFS mounted files you should probably restart the services once when the server comes online. - ```bash dotf service stop --all -df # confirm JuiceFS is mounted dotf service start --all ``` -### Object Storage (Servers only) - -In case you need to adjust anything regarding the /mnt/object_storage JuiceFS. -Ensure to shut down all services: - -```bash -dotf service stop --all -``` - -Unmount the volume: - -```bash -sudo systemctl stop juicefs -``` - -And optionally if you're going to do something with metadata you might need to stop redis too. - -```bash -cd ~/services/juicefs-redis/ -docker compose down --remove-orphans -``` - ### Adding a new system To add a new system you should follow these steps: diff --git a/ansible/tasks/servers/borg-backup.yml b/ansible/tasks/servers/borg-backup.yml index 232a02c..65a4dcb 100644 --- a/ansible/tasks/servers/borg-backup.yml +++ b/ansible/tasks/servers/borg-backup.yml @@ -19,7 +19,7 @@ borg_passphrase: "{{ lookup('community.general.onepassword', 'Borg Backup', vault='Dotfiles', field='password') }}" borg_config_dir: "{{ ansible_env.HOME }}/.config/borg" borg_backup_dir: "/mnt/services" - borg_repo_dir: "/mnt/object_storage/borg-repo" + borg_repo_dir: "/mnt/borg-backups" - name: Create Borg directories ansible.builtin.file: @@ -28,7 +28,7 @@ mode: "0755" loop: - "{{ borg_config_dir }}" - - "/mnt/object_storage" + - "/mnt/borg-backups" loop_control: loop_var: borg_dir become: true diff --git a/ansible/tasks/servers/borg-local-sync.yml b/ansible/tasks/servers/borg-local-sync.yml deleted file mode 100644 index e354bce..0000000 --- a/ansible/tasks/servers/borg-local-sync.yml +++ /dev/null @@ -1,95 +0,0 @@ ---- -- name: Borg Local Sync Installation and Configuration - block: - - name: Set Borg backup facts - ansible.builtin.set_fact: - borg_passphrase: "{{ lookup('community.general.onepassword', 'Borg Backup', vault='Dotfiles', field='password') }}" - borg_config_dir: "{{ ansible_env.HOME }}/.config/borg" - borg_backup_dir: "/mnt/services" - borg_repo_dir: "/mnt/object_storage/borg-repo" - - - name: Create Borg local sync script - template: - src: borg-local-sync.sh.j2 - dest: /usr/local/bin/borg-local-sync.sh - mode: "0755" - owner: root - group: root - become: yes - tags: - - borg-local-sync - - - name: Create Borg local sync systemd service - template: - src: borg-local-sync.service.j2 - dest: /etc/systemd/system/borg-local-sync.service - mode: "0644" - owner: root - group: root - become: yes - notify: - - reload systemd - tags: - - borg-local-sync - - - name: Create Borg local sync systemd timer - template: - src: borg-local-sync.timer.j2 - dest: /etc/systemd/system/borg-local-sync.timer - mode: "0644" - owner: root - group: root - become: yes - notify: - - reload systemd - - restart borg-local-sync-timer - tags: - - borg-local-sync - - - name: Create log file for Borg local sync - file: - path: /var/log/borg-local-sync.log - state: touch - owner: root - group: root - mode: "0644" - become: yes - tags: - - borg-local-sync - - - name: Enable and start Borg local sync timer - systemd: - name: borg-local-sync.timer - enabled: yes - state: started - daemon_reload: yes - become: yes - tags: - - borg-local-sync - - - name: Add logrotate configuration for Borg local sync - copy: - content: | - /var/log/borg-local-sync.log { - daily - rotate 30 - compress - delaycompress - missingok - notifempty - create 644 root root - } - dest: /etc/logrotate.d/borg-local-sync - mode: "0644" - owner: root - group: root - become: yes - tags: - - borg-local-sync - - borg - - backup - - tags: - - borg-local-sync - - borg - - backup diff --git a/ansible/tasks/servers/juicefs.yml b/ansible/tasks/servers/juicefs.yml deleted file mode 100644 index aa69230..0000000 --- a/ansible/tasks/servers/juicefs.yml +++ /dev/null @@ -1,94 +0,0 @@ ---- -- name: JuiceFS Installation and Configuration - block: - - name: Check if JuiceFS is already installed - ansible.builtin.command: which juicefs - register: juicefs_check - ignore_errors: true - changed_when: false - - - name: Install JuiceFS using the automatic installer - ansible.builtin.shell: curl -sSL https://d.juicefs.com/install | sh - - register: juicefs_installation - when: juicefs_check.rc != 0 - become: true - - - name: Verify JuiceFS installation - ansible.builtin.command: juicefs version - register: juicefs_version - changed_when: false - when: juicefs_check.rc != 0 or juicefs_installation.changed - - - name: Create mount directory - ansible.builtin.file: - path: /mnt/object_storage - state: directory - mode: "0755" - become: true - - - name: Create cache directory - ansible.builtin.file: - path: /var/jfsCache - state: directory - mode: "0755" - become: true - - - name: Configure JuiceFS network performance optimizations - ansible.builtin.sysctl: - name: "{{ item.name }}" - value: "{{ item.value }}" - state: present - reload: true - become: true - loop: - - { name: "net.core.rmem_max", value: "16777216" } - - { name: "net.core.wmem_max", value: "16777216" } - - { name: "net.ipv4.tcp_rmem", value: "4096 87380 16777216" } - - { name: "net.ipv4.tcp_wmem", value: "4096 65536 16777216" } - - - name: Set JuiceFS facts - ansible.builtin.set_fact: - hetzner_access_key: "{{ lookup('community.general.onepassword', 'Hetzner Object Storage Bucket', vault='Dotfiles', field='AWS_ACCESS_KEY_ID') }}" - hetzner_secret_key: - "{{ lookup('community.general.onepassword', 'Hetzner Object Storage Bucket', vault='Dotfiles', field='AWS_SECRET_ACCESS_KEY') - }}" - redis_password: "{{ lookup('community.general.onepassword', 'JuiceFS (Redis)', vault='Dotfiles', field='password') }}" - - - name: Create JuiceFS systemd service file - ansible.builtin.template: - src: templates/juicefs.service.j2 - dest: /etc/systemd/system/juicefs.service - owner: root - group: root - mode: "0644" - become: true - - - name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: true - become: true - - - name: Include JuiceFS Redis tasks - ansible.builtin.include_tasks: services/redis/redis.yml - when: inventory_hostname == 'mennos-server' - - - name: Enable and start JuiceFS service - ansible.builtin.systemd: - name: juicefs.service - enabled: true - state: started - become: true - - - name: Check if JuiceFS is mounted - ansible.builtin.shell: df -h | grep /mnt/object_storage - become: true - register: mount_check - ignore_errors: true - changed_when: false - - - name: Display mount status - ansible.builtin.debug: - msg: "JuiceFS is successfully mounted at /mnt/object_storage" - when: mount_check.rc == 0 - tags: - - juicefs diff --git a/ansible/tasks/servers/server.yml b/ansible/tasks/servers/server.yml index ba7090f..1d09d51 100644 --- a/ansible/tasks/servers/server.yml +++ b/ansible/tasks/servers/server.yml @@ -1,165 +1,160 @@ --- - name: Server setup block: - - name: Ensure openssh-server is installed on Arch-based systems - ansible.builtin.package: - name: openssh - state: present - when: ansible_pkg_mgr == 'pacman' + - name: Ensure openssh-server is installed on Arch-based systems + ansible.builtin.package: + name: openssh + state: present + when: ansible_pkg_mgr == 'pacman' - - name: Ensure openssh-server is installed on non-Arch systems - ansible.builtin.package: - name: openssh-server - state: present - when: ansible_pkg_mgr != 'pacman' + - name: Ensure openssh-server is installed on non-Arch systems + ansible.builtin.package: + name: openssh-server + state: present + when: ansible_pkg_mgr != 'pacman' - - name: Ensure Borg is installed on Arch-based systems - ansible.builtin.package: - name: borg - state: present - become: true - when: ansible_pkg_mgr == 'pacman' + - name: Ensure Borg is installed on Arch-based systems + ansible.builtin.package: + name: borg + state: present + become: true + when: ansible_pkg_mgr == 'pacman' - - name: Ensure Borg is installed on Debian/Ubuntu systems - ansible.builtin.package: - name: borgbackup - state: present - become: true - when: ansible_pkg_mgr != 'pacman' + - name: Ensure Borg is installed on Debian/Ubuntu systems + ansible.builtin.package: + name: borgbackup + state: present + become: true + when: ansible_pkg_mgr != 'pacman' - - name: Include JuiceFS tasks - ansible.builtin.include_tasks: juicefs.yml - tags: - - juicefs + - name: Include Dynamic DNS tasks + ansible.builtin.include_tasks: dynamic-dns.yml + tags: + - dynamic-dns - - name: Include Dynamic DNS tasks - ansible.builtin.include_tasks: dynamic-dns.yml - tags: - - dynamic-dns + - name: Include Borg Backup tasks + ansible.builtin.include_tasks: borg-backup.yml + tags: + - borg-backup - - name: Include Borg Backup tasks - ansible.builtin.include_tasks: borg-backup.yml - tags: - - borg-backup + - name: Include Borg Local Sync tasks + ansible.builtin.include_tasks: borg-local-sync.yml + tags: + - borg-local-sync - - name: Include Borg Local Sync tasks - ansible.builtin.include_tasks: borg-local-sync.yml - tags: - - borg-local-sync + - name: System performance optimizations + ansible.posix.sysctl: + name: "{{ item.name }}" + value: "{{ item.value }}" + state: present + reload: true + become: true + loop: + - { name: "fs.file-max", value: "2097152" } # Max open files for the entire system + - { name: "vm.max_map_count", value: "16777216" } # Max memory map areas a process can have + - { name: "vm.swappiness", value: "10" } # Controls how aggressively the kernel swaps out memory + - { name: "vm.vfs_cache_pressure", value: "50" } # Controls kernel's tendency to reclaim memory for directory/inode caches + - { name: "net.core.somaxconn", value: "65535" } # Max pending connections for a listening socket + - { name: "net.core.netdev_max_backlog", value: "65535" } # Max packets queued on network interface input + - { name: "net.ipv4.tcp_fin_timeout", value: "30" } # How long sockets stay in FIN-WAIT-2 state + - { name: "net.ipv4.tcp_tw_reuse", value: "1" } # Allows reusing TIME_WAIT sockets for new outgoing connections - - name: System performance optimizations - ansible.posix.sysctl: - name: "{{ item.name }}" - value: "{{ item.value }}" - state: present - reload: true - become: true - loop: - - { name: "fs.file-max", value: "2097152" } # Max open files for the entire system - - { name: "vm.max_map_count", value: "16777216" } # Max memory map areas a process can have - - { name: "vm.swappiness", value: "10" } # Controls how aggressively the kernel swaps out memory - - { name: "vm.vfs_cache_pressure", value: "50" } # Controls kernel's tendency to reclaim memory for directory/inode caches - - { name: "net.core.somaxconn", value: "65535" } # Max pending connections for a listening socket - - { name: "net.core.netdev_max_backlog", value: "65535" } # Max packets queued on network interface input - - { name: "net.ipv4.tcp_fin_timeout", value: "30" } # How long sockets stay in FIN-WAIT-2 state - - { name: "net.ipv4.tcp_tw_reuse", value: "1" } # Allows reusing TIME_WAIT sockets for new outgoing connections - - - name: Include service tasks - ansible.builtin.include_tasks: "services/{{ item.name }}/{{ item.name }}.yml" - loop: "{{ services | selectattr('enabled', 'equalto', true) | selectattr('hosts', 'contains', inventory_hostname) | list if specific_service is not defined else services | selectattr('name', 'equalto', specific_service) | selectattr('enabled', 'equalto', true) | selectattr('hosts', 'contains', inventory_hostname) | list }}" - loop_control: - label: "{{ item.name }}" - tags: - - services - - always + - name: Include service tasks + ansible.builtin.include_tasks: "services/{{ item.name }}/{{ item.name }}.yml" + loop: "{{ services | selectattr('enabled', 'equalto', true) | selectattr('hosts', 'contains', inventory_hostname) | list if specific_service is not defined else services | selectattr('name', 'equalto', specific_service) | selectattr('enabled', 'equalto', true) | selectattr('hosts', 'contains', inventory_hostname) | list }}" + loop_control: + label: "{{ item.name }}" + tags: + - services + - always vars: - services: - - name: dashy - enabled: true - hosts: - - mennos-server - - name: gitea - enabled: true - hosts: - - mennos-server - - name: factorio - enabled: true - hosts: - - mennos-server - - name: dozzle - enabled: true - hosts: - - mennos-server - - name: beszel - enabled: true - hosts: - - mennos-server - - name: caddy - enabled: true - hosts: - - mennos-server - - name: golink - enabled: true - hosts: - - mennos-server - - name: immich - enabled: true - hosts: - - mennos-server - - name: plex - enabled: true - hosts: - - mennos-server - - name: tautulli - enabled: true - hosts: - - mennos-server - - name: downloaders - enabled: true - hosts: - - mennos-server - - name: wireguard - enabled: true - hosts: - - mennos-server - - name: nextcloud - enabled: true - hosts: - - mennos-server - - name: cloudreve - enabled: true - hosts: - - mennos-server - - name: echoip - enabled: true - hosts: - - mennos-server - - name: arr-stack - enabled: true - hosts: - - mennos-server - - name: home-assistant - enabled: true - hosts: - - mennos-server - - name: privatebin - enabled: true - hosts: - - mennos-server - - name: unifi-network-application - enabled: true - hosts: - - mennos-server - - name: avorion - enabled: false - hosts: - - mennos-server - - name: sathub - enabled: true - hosts: - - mennos-server - - name: necesse - enabled: true - hosts: - - mennos-server + services: + - name: dashy + enabled: true + hosts: + - mennos-server + - name: gitea + enabled: true + hosts: + - mennos-server + - name: factorio + enabled: true + hosts: + - mennos-server + - name: dozzle + enabled: true + hosts: + - mennos-server + - name: beszel + enabled: true + hosts: + - mennos-server + - name: caddy + enabled: true + hosts: + - mennos-server + - name: golink + enabled: true + hosts: + - mennos-server + - name: immich + enabled: true + hosts: + - mennos-server + - name: plex + enabled: true + hosts: + - mennos-server + - name: tautulli + enabled: true + hosts: + - mennos-server + - name: downloaders + enabled: true + hosts: + - mennos-server + - name: wireguard + enabled: true + hosts: + - mennos-server + - name: nextcloud + enabled: true + hosts: + - mennos-server + - name: cloudreve + enabled: true + hosts: + - mennos-server + - name: echoip + enabled: true + hosts: + - mennos-server + - name: arr-stack + enabled: true + hosts: + - mennos-server + - name: home-assistant + enabled: true + hosts: + - mennos-server + - name: privatebin + enabled: true + hosts: + - mennos-server + - name: unifi-network-application + enabled: true + hosts: + - mennos-server + - name: avorion + enabled: false + hosts: + - mennos-server + - name: sathub + enabled: true + hosts: + - mennos-server + - name: necesse + enabled: true + hosts: + - mennos-server diff --git a/ansible/tasks/servers/services/plex/docker-compose.yml.j2 b/ansible/tasks/servers/services/plex/docker-compose.yml.j2 index c67c5cf..14bad20 100644 --- a/ansible/tasks/servers/services/plex/docker-compose.yml.j2 +++ b/ansible/tasks/servers/services/plex/docker-compose.yml.j2 @@ -16,7 +16,6 @@ services: - {{ plex_data_dir }}/transcode:/transcode - /mnt/data/movies:/movies - /mnt/data/tvshows:/tvshows - - /mnt/object_storage/tvshows:/tvshows_slow - /mnt/data/music:/music deploy: resources: diff --git a/ansible/tasks/servers/services/redis/docker-compose.yml.j2 b/ansible/tasks/servers/services/redis/docker-compose.yml.j2 deleted file mode 100644 index 299a85e..0000000 --- a/ansible/tasks/servers/services/redis/docker-compose.yml.j2 +++ /dev/null @@ -1,26 +0,0 @@ -services: - juicefs-redis: - image: redis:latest - restart: always - ports: - - "6379:6379" - volumes: - - /mnt/services/redis:/data - command: ["redis-server", "--appendonly", "yes", "--requirepass", "{{ REDIS_PASSWORD }}"] - environment: - - TZ=Europe/Amsterdam - healthcheck: - test: ["CMD", "redis-cli", "-a", "{{ REDIS_PASSWORD }}", "ping"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 5s - networks: - - juicefs-network - deploy: - resources: - limits: - memory: 256M - -networks: - juicefs-network: diff --git a/ansible/tasks/servers/services/redis/redis.yml b/ansible/tasks/servers/services/redis/redis.yml deleted file mode 100644 index 3f043f3..0000000 --- a/ansible/tasks/servers/services/redis/redis.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- -- name: Deploy Redis for JuiceFS - block: - - name: Set Redis facts - ansible.builtin.set_fact: - redis_service_dir: "{{ ansible_env.HOME }}/.services/juicefs-redis" - redis_password: "{{ lookup('community.general.onepassword', 'JuiceFS (Redis)', vault='Dotfiles', field='password') }}" - - - name: Create Redis service directory - ansible.builtin.file: - path: "{{ redis_service_dir }}" - state: directory - mode: "0755" - - - name: Deploy Redis docker-compose.yml - ansible.builtin.template: - src: docker-compose.yml.j2 - dest: "{{ redis_service_dir }}/docker-compose.yml" - mode: "0644" - register: redis_compose - vars: - REDIS_PASSWORD: "{{ redis_password }}" - - - name: Check if juicefs.service exists - ansible.builtin.stat: - path: /etc/systemd/system/juicefs.service - register: juicefs_service_stat - - - name: Stop juicefs.service to umount JuiceFS - ansible.builtin.systemd: - name: juicefs.service - state: stopped - enabled: false - register: juicefs_stop - changed_when: juicefs_stop.changed - when: redis_compose.changed and juicefs_service_stat.stat.exists - become: true - - - name: List containers that are running - ansible.builtin.command: docker ps -q - register: docker_ps - changed_when: docker_ps.rc == 0 - when: redis_compose.changed - - - name: Stop all docker containers - ansible.builtin.command: docker stop {{ item }} - loop: "{{ docker_ps.stdout_lines }}" - register: docker_stop - changed_when: docker_stop.rc == 0 - when: redis_compose.changed - ignore_errors: true - - - name: Start Redis service - ansible.builtin.command: docker compose -f "{{ redis_service_dir }}/docker-compose.yml" up -d - register: redis_start - changed_when: redis_start.rc == 0 - - - name: Wait for Redis to be ready - ansible.builtin.wait_for: - host: localhost - port: 6379 - timeout: 30 - - - name: Start juicefs.service to mount JuiceFS - ansible.builtin.systemd: - name: juicefs.service - state: started - enabled: true - register: juicefs_start - changed_when: juicefs_start.changed - when: juicefs_service_stat.stat.exists - become: true - - - name: Restart containers that were stopped - ansible.builtin.command: docker start {{ item }} - loop: "{{ docker_stop.results | map(attribute='item') | list }}" - register: docker_restart - changed_when: docker_restart.rc == 0 - when: redis_compose.changed - tags: - - services - - redis diff --git a/ansible/templates/borg-backup.service.j2 b/ansible/templates/borg-backup.service.j2 index 14a62cd..54c784c 100644 --- a/ansible/templates/borg-backup.service.j2 +++ b/ansible/templates/borg-backup.service.j2 @@ -20,7 +20,7 @@ Environment="BORG_KEYS_DIR={{ borg_config_dir }}/keys" NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict -ReadWritePaths=/mnt/services /mnt/object_storage /var/log {{ borg_config_dir }} +ReadWritePaths=/mnt/services /mnt/borg-backups /var/log {{ borg_config_dir }} ProtectHome=read-only ProtectControlGroups=true RestrictRealtime=true diff --git a/ansible/templates/borg-local-sync.service.j2 b/ansible/templates/borg-local-sync.service.j2 deleted file mode 100644 index f31b5da..0000000 --- a/ansible/templates/borg-local-sync.service.j2 +++ /dev/null @@ -1,48 +0,0 @@ -[Unit] -Description=Borg Local Sync - Copy Borg repository to local storage -Documentation=man:borg(1) -After=network-online.target -Wants=network-online.target -# Ensure this runs after the main backup has completed -After=borg-backup.service - -[Service] -Type=oneshot -User=root -Group=root - -# Set up environment -Environment="PATH=/usr/local/bin:/usr/bin:/bin" -Environment="LANG=en_US.UTF-8" -Environment="LC_ALL=en_US.UTF-8" - -# Security settings -ProtectSystem=strict -ProtectHome=read-only -ReadWritePaths=/var/log /mnt/borg-backups {{ borg_config_dir }} -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictRealtime=yes -RestrictSUIDSGID=yes - -# Resource limits -MemoryMax=2G -CPUQuota=80% -IOWeight=200 - -# Timeout settings (local sync might take a while for initial copy) -TimeoutStartSec=3600 -TimeoutStopSec=300 - -# Execute the sync script -ExecStart=/usr/local/bin/borg-local-sync.sh - -# Logging -StandardOutput=journal -StandardError=journal -SyslogIdentifier=borg-local-sync - -[Install] -WantedBy=multi-user.target diff --git a/ansible/templates/borg-local-sync.sh.j2 b/ansible/templates/borg-local-sync.sh.j2 deleted file mode 100644 index 785e171..0000000 --- a/ansible/templates/borg-local-sync.sh.j2 +++ /dev/null @@ -1,227 +0,0 @@ -#!/bin/bash - -# Borg local sync script for creating local copies of cloud backups -# This script syncs the Borg repository from JuiceFS/S3 to local ZFS storage - -# Set environment variables -export BORG_REPO_SOURCE="{{ borg_repo_dir }}" -export BORG_REPO_LOCAL="/mnt/borg-backups" -export ZFS_POOL="datapool" -export ZFS_DATASET="datapool/borg-backups" -export MOUNT_POINT="/mnt/borg-backups" - -# Telegram notification variables -export TELEGRAM_BOT_TOKEN="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='password') }}" -export TELEGRAM_CHAT_ID="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='chat_id') }}" - -# Log function -log() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a /var/log/borg-local-sync.log -} - -# Telegram notification function -send_telegram() { - local message="$1" - local silent="${2:-false}" - - if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then - log "Telegram credentials not configured, skipping notification" - return - fi - - local payload=$(cat < /dev/null 2>&1 - - if [ $? -eq 0 ]; then - log "Telegram notification sent successfully" - else - log "Failed to send Telegram notification" - fi -} - -# Check if ZFS pool is available -check_zfs_pool() { - if ! zpool status "$ZFS_POOL" > /dev/null 2>&1; then - log "ERROR: ZFS pool $ZFS_POOL is not available" - send_telegram "❌ Borg Local Sync Failed - -❌ ZFS pool not available: $ZFS_POOL -🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') - -The 20TB USB drive may not be connected or the ZFS pool is not imported. -Please check the physical connection and run: sudo zpool import $ZFS_POOL" - return 1 - fi - - # Check if the specific ZFS dataset exists - if ! zfs list "$ZFS_DATASET" > /dev/null 2>&1; then - log "ERROR: ZFS dataset $ZFS_DATASET is not available" - send_telegram "❌ Borg Local Sync Failed - -❌ ZFS dataset not available: $ZFS_DATASET -🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') - -The ZFS dataset may not exist or be mounted. -Please check: sudo zfs create $ZFS_DATASET" - return 1 - fi - return 0 -} - -# Check if mount point is available -check_mount_point() { - if ! mountpoint -q "$MOUNT_POINT"; then - log "ERROR: Mount point $MOUNT_POINT is not mounted" - send_telegram "❌ Borg Local Sync Failed - -❌ Mount point not available: $MOUNT_POINT -🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') - -The ZFS dataset may not be mounted. -Please check: sudo zfs mount $ZFS_DATASET" - return 1 - fi - return 0 -} - -# Check if source repository is available -check_source_repo() { - if [ ! -d "$BORG_REPO_SOURCE" ]; then - log "ERROR: Source Borg repository not found: $BORG_REPO_SOURCE" - send_telegram "❌ Borg Local Sync Failed - -❌ Source repository not found: $BORG_REPO_SOURCE -🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') - -JuiceFS may not be mounted or the source repository path is incorrect." - return 1 - fi - return 0 -} - -# Check available space -check_space() { - local source_size=$(sudo du -sb "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1) - local available_space=$(df -B1 "$MOUNT_POINT" | tail -1 | awk '{print $4}') - - if [ -z "$source_size" ]; then - log "WARNING: Could not determine source repository size" - return 0 - fi - - # Add 20% buffer for safety - local required_space=$((source_size * 120 / 100)) - - if [ "$available_space" -lt "$required_space" ]; then - local source_gb=$((source_size / 1024 / 1024 / 1024)) - local available_gb=$((available_space / 1024 / 1024 / 1024)) - local required_gb=$((required_space / 1024 / 1024 / 1024)) - - log "ERROR: Insufficient space. Source: ${source_gb}GB, Available: ${available_gb}GB, Required: ${required_gb}GB" - send_telegram "❌ Borg Local Sync Failed - -❌ Insufficient disk space -📊 Source size: ${source_gb}GB -💾 Available: ${available_gb}GB -⚠️ Required: ${required_gb}GB (with 20% buffer) -🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') - -Please free up space on the local backup drive." - return 1 - fi - - return 0 -} - -# Perform the sync -sync_repository() { - log "Starting rsync of Borg repository" - - # Get initial sizes for reporting - local source_size_before=$(sudo du -sh "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1) - local dest_size_before="0B" - if [ -d "$BORG_REPO_LOCAL" ]; then - dest_size_before=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1) - fi - - # Perform the sync with detailed logging - sudo rsync -avh --delete --progress \ - --exclude="lock.exclusive" \ - --exclude="lock.roster" \ - "$BORG_REPO_SOURCE/" "$BORG_REPO_LOCAL/" 2>&1 | while read line; do - log "rsync: $line" - done - - local rsync_exit=${PIPESTATUS[0]} - - # Get final sizes for reporting - local dest_size_after=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1) - - if [ $rsync_exit -eq 0 ]; then - log "Rsync completed successfully" - send_telegram "🔒 Borg Local Sync Success - -✅ Local backup sync completed successfully -📂 Source: $BORG_REPO_SOURCE (${source_size_before}) -💾 Destination: $BORG_REPO_LOCAL (${dest_size_after}) -🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S') - -Local backup copy is now up to date." "true" - return 0 - else - log "Rsync failed with exit code: $rsync_exit" - send_telegram "❌ Borg Local Sync Failed - -❌ Rsync failed during repository sync -📂 Source: $BORG_REPO_SOURCE -💾 Destination: $BORG_REPO_LOCAL -🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S') - -Exit code: $rsync_exit -Check logs: /var/log/borg-local-sync.log" - return 1 - fi -} - -# Main execution -log "Starting Borg local sync process" - -# Run all pre-flight checks -if ! check_zfs_pool; then - exit 1 -fi - -if ! check_mount_point; then - exit 1 -fi - -if ! check_source_repo; then - exit 1 -fi - -if ! check_space; then - exit 1 -fi - -# All checks passed, proceed with sync -log "All pre-flight checks passed, starting sync" - -if sync_repository; then - log "Local sync completed successfully" - exit 0 -else - log "Local sync failed" - exit 1 -fi diff --git a/ansible/templates/borg-local-sync.timer.j2 b/ansible/templates/borg-local-sync.timer.j2 deleted file mode 100644 index acf54da..0000000 --- a/ansible/templates/borg-local-sync.timer.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Run Borg Local Sync daily -Documentation=man:borg(1) -Requires=borg-local-sync.service - -[Timer] -# Run daily at 3:00 AM (1 hour after main backup at 2:00 AM) -OnCalendar=*-*-* 03:00:00 -# Add randomization to prevent conflicts if multiple systems exist -RandomizedDelaySec=300 -# Ensure timer persists across reboots -Persistent=true -# Wake system from suspend if needed -WakeSystem=false - -[Install] -WantedBy=timers.target diff --git a/ansible/templates/juicefs.service.j2 b/ansible/templates/juicefs.service.j2 deleted file mode 100644 index fab00a9..0000000 --- a/ansible/templates/juicefs.service.j2 +++ /dev/null @@ -1,24 +0,0 @@ -[Unit] -Description=JuiceFS -After=network.target -Before=docker.service - -[Service] -Type=simple -ExecStart=/usr/local/bin/juicefs mount redis://:{{ redis_password }}@mennos-server:6379/0 /mnt/object_storage \ - --cache-dir=/var/jfsCache \ - --buffer-size=4096 \ - --prefetch=16 \ - --cache-size=131072 \ - --attr-cache=60 \ - --entry-cache=60 \ - --open-cache=60 \ - -o writeback_cache \ - --max-uploads=80 \ - --max-deletes=80 \ - --writeback \ - --upload-delay=30s -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/bin/actions/service.py b/bin/actions/service.py index bd3b193..adb70a2 100755 --- a/bin/actions/service.py +++ b/bin/actions/service.py @@ -2,10 +2,10 @@ """Manage Docker services.""" -import os -import sys -import subprocess import argparse +import os +import subprocess +import sys # Import helper functions sys.path.append(os.path.join(os.path.dirname(__file__), "..")) @@ -14,7 +14,7 @@ from helpers.functions import printfe, println # Base directory for Docker services $HOME/services SERVICES_DIR = os.path.join(os.path.expanduser("~"), ".services") # Protected services that should never be stopped -PROTECTED_SERVICES = ["juicefs-redis"] +PROTECTED_SERVICES = [] def get_service_path(service_name): diff --git a/bin/actions/source.py b/bin/actions/source.py index ce38d99..72fb427 100755 --- a/bin/actions/source.py +++ b/bin/actions/source.py @@ -3,8 +3,8 @@ """Generate export commands for Borg environment variables.""" import os -import sys import subprocess +import sys # Add the bin directory to the path sys.path.append(os.path.join(os.path.dirname(__file__), "..")) @@ -45,7 +45,7 @@ def main(): # Generate the export commands exports = [ - 'export BORG_REPO="/mnt/object_storage/borg-repo"', + 'export BORG_REPO="/mnt/borg-backups"', f'export BORG_PASSPHRASE="{passphrase}"', 'export BORG_CACHE_DIR="/home/menno/.config/borg/cache"', 'export BORG_CONFIG_DIR="/home/menno/.config/borg/config"', diff --git a/flake.lock b/flake.lock index 7b3e1a7..32bdb6f 100644 --- a/flake.lock +++ b/flake.lock @@ -25,11 +25,11 @@ ] }, "locked": { - "lastModified": 1758463745, - "narHash": "sha256-uhzsV0Q0I9j2y/rfweWeGif5AWe0MGrgZ/3TjpDYdGA=", + "lastModified": 1763992789, + "narHash": "sha256-WHkdBlw6oyxXIra/vQPYLtqY+3G8dUVZM8bEXk0t8x4=", "owner": "nix-community", "repo": "home-manager", - "rev": "3b955f5f0a942f9f60cdc9cacb7844335d0f21c3", + "rev": "44831a7eaba4360fb81f2acc5ea6de5fde90aaa3", "type": "github" }, "original": { @@ -41,11 +41,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1761597516, - "narHash": "sha256-wxX7u6D2rpkJLWkZ2E932SIvDJW8+ON/0Yy8+a5vsDU=", + "lastModified": 1764316264, + "narHash": "sha256-82L+EJU+40+FIdeG4gmUlOF1jeSwlf2AwMarrpdHF6o=", "owner": "nixos", "repo": "nixpkgs", - "rev": "daf6dc47aa4b44791372d6139ab7b25269184d55", + "rev": "9a7b80b6f82a71ea04270d7ba11b48855681c4b0", "type": "github" }, "original": { @@ -77,11 +77,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1761503988, - "narHash": "sha256-MlMZXCTtPeXq/cDtJcL2XM8wCN33XOT9V2dB3PLV6f0=", + "lastModified": 1764172006, + "narHash": "sha256-89VihsuY1WWscerecKG+pe6WVPzFQ3ImYSqQDye78Cs=", "owner": "brizzbuzz", "repo": "opnix", - "rev": "48fdb078b5a1cd0b20b501fccf6be2d1279d6fe6", + "rev": "eaacde99a78ef4fb10ee1a68513a98532611c882", "type": "github" }, "original": {