Move Borg repo to local backups and remove JuiceFS
Some checks failed
Ansible Lint Check / check-ansible (push) Failing after 1s
Nix Format Check / check-format (push) Failing after 1s
Python Lint Check / check-python (push) Failing after 1s

Use /mnt/borg-backups in place of /mnt/object_storage for Borg.

Remove JuiceFS and Redis artifacts (tasks, templates, service configs)
and delete borg-local-sync tooling. Update borg-backup service
ReadWritePaths, remove Plex slow tvshows mount, add system sysctl
performance tunings, and apply minor code and flake.lock updates.
This commit is contained in:
2025-12-11 11:45:54 +01:00
parent 94f9fbb0e2
commit a987adb86b
16 changed files with 165 additions and 810 deletions

View File

@@ -57,37 +57,11 @@ If you add a new system you should add the relevant files to these paths.
### Server reboots
In case you reboot a server, it's likely that this runs JuiceFS.
To be sure that every service is properly accessing JuiceFS mounted files you should probably restart the services once when the server comes online.
```bash
dotf service stop --all
df # confirm JuiceFS is mounted
dotf service start --all
```
### Object Storage (Servers only)
In case you need to adjust anything regarding the /mnt/object_storage JuiceFS.
Ensure to shut down all services:
```bash
dotf service stop --all
```
Unmount the volume:
```bash
sudo systemctl stop juicefs
```
And optionally if you're going to do something with metadata you might need to stop redis too.
```bash
cd ~/services/juicefs-redis/
docker compose down --remove-orphans
```
### Adding a new system
To add a new system you should follow these steps:

View File

@@ -19,7 +19,7 @@
borg_passphrase: "{{ lookup('community.general.onepassword', 'Borg Backup', vault='Dotfiles', field='password') }}"
borg_config_dir: "{{ ansible_env.HOME }}/.config/borg"
borg_backup_dir: "/mnt/services"
borg_repo_dir: "/mnt/object_storage/borg-repo"
borg_repo_dir: "/mnt/borg-backups"
- name: Create Borg directories
ansible.builtin.file:
@@ -28,7 +28,7 @@
mode: "0755"
loop:
- "{{ borg_config_dir }}"
- "/mnt/object_storage"
- "/mnt/borg-backups"
loop_control:
loop_var: borg_dir
become: true

View File

@@ -1,95 +0,0 @@
---
- name: Borg Local Sync Installation and Configuration
block:
- name: Set Borg backup facts
ansible.builtin.set_fact:
borg_passphrase: "{{ lookup('community.general.onepassword', 'Borg Backup', vault='Dotfiles', field='password') }}"
borg_config_dir: "{{ ansible_env.HOME }}/.config/borg"
borg_backup_dir: "/mnt/services"
borg_repo_dir: "/mnt/object_storage/borg-repo"
- name: Create Borg local sync script
template:
src: borg-local-sync.sh.j2
dest: /usr/local/bin/borg-local-sync.sh
mode: "0755"
owner: root
group: root
become: yes
tags:
- borg-local-sync
- name: Create Borg local sync systemd service
template:
src: borg-local-sync.service.j2
dest: /etc/systemd/system/borg-local-sync.service
mode: "0644"
owner: root
group: root
become: yes
notify:
- reload systemd
tags:
- borg-local-sync
- name: Create Borg local sync systemd timer
template:
src: borg-local-sync.timer.j2
dest: /etc/systemd/system/borg-local-sync.timer
mode: "0644"
owner: root
group: root
become: yes
notify:
- reload systemd
- restart borg-local-sync-timer
tags:
- borg-local-sync
- name: Create log file for Borg local sync
file:
path: /var/log/borg-local-sync.log
state: touch
owner: root
group: root
mode: "0644"
become: yes
tags:
- borg-local-sync
- name: Enable and start Borg local sync timer
systemd:
name: borg-local-sync.timer
enabled: yes
state: started
daemon_reload: yes
become: yes
tags:
- borg-local-sync
- name: Add logrotate configuration for Borg local sync
copy:
content: |
/var/log/borg-local-sync.log {
daily
rotate 30
compress
delaycompress
missingok
notifempty
create 644 root root
}
dest: /etc/logrotate.d/borg-local-sync
mode: "0644"
owner: root
group: root
become: yes
tags:
- borg-local-sync
- borg
- backup
tags:
- borg-local-sync
- borg
- backup

View File

@@ -1,94 +0,0 @@
---
- name: JuiceFS Installation and Configuration
block:
- name: Check if JuiceFS is already installed
ansible.builtin.command: which juicefs
register: juicefs_check
ignore_errors: true
changed_when: false
- name: Install JuiceFS using the automatic installer
ansible.builtin.shell: curl -sSL https://d.juicefs.com/install | sh -
register: juicefs_installation
when: juicefs_check.rc != 0
become: true
- name: Verify JuiceFS installation
ansible.builtin.command: juicefs version
register: juicefs_version
changed_when: false
when: juicefs_check.rc != 0 or juicefs_installation.changed
- name: Create mount directory
ansible.builtin.file:
path: /mnt/object_storage
state: directory
mode: "0755"
become: true
- name: Create cache directory
ansible.builtin.file:
path: /var/jfsCache
state: directory
mode: "0755"
become: true
- name: Configure JuiceFS network performance optimizations
ansible.builtin.sysctl:
name: "{{ item.name }}"
value: "{{ item.value }}"
state: present
reload: true
become: true
loop:
- { name: "net.core.rmem_max", value: "16777216" }
- { name: "net.core.wmem_max", value: "16777216" }
- { name: "net.ipv4.tcp_rmem", value: "4096 87380 16777216" }
- { name: "net.ipv4.tcp_wmem", value: "4096 65536 16777216" }
- name: Set JuiceFS facts
ansible.builtin.set_fact:
hetzner_access_key: "{{ lookup('community.general.onepassword', 'Hetzner Object Storage Bucket', vault='Dotfiles', field='AWS_ACCESS_KEY_ID') }}"
hetzner_secret_key:
"{{ lookup('community.general.onepassword', 'Hetzner Object Storage Bucket', vault='Dotfiles', field='AWS_SECRET_ACCESS_KEY')
}}"
redis_password: "{{ lookup('community.general.onepassword', 'JuiceFS (Redis)', vault='Dotfiles', field='password') }}"
- name: Create JuiceFS systemd service file
ansible.builtin.template:
src: templates/juicefs.service.j2
dest: /etc/systemd/system/juicefs.service
owner: root
group: root
mode: "0644"
become: true
- name: Reload systemd daemon
ansible.builtin.systemd:
daemon_reload: true
become: true
- name: Include JuiceFS Redis tasks
ansible.builtin.include_tasks: services/redis/redis.yml
when: inventory_hostname == 'mennos-server'
- name: Enable and start JuiceFS service
ansible.builtin.systemd:
name: juicefs.service
enabled: true
state: started
become: true
- name: Check if JuiceFS is mounted
ansible.builtin.shell: df -h | grep /mnt/object_storage
become: true
register: mount_check
ignore_errors: true
changed_when: false
- name: Display mount status
ansible.builtin.debug:
msg: "JuiceFS is successfully mounted at /mnt/object_storage"
when: mount_check.rc == 0
tags:
- juicefs

View File

@@ -27,11 +27,6 @@
become: true
when: ansible_pkg_mgr != 'pacman'
- name: Include JuiceFS tasks
ansible.builtin.include_tasks: juicefs.yml
tags:
- juicefs
- name: Include Dynamic DNS tasks
ansible.builtin.include_tasks: dynamic-dns.yml
tags:

View File

@@ -16,7 +16,6 @@ services:
- {{ plex_data_dir }}/transcode:/transcode
- /mnt/data/movies:/movies
- /mnt/data/tvshows:/tvshows
- /mnt/object_storage/tvshows:/tvshows_slow
- /mnt/data/music:/music
deploy:
resources:

View File

@@ -1,26 +0,0 @@
services:
juicefs-redis:
image: redis:latest
restart: always
ports:
- "6379:6379"
volumes:
- /mnt/services/redis:/data
command: ["redis-server", "--appendonly", "yes", "--requirepass", "{{ REDIS_PASSWORD }}"]
environment:
- TZ=Europe/Amsterdam
healthcheck:
test: ["CMD", "redis-cli", "-a", "{{ REDIS_PASSWORD }}", "ping"]
interval: 30s
timeout: 10s
retries: 3
start_period: 5s
networks:
- juicefs-network
deploy:
resources:
limits:
memory: 256M
networks:
juicefs-network:

View File

@@ -1,82 +0,0 @@
---
- name: Deploy Redis for JuiceFS
block:
- name: Set Redis facts
ansible.builtin.set_fact:
redis_service_dir: "{{ ansible_env.HOME }}/.services/juicefs-redis"
redis_password: "{{ lookup('community.general.onepassword', 'JuiceFS (Redis)', vault='Dotfiles', field='password') }}"
- name: Create Redis service directory
ansible.builtin.file:
path: "{{ redis_service_dir }}"
state: directory
mode: "0755"
- name: Deploy Redis docker-compose.yml
ansible.builtin.template:
src: docker-compose.yml.j2
dest: "{{ redis_service_dir }}/docker-compose.yml"
mode: "0644"
register: redis_compose
vars:
REDIS_PASSWORD: "{{ redis_password }}"
- name: Check if juicefs.service exists
ansible.builtin.stat:
path: /etc/systemd/system/juicefs.service
register: juicefs_service_stat
- name: Stop juicefs.service to umount JuiceFS
ansible.builtin.systemd:
name: juicefs.service
state: stopped
enabled: false
register: juicefs_stop
changed_when: juicefs_stop.changed
when: redis_compose.changed and juicefs_service_stat.stat.exists
become: true
- name: List containers that are running
ansible.builtin.command: docker ps -q
register: docker_ps
changed_when: docker_ps.rc == 0
when: redis_compose.changed
- name: Stop all docker containers
ansible.builtin.command: docker stop {{ item }}
loop: "{{ docker_ps.stdout_lines }}"
register: docker_stop
changed_when: docker_stop.rc == 0
when: redis_compose.changed
ignore_errors: true
- name: Start Redis service
ansible.builtin.command: docker compose -f "{{ redis_service_dir }}/docker-compose.yml" up -d
register: redis_start
changed_when: redis_start.rc == 0
- name: Wait for Redis to be ready
ansible.builtin.wait_for:
host: localhost
port: 6379
timeout: 30
- name: Start juicefs.service to mount JuiceFS
ansible.builtin.systemd:
name: juicefs.service
state: started
enabled: true
register: juicefs_start
changed_when: juicefs_start.changed
when: juicefs_service_stat.stat.exists
become: true
- name: Restart containers that were stopped
ansible.builtin.command: docker start {{ item }}
loop: "{{ docker_stop.results | map(attribute='item') | list }}"
register: docker_restart
changed_when: docker_restart.rc == 0
when: redis_compose.changed
tags:
- services
- redis

View File

@@ -20,7 +20,7 @@ Environment="BORG_KEYS_DIR={{ borg_config_dir }}/keys"
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ReadWritePaths=/mnt/services /mnt/object_storage /var/log {{ borg_config_dir }}
ReadWritePaths=/mnt/services /mnt/borg-backups /var/log {{ borg_config_dir }}
ProtectHome=read-only
ProtectControlGroups=true
RestrictRealtime=true

View File

@@ -1,48 +0,0 @@
[Unit]
Description=Borg Local Sync - Copy Borg repository to local storage
Documentation=man:borg(1)
After=network-online.target
Wants=network-online.target
# Ensure this runs after the main backup has completed
After=borg-backup.service
[Service]
Type=oneshot
User=root
Group=root
# Set up environment
Environment="PATH=/usr/local/bin:/usr/bin:/bin"
Environment="LANG=en_US.UTF-8"
Environment="LC_ALL=en_US.UTF-8"
# Security settings
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/var/log /mnt/borg-backups {{ borg_config_dir }}
PrivateTmp=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
RestrictRealtime=yes
RestrictSUIDSGID=yes
# Resource limits
MemoryMax=2G
CPUQuota=80%
IOWeight=200
# Timeout settings (local sync might take a while for initial copy)
TimeoutStartSec=3600
TimeoutStopSec=300
# Execute the sync script
ExecStart=/usr/local/bin/borg-local-sync.sh
# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=borg-local-sync
[Install]
WantedBy=multi-user.target

View File

@@ -1,227 +0,0 @@
#!/bin/bash
# Borg local sync script for creating local copies of cloud backups
# This script syncs the Borg repository from JuiceFS/S3 to local ZFS storage
# Set environment variables
export BORG_REPO_SOURCE="{{ borg_repo_dir }}"
export BORG_REPO_LOCAL="/mnt/borg-backups"
export ZFS_POOL="datapool"
export ZFS_DATASET="datapool/borg-backups"
export MOUNT_POINT="/mnt/borg-backups"
# Telegram notification variables
export TELEGRAM_BOT_TOKEN="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='password') }}"
export TELEGRAM_CHAT_ID="{{ lookup('community.general.onepassword', 'Telegram Home Server Bot', vault='Dotfiles', field='chat_id') }}"
# Log function
log() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a /var/log/borg-local-sync.log
}
# Telegram notification function
send_telegram() {
local message="$1"
local silent="${2:-false}"
if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then
log "Telegram credentials not configured, skipping notification"
return
fi
local payload=$(cat <<EOF
{
"chat_id": "$TELEGRAM_CHAT_ID",
"text": "$message",
"parse_mode": "HTML",
"disable_notification": $silent
}
EOF
)
curl -s -X POST \
-H "Content-Type: application/json" \
-d "$payload" \
"https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" > /dev/null 2>&1
if [ $? -eq 0 ]; then
log "Telegram notification sent successfully"
else
log "Failed to send Telegram notification"
fi
}
# Check if ZFS pool is available
check_zfs_pool() {
if ! zpool status "$ZFS_POOL" > /dev/null 2>&1; then
log "ERROR: ZFS pool $ZFS_POOL is not available"
send_telegram "❌ <b>Borg Local Sync Failed</b>
❌ ZFS pool not available: $ZFS_POOL
🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
The 20TB USB drive may not be connected or the ZFS pool is not imported.
Please check the physical connection and run: sudo zpool import $ZFS_POOL"
return 1
fi
# Check if the specific ZFS dataset exists
if ! zfs list "$ZFS_DATASET" > /dev/null 2>&1; then
log "ERROR: ZFS dataset $ZFS_DATASET is not available"
send_telegram "❌ <b>Borg Local Sync Failed</b>
❌ ZFS dataset not available: $ZFS_DATASET
🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
The ZFS dataset may not exist or be mounted.
Please check: sudo zfs create $ZFS_DATASET"
return 1
fi
return 0
}
# Check if mount point is available
check_mount_point() {
if ! mountpoint -q "$MOUNT_POINT"; then
log "ERROR: Mount point $MOUNT_POINT is not mounted"
send_telegram "❌ <b>Borg Local Sync Failed</b>
❌ Mount point not available: $MOUNT_POINT
🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
The ZFS dataset may not be mounted.
Please check: sudo zfs mount $ZFS_DATASET"
return 1
fi
return 0
}
# Check if source repository is available
check_source_repo() {
if [ ! -d "$BORG_REPO_SOURCE" ]; then
log "ERROR: Source Borg repository not found: $BORG_REPO_SOURCE"
send_telegram "❌ <b>Borg Local Sync Failed</b>
❌ Source repository not found: $BORG_REPO_SOURCE
🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
JuiceFS may not be mounted or the source repository path is incorrect."
return 1
fi
return 0
}
# Check available space
check_space() {
local source_size=$(sudo du -sb "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1)
local available_space=$(df -B1 "$MOUNT_POINT" | tail -1 | awk '{print $4}')
if [ -z "$source_size" ]; then
log "WARNING: Could not determine source repository size"
return 0
fi
# Add 20% buffer for safety
local required_space=$((source_size * 120 / 100))
if [ "$available_space" -lt "$required_space" ]; then
local source_gb=$((source_size / 1024 / 1024 / 1024))
local available_gb=$((available_space / 1024 / 1024 / 1024))
local required_gb=$((required_space / 1024 / 1024 / 1024))
log "ERROR: Insufficient space. Source: ${source_gb}GB, Available: ${available_gb}GB, Required: ${required_gb}GB"
send_telegram "❌ <b>Borg Local Sync Failed</b>
❌ Insufficient disk space
📊 Source size: ${source_gb}GB
💾 Available: ${available_gb}GB
⚠️ Required: ${required_gb}GB (with 20% buffer)
🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
Please free up space on the local backup drive."
return 1
fi
return 0
}
# Perform the sync
sync_repository() {
log "Starting rsync of Borg repository"
# Get initial sizes for reporting
local source_size_before=$(sudo du -sh "$BORG_REPO_SOURCE" 2>/dev/null | cut -f1)
local dest_size_before="0B"
if [ -d "$BORG_REPO_LOCAL" ]; then
dest_size_before=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1)
fi
# Perform the sync with detailed logging
sudo rsync -avh --delete --progress \
--exclude="lock.exclusive" \
--exclude="lock.roster" \
"$BORG_REPO_SOURCE/" "$BORG_REPO_LOCAL/" 2>&1 | while read line; do
log "rsync: $line"
done
local rsync_exit=${PIPESTATUS[0]}
# Get final sizes for reporting
local dest_size_after=$(sudo du -sh "$BORG_REPO_LOCAL" 2>/dev/null | cut -f1)
if [ $rsync_exit -eq 0 ]; then
log "Rsync completed successfully"
send_telegram "🔒 <b>Borg Local Sync Success</b>
✅ Local backup sync completed successfully
📂 Source: $BORG_REPO_SOURCE (${source_size_before})
💾 Destination: $BORG_REPO_LOCAL (${dest_size_after})
🕐 Completed: $(date '+%Y-%m-%d %H:%M:%S')
Local backup copy is now up to date." "true"
return 0
else
log "Rsync failed with exit code: $rsync_exit"
send_telegram "❌ <b>Borg Local Sync Failed</b>
❌ Rsync failed during repository sync
📂 Source: $BORG_REPO_SOURCE
💾 Destination: $BORG_REPO_LOCAL
🕐 Failed: $(date '+%Y-%m-%d %H:%M:%S')
Exit code: $rsync_exit
Check logs: /var/log/borg-local-sync.log"
return 1
fi
}
# Main execution
log "Starting Borg local sync process"
# Run all pre-flight checks
if ! check_zfs_pool; then
exit 1
fi
if ! check_mount_point; then
exit 1
fi
if ! check_source_repo; then
exit 1
fi
if ! check_space; then
exit 1
fi
# All checks passed, proceed with sync
log "All pre-flight checks passed, starting sync"
if sync_repository; then
log "Local sync completed successfully"
exit 0
else
log "Local sync failed"
exit 1
fi

View File

@@ -1,17 +0,0 @@
[Unit]
Description=Run Borg Local Sync daily
Documentation=man:borg(1)
Requires=borg-local-sync.service
[Timer]
# Run daily at 3:00 AM (1 hour after main backup at 2:00 AM)
OnCalendar=*-*-* 03:00:00
# Add randomization to prevent conflicts if multiple systems exist
RandomizedDelaySec=300
# Ensure timer persists across reboots
Persistent=true
# Wake system from suspend if needed
WakeSystem=false
[Install]
WantedBy=timers.target

View File

@@ -1,24 +0,0 @@
[Unit]
Description=JuiceFS
After=network.target
Before=docker.service
[Service]
Type=simple
ExecStart=/usr/local/bin/juicefs mount redis://:{{ redis_password }}@mennos-server:6379/0 /mnt/object_storage \
--cache-dir=/var/jfsCache \
--buffer-size=4096 \
--prefetch=16 \
--cache-size=131072 \
--attr-cache=60 \
--entry-cache=60 \
--open-cache=60 \
-o writeback_cache \
--max-uploads=80 \
--max-deletes=80 \
--writeback \
--upload-delay=30s
Restart=on-failure
[Install]
WantedBy=multi-user.target

View File

@@ -2,10 +2,10 @@
"""Manage Docker services."""
import os
import sys
import subprocess
import argparse
import os
import subprocess
import sys
# Import helper functions
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -14,7 +14,7 @@ from helpers.functions import printfe, println
# Base directory for Docker services $HOME/services
SERVICES_DIR = os.path.join(os.path.expanduser("~"), ".services")
# Protected services that should never be stopped
PROTECTED_SERVICES = ["juicefs-redis"]
PROTECTED_SERVICES = []
def get_service_path(service_name):

View File

@@ -3,8 +3,8 @@
"""Generate export commands for Borg environment variables."""
import os
import sys
import subprocess
import sys
# Add the bin directory to the path
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -45,7 +45,7 @@ def main():
# Generate the export commands
exports = [
'export BORG_REPO="/mnt/object_storage/borg-repo"',
'export BORG_REPO="/mnt/borg-backups"',
f'export BORG_PASSPHRASE="{passphrase}"',
'export BORG_CACHE_DIR="/home/menno/.config/borg/cache"',
'export BORG_CONFIG_DIR="/home/menno/.config/borg/config"',

18
flake.lock generated
View File

@@ -25,11 +25,11 @@
]
},
"locked": {
"lastModified": 1758463745,
"narHash": "sha256-uhzsV0Q0I9j2y/rfweWeGif5AWe0MGrgZ/3TjpDYdGA=",
"lastModified": 1763992789,
"narHash": "sha256-WHkdBlw6oyxXIra/vQPYLtqY+3G8dUVZM8bEXk0t8x4=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "3b955f5f0a942f9f60cdc9cacb7844335d0f21c3",
"rev": "44831a7eaba4360fb81f2acc5ea6de5fde90aaa3",
"type": "github"
},
"original": {
@@ -41,11 +41,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1761597516,
"narHash": "sha256-wxX7u6D2rpkJLWkZ2E932SIvDJW8+ON/0Yy8+a5vsDU=",
"lastModified": 1764316264,
"narHash": "sha256-82L+EJU+40+FIdeG4gmUlOF1jeSwlf2AwMarrpdHF6o=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "daf6dc47aa4b44791372d6139ab7b25269184d55",
"rev": "9a7b80b6f82a71ea04270d7ba11b48855681c4b0",
"type": "github"
},
"original": {
@@ -77,11 +77,11 @@
"nixpkgs": "nixpkgs_2"
},
"locked": {
"lastModified": 1761503988,
"narHash": "sha256-MlMZXCTtPeXq/cDtJcL2XM8wCN33XOT9V2dB3PLV6f0=",
"lastModified": 1764172006,
"narHash": "sha256-89VihsuY1WWscerecKG+pe6WVPzFQ3ImYSqQDye78Cs=",
"owner": "brizzbuzz",
"repo": "opnix",
"rev": "48fdb078b5a1cd0b20b501fccf6be2d1279d6fe6",
"rev": "eaacde99a78ef4fb10ee1a68513a98532611c882",
"type": "github"
},
"original": {