From 5edee3250978bdf1377170302ef30ddad2df4575 Mon Sep 17 00:00:00 2001
From: Menno van Leeuwen <menno@vleeuwen.me>
Date: Wed, 13 Aug 2025 13:26:27 +0200
Subject: [PATCH] feat: add llm command for managing sillytavern and koboldcpp

Signed-off-by: Menno van Leeuwen <menno@vleeuwen.me>
---
 config/ansible/tasks/global/utils/llm | 298 ++++++++++++++++++++++++++
 1 file changed, 298 insertions(+)
 create mode 100755 config/ansible/tasks/global/utils/llm

diff --git a/config/ansible/tasks/global/utils/llm b/config/ansible/tasks/global/utils/llm
new file mode 100755
index 0000000..a2372b0
--- /dev/null
+++ b/config/ansible/tasks/global/utils/llm
@@ -0,0 +1,298 @@
+#!/bin/bash
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m' # No Color
+
+# Configuration
+KOBOLD_PATH="/mnt/data/ai/llm/koboldcpp-linux-x64"
+KOBOLD_MODEL="/mnt/data/ai/llm/Mistral-Small-24B-Instruct-2501-Q4_K_M.gguf"  # Default model
+SILLYTAVERN_SCREEN="sillytavern"
+KOBOLD_SCREEN="koboldcpp"
+
+# Function to check if a screen session exists
+check_screen() {
+    screen -ls | grep -q "\.${1}\s"
+}
+
+# Function to list available models
+list_models() {
+    echo -e "${BLUE}Available models:${NC}"
+    ls -1 /mnt/data/ai/llm/*.gguf | nl -w2 -s'. '
+}
+
+# Function to select a model
+select_model() {
+    list_models
+    echo
+    read -p "Select model number (or press Enter for default): " model_num
+
+    if [[ -z "$model_num" ]]; then
+        echo -e "${YELLOW}Using default model: $(basename "$KOBOLD_MODEL")${NC}"
+    else
+        selected_model=$(ls -1 /mnt/data/ai/llm/*.gguf | sed -n "${model_num}p")
+        if [[ -n "$selected_model" ]]; then
+            KOBOLD_MODEL="$selected_model"
+            echo -e "${GREEN}Selected model: $(basename "$KOBOLD_MODEL")${NC}"
+        else
+            echo -e "${RED}Invalid selection. Using default model.${NC}"
+        fi
+    fi
+}
+
+# Function to start SillyTavern
+start_sillytavern() {
+    echo -e "${YELLOW}Starting SillyTavern in screen session '${SILLYTAVERN_SCREEN}'...${NC}"
+    screen -dmS "$SILLYTAVERN_SCREEN" bash -c "cd ~/SillyTavern && npm start -- --listen 0.0.0.0"
+    sleep 2
+    if check_screen "$SILLYTAVERN_SCREEN"; then
+        echo -e "${GREEN}✓ SillyTavern started successfully!${NC}"
+        echo -e "${BLUE}  Access at: http://0.0.0.0:8000${NC}"
+    else
+        echo -e "${RED}✗ Failed to start SillyTavern${NC}"
+    fi
+}
+
+# Function to start KoboldCPP
+start_koboldcpp() {
+    select_model
+    echo -e "${YELLOW}Starting KoboldCPP in screen session '${KOBOLD_SCREEN}'...${NC}"
+    screen -dmS "$KOBOLD_SCREEN" bash -c "cd /mnt/data/ai/llm && ./koboldcpp-linux-x64 --model '$KOBOLD_MODEL' --host 0.0.0.0 --port 5001 --contextsize 8192 --gpulayers 999"
+    sleep 2
+    if check_screen "$KOBOLD_SCREEN"; then
+        echo -e "${GREEN}✓ KoboldCPP started successfully!${NC}"
+        echo -e "${BLUE}  Model: $(basename "$KOBOLD_MODEL")${NC}"
+        echo -e "${BLUE}  Access at: http://0.0.0.0:5001${NC}"
+    else
+        echo -e "${RED}✗ Failed to start KoboldCPP${NC}"
+    fi
+}
+
+# Function to stop a service
+stop_service() {
+    local service=$1
+    local screen_name=$2
+
+    echo -e "${YELLOW}Stopping ${service}...${NC}"
+    screen -S "$screen_name" -X quit
+    sleep 1
+    if ! check_screen "$screen_name"; then
+        echo -e "${GREEN}✓ ${service} stopped successfully${NC}"
+    else
+        echo -e "${RED}✗ Failed to stop ${service}${NC}"
+    fi
+}
+
+# Function to show service status
+show_status() {
+    echo -e "${CYAN}╔═══════════════════════════════════════╗${NC}"
+    echo -e "${CYAN}║        Service Status Overview        ║${NC}"
+    echo -e "${CYAN}╚═══════════════════════════════════════╝${NC}"
+    echo
+
+    local st_running=false
+    local kc_running=false
+
+    # Check SillyTavern
+    if check_screen "$SILLYTAVERN_SCREEN"; then
+        st_running=true
+        echo -e " ${GREEN}●${NC} SillyTavern:  ${GREEN}Running${NC} (screen: ${SILLYTAVERN_SCREEN})"
+        echo -e "   ${BLUE}→ http://0.0.0.0:8000${NC}"
+    else
+        echo -e " ${RED}●${NC} SillyTavern:  ${RED}Not running${NC}"
+    fi
+
+    echo
+
+    # Check KoboldCPP
+    if check_screen "$KOBOLD_SCREEN"; then
+        kc_running=true
+        echo -e " ${GREEN}●${NC} KoboldCPP:    ${GREEN}Running${NC} (screen: ${KOBOLD_SCREEN})"
+        echo -e "   ${BLUE}→ http://0.0.0.0:5001${NC}"
+    else
+        echo -e " ${RED}●${NC} KoboldCPP:    ${RED}Not running${NC}"
+    fi
+
+    echo
+}
+
+# Function to handle service management
+manage_services() {
+    local st_running=$(check_screen "$SILLYTAVERN_SCREEN" && echo "true" || echo "false")
+    local kc_running=$(check_screen "$KOBOLD_SCREEN" && echo "true" || echo "false")
+
+    # If both services are running
+    if [[ "$st_running" == "true" ]] && [[ "$kc_running" == "true" ]]; then
+        echo -e "${GREEN}Both services are running.${NC}"
+        echo
+        echo "1) Attach to SillyTavern"
+        echo "2) Attach to KoboldCPP"
+        echo "3) Restart SillyTavern"
+        echo "4) Restart KoboldCPP"
+        echo "5) Stop all services"
+        echo "6) Exit"
+
+        read -p "Your choice (1-6): " choice
+
+        case $choice in
+            1)
+                echo -e "${BLUE}Attaching to SillyTavern... (Use Ctrl+A then D to detach)${NC}"
+                sleep 1
+                screen -r "$SILLYTAVERN_SCREEN"
+                ;;
+            2)
+                echo -e "${BLUE}Attaching to KoboldCPP... (Use Ctrl+A then D to detach)${NC}"
+                sleep 1
+                screen -r "$KOBOLD_SCREEN"
+                ;;
+            3)
+                stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
+                echo
+                start_sillytavern
+                ;;
+            4)
+                stop_service "KoboldCPP" "$KOBOLD_SCREEN"
+                echo
+                start_koboldcpp
+                ;;
+            5)
+                stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
+                stop_service "KoboldCPP" "$KOBOLD_SCREEN"
+                ;;
+            6)
+                exit 0
+                ;;
+            *)
+                echo -e "${RED}Invalid choice${NC}"
+                ;;
+        esac
+
+    # If only SillyTavern is running
+    elif [[ "$st_running" == "true" ]]; then
+        echo -e "${YELLOW}Only SillyTavern is running.${NC}"
+        echo
+        echo "1) Attach to SillyTavern"
+        echo "2) Start KoboldCPP"
+        echo "3) Restart SillyTavern"
+        echo "4) Stop SillyTavern"
+        echo "5) Exit"
+
+        read -p "Your choice (1-5): " choice
+
+        case $choice in
+            1)
+                echo -e "${BLUE}Attaching to SillyTavern... (Use Ctrl+A then D to detach)${NC}"
+                sleep 1
+                screen -r "$SILLYTAVERN_SCREEN"
+                ;;
+            2)
+                start_koboldcpp
+                ;;
+            3)
+                stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
+                echo
+                start_sillytavern
+                ;;
+            4)
+                stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
+                ;;
+            5)
+                exit 0
+                ;;
+            *)
+                echo -e "${RED}Invalid choice${NC}"
+                ;;
+        esac
+
+    # If only KoboldCPP is running
+    elif [[ "$kc_running" == "true" ]]; then
+        echo -e "${YELLOW}Only KoboldCPP is running.${NC}"
+        echo
+        echo "1) Attach to KoboldCPP"
+        echo "2) Start SillyTavern"
+        echo "3) Restart KoboldCPP"
+        echo "4) Stop KoboldCPP"
+        echo "5) Exit"
+
+        read -p "Your choice (1-5): " choice
+
+        case $choice in
+            1)
+                echo -e "${BLUE}Attaching to KoboldCPP... (Use Ctrl+A then D to detach)${NC}"
+                sleep 1
+                screen -r "$KOBOLD_SCREEN"
+                ;;
+            2)
+                start_sillytavern
+                ;;
+            3)
+                stop_service "KoboldCPP" "$KOBOLD_SCREEN"
+                echo
+                start_koboldcpp
+                ;;
+            4)
+                stop_service "KoboldCPP" "$KOBOLD_SCREEN"
+                ;;
+            5)
+                exit 0
+                ;;
+            *)
+                echo -e "${RED}Invalid choice${NC}"
+                ;;
+        esac
+
+    # If no services are running
+    else
+        echo -e "${YELLOW}No services are running.${NC}"
+        echo
+        echo "1) Start both services"
+        echo "2) Start SillyTavern only"
+        echo "3) Start KoboldCPP only"
+        echo "4) Exit"
+
+        read -p "Your choice (1-4): " choice
+
+        case $choice in
+            1)
+                start_sillytavern
+                echo
+                start_koboldcpp
+                ;;
+            2)
+                start_sillytavern
+                ;;
+            3)
+                start_koboldcpp
+                ;;
+            4)
+                exit 0
+                ;;
+            *)
+                echo -e "${RED}Invalid choice${NC}"
+                ;;
+        esac
+    fi
+}
+
+# Main script
+echo -e "${BLUE}╔═══════════════════════════════════════╗${NC}"
+echo -e "${BLUE}║         LLM Services Manager          ║${NC}"
+echo -e "${BLUE}╚═══════════════════════════════════════╝${NC}"
+echo
+
+# Show status
+show_status
+
+# Show separator and manage services
+echo -e "${CYAN}═══════════════════════════════════════${NC}"
+manage_services
+
+echo
+echo -e "${BLUE}Quick reference:${NC}"
+echo "• List sessions: screen -ls"
+echo "• Attach: screen -r <name>"
+echo "• Detach: Ctrl+A then D"