feat: add llm command for managing sillytavern and koboldcpp
Signed-off-by: Menno van Leeuwen <menno@vleeuwen.me>
This commit is contained in:
298
config/ansible/tasks/global/utils/llm
Executable file
298
config/ansible/tasks/global/utils/llm
Executable file
@@ -0,0 +1,298 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Colors for output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
BLUE='\033[0;34m'
|
||||||
|
CYAN='\033[0;36m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
# Configuration
|
||||||
|
KOBOLD_PATH="/mnt/data/ai/llm/koboldcpp-linux-x64"
|
||||||
|
KOBOLD_MODEL="/mnt/data/ai/llm/Mistral-Small-24B-Instruct-2501-Q4_K_M.gguf" # Default model
|
||||||
|
SILLYTAVERN_SCREEN="sillytavern"
|
||||||
|
KOBOLD_SCREEN="koboldcpp"
|
||||||
|
|
||||||
|
# Function to check if a screen session exists
|
||||||
|
check_screen() {
|
||||||
|
screen -ls | grep -q "\.${1}\s"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Function to list available models
|
||||||
|
list_models() {
|
||||||
|
echo -e "${BLUE}Available models:${NC}"
|
||||||
|
ls -1 /mnt/data/ai/llm/*.gguf | nl -w2 -s'. '
|
||||||
|
}
|
||||||
|
|
||||||
|
# Function to select a model
|
||||||
|
select_model() {
|
||||||
|
list_models
|
||||||
|
echo
|
||||||
|
read -p "Select model number (or press Enter for default): " model_num
|
||||||
|
|
||||||
|
if [[ -z "$model_num" ]]; then
|
||||||
|
echo -e "${YELLOW}Using default model: $(basename "$KOBOLD_MODEL")${NC}"
|
||||||
|
else
|
||||||
|
selected_model=$(ls -1 /mnt/data/ai/llm/*.gguf | sed -n "${model_num}p")
|
||||||
|
if [[ -n "$selected_model" ]]; then
|
||||||
|
KOBOLD_MODEL="$selected_model"
|
||||||
|
echo -e "${GREEN}Selected model: $(basename "$KOBOLD_MODEL")${NC}"
|
||||||
|
else
|
||||||
|
echo -e "${RED}Invalid selection. Using default model.${NC}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Function to start SillyTavern
|
||||||
|
start_sillytavern() {
|
||||||
|
echo -e "${YELLOW}Starting SillyTavern in screen session '${SILLYTAVERN_SCREEN}'...${NC}"
|
||||||
|
screen -dmS "$SILLYTAVERN_SCREEN" bash -c "cd ~/SillyTavern && npm start -- --listen 0.0.0.0"
|
||||||
|
sleep 2
|
||||||
|
if check_screen "$SILLYTAVERN_SCREEN"; then
|
||||||
|
echo -e "${GREEN}✓ SillyTavern started successfully!${NC}"
|
||||||
|
echo -e "${BLUE} Access at: http://0.0.0.0:8000${NC}"
|
||||||
|
else
|
||||||
|
echo -e "${RED}✗ Failed to start SillyTavern${NC}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Function to start KoboldCPP
|
||||||
|
start_koboldcpp() {
|
||||||
|
select_model
|
||||||
|
echo -e "${YELLOW}Starting KoboldCPP in screen session '${KOBOLD_SCREEN}'...${NC}"
|
||||||
|
screen -dmS "$KOBOLD_SCREEN" bash -c "cd /mnt/data/ai/llm && ./koboldcpp-linux-x64 --model '$KOBOLD_MODEL' --host 0.0.0.0 --port 5001 --contextsize 8192 --gpulayers 999"
|
||||||
|
sleep 2
|
||||||
|
if check_screen "$KOBOLD_SCREEN"; then
|
||||||
|
echo -e "${GREEN}✓ KoboldCPP started successfully!${NC}"
|
||||||
|
echo -e "${BLUE} Model: $(basename "$KOBOLD_MODEL")${NC}"
|
||||||
|
echo -e "${BLUE} Access at: http://0.0.0.0:5001${NC}"
|
||||||
|
else
|
||||||
|
echo -e "${RED}✗ Failed to start KoboldCPP${NC}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Function to stop a service
|
||||||
|
stop_service() {
|
||||||
|
local service=$1
|
||||||
|
local screen_name=$2
|
||||||
|
|
||||||
|
echo -e "${YELLOW}Stopping ${service}...${NC}"
|
||||||
|
screen -S "$screen_name" -X quit
|
||||||
|
sleep 1
|
||||||
|
if ! check_screen "$screen_name"; then
|
||||||
|
echo -e "${GREEN}✓ ${service} stopped successfully${NC}"
|
||||||
|
else
|
||||||
|
echo -e "${RED}✗ Failed to stop ${service}${NC}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Function to show service status
|
||||||
|
show_status() {
|
||||||
|
echo -e "${CYAN}╔═══════════════════════════════════════╗${NC}"
|
||||||
|
echo -e "${CYAN}║ Service Status Overview ║${NC}"
|
||||||
|
echo -e "${CYAN}╚═══════════════════════════════════════╝${NC}"
|
||||||
|
echo
|
||||||
|
|
||||||
|
local st_running=false
|
||||||
|
local kc_running=false
|
||||||
|
|
||||||
|
# Check SillyTavern
|
||||||
|
if check_screen "$SILLYTAVERN_SCREEN"; then
|
||||||
|
st_running=true
|
||||||
|
echo -e " ${GREEN}●${NC} SillyTavern: ${GREEN}Running${NC} (screen: ${SILLYTAVERN_SCREEN})"
|
||||||
|
echo -e " ${BLUE}→ http://0.0.0.0:8000${NC}"
|
||||||
|
else
|
||||||
|
echo -e " ${RED}●${NC} SillyTavern: ${RED}Not running${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Check KoboldCPP
|
||||||
|
if check_screen "$KOBOLD_SCREEN"; then
|
||||||
|
kc_running=true
|
||||||
|
echo -e " ${GREEN}●${NC} KoboldCPP: ${GREEN}Running${NC} (screen: ${KOBOLD_SCREEN})"
|
||||||
|
echo -e " ${BLUE}→ http://0.0.0.0:5001${NC}"
|
||||||
|
else
|
||||||
|
echo -e " ${RED}●${NC} KoboldCPP: ${RED}Not running${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
# Function to handle service management
|
||||||
|
manage_services() {
|
||||||
|
local st_running=$(check_screen "$SILLYTAVERN_SCREEN" && echo "true" || echo "false")
|
||||||
|
local kc_running=$(check_screen "$KOBOLD_SCREEN" && echo "true" || echo "false")
|
||||||
|
|
||||||
|
# If both services are running
|
||||||
|
if [[ "$st_running" == "true" ]] && [[ "$kc_running" == "true" ]]; then
|
||||||
|
echo -e "${GREEN}Both services are running.${NC}"
|
||||||
|
echo
|
||||||
|
echo "1) Attach to SillyTavern"
|
||||||
|
echo "2) Attach to KoboldCPP"
|
||||||
|
echo "3) Restart SillyTavern"
|
||||||
|
echo "4) Restart KoboldCPP"
|
||||||
|
echo "5) Stop all services"
|
||||||
|
echo "6) Exit"
|
||||||
|
|
||||||
|
read -p "Your choice (1-6): " choice
|
||||||
|
|
||||||
|
case $choice in
|
||||||
|
1)
|
||||||
|
echo -e "${BLUE}Attaching to SillyTavern... (Use Ctrl+A then D to detach)${NC}"
|
||||||
|
sleep 1
|
||||||
|
screen -r "$SILLYTAVERN_SCREEN"
|
||||||
|
;;
|
||||||
|
2)
|
||||||
|
echo -e "${BLUE}Attaching to KoboldCPP... (Use Ctrl+A then D to detach)${NC}"
|
||||||
|
sleep 1
|
||||||
|
screen -r "$KOBOLD_SCREEN"
|
||||||
|
;;
|
||||||
|
3)
|
||||||
|
stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
|
||||||
|
echo
|
||||||
|
start_sillytavern
|
||||||
|
;;
|
||||||
|
4)
|
||||||
|
stop_service "KoboldCPP" "$KOBOLD_SCREEN"
|
||||||
|
echo
|
||||||
|
start_koboldcpp
|
||||||
|
;;
|
||||||
|
5)
|
||||||
|
stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
|
||||||
|
stop_service "KoboldCPP" "$KOBOLD_SCREEN"
|
||||||
|
;;
|
||||||
|
6)
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo -e "${RED}Invalid choice${NC}"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# If only SillyTavern is running
|
||||||
|
elif [[ "$st_running" == "true" ]]; then
|
||||||
|
echo -e "${YELLOW}Only SillyTavern is running.${NC}"
|
||||||
|
echo
|
||||||
|
echo "1) Attach to SillyTavern"
|
||||||
|
echo "2) Start KoboldCPP"
|
||||||
|
echo "3) Restart SillyTavern"
|
||||||
|
echo "4) Stop SillyTavern"
|
||||||
|
echo "5) Exit"
|
||||||
|
|
||||||
|
read -p "Your choice (1-5): " choice
|
||||||
|
|
||||||
|
case $choice in
|
||||||
|
1)
|
||||||
|
echo -e "${BLUE}Attaching to SillyTavern... (Use Ctrl+A then D to detach)${NC}"
|
||||||
|
sleep 1
|
||||||
|
screen -r "$SILLYTAVERN_SCREEN"
|
||||||
|
;;
|
||||||
|
2)
|
||||||
|
start_koboldcpp
|
||||||
|
;;
|
||||||
|
3)
|
||||||
|
stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
|
||||||
|
echo
|
||||||
|
start_sillytavern
|
||||||
|
;;
|
||||||
|
4)
|
||||||
|
stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
|
||||||
|
;;
|
||||||
|
5)
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo -e "${RED}Invalid choice${NC}"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# If only KoboldCPP is running
|
||||||
|
elif [[ "$kc_running" == "true" ]]; then
|
||||||
|
echo -e "${YELLOW}Only KoboldCPP is running.${NC}"
|
||||||
|
echo
|
||||||
|
echo "1) Attach to KoboldCPP"
|
||||||
|
echo "2) Start SillyTavern"
|
||||||
|
echo "3) Restart KoboldCPP"
|
||||||
|
echo "4) Stop KoboldCPP"
|
||||||
|
echo "5) Exit"
|
||||||
|
|
||||||
|
read -p "Your choice (1-5): " choice
|
||||||
|
|
||||||
|
case $choice in
|
||||||
|
1)
|
||||||
|
echo -e "${BLUE}Attaching to KoboldCPP... (Use Ctrl+A then D to detach)${NC}"
|
||||||
|
sleep 1
|
||||||
|
screen -r "$KOBOLD_SCREEN"
|
||||||
|
;;
|
||||||
|
2)
|
||||||
|
start_sillytavern
|
||||||
|
;;
|
||||||
|
3)
|
||||||
|
stop_service "KoboldCPP" "$KOBOLD_SCREEN"
|
||||||
|
echo
|
||||||
|
start_koboldcpp
|
||||||
|
;;
|
||||||
|
4)
|
||||||
|
stop_service "KoboldCPP" "$KOBOLD_SCREEN"
|
||||||
|
;;
|
||||||
|
5)
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo -e "${RED}Invalid choice${NC}"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# If no services are running
|
||||||
|
else
|
||||||
|
echo -e "${YELLOW}No services are running.${NC}"
|
||||||
|
echo
|
||||||
|
echo "1) Start both services"
|
||||||
|
echo "2) Start SillyTavern only"
|
||||||
|
echo "3) Start KoboldCPP only"
|
||||||
|
echo "4) Exit"
|
||||||
|
|
||||||
|
read -p "Your choice (1-4): " choice
|
||||||
|
|
||||||
|
case $choice in
|
||||||
|
1)
|
||||||
|
start_sillytavern
|
||||||
|
echo
|
||||||
|
start_koboldcpp
|
||||||
|
;;
|
||||||
|
2)
|
||||||
|
start_sillytavern
|
||||||
|
;;
|
||||||
|
3)
|
||||||
|
start_koboldcpp
|
||||||
|
;;
|
||||||
|
4)
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo -e "${RED}Invalid choice${NC}"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Main script
|
||||||
|
echo -e "${BLUE}╔═══════════════════════════════════════╗${NC}"
|
||||||
|
echo -e "${BLUE}║ LLM Services Manager ║${NC}"
|
||||||
|
echo -e "${BLUE}╚═══════════════════════════════════════╝${NC}"
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Show status
|
||||||
|
show_status
|
||||||
|
|
||||||
|
# Show separator and manage services
|
||||||
|
echo -e "${CYAN}═══════════════════════════════════════${NC}"
|
||||||
|
manage_services
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo -e "${BLUE}Quick reference:${NC}"
|
||||||
|
echo "• List sessions: screen -ls"
|
||||||
|
echo "• Attach: screen -r <name>"
|
||||||
|
echo "• Detach: Ctrl+A then D"
|
||||||
Reference in New Issue
Block a user