feat: add llm command for managing sillytavern and koboldcpp
Signed-off-by: Menno van Leeuwen <menno@vleeuwen.me>
This commit is contained in:
298
config/ansible/tasks/global/utils/llm
Executable file
298
config/ansible/tasks/global/utils/llm
Executable file
@@ -0,0 +1,298 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
KOBOLD_PATH="/mnt/data/ai/llm/koboldcpp-linux-x64"
|
||||
KOBOLD_MODEL="/mnt/data/ai/llm/Mistral-Small-24B-Instruct-2501-Q4_K_M.gguf" # Default model
|
||||
SILLYTAVERN_SCREEN="sillytavern"
|
||||
KOBOLD_SCREEN="koboldcpp"
|
||||
|
||||
# Function to check if a screen session exists
|
||||
check_screen() {
|
||||
screen -ls | grep -q "\.${1}\s"
|
||||
}
|
||||
|
||||
# Function to list available models
|
||||
list_models() {
|
||||
echo -e "${BLUE}Available models:${NC}"
|
||||
ls -1 /mnt/data/ai/llm/*.gguf | nl -w2 -s'. '
|
||||
}
|
||||
|
||||
# Function to select a model
|
||||
select_model() {
|
||||
list_models
|
||||
echo
|
||||
read -p "Select model number (or press Enter for default): " model_num
|
||||
|
||||
if [[ -z "$model_num" ]]; then
|
||||
echo -e "${YELLOW}Using default model: $(basename "$KOBOLD_MODEL")${NC}"
|
||||
else
|
||||
selected_model=$(ls -1 /mnt/data/ai/llm/*.gguf | sed -n "${model_num}p")
|
||||
if [[ -n "$selected_model" ]]; then
|
||||
KOBOLD_MODEL="$selected_model"
|
||||
echo -e "${GREEN}Selected model: $(basename "$KOBOLD_MODEL")${NC}"
|
||||
else
|
||||
echo -e "${RED}Invalid selection. Using default model.${NC}"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to start SillyTavern
|
||||
start_sillytavern() {
|
||||
echo -e "${YELLOW}Starting SillyTavern in screen session '${SILLYTAVERN_SCREEN}'...${NC}"
|
||||
screen -dmS "$SILLYTAVERN_SCREEN" bash -c "cd ~/SillyTavern && npm start -- --listen 0.0.0.0"
|
||||
sleep 2
|
||||
if check_screen "$SILLYTAVERN_SCREEN"; then
|
||||
echo -e "${GREEN}✓ SillyTavern started successfully!${NC}"
|
||||
echo -e "${BLUE} Access at: http://0.0.0.0:8000${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Failed to start SillyTavern${NC}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to start KoboldCPP
|
||||
start_koboldcpp() {
|
||||
select_model
|
||||
echo -e "${YELLOW}Starting KoboldCPP in screen session '${KOBOLD_SCREEN}'...${NC}"
|
||||
screen -dmS "$KOBOLD_SCREEN" bash -c "cd /mnt/data/ai/llm && ./koboldcpp-linux-x64 --model '$KOBOLD_MODEL' --host 0.0.0.0 --port 5001 --contextsize 8192 --gpulayers 999"
|
||||
sleep 2
|
||||
if check_screen "$KOBOLD_SCREEN"; then
|
||||
echo -e "${GREEN}✓ KoboldCPP started successfully!${NC}"
|
||||
echo -e "${BLUE} Model: $(basename "$KOBOLD_MODEL")${NC}"
|
||||
echo -e "${BLUE} Access at: http://0.0.0.0:5001${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Failed to start KoboldCPP${NC}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to stop a service
|
||||
stop_service() {
|
||||
local service=$1
|
||||
local screen_name=$2
|
||||
|
||||
echo -e "${YELLOW}Stopping ${service}...${NC}"
|
||||
screen -S "$screen_name" -X quit
|
||||
sleep 1
|
||||
if ! check_screen "$screen_name"; then
|
||||
echo -e "${GREEN}✓ ${service} stopped successfully${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Failed to stop ${service}${NC}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to show service status
|
||||
show_status() {
|
||||
echo -e "${CYAN}╔═══════════════════════════════════════╗${NC}"
|
||||
echo -e "${CYAN}║ Service Status Overview ║${NC}"
|
||||
echo -e "${CYAN}╚═══════════════════════════════════════╝${NC}"
|
||||
echo
|
||||
|
||||
local st_running=false
|
||||
local kc_running=false
|
||||
|
||||
# Check SillyTavern
|
||||
if check_screen "$SILLYTAVERN_SCREEN"; then
|
||||
st_running=true
|
||||
echo -e " ${GREEN}●${NC} SillyTavern: ${GREEN}Running${NC} (screen: ${SILLYTAVERN_SCREEN})"
|
||||
echo -e " ${BLUE}→ http://0.0.0.0:8000${NC}"
|
||||
else
|
||||
echo -e " ${RED}●${NC} SillyTavern: ${RED}Not running${NC}"
|
||||
fi
|
||||
|
||||
echo
|
||||
|
||||
# Check KoboldCPP
|
||||
if check_screen "$KOBOLD_SCREEN"; then
|
||||
kc_running=true
|
||||
echo -e " ${GREEN}●${NC} KoboldCPP: ${GREEN}Running${NC} (screen: ${KOBOLD_SCREEN})"
|
||||
echo -e " ${BLUE}→ http://0.0.0.0:5001${NC}"
|
||||
else
|
||||
echo -e " ${RED}●${NC} KoboldCPP: ${RED}Not running${NC}"
|
||||
fi
|
||||
|
||||
echo
|
||||
}
|
||||
|
||||
# Function to handle service management
|
||||
manage_services() {
|
||||
local st_running=$(check_screen "$SILLYTAVERN_SCREEN" && echo "true" || echo "false")
|
||||
local kc_running=$(check_screen "$KOBOLD_SCREEN" && echo "true" || echo "false")
|
||||
|
||||
# If both services are running
|
||||
if [[ "$st_running" == "true" ]] && [[ "$kc_running" == "true" ]]; then
|
||||
echo -e "${GREEN}Both services are running.${NC}"
|
||||
echo
|
||||
echo "1) Attach to SillyTavern"
|
||||
echo "2) Attach to KoboldCPP"
|
||||
echo "3) Restart SillyTavern"
|
||||
echo "4) Restart KoboldCPP"
|
||||
echo "5) Stop all services"
|
||||
echo "6) Exit"
|
||||
|
||||
read -p "Your choice (1-6): " choice
|
||||
|
||||
case $choice in
|
||||
1)
|
||||
echo -e "${BLUE}Attaching to SillyTavern... (Use Ctrl+A then D to detach)${NC}"
|
||||
sleep 1
|
||||
screen -r "$SILLYTAVERN_SCREEN"
|
||||
;;
|
||||
2)
|
||||
echo -e "${BLUE}Attaching to KoboldCPP... (Use Ctrl+A then D to detach)${NC}"
|
||||
sleep 1
|
||||
screen -r "$KOBOLD_SCREEN"
|
||||
;;
|
||||
3)
|
||||
stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
|
||||
echo
|
||||
start_sillytavern
|
||||
;;
|
||||
4)
|
||||
stop_service "KoboldCPP" "$KOBOLD_SCREEN"
|
||||
echo
|
||||
start_koboldcpp
|
||||
;;
|
||||
5)
|
||||
stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
|
||||
stop_service "KoboldCPP" "$KOBOLD_SCREEN"
|
||||
;;
|
||||
6)
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Invalid choice${NC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# If only SillyTavern is running
|
||||
elif [[ "$st_running" == "true" ]]; then
|
||||
echo -e "${YELLOW}Only SillyTavern is running.${NC}"
|
||||
echo
|
||||
echo "1) Attach to SillyTavern"
|
||||
echo "2) Start KoboldCPP"
|
||||
echo "3) Restart SillyTavern"
|
||||
echo "4) Stop SillyTavern"
|
||||
echo "5) Exit"
|
||||
|
||||
read -p "Your choice (1-5): " choice
|
||||
|
||||
case $choice in
|
||||
1)
|
||||
echo -e "${BLUE}Attaching to SillyTavern... (Use Ctrl+A then D to detach)${NC}"
|
||||
sleep 1
|
||||
screen -r "$SILLYTAVERN_SCREEN"
|
||||
;;
|
||||
2)
|
||||
start_koboldcpp
|
||||
;;
|
||||
3)
|
||||
stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
|
||||
echo
|
||||
start_sillytavern
|
||||
;;
|
||||
4)
|
||||
stop_service "SillyTavern" "$SILLYTAVERN_SCREEN"
|
||||
;;
|
||||
5)
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Invalid choice${NC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# If only KoboldCPP is running
|
||||
elif [[ "$kc_running" == "true" ]]; then
|
||||
echo -e "${YELLOW}Only KoboldCPP is running.${NC}"
|
||||
echo
|
||||
echo "1) Attach to KoboldCPP"
|
||||
echo "2) Start SillyTavern"
|
||||
echo "3) Restart KoboldCPP"
|
||||
echo "4) Stop KoboldCPP"
|
||||
echo "5) Exit"
|
||||
|
||||
read -p "Your choice (1-5): " choice
|
||||
|
||||
case $choice in
|
||||
1)
|
||||
echo -e "${BLUE}Attaching to KoboldCPP... (Use Ctrl+A then D to detach)${NC}"
|
||||
sleep 1
|
||||
screen -r "$KOBOLD_SCREEN"
|
||||
;;
|
||||
2)
|
||||
start_sillytavern
|
||||
;;
|
||||
3)
|
||||
stop_service "KoboldCPP" "$KOBOLD_SCREEN"
|
||||
echo
|
||||
start_koboldcpp
|
||||
;;
|
||||
4)
|
||||
stop_service "KoboldCPP" "$KOBOLD_SCREEN"
|
||||
;;
|
||||
5)
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Invalid choice${NC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# If no services are running
|
||||
else
|
||||
echo -e "${YELLOW}No services are running.${NC}"
|
||||
echo
|
||||
echo "1) Start both services"
|
||||
echo "2) Start SillyTavern only"
|
||||
echo "3) Start KoboldCPP only"
|
||||
echo "4) Exit"
|
||||
|
||||
read -p "Your choice (1-4): " choice
|
||||
|
||||
case $choice in
|
||||
1)
|
||||
start_sillytavern
|
||||
echo
|
||||
start_koboldcpp
|
||||
;;
|
||||
2)
|
||||
start_sillytavern
|
||||
;;
|
||||
3)
|
||||
start_koboldcpp
|
||||
;;
|
||||
4)
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Invalid choice${NC}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
# Main script
|
||||
echo -e "${BLUE}╔═══════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ LLM Services Manager ║${NC}"
|
||||
echo -e "${BLUE}╚═══════════════════════════════════════╝${NC}"
|
||||
echo
|
||||
|
||||
# Show status
|
||||
show_status
|
||||
|
||||
# Show separator and manage services
|
||||
echo -e "${CYAN}═══════════════════════════════════════${NC}"
|
||||
manage_services
|
||||
|
||||
echo
|
||||
echo -e "${BLUE}Quick reference:${NC}"
|
||||
echo "• List sessions: screen -ls"
|
||||
echo "• Attach: screen -r <name>"
|
||||
echo "• Detach: Ctrl+A then D"
|
||||
Reference in New Issue
Block a user