550 lines
22 KiB
Bash
Executable File
550 lines
22 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# VRAM/Context Detection Script
|
|
# Detects GPU VRAM, system RAM, and model context window to recommend
|
|
# a safe VRAM context window for task decomposition.
|
|
#
|
|
# Usage: ./vram_detect.sh [model_name]
|
|
# - If model_name is provided, looks up its context window
|
|
# - Otherwise, tries to detect from API config or config.md
|
|
|
|
set -uo pipefail # Don't exit on error - we want to continue even if detection fails
|
|
|
|
# ─── GPU VRAM Detection ───
|
|
detect_gpu_vram() {
|
|
local total_vram_kb=0
|
|
local vram_per_gpu_kb=0
|
|
local num_gpus=0
|
|
|
|
# Try nvidia-smi first (NVIDIA GPUs)
|
|
if command -v nvidia-smi &>/dev/null; then
|
|
local vram_kb
|
|
# Use timeout to avoid hanging on nvidia-smi (e.g., driver not loaded)
|
|
vram_kb=$(timeout 5 nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ' || true)
|
|
# Validate that vram_kb is a positive number
|
|
if [[ -n "$vram_kb" && "$vram_kb" =~ ^[0-9]+$ && "$vram_kb" -gt 0 ]]; then
|
|
total_vram_kb=$((vram_kb * 1024)) # MB → KB
|
|
vram_per_gpu_kb=$((total_vram_kb / (num_gpus+1)))
|
|
num_gpus=1
|
|
echo "GPU: NVIDIA (nvidia-smi available)"
|
|
echo "VRAM per GPU: $((vram_kb / 1024))GB ($vram_kb MB)"
|
|
else
|
|
echo "GPU: NVIDIA (nvidia-smi available but driver not responding)"
|
|
fi
|
|
fi
|
|
|
|
# Fallback: lspci
|
|
if [[ $total_vram_kb -eq 0 && $num_gpus -eq 0 ]]; then
|
|
local gpu_info
|
|
gpu_info=$(lspci 2>/dev/null | grep -i -E 'VGA|3D|Display' | head -5)
|
|
if [[ -n "$gpu_info" ]]; then
|
|
echo "GPU detected: $gpu_info"
|
|
# Try to get VRAM from lspci -vnn memory regions
|
|
# GPUs show VRAM as Memory regions in lspci
|
|
# Parse patterns like: Memory at f800000000 (64-bit, prefetchable) [size=256M]
|
|
local total_vram_mb=0
|
|
while IFS= read -r line; do
|
|
# Extract the size value from [size=256M] pattern
|
|
local size_num
|
|
size_num=$(echo "$line" | grep -oE '[0-9]+(M|G|K)' | head -1 || true)
|
|
if [[ -n "$size_num" ]]; then
|
|
local size_val
|
|
size_val=$(echo "$size_num" | grep -oE '[0-9]+')
|
|
local size_unit
|
|
size_unit=$(echo "$size_num" | grep -oE '(M|G|K)')
|
|
if [[ -n "$size_val" && -n "$size_unit" ]]; then
|
|
case "$size_unit" in
|
|
M) total_vram_mb=$((total_vram_mb + size_val)) ;;
|
|
G) total_vram_mb=$((total_vram_mb + size_val * 1024)) ;;
|
|
K) total_vram_mb=$((total_vram_mb + size_val / 1024)) ;;
|
|
esac
|
|
fi
|
|
fi
|
|
done < <(lspci -vnn 2>/dev/null | grep -i -A 15 -E 'VGA|3D|Display' | grep -i 'Memory at')
|
|
if [[ $total_vram_mb -gt 0 ]]; then
|
|
local total_vram_gb=$((total_vram_mb / 1024))
|
|
local total_vram_mb_remain=$((total_vram_mb % 1024))
|
|
echo "VRAM: $total_vram_mb MB ($total_vram_gb GB $total_vram_mb_remain MB)"
|
|
else
|
|
echo "VRAM: Could not determine from lspci"
|
|
fi
|
|
# Check for AMD GPU via amdgpu sysfs
|
|
if lspci -vnn 2>/dev/null | grep -qi 'amd\|ati'; then
|
|
local amdgpu_info
|
|
amdgpu_info=$(ls /sys/kernel/debug/amdgpu/ 2>/dev/null | head -1)
|
|
if [[ -n "$amdgpu_info" ]]; then
|
|
local vram_total
|
|
vram_total=$(cat /sys/kernel/debug/amdgpu/${amdgpu_info}/vram_total 2>/dev/null || echo 0)
|
|
if [[ "$vram_total" -gt 0 ]]; then
|
|
local vram_gb=$((vram_total / 1024 / 1024 / 1024))
|
|
local vram_mb=$((vram_total / 1024 / 1024))
|
|
echo "AMD GPU VRAM: ${vram_gb}GB (${vram_mb}MB)"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
echo "Total VRAM: $((total_vram_kb / 1024 / 1024))GB"
|
|
echo "VRAM per GPU: $((vram_per_gpu_kb / 1024 / 1024))GB"
|
|
echo "Num GPUs: $num_gpus"
|
|
}
|
|
|
|
# ─── System RAM Detection ───
|
|
detect_ram() {
|
|
local total_kb=0
|
|
local available_kb=0
|
|
|
|
if [[ -f /proc/meminfo ]]; then
|
|
total_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}')
|
|
available_kb=$(grep MemAvailable /proc/meminfo | awk '{print $2}')
|
|
if [[ $total_kb -gt 0 ]]; then
|
|
echo "RAM: $((total_kb / 1024 / 1024))GB total, $((available_kb / 1024 / 1024))GB available"
|
|
echo "$available_kb $total_kb"
|
|
fi
|
|
elif command -v sysctl &>/dev/null; then
|
|
total_kb=$(sysctl -n hw.memsize 2>/dev/null | awk '{print $1 / 1024}')
|
|
if [[ -n "$total_kb" && "$total_kb" -gt 0 ]]; then
|
|
echo "RAM: $((total_kb / 1024))GB total"
|
|
echo "$total_kb $total_kb" # Assume all available
|
|
fi
|
|
else
|
|
echo "RAM: Could not detect"
|
|
echo "0 0"
|
|
fi
|
|
}
|
|
|
|
# ─── Model Context Window Detection ───
|
|
detect_model_context() {
|
|
local model_name="$1"
|
|
local context_kb=0
|
|
|
|
# If model name provided, look it up
|
|
if [[ -n "$model_name" ]]; then
|
|
case "$model_name" in
|
|
gpt-4o|gpt-4o-2024-05-13|gpt-4o-2024-08-06)
|
|
context_kb=128000; echo "Model: $model_name"
|
|
echo "Context window: 128k tokens"
|
|
;;
|
|
gpt-4o-mini|gpt-4o-mini-2024-07-18)
|
|
context_kb=128000; echo "Model: $model_name"
|
|
echo "Context window: 128k tokens"
|
|
;;
|
|
gpt-4-turbo|gpt-4-turbo-2024-04-09)
|
|
context_kb=128000; echo "Model: $model_name"
|
|
echo "Context window: 128k tokens"
|
|
;;
|
|
gpt-4|gpt-4-0125-preview|gpt-4-1106-preview)
|
|
context_kb=128000; echo "Model: $model_name"
|
|
echo "Context window: 128k tokens"
|
|
;;
|
|
claude-3-5-sonnet|claude-3-5-sonnet-20241022)
|
|
context_kb=200000; echo "Model: $model_name"
|
|
echo "Context window: 200k tokens"
|
|
;;
|
|
claude-3-5-haiku|claude-3-5-haiku-20241022)
|
|
context_kb=200000; echo "Model: $model_name"
|
|
echo "Context window: 200k tokens"
|
|
;;
|
|
claude-3-opus|claude-3-opus-20240229)
|
|
context_kb=200000; echo "Model: $model_name"
|
|
echo "Context window: 200k tokens"
|
|
;;
|
|
claude-3-sonnet|claude-3-sonnet-20240229)
|
|
context_kb=200000; echo "Model: $model_name"
|
|
echo "Context window: 200k tokens"
|
|
;;
|
|
claude-3-haiku|claude-3-haiku-20240307)
|
|
context_kb=200000; echo "Model: $model_name"
|
|
echo "Context window: 200k tokens"
|
|
;;
|
|
claude-2|claude-2.1)
|
|
context_kb=200000; echo "Model: $model_name"
|
|
echo "Context window: 200k tokens"
|
|
;;
|
|
*)
|
|
echo "Model: $model_name (unknown context window)"
|
|
echo "0"
|
|
;;
|
|
esac
|
|
echo "$context_kb"
|
|
return
|
|
fi
|
|
|
|
# Try to detect from config.md (global framework model settings)
|
|
local project_dir="${1:-.}"
|
|
local config_md="${HOME}/.agent-framework/config.md"
|
|
local model_from_config=""
|
|
local override_context=""
|
|
if [[ -f "$config_md" ]]; then
|
|
# Check for model name
|
|
model_from_config=$(grep -i "model:" "$config_md" 2>/dev/null | grep -v "#" | grep -v "model_context" | grep -v "override" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]')
|
|
# Check for override context window
|
|
override_context=$(grep -i "override context" "$config_md" 2>/dev/null | grep -v "#" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]')
|
|
fi
|
|
|
|
# If config.md specifies a model, use it
|
|
if [[ -n "$model_from_config" ]]; then
|
|
echo "Found model in config.md: $model_from_config"
|
|
# Use override context window if specified in config.md
|
|
if [[ -n "$override_context" && "$override_context" != "auto" ]]; then
|
|
echo "Using override context window from config.md: $override_context"
|
|
local override_kb
|
|
override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true)
|
|
if [[ -n "$override_kb" ]]; then
|
|
echo "Context window: ${override_context} tokens (override)"
|
|
echo "$((override_kb * 1000))"
|
|
return
|
|
fi
|
|
fi
|
|
detect_model_context "$model_from_config"
|
|
return
|
|
fi
|
|
|
|
# Try to detect from AGENT.md (project-level model override)
|
|
local agent_md="${project_dir}/.agent-framework/AGENT.md"
|
|
if [[ -f "$agent_md" ]]; then
|
|
local model_line
|
|
model_line=$(grep -i "model" "$agent_md" 2>/dev/null | grep -v "#" | grep -v "target" | grep -v "headroom" | grep -v "peak" | grep -v "Auto-detect" | head -1)
|
|
if [[ -n "$model_line" ]]; then
|
|
echo "Found model in AGENT.md: $model_line"
|
|
# Extract model name from the line
|
|
local model
|
|
model=$(echo "$model_line" | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]')
|
|
if [[ -n "$model" ]]; then
|
|
# Use override context window if specified in config.md
|
|
if [[ -n "$override_context" && "$override_context" != "auto" ]]; then
|
|
echo "Using override context window from config.md: $override_context"
|
|
# Convert override_context to kb (e.g., 128k -> 128000, 200k -> 200000)
|
|
local override_kb
|
|
override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true)
|
|
if [[ -n "$override_kb" ]]; then
|
|
echo "Context window: ${override_context} tokens (override)"
|
|
echo "$((override_kb * 1000))"
|
|
return
|
|
fi
|
|
fi
|
|
detect_model_context "$model"
|
|
return
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Try to detect from common API config files
|
|
local config_files=(
|
|
".env"
|
|
".env.local"
|
|
"config.yaml"
|
|
"config.yml"
|
|
"config.json"
|
|
"settings.yaml"
|
|
".agent-framework/config.yaml"
|
|
".agent-framework/config.json"
|
|
)
|
|
|
|
for config_file in "${config_files[@]}"; do
|
|
local abs_file=""
|
|
for candidate in "${project_dir}/${config_file}" "${project_dir}/.agent-framework/${config_file}"; do
|
|
if [[ -f "$candidate" ]]; then
|
|
abs_file="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ -n "$abs_file" ]]; then
|
|
local model
|
|
model=$(grep -i "model" "$abs_file" 2>/dev/null | grep -v "#" | grep -v "context" | grep -v "max_tokens" | grep -v "temperature" | grep -v "stream" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]')
|
|
if [[ -n "$model" ]]; then
|
|
echo "Found model in $abs_file: $model"
|
|
# Use override context window if specified in config.md
|
|
if [[ -n "$override_context" && "$override_context" != "auto" ]]; then
|
|
echo "Using override context window from config.md: $override_context"
|
|
local override_kb
|
|
override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true)
|
|
if [[ -n "$override_kb" ]]; then
|
|
echo "Context window: ${override_context} tokens (override)"
|
|
echo "$((override_kb * 1000))"
|
|
return
|
|
fi
|
|
fi
|
|
detect_model_context "$model"
|
|
return
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo "Model: Unknown (could not detect from AGENT.md or config files)"
|
|
echo "0"
|
|
}
|
|
|
|
# ─── Agent Framework Overhead Calculation ───
|
|
calculate_overhead() {
|
|
local project_dir="${1:-.}"
|
|
local overhead_tokens=0
|
|
|
|
# Count tokens for the framework files that are loaded during orchestration
|
|
# These are the files loaded during the most common phase (orchestration):
|
|
# AGENT.md + RULES.md + workflow.md + orchestrate.md
|
|
# Note: other phase files (decompose.md, implement.md, etc.) are only loaded during
|
|
# their specific phases, so they don't contribute to the peak context during orchestration.
|
|
local framework_files=(
|
|
"${project_dir}/.agent-framework/AGENT.md"
|
|
"${project_dir}/.agent-framework/RULES.md"
|
|
"${project_dir}/.agent-framework/prompts/workflow.md"
|
|
"${project_dir}/.agent-framework/prompts/orchestrate.md"
|
|
)
|
|
# Fallback: check home directory if project dir doesn't have framework
|
|
if [[ ! -f "${project_dir}/.agent-framework/AGENT.md" ]]; then
|
|
framework_files=(
|
|
"${HOME}/.agent-framework/AGENT.md"
|
|
"${HOME}/.agent-framework/RULES.md"
|
|
"${HOME}/.agent-framework/prompts/workflow.md"
|
|
"${HOME}/.agent-framework/prompts/orchestrate.md"
|
|
)
|
|
fi
|
|
|
|
for file in "${framework_files[@]}"; do
|
|
if [[ -f "$file" ]]; then
|
|
# Rough estimate: 1 token ≈ 4 characters (English text)
|
|
local chars
|
|
chars=$(wc -c < "$file" 2>/dev/null || echo 0)
|
|
local tokens=$((chars / 4))
|
|
overhead_tokens=$((overhead_tokens + tokens))
|
|
echo " ${file##*/}: ~${tokens} tokens"
|
|
fi
|
|
done
|
|
|
|
echo "Framework overhead: ~${overhead_tokens} tokens"
|
|
echo "$overhead_tokens"
|
|
}
|
|
|
|
# ─── Recommendation Engine ───
|
|
recommend_context() {
|
|
local gpu_vram_gb="$1"
|
|
local ram_gb="$2"
|
|
local model_context_kb="$3"
|
|
local overhead_tokens="$4"
|
|
|
|
# Read VRAM config from config.md if it exists
|
|
local config_md="${HOME}/.agent-framework/config.md"
|
|
local auto_detect="Yes"
|
|
local target_context_kb=0
|
|
local override_headroom=25
|
|
local override_max_peak_kb=0
|
|
if [[ -f "$config_md" ]]; then
|
|
auto_detect=$(grep -i "auto-detect:" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]' || true)
|
|
target_context_kb=$(grep -i "target.*context" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true)
|
|
override_headroom=$(grep -i "headroom" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true)
|
|
override_max_peak_kb=$(grep -i "max peak" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true)
|
|
fi
|
|
|
|
# If auto-detect is disabled, use the manually specified values
|
|
if [[ -n "$auto_detect" && "$auto_detect" == "No" ]]; then
|
|
if [[ -n "$target_context_kb" ]]; then
|
|
local max_peak_kb=${override_max_peak_kb:-0}
|
|
if [[ $max_peak_kb -eq 0 && $headroom_pct -gt 0 ]]; then
|
|
max_peak_kb=$((target_context_kb * (100 - headroom_pct) / 100))
|
|
fi
|
|
echo "$headroom_pct"
|
|
echo "$target_context_kb"
|
|
echo "$max_peak_kb"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
local recommended_kb=0
|
|
local headroom_pct=${override_headroom:-25} # Use override from config.md, or default to 25%
|
|
|
|
# Output headroom_pct first (for parent to read)
|
|
# Then output recommended_kb
|
|
# Then output max_peak_kb (only for manual mode)
|
|
echo "$headroom_pct"
|
|
|
|
# Recommendation logic:
|
|
# 1. If GPU VRAM >= 4GB: use VRAM (practical for local inference)
|
|
# 2. If model context window is available: use it (for API inference)
|
|
# 3. If GPU VRAM < 4GB but > 0: use RAM (VRAM too small for local inference)
|
|
# 4. If no GPU VRAM and no model: use RAM as fallback
|
|
|
|
# If GPU VRAM >= 4GB, base it on VRAM
|
|
if [[ $gpu_vram_gb -ge 4 ]]; then
|
|
# Rule of thumb: 1GB VRAM ≈ 4k tokens for local LLMs
|
|
# But we need to leave room for the model itself
|
|
# For a model, each ~8k context tokens takes about ~3-5MB of GPU VRAM
|
|
# So VRAM available for context = VRAM - model size - agent overhead
|
|
# Conservative: 1GB VRAM ≈ 2k context tokens
|
|
local vram_context_kb=$((gpu_vram_gb * 2000))
|
|
|
|
# Leave headroom for the model itself and agent overhead
|
|
recommended_kb=$((vram_context_kb * (100 - headroom_pct) / 100))
|
|
# If model context window is available, use it (for API inference)
|
|
elif [[ $model_context_kb -gt 0 ]]; then
|
|
# For API-based, we're limited by the model's context window
|
|
# But we don't want to use the full window due to overhead
|
|
recommended_kb=$((model_context_kb * (100 - headroom_pct) / 100))
|
|
# Fallback: use RAM to estimate
|
|
else
|
|
# Moderate estimate for low-VRAM systems where VRAM is too small for local inference
|
|
# but RAM is available. Use 0.75k tokens per GB of RAM as a moderate estimate.
|
|
# This balances between being too conservative (0.5k/GB) and too generous (1k/GB).
|
|
local ram_context_kb=$((ram_gb * 750))
|
|
recommended_kb=$((ram_context_kb * (100 - headroom_pct) / 100))
|
|
fi
|
|
|
|
# Subtract framework overhead
|
|
local net_kb=$((recommended_kb - overhead_tokens))
|
|
if [[ $net_kb -lt 0 ]]; then
|
|
net_kb=0
|
|
fi
|
|
|
|
# Output: headroom_pct, recommended_kb
|
|
echo "$net_kb"
|
|
}
|
|
|
|
# ─── Main ───
|
|
main() {
|
|
local model_name=""
|
|
local project_dir="."
|
|
|
|
# Parse arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--model|-m)
|
|
model_name="$2"
|
|
shift 2
|
|
;;
|
|
--project|-p)
|
|
project_dir="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
# Could be model name as first argument
|
|
if [[ -z "$model_name" ]]; then
|
|
model_name="$1"
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
echo "=== VRAM / Context Detection ==="
|
|
echo ""
|
|
|
|
# Detect GPU VRAM
|
|
echo "--- GPU VRAM ---"
|
|
detect_gpu_vram
|
|
local gpu_vram_kb=0
|
|
local gpu_vram_gb=0
|
|
# Use timeout to avoid hanging
|
|
gpu_vram_kb=$(timeout 5 nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ' || true)
|
|
# Validate that vram_kb is a positive number
|
|
if [[ -z "$gpu_vram_kb" || ! "$gpu_vram_kb" =~ ^[0-9]+$ || "$gpu_vram_kb" -le 0 ]]; then
|
|
gpu_vram_kb=0
|
|
fi
|
|
# If nvidia-smi didn't work, try to detect from lspci (AMD GPUs)
|
|
if [[ $gpu_vram_kb -eq 0 ]]; then
|
|
echo " nvidia-smi failed, checking lspci for AMD GPU VRAM..."
|
|
local total_vram_mb=0
|
|
while IFS= read -r line; do
|
|
local size_num
|
|
size_num=$(echo "$line" | grep -oE '[0-9]+(M|G|K)' | head -1 || true)
|
|
if [[ -n "$size_num" ]]; then
|
|
local size_val
|
|
size_val=$(echo "$size_num" | grep -oE '[0-9]+')
|
|
local size_unit
|
|
size_unit=$(echo "$size_num" | grep -oE '(M|G|K)')
|
|
if [[ -n "$size_val" && -n "$size_unit" ]]; then
|
|
case "$size_unit" in
|
|
M) total_vram_mb=$((total_vram_mb + size_val)) ;;
|
|
G) total_vram_mb=$((total_vram_mb + size_val * 1024)) ;;
|
|
K) total_vram_mb=$((total_vram_mb + size_val / 1024)) ;;
|
|
esac
|
|
fi
|
|
fi
|
|
done < <(lspci -vnn 2>/dev/null | grep -i -A 15 -E 'VGA|3D|Display' | grep -i 'Memory at')
|
|
if [[ $total_vram_mb -gt 0 ]]; then
|
|
gpu_vram_kb=$((total_vram_mb * 1024))
|
|
gpu_vram_gb=$((total_vram_mb / 1024))
|
|
echo " AMD GPU VRAM from lspci: ${gpu_vram_gb}GB ($total_vram_mb MB)"
|
|
else
|
|
echo " No VRAM found from lspci"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Detect RAM
|
|
echo "--- RAM ---"
|
|
detect_ram
|
|
local ram_kb
|
|
ram_kb=$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print $2}' || echo 0)
|
|
local ram_gb=$((ram_kb / 1024 / 1024))
|
|
echo ""
|
|
|
|
# Detect model context window
|
|
echo "--- Model Context Window ---"
|
|
detect_model_context "$model_name"
|
|
local model_context_kb
|
|
model_context_kb=$(detect_model_context "$model_name" | tail -1)
|
|
echo ""
|
|
|
|
# Calculate framework overhead
|
|
echo "--- Framework Overhead ---"
|
|
calculate_overhead "$project_dir"
|
|
local overhead_tokens
|
|
overhead_tokens=$(calculate_overhead "$project_dir" | tail -1)
|
|
echo ""
|
|
|
|
# Recommend context window (also outputs headroom_pct and recommended_kb)
|
|
echo "--- Recommendation ---"
|
|
local recommendation_output
|
|
recommendation_output=$(recommend_context "$gpu_vram_gb" "$ram_gb" "$model_context_kb" "$overhead_tokens")
|
|
local line_count
|
|
line_count=$(echo "$recommendation_output" | wc -l)
|
|
local recommended_kb
|
|
local headroom_pct
|
|
local max_peak_kb
|
|
if [[ $line_count -ge 3 ]]; then
|
|
# Manual mode: outputs headroom_pct, recommended_kb, max_peak_kb
|
|
headroom_pct=$(echo "$recommendation_output" | head -1 | tr -d '[:space:]')
|
|
recommended_kb=$(echo "$recommendation_output" | sed -n '2p' | tr -d '[:space:]')
|
|
max_peak_kb=$(echo "$recommendation_output" | tail -1 | tr -d '[:space:]')
|
|
else
|
|
# Auto-detect mode: outputs headroom_pct, recommended_kb
|
|
headroom_pct=$(echo "$recommendation_output" | head -1 | tr -d '[:space:]')
|
|
recommended_kb=$(echo "$recommendation_output" | tail -1 | tr -d '[:space:]')
|
|
# Calculate max peak context based on headroom
|
|
max_peak_kb=$((recommended_kb * (100 - headroom_pct) / 100))
|
|
fi
|
|
|
|
# Convert to human-readable
|
|
local recommended_k
|
|
if [[ $recommended_kb -gt 0 ]]; then
|
|
recommended_k=$((recommended_kb / 1000))
|
|
else
|
|
recommended_k=8 # Default fallback
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== Recommended Configuration ==="
|
|
echo "Target context: ${recommended_k}k tokens"
|
|
echo "Headroom: ${headroom_pct}%"
|
|
echo "Max peak context per sub-task: $((recommended_k * (100 - headroom_pct) / 100))k tokens"
|
|
|
|
# Output as JSON for programmatic use
|
|
echo ""
|
|
echo "=== JSON Output ==="
|
|
cat <<EOF
|
|
{
|
|
"gpu_vram_gb": $gpu_vram_gb,
|
|
"ram_gb": $ram_gb,
|
|
"model_context_kb": $model_context_kb,
|
|
"framework_overhead_tokens": $overhead_tokens,
|
|
"recommended_kb": $recommended_kb,
|
|
"recommended_k": $recommended_k,
|
|
"headroom": 0.25,
|
|
"max_peak_context_kb": $max_peak_kb
|
|
}
|
|
EOF
|
|
}
|
|
|
|
main "$@"
|