#!/usr/bin/env bash # VRAM/Context Detection Script # Detects GPU VRAM, system RAM, and model context window to recommend # a safe VRAM context window for task decomposition. # # Usage: ./vram_detect.sh [model_name] # - If model_name is provided, looks up its context window # - Otherwise, tries to detect from API config or config.md set -uo pipefail # Don't exit on error - we want to continue even if detection fails # ─── GPU VRAM Detection ─── detect_gpu_vram() { local total_vram_kb=0 local vram_per_gpu_kb=0 local num_gpus=0 # Try nvidia-smi first (NVIDIA GPUs) if command -v nvidia-smi &>/dev/null; then local vram_kb # Use timeout to avoid hanging on nvidia-smi (e.g., driver not loaded) vram_kb=$(timeout 5 nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ' || true) # Validate that vram_kb is a positive number if [[ -n "$vram_kb" && "$vram_kb" =~ ^[0-9]+$ && "$vram_kb" -gt 0 ]]; then total_vram_kb=$((vram_kb * 1024)) # MB → KB vram_per_gpu_kb=$((total_vram_kb / (num_gpus+1))) num_gpus=1 echo "GPU: NVIDIA (nvidia-smi available)" echo "VRAM per GPU: $((vram_kb / 1024))GB ($vram_kb MB)" else echo "GPU: NVIDIA (nvidia-smi available but driver not responding)" fi fi # Fallback: lspci if [[ $total_vram_kb -eq 0 && $num_gpus -eq 0 ]]; then local gpu_info gpu_info=$(lspci 2>/dev/null | grep -i -E 'VGA|3D|Display' | head -5) if [[ -n "$gpu_info" ]]; then echo "GPU detected: $gpu_info" # Try to get VRAM from lspci -vnn memory regions # GPUs show VRAM as Memory regions in lspci # Parse patterns like: Memory at f800000000 (64-bit, prefetchable) [size=256M] local total_vram_mb=0 while IFS= read -r line; do # Extract the size value from [size=256M] pattern local size_num size_num=$(echo "$line" | grep -oE '[0-9]+(M|G|K)' | head -1 || true) if [[ -n "$size_num" ]]; then local size_val size_val=$(echo "$size_num" | grep -oE '[0-9]+') local size_unit size_unit=$(echo "$size_num" | grep -oE '(M|G|K)') if [[ -n "$size_val" && -n "$size_unit" ]]; then case "$size_unit" in M) total_vram_mb=$((total_vram_mb + size_val)) ;; G) total_vram_mb=$((total_vram_mb + size_val * 1024)) ;; K) total_vram_mb=$((total_vram_mb + size_val / 1024)) ;; esac fi fi done < <(lspci -vnn 2>/dev/null | grep -i -A 15 -E 'VGA|3D|Display' | grep -i 'Memory at') if [[ $total_vram_mb -gt 0 ]]; then local total_vram_gb=$((total_vram_mb / 1024)) local total_vram_mb_remain=$((total_vram_mb % 1024)) echo "VRAM: $total_vram_mb MB ($total_vram_gb GB $total_vram_mb_remain MB)" else echo "VRAM: Could not determine from lspci" fi # Check for AMD GPU via amdgpu sysfs if lspci -vnn 2>/dev/null | grep -qi 'amd\|ati'; then local amdgpu_info amdgpu_info=$(ls /sys/kernel/debug/amdgpu/ 2>/dev/null | head -1) if [[ -n "$amdgpu_info" ]]; then local vram_total vram_total=$(cat /sys/kernel/debug/amdgpu/${amdgpu_info}/vram_total 2>/dev/null || echo 0) if [[ "$vram_total" -gt 0 ]]; then local vram_gb=$((vram_total / 1024 / 1024 / 1024)) local vram_mb=$((vram_total / 1024 / 1024)) echo "AMD GPU VRAM: ${vram_gb}GB (${vram_mb}MB)" fi fi fi fi fi echo "Total VRAM: $((total_vram_kb / 1024 / 1024))GB" echo "VRAM per GPU: $((vram_per_gpu_kb / 1024 / 1024))GB" echo "Num GPUs: $num_gpus" } # ─── System RAM Detection ─── detect_ram() { local total_kb=0 local available_kb=0 if [[ -f /proc/meminfo ]]; then total_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}') available_kb=$(grep MemAvailable /proc/meminfo | awk '{print $2}') if [[ $total_kb -gt 0 ]]; then echo "RAM: $((total_kb / 1024 / 1024))GB total, $((available_kb / 1024 / 1024))GB available" echo "$available_kb $total_kb" fi elif command -v sysctl &>/dev/null; then total_kb=$(sysctl -n hw.memsize 2>/dev/null | awk '{print $1 / 1024}') if [[ -n "$total_kb" && "$total_kb" -gt 0 ]]; then echo "RAM: $((total_kb / 1024))GB total" echo "$total_kb $total_kb" # Assume all available fi else echo "RAM: Could not detect" echo "0 0" fi } # ─── Model Context Window Detection ─── detect_model_context() { local model_name="$1" local context_kb=0 # If model name provided, look it up if [[ -n "$model_name" ]]; then case "$model_name" in gpt-4o|gpt-4o-2024-05-13|gpt-4o-2024-08-06) context_kb=128000; echo "Model: $model_name" echo "Context window: 128k tokens" ;; gpt-4o-mini|gpt-4o-mini-2024-07-18) context_kb=128000; echo "Model: $model_name" echo "Context window: 128k tokens" ;; gpt-4-turbo|gpt-4-turbo-2024-04-09) context_kb=128000; echo "Model: $model_name" echo "Context window: 128k tokens" ;; gpt-4|gpt-4-0125-preview|gpt-4-1106-preview) context_kb=128000; echo "Model: $model_name" echo "Context window: 128k tokens" ;; claude-3-5-sonnet|claude-3-5-sonnet-20241022) context_kb=200000; echo "Model: $model_name" echo "Context window: 200k tokens" ;; claude-3-5-haiku|claude-3-5-haiku-20241022) context_kb=200000; echo "Model: $model_name" echo "Context window: 200k tokens" ;; claude-3-opus|claude-3-opus-20240229) context_kb=200000; echo "Model: $model_name" echo "Context window: 200k tokens" ;; claude-3-sonnet|claude-3-sonnet-20240229) context_kb=200000; echo "Model: $model_name" echo "Context window: 200k tokens" ;; claude-3-haiku|claude-3-haiku-20240307) context_kb=200000; echo "Model: $model_name" echo "Context window: 200k tokens" ;; claude-2|claude-2.1) context_kb=200000; echo "Model: $model_name" echo "Context window: 200k tokens" ;; *) echo "Model: $model_name (unknown context window)" echo "0" ;; esac echo "$context_kb" return fi # Try to detect from config.md (global framework model settings) local project_dir="${1:-.}" local config_md="${HOME}/.automaton/config.md" local model_from_config="" local override_context="" if [[ -f "$config_md" ]]; then # Check for model name model_from_config=$(grep -i "model:" "$config_md" 2>/dev/null | grep -v "#" | grep -v "model_context" | grep -v "override" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]') # Check for override context window override_context=$(grep -i "override context" "$config_md" 2>/dev/null | grep -v "#" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]') fi # If config.md specifies a model, use it if [[ -n "$model_from_config" ]]; then echo "Found model in config.md: $model_from_config" # Use override context window if specified in config.md if [[ -n "$override_context" && "$override_context" != "auto" ]]; then echo "Using override context window from config.md: $override_context" local override_kb override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true) if [[ -n "$override_kb" ]]; then echo "Context window: ${override_context} tokens (override)" echo "$((override_kb * 1000))" return fi fi detect_model_context "$model_from_config" return fi # Try to detect from .agent.md (project-level model override) local agent_md="${project_dir}/.automaton/.agent.md" if [[ -f "$agent_md" ]]; then local model_line model_line=$(grep -i "model" "$agent_md" 2>/dev/null | grep -v "#" | grep -v "target" | grep -v "headroom" | grep -v "peak" | grep -v "Auto-detect" | head -1) if [[ -n "$model_line" ]]; then echo "Found model in .agent.md: $model_line" # Extract model name from the line local model model=$(echo "$model_line" | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]') if [[ -n "$model" ]]; then # Use override context window if specified in config.md if [[ -n "$override_context" && "$override_context" != "auto" ]]; then echo "Using override context window from config.md: $override_context" # Convert override_context to kb (e.g., 128k -> 128000, 200k -> 200000) local override_kb override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true) if [[ -n "$override_kb" ]]; then echo "Context window: ${override_context} tokens (override)" echo "$((override_kb * 1000))" return fi fi detect_model_context "$model" return fi fi fi # Try to detect from common API config files local config_files=( ".env" ".env.local" "config.yaml" "config.yml" "config.json" "settings.yaml" ".automaton/config.yaml" ".automaton/config.json" ) for config_file in "${config_files[@]}"; do local abs_file="" for candidate in "${project_dir}/${config_file}" "${project_dir}/.automaton/${config_file}"; do if [[ -f "$candidate" ]]; then abs_file="$candidate" break fi done if [[ -n "$abs_file" ]]; then local model model=$(grep -i "model" "$abs_file" 2>/dev/null | grep -v "#" | grep -v "context" | grep -v "max_tokens" | grep -v "temperature" | grep -v "stream" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]') if [[ -n "$model" ]]; then echo "Found model in $abs_file: $model" # Use override context window if specified in config.md if [[ -n "$override_context" && "$override_context" != "auto" ]]; then echo "Using override context window from config.md: $override_context" local override_kb override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true) if [[ -n "$override_kb" ]]; then echo "Context window: ${override_context} tokens (override)" echo "$((override_kb * 1000))" return fi fi detect_model_context "$model" return fi fi done echo "Model: Unknown (could not detect from .agent.md or config files)" echo "0" } # ─── Agent Framework Overhead Calculation ─── calculate_overhead() { local project_dir="${1:-.}" local overhead_tokens=0 # Count tokens for the framework files that are loaded during orchestration # These are the files loaded during the most common phase (orchestration): # .agent.md + .rules.md + workflow.md + orchestrate.md # Note: other phase files (decompose.md, implement.md, etc.) are only loaded during # their specific phases, so they don't contribute to the peak context during orchestration. local framework_files=( "${project_dir}/.automaton/.agent.md" "${project_dir}/.automaton/.rules.md" "${project_dir}/.automaton/prompts/workflow.md" "${project_dir}/.automaton/prompts/orchestrate.md" ) # Fallback: check home directory if project dir doesn't have framework if [[ ! -f "${project_dir}/.automaton/.agent.md" ]]; then framework_files=( "${HOME}/.automaton/.agent.md" "${HOME}/.automaton/.rules.md" "${HOME}/.automaton/prompts/workflow.md" "${HOME}/.automaton/prompts/orchestrate.md" ) fi for file in "${framework_files[@]}"; do if [[ -f "$file" ]]; then # Rough estimate: 1 token ≈ 4 characters (English text) local chars chars=$(wc -c < "$file" 2>/dev/null || echo 0) local tokens=$((chars / 4)) overhead_tokens=$((overhead_tokens + tokens)) echo " ${file##*/}: ~${tokens} tokens" fi done echo "Framework overhead: ~${overhead_tokens} tokens" echo "$overhead_tokens" } # ─── Recommendation Engine ─── recommend_context() { local gpu_vram_gb="$1" local ram_gb="$2" local model_context_kb="$3" local overhead_tokens="$4" # Read VRAM config from config.md if it exists local config_md="${HOME}/.automaton/config.md" local auto_detect="Yes" local target_context_kb=0 local override_headroom=25 local override_max_peak_kb=0 if [[ -f "$config_md" ]]; then auto_detect=$(grep -i "auto-detect:" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]' || true) target_context_kb=$(grep -i "target.*context" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true) override_headroom=$(grep -i "headroom" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true) override_max_peak_kb=$(grep -i "max peak" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true) fi # If auto-detect is disabled, use the manually specified values if [[ -n "$auto_detect" && "$auto_detect" == "No" ]]; then if [[ -n "$target_context_kb" ]]; then local max_peak_kb=${override_max_peak_kb:-0} if [[ $max_peak_kb -eq 0 && $headroom_pct -gt 0 ]]; then max_peak_kb=$((target_context_kb * (100 - headroom_pct) / 100)) fi echo "$headroom_pct" echo "$target_context_kb" echo "$max_peak_kb" return fi fi local recommended_kb=0 local headroom_pct=${override_headroom:-25} # Use override from config.md, or default to 25% # Output headroom_pct first (for parent to read) # Then output recommended_kb # Then output max_peak_kb (only for manual mode) echo "$headroom_pct" # Recommendation logic: # 1. If GPU VRAM >= 4GB: use VRAM (practical for local inference) # 2. If model context window is available: use it (for API inference) # 3. If GPU VRAM < 4GB but > 0: use RAM (VRAM too small for local inference) # 4. If no GPU VRAM and no model: use RAM as fallback # If GPU VRAM >= 4GB, base it on VRAM if [[ $gpu_vram_gb -ge 4 ]]; then # Rule of thumb: 1GB VRAM ≈ 4k tokens for local LLMs # But we need to leave room for the model itself # For a model, each ~8k context tokens takes about ~3-5MB of GPU VRAM # So VRAM available for context = VRAM - model size - agent overhead # Conservative: 1GB VRAM ≈ 2k context tokens local vram_context_kb=$((gpu_vram_gb * 2000)) # Leave headroom for the model itself and agent overhead recommended_kb=$((vram_context_kb * (100 - headroom_pct) / 100)) # If model context window is available, use it (for API inference) elif [[ $model_context_kb -gt 0 ]]; then # For API-based, we're limited by the model's context window # But we don't want to use the full window due to overhead recommended_kb=$((model_context_kb * (100 - headroom_pct) / 100)) # Fallback: use RAM to estimate else # Moderate estimate for low-VRAM systems where VRAM is too small for local inference # but RAM is available. Use 0.75k tokens per GB of RAM as a moderate estimate. # This balances between being too conservative (0.5k/GB) and too generous (1k/GB). local ram_context_kb=$((ram_gb * 750)) recommended_kb=$((ram_context_kb * (100 - headroom_pct) / 100)) fi # Subtract framework overhead local net_kb=$((recommended_kb - overhead_tokens)) if [[ $net_kb -lt 0 ]]; then net_kb=0 fi # Output: headroom_pct, recommended_kb echo "$net_kb" } # ─── Main ─── main() { local model_name="" local project_dir="." # Parse arguments while [[ $# -gt 0 ]]; do case "$1" in --model|-m) model_name="$2" shift 2 ;; --project|-p) project_dir="$2" shift 2 ;; *) # Could be model name as first argument if [[ -z "$model_name" ]]; then model_name="$1" fi shift ;; esac done echo "=== VRAM / Context Detection ===" echo "" # Detect GPU VRAM echo "--- GPU VRAM ---" detect_gpu_vram local gpu_vram_kb=0 local gpu_vram_gb=0 # Use timeout to avoid hanging gpu_vram_kb=$(timeout 5 nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ' || true) # Validate that vram_kb is a positive number if [[ -z "$gpu_vram_kb" || ! "$gpu_vram_kb" =~ ^[0-9]+$ || "$gpu_vram_kb" -le 0 ]]; then gpu_vram_kb=0 fi # If nvidia-smi didn't work, try to detect from lspci (AMD GPUs) if [[ $gpu_vram_kb -eq 0 ]]; then echo " nvidia-smi failed, checking lspci for AMD GPU VRAM..." local total_vram_mb=0 while IFS= read -r line; do local size_num size_num=$(echo "$line" | grep -oE '[0-9]+(M|G|K)' | head -1 || true) if [[ -n "$size_num" ]]; then local size_val size_val=$(echo "$size_num" | grep -oE '[0-9]+') local size_unit size_unit=$(echo "$size_num" | grep -oE '(M|G|K)') if [[ -n "$size_val" && -n "$size_unit" ]]; then case "$size_unit" in M) total_vram_mb=$((total_vram_mb + size_val)) ;; G) total_vram_mb=$((total_vram_mb + size_val * 1024)) ;; K) total_vram_mb=$((total_vram_mb + size_val / 1024)) ;; esac fi fi done < <(lspci -vnn 2>/dev/null | grep -i -A 15 -E 'VGA|3D|Display' | grep -i 'Memory at') if [[ $total_vram_mb -gt 0 ]]; then gpu_vram_kb=$((total_vram_mb * 1024)) gpu_vram_gb=$((total_vram_mb / 1024)) echo " AMD GPU VRAM from lspci: ${gpu_vram_gb}GB ($total_vram_mb MB)" else echo " No VRAM found from lspci" fi fi echo "" # Detect RAM echo "--- RAM ---" detect_ram local ram_kb ram_kb=$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print $2}' || echo 0) local ram_gb=$((ram_kb / 1024 / 1024)) echo "" # Detect model context window echo "--- Model Context Window ---" detect_model_context "$model_name" local model_context_kb model_context_kb=$(detect_model_context "$model_name" | tail -1) echo "" # Calculate framework overhead echo "--- Framework Overhead ---" calculate_overhead "$project_dir" local overhead_tokens overhead_tokens=$(calculate_overhead "$project_dir" | tail -1) echo "" # Recommend context window (also outputs headroom_pct and recommended_kb) echo "--- Recommendation ---" local recommendation_output recommendation_output=$(recommend_context "$gpu_vram_gb" "$ram_gb" "$model_context_kb" "$overhead_tokens") local line_count line_count=$(echo "$recommendation_output" | wc -l) local recommended_kb local headroom_pct local max_peak_kb if [[ $line_count -ge 3 ]]; then # Manual mode: outputs headroom_pct, recommended_kb, max_peak_kb headroom_pct=$(echo "$recommendation_output" | head -1 | tr -d '[:space:]') recommended_kb=$(echo "$recommendation_output" | sed -n '2p' | tr -d '[:space:]') max_peak_kb=$(echo "$recommendation_output" | tail -1 | tr -d '[:space:]') else # Auto-detect mode: outputs headroom_pct, recommended_kb headroom_pct=$(echo "$recommendation_output" | head -1 | tr -d '[:space:]') recommended_kb=$(echo "$recommendation_output" | tail -1 | tr -d '[:space:]') # Calculate max peak context based on headroom max_peak_kb=$((recommended_kb * (100 - headroom_pct) / 100)) fi # Convert to human-readable local recommended_k if [[ $recommended_kb -gt 0 ]]; then recommended_k=$((recommended_kb / 1000)) else recommended_k=8 # Default fallback fi echo "" echo "=== Recommended Configuration ===" echo "Target context: ${recommended_k}k tokens" echo "Headroom: ${headroom_pct}%" echo "Max peak context per sub-task: $((recommended_k * (100 - headroom_pct) / 100))k tokens" # Output as JSON for programmatic use echo "" echo "=== JSON Output ===" cat <