diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..f6d301a --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + + - name: Compile Python files + run: | + python -m py_compile automaton/**/*.py automaton/dashboard/**/*.py + + - name: Run pytest + run: | + python -m pytest tests/ -v + + - name: Syntax-check shell scripts + run: | + bash -n scripts/*.sh diff --git a/.gitignore b/.gitignore index c18dd8d..586842a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ __pycache__/ +.pytest_cache/ +*.egg-info/ +.venv/ +venv/ diff --git a/.rules.md b/.rules.md index 3bef11e..e8d7cfe 100644 --- a/.rules.md +++ b/.rules.md @@ -29,3 +29,8 @@ - Never scan, read, or modify tasks in other projects on the system - The `{project}` in the Orchestrator prompt is the project root where the session started — do not broaden scope - Past failure: agent scanned tasks in `~/ai-env/projects/invest-copilot/` while working on the automaton framework itself, wasting time on unrelated work across project boundaries + +## Artifact Integrity +- Write task artifacts atomically to avoid partial reads by the dashboard or other agents +- Prefer writing to a temporary file and renaming it into place +- Avoid leaving empty or malformed artifacts in the task folder diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..fed4f1b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,102 @@ +# AGENTS.md — Automaton Framework + +This file contains the information coding agents need to work effectively on the automaton framework itself. + +## Project Overview + +Automaton is a **prompt-driven, contract-based workflow framework** for LLM agents. It is intentionally not an agent harness: the framework provides prompts, conventions, scripts, and a dashboard, but enforcement is soft and relies on agent discipline. + +## Repository Layout + +``` +~/.automaton/ +├── .agent.md # Global router (autopilot mode, task routing) +├── .rules.md # Global rules and failure modes +├── system-prompt.md # Session startup prompt +├── config.md # Global VRAM/model configuration +├── README.md # Human-facing documentation +├── AGENTS.md # This file +├── CHANGELOG.md # Release notes +├── scripts/ # Bash/Python helper scripts +│ ├── install.sh +│ ├── update.sh +│ ├── migrate-project.sh +│ ├── vram_detect.py +│ └── dashboard.sh +├── prompts/ # Phase-specific LLM prompts +│ ├── orchestrate.md +│ ├── research.md +│ ├── implement.md +│ └── ... +├── contracts/ # Contract checklists +├── templates/ # Task templates +│ └── tasks/ +│ ├── bad-impl/ +│ ├── research-task/ +│ └── subtask-parent/ +├── automaton/ # Python dashboard package +│ └── dashboard/ +│ ├── __main__.py +│ ├── config.py +│ ├── core/ +│ └── ui/ +├── tests/ # pytest suite +└── tasks/ # Framework development tasks +``` + +## Build & Test Commands + +```bash +# Compile all Python files +python -m py_compile automaton/**/*.py automaton/dashboard/**/*.py + +# Run the test suite +python -m pytest tests/ -v + +# Run a single test file +python -m pytest tests/test_task.py -v + +# Syntax-check shell scripts +bash -n scripts/*.sh + +# Start the dashboard +python -m automaton.dashboard +``` + +## Conventions + +- **Prompts** live in `prompts/` and use `{placeholder}` syntax. +- **Task paths** must always be written as `{project}/.automaton/tasks/{task-name}/`. +- **Scripts** should be written in Python if they need complex parsing or testing; Bash is OK for simple glue. +- **Tests** are required for any new Python code or significant script logic. +- **No orchestrator runtime** — keep the framework prompt-driven. Do not add an agent harness. +- **No Rust rewrite** — Python/Bash are the implementation languages. + +## Adding or Updating Prompts + +1. Edit the relevant file in `prompts/`. +2. Run `python -m pytest tests/test_prompt_paths.py` to ensure task paths are canonical. +3. Update `CHANGELOG.md` under `[unreleased]`. + +## Adding a New Script + +1. Place the script in `scripts/`. +2. Make it executable if it is entry-point code (`chmod +x`). +3. Add tests in `tests/` if the script is Python. +4. Update `README.md` and any prompts that reference it. + +## Dashboard Development + +- The dashboard is a **web application** served by a Python HTTP server. +- Static assets are in `automaton/dashboard/html/`. +- Core logic is in `automaton/dashboard/core/`. +- The dashboard is scope-aware: framework mode when run from `~/.automaton/`, project mode otherwise. + +## CI + +Gitea CI runs on every push: +- `python -m py_compile` +- `python -m pytest tests/` +- `bash -n scripts/*.sh` + +See `.gitea/workflows/ci.yml`. diff --git a/CHANGELOG.md b/CHANGELOG.md index d424a91..1a86bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## [unreleased] ### Added +- pytest test suite covering dashboard core, app security, and VRAM detection (#add-pytest-test-suite) +- Root `pyproject.toml` with optional test/dashboard dependency groups (#add-pytest-test-suite) +- `AGENTS.md` with build/test commands and conventions (#developer-experience-gitea-ci) +- `.gitea/workflows/ci.yml` running py_compile, pytest, and shell script syntax checks (#developer-experience-gitea-ci) +- `templates/README.md` documenting the task template examples (#developer-experience-gitea-ci) - Blocked phase column between Verification and Resolution on dashboard (#additive-extension-model) - Framework self-enforcement rules in .rules.md and system-prompt.md (#framework-self-enforcement) - Additive extension model: projects extend via extensions/ dir, never copy framework files (#additive-extension-model) @@ -10,6 +15,10 @@ - Framework audit: comprehensive self-consistency check with RESEARCH.md (#framework-audit) ### Changed +- All prompts now use the canonical task path `{project}/.automaton/tasks/{task-name}/` (#standardize-task-path-conventions) +- `scripts/vram_detect.sh` rewritten as `scripts/vram_detect.py` for testability and correctness (#rewrite-vram-detection-python) +- `tasks/dashboard-spec.md` reconciled with the implemented web dashboard (#reconcile-dashboard-spec) +- `automaton/dashboard/README.md` and help modal shortcuts now match the web UI (#reconcile-dashboard-spec) - prompts/orchestrate.md: always reads prompts/contracts/scripts from global, project extensions are additive (#additive-extension-model) - prompts/onboarding.md: removed diff/merge upgrade, replaced with migration check (#additive-extension-model) - README.md: updated upgrade docs for new additive model (#additive-extension-model); added Dashboard section (#dashboard-task-review) @@ -25,6 +34,12 @@ - automaton/dashboard/core/refresh.py: improved inotify error handling with explicit fallback messages (#implement-task) ### Fixed +- VRAM detection: undefined headroom, hardcoded JSON headroom, and code-block config parsing (#rewrite-vram-detection-python) +- VRAM detection: 10KB file-read limit now enforced for API config files (#rewrite-vram-detection-python) +- Dashboard static file serving: replaced string-prefix path traversal check with `Path.relative_to()` (#harden-dashboard-security-scripts) +- Dashboard task name validation: restricted to `[A-Za-z0-9_-]+` (#harden-dashboard-security-scripts) +- `scripts/update.sh`: now warns and aborts on uncommitted changes before pulling (#harden-dashboard-security-scripts) +- README/install.sh: replaced placeholder repository URL with real Gitea URL (#harden-dashboard-security-scripts) - State machine: IMPLEMENTATION.md was never checked in determine_task_state(), tasks showed as RESEARCH (#implement-task) - State machine: DOC_REVIEW checked after BUG_REPORT — wrong priority order (#implement-task) - Path traversal: review API accepted task names with ../ allowing writes outside tasks directory (#dashboard-task-review) @@ -32,6 +47,12 @@ - Review parsing: comment extraction used fragile conditional, falsy comments (e.g., "0") skipped (#implement-task) - Board display: approved planning tasks stayed in Planning column instead of advancing to Design (#dashboard-task-review) +### Removed +- `automaton/dashboard/themes.py` (vestigial ANSI theme stub) (#reconcile-dashboard-spec) +- `automaton/dashboard/core/refresh.py` (half-implemented file watcher; dashboard uses JS polling) (#remove-file-system-watcher) +- `templates/contract-template.md` (unused) (#developer-experience-gitea-ci) +- `automaton/dashboard/pyproject.toml` (consolidated into root `pyproject.toml`) (#add-pytest-test-suite) + ### Migration - Project migration script for old-model projects: scripts/migrate-project.sh (#project-migration) - Project migration detection in onboarding.md (#project-migration) diff --git a/README.md b/README.md index fc02b66..a1a8322 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Before you can use the framework in any project, you must install the core logic ```bash # Clone the framework into the global config directory -git clone [INSERT_FRAMEWORK_REPO_URL_HERE] ~/.automaton +git clone http://10.37.0.86:3003/hermes/automaton ~/.automaton # Enter the directory cd ~/.automaton @@ -96,7 +96,7 @@ For low-VRAM systems (8GB, 16GB), set your VRAM limits in `~/.automaton/config.m - **Max peak context per sub-task**: 12k tokens ``` -**Auto-detection**: When `Auto-detect: Yes`, the Orchestrator runs `scripts/vram_detect.sh` to probe: +**Auto-detection**: When `Auto-detect: Yes`, the Orchestrator runs `scripts/vram_detect.py` to probe: - GPU VRAM (via `nvidia-smi`) - System RAM (via `free`) - Model context window (from config.md or API config files) @@ -173,7 +173,7 @@ When a task is decomposed, the Orchestrator creates sub-tasks under `tasks/{pare - `workflow.md`: The state machine governing the Autopilot lifecycle. - `test_design.md`: Produces a TEST_PLAN.md — an explicit test specification before implementation. - `decompose.md`: Breaks a task into VRAM-sized sub-tasks. -- `scripts/vram_detect.sh`: Auto-detects GPU VRAM, RAM, model context window, and framework overhead. +- `scripts/vram_detect.py`: Auto-detects GPU VRAM, RAM, model context window, and framework overhead. - `contracts/vram_config.md`: Contract for VRAM-aware task decomposition. ## Layered File System diff --git a/automaton/dashboard/README.md b/automaton/dashboard/README.md index 58ed57f..90899a2 100644 --- a/automaton/dashboard/README.md +++ b/automaton/dashboard/README.md @@ -67,7 +67,6 @@ Shows task progress through phases as a timeline with wave visualization for dec | `s` | Focus search | | `?` | Show help | | `Esc` | Close modals / clear search | -| `q` | Quit | ## Configuration @@ -97,7 +96,7 @@ Create a `dashboard-config.json` file in your project's `.automaton/` directory: - **Default**: Bright colors on dark background - **Dark**: Dimmed colors for a darker appearance -- **Light**: Softer colors for light terminals +- **Light**: Softer colors for light displays ## Phase Mapping @@ -123,16 +122,13 @@ automaton/dashboard/ ├── __init__.py # Package marker ├── __main__.py # Entry point (python -m automaton.dashboard) ├── config.py # Configuration management -├── themes.py # Color themes (for TUI fallback) -├── pyproject.toml # Python package config ├── README.md # This file ├── core/ │ ├── scope.py # Scope detection (framework vs project mode) │ ├── task.py # Task model, state machine, artifact parsing │ ├── board.py # Kanban board logic │ ├── stats.py # Statistics calculations -│ ├── timeline.py # Timeline data -│ └── refresh.py # File system watcher (for future use) +│ └── timeline.py # Timeline data └── ui/ ├── app.py # Web server (HTTP + API endpoints) └── html/ diff --git a/automaton/dashboard/__pycache__/__init__.cpython-314.pyc b/automaton/dashboard/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index 0d96e2e..0000000 Binary files a/automaton/dashboard/__pycache__/__init__.cpython-314.pyc and /dev/null differ diff --git a/automaton/dashboard/__pycache__/__main__.cpython-314.pyc b/automaton/dashboard/__pycache__/__main__.cpython-314.pyc deleted file mode 100644 index 3218313..0000000 Binary files a/automaton/dashboard/__pycache__/__main__.cpython-314.pyc and /dev/null differ diff --git a/automaton/dashboard/__pycache__/config.cpython-314.pyc b/automaton/dashboard/__pycache__/config.cpython-314.pyc deleted file mode 100644 index fa5d96b..0000000 Binary files a/automaton/dashboard/__pycache__/config.cpython-314.pyc and /dev/null differ diff --git a/automaton/dashboard/core/__pycache__/__init__.cpython-314.pyc b/automaton/dashboard/core/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index 99621a2..0000000 Binary files a/automaton/dashboard/core/__pycache__/__init__.cpython-314.pyc and /dev/null differ diff --git a/automaton/dashboard/core/__pycache__/scope.cpython-314.pyc b/automaton/dashboard/core/__pycache__/scope.cpython-314.pyc deleted file mode 100644 index 7326c8d..0000000 Binary files a/automaton/dashboard/core/__pycache__/scope.cpython-314.pyc and /dev/null differ diff --git a/automaton/dashboard/core/__pycache__/task.cpython-314.pyc b/automaton/dashboard/core/__pycache__/task.cpython-314.pyc deleted file mode 100644 index 1a3b194..0000000 Binary files a/automaton/dashboard/core/__pycache__/task.cpython-314.pyc and /dev/null differ diff --git a/automaton/dashboard/core/refresh.py b/automaton/dashboard/core/refresh.py deleted file mode 100644 index 470827e..0000000 --- a/automaton/dashboard/core/refresh.py +++ /dev/null @@ -1,90 +0,0 @@ -"""File system watcher for auto-refresh.""" - -import time -from pathlib import Path -from typing import Callable, Optional -from threading import Thread, Event - -try: - import inotify.adapters - HAS_INOTIFY = True -except ImportError: - HAS_INOTIFY = False - - -class FileSystemWatcher: - def __init__(self, tasks_dir: Path, callback: Optional[Callable] = None): - self.tasks_dir = tasks_dir - self.callback = callback - self._running = False - self._thread: Optional[Thread] = None - self._stop_event = Event() - - def start(self) -> None: - if not self.tasks_dir.exists(): - return - self._running = True - self._stop_event.clear() - if HAS_INOTIFY: - self._thread = Thread(target=self._watch_inotify, daemon=True) - else: - self._thread = Thread(target=self._watch_polling, daemon=True) - self._thread.start() - - def stop(self) -> None: - self._running = False - self._stop_event.set() - if self._thread and self._thread.is_alive(): - self._thread.join(timeout=1) - - def _watch_inotify(self) -> None: - try: - watcher = inotify.adapters.Inotify() - watcher.add_watch(self.tasks_dir) - except Exception as e: - print(f"Warning: inotify init failed ({e}), falling back to polling") - self._watch_polling() - return - try: - while self._running and not self._stop_event.is_set(): - try: - events = watcher.inotify_read(timeout_ms=1000) - for event in events: - if self._stop_event.is_set(): - break - if self.callback: - self.callback() - except Exception: - if self.callback: - self.callback() - time.sleep(1) - watcher.remove_watch(self.tasks_dir) - except Exception as e: - print(f"Warning: inotify watch failed ({e}), falling back to polling") - self._watch_polling() - - def _watch_polling(self) -> None: - last_hash = _dir_hash(self.tasks_dir) - while self._running and not self._stop_event.is_set(): - time.sleep(2) - if self._stop_event.is_set(): - break - current_hash = _dir_hash(self.tasks_dir) - if current_hash != last_hash: - last_hash = current_hash - if self.callback: - self.callback() - - -def _dir_hash(directory: Path) -> str: - if not directory.exists(): - return "" - entries = [] - for item in sorted(directory.rglob("*")): - if item.is_file(): - try: - stat = item.stat() - entries.append(f"{item.name}:{stat.st_mtime}:{stat.st_size}") - except (OSError, IOError): - pass - return "|".join(entries) diff --git a/automaton/dashboard/html/index.html b/automaton/dashboard/html/index.html index 81018dc..0150437 100644 --- a/automaton/dashboard/html/index.html +++ b/automaton/dashboard/html/index.html @@ -110,12 +110,11 @@ 1Board view 2Statistics view 3Timeline view - tToggle wave display on Timeline + tCycle themes (default → dark → light) rManual refresh fToggle filter bar sFocus search EscClose modals / clear search - qQuit ?Show this help diff --git a/automaton/dashboard/pyproject.toml b/automaton/dashboard/pyproject.toml deleted file mode 100644 index 321b172..0000000 --- a/automaton/dashboard/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -[build-system] -requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" - -[project] -name = "automaton" -version = "0.1.0" -description = "Automaton framework - dashboard and tools" -requires-python = ">=3.9" - -[project.scripts] -automaton-dashboard = "automaton.dashboard.__main__:main" - -[tool.setuptools.packages.find] -include = ["automaton.dashboard*"] diff --git a/automaton/dashboard/themes.py b/automaton/dashboard/themes.py deleted file mode 100644 index c044d10..0000000 --- a/automaton/dashboard/themes.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Color themes for the dashboard.""" - -from typing import Dict - -THEMES = { - "default": {}, - "dark": {}, - "light": {}, -} - -# ANSI escape sequences (not needed for web dashboard but kept for compat) -RESET = "\033[0m" -BOLD = "\033[1m" - -def get_theme(theme_name: str = "default") -> Dict[str, str]: - return THEMES.get(theme_name, THEMES["default"]) - -def colorize(text: str, color_code: str) -> str: - return f"{color_code}{text}{RESET}" diff --git a/automaton/dashboard/ui/__pycache__/__init__.cpython-314.pyc b/automaton/dashboard/ui/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index f31641f..0000000 Binary files a/automaton/dashboard/ui/__pycache__/__init__.cpython-314.pyc and /dev/null differ diff --git a/automaton/dashboard/ui/__pycache__/app.cpython-314.pyc b/automaton/dashboard/ui/__pycache__/app.cpython-314.pyc deleted file mode 100644 index 810391e..0000000 Binary files a/automaton/dashboard/ui/__pycache__/app.cpython-314.pyc and /dev/null differ diff --git a/automaton/dashboard/ui/app.py b/automaton/dashboard/ui/app.py index da28e5e..0fdb1f1 100644 --- a/automaton/dashboard/ui/app.py +++ b/automaton/dashboard/ui/app.py @@ -4,6 +4,7 @@ import json import mimetypes import os import posixpath +import re import sys from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path @@ -77,23 +78,21 @@ class DashboardHandler(SimpleHTTPRequestHandler): try: resolved = full_path.resolve() base = self.dashboard_path.resolve() - # Ensure the resolved path starts with the base directory - if not str(resolved).startswith(str(base) + '/'): - self._send_error(403, "Forbidden") - return - except Exception: + # Ensure the resolved path is inside the base directory + resolved.relative_to(base) + except (ValueError, RuntimeError, OSError): self._send_error(403, "Forbidden") return - if not full_path.exists() or not full_path.is_file(): + if not resolved.exists() or not resolved.is_file(): self._send_error(404, "File not found") return # Determine content type - content_type, encoding = mimetypes.guess_type(str(full_path)) + content_type, encoding = mimetypes.guess_type(str(resolved)) if not content_type: content_type = 'application/octet-stream' # Serve the file try: - with open(full_path, 'rb') as f: + with open(resolved, 'rb') as f: data = f.read() self.send_response(200) self.send_header("Content-Type", content_type) @@ -197,8 +196,14 @@ class DashboardHandler(SimpleHTTPRequestHandler): REVIEW_FILE = "REVIEW.md" - def _validate_task_name(self, task_name: str) -> bool: - return bool(task_name) and "/" not in task_name and "\\" not in task_name and ".." not in task_name + @staticmethod + def _validate_task_name(task_name: str) -> bool: + if not task_name: + return False + # Allow kebab-case names with letters, digits, hyphens, and underscores. + if not re.fullmatch(r"[A-Za-z0-9_-]+", task_name): + return False + return True def _get_review_path(self, task_name: str) -> Path | None: project_root = find_automaton_root() diff --git a/prompts/adversarial_bug_find.md b/prompts/adversarial_bug_find.md index 64dbb5e..ddb8a0e 100644 --- a/prompts/adversarial_bug_find.md +++ b/prompts/adversarial_bug_find.md @@ -2,9 +2,9 @@ You are the Adversarial Bug Finder. ## Read These Files -1. {project}/tasks/{task-name}/SPEC.md -2. {project}/tasks/{task-name}/VRAM_CONFIG.md (if exists) -3. {project}/tasks/{task-name}/PARENT_SPEC.md (if exists) +1. {project}/.automaton/tasks/{task-name}/SPEC.md +2. {project}/.automaton/tasks/{task-name}/VRAM_CONFIG.md (if exists) +3. {project}/.automaton/tasks/{task-name}/PARENT_SPEC.md (if exists) 4. The code Your job is to find bugs that are difficult to spot, such as complex logic errors, race conditions, and performance bottlenecks. Be more aggressive and exhaustive than a standard bug finder. diff --git a/prompts/bug_finder.md b/prompts/bug_finder.md index f4c6fb3..f78e07a 100644 --- a/prompts/bug_finder.md +++ b/prompts/bug_finder.md @@ -2,11 +2,11 @@ You are the Bug Finder. Your job is to find every bug, deviation from spec, and ## Read These Files -1. {project}/tasks/{task-name}/SPEC.md -2. {project}/tasks/{task-name}/{task-name}_CONTRACT.md (if exists) -3. {project}/tasks/{task-name}/IMPLEMENTATION.md (if exists) -4. {project}/tasks/{task-name}/VRAM_CONFIG.md (if exists) -5. {project}/tasks/{task-name}/PARENT_SPEC.md (if exists) +1. {project}/.automaton/tasks/{task-name}/SPEC.md +2. {project}/.automaton/tasks/{task-name}/{task-name}_CONTRACT.md (if exists) +3. {project}/.automaton/tasks/{task-name}/IMPLEMENTATION.md (if exists) +4. {project}/.automaton/tasks/{task-name}/VRAM_CONFIG.md (if exists) +5. {project}/.automaton/tasks/{task-name}/PARENT_SPEC.md (if exists) ## Task @@ -22,7 +22,7 @@ You are the Bug Finder. Your job is to find every bug, deviation from spec, and ## Output Format -Produce a BUG_REPORT.md at {project}/tasks/{task-name}/BUG_REPORT.md: +Produce a BUG_REPORT.md at {project}/.automaton/tasks/{task-name}/BUG_REPORT.md: ```markdown # Bug Report: {task-name} diff --git a/prompts/decompose.md b/prompts/decompose.md index e64622e..e34f350 100644 --- a/prompts/decompose.md +++ b/prompts/decompose.md @@ -4,11 +4,11 @@ Your only job is to take a completed SPEC.md and break it into the smallest poss ## Read These Files -1. {project}/tasks/{task-name}/SPEC.md +1. {project}/.automaton/tasks/{task-name}/SPEC.md 2. {project}/.automaton/.rules.md (if exists — project override) OR ~/.automaton/.rules.md (global default) — project-specific rules 3. ~/.automaton/config.md — Global framework configuration (VRAM, model settings) 4. {project}/.automaton/.agent.md (if exists — project override) OR ~/.automaton/.agent.md (global default) — project agent config -5. {project}/.automaton/scripts/vram_detect.sh (if exists — project override) OR ~/.automaton/scripts/vram_detect.sh (global default) — VRAM detection +5. {project}/.automaton/scripts/vram_detect.py (if exists — project override) OR ~/.automaton/scripts/vram_detect.py (global default) — VRAM detection ## Task @@ -98,7 +98,7 @@ Before decomposing, analyze the SPEC.md: 6. Estimate the token budget for the full task (sum of all requirements' SPEC + DESIGN + TEST files) 7. **Detect VRAM limits**: - Check `~/.automaton/config.md` for VRAM Configuration section - - If `Auto-detect: Yes`, run `{project}/.automaton/scripts/vram_detect.sh` to probe GPU VRAM, RAM, and model context window + - If `Auto-detect: Yes`, run `{project}/.automaton/scripts/vram_detect.py` to probe GPU VRAM, RAM, and model context window - If `Auto-detect: No`, use the manually specified values from config.md - Report the detected VRAM limits 8. **Detect model context window**: @@ -156,7 +156,7 @@ Only produce the DECOMPOSITION.md after the user says "APPROVED" or equivalent. ## Output -Produce a file called DECOMPOSITION.md at {project}/tasks/{task-name}/DECOMPOSITION.md that contains: +Produce a file called DECOMPOSITION.md at {project}/.automaton/tasks/{task-name}/DECOMPOSITION.md that contains: ```markdown # Task Decomposition diff --git a/prompts/design.md b/prompts/design.md index 5d50154..d2f7b87 100644 --- a/prompts/design.md +++ b/prompts/design.md @@ -4,7 +4,7 @@ Your job is to create a clear, actionable design for the project based on the sp ## Read These Files -1. {project}/tasks/{task-name}/SPEC.md +1. {project}/.automaton/tasks/{task-name}/SPEC.md 2. {project}/.automaton/.rules.md (if exists — project override) OR ~/.automaton/.rules.md (global default) — project-specific rules ## Task diff --git a/prompts/doc_review.md b/prompts/doc_review.md index 420a41d..feea31a 100644 --- a/prompts/doc_review.md +++ b/prompts/doc_review.md @@ -2,10 +2,10 @@ You are in Documentation Review mode. ## Read These Files -1. {project}/tasks/{task-name}/DESIGN.md — Look for the "Documentation Plan" section -2. {project}/tasks/{task-name}/SPEC.md — Check what the spec requires -3. {project}/tasks/{task-name}/VRAM_CONFIG.md (if exists) -4. {project}/tasks/{task-name}/PARENT_SPEC.md (if exists) +1. {project}/.automaton/tasks/{task-name}/DESIGN.md — Look for the "Documentation Plan" section +2. {project}/.automaton/tasks/{task-name}/SPEC.md — Check what the spec requires +3. {project}/.automaton/tasks/{task-name}/VRAM_CONFIG.md (if exists) +4. {project}/.automaton/tasks/{task-name}/PARENT_SPEC.md (if exists) 5. Any existing documentation files mentioned in the DESIGN.md Documentation Plan 6. The code that was implemented (implementation artifacts) @@ -42,7 +42,7 @@ You are in Documentation Review mode. ## Output -If the DESIGN.md has a Documentation Plan section, produce a `DOC_REVIEW.md` at `{project}/tasks/{task-name}/DOC_REVIEW.md` with: +If the DESIGN.md has a Documentation Plan section, produce a `DOC_REVIEW.md` at `{project}/.automaton/tasks/{task-name}/DOC_REVIEW.md` with: ```markdown # Documentation Review: {task-name} diff --git a/prompts/implement.md b/prompts/implement.md index 7506160..12c11b7 100644 --- a/prompts/implement.md +++ b/prompts/implement.md @@ -2,14 +2,14 @@ You are in implementation mode. ## Read These Files -1. {project}/tasks/{task-name}/SPEC.md +1. {project}/.automaton/tasks/{task-name}/SPEC.md 2. {project}/.automaton/.rules.md (if exists — project override) OR ~/.automaton/.rules.md (global default) — project-specific rules 3. {project}/.automaton/.agent.md (if exists — project override) OR ~/.automaton/.agent.md (global default) — project agent config -4. {project}/tasks/{task-name}/{task-name}_CONTRACT.md (if exists) -5. {project}/tasks/{task-name}/DESIGN.md (if exists) -6. {project}/tasks/{task-name}/TEST_PLAN.md (if exists) -7. {project}/tasks/{task-name}/VRAM_CONFIG.md (if exists — for low-VRAM systems) -8. {project}/tasks/{task-name}/PARENT_SPEC.md (if exists — for sub-tasks) +4. {project}/.automaton/tasks/{task-name}/{task-name}_CONTRACT.md (if exists) +5. {project}/.automaton/tasks/{task-name}/DESIGN.md (if exists) +6. {project}/.automaton/tasks/{task-name}/TEST_PLAN.md (if exists) +7. {project}/.automaton/tasks/{task-name}/VRAM_CONFIG.md (if exists — for low-VRAM systems) +8. {project}/.automaton/tasks/{task-name}/PARENT_SPEC.md (if exists — for sub-tasks) ## Task diff --git a/prompts/onboarding.md b/prompts/onboarding.md index bb0cc73..217f650 100644 --- a/prompts/onboarding.md +++ b/prompts/onboarding.md @@ -9,7 +9,7 @@ Your only job is to set up the minimal agent framework structure in the target p 3. ~/.automaton/.onboarding.md — human reference for drop-in vs from-scratch scenarios 4. {project}/.automaton/.agent.md (if it exists — project override) 5. {project}/.automaton/.rules.md (if it exists — project override) -6. ~/.automaton/scripts/vram_detect.sh (if exists — for VRAM detection) +6. ~/.automaton/scripts/vram_detect.py (if exists — for VRAM detection) ## Task @@ -37,10 +37,10 @@ Your only job is to set up the minimal agent framework structure in the target p Check if VRAM configuration is available in `~/.automaton/config.md`: 1. Read `~/.automaton/config.md` to check for VRAM Configuration section. 2. If VRAM Configuration section exists, note the values. -3. If VRAM Configuration section does NOT exist, check if VRAM detection is available: `~/.automaton/scripts/vram_detect.sh`. +3. If VRAM Configuration section does NOT exist, check if VRAM detection is available: `~/.automaton/scripts/vram_detect.py`. 4. If available, run it to get VRAM recommendations: ``` - cd ~/.automaton && bash ~/.automaton/scripts/vram_detect.sh + cd ~/.automaton && python ~/.automaton/scripts/vram_detect.py ``` 5. Parse the JSON output for `recommended_k`, `max_peak_context_kb`, and `headroom`. 6. Add a VRAM Configuration section to `~/.automaton/config.md`: diff --git a/prompts/orchestrate.md b/prompts/orchestrate.md index 85190a6..d69be67 100644 --- a/prompts/orchestrate.md +++ b/prompts/orchestrate.md @@ -27,7 +27,7 @@ When VRAM configuration is needed (during task decomposition, sub-task creation, ### Detection Priority -1. **Auto-detect via script**: Check if `{project}/.automaton/scripts/vram_detect.sh` exists. If it does, run it to probe GPU VRAM, RAM, and model context window. Parse the JSON output for `recommended_kb`, `headroom`, and `max_peak_context_kb`. +1. **Auto-detect via script**: Check if `{project}/.automaton/scripts/vram_detect.py` exists. If it does, run it to probe GPU VRAM, RAM, and model context window. Parse the JSON output for `recommended_kb`, `headroom`, and `max_peak_context_kb`. 2. **Auto-detect via API config**: If the script is not available, try to detect the model name from `.agent.md` or config files (`.env`, `config.yaml`, etc.) and look up its context window. **Important**: Only read the specific lines needed (e.g., the model name line), not the entire file. Limit file reads to 10KB to prevent memory exhaustion. 3. **Manual override**: Check if `~/.automaton/config.md` has `Auto-detect: No` under VRAM Configuration. If so, use the manually specified values. 4. **Fallback**: Use 8k tokens as default, with 25% headroom. @@ -52,7 +52,7 @@ When model context window is needed, the Orchestrator MUST attempt to detect it #### Detection Priority -1. **Auto-detect via script**: Check if `{project}/.automaton/scripts/vram_detect.sh` exists. If it does, run it to detect the model name and its context window. Parse the JSON output for `model_context_kb`. +1. **Auto-detect via script**: Check if `{project}/.automaton/scripts/vram_detect.py` exists. If it does, run it to detect the model name and its context window. Parse the JSON output for `model_context_kb`. 2. **Auto-detect via config**: Check `~/.automaton/config.md` for the model name and override context window. 3. **Auto-detect via API config**: If the script is not available, try to detect the model name from `.agent.md` or config files (`.env`, `config.yaml`, etc.) and look up its context window. **Important**: Only read the specific lines needed (e.g., the model name line), not the entire file. Limit file reads to 10KB to prevent memory exhaustion. 4. **Fallback**: Use 128k tokens as default (common for modern models). @@ -372,7 +372,7 @@ When decomposing, the Orchestrator creates sub-tasks under the parent task's `su When a parent task reaches the **Decomposition** phase (has `SPEC.md` and `DECOMPOSITION.md`): 1. **Read `~/.automaton/config.md`** to get the VRAM configuration and check if auto-detect is enabled. -2. **If Auto-detect: Yes**, run `{project}/.automaton/scripts/vram_detect.sh` to detect VRAM limits. Parse the JSON output for `recommended_kb`, `headroom`, and `max_peak_context_kb`. Report the detection results. +2. **If Auto-detect: Yes**, run `{project}/.automaton/scripts/vram_detect.py` to detect VRAM limits. Parse the JSON output for `recommended_kb`, `headroom`, and `max_peak_context_kb`. Report the detection results. 3. **If Auto-detect: No**, use the manually specified values from config.md. 4. **Read `DECOMPOSITION.md`** to extract all sub-task names, dependencies, and their estimated token budgets. 5. **Verify VRAM constraints**: diff --git a/prompts/referee.md b/prompts/referee.md index 7b23661..18f1007 100644 --- a/prompts/referee.md +++ b/prompts/referee.md @@ -2,16 +2,16 @@ You are the Referee. Your job is to objectively evaluate whether the implementat ## Read These Files -1. {project}/tasks/{task-name}/SPEC.md -2. {project}/tasks/{task-name}/{task-name}_CONTRACT.md (if exists) -3. {project}/tasks/{task-name}/BUG_REPORT.md (if exists) -4. {project}/tasks/{task-name}/ADVERSARIAL_BUG_REPORT.md (if exists) -5. {project}/tasks/{task-name}/DOC_REVIEW.md (if exists) -6. {project}/tasks/{task-name}/IMPLEMENTATION.md (if exists) -7. {project}/tasks/{task-name}/DESIGN.md (if exists) -8. {project}/tasks/{task-name}/TEST_PLAN.md (if exists) -9. {project}/tasks/{task-name}/VRAM_CONFIG.md (if exists) -10. {project}/tasks/{task-name}/PARENT_SPEC.md (if exists) +1. {project}/.automaton/tasks/{task-name}/SPEC.md +2. {project}/.automaton/tasks/{task-name}/{task-name}_CONTRACT.md (if exists) +3. {project}/.automaton/tasks/{task-name}/BUG_REPORT.md (if exists) +4. {project}/.automaton/tasks/{task-name}/ADVERSARIAL_BUG_REPORT.md (if exists) +5. {project}/.automaton/tasks/{task-name}/DOC_REVIEW.md (if exists) +6. {project}/.automaton/tasks/{task-name}/IMPLEMENTATION.md (if exists) +7. {project}/.automaton/tasks/{task-name}/DESIGN.md (if exists) +8. {project}/.automaton/tasks/{task-name}/TEST_PLAN.md (if exists) +9. {project}/.automaton/tasks/{task-name}/VRAM_CONFIG.md (if exists) +10. {project}/.automaton/tasks/{task-name}/PARENT_SPEC.md (if exists) ## Task @@ -56,7 +56,7 @@ You are the Referee. Your job is to objectively evaluate whether the implementat ## Verdict -Produce a VERDICT.md at {project}/tasks/{task-name}/VERDICT.md with: +Produce a VERDICT.md at {project}/.automaton/tasks/{task-name}/VERDICT.md with: ```markdown # Verdict: {task-name} diff --git a/prompts/research.md b/prompts/research.md index 16e3be1..3d651fd 100644 --- a/prompts/research.md +++ b/prompts/research.md @@ -80,7 +80,7 @@ Only produce the SPEC.md after the user says "APPROVED" or equivalent. ## Output -Produce a file called SPEC.md at {project}/tasks/{task-name}/SPEC.md that contains: +Produce a file called SPEC.md at {project}/.automaton/tasks/{task-name}/SPEC.md that contains: - Clear goal - Exact requirements (numbered) diff --git a/prompts/test_design.md b/prompts/test_design.md index b673d1d..558ec6f 100644 --- a/prompts/test_design.md +++ b/prompts/test_design.md @@ -4,8 +4,8 @@ Your only job is to produce a comprehensive, explicit test specification for the ## Read These Files -1. {project}/tasks/{task-name}/SPEC.md — Requirements and acceptance criteria -2. {project}/tasks/{task-name}/DESIGN.md — Architecture and data model (if exists) +1. {project}/.automaton/tasks/{task-name}/SPEC.md — Requirements and acceptance criteria +2. {project}/.automaton/tasks/{task-name}/DESIGN.md — Architecture and data model (if exists) 3. {project}/.automaton/.rules.md (if exists — project override) OR ~/.automaton/.rules.md (global default) — Project constraints ## Task @@ -77,7 +77,7 @@ Only produce the TEST_PLAN.md after the user says "APPROVED" or equivalent. ## Output -Produce a file called TEST_PLAN.md at {project}/tasks/{task-name}/TEST_PLAN.md that contains: +Produce a file called TEST_PLAN.md at {project}/.automaton/tasks/{task-name}/TEST_PLAN.md that contains: ```markdown # Test Plan: {task-name} diff --git a/prompts/workflow.md b/prompts/workflow.md index f50360a..cc2aa99 100644 --- a/prompts/workflow.md +++ b/prompts/workflow.md @@ -24,7 +24,7 @@ The Orchestrator is responsible for creating new task folders automatically — ### New tasks from user input When the Orchestrator detects a new task description: 1. Generate a kebab-case task name from the description -2. Create `{project}/tasks/{task-name}/` (empty — no artifact files) +2. Create `{project}/.automaton/tasks/{task-name}/` (empty — no artifact files) 3. Move the task to the **Research** phase The Orchestrator also scans for tasks that have `VERDICT.md` with `PASS` and removes them from the active task list (they can be archived but not auto-deleted). diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ebfc3aa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "automaton" +version = "0.2.0" +description = "Automaton framework - contract-based operating system for LLM agents" +requires-python = ">=3.9" + +[project.optional-dependencies] +test = ["pytest>=7.0"] +dashboard = ["inotify>=0.2"] + +[project.scripts] +automaton-dashboard = "automaton.dashboard.__main__:main" + +[tool.setuptools.packages.find] +include = ["automaton*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/scripts/install.sh b/scripts/install.sh index 8934ccc..d8c7a3a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -10,7 +10,7 @@ if [ -d "$FRAMEWORK_DIR" ]; then fi echo "Cloning automaton to $FRAMEWORK_DIR..." -git clone https://gitea.yourdomain.com/you/automaton.git "$FRAMEWORK_DIR" +git clone http://10.37.0.86:3003/hermes/automaton "$FRAMEWORK_DIR" echo "" echo "=== VRAM / Context Detection ===" @@ -18,23 +18,23 @@ echo "Detecting your system's VRAM to recommend task decomposition settings..." echo "" # Run VRAM detection script if it exists -if [ -f "$FRAMEWORK_DIR/scripts/vram_detect.sh" ]; then +if [ -f "$FRAMEWORK_DIR/scripts/vram_detect.py" ]; then # Run in project-dir context so it can read framework overhead - detection_output=$(cd "$FRAMEWORK_DIR" && bash "$FRAMEWORK_DIR/scripts/vram_detect.sh" 2>&1) - - # Extract JSON output (last section after "=== JSON Output ===") - json_output=$(echo "$detection_output" | sed -n '/=== JSON Output ===/,/EOF/p' | grep -v '=== JSON Output ===' | grep -v '^EOF$') - + detection_output=$(cd "$FRAMEWORK_DIR" && python3 "$FRAMEWORK_DIR/scripts/vram_detect.py" 2>&1) + + # Extract JSON output (the block after "=== JSON Output ===") + json_output=$(echo "$detection_output" | sed -n '/=== JSON Output ===/,$p' | tail -n +2) + if [ -n "$json_output" ]; then echo "$detection_output" - - # Extract key values from JSON for display - recommended_k=$(echo "$json_output" | grep '"recommended_k"' | grep -oP '\d+') - max_peak_kb=$(echo "$json_output" | grep '"max_peak_context_kb"' | grep -oP '\d+') - headroom=$(echo "$json_output" | grep '"headroom"' | grep -oP '\d+\.\d+') - gpu_vram=$(echo "$json_output" | grep '"gpu_vram_gb"' | grep -oP '\d+') - ram_gb=$(echo "$json_output" | grep '"ram_gb"' | grep -oP '\d+') - model_context=$(echo "$json_output" | grep '"model_context_kb"' | grep -oP '\d+') + + # Extract key values from JSON using Python + recommended_k=$(echo "$json_output" | python3 -c 'import json,sys; print(json.load(sys.stdin)["recommended_k"])') + max_peak_kb=$(echo "$json_output" | python3 -c 'import json,sys; print(json.load(sys.stdin)["max_peak_context_kb"])') + headroom=$(echo "$json_output" | python3 -c 'import json,sys; print(json.load(sys.stdin)["headroom"])') + gpu_vram=$(echo "$json_output" | python3 -c 'import json,sys; print(json.load(sys.stdin)["gpu_vram_gb"])') + ram_gb=$(echo "$json_output" | python3 -c 'import json,sys; print(json.load(sys.stdin)["ram_gb"])') + model_context=$(echo "$json_output" | python3 -c 'import json,sys; print(json.load(sys.stdin)["model_context_kb"])') echo "" echo "=== Recommended VRAM Configuration ===" diff --git a/scripts/update.sh b/scripts/update.sh index 90a4d68..922653f 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -19,6 +19,13 @@ if [ ! -d ".git" ]; then exit 1 fi +# Check for uncommitted changes +if ! git diff --quiet HEAD || ! git diff --cached --quiet HEAD; then + echo "ERROR: $FRAMEWORK_DIR has uncommitted changes." + echo "Commit or stash them before updating to avoid losing work." + exit 1 +fi + # Fetch and pull latest changes git pull origin main diff --git a/scripts/vram_detect.py b/scripts/vram_detect.py new file mode 100755 index 0000000..2d4deb0 --- /dev/null +++ b/scripts/vram_detect.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 +"""VRAM/Context Detection Script for automaton. + +Detects GPU VRAM, system RAM, and model context window to recommend a safe +context window for task decomposition. + +Usage: + python vram_detect.py [model_name] [--model model_name] [--project project_dir] +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Optional + +# Known model context windows in tokens. +MODEL_CONTEXT_WINDOWS: dict[str, int] = { + "gpt-4o": 128_000, + "gpt-4o-2024-05-13": 128_000, + "gpt-4o-2024-08-06": 128_000, + "gpt-4o-mini": 128_000, + "gpt-4o-mini-2024-07-18": 128_000, + "gpt-4-turbo": 128_000, + "gpt-4-turbo-2024-04-09": 128_000, + "gpt-4": 128_000, + "gpt-4-0125-preview": 128_000, + "gpt-4-1106-preview": 128_000, + "claude-3-5-sonnet": 200_000, + "claude-3-5-sonnet-20241022": 200_000, + "claude-3-5-haiku": 200_000, + "claude-3-5-haiku-20241022": 200_000, + "claude-3-opus": 200_000, + "claude-3-opus-20240229": 200_000, + "claude-3-sonnet": 200_000, + "claude-3-sonnet-20240229": 200_000, + "claude-3-haiku": 200_000, + "claude-3-haiku-20240307": 200_000, + "claude-2": 200_000, + "claude-2.1": 200_000, +} + +DEFAULT_FALLBACK_CONTEXT_TOKENS = 128_000 +DEFAULT_HEADROOM_PCT = 25 +MAX_CONFIG_READ_BYTES = 10 * 1024 # 10KB limit per prompt requirement + + +def run_command(cmd: list[str], timeout: float = 5.0) -> Optional[str]: + """Run a command and return stdout, or None on failure.""" + if not shutil.which(cmd[0]): + return None + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + if result.returncode != 0: + return None + return result.stdout.strip() + except (subprocess.TimeoutExpired, OSError): + return None + + +def detect_gpu_vram() -> tuple[int, int, int]: + """Detect GPU VRAM and return (total_vram_kb, vram_per_gpu_kb, num_gpus).""" + total_vram_kb = 0 + num_gpus = 0 + + # Try nvidia-smi first. + nvidia_output = run_command( + ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"] + ) + if nvidia_output: + lines = [line.strip() for line in nvidia_output.splitlines() if line.strip()] + if lines: + try: + vram_mb = int(lines[0]) + if vram_mb > 0: + total_vram_kb = vram_mb * 1024 + num_gpus = len(lines) + print("GPU: NVIDIA (nvidia-smi available)") + print(f"VRAM per GPU: {vram_mb // 1024}GB ({vram_mb} MB)") + print(f"Num GPUs: {num_gpus}") + except ValueError: + print("GPU: NVIDIA (nvidia-smi available but driver not responding)") + + # Fallback: lspci for AMD/others. + if total_vram_kb == 0: + lspci_output = run_command(["lspci"]) + if lspci_output and re.search(r"VGA|3D|Display", lspci_output, re.IGNORECASE): + print("GPU detected via lspci") + vram_mb = _detect_amd_vram_from_lspci() + if vram_mb > 0: + total_vram_kb = vram_mb * 1024 + num_gpus = 1 + print(f"VRAM: {vram_mb} MB ({vram_mb // 1024}GB)") + + vram_per_gpu_kb = total_vram_kb // num_gpus if num_gpus > 0 else 0 + return total_vram_kb, vram_per_gpu_kb, num_gpus + + +def _detect_amd_vram_from_lspci() -> int: + """Attempt to sum memory region sizes from lspci -vnn for VGA/3D/Display devices.""" + output = run_command(["lspci", "-vnn"]) + if not output: + return 0 + + total_mb = 0 + # Split into device blocks. Each block starts with a bus address like "71:00.0". + blocks = re.split(r"\n\n", output) + for block in blocks: + if not re.search(r"VGA|3D|Display", block, re.IGNORECASE): + continue + for line in block.splitlines(): + match = re.search(r"Memory at [0-9a-fA-Fx]+ \(.*\) \[size=(\d+)([MGK])\]", line) + if match: + value = int(match.group(1)) + unit = match.group(2).upper() + if unit == "G": + total_mb += value * 1024 + elif unit == "M": + total_mb += value + elif unit == "K": + total_mb += value // 1024 + return total_mb + + +def detect_ram() -> tuple[int, int]: + """Detect total and available RAM in KB.""" + meminfo = Path("/proc/meminfo") + if meminfo.exists(): + try: + text = meminfo.read_text(encoding="utf-8") + total_kb = _parse_meminfo_value(text, "MemTotal") + available_kb = _parse_meminfo_value(text, "MemAvailable") or total_kb + if total_kb > 0: + print(f"RAM: {total_kb // 1024 // 1024}GB total, {available_kb // 1024 // 1024}GB available") + return total_kb, available_kb + except (OSError, ValueError): + pass + + sysctl_output = run_command(["sysctl", "-n", "hw.memsize"]) + if sysctl_output: + try: + total_kb = int(sysctl_output) // 1024 + if total_kb > 0: + print(f"RAM: {total_kb // 1024 // 1024}GB total") + return total_kb, total_kb + except ValueError: + pass + + print("RAM: Could not detect") + return 0, 0 + + +def _parse_meminfo_value(text: str, key: str) -> int: + """Parse a value in KB from /proc/meminfo.""" + for line in text.splitlines(): + if line.startswith(key + ":"): + parts = line.split() + if len(parts) >= 2: + return int(parts[1]) + return 0 + + +def detect_model_context( + model_name: Optional[str] = None, + project_dir: Optional[Path] = None, +) -> int: + """Detect model context window in tokens.""" + if model_name: + return _lookup_model_context(model_name) + + # Try config.md (global framework model settings). + config_md = Path.home() / ".automaton" / "config.md" + model_from_config, override_context = _parse_config_model(config_md) + if model_from_config: + print(f"Found model in config.md: {model_from_config}") + if override_context and override_context != "auto": + print(f"Using override context window from config.md: {override_context}") + return override_context + return _lookup_model_context(model_from_config) + + # Try .agent.md (project-level model override). + if project_dir: + agent_md = project_dir / ".automaton" / ".agent.md" + model_from_agent, _ = _parse_config_model(agent_md) + if model_from_agent: + print(f"Found model in .agent.md: {model_from_agent}") + if override_context and override_context != "auto": + print(f"Using override context window from config.md: {override_context}") + return override_context + return _lookup_model_context(model_from_agent) + + # Try common API config files. + config_files = [ + ".env", + ".env.local", + "config.yaml", + "config.yml", + "config.json", + "settings.yaml", + ".automaton/config.yaml", + ".automaton/config.json", + ] + if project_dir: + for config_file in config_files: + for candidate in [project_dir / config_file, project_dir / ".automaton" / config_file]: + if candidate.exists(): + model = _extract_model_from_file(candidate) + if model: + print(f"Found model in {candidate}: {model}") + if override_context and override_context != "auto": + print(f"Using override context window from config.md: {override_context}") + return override_context + return _lookup_model_context(model) + + print("Model: Unknown (could not detect from .agent.md or config files)") + return 0 + + +def _lookup_model_context(model_name: str) -> int: + """Look up context window for a known model name.""" + # Strip common version/date suffixes for lookup. + for key in MODEL_CONTEXT_WINDOWS: + if model_name.lower().startswith(key.lower()): + print(f"Model: {model_name}") + print(f"Context window: {MODEL_CONTEXT_WINDOWS[key] // 1000}k tokens") + return MODEL_CONTEXT_WINDOWS[key] + print(f"Model: {model_name} (unknown context window)") + return 0 + + +def _parse_config_model(config_path: Path) -> tuple[Optional[str], Optional[int]]: + """Parse model name and override context window from a config file.""" + if not config_path.exists(): + return None, None + try: + text = config_path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None, None + + model_name = None + override_tokens: Optional[int] = None + in_code_block = False + + for raw_line in text.splitlines(): + line = raw_line.strip() + if line.startswith("```"): + in_code_block = not in_code_block + continue + if in_code_block: + continue + if line.startswith("#"): + continue + if re.search(r"^[-*]?\s*\*\*Model\*\*:|^\s*model\s*[:=]", line, re.IGNORECASE): + if "override" in line.lower(): + continue + value = _extract_value(line) + if value and value.lower() != "auto": + model_name = value + if re.search(r"^[-*]?\s*\*\*Override context( window)?\*\*:|^\s*override.*context\s*[:=]", line, re.IGNORECASE): + value = _extract_value(line) + if value and value.lower() != "auto": + override_tokens = _parse_token_value(value) + + return model_name, override_tokens + + +def _extract_model_from_file(path: Path) -> Optional[str]: + """Extract model name from an API config file, reading at most 10KB.""" + try: + raw = path.read_bytes() + text = raw[:MAX_CONFIG_READ_BYTES].decode("utf-8", errors="replace") + except OSError: + return None + + for line in text.splitlines(): + line = line.strip() + if line.startswith("#"): + continue + if re.search(r"model\s*[:=]", line, re.IGNORECASE): + # Skip non-model keys like max_tokens, temperature, stream. + key = line.split("=", 1)[0].split(":", 1)[0].strip().lower() + if any(bad in key for bad in ["context", "max_tokens", "temperature", "stream"]): + continue + value = _extract_value(line) + if value and value.lower() != "auto": + return value + return None + + +def _extract_value(line: str) -> Optional[str]: + """Extract the value after ':' or '=' from a key-value line.""" + for sep in [":", "="]: + if sep in line: + value = line.split(sep, 1)[1].strip() + value = value.split("#", 1)[0].strip() + value = value.strip('"').strip("'") + return value if value else None + return None + + +def _parse_token_value(value: str) -> Optional[int]: + """Parse a token value like '128k', '5.6k', or '128000' into an integer.""" + value = value.strip().lower() + match = re.search(r"(\d+(?:\.\d+)?)\s*k?", value) + if not match: + return None + number = float(match.group(1)) + if "k" in value: + return int(number * 1000) + return int(number) + + +def calculate_overhead(project_dir: Optional[Path] = None) -> int: + """Estimate framework overhead in tokens.""" + if project_dir: + base_files = [ + project_dir / ".automaton" / ".agent.md", + project_dir / ".automaton" / ".rules.md", + project_dir / ".automaton" / "prompts" / "workflow.md", + project_dir / ".automaton" / "prompts" / "orchestrate.md", + ] + if not base_files[0].exists(): + base_files = [ + Path.home() / ".automaton" / ".agent.md", + Path.home() / ".automaton" / ".rules.md", + Path.home() / ".automaton" / "prompts" / "workflow.md", + Path.home() / ".automaton" / "prompts" / "orchestrate.md", + ] + else: + base_files = [ + Path.home() / ".automaton" / ".agent.md", + Path.home() / ".automaton" / ".rules.md", + Path.home() / ".automaton" / "prompts" / "workflow.md", + Path.home() / ".automaton" / "prompts" / "orchestrate.md", + ] + + overhead_tokens = 0 + for file_path in base_files: + if file_path.exists(): + try: + chars = len(file_path.read_text(encoding="utf-8", errors="replace")) + tokens = chars // 4 + overhead_tokens += tokens + print(f" {file_path.name}: ~{tokens} tokens") + except OSError: + pass + + print(f"Framework overhead: ~{overhead_tokens} tokens") + return overhead_tokens + + +def _iter_config_lines(text: str, section_header: str) -> list[str]: + """Return non-code-block lines within a markdown section.""" + lines: list[str] = [] + in_section = False + in_code_block = False + for raw_line in text.splitlines(): + line = raw_line.strip() + if line.startswith("```"): + in_code_block = not in_code_block + continue + if in_code_block: + continue + if line.startswith(section_header): + in_section = True + continue + if in_section and line.startswith("##"): + in_section = False + if not in_section: + continue + lines.append(line) + return lines + + +def parse_vram_config(config_path: Path) -> dict[str, object]: + """Parse VRAM configuration from config.md.""" + defaults: dict[str, object] = { + "auto_detect": True, + "target_context_kb": 0, + "headroom_pct": DEFAULT_HEADROOM_PCT, + "max_peak_kb": 0, + } + if not config_path.exists(): + return defaults + + try: + text = config_path.read_text(encoding="utf-8", errors="replace") + except OSError: + return defaults + + for line in _iter_config_lines(text, "## VRAM Configuration"): + if line.startswith("#"): + continue + lower = line.lower() + if "auto-detect" in lower: + value = _extract_value(line) + if value: + defaults["auto_detect"] = value.lower() in ("yes", "true", "1") + elif "target" in lower and "context" in lower: + value = _extract_value(line) + if value: + parsed = _parse_token_value(value) + if parsed: + defaults["target_context_kb"] = parsed + elif "headroom" in lower: + value = _extract_value(line) + if value: + match = re.search(r"(\d+)", value) + if match: + defaults["headroom_pct"] = int(match.group(1)) + elif "max peak" in lower: + value = _extract_value(line) + if value: + parsed = _parse_token_value(value) + if parsed: + defaults["max_peak_kb"] = parsed + + return defaults + + +def recommend_context( + gpu_vram_gb: int, + ram_gb: int, + model_context_kb: int, + overhead_tokens: int, + config: dict[str, object], +) -> tuple[int, int, int]: + """Return (headroom_pct, recommended_kb, max_peak_kb).""" + headroom_pct = int(config.get("headroom_pct", DEFAULT_HEADROOM_PCT)) + + # Manual override mode. + if not config.get("auto_detect", True): + target_kb = int(config.get("target_context_kb", 0)) + max_peak_kb = int(config.get("max_peak_kb", 0)) + if target_kb > 0: + if max_peak_kb == 0 and headroom_pct > 0: + max_peak_kb = target_kb * (100 - headroom_pct) // 100 + return headroom_pct, target_kb, max_peak_kb + + recommended_kb = 0 + + if gpu_vram_gb >= 4: + # Conservative: 1GB VRAM ≈ 2k context tokens. + vram_context_kb = gpu_vram_gb * 2000 + recommended_kb = vram_context_kb * (100 - headroom_pct) // 100 + elif model_context_kb > 0: + recommended_kb = model_context_kb * (100 - headroom_pct) // 100 + else: + # RAM fallback: 0.75k tokens per GB. + ram_context_kb = ram_gb * 750 + recommended_kb = ram_context_kb * (100 - headroom_pct) // 100 + + # Subtract framework overhead. + net_kb = max(0, recommended_kb - overhead_tokens) + + # Calculate max peak context based on headroom. + max_peak_kb = net_kb * (100 - headroom_pct) // 100 + + return headroom_pct, net_kb, max_peak_kb + + +def main() -> int: + parser = argparse.ArgumentParser(description="Detect VRAM/context for automaton") + parser.add_argument("model", nargs="?", help="Model name") + parser.add_argument("--model", "-m", dest="model_flag", help="Model name") + parser.add_argument("--project", "-p", type=Path, help="Project directory") + args = parser.parse_args() + + model_name = args.model_flag or args.model + project_dir = args.project.resolve() if args.project else None + + print("=== VRAM / Context Detection ===\n") + + print("--- GPU VRAM ---") + total_vram_kb, vram_per_gpu_kb, num_gpus = detect_gpu_vram() + gpu_vram_gb = total_vram_kb // 1024 // 1024 + print(f"Total VRAM: {gpu_vram_gb}GB") + print(f"VRAM per GPU: {vram_per_gpu_kb // 1024 // 1024}GB") + print(f"Num GPUs: {num_gpus}\n") + + print("--- RAM ---") + ram_kb, _ = detect_ram() + ram_gb = ram_kb // 1024 // 1024 + print(f"RAM: {ram_gb}GB\n") + + print("--- Model Context Window ---") + model_context_kb = detect_model_context(model_name, project_dir) + print() + + print("--- Framework Overhead ---") + overhead_tokens = calculate_overhead(project_dir) + print() + + print("--- Recommendation ---") + config_path = Path.home() / ".automaton" / "config.md" + config = parse_vram_config(config_path) + headroom_pct, recommended_kb, max_peak_kb = recommend_context( + gpu_vram_gb, ram_gb, model_context_kb, overhead_tokens, config + ) + + recommended_k = recommended_kb // 1000 if recommended_kb > 0 else 8 + max_peak_k = max_peak_kb // 1000 if max_peak_kb > 0 else 6 + + print(f"Target context: {recommended_k}k tokens") + print(f"Headroom: {headroom_pct}%") + print(f"Max peak context per sub-task: {max_peak_k}k tokens") + + print("\n=== JSON Output ===") + output = { + "gpu_vram_gb": gpu_vram_gb, + "ram_gb": ram_gb, + "model_context_kb": model_context_kb, + "framework_overhead_tokens": overhead_tokens, + "recommended_kb": recommended_kb, + "recommended_k": recommended_k, + "headroom": headroom_pct / 100.0, + "max_peak_context_kb": max_peak_kb, + } + print(json.dumps(output, indent=4)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vram_detect.sh b/scripts/vram_detect.sh deleted file mode 100755 index 0fcb536..0000000 --- a/scripts/vram_detect.sh +++ /dev/null @@ -1,549 +0,0 @@ -#!/usr/bin/env bash -# VRAM/Context Detection Script -# Detects GPU VRAM, system RAM, and model context window to recommend -# a safe VRAM context window for task decomposition. -# -# Usage: ./vram_detect.sh [model_name] -# - If model_name is provided, looks up its context window -# - Otherwise, tries to detect from API config or config.md - -set -uo pipefail # Don't exit on error - we want to continue even if detection fails - -# ─── GPU VRAM Detection ─── -detect_gpu_vram() { - local total_vram_kb=0 - local vram_per_gpu_kb=0 - local num_gpus=0 - - # Try nvidia-smi first (NVIDIA GPUs) - if command -v nvidia-smi &>/dev/null; then - local vram_kb - # Use timeout to avoid hanging on nvidia-smi (e.g., driver not loaded) - vram_kb=$(timeout 5 nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ' || true) - # Validate that vram_kb is a positive number - if [[ -n "$vram_kb" && "$vram_kb" =~ ^[0-9]+$ && "$vram_kb" -gt 0 ]]; then - total_vram_kb=$((vram_kb * 1024)) # MB → KB - vram_per_gpu_kb=$((total_vram_kb / (num_gpus+1))) - num_gpus=1 - echo "GPU: NVIDIA (nvidia-smi available)" - echo "VRAM per GPU: $((vram_kb / 1024))GB ($vram_kb MB)" - else - echo "GPU: NVIDIA (nvidia-smi available but driver not responding)" - fi - fi - - # Fallback: lspci - if [[ $total_vram_kb -eq 0 && $num_gpus -eq 0 ]]; then - local gpu_info - gpu_info=$(lspci 2>/dev/null | grep -i -E 'VGA|3D|Display' | head -5) - if [[ -n "$gpu_info" ]]; then - echo "GPU detected: $gpu_info" - # Try to get VRAM from lspci -vnn memory regions - # GPUs show VRAM as Memory regions in lspci - # Parse patterns like: Memory at f800000000 (64-bit, prefetchable) [size=256M] - local total_vram_mb=0 - while IFS= read -r line; do - # Extract the size value from [size=256M] pattern - local size_num - size_num=$(echo "$line" | grep -oE '[0-9]+(M|G|K)' | head -1 || true) - if [[ -n "$size_num" ]]; then - local size_val - size_val=$(echo "$size_num" | grep -oE '[0-9]+') - local size_unit - size_unit=$(echo "$size_num" | grep -oE '(M|G|K)') - if [[ -n "$size_val" && -n "$size_unit" ]]; then - case "$size_unit" in - M) total_vram_mb=$((total_vram_mb + size_val)) ;; - G) total_vram_mb=$((total_vram_mb + size_val * 1024)) ;; - K) total_vram_mb=$((total_vram_mb + size_val / 1024)) ;; - esac - fi - fi - done < <(lspci -vnn 2>/dev/null | grep -i -A 15 -E 'VGA|3D|Display' | grep -i 'Memory at') - if [[ $total_vram_mb -gt 0 ]]; then - local total_vram_gb=$((total_vram_mb / 1024)) - local total_vram_mb_remain=$((total_vram_mb % 1024)) - echo "VRAM: $total_vram_mb MB ($total_vram_gb GB $total_vram_mb_remain MB)" - else - echo "VRAM: Could not determine from lspci" - fi - # Check for AMD GPU via amdgpu sysfs - if lspci -vnn 2>/dev/null | grep -qi 'amd\|ati'; then - local amdgpu_info - amdgpu_info=$(ls /sys/kernel/debug/amdgpu/ 2>/dev/null | head -1) - if [[ -n "$amdgpu_info" ]]; then - local vram_total - vram_total=$(cat /sys/kernel/debug/amdgpu/${amdgpu_info}/vram_total 2>/dev/null || echo 0) - if [[ "$vram_total" -gt 0 ]]; then - local vram_gb=$((vram_total / 1024 / 1024 / 1024)) - local vram_mb=$((vram_total / 1024 / 1024)) - echo "AMD GPU VRAM: ${vram_gb}GB (${vram_mb}MB)" - fi - fi - fi - fi - fi - - echo "Total VRAM: $((total_vram_kb / 1024 / 1024))GB" - echo "VRAM per GPU: $((vram_per_gpu_kb / 1024 / 1024))GB" - echo "Num GPUs: $num_gpus" -} - -# ─── System RAM Detection ─── -detect_ram() { - local total_kb=0 - local available_kb=0 - - if [[ -f /proc/meminfo ]]; then - total_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}') - available_kb=$(grep MemAvailable /proc/meminfo | awk '{print $2}') - if [[ $total_kb -gt 0 ]]; then - echo "RAM: $((total_kb / 1024 / 1024))GB total, $((available_kb / 1024 / 1024))GB available" - echo "$available_kb $total_kb" - fi - elif command -v sysctl &>/dev/null; then - total_kb=$(sysctl -n hw.memsize 2>/dev/null | awk '{print $1 / 1024}') - if [[ -n "$total_kb" && "$total_kb" -gt 0 ]]; then - echo "RAM: $((total_kb / 1024))GB total" - echo "$total_kb $total_kb" # Assume all available - fi - else - echo "RAM: Could not detect" - echo "0 0" - fi -} - -# ─── Model Context Window Detection ─── -detect_model_context() { - local model_name="$1" - local context_kb=0 - - # If model name provided, look it up - if [[ -n "$model_name" ]]; then - case "$model_name" in - gpt-4o|gpt-4o-2024-05-13|gpt-4o-2024-08-06) - context_kb=128000; echo "Model: $model_name" - echo "Context window: 128k tokens" - ;; - gpt-4o-mini|gpt-4o-mini-2024-07-18) - context_kb=128000; echo "Model: $model_name" - echo "Context window: 128k tokens" - ;; - gpt-4-turbo|gpt-4-turbo-2024-04-09) - context_kb=128000; echo "Model: $model_name" - echo "Context window: 128k tokens" - ;; - gpt-4|gpt-4-0125-preview|gpt-4-1106-preview) - context_kb=128000; echo "Model: $model_name" - echo "Context window: 128k tokens" - ;; - claude-3-5-sonnet|claude-3-5-sonnet-20241022) - context_kb=200000; echo "Model: $model_name" - echo "Context window: 200k tokens" - ;; - claude-3-5-haiku|claude-3-5-haiku-20241022) - context_kb=200000; echo "Model: $model_name" - echo "Context window: 200k tokens" - ;; - claude-3-opus|claude-3-opus-20240229) - context_kb=200000; echo "Model: $model_name" - echo "Context window: 200k tokens" - ;; - claude-3-sonnet|claude-3-sonnet-20240229) - context_kb=200000; echo "Model: $model_name" - echo "Context window: 200k tokens" - ;; - claude-3-haiku|claude-3-haiku-20240307) - context_kb=200000; echo "Model: $model_name" - echo "Context window: 200k tokens" - ;; - claude-2|claude-2.1) - context_kb=200000; echo "Model: $model_name" - echo "Context window: 200k tokens" - ;; - *) - echo "Model: $model_name (unknown context window)" - echo "0" - ;; - esac - echo "$context_kb" - return - fi - - # Try to detect from config.md (global framework model settings) - local project_dir="${1:-.}" - local config_md="${HOME}/.automaton/config.md" - local model_from_config="" - local override_context="" - if [[ -f "$config_md" ]]; then - # Check for model name - model_from_config=$(grep -i "model:" "$config_md" 2>/dev/null | grep -v "#" | grep -v "model_context" | grep -v "override" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]') - # Check for override context window - override_context=$(grep -i "override context" "$config_md" 2>/dev/null | grep -v "#" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]') - fi - - # If config.md specifies a model, use it - if [[ -n "$model_from_config" ]]; then - echo "Found model in config.md: $model_from_config" - # Use override context window if specified in config.md - if [[ -n "$override_context" && "$override_context" != "auto" ]]; then - echo "Using override context window from config.md: $override_context" - local override_kb - override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true) - if [[ -n "$override_kb" ]]; then - echo "Context window: ${override_context} tokens (override)" - echo "$((override_kb * 1000))" - return - fi - fi - detect_model_context "$model_from_config" - return - fi - - # Try to detect from .agent.md (project-level model override) - local agent_md="${project_dir}/.automaton/.agent.md" - if [[ -f "$agent_md" ]]; then - local model_line - model_line=$(grep -i "model" "$agent_md" 2>/dev/null | grep -v "#" | grep -v "target" | grep -v "headroom" | grep -v "peak" | grep -v "Auto-detect" | head -1) - if [[ -n "$model_line" ]]; then - echo "Found model in .agent.md: $model_line" - # Extract model name from the line - local model - model=$(echo "$model_line" | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]') - if [[ -n "$model" ]]; then - # Use override context window if specified in config.md - if [[ -n "$override_context" && "$override_context" != "auto" ]]; then - echo "Using override context window from config.md: $override_context" - # Convert override_context to kb (e.g., 128k -> 128000, 200k -> 200000) - local override_kb - override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true) - if [[ -n "$override_kb" ]]; then - echo "Context window: ${override_context} tokens (override)" - echo "$((override_kb * 1000))" - return - fi - fi - detect_model_context "$model" - return - fi - fi - fi - - # Try to detect from common API config files - local config_files=( - ".env" - ".env.local" - "config.yaml" - "config.yml" - "config.json" - "settings.yaml" - ".automaton/config.yaml" - ".automaton/config.json" - ) - - for config_file in "${config_files[@]}"; do - local abs_file="" - for candidate in "${project_dir}/${config_file}" "${project_dir}/.automaton/${config_file}"; do - if [[ -f "$candidate" ]]; then - abs_file="$candidate" - break - fi - done - - if [[ -n "$abs_file" ]]; then - local model - model=$(grep -i "model" "$abs_file" 2>/dev/null | grep -v "#" | grep -v "context" | grep -v "max_tokens" | grep -v "temperature" | grep -v "stream" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]') - if [[ -n "$model" ]]; then - echo "Found model in $abs_file: $model" - # Use override context window if specified in config.md - if [[ -n "$override_context" && "$override_context" != "auto" ]]; then - echo "Using override context window from config.md: $override_context" - local override_kb - override_kb=$(echo "$override_context" | sed 's/[kK]$//' | grep -oP '[0-9]+' || true) - if [[ -n "$override_kb" ]]; then - echo "Context window: ${override_context} tokens (override)" - echo "$((override_kb * 1000))" - return - fi - fi - detect_model_context "$model" - return - fi - fi - done - - echo "Model: Unknown (could not detect from .agent.md or config files)" - echo "0" -} - -# ─── Agent Framework Overhead Calculation ─── -calculate_overhead() { - local project_dir="${1:-.}" - local overhead_tokens=0 - - # Count tokens for the framework files that are loaded during orchestration - # These are the files loaded during the most common phase (orchestration): - # .agent.md + .rules.md + workflow.md + orchestrate.md - # Note: other phase files (decompose.md, implement.md, etc.) are only loaded during - # their specific phases, so they don't contribute to the peak context during orchestration. - local framework_files=( - "${project_dir}/.automaton/.agent.md" - "${project_dir}/.automaton/.rules.md" - "${project_dir}/.automaton/prompts/workflow.md" - "${project_dir}/.automaton/prompts/orchestrate.md" - ) - # Fallback: check home directory if project dir doesn't have framework - if [[ ! -f "${project_dir}/.automaton/.agent.md" ]]; then - framework_files=( - "${HOME}/.automaton/.agent.md" - "${HOME}/.automaton/.rules.md" - "${HOME}/.automaton/prompts/workflow.md" - "${HOME}/.automaton/prompts/orchestrate.md" - ) - fi - - for file in "${framework_files[@]}"; do - if [[ -f "$file" ]]; then - # Rough estimate: 1 token ≈ 4 characters (English text) - local chars - chars=$(wc -c < "$file" 2>/dev/null || echo 0) - local tokens=$((chars / 4)) - overhead_tokens=$((overhead_tokens + tokens)) - echo " ${file##*/}: ~${tokens} tokens" - fi - done - - echo "Framework overhead: ~${overhead_tokens} tokens" - echo "$overhead_tokens" -} - -# ─── Recommendation Engine ─── -recommend_context() { - local gpu_vram_gb="$1" - local ram_gb="$2" - local model_context_kb="$3" - local overhead_tokens="$4" - - # Read VRAM config from config.md if it exists - local config_md="${HOME}/.automaton/config.md" - local auto_detect="Yes" - local target_context_kb=0 - local override_headroom=25 - local override_max_peak_kb=0 - if [[ -f "$config_md" ]]; then - auto_detect=$(grep -i "auto-detect:" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | tr -d '[:space:]' || true) - target_context_kb=$(grep -i "target.*context" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true) - override_headroom=$(grep -i "headroom" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true) - override_max_peak_kb=$(grep -i "max peak" "$config_md" 2>/dev/null | grep -v "#" | grep -i "vram" | head -1 | sed -E 's/.*[:=[:space:]]+//i' | grep -oP '\d+' || true) - fi - - # If auto-detect is disabled, use the manually specified values - if [[ -n "$auto_detect" && "$auto_detect" == "No" ]]; then - if [[ -n "$target_context_kb" ]]; then - local max_peak_kb=${override_max_peak_kb:-0} - if [[ $max_peak_kb -eq 0 && $headroom_pct -gt 0 ]]; then - max_peak_kb=$((target_context_kb * (100 - headroom_pct) / 100)) - fi - echo "$headroom_pct" - echo "$target_context_kb" - echo "$max_peak_kb" - return - fi - fi - - local recommended_kb=0 - local headroom_pct=${override_headroom:-25} # Use override from config.md, or default to 25% - - # Output headroom_pct first (for parent to read) - # Then output recommended_kb - # Then output max_peak_kb (only for manual mode) - echo "$headroom_pct" - - # Recommendation logic: - # 1. If GPU VRAM >= 4GB: use VRAM (practical for local inference) - # 2. If model context window is available: use it (for API inference) - # 3. If GPU VRAM < 4GB but > 0: use RAM (VRAM too small for local inference) - # 4. If no GPU VRAM and no model: use RAM as fallback - - # If GPU VRAM >= 4GB, base it on VRAM - if [[ $gpu_vram_gb -ge 4 ]]; then - # Rule of thumb: 1GB VRAM ≈ 4k tokens for local LLMs - # But we need to leave room for the model itself - # For a model, each ~8k context tokens takes about ~3-5MB of GPU VRAM - # So VRAM available for context = VRAM - model size - agent overhead - # Conservative: 1GB VRAM ≈ 2k context tokens - local vram_context_kb=$((gpu_vram_gb * 2000)) - - # Leave headroom for the model itself and agent overhead - recommended_kb=$((vram_context_kb * (100 - headroom_pct) / 100)) - # If model context window is available, use it (for API inference) - elif [[ $model_context_kb -gt 0 ]]; then - # For API-based, we're limited by the model's context window - # But we don't want to use the full window due to overhead - recommended_kb=$((model_context_kb * (100 - headroom_pct) / 100)) - # Fallback: use RAM to estimate - else - # Moderate estimate for low-VRAM systems where VRAM is too small for local inference - # but RAM is available. Use 0.75k tokens per GB of RAM as a moderate estimate. - # This balances between being too conservative (0.5k/GB) and too generous (1k/GB). - local ram_context_kb=$((ram_gb * 750)) - recommended_kb=$((ram_context_kb * (100 - headroom_pct) / 100)) - fi - - # Subtract framework overhead - local net_kb=$((recommended_kb - overhead_tokens)) - if [[ $net_kb -lt 0 ]]; then - net_kb=0 - fi - - # Output: headroom_pct, recommended_kb - echo "$net_kb" -} - -# ─── Main ─── -main() { - local model_name="" - local project_dir="." - - # Parse arguments - while [[ $# -gt 0 ]]; do - case "$1" in - --model|-m) - model_name="$2" - shift 2 - ;; - --project|-p) - project_dir="$2" - shift 2 - ;; - *) - # Could be model name as first argument - if [[ -z "$model_name" ]]; then - model_name="$1" - fi - shift - ;; - esac - done - - echo "=== VRAM / Context Detection ===" - echo "" - - # Detect GPU VRAM - echo "--- GPU VRAM ---" - detect_gpu_vram - local gpu_vram_kb=0 - local gpu_vram_gb=0 - # Use timeout to avoid hanging - gpu_vram_kb=$(timeout 5 nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ' || true) - # Validate that vram_kb is a positive number - if [[ -z "$gpu_vram_kb" || ! "$gpu_vram_kb" =~ ^[0-9]+$ || "$gpu_vram_kb" -le 0 ]]; then - gpu_vram_kb=0 - fi - # If nvidia-smi didn't work, try to detect from lspci (AMD GPUs) - if [[ $gpu_vram_kb -eq 0 ]]; then - echo " nvidia-smi failed, checking lspci for AMD GPU VRAM..." - local total_vram_mb=0 - while IFS= read -r line; do - local size_num - size_num=$(echo "$line" | grep -oE '[0-9]+(M|G|K)' | head -1 || true) - if [[ -n "$size_num" ]]; then - local size_val - size_val=$(echo "$size_num" | grep -oE '[0-9]+') - local size_unit - size_unit=$(echo "$size_num" | grep -oE '(M|G|K)') - if [[ -n "$size_val" && -n "$size_unit" ]]; then - case "$size_unit" in - M) total_vram_mb=$((total_vram_mb + size_val)) ;; - G) total_vram_mb=$((total_vram_mb + size_val * 1024)) ;; - K) total_vram_mb=$((total_vram_mb + size_val / 1024)) ;; - esac - fi - fi - done < <(lspci -vnn 2>/dev/null | grep -i -A 15 -E 'VGA|3D|Display' | grep -i 'Memory at') - if [[ $total_vram_mb -gt 0 ]]; then - gpu_vram_kb=$((total_vram_mb * 1024)) - gpu_vram_gb=$((total_vram_mb / 1024)) - echo " AMD GPU VRAM from lspci: ${gpu_vram_gb}GB ($total_vram_mb MB)" - else - echo " No VRAM found from lspci" - fi - fi - echo "" - - # Detect RAM - echo "--- RAM ---" - detect_ram - local ram_kb - ram_kb=$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print $2}' || echo 0) - local ram_gb=$((ram_kb / 1024 / 1024)) - echo "" - - # Detect model context window - echo "--- Model Context Window ---" - detect_model_context "$model_name" - local model_context_kb - model_context_kb=$(detect_model_context "$model_name" | tail -1) - echo "" - - # Calculate framework overhead - echo "--- Framework Overhead ---" - calculate_overhead "$project_dir" - local overhead_tokens - overhead_tokens=$(calculate_overhead "$project_dir" | tail -1) - echo "" - - # Recommend context window (also outputs headroom_pct and recommended_kb) - echo "--- Recommendation ---" - local recommendation_output - recommendation_output=$(recommend_context "$gpu_vram_gb" "$ram_gb" "$model_context_kb" "$overhead_tokens") - local line_count - line_count=$(echo "$recommendation_output" | wc -l) - local recommended_kb - local headroom_pct - local max_peak_kb - if [[ $line_count -ge 3 ]]; then - # Manual mode: outputs headroom_pct, recommended_kb, max_peak_kb - headroom_pct=$(echo "$recommendation_output" | head -1 | tr -d '[:space:]') - recommended_kb=$(echo "$recommendation_output" | sed -n '2p' | tr -d '[:space:]') - max_peak_kb=$(echo "$recommendation_output" | tail -1 | tr -d '[:space:]') - else - # Auto-detect mode: outputs headroom_pct, recommended_kb - headroom_pct=$(echo "$recommendation_output" | head -1 | tr -d '[:space:]') - recommended_kb=$(echo "$recommendation_output" | tail -1 | tr -d '[:space:]') - # Calculate max peak context based on headroom - max_peak_kb=$((recommended_kb * (100 - headroom_pct) / 100)) - fi - - # Convert to human-readable - local recommended_k - if [[ $recommended_kb -gt 0 ]]; then - recommended_k=$((recommended_kb / 1000)) - else - recommended_k=8 # Default fallback - fi - - echo "" - echo "=== Recommended Configuration ===" - echo "Target context: ${recommended_k}k tokens" - echo "Headroom: ${headroom_pct}%" - echo "Max peak context per sub-task: $((recommended_k * (100 - headroom_pct) / 100))k tokens" - - # Output as JSON for programmatic use - echo "" - echo "=== JSON Output ===" - cat < None: + assert DashboardHandler._validate_task_name("good-task") is True + assert DashboardHandler._validate_task_name("bad/../task") is False + assert DashboardHandler._validate_task_name("bad\\task") is False + assert DashboardHandler._validate_task_name("") is False + + +def test_find_tasks_dir(tmp_path: Path) -> None: + (tmp_path / ".automaton" / "tasks").mkdir(parents=True) + tasks_dir = DashboardHandler._find_tasks_dir(tmp_path) + assert tasks_dir == tmp_path / ".automaton" / "tasks" + + +def test_find_tasks_dir_missing(tmp_path: Path) -> None: + tasks_dir = DashboardHandler._find_tasks_dir(tmp_path) + assert tasks_dir == tmp_path / ".automaton" / "tasks" + + +def test_path_traversal_attempt() -> None: + """Task names with path traversal should be rejected.""" + assert DashboardHandler._validate_task_name("../etc/passwd") is False + assert DashboardHandler._validate_task_name("task%2f..%2fetc") is False + + +def test_static_path_traversal_symlink(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Static file serving must reject symlinks that resolve outside the html directory.""" + html_dir = tmp_path / "html" + html_dir.mkdir() + outside = tmp_path / "secret.txt" + outside.write_text("secret") + symlink = html_dir / "link.txt" + symlink.symlink_to(outside) + + monkeypatch.setattr(DashboardHandler, "dashboard_path", html_dir) + + handler = DashboardHandler.__new__(DashboardHandler) + handler.path = "/link.txt" + errors: list[tuple[int, str]] = [] + + def capture_error(code: int, message: str) -> None: + errors.append((code, message)) + + handler._send_error = capture_error + handler._serve_static() + assert errors == [(403, "Forbidden")] + + +def test_static_valid_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Static file serving returns a valid html file.""" + html_dir = tmp_path / "html" + html_dir.mkdir() + (html_dir / "index.html").write_text("") + + monkeypatch.setattr(DashboardHandler, "dashboard_path", html_dir) + + handler = DashboardHandler.__new__(DashboardHandler) + handler.path = "/" + + response_status: list[int] = [] + response_headers: list[tuple[str, str]] = [] + + def fake_send_response(code: int) -> None: + response_status.append(code) + + def fake_send_header(key: str, value: str) -> None: + response_headers.append((key, value)) + + handler.send_response = fake_send_response + handler.send_header = fake_send_header + handler.end_headers = lambda: None + handler.wfile = io.BytesIO() + handler._send_error = lambda code, msg: None + + handler._serve_static() + assert response_status == [200] + assert any(h[0] == "Content-Type" and h[1] == "text/html" for h in response_headers) + assert handler.wfile.getvalue() == b"" diff --git a/tests/test_board.py b/tests/test_board.py new file mode 100644 index 0000000..7ffe582 --- /dev/null +++ b/tests/test_board.py @@ -0,0 +1,54 @@ +"""Tests for automaton.dashboard.core.board.""" + +from pathlib import Path + +import pytest + +from automaton.dashboard.core.board import KanbanBoard +from automaton.dashboard.core.task import determine_task_state + + +def _task(name: str, tmp_path: Path, artifacts: dict[str, str]) -> object: + from automaton.dashboard.core.task import Task + + task_dir = tmp_path / name + task_dir.mkdir(parents=True) + for filename, content in artifacts.items(): + (task_dir / filename).write_text(content) + state, artifact_map = determine_task_state(task_dir) + return Task(name=name, folder_path=task_dir, state=state, artifacts=artifact_map) + + +def test_board_columns(tmp_path: Path) -> None: + tasks = [ + _task("research", tmp_path, {"SPEC.md": "# Spec"}), + _task("done", tmp_path, {"VERDICT.md": "## Status: PASS"}), + ] + board = KanbanBoard(tasks) + assert board.columns[tasks[0].state] == [tasks[0]] + assert board.columns[tasks[1].state] == [tasks[1]] + + +def test_board_filter_by_phase(tmp_path: Path) -> None: + research = _task("research", tmp_path, {"SPEC.md": "# Spec"}) + done = _task("done", tmp_path, {"VERDICT.md": "## Status: PASS"}) + board = KanbanBoard([research, done]) + filtered = board.filter_columns(phase_filter=research.state) + assert research in filtered[research.state] + assert done not in filtered[research.state] + + +def test_board_search(tmp_path: Path) -> None: + apple = _task("apple-task", tmp_path, {"SPEC.md": "# Spec"}) + banana = _task("banana-task", tmp_path, {"SPEC.md": "# Spec"}) + board = KanbanBoard([apple, banana]) + filtered = board.filter_columns(search_query="apple") + assert apple in filtered[apple.state] + assert banana not in filtered[banana.state] + + +def test_board_wip_count(tmp_path: Path) -> None: + research = _task("research", tmp_path, {"SPEC.md": "# Spec"}) + done = _task("done", tmp_path, {"VERDICT.md": "## Status: PASS"}) + board = KanbanBoard([research, done]) + assert board.get_wip_count() == 1 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..6204b12 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,55 @@ +"""Tests for automaton.dashboard.config.""" + +import json +from pathlib import Path + +import pytest + +from automaton.dashboard.config import DashboardConfig, get_config_path + + +def test_config_defaults() -> None: + config = DashboardConfig() + assert config.auto_refresh_interval == 2 + assert config.default_view == "board" + assert config.column_width == 30 + assert config.show_timelines is True + assert config.theme == "default" + + +def test_config_validation() -> None: + config = DashboardConfig(auto_refresh_interval=0, default_view="invalid", column_width=5, theme="neon") + errors = config.validate() + assert any("auto_refresh_interval" in e for e in errors) + assert any("default_view" in e for e in errors) + assert any("column_width" in e for e in errors) + assert any("theme" in e for e in errors) + + +def test_config_from_dict() -> None: + config = DashboardConfig.from_dict({"theme": "dark", "column_width": 50}) + assert config.theme == "dark" + assert config.column_width == 50 + assert config.default_view == "board" # default preserved + + +def test_config_from_file(tmp_path: Path) -> None: + config_path = tmp_path / "dashboard-config.json" + config_path.write_text(json.dumps({"theme": "light", "auto_refresh_interval": 5})) + config = DashboardConfig.from_file(config_path) + assert config.theme == "light" + assert config.auto_refresh_interval == 5 + + +def test_config_save(tmp_path: Path) -> None: + config_path = tmp_path / "subdir" / "dashboard-config.json" + config = DashboardConfig(theme="dark") + config.save(config_path) + assert config_path.exists() + data = json.loads(config_path.read_text()) + assert data["theme"] == "dark" + + +def test_get_config_path(tmp_path: Path) -> None: + path = get_config_path(tmp_path) + assert path == tmp_path / ".automaton" / "dashboard-config.json" diff --git a/tests/test_prompt_paths.py b/tests/test_prompt_paths.py new file mode 100644 index 0000000..84b160b --- /dev/null +++ b/tests/test_prompt_paths.py @@ -0,0 +1,44 @@ +"""Regression test for task path conventions in prompts and templates.""" + +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +PROMPTS_DIR = ROOT / "prompts" +TEMPLATES_DIR = ROOT / "templates" + +# Legacy path pattern that should no longer appear. +LEGACY_PATH = re.compile(r"\{project\}/tasks/\{task-name\}/") +# Canonical path pattern that should be used instead. +CANONICAL_PATH = re.compile(r"\{project\}/\.automaton/tasks/\{task-name\}/") + + +def _markdown_files(*directories: Path) -> list[Path]: + files = [] + for directory in directories: + if directory.exists(): + files.extend(directory.rglob("*.md")) + return files + + +@pytest.mark.parametrize("path", _markdown_files(PROMPTS_DIR, TEMPLATES_DIR)) +def test_no_legacy_task_paths(path: Path) -> None: + """Every prompt/template must use the canonical .automaton/tasks path.""" + text = path.read_text(encoding="utf-8") + legacy_matches = LEGACY_PATH.findall(text) + assert not legacy_matches, ( + f"Found legacy task path in {path.relative_to(ROOT)}: {legacy_matches}\n" + "Use {project}/.automaton/tasks/{task-name}/ instead." + ) + + +def test_canonical_path_present_in_prompts() -> None: + """At least one prompt uses the canonical path (sanity check).""" + found = False + for path in _markdown_files(PROMPTS_DIR): + if CANONICAL_PATH.search(path.read_text(encoding="utf-8")): + found = True + break + assert found, "No prompt uses the canonical {project}/.automaton/tasks/{task-name}/ path" diff --git a/tests/test_scope.py b/tests/test_scope.py new file mode 100644 index 0000000..15ea67b --- /dev/null +++ b/tests/test_scope.py @@ -0,0 +1,62 @@ +"""Tests for automaton.dashboard.core.scope.""" + +from pathlib import Path + +import pytest + +from automaton.dashboard.core.scope import detect_scope, find_automaton_root + + +@pytest.fixture +def temp_project(tmp_path: Path) -> Path: + """Create a temporary automaton project.""" + (tmp_path / ".automaton").mkdir() + return tmp_path + + +@pytest.fixture +def nested_project(tmp_path: Path) -> Path: + """Create a nested automaton project.""" + project = tmp_path / "my-project" + (project / ".automaton").mkdir(parents=True) + return project + + +def test_find_automaton_root_in_project(temp_project: Path) -> None: + root = find_automaton_root(temp_project) + assert root == temp_project + + +def test_find_automaton_root_from_subdir(temp_project: Path) -> None: + subdir = temp_project / "src" / "deep" + subdir.mkdir(parents=True) + root = find_automaton_root(subdir) + assert root == temp_project + + +def test_find_automaton_root_not_found(tmp_path: Path) -> None: + root = find_automaton_root(tmp_path) + assert root is None + + +def test_detect_scope_project(temp_project: Path) -> None: + root, scope = detect_scope(temp_project) + assert root == temp_project + assert scope == "project" + + +def test_detect_scope_framework(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Framework mode is detected when .automaton is directly in home.""" + fake_home = tmp_path / "home" + fake_home.mkdir(parents=True) + (fake_home / ".automaton").mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + root, scope = detect_scope(fake_home) + assert root == fake_home + assert scope == "framework" + + +def test_detect_scope_none(tmp_path: Path) -> None: + root, scope = detect_scope(tmp_path) + assert root is None + assert scope == "none" diff --git a/tests/test_stats.py b/tests/test_stats.py new file mode 100644 index 0000000..6407d2f --- /dev/null +++ b/tests/test_stats.py @@ -0,0 +1,56 @@ +"""Tests for automaton.dashboard.core.stats.""" + +from pathlib import Path + +import pytest + +from automaton.dashboard.core.stats import TaskStats +from automaton.dashboard.core.task import determine_task_state + + +def _task(name: str, tmp_path: Path, artifacts: dict[str, str]) -> object: + from automaton.dashboard.core.task import Task + + task_dir = tmp_path / name + task_dir.mkdir(parents=True) + for filename, content in artifacts.items(): + (task_dir / filename).write_text(content) + state, artifact_map = determine_task_state(task_dir) + return Task(name=name, folder_path=task_dir, state=state, artifacts=artifact_map) + + +def test_empty_stats() -> None: + stats = TaskStats([]) + assert stats.total_tasks == 0 + assert stats.pass_rate == 0.0 + assert stats.fail_rate == 0.0 + + +def test_counts(tmp_path: Path) -> None: + tasks = [ + _task("done-1", tmp_path, {"VERDICT.md": "## Status: PASS"}), + _task("done-2", tmp_path, {"VERDICT.md": "## Status: PASS"}), + _task("blocked", tmp_path, {"VERDICT.md": "## Status: FAIL"}), + _task("research", tmp_path, {"SPEC.md": "# Spec"}), + ] + stats = TaskStats(tasks) + assert stats.total_tasks == 4 + assert stats.pass_count == 2 + assert stats.fail_count == 1 + assert stats.in_progress_count == 1 + assert stats.backlog_count == 0 + assert stats.pass_rate == 50.0 + + +def test_sub_task_stats(tmp_path: Path) -> None: + from automaton.dashboard.core.task import SubTask, TaskState + + parent = _task("parent", tmp_path, {"SPEC.md": "# Spec"}) + parent.sub_tasks = [ + SubTask(name="a", state=TaskState.DONE, has_spec=True, has_verdict=True, verdict_status="PASS"), + SubTask(name="b", state=TaskState.BLOCKED, has_spec=True, has_verdict=True, verdict_status="FAIL"), + ] + stats = TaskStats([parent]) + assert stats.sub_task_stats == { + "parent": {"total": 2, "passed": 1, "failed": 1, "needs_review": 0, "incomplete": 0} + } diff --git a/tests/test_task.py b/tests/test_task.py new file mode 100644 index 0000000..b1da237 --- /dev/null +++ b/tests/test_task.py @@ -0,0 +1,134 @@ +"""Tests for automaton.dashboard.core.task.""" + +from pathlib import Path + +import pytest + +from automaton.dashboard.core.task import ( + determine_task_state, + discover_tasks, + parse_sub_tasks, + TaskState, +) + + +def _make_task(tmp_path: Path, name: str, artifacts: dict[str, str]) -> Path: + """Create a task folder with the given artifact contents.""" + task_dir = tmp_path / name + task_dir.mkdir(parents=True) + for filename, content in artifacts.items(): + (task_dir / filename).write_text(content) + return task_dir + + +def test_backlog_state(tmp_path: Path) -> None: + task_dir = _make_task(tmp_path, "backlog-task", {}) + state, artifacts = determine_task_state(task_dir) + assert state == TaskState.BACKLOG + assert not artifacts + + +def test_research_state(tmp_path: Path) -> None: + task_dir = _make_task(tmp_path, "research-task", {"SPEC.md": "# Spec"}) + state, artifacts = determine_task_state(task_dir) + assert state == TaskState.RESEARCH + assert "SPEC.md" in artifacts + + +def test_implementation_state(tmp_path: Path) -> None: + task_dir = _make_task( + tmp_path, + "impl-task", + {"SPEC.md": "# Spec", "IMPLEMENTATION.md": "# Impl"}, + ) + state, _ = determine_task_state(task_dir) + assert state == TaskState.IMPLEMENT + + +def test_bug_find_state(tmp_path: Path) -> None: + task_dir = _make_task( + tmp_path, + "bug-task", + {"SPEC.md": "# Spec", "IMPLEMENTATION.md": "# Impl", "BUG_REPORT.md": "# Bugs"}, + ) + state, _ = determine_task_state(task_dir) + assert state == TaskState.BUG_FIND + + +def test_adv_bug_find_state(tmp_path: Path) -> None: + task_dir = _make_task( + tmp_path, + "adv-task", + { + "SPEC.md": "# Spec", + "IMPLEMENTATION.md": "# Impl", + "BUG_REPORT.md": "# Bugs", + "ADVERSARIAL_BUG_REPORT.md": "# Adv", + }, + ) + state, _ = determine_task_state(task_dir) + assert state == TaskState.ADV_BUG_FIND + + +def test_done_state(tmp_path: Path) -> None: + task_dir = _make_task( + tmp_path, + "done-task", + {"SPEC.md": "# Spec", "VERDICT.md": "## Status: PASS"}, + ) + state, _ = determine_task_state(task_dir) + assert state == TaskState.DONE + + +def test_blocked_state(tmp_path: Path) -> None: + task_dir = _make_task( + tmp_path, + "blocked-task", + {"SPEC.md": "# Spec", "VERDICT.md": "## Status: FAIL"}, + ) + state, _ = determine_task_state(task_dir) + assert state == TaskState.BLOCKED + + +def test_empty_verdict_is_blocked(tmp_path: Path) -> None: + task_dir = _make_task(tmp_path, "empty-verdict", {"VERDICT.md": ""}) + state, _ = determine_task_state(task_dir) + assert state == TaskState.BLOCKED + + +def test_parse_sub_tasks(tmp_path: Path) -> None: + parent = tmp_path / "parent" + parent.mkdir() + sub = parent / "subtasks" / "sub-a" + sub.mkdir(parents=True) + (sub / "SPEC.md").write_text("# Spec") + (sub / "VERDICT.md").write_text("## Status: PASS") + + sub_tasks = parse_sub_tasks(parent) + assert len(sub_tasks) == 1 + assert sub_tasks[0].name == "sub-a" + assert sub_tasks[0].has_verdict is True + assert sub_tasks[0].verdict_status == "PASS" + + +def test_discover_tasks(tmp_path: Path) -> None: + _make_task(tmp_path, "aaa", {"SPEC.md": "# Spec"}) + _make_task(tmp_path, "zzz", {"VERDICT.md": "## Status: PASS"}) + tasks = discover_tasks(tmp_path) + assert len(tasks) == 2 + # Done task should be first (sorted by advancement). + assert tasks[0].state == TaskState.DONE + assert tasks[1].state == TaskState.RESEARCH + + +def test_discover_tasks_skips_subtasks_root(tmp_path: Path) -> None: + parent = tmp_path / "parent" + parent.mkdir() + (parent / "SPEC.md").write_text("# Spec") + (parent / "subtasks" / "sub-a").mkdir(parents=True) + (parent / "subtasks" / "sub-a" / "SPEC.md").write_text("# Sub") + + tasks = discover_tasks(tmp_path) + assert len(tasks) == 1 + assert tasks[0].name == "parent" + assert len(tasks[0].sub_tasks) == 1 diff --git a/tests/test_vram_detect.py b/tests/test_vram_detect.py new file mode 100644 index 0000000..2a6bff4 --- /dev/null +++ b/tests/test_vram_detect.py @@ -0,0 +1,110 @@ +"""Tests for scripts/vram_detect.py.""" + +from pathlib import Path + +import pytest + +import scripts.vram_detect as vram + + +def test_lookup_model_context() -> None: + assert vram._lookup_model_context("gpt-4o") == 128_000 + assert vram._lookup_model_context("claude-3-5-sonnet") == 200_000 + assert vram._lookup_model_context("unknown-model") == 0 + + +def test_parse_token_value() -> None: + assert vram._parse_token_value("128k") == 128_000 + assert vram._parse_token_value("128K") == 128_000 + assert vram._parse_token_value("128000") == 128_000 + assert vram._parse_token_value("nonsense") is None + + +def test_extract_value() -> None: + assert vram._extract_value("- **Model**: gpt-4o") == "gpt-4o" + assert vram._extract_value("model = gpt-4o # comment") == "gpt-4o" + assert vram._extract_value("- **Model**: auto") == "auto" + + +def test_parse_config_model(tmp_path: Path) -> None: + config = tmp_path / "config.md" + config.write_text( + "## VRAM Configuration\n" + "- **Auto-detect**: Yes\n" + "- **Target context**: 16k tokens\n" + "- **Headroom**: 25%\n" + "\n" + "## Model Configuration\n" + "- **Model**: gpt-4o\n" + "- **Override context window**: 128k\n" + ) + model, override = vram._parse_config_model(config) + assert model == "gpt-4o" + assert override == 128_000 + + +def test_parse_config_model_skips_code_blocks(tmp_path: Path) -> None: + """Example code blocks should not be parsed as live config.""" + config = tmp_path / "config.md" + config.write_text( + "## VRAM Configuration\n" + "- **Headroom**: 25%\n" + "```\n" + "- **Headroom**: 99%\n" + "```\n" + ) + parsed = vram.parse_vram_config(config) + assert parsed["headroom_pct"] == 25 + + +def test_parse_vram_config_manual(tmp_path: Path) -> None: + config = tmp_path / "config.md" + config.write_text( + "## VRAM Configuration\n" + "- **Auto-detect**: No\n" + "- **Target context**: 8k\n" + "- **Headroom**: 30%\n" + "- **Max peak context per sub-task**: 5.6k\n" + ) + parsed = vram.parse_vram_config(config) + assert parsed["auto_detect"] is False + assert parsed["target_context_kb"] == 8000 + assert parsed["headroom_pct"] == 30 + assert parsed["max_peak_kb"] == 5600 + + +def test_recommend_context_api_model() -> None: + config = {"auto_detect": True, "headroom_pct": 25, "target_context_kb": 0, "max_peak_kb": 0} + _, recommended_kb, max_peak_kb = vram.recommend_context( + gpu_vram_gb=0, + ram_gb=16, + model_context_kb=128_000, + overhead_tokens=4000, + config=config, + ) + assert recommended_kb > 0 + assert max_peak_kb > 0 + assert recommended_kb <= 128_000 * 0.75 # after headroom + + +def test_recommend_context_manual_mode() -> None: + config = {"auto_detect": False, "headroom_pct": 30, "target_context_kb": 8000, "max_peak_kb": 0} + headroom, recommended_kb, max_peak_kb = vram.recommend_context( + gpu_vram_gb=0, + ram_gb=16, + model_context_kb=0, + overhead_tokens=0, + config=config, + ) + assert headroom == 30 + assert recommended_kb == 8000 + assert max_peak_kb == 5600 + + +def test_extract_model_from_file_respects_10kb_limit(tmp_path: Path) -> None: + """Only the first 10KB of an API config file is scanned.""" + env_file = tmp_path / ".env" + # Put the model name far beyond 10KB. + env_file.write_text("x" * 11_000 + "\nMODEL=far-away-model\n") + model = vram._extract_model_from_file(env_file) + assert model is None