State Enforcement (v2.0):
- .state file as single source of truth for task phase
- Approval gates for research, decomposition, design, test_design
- status.py --transition refuses illegal phase transitions
- status.py --validate-folder detects out-of-order artifacts
- status.py --audit checks all tasks for violations
- status.py --create-task is the only valid way to create tasks
- Pre-v2.0 tasks without .state are UNTRACKED -- all commands refuse them
- New --upgrade command bootstraps .state files for existing tasks
Project Scoping:
- --project flag added to all status.py commands across 16+ files
- _find_project_dir errors instead of silently falling back to ~/.automaton/
- --scope-check marks framework files OUT_OF_SCOPE when working on a project
- Dashboard handlers use stored project_root instead of re-detecting from CWD
- Prompts reference ~/.automaton/scripts/vram_detect.py (not {project}/.automaton/)
Harness Integration:
- status.py --can-edit now supports project-level checks (no --task required)
- --can-edit --file checks file scope without --task
- --json output for machine-readable harness integration
- opencode plugin (plugins/automaton-guard/plugin.ts) intercepts edit/write
- Git pre-commit hook (scripts/git-hooks/pre-commit) blocks commits without task
- Formal integration contract (contracts/harness-integration.md)
Other:
- upgrade.sh delegates to status.py --upgrade instead of manual heuristics
- Phase prompts reference --project {project} for multi-project scoping
- 200 tests passing (14 new)
This commit is contained in:
@@ -0,0 +1,254 @@
|
||||
"""Framework self-consistency tests.
|
||||
|
||||
These tests enforce structural rules on the framework itself — prompt
|
||||
consistency, canonical paths, state machine integrity, and configuration
|
||||
validity. They catch regressions that unit tests alone cannot.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
PROMPTS_DIR = ROOT / "prompts"
|
||||
RULES_FILE = ROOT / ".rules.md"
|
||||
PYPROJECT_FILE = ROOT / "pyproject.toml"
|
||||
CI_FILE = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
CSS_FILE = ROOT / "automaton" / "dashboard" / "html" / "styles.css"
|
||||
JS_FILE = ROOT / "automaton" / "dashboard" / "html" / "dashboard.js"
|
||||
|
||||
|
||||
class TestDeliveryPromptsHaveStopConditions:
|
||||
"""R1.A: Every delivery prompt must contain a stop condition block."""
|
||||
|
||||
EXCLUDED = {"orchestrate.md", "compaction.md", "workflow.md", "subtask_management.md", "onboarding.md"}
|
||||
|
||||
@pytest.fixture()
|
||||
def delivery_prompts(self):
|
||||
if not PROMPTS_DIR.exists():
|
||||
pytest.skip("prompts/ directory not found")
|
||||
return [
|
||||
f
|
||||
for f in sorted(PROMPTS_DIR.iterdir())
|
||||
if f.is_file() and f.suffix == ".md" and f.name not in self.EXCLUDED
|
||||
]
|
||||
|
||||
def test_each_prompt_has_stop_condition(self, delivery_prompts):
|
||||
for prompt in delivery_prompts:
|
||||
content = prompt.read_text()
|
||||
has_stop = "## Stop Condition" in content or "STOP CONDITION" in content or "CONTRACT_MET" in content
|
||||
assert has_stop, f"{prompt.name} is missing a stop condition block"
|
||||
|
||||
|
||||
class TestNoHardcodedURLs:
|
||||
"""R1.B: Prompts, contracts, and templates must not contain hardcoded repo URLs."""
|
||||
|
||||
ALLOWED_FILES = {"install.sh"}
|
||||
|
||||
def _check_dir(self, directory: Path, pattern: re.Pattern):
|
||||
violations = []
|
||||
if not directory.exists():
|
||||
return violations
|
||||
for f in directory.rglob("*.md"):
|
||||
if f.name in self.ALLOWED_FILES:
|
||||
continue
|
||||
content = f.read_text()
|
||||
for line_no, line in enumerate(content.splitlines(), 1):
|
||||
if pattern.search(line):
|
||||
violations.append(f"{f.relative_to(ROOT)}:{line_no}")
|
||||
return violations
|
||||
|
||||
def test_no_hardcoded_ip_urls(self):
|
||||
pattern = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}[:/]")
|
||||
dirs = [PROMPTS_DIR, ROOT / "contracts", ROOT / "templates"]
|
||||
violations = []
|
||||
for d in dirs:
|
||||
violations.extend(self._check_dir(d, pattern))
|
||||
assert not violations, f"Hardcoded IP URLs found in: {violations}"
|
||||
|
||||
def test_no_hardcoded_localhost_ports(self):
|
||||
pattern = re.compile(r"localhost:\d{4,5}")
|
||||
dirs = [PROMPTS_DIR, ROOT / "contracts", ROOT / "templates"]
|
||||
violations = []
|
||||
for d in dirs:
|
||||
violations.extend(self._check_dir(d, pattern))
|
||||
assert not violations, f"Hardcoded localhost URLs found in: {violations}"
|
||||
|
||||
|
||||
class TestRulesMdSections:
|
||||
"""R1.C & R1.D: .rules.md must contain mandatory sections."""
|
||||
|
||||
MANDATORY_SECTIONS = [
|
||||
"Task-Driven Development",
|
||||
"VRAM",
|
||||
"Changelog",
|
||||
"Session Discipline",
|
||||
"Scope Confinement",
|
||||
"Artifact Integrity",
|
||||
]
|
||||
|
||||
def test_mandatory_sections_present(self):
|
||||
if not RULES_FILE.exists():
|
||||
pytest.skip(".rules.md not found")
|
||||
content = RULES_FILE.read_text()
|
||||
for section in self.MANDATORY_SECTIONS:
|
||||
assert section in content, f".rules.md missing mandatory section: {section}"
|
||||
|
||||
def test_self_improvement_has_example(self):
|
||||
if not RULES_FILE.exists():
|
||||
pytest.skip(".rules.md not found")
|
||||
content = RULES_FILE.read_text()
|
||||
assert "Past failure" in content, ".rules.md Self-Improvement section must reference at least one real failure mode"
|
||||
|
||||
|
||||
class TestCanonicalTaskPaths:
|
||||
"""R1.E: All task path references in prompts must use the canonical format."""
|
||||
|
||||
CANONICAL_PATTERN = re.compile(r"\{project\}/\.automaton/tasks/\w")
|
||||
DEPRECATED_PATTERN = re.compile(r"\{project\}/tasks/[\w-]+/")
|
||||
|
||||
def test_no_deprecated_task_paths(self):
|
||||
if not PROMPTS_DIR.exists():
|
||||
pytest.skip("prompts/ directory not found")
|
||||
violations = []
|
||||
for f in sorted(PROMPTS_DIR.iterdir()):
|
||||
if not f.is_file() or f.suffix != ".md":
|
||||
continue
|
||||
content = f.read_text()
|
||||
for line_no, line in enumerate(content.splitlines(), 1):
|
||||
if self.DEPRECATED_PATTERN.search(line):
|
||||
violations.append(f"{f.name}:{line_no}: {line.strip()}")
|
||||
assert not violations, f"Deprecated task paths found: {violations}"
|
||||
|
||||
|
||||
class TestPyprojectNoStaleExtras:
|
||||
"""R1.F: pyproject.toml must not reference inotify."""
|
||||
|
||||
def test_no_inotify_dependency(self):
|
||||
if not PYPROJECT_FILE.exists():
|
||||
pytest.skip("pyproject.toml not found")
|
||||
content = PYPROJECT_FILE.read_text()
|
||||
assert "inotify" not in content, "pyproject.toml still references inotify"
|
||||
|
||||
|
||||
class TestDashboardCSSThemes:
|
||||
"""R1.G: Light and dark themes must define the same variable set."""
|
||||
|
||||
def _extract_vars(self, section: str) -> set:
|
||||
pattern = re.compile(r"--([\w-]+)\s*:", re.MULTILINE)
|
||||
return set(pattern.findall(section))
|
||||
|
||||
def test_theme_variable_parity(self):
|
||||
if not CSS_FILE.exists():
|
||||
pytest.skip("styles.css not found")
|
||||
content = CSS_FILE.read_text()
|
||||
root_match = re.search(r":root\s*\{([^}]+)\}", content, re.DOTALL)
|
||||
light_match = re.search(r'\[data-theme="light"\]\s*\{([^}]+)\}', content, re.DOTALL)
|
||||
dark_match = re.search(r'\[data-theme="dark"\]\s*\{([^}]+)\}', content, re.DOTALL)
|
||||
if not root_match:
|
||||
pytest.skip(":root CSS variables not found")
|
||||
root_vars = self._extract_vars(root_match.group(1))
|
||||
theme_vars = set()
|
||||
if light_match:
|
||||
theme_vars |= self._extract_vars(light_match.group(1))
|
||||
if dark_match:
|
||||
theme_vars |= self._extract_vars(dark_match.group(1))
|
||||
if not theme_vars:
|
||||
pytest.skip("No theme sections found in CSS")
|
||||
structural_vars = {"radius-sm", "radius-md", "radius-lg"}
|
||||
themable_root_vars = root_vars - structural_vars
|
||||
missing_from_themes = themable_root_vars - theme_vars
|
||||
assert not missing_from_themes, f"CSS variables defined in :root but missing from theme overrides: {missing_from_themes}"
|
||||
|
||||
|
||||
class TestVerdictParsingRegression:
|
||||
"""R3: Verdict parsing regression tests for the critical false-BLOCKED bug."""
|
||||
|
||||
def test_pass_verdict_mentioning_fail_is_done(self):
|
||||
from automaton.dashboard.core.task import determine_task_state, TaskState
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
task_dir = Path(tmp) / "test-task"
|
||||
task_dir.mkdir()
|
||||
(task_dir / "VERDICT.md").write_text("## Status: PASS\n\nThe bug in the FAIL case is now fixed.\n")
|
||||
state, _ = determine_task_state(task_dir)
|
||||
assert state == TaskState.DONE, f"Expected DONE, got {state}"
|
||||
|
||||
def test_pass_verdict_with_needs_review_mention_is_done(self):
|
||||
from automaton.dashboard.core.task import determine_task_state, TaskState
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
task_dir = Path(tmp) / "test-task"
|
||||
task_dir.mkdir()
|
||||
(task_dir / "VERDICT.md").write_text("## Status: PASS\n\nPreviously flagged as NEEDS_REVIEW but resolved.\n")
|
||||
state, _ = determine_task_state(task_dir)
|
||||
assert state == TaskState.DONE, f"Expected DONE, got {state}"
|
||||
|
||||
def test_structured_fail_verdict_is_blocked(self):
|
||||
from automaton.dashboard.core.task import determine_task_state, TaskState
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
task_dir = Path(tmp) / "test-task"
|
||||
task_dir.mkdir()
|
||||
(task_dir / "VERDICT.md").write_text("## Status: FAIL\n\nThe implementation has a critical bug.\n")
|
||||
state, _ = determine_task_state(task_dir)
|
||||
assert state == TaskState.BLOCKED
|
||||
|
||||
def test_implementation_alone_is_bug_find(self):
|
||||
from automaton.dashboard.core.task import determine_task_state, TaskState
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
task_dir = Path(tmp) / "test-task"
|
||||
task_dir.mkdir()
|
||||
(task_dir / "IMPLEMENTATION.md").write_text("# Implementation\nDone.\n")
|
||||
state, _ = determine_task_state(task_dir)
|
||||
assert state == TaskState.BUG_FIND
|
||||
|
||||
def test_empty_verdict_is_blocked(self):
|
||||
from automaton.dashboard.core.task import determine_task_state, TaskState
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
task_dir = Path(tmp) / "test-task"
|
||||
task_dir.mkdir()
|
||||
(task_dir / "VERDICT.md").write_text("")
|
||||
state, _ = determine_task_state(task_dir)
|
||||
assert state == TaskState.BLOCKED
|
||||
|
||||
def test_adv_bug_report_alone_is_bug_find(self):
|
||||
from automaton.dashboard.core.task import determine_task_state, TaskState
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
task_dir = Path(tmp) / "test-task"
|
||||
task_dir.mkdir()
|
||||
(task_dir / "ADVERSARIAL_BUG_REPORT.md").write_text("# Bug Report\nFound issue.\n")
|
||||
state, _ = determine_task_state(task_dir)
|
||||
assert state == TaskState.BUG_FIND
|
||||
|
||||
|
||||
class TestCIWorkflowValidation:
|
||||
"""R4: CI workflow must compile, test, and check shell scripts."""
|
||||
|
||||
def test_ci_runs_py_compile(self):
|
||||
if not CI_FILE.exists():
|
||||
pytest.skip("CI workflow not found")
|
||||
content = CI_FILE.read_text()
|
||||
assert "py_compile" in content, "CI must run py_compile"
|
||||
|
||||
def test_ci_runs_pytest(self):
|
||||
if not CI_FILE.exists():
|
||||
pytest.skip("CI workflow not found")
|
||||
content = CI_FILE.read_text()
|
||||
assert "pytest" in content, "CI must run pytest"
|
||||
|
||||
def test_ci_checks_shell_scripts(self):
|
||||
if not CI_FILE.exists():
|
||||
pytest.skip("CI workflow not found")
|
||||
content = CI_FILE.read_text()
|
||||
assert "bash -n" in content, "CI must syntax-check shell scripts"
|
||||
Reference in New Issue
Block a user