Files

765 lines
34 KiB
Python
Raw Permalink Normal View History

"""Tests for status.py enforcement script."""
import os
import sys
import tempfile
import pytest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
@pytest.fixture
def tmp_project(tmp_path):
"""Create a temporary project with .automaton/tasks structure."""
auto_dir = tmp_path / ".automaton"
tasks_dir = auto_dir / "tasks"
tasks_dir.mkdir(parents=True)
return tmp_path
@pytest.fixture
def status_script():
"""Return path to status.py."""
return Path.home() / ".automaton" / "scripts" / "status.py"
def _run_status(args, project=None):
"""Run status.py with given args and return (stdout, exit_code)."""
import subprocess
cmd = [sys.executable, str(Path.home() / ".automaton" / "scripts" / "status.py")]
if project:
cmd.extend(["--project", str(project)])
cmd.extend(args)
result = subprocess.run(cmd, capture_output=True, text=True)
return result.stdout.strip(), result.returncode
def _create_task(project, task_name, phase=None):
"""Create a task and optionally set its phase."""
out, code = _run_status(["--create-task", task_name], project)
assert code == 0, f"Failed to create task: {out}"
if phase:
task_dir = project / ".automaton" / "tasks" / task_name
(task_dir / ".state").write_text(f"{phase}\n")
return project / ".automaton" / "tasks" / task_name
class TestCreateTask:
def test_create_valid_task(self, tmp_project):
out, code = _run_status(["--create-task", "my-test-task"], tmp_project)
assert code == 0
assert "Created task 'my-test-task'" in out
task_dir = tmp_project / ".automaton" / "tasks" / "my-test-task"
assert task_dir.exists()
assert (task_dir / ".state").read_text().strip() == "new"
assert (task_dir / ".state.approvals").exists()
def test_create_task_rejects_spaces(self, tmp_project):
out, code = _run_status(["--create-task", "my test task"], tmp_project)
assert code == 2
assert "kebab-case" in out
def test_create_task_rejects_uppercase(self, tmp_project):
out, code = _run_status(["--create-task", "My-Task"], tmp_project)
assert code == 2
def test_create_task_rejects_duplicate(self, tmp_project):
_run_status(["--create-task", "my-task"], tmp_project)
out, code = _run_status(["--create-task", "my-task"], tmp_project)
assert code == 2
assert "already exists" in out
class TestStateTransitions:
def test_transition_new_to_research(self, tmp_project):
task_dir = _create_task(tmp_project, "trans-test", "new")
out, code = _run_status(["--task", "trans-test", "--transition", "research"], tmp_project)
assert code == 0
assert "Transitioned" in out
assert (task_dir / ".state").read_text().strip() == "research"
def test_illegal_transition_refused(self, tmp_project):
task_dir = _create_task(tmp_project, "illegal-test", "new")
out, code = _run_status(["--task", "illegal-test", "--transition", "implement"], tmp_project)
assert code == 1
assert "Cannot transition" in out
def test_approval_gate_blocks_transition(self, tmp_project):
task_dir = _create_task(tmp_project, "approval-test", "research:awaiting_approval")
out, code = _run_status(["--task", "approval-test", "--transition", "design"], tmp_project)
assert code == 1
assert "research:approved" in out
class TestApprovalGates:
def test_approve_transitions_to_approved(self, tmp_project):
task_dir = _create_task(tmp_project, "approve-test", "research:awaiting_approval")
out, code = _run_status(["--approve", "--task", "approve-test"], tmp_project)
assert code == 0
assert "Approved" in out
assert (task_dir / ".state").read_text().strip() == "research:approved"
def test_approve_records_in_log(self, tmp_project):
task_dir = _create_task(tmp_project, "log-test", "research:awaiting_approval")
_run_status(["--approve", "--task", "log-test"], tmp_project)
approvals = (task_dir / ".state.approvals").read_text()
assert "research:approved" in approvals
assert "user" in approvals
def test_approve_refused_for_wrong_phase(self, tmp_project):
_create_task(tmp_project, "wrong-phase", "implement")
out, code = _run_status(["--approve", "--task", "wrong-phase"], tmp_project)
assert "does not require approval" in out
def test_approve_refused_without_awaiting(self, tmp_project):
_create_task(tmp_project, "no-await", "research")
out, code = _run_status(["--approve", "--task", "no-await"], tmp_project)
assert code == 1
assert "not awaiting approval" in out
class TestValidateFolder:
def test_valid_folder_passes(self, tmp_project):
task_dir = _create_task(tmp_project, "valid-test", "research")
# Write SPEC.md (expected artifact for research)
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--validate-folder", "--task", "valid-test"], tmp_project)
assert "PASS" in out
def test_folder_without_state_flagged(self, tmp_project):
tasks_dir = tmp_project / ".automaton" / "tasks"
task_dir = tasks_dir / "manual-task"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--validate-folder", "--task", "manual-task"], tmp_project)
assert "no .state file" in out or "manually" in out
def test_out_of_order_artifacts_flagged(self, tmp_project):
task_dir = _create_task(tmp_project, "skip-test", "research")
(task_dir / "SPEC.md").write_text("# Spec")
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
out, code = _run_status(["--validate-folder", "--task", "skip-test"], tmp_project)
assert "FAIL" in out
assert "IMPLEMENTATION.md" in out
class TestArtifactValidation:
def test_missing_required_artifact_blocks_transition(self, tmp_project):
_create_task(tmp_project, "no-spec", "research")
out, code = _run_status(["--task", "no-spec", "--transition", "design"], tmp_project)
assert code == 1
assert "SPEC.md" in out
def test_forbidden_artifact_blocks_transition(self, tmp_project):
task_dir = _create_task(tmp_project, "skip-artifact", "research")
(task_dir / "SPEC.md").write_text("# Spec")
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
out, code = _run_status(["--task", "skip-artifact", "--transition", "design"], tmp_project)
assert code == 1
assert "out-of-order" in out.lower() or "forbidden" in out.lower()
class TestShowTask:
def test_show_task_displays_phase(self, tmp_project):
_create_task(tmp_project, "show-test", "implement")
out, code = _run_status(["--task", "show-test"], tmp_project)
assert code == 0
assert "Phase: implement" in out
def test_show_task_displays_allowed_forbidden(self, tmp_project):
_create_task(tmp_project, "show-impl", "implement")
out, code = _run_status(["--task", "show-impl"], tmp_project)
assert "Allowed actions" in out
assert "Edit code" in out
assert "Forbidden actions" in out
class TestList:
def test_list_shows_tasks(self, tmp_project):
_create_task(tmp_project, "list-a", "research")
_create_task(tmp_project, "list-b", "implement")
out, code = _run_status(["--list"], tmp_project)
assert code == 0
assert "list-a" in out
assert "list-b" in out
class TestToolHooks:
def test_can_edit_denied_in_research(self, tmp_project):
_create_task(tmp_project, "edit-no", "research")
out, code = _run_status(["--can-edit", "--task", "edit-no"], tmp_project)
assert code == 1
assert "DENIED" in out
def test_can_edit_allowed_in_implement(self, tmp_project):
_create_task(tmp_project, "edit-yes", "implement")
out, code = _run_status(["--can-edit", "--task", "edit-yes"], tmp_project)
assert code == 0
assert "ALLOWED" in out
def test_scope_check_in_scope(self, tmp_project):
test_file = tmp_project / "src" / "main.py"
test_file.parent.mkdir(parents=True, exist_ok=True)
test_file.write_text("# test")
out, code = _run_status(["--scope-check", "--task", "scope-test", "--file", str(test_file)], tmp_project)
assert "IN_SCOPE" in out
def test_scope_check_out_of_scope(self, tmp_project):
_create_task(tmp_project, "scope-test", "research")
out, code = _run_status(["--scope-check", "--task", "scope-test", "--file", "/tmp/some_random_file.py"], tmp_project)
assert code == 1
assert "OUT_OF_SCOPE" in out
def test_scope_check_framework_out_of_scope_for_project(self, tmp_project):
_create_task(tmp_project, "scope-proj", "research")
framework_file = Path.home() / ".automaton" / "scripts" / "status.py"
if framework_file.exists():
out, code = _run_status(["--scope-check", "--task", "scope-proj", "--file", str(framework_file)], tmp_project)
assert code == 1
assert "OUT_OF_SCOPE" in out
class TestProjectScoping:
def test_no_project_errors_without_flag(self):
import subprocess
cmd = [sys.executable, str(Path.home() / ".automaton" / "scripts" / "status.py"), "--list"]
result = subprocess.run(cmd, capture_output=True, text=True, cwd="/tmp")
assert result.returncode != 0
def test_project_flag_targets_correct_tasks(self, tmp_project):
_create_task(tmp_project, "scoped-task", "research")
out, code = _run_status(["--list"], tmp_project)
assert code == 0
assert "scoped-task" in out
class TestUntrackedTasks:
def test_transition_refuses_untracked_task(self, tmp_project):
tasks_dir = tmp_project / ".automaton" / "tasks"
task_dir = tasks_dir / "untracked-task"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--transition", "design", "--task", "untracked-task"], tmp_project)
assert code == 1
assert "no .state file" in out
def test_can_edit_refuses_untracked_task(self, tmp_project):
tasks_dir = tmp_project / ".automaton" / "tasks"
task_dir = tasks_dir / "untracked-edit"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--can-edit", "--task", "untracked-edit"], tmp_project)
assert code == 1
assert "no .state file" in out
def test_show_task_refuses_untracked_task(self, tmp_project):
tasks_dir = tmp_project / ".automaton" / "tasks"
task_dir = tasks_dir / "untracked-show"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--task", "untracked-show"], tmp_project)
assert code == 1
assert "no .state file" in out
def test_list_shows_untracked_task(self, tmp_project):
tasks_dir = tmp_project / ".automaton" / "tasks"
task_dir = tasks_dir / "untracked-list"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--list"], tmp_project)
assert code == 0
assert "UNTRACKED" in out
def test_upgrade_bootstraps_state_file(self, tmp_project):
tasks_dir = tmp_project / ".automaton" / "tasks"
task_dir = tasks_dir / "upgrade-test"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--upgrade", "--task", "upgrade-test"], tmp_project)
assert code == 0
assert "Bootstrapped" in out
state_file = task_dir / ".state"
assert state_file.exists()
def test_upgrade_all_tasks(self, tmp_project):
tasks_dir = tmp_project / ".automaton" / "tasks"
t1 = tasks_dir / "task-a"
t1.mkdir()
(t1 / "SPEC.md").write_text("# Spec")
t2 = tasks_dir / "task-b"
t2.mkdir()
out, code = _run_status(["--upgrade"], tmp_project)
assert code == 0
assert (t1 / ".state").exists()
assert (t2 / ".state").exists()
class TestCanEditProject:
def test_can_edit_no_task_allowed_when_implement(self, tmp_project):
_create_task(tmp_project, "edit-impl", "implement")
out, code = _run_status(["--can-edit"], tmp_project)
assert code == 0
assert "ALLOWED" in out
assert "edit-impl" in out
def test_can_edit_no_task_allowed_when_doc_review(self, tmp_project):
_create_task(tmp_project, "edit-doc", "doc_review")
out, code = _run_status(["--can-edit"], tmp_project)
assert code == 0
assert "ALLOWED" in out
assert "edit-doc" in out
def test_can_edit_no_task_denied_when_no_edit_phase(self, tmp_project):
_create_task(tmp_project, "edit-res", "research")
out, code = _run_status(["--can-edit"], tmp_project)
assert code == 1
assert "DENIED" in out
def test_can_edit_no_task_denied_when_no_tasks(self, tmp_project):
out, code = _run_status(["--can-edit"], tmp_project)
assert code == 1
assert "DENIED" in out
def test_can_edit_with_file_scope(self, tmp_project):
_create_task(tmp_project, "edit-scope", "implement")
test_file = tmp_project / "src" / "main.py"
test_file.parent.mkdir(parents=True, exist_ok=True)
test_file.write_text("# test")
out, code = _run_status(["--can-edit", "--file", str(test_file)], tmp_project)
assert code == 0
assert "ALLOWED" in out
def test_can_edit_with_file_out_of_scope(self, tmp_project):
_create_task(tmp_project, "edit-scope-out", "implement")
out, code = _run_status(["--can-edit", "--file", "/tmp/some_random_file.py"], tmp_project)
assert code == 1
assert "outside project" in out
def test_can_edit_json_output(self, tmp_project):
_create_task(tmp_project, "edit-json", "implement")
out, code = _run_status(["--can-edit", "--json"], tmp_project)
assert code == 0
assert '"allowed": true' in out.lower() or '"allowed": True' in out
def test_can_edit_json_denied(self, tmp_project):
_create_task(tmp_project, "edit-json-denied", "research")
out, code = _run_status(["--can-edit", "--json"], tmp_project)
assert code == 1
assert '"allowed": false' in out.lower() or '"allowed": False' in out
class TestAudit:
def test_audit_reports_manual_task(self, tmp_project):
tasks_dir = tmp_project / ".automaton" / "tasks"
task_dir = tasks_dir / "manual-only"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--audit"], tmp_project)
assert "manually created" in out
def test_audit_clean_task(self, tmp_project):
task_dir = _create_task(tmp_project, "clean-audit", "research")
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--audit"], tmp_project)
assert "PASS" in out
class TestInferStateFromArtifacts:
def test_spec_only_infers_research(self, tmp_project):
task_dir = tmp_project / ".automaton" / "tasks" / "spec-only"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--upgrade", "--task", "spec-only"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "research"
def test_implementation_only_infers_code_review(self, tmp_project):
task_dir = tmp_project / ".automaton" / "tasks" / "impl-only"
task_dir.mkdir()
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
out, code = _run_status(["--upgrade", "--task", "impl-only"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "code_review"
def test_bug_report_only_infers_adversarial_bug_find(self, tmp_project):
task_dir = tmp_project / ".automaton" / "tasks" / "bug-only"
task_dir.mkdir()
(task_dir / "BUG_REPORT.md").write_text("# Bugs")
out, code = _run_status(["--upgrade", "--task", "bug-only"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "adversarial_bug_find"
def test_spec_and_decomposition_infers_decomposition(self, tmp_project):
task_dir = tmp_project / ".automaton" / "tasks" / "decomp-test"
task_dir.mkdir()
(task_dir / "SPEC.md").write_text("# Spec")
(task_dir / "DECOMPOSITION.md").write_text("# Decomp")
out, code = _run_status(["--upgrade", "--task", "decomp-test"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "decomposition"
class TestValidateFolderCorruptedState:
def test_corrupted_empty_state_file_errors(self, tmp_project):
task_dir = tmp_project / ".automaton" / "tasks" / "corrupt-test"
task_dir.mkdir()
(task_dir / ".state").write_text("")
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--validate-folder", "--task", "corrupt-test"], tmp_project)
assert code == 1
assert "corrupted" in out.lower()
def test_whitespace_state_file_errors(self, tmp_project):
task_dir = tmp_project / ".automaton" / "tasks" / "ws-test"
task_dir.mkdir()
(task_dir / ".state").write_text(" \n")
(task_dir / "SPEC.md").write_text("# Spec")
out, code = _run_status(["--validate-folder", "--task", "ws-test"], tmp_project)
assert code == 1
assert "corrupted" in out.lower()
class TestCodeReviewPhase:
def test_transition_implement_to_code_review(self, tmp_project):
task_dir = _create_task(tmp_project, "cr-trans", "implement")
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
out, code = _run_status(["--task", "cr-trans", "--transition", "code_review"], tmp_project)
assert code == 0
assert "Transitioned" in out
assert (task_dir / ".state").read_text().strip() == "code_review"
def test_transition_code_review_to_awaiting_approval(self, tmp_project):
task_dir = _create_task(tmp_project, "cr-await", "code_review")
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
(task_dir / "CODE_REVIEW.md").write_text("# Code Review")
out, code = _run_status(["--task", "cr-await", "--transition", "code_review:awaiting_approval"], tmp_project)
assert code == 0
assert (task_dir / ".state").read_text().strip() == "code_review:awaiting_approval"
def test_code_review_approval_gate_blocks_skip(self, tmp_project):
task_dir = _create_task(tmp_project, "cr-approval-block", "code_review:awaiting_approval")
out, code = _run_status(["--task", "cr-approval-block", "--transition", "bug_find"], tmp_project)
assert code == 1
assert "code_review:approved" in out
def test_code_review_approved_to_bug_find(self, tmp_project):
task_dir = _create_task(tmp_project, "cr-approved", "code_review:approved")
(task_dir / "CODE_REVIEW.md").write_text("# Code Review")
out, code = _run_status(["--task", "cr-approved", "--transition", "bug_find"], tmp_project)
assert code == 0
assert (task_dir / ".state").read_text().strip() == "bug_find"
def test_code_review_approvable(self, tmp_project):
task_dir = _create_task(tmp_project, "cr-approvable", "code_review:awaiting_approval")
out, code = _run_status(["--approve", "--task", "cr-approvable"], tmp_project)
assert code == 0
assert "Approved" in out
assert (task_dir / ".state").read_text().strip() == "code_review:approved"
def test_code_review_approve_records_log(self, tmp_project):
task_dir = _create_task(tmp_project, "cr-log", "code_review:awaiting_approval")
_run_status(["--approve", "--task", "cr-log"], tmp_project)
approvals = (task_dir / ".state.approvals").read_text()
assert "code_review:approved" in approvals
def test_code_review_not_editable(self, tmp_project):
_create_task(tmp_project, "cr-noedit", "code_review")
out, code = _run_status(["--can-edit", "--task", "cr-noedit"], tmp_project)
assert code == 1
assert "DENIED" in out
def test_code_review_forbids_bug_report_artifact(self, tmp_project):
task_dir = _create_task(tmp_project, "cr-forbidden", "code_review")
(task_dir / "BUG_REPORT.md").write_text("# Bugs")
out, code = _run_status(["--validate-folder", "--task", "cr-forbidden"], tmp_project)
assert code == 1
assert "BUG_REPORT.md" in out
class TestImplementerTracking:
def test_transition_records_implementer_when_lock_exists(self, tmp_project):
task_dir = _create_task(tmp_project, "impl-track", "implement")
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
lock_file = task_dir / ".state.lock"
from datetime import datetime, timedelta, timezone
now = datetime.now(timezone.utc)
expires = now + timedelta(minutes=30)
lock_file.write_text(f"agent: builder-agent\nphase: implement\nclaimed: {now.isoformat()}\nexpires: {expires.isoformat()}\n")
out, code = _run_status(["--task", "impl-track", "--transition", "code_review"], tmp_project)
assert code == 0
impl_file = task_dir / ".state.implementer"
assert impl_file.exists()
assert impl_file.read_text().strip() == "builder-agent"
def test_no_lock_does_not_block_transition(self, tmp_project):
task_dir = _create_task(tmp_project, "impl-nolock", "implement")
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
out, code = _run_status(["--task", "impl-nolock", "--transition", "code_review"], tmp_project)
assert code == 0
assert not (task_dir / ".state.implementer").exists()
class TestReviewerEnforcement:
def test_same_implementer_cannot_claim_code_review(self, tmp_project):
task_dir = _create_task(tmp_project, "same-reviewer", "code_review")
(task_dir / ".state.implementer").write_text("builder-agent\n")
agent_file = tmp_project / ".automaton" / ".agent.md"
agent_file.write_text("""## Agent Configuration
Mode: multi-agent
Agents:
- id: builder-agent
phases: [code_review, implement]
Lock timeout: 30m
""")
out, code = _run_status(["--claim", "--task", "same-reviewer", "--agent", "builder-agent"], tmp_project)
assert code == 1
assert "cannot claim" in out
def test_different_reviewer_can_claim_code_review(self, tmp_project):
task_dir = _create_task(tmp_project, "diff-reviewer", "code_review")
(task_dir / ".state.implementer").write_text("builder-agent\n")
agent_file = tmp_project / ".automaton" / ".agent.md"
agent_file.write_text("""## Agent Configuration
Mode: multi-agent
Agents:
- id: reviewer-agent
phases: [code_review]
Lock timeout: 30m
""")
out, code = _run_status(["--claim", "--task", "diff-reviewer", "--agent", "reviewer-agent"], tmp_project)
assert code == 0
assert "Claimed" in out
class TestCodeReviewInference:
def test_code_review_artifact_infers_bug_find(self, tmp_project):
task_dir = tmp_project / ".automaton" / "tasks" / "cr-infer"
task_dir.mkdir()
(task_dir / "CODE_REVIEW.md").write_text("# Review")
out, code = _run_status(["--upgrade", "--task", "cr-infer"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "bug_find"
def test_impl_and_code_review_infers_bug_find(self, tmp_project):
task_dir = tmp_project / ".automaton" / "tasks" / "cr-plus-impl"
task_dir.mkdir()
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
(task_dir / "CODE_REVIEW.md").write_text("# Review")
out, code = _run_status(["--upgrade", "--task", "cr-plus-impl"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "bug_find"
class TestCodeReviewShowTask:
def test_show_task_code_review(self, tmp_project):
_create_task(tmp_project, "cr-show", "code_review")
out, code = _run_status(["--task", "cr-show"], tmp_project)
assert code == 0
assert "Phase: code_review" in out
assert "Next artifact needed: CODE_REVIEW.md" in out
class TestCodeReviewListStates:
def test_list_states_includes_code_review(self, tmp_project):
out, code = _run_status(["--list-states"], tmp_project)
assert code == 0
assert "code_review * requires approval" in out
assert "code_review:awaiting_approval" in out
assert "code_review:approved" in out
class TestCat3AuditRegularProjectPaths:
"""Bug 1: Category 3 audit must recognize .automaton/tasks/ paths for regular projects."""
def test_regular_project_task_path_not_flagged(self, tmp_project):
"""Files inside .automaton/tasks/ in a regular project should NOT be flagged as unauthorized."""
import subprocess
task_dir = _create_task(tmp_project, "edit-task", "implement")
# Simulate a change inside the task folder
(task_dir / "IMPLEMENTATION.md").write_text("# Impl")
subprocess.run(["git", "init"], cwd=str(tmp_project), capture_output=True)
subprocess.run(["git", "add", "-A"], cwd=str(tmp_project), capture_output=True)
subprocess.run(["git", "commit", "-m", "init"], cwd=str(tmp_project), capture_output=True)
# Make a change inside task folder
(task_dir / "IMPLEMENTATION.md").write_text("# Updated Impl")
out, code = _run_status(["--audit"], tmp_project)
# The task file should NOT appear as unauthorized
assert ".automaton/tasks/edit-task/IMPLEMENTATION.md" not in out or "[PASS]" in out
class TestVerdictPassInference:
"""Bug 4: _infer_state_from_artifacts must not use substring 'PASS' search."""
def test_fail_verdict_with_pass_in_body_is_human_intervention(self, tmp_project):
"""A FAIL verdict mentioning 'PASS' in body must be human_intervention, not complete."""
task_dir = tmp_project / ".automaton" / "tasks" / "verdict-fail-pass"
task_dir.mkdir(parents=True)
(task_dir / "VERDICT.md").write_text(
"# Verdict\n## Status: FAIL\n\nAll unit tests PASS but spec is not met.\n"
)
out, code = _run_status(["--upgrade", "--task", "verdict-fail-pass"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "human_intervention", f"Expected human_intervention, got {state}"
def test_pass_verdict_is_complete(self, tmp_project):
"""A PASS verdict with ## Status: PASS should be complete."""
task_dir = tmp_project / ".automaton" / "tasks" / "verdict-pass"
task_dir.mkdir(parents=True)
(task_dir / "VERDICT.md").write_text("# Verdict\n## Status: PASS\n\nAll good.\n")
out, code = _run_status(["--upgrade", "--task", "verdict-pass"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "complete", f"Expected complete, got {state}"
def test_needs_review_verdict_is_human_intervention(self, tmp_project):
"""A NEEDS_REVIEW verdict should be human_intervention."""
task_dir = tmp_project / ".automaton" / "tasks" / "verdict-nr"
task_dir.mkdir(parents=True)
(task_dir / "VERDICT.md").write_text("# Verdict\n## Status: NEEDS_REVIEW\n\nNeeds manual review.\n")
out, code = _run_status(["--upgrade", "--task", "verdict-nr"], tmp_project)
assert code == 0
state = (task_dir / ".state").read_text().strip()
assert state == "human_intervention", f"Expected human_intervention, got {state}"
class TestCanEditPathPrefix:
"""Bug 7: --can-edit and --scope-check must not match sibling directories."""
def test_scope_check_rejects_sibling_directory(self, tmp_project):
"""A file in a sibling directory (e.g. project-evil) must be OUT_OF_SCOPE."""
_create_task(tmp_project, "scope-sibling", "implement")
sibling = tmp_project.parent / (tmp_project.name + "-evil")
sibling.mkdir(exist_ok=True)
evil_file = sibling / "file.py"
evil_file.write_text("# evil")
out, code = _run_status(
["--scope-check", "--task", "scope-sibling", "--file", str(evil_file)],
tmp_project,
)
assert code == 1
assert "OUT_OF_SCOPE" in out
def test_scope_check_accepts_subdirectory(self, tmp_project):
"""A file inside the project directory should be IN_SCOPE."""
_create_task(tmp_project, "scope-sub", "implement")
sub = tmp_project / "subdir"
sub.mkdir()
test_file = sub / "file.py"
test_file.write_text("# ok")
out, code = _run_status(
["--scope-check", "--task", "scope-sub", "--file", str(test_file)],
tmp_project,
)
assert code == 0
assert "IN_SCOPE" in out
def test_can_edit_task_rejects_sibling_directory(self, tmp_project):
"""--can-edit --task --file must reject files in sibling directories."""
task_dir = _create_task(tmp_project, "edit-sibling", "implement")
sibling = tmp_project.parent / (tmp_project.name + "-evil")
sibling.mkdir(exist_ok=True)
evil_file = sibling / "file.py"
evil_file.write_text("# evil")
out, code = _run_status(
["--can-edit", "--task", "edit-sibling", "--file", str(evil_file)],
tmp_project,
)
assert code == 1
assert "OUT_OF_SCOPE" in out
class TestStateLastEdit:
"""Bug 9: --can-edit should use .state.lastedit for staleness, not .state mtime."""
def test_can_edit_creates_lastedit_on_allowed(self, tmp_project):
"""ALLOWED response should create .state.lastedit file."""
task_dir = _create_task(tmp_project, "lastedit-create", "implement")
assert not (task_dir / ".state.lastedit").exists()
out, code = _run_status(["--can-edit", "--task", "lastedit-create"], tmp_project)
assert code == 0
assert "ALLOWED" in out
assert (task_dir / ".state.lastedit").exists()
def test_can_edit_creates_lastedit_with_file_scope(self, tmp_project):
"""ALLOWED with --file should create .state.lastedit for primary task."""
task_dir = _create_task(tmp_project, "lastedit-file", "implement")
test_file = tmp_project / "src" / "main.py"
test_file.parent.mkdir(parents=True, exist_ok=True)
test_file.write_text("# test")
assert not (task_dir / ".state.lastedit").exists()
out, code = _run_status(
["--can-edit", "--file", str(test_file)],
tmp_project,
)
assert code == 0
assert "ALLOWED" in out
assert (task_dir / ".state.lastedit").exists()
def test_stale_uses_lastedit_not_state_mtime(self, tmp_project):
"""If .state is old but .state.lastedit is recent, task should not be stale."""
import time
task_dir = _create_task(tmp_project, "stale-lastedit", "implement")
# Make .state old (31 minutes ago)
state_file = task_dir / ".state"
old_time = time.time() - 31 * 60
os.utime(state_file, (old_time, old_time))
# Make .state.lastedit recent (1 minute ago)
lastedit = task_dir / ".state.lastedit"
lastedit.touch()
recent_time = time.time() - 60
os.utime(lastedit, (recent_time, recent_time))
out, code = _run_status(["--can-edit", "--task", "stale-lastedit"], tmp_project)
assert code == 0
assert "ALLOWED" in out
def test_stale_when_lastedit_old(self, tmp_project):
"""If .state.lastedit is old, task should be denied as stale."""
import time
task_dir = _create_task(tmp_project, "stale-old", "implement")
# Make .state recent (1 minute ago)
state_file = task_dir / ".state"
recent_time = time.time() - 60
os.utime(state_file, (recent_time, recent_time))
# Make .state.lastedit old (31 minutes ago)
lastedit = task_dir / ".state.lastedit"
lastedit.touch()
old_time = time.time() - 31 * 60
os.utime(lastedit, (old_time, old_time))
out, code = _run_status(["--can-edit", "--task", "stale-old"], tmp_project)
assert code == 1
assert "stale" in out.lower()
def test_falls_back_to_state_mtime_without_lastedit(self, tmp_project):
"""Without .state.lastedit, should fall back to .state mtime (backward compat)."""
import time
task_dir = _create_task(tmp_project, "fallback-state", "implement")
# No .state.lastedit — .state is 31 minutes old
state_file = task_dir / ".state"
old_time = time.time() - 31 * 60
os.utime(state_file, (old_time, old_time))
out, code = _run_status(["--can-edit", "--task", "fallback-state"], tmp_project)
assert code == 1
assert "stale" in out.lower()
class TestTestPlanPhaseMapping:
"""Bug 10: TEST_PLAN.md should map to test_design, not implement."""
def test_test_plan_maps_to_test_design(self, tmp_project):
"""--upgrade should infer test_design when TEST_PLAN.md exists (no .state)."""
task_dir = tmp_project / ".automaton" / "tasks" / "testplan-infer"
task_dir.mkdir(parents=True)
(task_dir / "SPEC.md").write_text("# Spec")
(task_dir / "TEST_PLAN.md").write_text("# Tests")
out, code = _run_status(["--upgrade", "--task", "testplan-infer"], tmp_project)
assert code == 0
assert "test_design" in out
assert (task_dir / ".state").read_text().strip() == "test_design"