"""Tests for model-divergence enforcement module. Covers: manifest loading, mode detection, conflict matrix, --transition --model, --claim --model, --audit model_divergence, detect_models.py, and loop model binding. """ import importlib.util import json import os import sys import pytest from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) _RUNNER_PATH = Path.home() / ".automaton" / "scripts" / "loop-runner.py" _runner_spec = importlib.util.spec_from_file_location("loop_runner_mde", _RUNNER_PATH) _runner_mod = importlib.util.module_from_spec(_runner_spec) _runner_spec.loader.exec_module(_runner_mod) # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest.fixture def tmp_project(tmp_path): auto_dir = tmp_path / ".automaton" tasks_dir = auto_dir / "tasks" tasks_dir.mkdir(parents=True) return tmp_path @pytest.fixture def task_dir(tmp_project): task_path = tmp_project / ".automaton" / "tasks" / "test-task" task_path.mkdir(parents=True) (task_path / ".state").write_text("new\n") (task_path / ".state.approvals").write_text("") return task_path def _write_models(project, models_list, default=None, advised=False): """Write a models.json to the project's .automaton/.""" auto = project / ".automaton" auto.mkdir(parents=True, exist_ok=True) data = { "schema_version": 1, "default": default or (models_list[0]["name"] if models_list else None), "advised": advised, "models": models_list, } (auto / "models.json").write_text(json.dumps(data, indent=2)) return auto / "models.json" def _run_status(args, project=None): """Run status.py with given args and return (stdout, exit_code).""" import subprocess cmd = [sys.executable, str(Path.home() / ".automaton" / "scripts" / "status.py")] if project: cmd.extend(["--project", str(project)]) cmd.extend(args) result = subprocess.run(cmd, capture_output=True, text=True) return result.stdout.strip(), result.returncode # =========================================================================== # Subtask 1: Manifest + Detection # =========================================================================== class TestManifestLoading: """Test _load_models_manifest and _get_model_mode.""" def test_missing_manifest_is_single_mode(self, tmp_project): """No models.json → single-LLM mode (backward compatible).""" # Import the module functions directly from status import _load_models_manifest, _get_model_mode assert (tmp_project / ".automaton" / "models.json").exists() is False manifest = _load_models_manifest(str(tmp_project)) assert manifest is None mode = _get_model_mode(manifest) assert mode == "single" def test_empty_models_is_single_mode(self, tmp_project): """models.json with 0 models → single-LLM mode.""" _write_models(tmp_project, []) from status import _load_models_manifest, _get_model_mode manifest = _load_models_manifest(str(tmp_project)) assert manifest is not None mode = _get_model_mode(manifest) assert mode == "single" def test_one_model_is_single_mode(self, tmp_project): """models.json with 1 model → single-LLM mode.""" _write_models(tmp_project, [{"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}]) from status import _load_models_manifest, _get_model_mode manifest = _load_models_manifest(str(tmp_project)) assert manifest is not None mode = _get_model_mode(manifest) assert mode == "single" def test_two_models_is_multi_llm_mode(self, tmp_project): """models.json with 2+ models → multi-LLM mode.""" _write_models(tmp_project, [ {"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}, {"name": "model-b", "provider": "test", "context_window": 4096, "location": "remote"}, ]) from status import _load_models_manifest, _get_model_mode manifest = _load_models_manifest(str(tmp_project)) assert manifest is not None mode = _get_model_mode(manifest) assert mode == "multi-llm" def test_project_manifest_preferred(self, tmp_project): """Project-level models.json takes priority over global.""" auto = tmp_project / ".automaton" auto.mkdir(parents=True, exist_ok=True) (auto / "models.json").write_text(json.dumps({ "schema_version": 1, "default": "project-model", "advised": False, "models": [{"name": "project-model", "provider": "test", "context_window": 4096, "location": "remote"}], })) # Write a global one too global_models = Path.home() / ".automaton" / "models.json" existed = global_models.exists() if not existed: global_models.write_text(json.dumps({ "schema_version": 1, "default": "global-model", "advised": False, "models": [{"name": "global-model", "provider": "test", "context_window": 4096, "location": "remote"}], })) try: from status import _load_models_manifest manifest = _load_models_manifest(str(tmp_project)) assert manifest is not None assert manifest["default"] == "project-model" finally: if not existed: global_models.unlink() # =========================================================================== # Subtask 2: Conflict Matrix + Interactive Enforcement # =========================================================================== class TestConflictMatrix: def test_no_conflict_for_different_models(self): from status import _check_conflict state = {"implement": "model-a", "code_review": "model-b"} result = _check_conflict(state, "code_review", "model-b") assert result is None def test_conflict_code_review_same_as_implement(self): from status import _check_conflict state = {"implement": "model-a"} result = _check_conflict(state, "code_review", "model-a") assert result == "implement" def test_conflict_bug_find_same_as_implement(self): from status import _check_conflict state = {"implement": "model-a"} result = _check_conflict(state, "bug_find", "model-a") assert result == "implement" def test_conflict_adversarial_same_as_bug_find(self): from status import _check_conflict state = {"bug_find": "model-a"} result = _check_conflict(state, "adversarial_bug_find", "model-a") assert result == "bug_find" def test_conflict_adversarial_same_as_implement(self): from status import _check_conflict state = {"implement": "model-a"} result = _check_conflict(state, "adversarial_bug_find", "model-a") assert result == "implement" def test_conflict_referee_same_as_implement(self): from status import _check_conflict state = {"implement": "model-a", "bug_find": "model-b"} result = _check_conflict(state, "referee", "model-a") assert result == "implement" def test_conflict_referee_same_as_bug_find(self): from status import _check_conflict state = {"implement": "model-b", "bug_find": "model-a"} result = _check_conflict(state, "referee", "model-a") assert result == "bug_find" def test_conflict_referee_same_as_adversarial(self): from status import _check_conflict state = {"adversarial_bug_find": "model-a"} result = _check_conflict(state, "referee", "model-a") assert result == "adversarial_bug_find" def test_no_conflict_for_doc_review(self): from status import _check_conflict state = {"implement": "model-a"} result = _check_conflict(state, "doc_review", "model-a") assert result is None # doc_review has no conflicts def test_loop_verify_conflict_with_loop_implement(self): from status import _check_conflict state = {"loop-implement": "model-a"} result = _check_conflict(state, "loop-verify", "model-a") assert result == "loop-implement" class TestTransitionModel: def _transition_through(self, tmp_project, task_dir, task_name, phases): """Helper: transition through a list of phases, using --model.""" for i, (phase, model) in enumerate(phases): out, code = _run_status(["--task", task_name, "--transition", phase, "--model", model], tmp_project) assert code == 0, f"Transition to {phase} failed: {out}" if i < len(phases) - 1: # Write the required artifact for the current phase artifact_map = { "implement": "IMPLEMENTATION.md", "code_review": "CODE_REVIEW.md", "bug_find": "BUG_REPORT.md", "adversarial_bug_find": "ADVERSARIAL_BUG_REPORT.md", "doc_review": "DOC_REVIEW.md", "referee": "VERDICT.md", } art = artifact_map.get(phase) if art: (task_dir / art).write_text(f"# {art}\n") def test_transition_with_model_single_mode(self, tmp_project, task_dir): """Single-LLM: --transition --model records the model.""" _write_models(tmp_project, [{"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}]) (task_dir / ".state").write_text("research\n") (task_dir / "SPEC.md").write_text("# Spec\n") out, code = _run_status(["--task", "test-task", "--transition", "implement", "--model", "model-a"], tmp_project) assert code == 0, f"Transition failed: {out}" models_file = task_dir / ".state.models" assert models_file.exists() data = json.loads(models_file.read_text()) assert data.get("implement") == "model-a" def test_transition_with_model_multi_llm(self, tmp_project, task_dir): """Multi-LLM: --transition --model records and passes.""" _write_models(tmp_project, [ {"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}, {"name": "model-b", "provider": "test", "context_window": 4096, "location": "remote"}, ]) (task_dir / ".state").write_text("research\n") (task_dir / "SPEC.md").write_text("# Spec\n") out, code = _run_status(["--task", "test-task", "--transition", "implement", "--model", "model-a"], tmp_project) assert code == 0, f"Transition failed: {out}" models_file = task_dir / ".state.models" assert models_file.exists() data = json.loads(models_file.read_text()) assert data.get("implement") == "model-a" def test_transition_refuses_conflict(self, tmp_project, task_dir): """Multi-LLM: --transition refuses conflict-matrix violation.""" _write_models(tmp_project, [ {"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}, {"name": "model-b", "provider": "test", "context_window": 4096, "location": "remote"}, ]) (task_dir / ".state").write_text("research\n") (task_dir / "SPEC.md").write_text("# Spec\n") out, code = _run_status(["--task", "test-task", "--transition", "implement", "--model", "model-a"], tmp_project) assert code == 0 (task_dir / "IMPLEMENTATION.md").write_text("# Impl\n") out, code = _run_status(["--task", "test-task", "--transition", "code_review", "--model", "model-a"], tmp_project) assert code == 1, f"Should have refused conflict: {out}" assert "conflict" in out or "conflicts" in out def test_transition_auto_assigns_non_conflicting(self, tmp_project, task_dir): """Multi-LLM: auto-assigns next-available non-conflicting model.""" _write_models(tmp_project, [ {"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}, {"name": "model-b", "provider": "test", "context_window": 4096, "location": "remote"}, ]) (task_dir / ".state").write_text("research\n") (task_dir / "SPEC.md").write_text("# Spec\n") out, code = _run_status(["--task", "test-task", "--transition", "implement", "--model", "model-a"], tmp_project) assert code == 0 (task_dir / "IMPLEMENTATION.md").write_text("# Impl\n") out, code = _run_status(["--task", "test-task", "--transition", "code_review"], tmp_project) # Should auto-assign model-b (the only non-conflicting model) assert code == 0, f"Auto-assign failed: {out}" data = json.loads((task_dir / ".state.models").read_text()) assert data.get("code_review") == "model-b" def test_transition_single_model_fills_all_roles(self, tmp_project, task_dir): """Single-LLM: one model can fill all roles without conflict checks.""" _write_models(tmp_project, [ {"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}, ]) (task_dir / ".state").write_text("research\n") (task_dir / "SPEC.md").write_text("# Spec\n") out, code = _run_status(["--task", "test-task", "--transition", "implement", "--model", "model-a"], tmp_project) assert code == 0 (task_dir / "IMPLEMENTATION.md").write_text("# Impl\n") out, code = _run_status(["--task", "test-task", "--transition", "code_review", "--model", "model-a"], tmp_project) # Single-LLM mode: same model re-used for conflicting role is fine assert code == 0, f"Single-LLM should allow same model: {out}" class TestClaimModel: def test_claim_with_model_single_mode(self, tmp_project, task_dir): """--claim --model works in single-LLM mode (no conflict check).""" _write_models(tmp_project, [{"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}]) agent_file = tmp_project / ".automaton" / ".agent.md" agent_file.write_text("## Agent Configuration\nMode: multi-agent\n- id: agent-a\n phases: [*]\n") (task_dir / ".state").write_text("research\n") out, code = _run_status(["--task", "test-task", "--claim", "--agent", "agent-a", "--model", "model-a"], tmp_project) assert code == 0, f"Claim failed: {out}" class TestAuditModelDivergence: def test_audit_no_violations_single_mode(self, tmp_project): """--audit in single-LLM mode → no model-divergence violations.""" _write_models(tmp_project, [{"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}]) out, code = _run_status(["--audit"], tmp_project) assert code == 0 or "No model-divergence violations" in out def test_audit_detects_violations(self, tmp_project): """--audit flags conflict-matrix violations in multi-LLM mode.""" _write_models(tmp_project, [ {"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}, {"name": "model-b", "provider": "test", "context_window": 4096, "location": "remote"}, ]) # Create a task with conflicting models task_path = tmp_project / ".automaton" / "tasks" / "conflict-task" task_path.mkdir() (task_path / ".state").write_text("code_review\n") (task_path / ".state.approvals").write_text("") # Write .state.models with conflict: implement and code_review use same model (task_path / ".state.models").write_text(json.dumps({ "implement": "model-a", "code_review": "model-a", })) out, code = _run_status(["--audit"], tmp_project) assert "model-divergence" in out or "Model-Divergence" in out def test_audit_json_includes_model_divergence(self, tmp_project): """--audit --json includes model_divergence violations.""" _write_models(tmp_project, [ {"name": "model-a", "provider": "test", "context_window": 4096, "location": "remote"}, {"name": "model-b", "provider": "test", "context_window": 4096, "location": "remote"}, ]) task_path = tmp_project / ".automaton" / "tasks" / "conflict-task" task_path.mkdir() (task_path / ".state").write_text("code_review\n") (task_path / ".state.approvals").write_text("") (task_path / ".state.models").write_text(json.dumps({ "implement": "model-a", "code_review": "model-a", })) out, code = _run_status(["--audit", "--json"], tmp_project) data = json.loads(out.splitlines()[-1]) violations = data.get("violations", []) md_violations = [v for v in violations if v.get("category") == 6] assert len(md_violations) >= 1 # =========================================================================== # Subtask 3: Loop Enforcement # =========================================================================== class TestLoopModelBinding: def test_invoke_harness_model_substitution(self): """_substitute replaces {model} token when model is in mapping.""" result = _runner_mod._substitute("opencode run --model {model} --dir {cwd}", {"model": "model-a", "cwd": "/tmp"}) assert "model-a" in result assert "{model}" not in result def test_invoke_harness_no_substitution_when_no_model(self): """_substitute leaves {model} unchanged when no model in mapping.""" result = _runner_mod._substitute("opencode run --model {model}", {}) assert "{model}" in result # =========================================================================== # detect_models.py # =========================================================================== class TestDetectModels: def test_build_candidate_with_no_config(self): """detect_models builds empty candidate when no opencode.json.""" from detect_models import build_candidate_models candidate = build_candidate_models(probe_local=False) assert "models" in candidate assert "schema_version" in candidate assert candidate["schema_version"] == 1 def test_build_candidate_parses_opencode_json(self, tmp_path): """detect_models parses opencode.json provider models.""" opencode_json = tmp_path / "opencode.json" opencode_json.write_text(json.dumps({ "model": "model-a", "providers": {"provider-x": {"model": "model-b"}}, })) # Temporarily chdir so _find_opencode_json picks it up old_cwd = os.getcwd() os.chdir(str(tmp_path)) try: from detect_models import build_candidate_models candidate = build_candidate_models(probe_local=False) names = {m["name"] for m in candidate["models"]} assert "model-a" in names assert "model-b" in names finally: os.chdir(old_cwd) def test_write_models_file(self, tmp_path): """detect_models writes candidate to models.json.""" from detect_models import write_models_file candidate = {"schema_version": 1, "default": None, "advised": True, "models": []} target = tmp_path / "models.json" # Temporarily patch AUTOMATON_DIR import detect_models old_dir = detect_models.AUTOMATON_DIR detect_models.AUTOMATON_DIR = tmp_path try: written = write_models_file(candidate, force=True) assert written is True assert target.exists() data = json.loads(target.read_text()) assert data["schema_version"] == 1 finally: detect_models.AUTOMATON_DIR = old_dir def test_write_models_file_does_not_overwrite(self, tmp_path): """detect_models does not overwrite existing models.json without --force.""" target = tmp_path / "models.json" target.write_text('{"existing": true}') from detect_models import write_models_file import detect_models old_dir = detect_models.AUTOMATON_DIR detect_models.AUTOMATON_DIR = tmp_path try: written = write_models_file({"schema_version": 1, "default": None, "advised": True, "models": []}) assert written is False data = json.loads(target.read_text()) assert data.get("existing") is True finally: detect_models.AUTOMATON_DIR = old_dir # =========================================================================== # .state.models Read/Write # =========================================================================== class TestStateModels: def test_write_and_read_state_models(self, task_dir): from status import _write_state_models, _read_state_models data = {"implement": "model-a", "code_review": "model-b"} _write_state_models(task_dir, data) assert (task_dir / ".state.models").exists() read_back = _read_state_models(task_dir) assert read_back == data def test_read_missing_state_models(self, task_dir): from status import _read_state_models assert _read_state_models(task_dir) == {} def test_non_artifact_file_state_models(self, task_dir): """Verify .state.models is in NON_ARTIFACT_FILES so audit doesn't flag it.""" from status import NON_ARTIFACT_FILES assert ".state.models" in NON_ARTIFACT_FILES