172 lines
7.1 KiB
Python
172 lines
7.1 KiB
Python
|
"""Tests for the harness command template fix (task fix-harness-command-template).
|
||
|
|
|
||
|
|
Exercises `_invoke_harness` directly with the new default command shape
|
||
|
|
(`--dir {cwd} {prompt_content}`) and custom commands. No live LLM calls.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import importlib.util
|
||
|
|
import json
|
||
|
|
import subprocess
|
||
|
|
from pathlib import Path
|
||
|
|
from unittest.mock import patch
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
_RUNNER_PATH = Path.home() / ".automaton" / "scripts" / "loop-runner.py"
|
||
|
|
_spec = importlib.util.spec_from_file_location("loop_runner_fix", _RUNNER_PATH)
|
||
|
|
lr = importlib.util.module_from_spec(_spec)
|
||
|
|
_spec.loader.exec_module(lr)
|
||
|
|
|
||
|
|
|
||
|
|
def _make_loop_with_prompt(tmp_path: Path, name: str = "hc-loop",
|
||
|
|
prompt_text: str = "hello world") -> tuple[Path, Path]:
|
||
|
|
lp = tmp_path / ".automaton" / "loops" / name
|
||
|
|
lp.mkdir(parents=True, exist_ok=True)
|
||
|
|
(lp / "loop.json").write_text(json.dumps({
|
||
|
|
"name": name, "description": "hc test",
|
||
|
|
"schedule": {"interval_seconds": 3600},
|
||
|
|
"brakes": {"max_iterations": 25, "max_budget_usd": None,
|
||
|
|
"score_plateau_window": 5},
|
||
|
|
"blast_radius": {"file_scope": [], "use_worktree": False},
|
||
|
|
"work_source": {"kind": "single"},
|
||
|
|
"roles": {"implement": {"prompt": "test-impl.md"},
|
||
|
|
"verify": {"prompt": "test-verify.md"},
|
||
|
|
"orchestrate": {"prompt": "test-orch.md"}},
|
||
|
|
}) + "\n")
|
||
|
|
(lp / ".state.loop").write_text(json.dumps({
|
||
|
|
"schema_version": 1, "name": name, "status": "running",
|
||
|
|
"halt_reason": None, "iteration_count": 0, "resumed_count": 0,
|
||
|
|
"last_tick_at": None, "last_verdict": None, "score_history": [],
|
||
|
|
"current_task": "demo", "worktree_branch": None, "worktree_path": None,
|
||
|
|
}, indent=2, sort_keys=True) + "\n")
|
||
|
|
(lp / ".state.log").write_text("")
|
||
|
|
for ref in ("test-impl.md", "test-verify.md", "test-orch.md"):
|
||
|
|
(lp / ref).write_text(f"{prompt_text}\n")
|
||
|
|
return lp, lp / "test-impl.md"
|
||
|
|
|
||
|
|
|
||
|
|
class TestDefaultCommand:
|
||
|
|
def test_default_uses_dir_not_cwd(self, tmp_path, monkeypatch):
|
||
|
|
lp, prompt = _make_loop_with_prompt(tmp_path)
|
||
|
|
captured = {}
|
||
|
|
|
||
|
|
def fake_run(argv, *a, **kw):
|
||
|
|
captured["argv"] = list(argv)
|
||
|
|
class R: pass
|
||
|
|
r = R(); r.stdout = ""; r.stderr = ""; r.returncode = 0
|
||
|
|
return r
|
||
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
|
lr._invoke_harness(None, "implement", str(prompt), str(tmp_path),
|
||
|
|
loop_path=lp, tick_num=1)
|
||
|
|
argv = captured["argv"]
|
||
|
|
assert "--dir" in argv
|
||
|
|
assert "--cwd" not in argv
|
||
|
|
assert "--prompt-file" not in argv
|
||
|
|
|
||
|
|
def test_default_passes_prompt_content(self, tmp_path, monkeypatch):
|
||
|
|
lp, prompt = _make_loop_with_prompt(tmp_path, prompt_text="hello world")
|
||
|
|
captured = {}
|
||
|
|
|
||
|
|
def fake_run(argv, *a, **kw):
|
||
|
|
captured["argv"] = list(argv)
|
||
|
|
class R: pass
|
||
|
|
r = R(); r.stdout = ""; r.stderr = ""; r.returncode = 0
|
||
|
|
return r
|
||
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
|
lr._invoke_harness(None, "implement", str(prompt), str(tmp_path),
|
||
|
|
loop_path=lp, tick_num=1)
|
||
|
|
argv = captured["argv"]
|
||
|
|
assert any(a.rstrip() == "hello world" for a in argv)
|
||
|
|
assert argv[-1].rstrip() == "hello world"
|
||
|
|
|
||
|
|
def test_prompt_content_handles_special_chars(self, tmp_path, monkeypatch):
|
||
|
|
weird = "hello 'world' with $vars and \"quotes\""
|
||
|
|
lp, prompt = _make_loop_with_prompt(tmp_path, prompt_text=weird)
|
||
|
|
captured = {}
|
||
|
|
|
||
|
|
def fake_run(argv, *a, **kw):
|
||
|
|
captured["argv"] = list(argv)
|
||
|
|
class R: pass
|
||
|
|
r = R(); r.stdout = ""; r.stderr = ""; r.returncode = 0
|
||
|
|
return r
|
||
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
|
lr._invoke_harness(None, "implement", str(prompt), str(tmp_path),
|
||
|
|
loop_path=lp, tick_num=1)
|
||
|
|
argv = captured["argv"]
|
||
|
|
assert weird in argv or (weird + "\n") in argv
|
||
|
|
matching = [a for a in argv if a.rstrip() == weird]
|
||
|
|
assert len(matching) == 1
|
||
|
|
|
||
|
|
def test_prompt_token_still_available(self, tmp_path, monkeypatch):
|
||
|
|
lp, prompt = _make_loop_with_prompt(tmp_path)
|
||
|
|
captured = {}
|
||
|
|
|
||
|
|
def fake_run(argv, *a, **kw):
|
||
|
|
captured["argv"] = list(argv)
|
||
|
|
class R: pass
|
||
|
|
r = R(); r.stdout = ""; r.stderr = ""; r.returncode = 0
|
||
|
|
return r
|
||
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
|
lr._invoke_harness({"command": ["cat", "{prompt}"]},
|
||
|
|
"implement", str(prompt), str(tmp_path),
|
||
|
|
loop_path=lp, tick_num=1)
|
||
|
|
argv = captured["argv"]
|
||
|
|
assert argv[0] == "cat"
|
||
|
|
assert argv[1].endswith("test-impl.md") or "outputs" in argv[1]
|
||
|
|
|
||
|
|
def test_custom_command_with_cwd_still_works(self, tmp_path, monkeypatch):
|
||
|
|
lp, prompt = _make_loop_with_prompt(tmp_path)
|
||
|
|
captured = {}
|
||
|
|
|
||
|
|
def fake_run(argv, *a, **kw):
|
||
|
|
captured["argv"] = list(argv)
|
||
|
|
class R: pass
|
||
|
|
r = R(); r.stdout = ""; r.stderr = ""; r.returncode = 0
|
||
|
|
return r
|
||
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
|
lr._invoke_harness({"command": ["my-tool", "--cwd", "{cwd}", "{prompt_content}"]},
|
||
|
|
"implement", str(prompt), str(tmp_path),
|
||
|
|
loop_path=lp, tick_num=1)
|
||
|
|
argv = captured["argv"]
|
||
|
|
assert "--cwd" in argv
|
||
|
|
assert argv[0] == "my-tool"
|
||
|
|
cwd_idx = argv.index("--cwd")
|
||
|
|
assert argv[cwd_idx + 1] == str(tmp_path)
|
||
|
|
|
||
|
|
def test_empty_command_falls_back_to_new_default(self, tmp_path, monkeypatch):
|
||
|
|
lp, prompt = _make_loop_with_prompt(tmp_path)
|
||
|
|
captured = {}
|
||
|
|
|
||
|
|
def fake_run(argv, *a, **kw):
|
||
|
|
captured["argv"] = list(argv)
|
||
|
|
class R: pass
|
||
|
|
r = R(); r.stdout = ""; r.stderr = ""; r.returncode = 0
|
||
|
|
return r
|
||
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
|
lr._invoke_harness({"command": []},
|
||
|
|
"implement", str(prompt), str(tmp_path),
|
||
|
|
loop_path=lp, tick_num=1)
|
||
|
|
argv = captured["argv"]
|
||
|
|
assert "--dir" in argv
|
||
|
|
assert "--prompt-file" not in argv
|
||
|
|
assert "--cwd" not in argv
|
||
|
|
|
||
|
|
|
||
|
|
class TestPiShapedCommand:
|
||
|
|
def test_pi_shaped_command_substitutes_correctly(self, tmp_path, monkeypatch):
|
||
|
|
lp, prompt = _make_loop_with_prompt(tmp_path, prompt_text="implement the lock")
|
||
|
|
captured = {}
|
||
|
|
|
||
|
|
def fake_run(argv, *a, **kw):
|
||
|
|
captured["argv"] = list(argv)
|
||
|
|
class R: pass
|
||
|
|
r = R(); r.stdout = ""; r.stderr = ""; r.returncode = 0
|
||
|
|
return r
|
||
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
|
lr._invoke_harness(
|
||
|
|
{"command": ["pi", "run", "--cwd", "{cwd}", "{prompt_content}"]},
|
||
|
|
"implement", str(prompt), str(tmp_path),
|
||
|
|
loop_path=lp, tick_num=1)
|
||
|
|
argv = captured["argv"]
|
||
|
|
assert argv[:4] == ["pi", "run", "--cwd", str(tmp_path)]
|
||
|
|
assert argv[4].rstrip() == "implement the lock"
|