fix(sdk): harden command examples and run wheel regressions

This commit is contained in:
LogosVeritas committed 2026-10-26 10:19:06 +08:00
1 parent e122cfb627
commit 5771a93f2e
24 files changed
+687 -82

No files matched your search

@@ -0,0 +1,67 @@
"""Run native command regressions against installed SDK and runtime wheels."""
from __future__ import annotations
import os
import runpy
import sys
import tempfile
from pathlib import Path
import pytest
TEST_FILES = (
"test_commands_runtime.py",
"test_goal_runtime.py",
"test_plan_runtime.py",
"test_plan_example.py",
"test_plan_input.py",
)
class RequireExecutedTests:
"""Treat skipped collection or execution as a failed installed-wheel check."""
def __init__(self) -> None:
self.skipped: set[str] = set()
def pytest_collectreport(self, report: pytest.CollectReport) -> None:
if report.skipped:
self.skipped.add(report.nodeid)
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
if report.skipped:
self.skipped.add(report.nodeid)
def pytest_sessionfinish(self, session: pytest.Session) -> None:
if self.skipped:
print("Installed-wheel native tests skipped: " + ", ".join(sorted(self.skipped)), file=sys.stderr)
session.exitstatus = pytest.ExitCode.TESTS_FAILED
def main() -> int:
"""Verify wheel imports, then run every selected regression without repository pytest configuration."""
tests = Path(__file__).resolve().parent
smoke = runpy.run_path(tests.parents[2] / "scripts/smoke-python-runtime.py")
for name in ("PYTHONPATH", "DSH_RUNTIME_MODE", "PYTEST_ADDOPTS", "PYTEST_PLUGINS"):
os.environ.pop(name, None)
os.environ["DSH_COMMANDS_TEST_RUNTIME"] = "installed"
os.environ["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1"
previous_cwd = Path.cwd()
with tempfile.TemporaryDirectory(prefix="dsh-sdk-native-") as temporary:
try:
os.chdir(temporary)
smoke["assert_installed_wheel_environment"]()
config = Path(temporary) / "pytest.ini"
config.write_text("[pytest]\n", encoding="utf-8")
return pytest.main([
"-c", str(config), "--noconftest", "--import-mode=prepend", "-ra",
*(str(tests / name) for name in TEST_FILES),
], plugins=[RequireExecutedTests()])
finally:
os.chdir(previous_cwd)
if __name__ == "__main__":
raise SystemExit(main())
+114
View File
@@ -0,0 +1,114 @@
"""Goal observation deadlines and runtime cleanup through the runnable example."""
from __future__ import annotations
import json
import runpy
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import pytest
from deepseek_harness import DeepSeekHarness
_EXAMPLE = Path(__file__).resolve().parents[1] / "examples/goal.py"
@pytest.mark.parametrize("pending", ["command", "goal-event", "wrapup", "complete"])
def test_goal_example_bounds_pending_work_and_reaps_runtime(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], pending: str,
) -> None:
peer = tmp_path / "goal-peer.py"
observed = tmp_path / "observed.jsonl"
peer.write_text('''import json
import sys
from pathlib import Path
pending, observed = sys.argv[1:]
def send(frame):
print(json.dumps({"jsonrpc": "2.0", **frame}), flush=True)
for line in sys.stdin:
frame = json.loads(line)
with Path(observed).open("a", encoding="utf-8") as stream:
stream.write(json.dumps(frame) + "\\n")
method = frame["method"]
if method == "initialize":
send({"id": frame["id"], "result": {"serverInfo": {"name": "goal-example-peer"}}})
elif method == "session/query":
assert frame["params"]["text"] == "/goal Finish this task"
session = frame["params"]["sessionId"]
if pending == "command":
continue
if pending in ("wrapup", "complete"):
send({"method": "session.event", "params": {"sessionId": session, "event": {
"type": "goal/change", "data": {"operation": "complete"}}}})
send({"id": frame["id"], "result": {"kind": "command", "execution": {
"commandId": "goal-command", "result": {"kind": "success", "text": "Goal set"}}}})
elif method == "session/wait":
if pending == "wrapup":
continue
send({"method": "session.event", "params": {"sessionId": session, "event": {
"type": "assistant/message", "data": {"message": {
"content": [{"type": "text", "text": "Closing reply"}]}}}}})
send({"id": frame["id"], "result": {}})
elif method == "shutdown":
send({"id": frame["id"], "result": {}})
break
else:
raise AssertionError(method)
''', encoding="utf-8")
example = runpy.run_path(str(_EXAMPLE))
harnesses: list[DeepSeekHarness] = []
def configured_harness(**options: object) -> DeepSeekHarness:
harness = DeepSeekHarness(
**options, _launch_args=(sys.executable, str(peer), pending, str(observed)),
initialize_timeout_seconds=10, shutdown_timeout_seconds=10,
)
harnesses.append(harness)
return harness
monkeypatch.setitem(example["main"].__globals__, "DeepSeekHarness", configured_harness)
timeout = "10" if pending == "complete" else "1"
monkeypatch.setattr(sys, "argv", [str(_EXAMPLE), "--dsh-home", str(tmp_path / "home"),
"--workspace", str(tmp_path), "--timeout-seconds", timeout, "Finish this task"])
# The outer bound also releases an implementation that forgets its own deadline.
with ThreadPoolExecutor(max_workers=1) as workers:
execution = workers.submit(example["main"])
try:
if pending == "complete":
execution.result(timeout=30)
else:
expected = {"command": "session/query timed out", "goal-event": "SDK notification",
"wrapup": "session/wait timed out"}[pending]
with pytest.raises(TimeoutError, match=expected):
execution.result(timeout=30)
finally:
for harness in harnesses:
harness.close()
[harness] = harnesses
assert harness.client._proc is None
frames = [json.loads(line) for line in observed.read_text(encoding="utf-8").splitlines()]
methods = [frame["method"] for frame in frames]
assert methods == ["initialize", "session/query", *(["session/wait"] if pending in {"wrapup", "complete"} else []), "shutdown"]
if pending == "complete":
output = capsys.readouterr().out
assert "Goal: complete" in output
assert "Closing reply" in output
@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf", "-inf"])
def test_goal_example_rejects_unbounded_or_nonpositive_timeout(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], timeout: str,
) -> None:
example = runpy.run_path(str(_EXAMPLE))
monkeypatch.setattr(sys, "argv", [str(_EXAMPLE), "--dsh-home", str(tmp_path / "home"),
f"--timeout-seconds={timeout}", "Finish this task"])
with pytest.raises(SystemExit) as error:
example["main"]()
assert error.value.code == 2
assert "timeout-seconds must be finite and positive" in capsys.readouterr().err
+7 -10
View File
@@ -40,18 +40,14 @@ def test_plan_example_reports_truncation_and_accepts_explicit_dismissal(
return response(*args).replace(b'"stop_reason": "end_turn"', b'"stop_reason": "max_tokens"')
monkeypatch.setattr(runtime_helpers, "_messages_response", truncated_response)
choices: list[str] = []
def dismiss(prompt: str) -> str:
choices.append(prompt)
return "d"
monkeypatch.setattr("builtins.input", dismiss)
terminal_input = tmp_path / "stdin.txt"
terminal_input.write_text("d\n", encoding="utf-8")
monkeypatch.setenv("DSH_TELEMETRY_DISABLED", "1")
monkeypatch.setenv("DSH_TOOLS_MODE", "native")
monkeypatch.setattr(sys, "argv", [str(_EXAMPLE), "--dsh-home", str(tmp_path / "home"),
"--workspace", str(workspace), "Plan a small change."])
with _ModelServer([completion]) as model:
with terminal_input.open(encoding="utf-8") as terminal, _ModelServer([completion]) as model:
monkeypatch.setattr(sys, "stdin", terminal)
def configured_harness(**options: object) -> DeepSeekHarness:
return DeepSeekHarness(
**options, dsh_bin=_dsh_bin(), patches=(str(patch),),
@@ -73,8 +69,9 @@ def test_plan_example_reports_truncation_and_accepts_explicit_dismissal(
"max-tokens" if outcome == "max-tokens" else "completed",
]
assert [event["data"] for event in _of_type(events, "plan/mode")] == [{"active": True}]
assert len(choices) == (0 if outcome == "max-tokens" else 1)
assert ("PARTIAL_PLAN" if outcome == "max-tokens" else _PLAN) in capsys.readouterr().out
output = capsys.readouterr().out
assert ("[a] Approve" in output) == (outcome == "dismiss")
assert ("PARTIAL_PLAN" if outcome == "max-tokens" else _PLAN) in output
def test_plan_display_rejects_root_abort_without_rejecting_descendant_abort() -> None:
+253
View File
@@ -0,0 +1,253 @@
"""Plan example input uses real pipes and releases readers before process exit."""
from __future__ import annotations
import json
import os
import runpy
import signal
import subprocess
import sys
import threading
import time
from contextlib import ExitStack
from pathlib import Path
import pytest
_EXAMPLE = Path(__file__).resolve().parents[1] / "examples/plan.py"
_WAIT_SECONDS = 15
_PEER = '''import json
from pathlib import Path
import sys
import threading
root = Path(sys.argv[1])
lock = threading.Lock()
stop = threading.Event()
wait_id = None
serial = 0
def send(frame):
with lock:
print(json.dumps({"jsonrpc": "2.0", **frame}), flush=True)
def ask(session_id):
global serial
serial += 1
send({"id": "ask-" + str(serial), "method": "user-question/ask", "params": {
"requestId": "review-" + str(serial), "sessionId": session_id,
"questions": [{"id": "plan-review", "question": "Review the plan?", "detail": "# Small change",
"options": [{"label": "Approve"}, {"label": "Keep planning"}],
"intent": {"kind": "plan-review", "approve": "Approve"}}],
}})
def cancel():
while not stop.wait(0.01):
if (root / "cancel").exists():
send({"method": "user-question.cancelled", "params": {"requestId": "review-1"}})
send({"id": wait_id, "result": {}})
return
watcher = threading.Thread(target=cancel)
watcher.start()
try:
for line in sys.stdin:
frame = json.loads(line)
method = frame.get("method")
if method is None:
with (root / "answers.jsonl").open("a") as output:
output.write(json.dumps(frame) + "\\n")
result = frame.get("result", {})
answers = result.get("answers", [])
if answers and answers[0].get("selected") == ["Keep planning"]:
ask(session_id)
else:
send({"id": wait_id, "result": {}})
elif method == "initialize":
send({"id": frame["id"], "result": {"serverInfo": {"name": "plan-input-peer"}}})
elif method == "session/query":
send({"id": frame["id"], "result": {"kind": "command", "execution": {
"commandId": "plan-command", "result": {"kind": "success", "text": "Plan mode entered"}}}})
elif method == "session/wait":
wait_id = frame["id"]
session_id = frame["params"]["sessionId"]
ask(session_id)
elif method == "shutdown":
(root / "shutdown").write_text("done")
send({"id": frame["id"], "result": {}})
break
else:
raise AssertionError(method)
finally:
stop.set()
watcher.join()
'''
_WRAPPER = '''import json
from pathlib import Path
import runpy
import subprocess
import sys
import threading
from deepseek_harness import DeepSeekHarness
if sys.platform != "win32":
import resource
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
example_path, peer, root_argument = sys.argv[1:]
root = Path(root_argument)
example = runpy.run_path(example_path)
readers = []
popen = subprocess.Popen
def record_reader(*args, **kwargs):
process = popen(*args, **kwargs)
if args and len(args[0]) > 1 and args[0][1] == "-c":
readers.append(process)
return process
subprocess.Popen = record_reader
def configured_harness(**options):
answer = options.pop("on_user_question")
def record_answer(request, cancelled):
result = answer(request, cancelled)
with (root / "callbacks.jsonl").open("a") as output:
output.write(json.dumps(result) + "\\n")
return result
return DeepSeekHarness(
**options, on_user_question=record_answer, _launch_args=(sys.executable, peer, str(root)),
initialize_timeout_seconds=5, shutdown_timeout_seconds=1,
)
example["main"].__globals__["DeepSeekHarness"] = configured_harness
sys.argv = [example_path, "--dsh-home", str(root / "home"), "--workspace", str(root), "Plan a change."]
try:
example["main"]()
except KeyboardInterrupt:
raise SystemExit(130)
finally:
(root / "cleanup.json").write_text(json.dumps({
"readers": [{"pid": reader.pid, "returncode": reader.poll()} for reader in readers],
"callbacks": [thread.name for thread in threading.enumerate() if thread.name == "dsh-user-question"],
}))
'''
def _wait_prompt(process: subprocess.Popen[bytes], output: Path, marker: str) -> None:
deadline = time.monotonic() + _WAIT_SECONDS
while marker not in output.read_text(encoding="utf-8"):
assert process.poll() is None, output.read_text(encoding="utf-8")
if time.monotonic() >= deadline:
raise AssertionError(f"plan input did not reach {marker!r}")
time.sleep(0.01)
def _run(
tmp_path: Path, data: bytes, *, stage: str | None = None, action: str | None = None, terminal: bool = False,
) -> list[dict]:
peer = tmp_path / "peer.py"
peer.write_text(_PEER, encoding="utf-8")
wrapper = tmp_path / "wrapper.py"
wrapper.write_text(_WRAPPER, encoding="utf-8")
output = tmp_path / "stdout.txt"
errors = tmp_path / "stderr.txt"
env = {**os.environ, "PYTHONIOENCODING": "utf-8"}
with ExitStack() as stack, output.open("wb") as stdout, errors.open("wb") as stderr:
if terminal:
master, slave = os.openpty()
writing = stack.enter_context(os.fdopen(master, "wb", buffering=0))
reading = stack.enter_context(os.fdopen(slave, "rb", buffering=0))
child_input = reading
else:
child_input = subprocess.PIPE
process = subprocess.Popen(
[sys.executable, str(wrapper), str(_EXAMPLE), str(peer), str(tmp_path)],
stdin=child_input, stdout=stdout, stderr=stderr, env=env, start_new_session=os.name != "nt",
)
if not terminal:
assert process.stdin is not None
writing = stack.enter_context(process.stdin)
try:
if data:
writing.write(data)
writing.flush()
if action is None and not terminal:
writing.close()
elif action is not None:
marker = "Feedback: " if stage == "feedback" else "[d] Dismiss: "
_wait_prompt(process, output, marker)
if action == "interrupt":
process.send_signal(signal.SIGINT)
else:
(tmp_path / "cancel").write_text("cancel")
expected = 130 if action == "interrupt" else 0
assert process.wait(timeout=_WAIT_SECONDS) == expected, errors.read_text(encoding="utf-8")
finally:
if os.name != "nt":
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError as error:
# Successful example teardown has already removed the process group.
pass
elif process.poll() is None:
taskkill = Path(os.environ["SystemRoot"]) / "System32" / "taskkill.exe"
subprocess.run([str(taskkill), "/PID", str(process.pid), "/T", "/F"],
stdin=subprocess.DEVNULL, capture_output=True, check=False, timeout=_WAIT_SECONDS)
process.wait(timeout=_WAIT_SECONDS)
assert "_enter_buffered_busy" not in errors.read_text(encoding="utf-8")
assert (tmp_path / "shutdown").read_text() == "done"
cleanup = json.loads((tmp_path / "cleanup.json").read_text())
assert cleanup["readers"] and all(reader["returncode"] is not None for reader in cleanup["readers"])
assert cleanup["callbacks"] == []
return [json.loads(line) for line in (tmp_path / "callbacks.jsonl").read_text().splitlines()]
@pytest.mark.parametrize("data,expected", [
(b"a", [{"answers": [{"id": "plan-review", "selected": ["Approve"]}]}]),
(b"d\n", [{"cancelled": True}]),
(b"", [{"cancelled": True}]),
("invalid\nk\n继续检查\r\nd\n".encode(), [
{"answers": [{"id": "plan-review", "selected": ["Keep planning"], "custom": "继续检查"}]},
{"cancelled": True},
]),
])
def test_plan_input_preserves_piped_lines_and_eof(tmp_path: Path, data: bytes, expected: list[dict]) -> None:
assert _run(tmp_path, data) == expected
def test_plan_input_print_failure_reaps_reader_before_owner_exit(monkeypatch: pytest.MonkeyPatch) -> None:
example = runpy.run_path(str(_EXAMPLE))
owner_type = example["_PlanReviews"]
readers = []
popen = subprocess.Popen
def record_reader(*args, **kwargs):
reader = popen(*args, **kwargs)
readers.append(reader)
return reader
def fail_print(*_args, **_kwargs):
raise BrokenPipeError("terminal output closed")
with ExitStack() as stack, monkeypatch.context() as patch:
read_fd, write_fd = os.pipe()
terminal = stack.enter_context(os.fdopen(read_fd, "r", encoding="utf-8"))
stack.enter_context(os.fdopen(write_fd, "w", encoding="utf-8"))
patch.setattr(sys, "stdin", terminal)
patch.setattr(subprocess, "Popen", record_reader)
patch.setitem(owner_type._read_line.__globals__, "print", fail_print)
with owner_type() as owner:
with pytest.raises(BrokenPipeError, match="terminal output closed"):
owner._read_line("Review: ", threading.Event())
assert len(readers) == 1 and readers[0].poll() is not None
assert owner._readers == []
@pytest.mark.parametrize("stage", ["choice", "feedback"])
def test_plan_input_cancellation_reaps_open_pipe_reader(tmp_path: Path, stage: str) -> None:
data = b"k\npartial feedback" if stage == "feedback" else b"partial choice"
assert _run(tmp_path, data, stage=stage, action="protocol") == [{"cancelled": True}]
assert not (tmp_path / "answers.jsonl").exists()
+36
View File
@@ -0,0 +1,36 @@
"""POSIX terminal input and SIGINT cleanup for the plan example."""
from pathlib import Path
import sys
import pytest
from test_plan_input import _run
import test_plan_input
pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX PTYs and SIGINT delivery")
def test_plan_input_preserves_terminal_feedback(tmp_path: Path) -> None:
assert _run(tmp_path, "k\n继续检查\nd\n".encode(), terminal=True) == [
{"answers": [{"id": "plan-review", "selected": ["Keep planning"], "custom": "继续检查"}]},
{"cancelled": True},
]
@pytest.mark.parametrize("stage", ["choice", "feedback"])
def test_plan_input_interrupt_reaps_open_pipe_reader(tmp_path: Path, stage: str) -> None:
data = b"k\npartial feedback" if stage == "feedback" else b"partial choice"
assert _run(tmp_path, data, stage=stage, action="interrupt") == [{"cancelled": True}]
assert not (tmp_path / "answers.jsonl").exists()
def test_plan_input_cancellation_ignores_inherited_sigterm_disposition(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
) -> None:
wrapper = test_plan_input._WRAPPER.replace(
"import json\n", "import json\nimport signal\nsignal.signal(signal.SIGTERM, signal.SIG_IGN)\n", 1,
)
monkeypatch.setattr(test_plan_input, "_WRAPPER", wrapper)
assert _run(tmp_path, b"partial choice", stage="choice", action="protocol") == [{"cancelled": True}]