Merge master into feat/subagent-names
This commit is contained in:
commit
4514b99d9c
2829 files changed
+180072
-13757
No files matched your search
@@ -0,0 +1,69 @@
|
||||
"""Run native command regressions against installed SDK and runtime wheels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import runpy
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
TEST_FILES = (
|
||||
"test_commands_runtime.py",
|
||||
"test_command_attachments_runtime.py",
|
||||
"test_goal_runtime.py",
|
||||
"test_goal_result_text_runtime.py",
|
||||
"test_plan_runtime.py",
|
||||
"test_plan_example.py",
|
||||
"test_plan_input.py",
|
||||
)
|
||||
|
||||
|
||||
class RequireExecutedTests:
|
||||
"""Treat skipped collection or execution as a failed installed-wheel check."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.skipped: set[str] = set()
|
||||
|
||||
def pytest_collectreport(self, report: pytest.CollectReport) -> None:
|
||||
if report.skipped:
|
||||
self.skipped.add(report.nodeid)
|
||||
|
||||
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
|
||||
if report.skipped:
|
||||
self.skipped.add(report.nodeid)
|
||||
|
||||
def pytest_sessionfinish(self, session: pytest.Session) -> None:
|
||||
if self.skipped:
|
||||
print("Installed-wheel native tests skipped: " + ", ".join(sorted(self.skipped)), file=sys.stderr)
|
||||
session.exitstatus = pytest.ExitCode.TESTS_FAILED
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Verify wheel imports, then run every selected regression without repository pytest configuration."""
|
||||
tests = Path(__file__).resolve().parent
|
||||
smoke = runpy.run_path(tests.parents[2] / "scripts/smoke-python-runtime.py")
|
||||
for name in ("PYTHONPATH", "DSH_RUNTIME_MODE", "PYTEST_ADDOPTS", "PYTEST_PLUGINS"):
|
||||
os.environ.pop(name, None)
|
||||
os.environ["DSH_COMMANDS_TEST_RUNTIME"] = "installed"
|
||||
os.environ["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1"
|
||||
previous_cwd = Path.cwd()
|
||||
with tempfile.TemporaryDirectory(prefix="dsh-sdk-native-") as temporary:
|
||||
try:
|
||||
os.chdir(temporary)
|
||||
smoke["assert_installed_wheel_environment"]()
|
||||
config = Path(temporary) / "pytest.ini"
|
||||
config.write_text("[pytest]\n", encoding="utf-8")
|
||||
return pytest.main([
|
||||
"-c", str(config), "--noconftest", "--import-mode=prepend", "-ra",
|
||||
*(str(tests / name) for name in TEST_FILES),
|
||||
], plugins=[RequireExecutedTests()])
|
||||
finally:
|
||||
os.chdir(previous_cwd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -38,11 +38,11 @@ for line in sys.stdin:
|
||||
if method == "initialize":
|
||||
json.dump(msg.get("params"), open(os.environ["INIT_DUMP"], "w"))
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"serverInfo": {"name": "fake-runtime"}}}), flush=True)
|
||||
elif method == "session/prompt":
|
||||
elif method == "session/query":
|
||||
params = msg.get("params") or {}
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": params["sessionId"], "event": {"type": "agent/inbox/spliced", "data": {"target": "next-turn", "start": 0, "inserted": [{"id": "message-1"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": params["sessionId"], "status": "running"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"kind": "prompt", "messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "session.event",
|
||||
@@ -178,7 +178,7 @@ for line in sys.stdin:
|
||||
"DEEPSEEK_BASE_URL": "http://127.0.0.1:4321",
|
||||
},
|
||||
) as harness:
|
||||
result = harness.run("say hello", session_id="main")
|
||||
result = harness.run([{"type": "text", "text": "say hello"}], session_id="main")
|
||||
|
||||
assert result.final_response == "hello from runtime"
|
||||
assert result.finish_reason == "max-tokens"
|
||||
@@ -233,10 +233,10 @@ for line in sys.stdin:
|
||||
method = msg.get("method")
|
||||
if method == "initialize":
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"serverInfo": {"name": "fake-runtime"}}}), flush=True)
|
||||
elif method == "session/prompt":
|
||||
elif method == "session/query":
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": "main", "event": {"type": "agent/inbox/spliced", "data": {"target": "next-turn", "start": 0, "inserted": [{"id": "message-1"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": "main", "status": "running"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"kind": "prompt", "messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "subagent.started", "params": {"parentSessionId": "main", "childSessionId": "child"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": "main", "status": "idle"}}), flush=True)
|
||||
elif method == "shutdown":
|
||||
@@ -252,7 +252,7 @@ for line in sys.stdin:
|
||||
) as harness:
|
||||
session = harness.start_session("main")
|
||||
result = session.run(
|
||||
"spawn a helper",
|
||||
[{"type": "text", "text": "spawn a helper"}],
|
||||
on_notification=lambda notification: seen.append(notification.method),
|
||||
)
|
||||
|
||||
@@ -272,10 +272,10 @@ for line in sys.stdin:
|
||||
method = msg.get("method")
|
||||
if method == "initialize":
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"serverInfo": {"name": "fake-runtime"}}}), flush=True)
|
||||
elif method == "session/prompt":
|
||||
elif method == "session/query":
|
||||
params = msg.get("params") or {}
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": params["sessionId"], "event": {"type": "agent/inbox/spliced", "data": {"target": "next-turn", "start": 0, "inserted": [{"id": "message-1"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"kind": "prompt", "messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": params["sessionId"], "event": {"type": "turn/end", "data": {"turn": 1, "reason": {}}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": params["sessionId"], "status": "idle"}}), flush=True)
|
||||
elif method == "shutdown":
|
||||
@@ -292,7 +292,7 @@ for line in sys.stdin:
|
||||
SdkProtocolError,
|
||||
match=r"turn/end event requires a string data\.reason\.kind",
|
||||
):
|
||||
harness.run("reject malformed turn ending", session_id="main")
|
||||
harness.run([{"type": "text", "text": "reject malformed turn ending"}], session_id="main")
|
||||
|
||||
|
||||
def test_relative_cwd_is_absolute_in_process_environment_and_wire(
|
||||
@@ -346,10 +346,10 @@ for line in sys.stdin:
|
||||
method = msg.get("method")
|
||||
if method == "initialize":
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"serverInfo": {"name": "fake-runtime"}}}), flush=True)
|
||||
elif method == "session/prompt":
|
||||
elif method == "session/query":
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": "main", "event": {"type": "agent/inbox/spliced", "data": {"target": "next-turn", "start": 0, "inserted": [{"id": "message-1"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": "main", "status": "running"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"kind": "prompt", "messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "subagent.started", "params": {"parentSessionId": "main", "childSessionId": "child"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "subagent.finished", "params": {"parentSessionId": "main", "childSessionId": "child", "status": "ok", "stopReason": "completed"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": "main", "status": "idle"}}), flush=True)
|
||||
@@ -363,7 +363,7 @@ for line in sys.stdin:
|
||||
_launch_args=(sys.executable, str(script)),
|
||||
cwd=str(tmp_path),
|
||||
) as harness:
|
||||
result = harness.run("spawn a helper", session_id="main")
|
||||
result = harness.run([{"type": "text", "text": "spawn a helper"}], session_id="main")
|
||||
|
||||
assert [notification.method for notification in result.notifications] == [
|
||||
"session.event",
|
||||
@@ -388,11 +388,11 @@ for line in sys.stdin:
|
||||
method = msg.get("method")
|
||||
if method == "initialize":
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"serverInfo": {"name": "fake-runtime"}}}), flush=True)
|
||||
elif method == "session/prompt":
|
||||
elif method == "session/query":
|
||||
root = (msg.get("params") or {})["sessionId"]
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": root, "event": {"type": "agent/inbox/spliced", "data": {"target": "next-turn", "start": 0, "inserted": [{"id": "message-1"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": root, "status": "running"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"kind": "prompt", "messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "subagent.started", "params": {"parentSessionId": root, "childSessionId": "child"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": "child", "event": {"type": "assistant/message", "data": {"content": [{"type": "text", "text": "child response"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "subagent.started", "params": {"parentSessionId": "child", "childSessionId": "grandchild"}}), flush=True)
|
||||
@@ -413,7 +413,7 @@ for line in sys.stdin:
|
||||
cwd=str(tmp_path),
|
||||
) as harness:
|
||||
result = harness.run(
|
||||
"delegate recursively",
|
||||
[{"type": "text", "text": "delegate recursively"}],
|
||||
session_id="main",
|
||||
on_notification=lambda notification: seen.append(notification.method),
|
||||
)
|
||||
@@ -448,13 +448,13 @@ for line in sys.stdin:
|
||||
method = msg.get("method")
|
||||
if method == "initialize":
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"serverInfo": {"name": "fake-runtime"}}}), flush=True)
|
||||
elif method == "session/prompt":
|
||||
elif method == "session/query":
|
||||
params = msg.get("params") or {}
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": "other", "event": {"type": "assistant/message", "data": {"content": [{"type": "text", "text": "wrong session"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": "other", "status": "idle"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": params["sessionId"], "event": {"type": "agent/inbox/spliced", "data": {"target": "next-turn", "start": 0, "inserted": [{"id": "message-1"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": params["sessionId"], "status": "running"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"kind": "prompt", "messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": params["sessionId"], "event": {"type": "assistant/message", "data": {"content": [{"type": "text", "text": "right session"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": params["sessionId"], "status": "idle"}}), flush=True)
|
||||
elif method == "shutdown":
|
||||
@@ -467,7 +467,7 @@ for line in sys.stdin:
|
||||
_launch_args=(sys.executable, str(script)),
|
||||
cwd=str(tmp_path),
|
||||
) as harness:
|
||||
result = harness.run("stay in your lane", session_id="main")
|
||||
result = harness.run([{"type": "text", "text": "stay in your lane"}], session_id="main")
|
||||
|
||||
assert result.final_response == "right session"
|
||||
assert [notification.payload.get("sessionId") for notification in result.notifications] == ["main"] * 4
|
||||
@@ -485,11 +485,11 @@ for line in sys.stdin:
|
||||
method = msg.get("method")
|
||||
if method == "initialize":
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"serverInfo": {"name": "fake-runtime"}}}), flush=True)
|
||||
elif method == "session/prompt":
|
||||
elif method == "session/query":
|
||||
params = msg.get("params") or {}
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": params["sessionId"], "event": {"type": "agent/inbox/spliced", "data": {"target": "next-turn", "start": 0, "inserted": [{"id": "message-1"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": params["sessionId"], "status": "running"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"kind": "prompt", "messageId": "message-1"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": params["sessionId"], "event": {"type": "assistant/message", "data": {"content": [{"type": "text", "text": "ok"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": params["sessionId"], "status": "idle"}}), flush=True)
|
||||
elif method == "shutdown":
|
||||
@@ -499,7 +499,7 @@ for line in sys.stdin:
|
||||
)
|
||||
|
||||
with DeepSeekHarness(_launch_args=(sys.executable, str(script)), cwd=str(tmp_path)) as harness:
|
||||
result = harness.run("one turn", session_id="main")
|
||||
result = harness.run([{"type": "text", "text": "one turn"}], session_id="main")
|
||||
assert harness.client._notifications.qsize() == 0
|
||||
|
||||
|
||||
@@ -517,14 +517,14 @@ for line in sys.stdin:
|
||||
method = msg.get("method")
|
||||
if method == "initialize":
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"serverInfo": {"name": "fake-runtime"}}}), flush=True)
|
||||
elif method == "session/prompt":
|
||||
elif method == "session/query":
|
||||
turn += 1
|
||||
params = msg.get("params") or {}
|
||||
session_id = params["sessionId"]
|
||||
message_id = f"message-{turn}"
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": session_id, "event": {"type": "agent/inbox/spliced", "data": {"target": "next-turn", "start": 0, "inserted": [{"id": message_id}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": session_id, "status": "running"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"messageId": message_id}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": msg["id"], "result": {"kind": "prompt", "messageId": message_id}}), flush=True)
|
||||
if turn == 1:
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.event", "params": {"sessionId": session_id, "event": {"type": "assistant/message", "data": {"content": [{"type": "text", "text": "first"}]}}}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "method": "session.status", "params": {"sessionId": session_id, "status": "idle"}}), flush=True)
|
||||
@@ -539,8 +539,8 @@ for line in sys.stdin:
|
||||
)
|
||||
|
||||
with DeepSeekHarness(_launch_args=(sys.executable, str(script)), cwd=str(tmp_path)) as harness:
|
||||
first = harness.run("first turn", session_id="main")
|
||||
second = harness.run("second turn", session_id="main")
|
||||
first = harness.run([{"type": "text", "text": "first turn"}], session_id="main")
|
||||
second = harness.run([{"type": "text", "text": "second turn"}], session_id="main")
|
||||
|
||||
assert first.final_response == "first"
|
||||
assert second.final_response == "second"
|
||||
@@ -596,6 +596,15 @@ def test_client_keeps_unmatched_notifications_available_globally_while_subscribe
|
||||
assert notification.payload["sessionId"] == "other"
|
||||
|
||||
|
||||
def test_notification_timeout_keeps_subscription_available_for_later_events() -> None:
|
||||
client = HarnessClient()
|
||||
with client.subscribe_notifications() as subscription:
|
||||
with pytest.raises(TimeoutError, match="timed out waiting for an SDK notification"):
|
||||
subscription.next(timeout_seconds=0)
|
||||
client._handle_message({"jsonrpc": "2.0", "method": "tick", "params": {"value": 1}})
|
||||
assert subscription.next(timeout_seconds=0).payload == {"value": 1}
|
||||
|
||||
|
||||
def test_session_subscription_keeps_descendant_relationships_across_subscriptions() -> None:
|
||||
client = HarnessClient()
|
||||
with client.subscribe_session_notifications("main") as first:
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Encoded command images retain desktop command rules through the shipped SDK profile."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deepseek_harness.models import CommandExecution
|
||||
from deepseek_harness import RunResult
|
||||
from deepseek_harness.errors import JsonRpcError
|
||||
from test_commands_runtime import (
|
||||
_Completion, _SESSION_ID, _WAIT_SECONDS, _command_pair, _native_result, _of_type,
|
||||
_read_events, _request_texts, _runtime, _wait_activity,
|
||||
)
|
||||
from test_goal_runtime import _command, _until
|
||||
from test_plan_runtime import _GUIDANCE, _effective_system
|
||||
|
||||
|
||||
_IMAGE_MODEL = "deepseek-flash"
|
||||
_RED = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="
|
||||
_BLUE = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYPj/HwADAgH/5ncLrgAAAABJRU5ErkJggg=="
|
||||
|
||||
|
||||
def _image(name: str = "reference.png", data: str = _RED) -> dict:
|
||||
return {"type": "image", "data": data, "mimeType": "image/png", "name": name}
|
||||
|
||||
|
||||
def _request_images(request: dict) -> list[dict]:
|
||||
return [block for message in request["messages"] for block in message["content"]
|
||||
if block["type"] == "image"]
|
||||
|
||||
|
||||
def _assert_request_images(request: dict, *images: dict) -> None:
|
||||
assert _request_images(request) == [
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": image["data"]}}
|
||||
for image in images
|
||||
]
|
||||
|
||||
|
||||
def _image_messages(events: list[dict]) -> list[dict]:
|
||||
return [event for event in _of_type(events, "user/message")
|
||||
if any(block["type"] == "image" for block in event["data"]["content"])]
|
||||
|
||||
|
||||
def _assert_durable_image(block: dict, image: dict) -> None:
|
||||
data = base64.b64decode(image["data"])
|
||||
assert block == {"type": "image", "attachment": {
|
||||
"attachmentId": "sha256:" + hashlib.sha256(data).hexdigest(),
|
||||
"mediaType": "image/png", "width": 1, "height": 1,
|
||||
"bytes": len(data), "name": image["name"],
|
||||
}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("line,body,image_first", [
|
||||
("/plan Describe this picture.", "Describe this picture.", False), ("/plan", "", True),
|
||||
])
|
||||
def test_plan_image_reaches_the_model_and_persists_as_a_reference(
|
||||
tmp_path: Path, line: str, body: str, image_first: bool,
|
||||
) -> None:
|
||||
picture = _image()
|
||||
text = {"type": "text", "text": line}
|
||||
content = [picture, text] if image_first else [text, picture]
|
||||
with _runtime(tmp_path, [_Completion("IMAGE_PLAN_READY")], model=_IMAGE_MODEL) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
execution = runtime.call(session.run, content)
|
||||
assert isinstance(execution, RunResult)
|
||||
assert execution.finish_reason == "completed" and execution.final_response == "IMAGE_PLAN_READY"
|
||||
_wait_activity(runtime, session)
|
||||
[request] = runtime.model.captured()
|
||||
_assert_request_images(request, picture)
|
||||
assert _GUIDANCE in _effective_system(request)
|
||||
assert line not in _request_texts(request)
|
||||
if body:
|
||||
assert body in _request_texts(request)
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, execution, "plan")
|
||||
assert [event["data"] for event in _of_type(events, "plan/mode")] == [{"active": True}]
|
||||
[message] = _image_messages(events)
|
||||
content = message["data"]["content"]
|
||||
_assert_durable_image(content[0], picture)
|
||||
assert content[1:] == ([{"type": "text", "text": body}] if body else [])
|
||||
digest = content[0]["attachment"]["attachmentId"].removeprefix("sha256:")
|
||||
stored = tmp_path / "home" / "attachments" / "v1" / "objects" / digest[:2] / digest
|
||||
assert stored.read_bytes() == base64.b64decode(picture["data"])
|
||||
|
||||
|
||||
def test_plan_off_compact_and_permission_reject_images_without_applying_the_command(tmp_path: Path) -> None:
|
||||
picture = _image()
|
||||
with _runtime(tmp_path, [], model=_IMAGE_MODEL, initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
_command(runtime, session, "/plan")
|
||||
executions = []
|
||||
for line in ["/plan off", "/compact", "/permission danger-full-access"]:
|
||||
execution = runtime.call(session.run, [{"type": "text", "text": line}, picture])
|
||||
assert isinstance(execution, RunResult)
|
||||
assert execution.finish_reason == "error"
|
||||
execution = _native_result(execution)
|
||||
assert "attachment" in execution.result.text.lower()
|
||||
executions.append((line.split()[0][1:], execution))
|
||||
assert "Plan mode off" in _command(runtime, session, "/plan off").result.text
|
||||
assert runtime.model.captured() == []
|
||||
events = _read_events(runtime.logs)
|
||||
for name, execution in executions:
|
||||
_command_pair(events, execution, name)
|
||||
start = next(event["seq"] for event in _of_type(events, "command/run")
|
||||
if event["data"]["commandId"] == execution.command_id)
|
||||
done = next(event["seq"] for event in _of_type(events, "command/done")
|
||||
if event["data"]["commandId"] == execution.command_id)
|
||||
assert events[start + 1:done] == []
|
||||
assert [event["data"] for event in _of_type(events, "plan/mode")] == [{"active": True}, {"active": False}]
|
||||
assert [event["data"] for event in _of_type(events, "permission/preset")] == [{"preset": "workspace-write"}]
|
||||
assert not _image_messages(events) and not _of_type(events, "turn/start")
|
||||
|
||||
|
||||
def test_goal_create_and_edit_images_reach_following_requests_as_durable_references(tmp_path: Path) -> None:
|
||||
original = _image("original.png")
|
||||
revised = _image("revised.png", _BLUE)
|
||||
first = _Completion("ORIGINAL_GOAL_IMAGE", blocked=True)
|
||||
second = _Completion("REVISED_GOAL_IMAGE", blocked=True)
|
||||
with _runtime(tmp_path, [first, second], model=_IMAGE_MODEL) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
created = runtime.call(runtime.harness.client._query, session.id, [original, {"type": "text", "text": "/goal Inspect the original picture."}])
|
||||
assert isinstance(created, CommandExecution) and created.result.kind == "success"
|
||||
assert first.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
edited = runtime.call(runtime.harness.client._query, session.id, [revised, {"type": "text", "text": "/goal edit Inspect the revised picture."}])
|
||||
assert isinstance(edited, CommandExecution) and edited.result.kind == "success"
|
||||
_command(runtime, session, "/goal clear")
|
||||
assert not first.release.is_set()
|
||||
first.release.set()
|
||||
assert second.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
second.release.set()
|
||||
_wait_activity(runtime, session)
|
||||
before, after = runtime.model.captured()
|
||||
_assert_request_images(before, original)
|
||||
_assert_request_images(after, original, revised)
|
||||
assert "Reference attachments for the goal objective." in _request_texts(before)
|
||||
assert "Reference attachments for the goal objective." in _request_texts(after)
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, created, "goal")
|
||||
_command_pair(events, edited, "goal")
|
||||
changes = _of_type(events, "goal/change")
|
||||
assert [event["data"]["operation"] for event in changes] == ["create", "edit", "clear"]
|
||||
assert changes[1]["data"]["goal"]["objective"] == "Inspect the revised picture."
|
||||
messages = _image_messages(events)
|
||||
assert len(messages) == 2
|
||||
for message, picture in zip(messages, [original, revised]):
|
||||
_assert_durable_image(message["data"]["content"][0], picture)
|
||||
assert message["data"]["content"][1:] == [{
|
||||
"type": "text", "text": "Reference attachments for the goal objective.",
|
||||
}]
|
||||
|
||||
|
||||
def test_cleared_goal_waits_for_its_image_input_queued_during_compaction(tmp_path: Path) -> None:
|
||||
picture = _image()
|
||||
summary = _Completion("Retain the earlier task.", blocked=True)
|
||||
reply = _Completion("REFERENCE_INPUT_REPLY", blocked=True)
|
||||
with _runtime(tmp_path, [_Completion("EARLIER_REPLY"), summary, reply], model=_IMAGE_MODEL) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
runtime.call(session.run, "Retain this earlier task. " * 256)
|
||||
compacting = runtime.workers.submit(session.run, "/compact")
|
||||
assert summary.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
with runtime.harness.client.subscribe_session_notifications(session.id) as notices:
|
||||
running = runtime.workers.submit(session.run, [
|
||||
{"type": "text", "text": "/goal Inspect the reference."}, picture,
|
||||
])
|
||||
_until(notices, "goal/change", "create")
|
||||
cleared = session.run("/goal clear")
|
||||
assert cleared.finish_reason == "completed" and cleared.final_response == "Goal cleared."
|
||||
summary.release.set()
|
||||
assert compacting.result(timeout=_WAIT_SECONDS).finish_reason == "completed"
|
||||
assert reply.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
assert not running.done()
|
||||
_assert_request_images(runtime.model.captured()[-1], picture)
|
||||
reply.release.set()
|
||||
result = running.result(timeout=_WAIT_SECONDS)
|
||||
assert result.finish_reason == "aborted" and result.final_response == "REFERENCE_INPUT_REPLY"
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, result, "goal")
|
||||
assert [event["data"]["operation"] for event in _of_type(events, "goal/change")] == ["create", "clear"]
|
||||
|
||||
|
||||
def test_goal_controls_with_images_leave_the_active_request_and_goal_unchanged(tmp_path: Path) -> None:
|
||||
first = _Completion("CURRENT_GOAL_REQUEST_FINISHED", blocked=True)
|
||||
with _runtime(tmp_path, [first], model=_IMAGE_MODEL) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
with runtime.harness.client.subscribe_session_notifications(session.id) as notices:
|
||||
_command(runtime, session, "/goal Keep the current goal.")
|
||||
assert first.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
rejected = []
|
||||
for line in ["/goal", "/goal pause", "/goal resume", "/goal clear", "/goal edit", "/goal Duplicate goal"]:
|
||||
execution = runtime.call(session.run, [{"type": "text", "text": line}, _image()])
|
||||
assert isinstance(execution, RunResult) and execution.finish_reason == "error"
|
||||
execution = _native_result(execution)
|
||||
rejected.append(execution)
|
||||
assert "Status: active" in _command(runtime, session, "/goal").result.text
|
||||
assert not first.release.is_set() and len(runtime.model.captured()) == 1
|
||||
_command(runtime, session, "/goal clear")
|
||||
first.release.set()
|
||||
_wait_activity(runtime, session)
|
||||
assert _until(notices, "turn/end")["reason"]["kind"] == "completed"
|
||||
assert len(runtime.model.captured()) == 1
|
||||
events = _read_events(runtime.logs)
|
||||
for execution in rejected:
|
||||
_command_pair(events, execution, "goal")
|
||||
assert [event["data"]["operation"] for event in _of_type(events, "goal/change")] == ["create", "clear"]
|
||||
assert not _image_messages(events)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_first", [False, True])
|
||||
def test_prompt_images_queue_during_an_active_turn(tmp_path: Path, image_first: bool) -> None:
|
||||
picture = _image()
|
||||
text = {"type": "text", "text": "Inspect the image."}
|
||||
content = [picture, text] if image_first else [text, picture]
|
||||
first = _Completion("BEFORE_IMAGE", blocked=True)
|
||||
with _runtime(tmp_path, [first, _Completion("IMAGE_RECEIVED")], model=_IMAGE_MODEL) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
running = runtime.workers.submit(session.run, "Start without an image.")
|
||||
assert first.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
accepted = threading.Event()
|
||||
following = runtime.workers.submit(
|
||||
session.run, content,
|
||||
on_notification=lambda _notification: accepted.set(),
|
||||
)
|
||||
assert accepted.wait(_WAIT_SECONDS), "image prompt did not reach its inbox receipt"
|
||||
assert not first.release.is_set() and not following.done()
|
||||
first.release.set()
|
||||
assert isinstance(running.result(timeout=_WAIT_SECONDS), RunResult)
|
||||
result = following.result(timeout=_WAIT_SECONDS)
|
||||
assert isinstance(result, RunResult) and result.final_response == "IMAGE_RECEIVED"
|
||||
before, after = runtime.model.captured()
|
||||
_assert_request_images(before)
|
||||
_assert_request_images(after, picture)
|
||||
events = _read_events(runtime.logs)
|
||||
[message] = _image_messages(events)
|
||||
recorded = message["data"]["content"]
|
||||
_assert_durable_image(next(block for block in recorded if block["type"] == "image"), picture)
|
||||
assert [block["type"] for block in recorded] == [block["type"] for block in content]
|
||||
assert [block for block in recorded if block["type"] == "text"] == [text]
|
||||
assert not _of_type(events, "command/run")
|
||||
assert [event["data"] for event in _of_type(events, "step/start")] == [
|
||||
{"turn": 1, "step": 1}, {"turn": 2, "step": 1},
|
||||
]
|
||||
|
||||
|
||||
def test_top_level_run_accepts_an_image_without_text(tmp_path: Path) -> None:
|
||||
picture = _image()
|
||||
with _runtime(tmp_path, [_Completion("IMAGE_ONLY_REPLY")], model=_IMAGE_MODEL) as runtime:
|
||||
result = runtime.call(runtime.harness.run, [picture], session_id=_SESSION_ID)
|
||||
assert isinstance(result, RunResult) and result.final_response == "IMAGE_ONLY_REPLY"
|
||||
[request] = runtime.model.captured()
|
||||
_assert_request_images(request, picture)
|
||||
[message] = _image_messages(_read_events(runtime.logs))
|
||||
[block] = message["data"]["content"]
|
||||
_assert_durable_image(block, picture)
|
||||
|
||||
|
||||
def test_invalid_image_batch_cannot_enter_plan_mode_or_deliver_a_partial_prompt(tmp_path: Path) -> None:
|
||||
images = [_image(), _image("invalid.png", "not canonical base64")]
|
||||
with _runtime(tmp_path, [], model=_IMAGE_MODEL) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
with pytest.raises(JsonRpcError) as ambiguous:
|
||||
runtime.call(session.run, [
|
||||
{"type": "text", "text": "/plan off"}, _image(),
|
||||
{"type": "text", "text": "This is separate text, not another command argument."},
|
||||
])
|
||||
assert "command" in str(ambiguous.value).lower()
|
||||
rejected = runtime.call(session.run, [{"type": "text", "text": "/plan Inspect the pictures."}, *images])
|
||||
assert isinstance(rejected, RunResult) and rejected.finish_reason == "error"
|
||||
rejected = _native_result(rejected)
|
||||
assert "base64" in rejected.result.text
|
||||
with pytest.raises(JsonRpcError, match="base64"):
|
||||
runtime.call(session.run, [*images, {"type": "text", "text": "Inspect the pictures."}])
|
||||
assert "already inactive" in _command(runtime, session, "/plan off").result.text
|
||||
assert runtime.model.captured() == []
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, rejected, "plan")
|
||||
assert len(_of_type(events, "command/run")) == 2
|
||||
assert not _of_type(events, "plan/mode")
|
||||
assert not _of_type(events, "agent/inbox/spliced")
|
||||
assert not _of_type(events, "turn/start")
|
||||
assert not _image_messages(events)
|
||||
@@ -0,0 +1,668 @@
|
||||
"""Exercise Session commands through Python and the real sdk profile with a local model.
|
||||
|
||||
The default carrier is the repository's built dsh CLI. Set
|
||||
DSH_COMMANDS_TEST_RUNTIME=installed to run these same cases against the imported
|
||||
SDK wheel and its bundled runtime, including from outside the repository.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import closing, contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from http.client import HTTPConnection, RemoteDisconnected
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Callable, Iterator, TypeVar
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from deepseek_harness.models import CommandExecution
|
||||
from deepseek_harness import DeepSeekHarness, Notification, RunResult, Session
|
||||
from deepseek_harness.client import UserQuestionHandler
|
||||
|
||||
|
||||
_WAIT_SECONDS = 30
|
||||
_SESSION_ID = "commands-runtime"
|
||||
_MODEL = "deepseek-v4-flash"
|
||||
_HISTORY_MARKER = "DSH_COMMANDS_OLD_USER_INPUT"
|
||||
_FIRST_REPLY = "SDK_COMPACT_READY"
|
||||
_CHECKPOINT = "## Primary Request and Intent\n- DSH_COMMANDS_CHECKPOINT: retain the earlier task."
|
||||
_CONTINUE_PROMPT = "DSH_COMMANDS_CONTINUE: answer using the retained checkpoint."
|
||||
_CONTINUE_REPLY = "SDK_COMPACT_OK"
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Completion:
|
||||
text: str
|
||||
blocked: bool = False
|
||||
tool_call: tuple[str, dict] | Callable[[dict], tuple[str, dict]] | None = None
|
||||
allow_disconnect: bool = False
|
||||
entered: threading.Event = field(default_factory=threading.Event)
|
||||
release: threading.Event = field(default_factory=threading.Event)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.blocked:
|
||||
self.release.set()
|
||||
|
||||
|
||||
def _messages_response(text: str, model: str, ordinal: int, tool_call: tuple[str, dict] | None = None) -> bytes:
|
||||
if tool_call is None:
|
||||
block = {"type": "text", "text": ""}
|
||||
delta = {"type": "text_delta", "text": text}
|
||||
else:
|
||||
name, arguments = tool_call
|
||||
block = {"type": "tool_use", "id": f"commands-tool-{ordinal}", "name": name, "input": {}}
|
||||
delta = {"type": "input_json_delta", "partial_json": json.dumps(arguments)}
|
||||
events = [
|
||||
{"type": "message_start", "message": {
|
||||
"id": f"commands-response-{ordinal}", "type": "message", "role": "assistant",
|
||||
"model": model, "content": [], "stop_reason": None, "stop_sequence": None,
|
||||
"usage": {"input_tokens": 10000, "output_tokens": 0},
|
||||
}},
|
||||
{"type": "content_block_start", "index": 0, "content_block": block},
|
||||
{"type": "content_block_delta", "index": 0, "delta": delta},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{"type": "message_delta", "delta": {"stop_reason": "end_turn" if tool_call is None else "tool_use", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 32}},
|
||||
{"type": "message_stop"},
|
||||
]
|
||||
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode()
|
||||
|
||||
|
||||
class _ModelServer:
|
||||
def __init__(self, completions: list[_Completion]) -> None:
|
||||
self.completions = completions
|
||||
self.requests: list[dict] = []
|
||||
self.failures: list[BaseException] = []
|
||||
self.lock = threading.Lock()
|
||||
owner = self
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def setup(self) -> None:
|
||||
self.request.settimeout(_WAIT_SECONDS)
|
||||
super().setup()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
payload = self.rfile.read(int(self.headers["Content-Length"]))
|
||||
if self.path == "/v1/files":
|
||||
# The model fixture accepts inline images through the provider's Files fallback.
|
||||
self.send_error(404, "Images are accepted inline")
|
||||
return
|
||||
assert self.path == "/v1/messages", f"unexpected model endpoint: {self.path}"
|
||||
body = json.loads(payload)
|
||||
assert isinstance(body, dict) and isinstance(body.get("model"), str)
|
||||
with owner.lock:
|
||||
ordinal = len(owner.requests)
|
||||
owner.requests.append(body)
|
||||
assert ordinal < len(owner.completions), "unexpected additional model request"
|
||||
completion = owner.completions[ordinal]
|
||||
completion.entered.set()
|
||||
# The test owns release. This larger bound only contains a failed test.
|
||||
assert completion.release.wait(_WAIT_SECONDS * 2), "model barrier was not released"
|
||||
tool_call = completion.tool_call(body) if callable(completion.tool_call) else completion.tool_call
|
||||
payload = _messages_response(completion.text, body["model"], ordinal, tool_call)
|
||||
try:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
self.wfile.flush()
|
||||
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
|
||||
if not completion.allow_disconnect:
|
||||
raise
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
pass
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
# server_close joins request handlers after their barriers are released.
|
||||
daemon_threads = False
|
||||
|
||||
def handle_error(self, _request: object, _client_address: object) -> None:
|
||||
error = sys.exc_info()[1]
|
||||
assert error is not None
|
||||
with owner.lock:
|
||||
owner.failures.append(error)
|
||||
|
||||
self.server = Server(("127.0.0.1", 0), Handler)
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, name="commands-model", daemon=True)
|
||||
host, port = self.server.server_address
|
||||
self.url = f"http://{host}:{port}"
|
||||
|
||||
def __enter__(self) -> _ModelServer:
|
||||
try:
|
||||
self.thread.start()
|
||||
except BaseException:
|
||||
self.server.server_close()
|
||||
raise
|
||||
return self
|
||||
|
||||
def release_all(self) -> None:
|
||||
for completion in self.completions:
|
||||
completion.release.set()
|
||||
|
||||
def captured(self) -> list[dict]:
|
||||
with self.lock:
|
||||
return list(self.requests)
|
||||
|
||||
def __exit__(self, exc_type: object, _exc: object, _tb: object) -> None:
|
||||
self.release_all()
|
||||
self.server.shutdown()
|
||||
try:
|
||||
self.server.server_close()
|
||||
finally:
|
||||
self.thread.join(_WAIT_SECONDS)
|
||||
assert not self.thread.is_alive(), "model listener did not stop"
|
||||
if exc_type is None:
|
||||
assert not self.failures, [repr(error) for error in self.failures]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error,allow_disconnect,accepted", [
|
||||
(ConnectionAbortedError(10053, "response connection aborted"), True, True),
|
||||
(ConnectionAbortedError(10053, "response connection aborted"), False, False),
|
||||
(OSError(5, "unrelated response write failure"), True, False),
|
||||
])
|
||||
def test_model_server_only_allows_expected_connection_aborts(
|
||||
monkeypatch: pytest.MonkeyPatch, error: OSError, allow_disconnect: bool, accepted: bool,
|
||||
) -> None:
|
||||
model = _ModelServer([_Completion("response", allow_disconnect=allow_disconnect)])
|
||||
body = {"model": _MODEL}
|
||||
|
||||
def fail_response(_handler: BaseHTTPRequestHandler) -> None:
|
||||
raise error
|
||||
|
||||
# Keep the real HTTP handler and teardown; only the response write fails deterministically.
|
||||
monkeypatch.setattr(model.server.RequestHandlerClass, "end_headers", fail_response)
|
||||
|
||||
def exchange() -> None:
|
||||
with model, closing(HTTPConnection(*model.server.server_address, timeout=_WAIT_SECONDS)) as connection:
|
||||
connection.request("POST", "/v1/messages", body=json.dumps(body))
|
||||
with pytest.raises(RemoteDisconnected):
|
||||
connection.getresponse()
|
||||
|
||||
if accepted:
|
||||
exchange()
|
||||
else:
|
||||
with pytest.raises(AssertionError, match=type(error).__name__):
|
||||
exchange()
|
||||
assert model.captured() == [body]
|
||||
assert model.failures == ([] if accepted else [error])
|
||||
assert not model.thread.is_alive()
|
||||
|
||||
|
||||
def _dsh_bin() -> str | None:
|
||||
mode = os.environ.get("DSH_COMMANDS_TEST_RUNTIME", "built")
|
||||
if mode == "installed":
|
||||
return None
|
||||
if mode != "built":
|
||||
raise ValueError("DSH_COMMANDS_TEST_RUNTIME must be built or installed")
|
||||
repo = Path(__file__).resolve().parents[3]
|
||||
node = shutil.which("node")
|
||||
binary = repo / "apps/cli/lib/bin.js"
|
||||
if node is None or not binary.is_file():
|
||||
pytest.skip("requires Node and built dsh CLI, or DSH_COMMANDS_TEST_RUNTIME=installed")
|
||||
return str(binary)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Runtime:
|
||||
harness: DeepSeekHarness
|
||||
model: _ModelServer
|
||||
workers: ThreadPoolExecutor
|
||||
logs: Path
|
||||
|
||||
def call(self, operation: Callable[..., _T], *args: object, **kwargs: object) -> _T:
|
||||
return self.workers.submit(operation, *args, **kwargs).result(timeout=_WAIT_SECONDS)
|
||||
|
||||
|
||||
class _WaitResult(BaseModel):
|
||||
pass
|
||||
|
||||
|
||||
def _wait_activity(runtime: _Runtime, session: Session) -> None:
|
||||
# Command completion does not own the model activity that its native handler starts.
|
||||
runtime.call(runtime.harness.client.request, "session/wait", {"sessionId": session.id}, response_model=_WaitResult)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _runtime(
|
||||
tmp_path: Path, completions: list[_Completion], *, capture_patch: Path | None = None,
|
||||
initial_permission: str = "danger-full-access",
|
||||
on_user_question: UserQuestionHandler | None = None,
|
||||
max_goal_rounds: int | None = None,
|
||||
model: str = _MODEL,
|
||||
) -> Iterator[_Runtime]:
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir(exist_ok=True)
|
||||
home = tmp_path / "home"
|
||||
logs = tmp_path / "sessions"
|
||||
patch = tmp_path / "commands.patch.yml"
|
||||
# Raw JSONL permits an independent read at the compaction durability boundary.
|
||||
patch.write_text(
|
||||
"- id: session-persistence-jsonl\n config:\n root: " + json.dumps(str(logs))
|
||||
+ "\n compression: none\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
if max_goal_rounds is not None:
|
||||
with patch.open("a", encoding="utf-8") as extra:
|
||||
extra.write(f"- id: goal\n config:\n defaultMaxGoalRounds: {max_goal_rounds}\n")
|
||||
patches = (patch,) if capture_patch is None else (patch, capture_patch.resolve())
|
||||
dsh_bin = _dsh_bin()
|
||||
with _ModelServer(completions) as model_server, ThreadPoolExecutor(max_workers=2) as workers:
|
||||
harness = DeepSeekHarness(
|
||||
cwd=str(workspace), runtime_cwd=str(workspace), dsh_home=str(home),
|
||||
dsh_bin=dsh_bin, profile="sdk", patches=tuple(str(item) for item in patches), model=model,
|
||||
base_url=model_server.url, api_key="keyless-commands-runtime",
|
||||
initialize_timeout_seconds=60, request_timeout_seconds=20, shutdown_timeout_seconds=20,
|
||||
on_user_question=on_user_question,
|
||||
env={"DSH_PERMISSION_MODE": initial_permission,
|
||||
"DSH_TELEMETRY_DISABLED": "1", "DSH_TOOLS_MODE": "native"},
|
||||
)
|
||||
try:
|
||||
harness.start()
|
||||
yield _Runtime(harness, model_server, workers, logs)
|
||||
finally:
|
||||
# Also covers failed initialization: release HTTP, close the client, then join callers.
|
||||
model_server.release_all()
|
||||
harness.close()
|
||||
|
||||
|
||||
def _read_events(root: Path, *, live: bool = False, session_id: str = _SESSION_ID) -> list[dict]:
|
||||
paths = []
|
||||
for path in root.rglob("session*.jsonl"):
|
||||
with path.open(encoding="utf-8") as log:
|
||||
header = json.loads(log.readline())
|
||||
if header.get("id") == session_id:
|
||||
paths.append(path)
|
||||
assert len(paths) == 1, f"expected one Session log for {session_id}, found {paths}"
|
||||
content = paths[0].read_bytes()
|
||||
if not live:
|
||||
assert content.endswith(b"\n"), "closed Session log has an incomplete final row"
|
||||
# A live writer may append an unflushed suffix after the checkpoint we inspect.
|
||||
complete = content[:content.rfind(b"\n") + 1]
|
||||
records = [json.loads(line) for line in complete.splitlines()]
|
||||
assert records and records[0]["type"] == "session"
|
||||
assert records[0]["id"] == session_id
|
||||
events = records[1:]
|
||||
for seq, event in enumerate(events):
|
||||
assert isinstance(event, dict)
|
||||
assert event.setdefault("seq", seq) == seq
|
||||
return events
|
||||
|
||||
|
||||
def _of_type(events: list[dict], name: str) -> list[dict]:
|
||||
return [event for event in events if event["type"] == name]
|
||||
|
||||
|
||||
def _native_result(result: RunResult) -> CommandExecution:
|
||||
"""Read the native outcome retained in a high-level result's command lifecycle events."""
|
||||
started = next(event for event in result.events if event["type"] == "command/run")
|
||||
identity = started["data"]["commandId"]
|
||||
done = next(event["data"] for event in result.events
|
||||
if event["type"] == "command/done" and event["data"]["commandId"] == identity)
|
||||
return CommandExecution.model_validate({"commandId": identity, "result": {
|
||||
key: value for key, value in done.items() if key in ("kind", "text", "sourceEventSeq")
|
||||
}})
|
||||
|
||||
|
||||
def _command_pair(events: list[dict], execution: CommandExecution | RunResult, name: str = "compact") -> None:
|
||||
if isinstance(execution, RunResult):
|
||||
execution = _native_result(execution)
|
||||
runs = [event for event in _of_type(events, "command/run")
|
||||
if event["data"]["commandId"] == execution.command_id]
|
||||
done = [event for event in _of_type(events, "command/done")
|
||||
if event["data"]["commandId"] == execution.command_id]
|
||||
assert len(runs) == len(done) == 1
|
||||
assert runs[0]["data"]["name"] == name
|
||||
assert runs[0]["seq"] < done[0]["seq"]
|
||||
assert done[0]["data"]["kind"] == execution.result.kind
|
||||
assert done[0]["data"].get("text") == execution.result.text
|
||||
|
||||
|
||||
def _text_blocks(content: object) -> list[str]:
|
||||
if isinstance(content, str):
|
||||
return [content]
|
||||
assert isinstance(content, list)
|
||||
return [block["text"] for block in content if block.get("type") == "text"]
|
||||
|
||||
|
||||
def _request_texts(request: dict) -> list[str]:
|
||||
return [text for message in request["messages"] for text in _text_blocks(message["content"])]
|
||||
|
||||
|
||||
def _wait_command_done(next_notification: Callable[[], Notification]) -> dict:
|
||||
while True:
|
||||
notification = next_notification()
|
||||
if notification.method != "session.event" or notification.payload.get("sessionId") != _SESSION_ID:
|
||||
continue
|
||||
event = notification.payload.get("event")
|
||||
if isinstance(event, dict) and event.get("type") == "command/done":
|
||||
return event["data"]
|
||||
|
||||
|
||||
def test_compact_without_history_returns_without_a_model_turn(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, []) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
observer = runtime.harness.client.subscribe_session_notifications(_SESSION_ID)
|
||||
try:
|
||||
execution = runtime.call(session.run, " \n/压缩\t ")
|
||||
done = runtime.call(_wait_command_done, observer.next)
|
||||
except BaseException:
|
||||
# Wake a timed-out observer before unregistering its otherwise blocking queue.
|
||||
runtime.harness.close()
|
||||
raise
|
||||
finally:
|
||||
observer.close()
|
||||
assert isinstance(execution, RunResult)
|
||||
execution = _native_result(execution)
|
||||
assert done["commandId"] == execution.command_id
|
||||
assert execution.result.kind == "success"
|
||||
assert execution.result.source_event_seq is None
|
||||
assert runtime.model.captured() == []
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, execution)
|
||||
assert not {event["type"] for event in events} & {
|
||||
"turn/start", "step/start", "request/header", "compaction/start", "user/message",
|
||||
}
|
||||
|
||||
|
||||
def test_compact_rejects_busy_agent_before_the_model_is_released(tmp_path: Path) -> None:
|
||||
response = _Completion("DSH_COMMANDS_BUSY_REPLY", blocked=True)
|
||||
with _runtime(tmp_path, [response]) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
running = runtime.workers.submit(
|
||||
session.run, "Hold this request while compaction is attempted.",
|
||||
)
|
||||
assert response.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
assert not running.done() and not response.release.is_set()
|
||||
execution = runtime.call(session.run, "/compact")
|
||||
assert isinstance(execution, RunResult)
|
||||
execution = _native_result(execution)
|
||||
assert execution.result.kind == "error"
|
||||
assert not response.release.is_set()
|
||||
assert not running.done()
|
||||
assert len(runtime.model.captured()) == 1
|
||||
response.release.set()
|
||||
completed = running.result(timeout=_WAIT_SECONDS)
|
||||
assert isinstance(completed, RunResult)
|
||||
assert completed.final_response == response.text
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, execution)
|
||||
assert len(_of_type(events, "turn/start")) == len(_of_type(events, "turn/end")) == 1
|
||||
assert not _of_type(events, "compaction/start")
|
||||
|
||||
|
||||
def run_compaction_scenario(tmp_path: Path, *, capture_patch: Path | None = None) -> Path:
|
||||
"""Run the complete scenario and retain its real Session log under the caller's directory."""
|
||||
summary_response = _Completion(_CHECKPOINT, blocked=True)
|
||||
with _runtime(
|
||||
tmp_path, [_Completion(_FIRST_REPLY), summary_response, _Completion(_CONTINUE_REPLY)],
|
||||
capture_patch=capture_patch,
|
||||
) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
# The replaced span is deliberately larger than the framed summary.
|
||||
prompt = _HISTORY_MARKER + "\n" + "Keep the original request only in the stored history. " * 256
|
||||
first_result = runtime.call(session.run, prompt)
|
||||
assert isinstance(first_result, RunResult)
|
||||
assert first_result.final_response == _FIRST_REPLY
|
||||
|
||||
compacting = runtime.workers.submit(session.run, "/compact")
|
||||
assert summary_response.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
assert not summary_response.release.is_set()
|
||||
assert not compacting.done(), "command returned before its summary request completed"
|
||||
summary_response.release.set()
|
||||
execution = compacting.result(timeout=_WAIT_SECONDS)
|
||||
assert isinstance(execution, RunResult)
|
||||
execution = _native_result(execution)
|
||||
assert execution.result.kind == "success"
|
||||
assert execution.result.source_event_seq is not None
|
||||
|
||||
# The native compactor promises this prefix before returning, unlike command/done.
|
||||
checkpointed = _read_events(runtime.logs, live=True)
|
||||
summaries = _of_type(checkpointed, "compaction/summary")
|
||||
starts = _of_type(checkpointed, "compaction/start")
|
||||
ends = _of_type(checkpointed, "compaction/end")
|
||||
assert len(starts) == len(summaries) == len(ends) == 1
|
||||
summary = summaries[0]
|
||||
assert summary["seq"] == execution.result.source_event_seq
|
||||
assert summary["data"]["llmStreamCall"] is True
|
||||
assert summary["data"]["provider"] == "deepseek-official"
|
||||
assert summary["data"]["model"] == _MODEL
|
||||
for event in [starts[0], summary, ends[0]]:
|
||||
assert event["data"]["sourceCommandId"] == execution.command_id
|
||||
assert event["data"]["compactionId"] == summary["data"]["compactionId"]
|
||||
assert starts[0]["data"]["turn"] is None and ends[0]["data"]["turn"] is None
|
||||
assert "error" not in ends[0]["data"]
|
||||
checkpoint = checkpointed[summary["seq"] + 1]
|
||||
assert checkpoint["type"] == "user/message"
|
||||
assert checkpoint["data"]["source"]["kind"] == "compact-checkpoint"
|
||||
assert checkpoint["surfaceOp"]["op"] == "replace"
|
||||
checkpoint_text = "\n".join(_text_blocks(checkpoint["data"]["content"]))
|
||||
assert _CHECKPOINT in checkpoint_text
|
||||
assert "<compacted-summary>" in checkpoint_text and "</compacted-summary>" in checkpoint_text
|
||||
assert len(_of_type(checkpointed, "turn/start")) == len(_of_type(checkpointed, "turn/end")) == 1
|
||||
|
||||
continued = runtime.call(session.run, _CONTINUE_PROMPT)
|
||||
assert isinstance(continued, RunResult)
|
||||
assert continued.final_response == _CONTINUE_REPLY
|
||||
first, compaction, following = runtime.model.captured()
|
||||
assert first["system"] == compaction["system"]
|
||||
assert first["tools"] == compaction["tools"]
|
||||
initial_texts = _request_texts(first)
|
||||
summary_texts = _request_texts(compaction)
|
||||
assert summary_texts[:len(initial_texts)] == initial_texts
|
||||
assert "You are now acting as a compaction engine" in summary_texts[-1]
|
||||
following_texts = _request_texts(following)
|
||||
following_text = "\n".join(following_texts)
|
||||
assert _CHECKPOINT in following_text and _CONTINUE_PROMPT in following_text
|
||||
assert _FIRST_REPLY in following_text
|
||||
assert _HISTORY_MARKER not in following_text
|
||||
assert "/compact" not in following_texts
|
||||
assert execution.result.text not in following_text
|
||||
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, execution)
|
||||
assert len(_of_type(events, "turn/start")) == len(_of_type(events, "turn/end")) == 2
|
||||
assert len(_of_type(events, "step/start")) == len(_of_type(events, "step/end")) == 2
|
||||
assert _HISTORY_MARKER in json.dumps(events), "compaction erased the original stored input"
|
||||
return next(runtime.logs.rglob("session*.jsonl"))
|
||||
|
||||
|
||||
def test_compact_checkpoints_history_before_continuing_the_same_session(tmp_path: Path) -> None:
|
||||
run_compaction_scenario(tmp_path)
|
||||
|
||||
|
||||
def test_unknown_slash_and_compact_arguments_follow_the_plain_prompt_path(tmp_path: Path) -> None:
|
||||
prompts = ["/sdk-unregistered-command-fixture", "/compact extra"]
|
||||
replies = ["SDK_UNKNOWN_SLASH_REPLY", "SDK_COMPACT_ARGUMENTS_REPLY"]
|
||||
with _runtime(tmp_path, [_Completion(text) for text in replies]) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
for prompt, reply in zip(prompts, replies):
|
||||
result = runtime.call(session.run, prompt)
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.final_response == reply
|
||||
requests = runtime.model.captured()
|
||||
assert len(requests) == len(prompts)
|
||||
for prompt, request in zip(prompts, requests):
|
||||
assert prompt in _request_texts(request)
|
||||
events = _read_events(runtime.logs)
|
||||
assert not _of_type(events, "command/run")
|
||||
assert not _of_type(events, "compaction/start")
|
||||
assert len(_of_type(events, "turn/start")) == len(_of_type(events, "turn/end")) == 2
|
||||
|
||||
|
||||
def test_unimplemented_slash_remains_plain_text_without_recording_feedback(tmp_path: Path) -> None:
|
||||
input_text = "/feedback DSH_FEEDBACK_TEXT_INPUT"
|
||||
following = "DSH_AFTER_PLAN_TEXT_INPUT"
|
||||
reply_before = "DSH_FEEDBACK_TEXT_REPLY"
|
||||
reply = "DSH_AFTER_PLAN_TEXT_REPLY"
|
||||
with _runtime(tmp_path, [_Completion(reply_before), _Completion(reply)]) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
first = runtime.call(session.run, input_text)
|
||||
assert isinstance(first, RunResult)
|
||||
assert first.final_response == reply_before
|
||||
|
||||
result = runtime.call(session.run, following)
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.final_response == reply
|
||||
initial, continued = runtime.model.captured()
|
||||
assert input_text in _request_texts(initial)
|
||||
assert following in _request_texts(continued)
|
||||
assert initial["tools"] == continued["tools"]
|
||||
events = _read_events(runtime.logs)
|
||||
assert not _of_type(events, "command/run")
|
||||
assert not any(event["type"].startswith("feedback/") for event in events)
|
||||
assert len(_of_type(events, "turn/start")) == len(_of_type(events, "turn/end")) == 2
|
||||
|
||||
|
||||
_PERMISSION_FACTS = {"permission/preset", "sandbox/mode", "approval/policy"}
|
||||
_ASK_POLICY = "Approval prompts are enabled in this session"
|
||||
_NEVER_POLICY = "Approval prompts are disabled in this session"
|
||||
|
||||
|
||||
def _assert_request_permission(request: dict, sandbox: str, approval: str) -> None:
|
||||
contexts = [text for text in _request_texts(request) if text.startswith("Current runtime context.")]
|
||||
assert contexts, "model request has no runtime-context snapshot"
|
||||
assert f"Current DSH file policy: {sandbox}." in contexts[-1]
|
||||
assert (_ASK_POLICY if approval == "ask" else _NEVER_POLICY) in contexts[-1]
|
||||
|
||||
|
||||
def _permission_facts(events: list[dict]) -> list[dict]:
|
||||
return [{"type": event["type"], "data": event["data"]}
|
||||
for event in events if event["type"] in _PERMISSION_FACTS]
|
||||
|
||||
|
||||
def _permission_command(runtime: _Runtime, session: Session, preset: str) -> CommandExecution:
|
||||
result = runtime.call(session.run, f"/permission {preset}")
|
||||
assert isinstance(result, RunResult)
|
||||
return _native_result(result)
|
||||
|
||||
|
||||
def test_permission_changes_only_the_selected_session_and_retains_prompt_semantics(tmp_path: Path) -> None:
|
||||
completions = [_Completion(f"PERMISSION_REPLY_{index}") for index in range(9)]
|
||||
with _runtime(tmp_path, completions, initial_permission="workspace-write") as runtime:
|
||||
main = runtime.harness.start_session(_SESSION_ID)
|
||||
other = runtime.harness.start_session("permission-other")
|
||||
initial_config = dict(runtime.harness.config.env)
|
||||
for session, text in [(main, "Initial main policy; do not use tools."),
|
||||
(other, "Initial other policy; do not use tools.")]:
|
||||
assert isinstance(runtime.call(session.run, text), RunResult)
|
||||
|
||||
readonly = _permission_command(runtime, main, "read-only")
|
||||
invalid = _permission_command(runtime, main, "no-such-preset")
|
||||
repeated = _permission_command(runtime, main, "read-only")
|
||||
assert readonly.result.kind == repeated.result.kind == "success"
|
||||
assert invalid.result.kind == "error"
|
||||
assert "unknown preset" in invalid.result.text
|
||||
assert len(runtime.model.captured()) == 2, "idle permission commands woke the model"
|
||||
|
||||
assert isinstance(runtime.call(main.run, "Read-only policy after a plain query; do not use tools."), RunResult)
|
||||
full = _permission_command(runtime, main, "danger-full-access")
|
||||
assert full.result.kind == "success"
|
||||
assert len(runtime.model.captured()) == 3
|
||||
assert isinstance(runtime.call(main.run, "Full policy after a plain query; do not use tools."), RunResult)
|
||||
restored = _permission_command(runtime, main, "workspace-write")
|
||||
assert restored.result.kind == "success"
|
||||
assert isinstance(runtime.call(main.run, "Switched-back policy; do not use tools."), RunResult)
|
||||
|
||||
for session in [other, runtime.harness.start_session("permission-new")]:
|
||||
assert isinstance(runtime.call(session.run, "Report your own policy; do not use tools."), RunResult)
|
||||
runtime.call(runtime.harness.client.session_prompt, main.id, [{"type": "text", "text": "/permission danger-full-access"}])
|
||||
_wait_activity(runtime, main)
|
||||
assert isinstance(runtime.call(main.run, "/permission"), RunResult)
|
||||
assert runtime.harness.config.env == initial_config
|
||||
|
||||
requests = runtime.model.captured()
|
||||
assert len(requests) == 9
|
||||
for index in [0, 1, 4, 5, 6, 7, 8]:
|
||||
_assert_request_permission(requests[index], "workspace-write", "ask")
|
||||
_assert_request_permission(requests[2], "read-only", "ask")
|
||||
_assert_request_permission(requests[3], "danger-full-access", "never")
|
||||
for index in [2, 3, 4]:
|
||||
assert requests[index]["system"] == requests[0]["system"]
|
||||
assert requests[index]["tools"] == requests[0]["tools"]
|
||||
assert "/permission danger-full-access" in _request_texts(requests[7])
|
||||
assert "/permission" in _request_texts(requests[8])
|
||||
|
||||
events = _read_events(runtime.logs)
|
||||
notices = [event["data"]["content"] for event in _of_type(events, "user/message")
|
||||
if event["data"]["source"]["kind"] == "user-approval"]
|
||||
assert len(notices) == 2
|
||||
for content, request in zip(notices, requests[3:5]):
|
||||
assert all(text in _request_texts(request) for text in _text_blocks(content))
|
||||
executions = [readonly, invalid, repeated, full, restored]
|
||||
assert len(_of_type(events, "command/run")) == len(executions)
|
||||
for execution in executions:
|
||||
_command_pair(events, execution, "permission")
|
||||
for execution in [invalid, repeated]:
|
||||
start = next(event["seq"] for event in _of_type(events, "command/run")
|
||||
if event["data"]["commandId"] == execution.command_id)
|
||||
done = next(event["seq"] for event in _of_type(events, "command/done")
|
||||
if event["data"]["commandId"] == execution.command_id)
|
||||
assert not _permission_facts(events[start + 1:done])
|
||||
for session_id in [_SESSION_ID, "permission-other", "permission-new"]:
|
||||
state = {event["type"]: event["data"] for event in _permission_facts(_read_events(runtime.logs, session_id=session_id))}
|
||||
assert state["permission/preset"] == {"preset": "workspace-write"}
|
||||
assert state["sandbox/mode"]["mode"] == "workspace-write"
|
||||
assert state["approval/policy"] == {"policy": "ask"}
|
||||
|
||||
|
||||
def test_running_permission_change_keeps_the_native_extra_step_and_notifications(tmp_path: Path) -> None:
|
||||
first = _Completion("PERMISSION_FIRST_DONE", blocked=True)
|
||||
second = _Completion("PERMISSION_SWITCH_SEEN")
|
||||
with _runtime(tmp_path, [first, second], initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
observing = runtime.workers.submit(session.run, "Wait for the policy change, then finish without tools.")
|
||||
assert first.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
execution = _permission_command(runtime, session, "danger-full-access")
|
||||
assert execution.result.kind == "success"
|
||||
assert not first.release.is_set()
|
||||
assert not observing.done()
|
||||
assert len(runtime.model.captured()) == 1
|
||||
first.release.set()
|
||||
result = observing.result(timeout=_WAIT_SECONDS)
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.final_response == second.text
|
||||
requests = runtime.model.captured()
|
||||
assert len(requests) == 2
|
||||
_assert_request_permission(requests[0], "workspace-write", "ask")
|
||||
_assert_request_permission(requests[1], "danger-full-access", "never")
|
||||
assert requests[0]["system"] == requests[1]["system"]
|
||||
assert requests[0]["tools"] == requests[1]["tools"]
|
||||
assert runtime.harness.config.env["DSH_PERMISSION_MODE"] == "workspace-write"
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, execution, "permission")
|
||||
[notice] = [event["data"]["content"] for event in _of_type(events, "user/message")
|
||||
if event["data"]["source"]["kind"] == "user-approval"]
|
||||
for text in _text_blocks(notice):
|
||||
assert text not in _request_texts(requests[0])
|
||||
assert text in _request_texts(requests[1])
|
||||
assert [event["data"] for event in _of_type(events, "step/start")] == [
|
||||
{"turn": 1, "step": 1}, {"turn": 1, "step": 2},
|
||||
]
|
||||
assert len(_of_type(events, "turn/start")) == len(_of_type(events, "turn/end")) == 1
|
||||
|
||||
|
||||
def test_resumed_permission_survives_different_launch_defaults(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, [], initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
execution = _permission_command(runtime, session, "read-only")
|
||||
assert execution.result.kind == "success"
|
||||
assert runtime.model.captured() == []
|
||||
saved = _permission_facts(_read_events(runtime.logs))
|
||||
with _runtime(tmp_path, [_Completion("RESUMED_PERMISSION")], initial_permission="danger-full-access") as resumed:
|
||||
session = resumed.call(resumed.harness.resume_session, _SESSION_ID)
|
||||
assert isinstance(resumed.call(session.run, "Report the restored policy without using tools."), RunResult)
|
||||
[request] = resumed.model.captured()
|
||||
_assert_request_permission(request, "read-only", "ask")
|
||||
assert resumed.harness.config.env["DSH_PERMISSION_MODE"] == "danger-full-access"
|
||||
assert _permission_facts(_read_events(resumed.logs)) == saved
|
||||
@@ -27,13 +27,13 @@ for line in sys.stdin:
|
||||
message = json.loads(line)
|
||||
if message["method"] == "initialize":
|
||||
result = {"serverInfo": {"name": "audit-fixture"}}
|
||||
elif message["method"] == "session/prompt":
|
||||
elif message["method"] == "session/query":
|
||||
session_id = message["params"]["sessionId"]
|
||||
notify("session.event", {"sessionId": session_id, "event": {
|
||||
"type": "agent/inbox/spliced", "seq": 0, "time": 0,
|
||||
"data": {"target": "next-turn", "start": 0, "inserted": [{"id": "accepted"}]}}})
|
||||
notify("session.status", {"sessionId": session_id, "status": "running"})
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": message["id"], "result": {"messageId": "accepted"}}), flush=True)
|
||||
print(json.dumps({"jsonrpc": "2.0", "id": message["id"], "result": {"kind": "prompt", "messageId": "accepted"}}), flush=True)
|
||||
notify("session.event", {"sessionId": session_id, "event": {"type": "turn/start", "seq": 1, "time": 0, "data": {"turn": 0}}})
|
||||
notify("session.event", {"sessionId": session_id, "event": {"type": "assistant/message", "seq": 2, "time": 0,
|
||||
"data": {"message": {"role": "assistant", "content": [{"type": "text", "text": os.environ["MAIN_RESPONSE"]}]}}}})
|
||||
@@ -55,7 +55,7 @@ for line in sys.stdin:
|
||||
_launch_args=(sys.executable, str(script)), cwd=str(tmp_path),
|
||||
env={"MAIN_RESPONSE": expected["finalResponse"], "AUDIT_EVENTS": json.dumps(expected["auditEvents"])},
|
||||
) as harness:
|
||||
result = harness.run("Keep the original response.", session_id="sdk-audit")
|
||||
result = harness.run([{"type": "text", "text": "Keep the original response."}], session_id="sdk-audit")
|
||||
audit_notifications = [notification for notification in result.notifications
|
||||
if notification.method == "session.event"
|
||||
and notification.payload["event"]["type"] in ("plugin:translator/request", "plugin:translator/result")]
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Goal observation deadlines and runtime cleanup through the runnable example."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import runpy
|
||||
import sys
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deepseek_harness import DeepSeekHarness
|
||||
|
||||
|
||||
_EXAMPLE = Path(__file__).resolve().parents[1] / "examples/goal.py"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pending", ["command", "activity", "complete", "error", "interrupt"])
|
||||
def test_goal_example_bounds_the_whole_run_and_reaps_runtime(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], pending: str,
|
||||
) -> None:
|
||||
peer = tmp_path / "goal-peer.py"
|
||||
observed = tmp_path / "observed.jsonl"
|
||||
peer.write_text('''import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
pending, observed = sys.argv[1:]
|
||||
|
||||
def send(frame):
|
||||
print(json.dumps({"jsonrpc": "2.0", **frame}), flush=True)
|
||||
|
||||
for line in sys.stdin:
|
||||
frame = json.loads(line)
|
||||
with Path(observed).open("a", encoding="utf-8") as stream:
|
||||
stream.write(json.dumps(frame) + "\\n")
|
||||
method = frame["method"]
|
||||
if method == "initialize":
|
||||
send({"id": frame["id"], "result": {"serverInfo": {"name": "goal-example-peer"}}})
|
||||
elif method == "session/query":
|
||||
assert frame["params"]["contentBlocks"] == [{"type": "text", "text": "/goal Finish this task"}]
|
||||
if pending == "command":
|
||||
continue
|
||||
session = frame["params"]["sessionId"]
|
||||
for kind in ("command/run", "command/done"):
|
||||
send({"method": "session.event", "params": {"sessionId": session, "event": {
|
||||
"type": kind, "data": {"commandId": "goal-command"}}}})
|
||||
send({"id": frame["id"], "result": {"kind": "command", "execution": {
|
||||
"commandId": "goal-command", "result": {"kind": "success", "text": "Goal set"}}}})
|
||||
if pending in ("activity", "interrupt"):
|
||||
continue
|
||||
if pending == "complete":
|
||||
send({"method": "session.event", "params": {"sessionId": session, "event": {
|
||||
"type": "goal/change", "data": {"operation": "complete"}}}})
|
||||
send({"method": "session.event", "params": {"sessionId": session, "event": {
|
||||
"type": "assistant/message", "data": {"message": {
|
||||
"content": [{"type": "text", "text": "Closing reply"}]}}}}})
|
||||
send({"method": "session.query.completed", "params": {"sessionId": session,
|
||||
"commandId": "goal-command", "finalResponse": "Closing reply" if pending == "complete" else "Goal stopped",
|
||||
"finishReason": "completed" if pending == "complete" else "error"}})
|
||||
elif method == "shutdown":
|
||||
send({"id": frame["id"], "result": {}})
|
||||
break
|
||||
else:
|
||||
raise AssertionError(method)
|
||||
''', encoding="utf-8")
|
||||
example = runpy.run_path(str(_EXAMPLE))
|
||||
harnesses: list[DeepSeekHarness] = []
|
||||
|
||||
def configured_harness(**options: object) -> DeepSeekHarness:
|
||||
harness = DeepSeekHarness(
|
||||
**options, _launch_args=(sys.executable, str(peer), pending, str(observed)),
|
||||
initialize_timeout_seconds=10, shutdown_timeout_seconds=10,
|
||||
)
|
||||
harnesses.append(harness)
|
||||
return harness
|
||||
|
||||
monkeypatch.setitem(example["main"].__globals__, "DeepSeekHarness", configured_harness)
|
||||
if pending == "interrupt":
|
||||
entered = threading.Event()
|
||||
display = example["display"]
|
||||
def observe(notice):
|
||||
if notice.method == "session.event" and notice.payload["event"]["type"] == "command/done":
|
||||
entered.set()
|
||||
return display(notice)
|
||||
class InterruptedExecutor(ThreadPoolExecutor):
|
||||
def submit(self, *args, **kwargs):
|
||||
future = super().submit(*args, **kwargs)
|
||||
def interrupt(timeout=None):
|
||||
assert entered.wait(10), "command must be observed before interrupting its wait"
|
||||
raise KeyboardInterrupt
|
||||
monkeypatch.setattr(future, "result", interrupt)
|
||||
return future
|
||||
monkeypatch.setitem(example["main"].__globals__, "display", observe)
|
||||
monkeypatch.setitem(example["main"].__globals__, "ThreadPoolExecutor", InterruptedExecutor)
|
||||
timeout = "1" if pending in {"command", "activity"} else "10"
|
||||
monkeypatch.setattr(sys, "argv", [str(_EXAMPLE), "--dsh-home", str(tmp_path / "home"),
|
||||
"--workspace", str(tmp_path), "--timeout-seconds", timeout, "Finish this task"])
|
||||
with ThreadPoolExecutor(max_workers=1) as workers:
|
||||
execution = workers.submit(example["main"])
|
||||
try:
|
||||
if pending in {"command", "activity"}:
|
||||
with pytest.raises(TimeoutError, match="Goal observation timed out"):
|
||||
execution.result(timeout=30)
|
||||
elif pending == "interrupt":
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
execution.result(timeout=30)
|
||||
elif pending == "error":
|
||||
with pytest.raises(RuntimeError, match="Goal stopped"):
|
||||
execution.result(timeout=30)
|
||||
else:
|
||||
execution.result(timeout=30)
|
||||
finally:
|
||||
for harness in harnesses:
|
||||
harness.close()
|
||||
[harness] = harnesses
|
||||
assert harness.client._proc is None
|
||||
frames = [json.loads(line) for line in observed.read_text(encoding="utf-8").splitlines()]
|
||||
assert [frame["method"] for frame in frames] == ["initialize", "session/query", "shutdown"]
|
||||
if pending == "complete":
|
||||
output = capsys.readouterr().out
|
||||
assert "Goal: complete" in output
|
||||
assert "Closing reply" in output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf", "-inf"])
|
||||
def test_goal_example_rejects_unbounded_or_nonpositive_timeout(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], timeout: str,
|
||||
) -> None:
|
||||
example = runpy.run_path(str(_EXAMPLE))
|
||||
monkeypatch.setattr(sys, "argv", [str(_EXAMPLE), "--dsh-home", str(tmp_path / "home"),
|
||||
f"--timeout-seconds={timeout}", "Finish this task"])
|
||||
with pytest.raises(SystemExit) as error:
|
||||
example["main"]()
|
||||
assert error.value.code == 2
|
||||
assert "timeout-seconds must be finite and positive" in capsys.readouterr().err
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Goal run results preserve empty assistant text through the shipped SDK profile."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from test_commands_runtime import _Completion, _runtime, _SESSION_ID
|
||||
from test_goal_runtime import _completion_steps
|
||||
|
||||
|
||||
def test_completed_goal_without_assistant_text_returns_empty_result(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, _completion_steps(_Completion("")),
|
||||
initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
result = runtime.call(session.run, "/goal Finish the task.")
|
||||
assert result.final_response == "" and result.finish_reason == "completed"
|
||||
assert any(event["type"] == "goal/change" and event["data"]["operation"] == "complete"
|
||||
for event in result.events)
|
||||
cleared = runtime.call(session.run, "/goal clear")
|
||||
assert cleared.final_response == "Goal cleared." and cleared.finish_reason == "completed"
|
||||
assert len(runtime.model.captured()) == 3
|
||||
|
||||
|
||||
def test_blocked_goal_without_assistant_text_returns_empty_result(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, [_Completion("")], max_goal_rounds=1) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
result = runtime.call(session.run, "/goal Finish the task.")
|
||||
assert result.final_response == "" and result.finish_reason == "error"
|
||||
assert any(event["type"] == "goal/change" and event["data"]["goal"]["phase"] == "blocked"
|
||||
for event in result.events)
|
||||
assert len(runtime.model.captured()) == 1
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Native goal rounds and state changes through the Python SDK profile."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
|
||||
from deepseek_harness.models import CommandExecution
|
||||
from deepseek_harness import RunResult, Session
|
||||
from deepseek_harness.client import NotificationSubscription
|
||||
from test_commands_runtime import (
|
||||
_Completion, _Runtime, _SESSION_ID, _WAIT_SECONDS, _WaitResult, _assert_request_permission,
|
||||
_command_pair, _of_type, _permission_facts, _read_events,
|
||||
_request_texts, _runtime, _text_blocks, _wait_activity,
|
||||
)
|
||||
|
||||
|
||||
def _command(runtime: _Runtime, session: Session, text: str, *, kind: str = "success") -> CommandExecution:
|
||||
result = runtime.call(runtime.harness.client._query, session.id, text)
|
||||
assert isinstance(result, CommandExecution)
|
||||
assert result.result.kind == kind, result.result.text
|
||||
return result
|
||||
|
||||
|
||||
def _until(subscription: NotificationSubscription, event_type: str, operation: str | None = None) -> dict:
|
||||
deadline = time.monotonic() + _WAIT_SECONDS
|
||||
while True:
|
||||
notification = subscription.next(timeout_seconds=max(0, deadline - time.monotonic()))
|
||||
if notification.method != "session.event":
|
||||
continue
|
||||
event = notification.payload["event"]
|
||||
if event["type"] == event_type and (operation is None or event["data"].get("operation") == operation):
|
||||
return event["data"]
|
||||
|
||||
|
||||
def _complete_goal(request: dict) -> tuple[str, dict]:
|
||||
results = [block for message in request["messages"] for block in message["content"]
|
||||
if block["type"] == "tool_result"]
|
||||
current = json.loads("\n".join(_text_blocks(results[-1]["content"])))
|
||||
goal = current["goal"]
|
||||
assert goal["phase"] == "active" and current["activation"] == "armed"
|
||||
return "update_goal", {"goal_id": goal["id"], "revision": goal["revision"], "action": "complete"}
|
||||
|
||||
|
||||
def _completion_steps(final: _Completion | None = None) -> list[_Completion]:
|
||||
return [_Completion("", tool_call=("get_goal", {})), _Completion("", tool_call=_complete_goal),
|
||||
final if final is not None else _Completion("GOAL_DONE")]
|
||||
|
||||
|
||||
def test_goal_reads_and_invalid_controls_do_not_start_model_work(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, []) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
executions = [_command(runtime, session, "/目标")]
|
||||
for line in ["/goal edit", "/goal edit Missing goal", "/goal pause", "/goal resume"]:
|
||||
executions.append(_command(runtime, session, line, kind="error"))
|
||||
executions.append(_command(runtime, session, "/goal clear"))
|
||||
assert runtime.model.captured() == []
|
||||
events = _read_events(runtime.logs)
|
||||
assert not _of_type(events, "goal/change") and not _of_type(events, "turn/start")
|
||||
for execution in executions:
|
||||
_command_pair(events, execution, "goal")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("clear_before_reply", [False, True])
|
||||
def test_goal_continues_rounds_and_finishes_wrapup_after_completion_event(
|
||||
tmp_path: Path, clear_before_reply: bool,
|
||||
) -> None:
|
||||
final = _Completion("GOAL_DONE", blocked=True)
|
||||
with _runtime(tmp_path, [_Completion("PROGRESS"), *_completion_steps(final)],
|
||||
initial_permission="workspace-write", max_goal_rounds=3) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
with runtime.harness.client.subscribe_session_notifications(session.id) as events:
|
||||
running = runtime.workers.submit(session.run, "/目标 Verify the small change.")
|
||||
completed = _until(events, "goal/change", "complete")
|
||||
assert completed["goal"]["phase"] == "complete" and completed["roundsStarted"] == 2
|
||||
assert final.entered.wait(_WAIT_SECONDS)
|
||||
assert not running.done()
|
||||
if clear_before_reply:
|
||||
cleared = runtime.call(session.run, "/goal clear")
|
||||
assert cleared.final_response == "Goal cleared." and cleared.finish_reason == "completed"
|
||||
assert not running.done()
|
||||
final.release.set()
|
||||
result = running.result(timeout=_WAIT_SECONDS)
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.final_response == "GOAL_DONE" and result.finish_reason == "completed"
|
||||
operations = ["create", "complete", *(["clear"] if clear_before_reply else [])]
|
||||
assert [event["data"]["operation"] for event in result.events if event["type"] == "goal/change"] == operations
|
||||
_command_pair(_read_events(runtime.logs), result, "goal")
|
||||
|
||||
|
||||
def test_pause_aborts_then_edit_and_resume_use_revised_goal_without_changing_other_sessions(tmp_path: Path) -> None:
|
||||
first = _Completion("CANCELLED_RESPONSE", blocked=True, allow_disconnect=True)
|
||||
with _runtime(tmp_path, [first, *_completion_steps(), _Completion("RAW_GOAL_TEXT")]) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
with runtime.harness.client.subscribe_session_notifications(session.id) as events:
|
||||
_command(runtime, session, "/goal Original objective")
|
||||
assert first.entered.wait(_WAIT_SECONDS)
|
||||
_command(runtime, session, "/goal Duplicate objective", kind="error")
|
||||
_command(runtime, session, "/goal edit", kind="error")
|
||||
_command(runtime, session, "/goal resume", kind="error")
|
||||
_command(runtime, session, "/goal PAUSE")
|
||||
assert _until(events, "turn/end")["reason"]["kind"] == "aborted"
|
||||
assert not first.release.is_set()
|
||||
_command(runtime, session, "/goal edit Revised objective")
|
||||
assert len(runtime.model.captured()) == 1
|
||||
other = runtime.harness.start_session("goal-other")
|
||||
assert "No goal is currently set" in _command(runtime, other, "/goal").result.text
|
||||
_command(runtime, session, "/goal resume")
|
||||
_until(events, "goal/change", "complete")
|
||||
_wait_activity(runtime, session)
|
||||
assert 'Objective: "Revised objective"' in "\n".join(_request_texts(runtime.model.captured()[1]))
|
||||
_command(runtime, session, "/goal clear")
|
||||
_command(runtime, session, "/goal clear")
|
||||
runtime.call(runtime.harness.client.session_prompt, session.id, [{"type": "text", "text": "/goal Plain text objective"}])
|
||||
_wait_activity(runtime, session)
|
||||
assert "/goal Plain text objective" in _request_texts(runtime.model.captured()[-1])
|
||||
changes = _of_type(_read_events(runtime.logs), "goal/change")
|
||||
assert [e["data"]["operation"] for e in changes] == ["create", "pause", "edit", "resume", "complete", "clear"]
|
||||
assert not _of_type(_read_events(runtime.logs, session_id="goal-other"), "goal/change")
|
||||
|
||||
|
||||
def test_clear_keeps_current_request_and_stops_further_goal_rounds(tmp_path: Path) -> None:
|
||||
first = _Completion("CURRENT_ROUND_FINISHED", blocked=True)
|
||||
with _runtime(tmp_path, [first]) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
with runtime.harness.client.subscribe_session_notifications(session.id) as events:
|
||||
_command(runtime, session, "/goal Work until cleared")
|
||||
assert first.entered.wait(_WAIT_SECONDS)
|
||||
_command(runtime, session, "/goal clear")
|
||||
first.release.set()
|
||||
_wait_activity(runtime, session)
|
||||
assert _until(events, "turn/end")["reason"]["kind"] == "completed"
|
||||
assert len(runtime.model.captured()) == 1
|
||||
events = _read_events(runtime.logs)
|
||||
assert len(_of_type(events, "assistant/message")) == 1
|
||||
assert [e["data"]["operation"] for e in _of_type(events, "goal/change")] == ["create", "clear"]
|
||||
|
||||
|
||||
def test_round_limit_blocks_and_rejects_resume_without_another_request(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, [_Completion("ONE"), _Completion("TWO")], max_goal_rounds=2) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
with runtime.harness.client.subscribe_session_notifications(session.id) as events:
|
||||
_command(runtime, session, "/goal An unfinished objective")
|
||||
blocked = _until(events, "goal/change", "block")
|
||||
assert blocked["goal"]["blockedReason"]["code"] == "round-limit"
|
||||
assert blocked["roundsStarted"] == 2
|
||||
_command(runtime, session, "/goal resume", kind="error")
|
||||
_wait_activity(runtime, session)
|
||||
assert len(runtime.model.captured()) == 2
|
||||
|
||||
|
||||
def test_restored_active_goal_waits_for_explicit_resume(tmp_path: Path) -> None:
|
||||
first = _Completion("INTERRUPTED", blocked=True, allow_disconnect=True)
|
||||
with _runtime(tmp_path, [first], max_goal_rounds=3) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
runtime.call(runtime.harness.client.session_prompt, session.id, [{"type": "text", "text": "Start an ordinary task"}])
|
||||
assert first.entered.wait(_WAIT_SECONDS)
|
||||
_command(runtime, session, "/goal Persisted objective")
|
||||
runtime.harness.close()
|
||||
with _runtime(tmp_path, _completion_steps()) as runtime:
|
||||
session = runtime.call(runtime.harness.resume_session, _SESSION_ID)
|
||||
shown = _command(runtime, session, "/goal")
|
||||
assert "Status: active" in shown.result.text and "Activation: disarmed" in shown.result.text
|
||||
assert runtime.model.captured() == []
|
||||
with runtime.harness.client.subscribe_session_notifications(session.id) as events:
|
||||
_command(runtime, session, "/goal resume")
|
||||
_until(events, "goal/change", "complete")
|
||||
_wait_activity(runtime, session)
|
||||
assert 'Objective: "Persisted objective"\nRound: 1/3' in "\n".join(_request_texts(runtime.model.captured()[0]))
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Run the plan example against the SDK profile with controlled model replies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import runpy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deepseek_harness import DeepSeekHarness, Notification
|
||||
import test_commands_runtime as runtime_helpers
|
||||
from test_commands_runtime import _Completion, _ModelServer, _dsh_bin, _of_type, _read_events
|
||||
|
||||
|
||||
_EXAMPLE = Path(__file__).resolve().parents[1] / "examples/plan.py"
|
||||
_PLAN = "# Small change\n\n- Inspect the code.\n- Implement after approval."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", ["max-tokens", "dismiss"])
|
||||
def test_plan_example_reports_truncation_and_accepts_explicit_dismissal(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], outcome: str,
|
||||
) -> None:
|
||||
example = runpy.run_path(str(_EXAMPLE))
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
logs = tmp_path / "sessions"
|
||||
patch = tmp_path / "persistence.yml"
|
||||
patch.write_text(
|
||||
"- id: session-persistence-jsonl\n config:\n root: " + json.dumps(str(logs))
|
||||
+ "\n compression: none\n", encoding="utf-8",
|
||||
)
|
||||
completion = (_Completion("PARTIAL_PLAN") if outcome == "max-tokens"
|
||||
else _Completion("", tool_call=("exit_plan_mode", {"plan": _PLAN})))
|
||||
if outcome == "max-tokens":
|
||||
response = runtime_helpers._messages_response
|
||||
|
||||
def truncated_response(*args: object) -> bytes:
|
||||
return response(*args).replace(b'"stop_reason": "end_turn"', b'"stop_reason": "max_tokens"')
|
||||
|
||||
monkeypatch.setattr(runtime_helpers, "_messages_response", truncated_response)
|
||||
terminal_input = tmp_path / "stdin.txt"
|
||||
terminal_input.write_text("d\n", encoding="utf-8")
|
||||
monkeypatch.setenv("DSH_TELEMETRY_DISABLED", "1")
|
||||
monkeypatch.setenv("DSH_TOOLS_MODE", "native")
|
||||
monkeypatch.setattr(sys, "argv", [str(_EXAMPLE), "--dsh-home", str(tmp_path / "home"),
|
||||
"--workspace", str(workspace), "Plan a small change."])
|
||||
with terminal_input.open(encoding="utf-8") as terminal, _ModelServer([completion]) as model:
|
||||
monkeypatch.setattr(sys, "stdin", terminal)
|
||||
def configured_harness(**options: object) -> DeepSeekHarness:
|
||||
return DeepSeekHarness(
|
||||
**options, dsh_bin=_dsh_bin(), patches=(str(patch),),
|
||||
base_url=model.url, api_key="keyless-plan-example",
|
||||
initialize_timeout_seconds=30, request_timeout_seconds=20, shutdown_timeout_seconds=20,
|
||||
)
|
||||
|
||||
monkeypatch.setitem(example["main"].__globals__, "DeepSeekHarness", configured_harness)
|
||||
if outcome == "max-tokens":
|
||||
with pytest.raises(RuntimeError, match="max-tokens"):
|
||||
example["main"]()
|
||||
else:
|
||||
example["main"]()
|
||||
assert len(model.captured()) == 1
|
||||
[log] = logs.rglob("session*.jsonl")
|
||||
session_id = json.loads(log.read_text(encoding="utf-8").splitlines()[0])["id"]
|
||||
events = _read_events(logs, session_id=session_id)
|
||||
assert [event["data"]["reason"]["kind"] for event in _of_type(events, "turn/end")] == [
|
||||
"max-tokens" if outcome == "max-tokens" else "completed",
|
||||
]
|
||||
assert [event["data"] for event in _of_type(events, "plan/mode")] == [{"active": True}]
|
||||
output = capsys.readouterr().out
|
||||
assert ("[a] Approve" in output) == (outcome == "dismiss")
|
||||
assert ("PARTIAL_PLAN" if outcome == "max-tokens" else _PLAN) in output
|
||||
|
||||
|
||||
def test_plan_display_rejects_root_abort_without_rejecting_descendant_abort() -> None:
|
||||
display = runpy.run_path(str(_EXAMPLE))["display"]
|
||||
event = {"type": "turn/end", "data": {"turn": 1, "reason": {"kind": "aborted"}}}
|
||||
display(Notification(method="session.event", payload={"sessionId": "child", "event": event}), "root")
|
||||
with pytest.raises(RuntimeError, match="aborted"):
|
||||
display(Notification(method="session.event", payload={"sessionId": "root", "event": event}), "root")
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Plan example input uses real pipes and releases readers before process exit."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import runpy
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
_EXAMPLE = Path(__file__).resolve().parents[1] / "examples/plan.py"
|
||||
_WAIT_SECONDS = 15
|
||||
_PEER = '''import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
|
||||
root = Path(sys.argv[1])
|
||||
lock = threading.Lock()
|
||||
stop = threading.Event()
|
||||
session_id = None
|
||||
serial = 0
|
||||
|
||||
def send(frame):
|
||||
with lock:
|
||||
print(json.dumps({"jsonrpc": "2.0", **frame}), flush=True)
|
||||
|
||||
def ask(session_id):
|
||||
global serial
|
||||
serial += 1
|
||||
send({"id": "ask-" + str(serial), "method": "user-question/ask", "params": {
|
||||
"requestId": "review-" + str(serial), "sessionId": session_id,
|
||||
"questions": [{"id": "plan-review", "question": "Review the plan?", "detail": "# Small change",
|
||||
"options": [{"label": "Approve"}, {"label": "Keep planning"}],
|
||||
"intent": {"kind": "plan-review", "approve": "Approve"}}],
|
||||
}})
|
||||
|
||||
def finish():
|
||||
# These cases isolate input handling from native plan turn outcomes.
|
||||
send({"method": "session.query.completed", "params": {
|
||||
"sessionId": session_id, "commandId": "plan-command",
|
||||
"finalResponse": "", "finishReason": "completed",
|
||||
}})
|
||||
|
||||
def cancel():
|
||||
while not stop.wait(0.01):
|
||||
if (root / "cancel").exists():
|
||||
send({"method": "user-question.cancelled", "params": {"requestId": "review-" + str(serial)}})
|
||||
finish()
|
||||
return
|
||||
|
||||
watcher = threading.Thread(target=cancel)
|
||||
watcher.start()
|
||||
try:
|
||||
for line in sys.stdin:
|
||||
frame = json.loads(line)
|
||||
method = frame.get("method")
|
||||
if method is None:
|
||||
with (root / "answers.jsonl").open("a") as output:
|
||||
output.write(json.dumps(frame) + "\\n")
|
||||
result = frame.get("result", {})
|
||||
answers = result.get("answers", [])
|
||||
if answers and answers[0].get("selected") == ["Keep planning"]:
|
||||
ask(session_id)
|
||||
else:
|
||||
finish()
|
||||
elif method == "initialize":
|
||||
send({"id": frame["id"], "result": {"serverInfo": {"name": "plan-input-peer"}}})
|
||||
elif method == "session/query":
|
||||
session_id = frame["params"]["sessionId"]
|
||||
for event_type, data in [
|
||||
("command/run", {"commandId": "plan-command", "name": "plan"}),
|
||||
("command/done", {"commandId": "plan-command", "kind": "success"}),
|
||||
]:
|
||||
send({"method": "session.event", "params": {
|
||||
"sessionId": session_id, "event": {"type": event_type, "data": data},
|
||||
}})
|
||||
send({"id": frame["id"], "result": {"kind": "command", "execution": {
|
||||
"commandId": "plan-command", "result": {"kind": "success", "text": "Plan mode entered"}}}})
|
||||
ask(session_id)
|
||||
elif method == "shutdown":
|
||||
(root / "shutdown").write_text("done")
|
||||
send({"id": frame["id"], "result": {}})
|
||||
break
|
||||
else:
|
||||
raise AssertionError(method)
|
||||
finally:
|
||||
stop.set()
|
||||
watcher.join()
|
||||
'''
|
||||
_WRAPPER = '''import json
|
||||
from pathlib import Path
|
||||
import runpy
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from deepseek_harness import DeepSeekHarness
|
||||
|
||||
if sys.platform != "win32":
|
||||
import resource
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
|
||||
example_path, peer, root_argument, stdin_encoding = sys.argv[1:]
|
||||
if stdin_encoding:
|
||||
sys.stdin.reconfigure(encoding=stdin_encoding)
|
||||
root = Path(root_argument)
|
||||
example = runpy.run_path(example_path)
|
||||
readers = []
|
||||
popen = subprocess.Popen
|
||||
|
||||
def record_reader(*args, **kwargs):
|
||||
process = popen(*args, **kwargs)
|
||||
if args and len(args[0]) > 1 and args[0][1] == "-c":
|
||||
readers.append(process)
|
||||
return process
|
||||
|
||||
subprocess.Popen = record_reader
|
||||
|
||||
def configured_harness(**options):
|
||||
answer = options.pop("on_user_question")
|
||||
def record_answer(request, cancelled):
|
||||
result = answer(request, cancelled)
|
||||
with (root / "callbacks.jsonl").open("a") as output:
|
||||
output.write(json.dumps(result) + "\\n")
|
||||
return result
|
||||
return DeepSeekHarness(
|
||||
**options, on_user_question=record_answer, _launch_args=(sys.executable, peer, str(root)),
|
||||
initialize_timeout_seconds=5, shutdown_timeout_seconds=1,
|
||||
)
|
||||
|
||||
example["main"].__globals__["DeepSeekHarness"] = configured_harness
|
||||
sys.argv = [example_path, "--dsh-home", str(root / "home"), "--workspace", str(root), "Plan a change."]
|
||||
try:
|
||||
example["main"]()
|
||||
except KeyboardInterrupt:
|
||||
raise SystemExit(130)
|
||||
finally:
|
||||
(root / "cleanup.json").write_text(json.dumps({
|
||||
"readers": [{"pid": reader.pid, "returncode": reader.poll()} for reader in readers],
|
||||
"callbacks": [thread.name for thread in threading.enumerate() if thread.name == "dsh-user-question"],
|
||||
}))
|
||||
'''
|
||||
|
||||
|
||||
def _wait_prompt(process: subprocess.Popen[bytes], output: Path, marker: str) -> None:
|
||||
deadline = time.monotonic() + _WAIT_SECONDS
|
||||
while marker not in output.read_text(encoding="utf-8"):
|
||||
assert process.poll() is None, output.read_text(encoding="utf-8")
|
||||
if time.monotonic() >= deadline:
|
||||
raise AssertionError(f"plan input did not reach {marker!r}")
|
||||
time.sleep(0.01)
|
||||
|
||||
|
||||
def _run(
|
||||
tmp_path: Path, data: bytes, *, stage: str | None = None, action: str | None = None, terminal: bool = False,
|
||||
stdin_encoding: str | None = None, python_args: tuple[str, ...] = (), env: dict[str, str] | None = None,
|
||||
) -> list[dict]:
|
||||
peer = tmp_path / "peer.py"
|
||||
peer.write_text(_PEER, encoding="utf-8")
|
||||
wrapper = tmp_path / "wrapper.py"
|
||||
wrapper.write_text(_WRAPPER, encoding="utf-8")
|
||||
output = tmp_path / "stdout.txt"
|
||||
errors = tmp_path / "stderr.txt"
|
||||
if env is None:
|
||||
env = {**os.environ, "PYTHONIOENCODING": "utf-8"}
|
||||
with ExitStack() as stack, output.open("wb") as stdout, errors.open("wb") as stderr:
|
||||
if terminal:
|
||||
master, slave = os.openpty()
|
||||
writing = stack.enter_context(os.fdopen(master, "wb", buffering=0))
|
||||
reading = stack.enter_context(os.fdopen(slave, "rb", buffering=0))
|
||||
child_input = reading
|
||||
else:
|
||||
child_input = subprocess.PIPE
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, *python_args, str(wrapper), str(_EXAMPLE), str(peer), str(tmp_path), stdin_encoding or ""],
|
||||
stdin=child_input, stdout=stdout, stderr=stderr, env=env, start_new_session=os.name != "nt",
|
||||
)
|
||||
if not terminal:
|
||||
assert process.stdin is not None
|
||||
writing = stack.enter_context(process.stdin)
|
||||
try:
|
||||
if data:
|
||||
writing.write(data)
|
||||
writing.flush()
|
||||
if action is None and not terminal:
|
||||
writing.close()
|
||||
elif action is not None:
|
||||
marker = "Feedback: " if stage == "feedback" else "[d] Dismiss: "
|
||||
_wait_prompt(process, output, marker)
|
||||
if action == "interrupt":
|
||||
process.send_signal(signal.SIGINT)
|
||||
else:
|
||||
(tmp_path / "cancel").write_text("cancel")
|
||||
expected = 130 if action == "interrupt" else 0
|
||||
assert process.wait(timeout=_WAIT_SECONDS) == expected, errors.read_text(encoding="utf-8")
|
||||
finally:
|
||||
if os.name != "nt":
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError as error:
|
||||
# Successful example teardown has already removed the process group.
|
||||
pass
|
||||
elif process.poll() is None:
|
||||
taskkill = Path(os.environ["SystemRoot"]) / "System32" / "taskkill.exe"
|
||||
subprocess.run([str(taskkill), "/PID", str(process.pid), "/T", "/F"],
|
||||
stdin=subprocess.DEVNULL, capture_output=True, check=False, timeout=_WAIT_SECONDS)
|
||||
process.wait(timeout=_WAIT_SECONDS)
|
||||
assert "_enter_buffered_busy" not in errors.read_text(encoding="utf-8")
|
||||
assert (tmp_path / "shutdown").read_text() == "done"
|
||||
cleanup = json.loads((tmp_path / "cleanup.json").read_text())
|
||||
assert cleanup["readers"] and all(reader["returncode"] is not None for reader in cleanup["readers"])
|
||||
assert cleanup["callbacks"] == []
|
||||
answers = tmp_path / "answers.jsonl"
|
||||
if answers.is_file():
|
||||
responses = [json.loads(line) for line in answers.read_text().splitlines()]
|
||||
assert not [response for response in responses if "error" in response], responses
|
||||
return [json.loads(line) for line in (tmp_path / "callbacks.jsonl").read_text().splitlines()]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data,expected", [
|
||||
(b"a", [{"answers": [{"id": "plan-review", "selected": ["Approve"]}]}]),
|
||||
(b"d\n", [{"cancelled": True}]),
|
||||
(b"", [{"cancelled": True}]),
|
||||
("invalid\nk\n继续检查\r\nd\n".encode(), [
|
||||
{"answers": [{"id": "plan-review", "selected": ["Keep planning"], "custom": "继续检查"}]},
|
||||
{"cancelled": True},
|
||||
]),
|
||||
])
|
||||
def test_plan_input_preserves_piped_lines_and_eof(tmp_path: Path, data: bytes, expected: list[dict]) -> None:
|
||||
assert _run(tmp_path, data) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding,feedback", [
|
||||
("utf-8", "réviser Ċ encore"), ("latin-1", "réviser encore"),
|
||||
("utf-16-le", "réviser Ċ encore"), ("utf-16-be", "réviser Ċ encore"),
|
||||
("utf-32-le", "réviser Ċ encore"), ("utf-32-be", "réviser Ċ encore"),
|
||||
])
|
||||
def test_plan_input_preserves_reconfigured_stdin_feedback(tmp_path: Path, encoding: str, feedback: str) -> None:
|
||||
# U+010A contains byte 0x0a in UTF-16/32 without being a line ending.
|
||||
assert _run(tmp_path, f"k\r\n{feedback}\r\nd\r\n".encode(encoding), stdin_encoding=encoding) == [
|
||||
{"answers": [{"id": "plan-review", "selected": ["Keep planning"], "custom": feedback}]},
|
||||
{"cancelled": True},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["utf-8", "latin-1", "utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be"])
|
||||
@pytest.mark.parametrize("text", ["a", ""])
|
||||
def test_plan_input_preserves_reconfigured_stdin_at_eof(tmp_path: Path, encoding: str, text: str) -> None:
|
||||
assert _run(tmp_path, text.encode(encoding), stdin_encoding=encoding) == [
|
||||
{"answers": [{"id": "plan-review", "selected": ["Approve"]}]} if text else {"cancelled": True},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["utf-8", "utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be"])
|
||||
def test_plan_input_rejects_truncated_character_at_eof_and_reaps_reader(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, encoding: str,
|
||||
) -> None:
|
||||
owner_type = runpy.run_path(str(_EXAMPLE))["_PlanReviews"]
|
||||
encoded_input = tmp_path / "stdin.bin"
|
||||
encoded_input.write_bytes("é".encode(encoding)[:-1])
|
||||
readers = []
|
||||
popen = subprocess.Popen
|
||||
|
||||
def record_reader(*args, **kwargs):
|
||||
reader = popen(*args, **kwargs)
|
||||
readers.append(reader)
|
||||
return reader
|
||||
|
||||
with encoded_input.open(encoding=encoding) as terminal, monkeypatch.context() as patch:
|
||||
patch.setattr(sys, "stdin", terminal)
|
||||
patch.setattr(subprocess, "Popen", record_reader)
|
||||
with ThreadPoolExecutor(max_workers=1) as workers, owner_type() as owner:
|
||||
reading = workers.submit(owner._read_line, "Review: ", threading.Event())
|
||||
with pytest.raises(RuntimeError, match="UnicodeDecodeError"):
|
||||
reading.result(timeout=_WAIT_SECONDS)
|
||||
assert len(readers) == 1 and readers[0].poll() == 1
|
||||
assert owner._readers == []
|
||||
|
||||
|
||||
def test_plan_input_print_failure_reaps_reader_before_owner_exit(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
example = runpy.run_path(str(_EXAMPLE))
|
||||
owner_type = example["_PlanReviews"]
|
||||
readers = []
|
||||
popen = subprocess.Popen
|
||||
|
||||
def record_reader(*args, **kwargs):
|
||||
reader = popen(*args, **kwargs)
|
||||
readers.append(reader)
|
||||
return reader
|
||||
|
||||
def fail_print(*_args, **_kwargs):
|
||||
raise BrokenPipeError("terminal output closed")
|
||||
|
||||
with ExitStack() as stack, monkeypatch.context() as patch:
|
||||
read_fd, write_fd = os.pipe()
|
||||
terminal = stack.enter_context(os.fdopen(read_fd, "r", encoding="utf-8"))
|
||||
stack.enter_context(os.fdopen(write_fd, "w", encoding="utf-8"))
|
||||
patch.setattr(sys, "stdin", terminal)
|
||||
patch.setattr(subprocess, "Popen", record_reader)
|
||||
patch.setitem(owner_type._read_line.__globals__, "print", fail_print)
|
||||
with owner_type() as owner:
|
||||
with pytest.raises(BrokenPipeError, match="terminal output closed"):
|
||||
owner._read_line("Review: ", threading.Event())
|
||||
assert len(readers) == 1 and readers[0].poll() is not None
|
||||
assert owner._readers == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["choice", "feedback"])
|
||||
@pytest.mark.parametrize("encoding", ["utf-8", "utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be"])
|
||||
def test_plan_input_cancellation_reaps_open_pipe_reader(tmp_path: Path, stage: str, encoding: str) -> None:
|
||||
prefix = "k\npartial feedback" if stage == "feedback" else "partial choice"
|
||||
data = prefix.encode(encoding) + "é".encode(encoding)[:-1]
|
||||
assert _run(tmp_path, data, stage=stage, action="protocol", stdin_encoding=encoding) == [{"cancelled": True}]
|
||||
assert not (tmp_path / "answers.jsonl").exists()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""POSIX terminal input and SIGINT cleanup for the plan example."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from test_plan_input import _run
|
||||
import test_plan_input
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX PTYs and SIGINT delivery")
|
||||
|
||||
|
||||
def test_plan_input_preserves_terminal_feedback(tmp_path: Path) -> None:
|
||||
assert _run(tmp_path, "k\n继续检查\nd\n".encode(), terminal=True) == [
|
||||
{"answers": [{"id": "plan-review", "selected": ["Keep planning"], "custom": "继续检查"}]},
|
||||
{"cancelled": True},
|
||||
]
|
||||
|
||||
|
||||
def test_plan_input_preserves_reconfigured_terminal_feedback(tmp_path: Path) -> None:
|
||||
assert _run(tmp_path, "k\nréviser\nd\n".encode("latin-1"), terminal=True, stdin_encoding="latin-1") == [
|
||||
{"answers": [{"id": "plan-review", "selected": ["Keep planning"], "custom": "réviser"}]},
|
||||
{"cancelled": True},
|
||||
]
|
||||
|
||||
|
||||
def test_plan_input_preserves_explicit_utf8_mode_with_ascii_child(tmp_path: Path) -> None:
|
||||
env = {**os.environ, "LC_ALL": "C", "PYTHONUTF8": "0", "PYTHONCOERCECLOCALE": "0"}
|
||||
env.pop("PYTHONIOENCODING", None)
|
||||
assert _run(tmp_path, "k\n继续检查\nd\n".encode("utf-8"), python_args=("-X", "utf8"), env=env) == [
|
||||
{"answers": [{"id": "plan-review", "selected": ["Keep planning"], "custom": "继续检查"}]},
|
||||
{"cancelled": True},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["choice", "feedback"])
|
||||
def test_plan_input_interrupt_reaps_open_pipe_reader(tmp_path: Path, stage: str) -> None:
|
||||
data = b"k\npartial feedback" if stage == "feedback" else b"partial choice"
|
||||
assert _run(tmp_path, data, stage=stage, action="interrupt") == [{"cancelled": True}]
|
||||
assert not (tmp_path / "answers.jsonl").exists()
|
||||
|
||||
|
||||
def test_plan_input_cancellation_ignores_inherited_sigterm_disposition(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
wrapper = test_plan_input._WRAPPER.replace(
|
||||
"import json\n", "import json\nimport signal\nsignal.signal(signal.SIGTERM, signal.SIG_IGN)\n", 1,
|
||||
)
|
||||
monkeypatch.setattr(test_plan_input, "_WRAPPER", wrapper)
|
||||
assert _run(tmp_path, b"partial choice", stage="choice", action="protocol") == [{"cancelled": True}]
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Native plan guidance, input timing and reviewed exit through the real SDK profile."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deepseek_harness.models import CommandExecution
|
||||
from deepseek_harness import RunResult, Session, UserQuestionRequest
|
||||
from test_commands_runtime import (
|
||||
_Completion, _Runtime, _SESSION_ID, _WAIT_SECONDS, _assert_request_permission,
|
||||
_command_pair, _native_result, _of_type, _permission_facts, _read_events,
|
||||
_request_texts, _runtime, _text_blocks, _wait_activity,
|
||||
)
|
||||
|
||||
_PLAN = "# Implement the change\n\n- Inspect existing code.\n- Make the smallest change.\n- Run focused tests."
|
||||
_PLAN_REVISED = "# Revised plan\n\n- Include a rollback procedure."
|
||||
_GUIDANCE = "You are in plan mode."
|
||||
|
||||
|
||||
def _command(runtime: _Runtime, session: Session, text: str) -> CommandExecution:
|
||||
result = runtime.call(runtime.harness.client._query, session.id, text)
|
||||
assert isinstance(result, CommandExecution)
|
||||
assert result.result.kind == "success", result.result.text
|
||||
return result
|
||||
|
||||
|
||||
def _effective_system(request: dict) -> str:
|
||||
system = "\n".join(_text_blocks(request.get("system", "")))
|
||||
for message in request["messages"]:
|
||||
if message["role"] == "system":
|
||||
system = "\n".join(_text_blocks(message["content"]))
|
||||
return system
|
||||
|
||||
|
||||
def _tool_names(request: dict) -> set[str]:
|
||||
return {tool["name"] for tool in request.get("tools", [])}
|
||||
|
||||
|
||||
def _review(plan: str = _PLAN) -> _Completion:
|
||||
return _Completion("", tool_call=("exit_plan_mode", {"plan": plan}))
|
||||
|
||||
|
||||
def _answer(request: UserQuestionRequest, choice: str, feedback: str | None = None) -> dict:
|
||||
[question] = request.questions
|
||||
assert question["intent"]["kind"] == "plan-review"
|
||||
item = {"id": question["id"], "selected": [choice]}
|
||||
if feedback is not None:
|
||||
item["custom"] = feedback
|
||||
return {"answers": [item]}
|
||||
|
||||
|
||||
def test_bare_plan_and_off_change_only_plan_state_without_a_model_request(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, [], initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
on = _command(runtime, session, " \n/计划\t ")
|
||||
off = _command(runtime, session, "/plan off")
|
||||
repeated = _command(runtime, session, "/plan off")
|
||||
assert runtime.model.captured() == []
|
||||
assert "already inactive" in repeated.result.text
|
||||
assert runtime.harness.config.env["DSH_PERMISSION_MODE"] == "workspace-write"
|
||||
events = _read_events(runtime.logs)
|
||||
for execution in [on, off, repeated]:
|
||||
_command_pair(events, execution, "plan")
|
||||
assert [e["data"] for e in _of_type(events, "plan/mode")] == [{"active": True}, {"active": False}]
|
||||
assert not _of_type(events, "turn/start")
|
||||
assert [e["data"] for e in _permission_facts(events)] == [
|
||||
{"preset": "workspace-write"}, {"mode": "workspace-write"}, {"policy": "ask"},
|
||||
]
|
||||
|
||||
|
||||
def test_plan_body_uses_native_input_then_off_removes_guidance_but_keeps_exit_tool(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, [_Completion("PLAN_READY"), _Completion("AFTER_OFF")], initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
execution = _command(runtime, session, "/计划 Design this change without tools. ")
|
||||
_wait_activity(runtime, session)
|
||||
first = runtime.model.captured()[0]
|
||||
assert "Design this change without tools." in _request_texts(first)
|
||||
assert "/计划 Design this change without tools. " not in _request_texts(first)
|
||||
assert _GUIDANCE in _effective_system(first)
|
||||
assert "Do not edit or write files" in _effective_system(first)
|
||||
assert {"exit_plan_mode", "write", "edit"} <= _tool_names(first)
|
||||
_assert_request_permission(first, "workspace-write", "ask")
|
||||
_command(runtime, session, "/plan off")
|
||||
result = runtime.call(session.run, "Continue after the user changed the mode.")
|
||||
assert isinstance(result, RunResult) and result.final_response == "AFTER_OFF"
|
||||
second = runtime.model.captured()[1]
|
||||
assert _GUIDANCE not in _effective_system(second)
|
||||
assert _tool_names(first) == _tool_names(second)
|
||||
assert "The user switched this session back to the default mode." in _request_texts(second)
|
||||
_assert_request_permission(second, "workspace-write", "ask")
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, execution, "plan")
|
||||
assert len(_of_type(events, "turn/start")) == 2
|
||||
assert len(_permission_facts(events)) == 3
|
||||
|
||||
|
||||
def test_plan_body_steers_the_next_step_of_a_running_turn(tmp_path: Path) -> None:
|
||||
first = _Completion("FIRST_REPLY", blocked=True)
|
||||
with _runtime(tmp_path, [first, _Completion("PLANNING_REPLY")], initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
observing = runtime.workers.submit(session.run, "Start this task without tools.")
|
||||
assert first.entered.wait(_WAIT_SECONDS)
|
||||
execution = _command(runtime, session, "/plan Design a smaller change.")
|
||||
assert not first.release.is_set() and not observing.done()
|
||||
first.release.set()
|
||||
result = observing.result(timeout=_WAIT_SECONDS)
|
||||
assert isinstance(result, RunResult) and result.final_response == "PLANNING_REPLY"
|
||||
a, b = runtime.model.captured()
|
||||
assert _GUIDANCE not in _effective_system(a)
|
||||
assert _GUIDANCE in _effective_system(b)
|
||||
assert "Design a smaller change." in _request_texts(b)
|
||||
_assert_request_permission(b, "workspace-write", "ask")
|
||||
events = _read_events(runtime.logs)
|
||||
_command_pair(events, execution, "plan")
|
||||
assert [e["data"] for e in _of_type(events, "step/start")] == [{"turn": 1, "step": 1}, {"turn": 1, "step": 2}]
|
||||
assert len(_of_type(events, "turn/start")) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cancel", [False, True])
|
||||
def test_running_bare_selection_does_not_force_a_step_and_can_be_cancelled(tmp_path: Path, cancel: bool) -> None:
|
||||
first = _Completion("FIRST_REPLY", blocked=True)
|
||||
with _runtime(tmp_path, [first, _Completion("NEXT_REPLY")], initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
observing = runtime.workers.submit(session.run, "Start without tools.")
|
||||
assert first.entered.wait(_WAIT_SECONDS)
|
||||
_command(runtime, session, "/plan")
|
||||
if cancel:
|
||||
cancelled = _command(runtime, session, "/plan off")
|
||||
assert "cancelled" in cancelled.result.text
|
||||
assert len(runtime.model.captured()) == 1
|
||||
first.release.set()
|
||||
assert isinstance(observing.result(timeout=_WAIT_SECONDS), RunResult)
|
||||
assert len(runtime.model.captured()) == 1
|
||||
assert isinstance(runtime.call(session.run, "Inspect the next task without tools."), RunResult)
|
||||
assert (_GUIDANCE in _effective_system(runtime.model.captured()[1])) is (not cancel)
|
||||
events = _read_events(runtime.logs)
|
||||
assert len(_of_type(events, "turn/start")) == 2
|
||||
assert len(_of_type(events, "step/start")) == 2
|
||||
assert [e["data"] for e in _of_type(events, "plan/mode")] == ([] if cancel else [{"active": True}])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_choice", ["Approve", "Keep planning", "dismiss"])
|
||||
def test_review_approval_feedback_and_dismissal_keep_native_model_behavior(tmp_path: Path, first_choice: str) -> None:
|
||||
seen: list[str] = []
|
||||
def answer(request: UserQuestionRequest, cancelled: threading.Event):
|
||||
assert not cancelled.is_set()
|
||||
seen.append(request.questions[0]["detail"])
|
||||
if len(seen) == 1:
|
||||
if first_choice == "dismiss":
|
||||
return {"cancelled": True}
|
||||
return _answer(request, first_choice, "Include rollback." if first_choice == "Keep planning" else None)
|
||||
return _answer(request, "Approve")
|
||||
completions = [_review()]
|
||||
if first_choice == "Keep planning":
|
||||
completions.append(_review(_PLAN_REVISED))
|
||||
if first_choice != "dismiss":
|
||||
completions.append(_Completion("AFTER_APPROVAL"))
|
||||
with _runtime(tmp_path, completions, initial_permission="workspace-write", on_user_question=answer) as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
_command(runtime, session, "/plan Prepare a plan and present it for review.")
|
||||
_wait_activity(runtime, session)
|
||||
requests = runtime.model.captured()
|
||||
assert seen == ([_PLAN, _PLAN_REVISED] if first_choice == "Keep planning" else [_PLAN])
|
||||
assert _GUIDANCE in _effective_system(requests[0])
|
||||
if first_choice == "Keep planning":
|
||||
assert _GUIDANCE in _effective_system(requests[1])
|
||||
assert "Include rollback." in str(requests[1]["messages"])
|
||||
if first_choice != "dismiss":
|
||||
assert _GUIDANCE not in _effective_system(requests[-1])
|
||||
assert _tool_names(requests[0]) == _tool_names(requests[-1])
|
||||
else:
|
||||
assert len(requests) == 1, "dismissal started another model request"
|
||||
for request in requests:
|
||||
_assert_request_permission(request, "workspace-write", "ask")
|
||||
events = _read_events(runtime.logs)
|
||||
assert [e["data"]["active"] for e in _of_type(events, "plan/mode")] == ([True] if first_choice == "dismiss" else [True, False])
|
||||
assert len(_of_type(events, "turn/start")) == len(_of_type(events, "turn/end")) == 1
|
||||
assert len(_permission_facts(events)) == 3
|
||||
if first_choice == "dismiss":
|
||||
results = _of_type(events, "tool/result")
|
||||
assert len(results) == 1
|
||||
assert "The user dismissed the plan review" in str(results[0]["data"])
|
||||
|
||||
|
||||
def test_no_review_provider_keeps_plan_mode_and_reports_tool_failure(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, [_review(), _Completion("REVIEW_UNAVAILABLE")], initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
_command(runtime, session, "/plan Prepare a plan.")
|
||||
_wait_activity(runtime, session)
|
||||
first, second = runtime.model.captured()
|
||||
assert _GUIDANCE in _effective_system(first) and _GUIDANCE in _effective_system(second)
|
||||
assert "no user-questions answerer accepted the request" in str(second["messages"])
|
||||
_command(runtime, session, "/plan off")
|
||||
events = _read_events(runtime.logs)
|
||||
[result] = _of_type(events, "tool/result")
|
||||
assert result["data"]["message"]["isError"] is True
|
||||
assert result["data"]["error"]["code"] == "NO_PROVIDER"
|
||||
assert len(_permission_facts(events)) == 3
|
||||
|
||||
|
||||
def test_plan_state_survives_resume_without_changing_permissions(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, [], initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
_command(runtime, session, "/plan")
|
||||
with _runtime(tmp_path, [_Completion("RESTORED_PLAN")], initial_permission="workspace-write") as resumed:
|
||||
session = resumed.call(resumed.harness.resume_session, _SESSION_ID)
|
||||
assert isinstance(resumed.call(session.run, "Continue planning without tools."), RunResult)
|
||||
[request] = resumed.model.captured()
|
||||
assert _GUIDANCE in _effective_system(request)
|
||||
_assert_request_permission(request, "workspace-write", "ask")
|
||||
events = _read_events(resumed.logs)
|
||||
assert [e["data"] for e in _of_type(events, "plan/mode")] == [{"active": True}]
|
||||
|
||||
|
||||
def test_top_level_plan_result_identifies_the_session_for_wait_continue_and_resume(tmp_path: Path) -> None:
|
||||
first = _Completion("PLAN_READY", blocked=True)
|
||||
with _runtime(tmp_path, [first, _Completion("CONTINUED_PLAN")], initial_permission="workspace-write") as runtime:
|
||||
running = runtime.workers.submit(runtime.harness.run, "/plan Design this change without tools.")
|
||||
assert first.entered.wait(_WAIT_SECONDS), runtime.model.failures
|
||||
assert not running.done()
|
||||
first.release.set()
|
||||
execution = running.result(timeout=_WAIT_SECONDS)
|
||||
assert isinstance(execution, RunResult)
|
||||
assert execution.finish_reason == "completed" and execution.final_response == "PLAN_READY"
|
||||
session = runtime.harness.start_session(execution.session_id)
|
||||
continued = runtime.call(runtime.harness.run, "Continue this plan without tools.", session_id=execution.session_id)
|
||||
assert isinstance(continued, RunResult)
|
||||
assert continued.session_id == execution.session_id
|
||||
assert continued.final_response == "CONTINUED_PLAN"
|
||||
assert "Design this change without tools." in _request_texts(runtime.model.captured()[1])
|
||||
with _runtime(tmp_path, [_Completion("RESTORED_PLAN")], initial_permission="workspace-write") as resumed:
|
||||
session = resumed.call(resumed.harness.resume_session, execution.session_id)
|
||||
result = resumed.call(session.run, "Finish reviewing this plan without tools.")
|
||||
assert isinstance(result, RunResult) and result.session_id == execution.session_id
|
||||
assert result.final_response == "RESTORED_PLAN"
|
||||
[request] = resumed.model.captured()
|
||||
assert _GUIDANCE in _effective_system(request)
|
||||
assert "Design this change without tools." in _request_texts(request)
|
||||
assert "Continue this plan without tools." in _request_texts(request)
|
||||
events = _read_events(resumed.logs, session_id=execution.session_id)
|
||||
_command_pair(events, execution, "plan")
|
||||
assert len(_of_type(events, "turn/start")) == 3
|
||||
|
||||
|
||||
def test_plan_state_is_local_and_literal_prompts_do_not_toggle_it(tmp_path: Path) -> None:
|
||||
with _runtime(tmp_path, [_Completion("OTHER"), _Completion("PLANNING"), _Completion("RAW_INPUT")],
|
||||
initial_permission="workspace-write") as runtime:
|
||||
session = runtime.harness.start_session(_SESSION_ID)
|
||||
_command(runtime, session, "/plan")
|
||||
other = runtime.harness.start_session("plan-other")
|
||||
assert isinstance(runtime.call(other.run, "Inspect your own task without tools."), RunResult)
|
||||
assert isinstance(runtime.call(session.run, "Inspect the plan without tools."), RunResult)
|
||||
runtime.call(runtime.harness.client.session_prompt, session.id, [{"type": "text", "text": "/plan off"}])
|
||||
_wait_activity(runtime, session)
|
||||
first, second, third = runtime.model.captured()
|
||||
assert _GUIDANCE not in _effective_system(first)
|
||||
assert _GUIDANCE in _effective_system(second)
|
||||
assert _GUIDANCE in _effective_system(third)
|
||||
assert "/plan off" in _request_texts(third)
|
||||
assert not _of_type(_read_events(runtime.logs, session_id="plan-other"), "plan/mode")
|
||||
assert [e["data"] for e in _of_type(_read_events(runtime.logs), "plan/mode")] == [{"active": True}]
|
||||
@@ -0,0 +1,454 @@
|
||||
"""Unified blocking input, native command results and independent run observation."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from deepseek_harness.models import CommandExecution
|
||||
from deepseek_harness import DeepSeekHarness, Notification, RunResult, SdkProtocolError
|
||||
from deepseek_harness.errors import JsonRpcError
|
||||
|
||||
|
||||
class Ack(BaseModel):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def peer(tmp_path: Path) -> Path:
|
||||
script = tmp_path / "queries.py"
|
||||
script.write_text('''import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
serial = 0
|
||||
pending_commands = []
|
||||
pending_prompts = []
|
||||
pending_plans = []
|
||||
|
||||
def send(frame):
|
||||
print(json.dumps({"jsonrpc": "2.0", **frame}), flush=True)
|
||||
|
||||
def event(session, type, data):
|
||||
send({"method": "session.event", "params": {"sessionId": session, "event": {"type": type, "data": data}}})
|
||||
|
||||
def input_text(frame):
|
||||
return "".join(block["text"] for block in frame["params"]["contentBlocks"] if block["type"] == "text")
|
||||
|
||||
def command(frame, number):
|
||||
session = frame["params"]["sessionId"]
|
||||
identity = "command-" + str(number)
|
||||
event(session, "command/run", {"commandId": identity})
|
||||
event(session, "command/done", {"commandId": identity, "kind": "success"})
|
||||
send({"method": "session.query.completed", "params": {"sessionId": session, "commandId": identity,
|
||||
"finalResponse": "command completed", "finishReason": "completed"}})
|
||||
send({"id": frame["id"], "result": {"kind": "command", "execution": {
|
||||
"commandId": identity, "result": {"kind": "success", "text": "command completed"}}}})
|
||||
|
||||
def receipt(frame, number):
|
||||
session = frame["params"]["sessionId"]
|
||||
send({"method": "session.status", "params": {"sessionId": session, "status": "idle"}})
|
||||
event(session, "agent/inbox/spliced", {"inserted": [{"id": "input-" + str(number)}]})
|
||||
|
||||
def finish(frame):
|
||||
session = frame["params"]["sessionId"]
|
||||
event(session, "assistant/message", {"message": {"content": [
|
||||
{"type": "text", "text": "answer for " + input_text(frame)}]}})
|
||||
event(session, "turn/end", {"turn": 1, "reason": {"kind": "completed"}})
|
||||
send({"method": "session.status", "params": {"sessionId": session, "status": "idle"}})
|
||||
|
||||
for line in sys.stdin:
|
||||
frame = json.loads(line)
|
||||
with open(os.environ["REQUEST_LOG"], "a", encoding="utf-8") as log:
|
||||
log.write(line)
|
||||
method = frame["method"]
|
||||
if method == "initialize":
|
||||
send({"id": frame["id"], "result": {"serverInfo": {"name": "query-peer"}}})
|
||||
elif method == "session/prompt":
|
||||
send({"id": frame["id"], "result": {"messageId": "literal-input"}})
|
||||
elif method == "session/query":
|
||||
serial += 1
|
||||
params = frame["params"]
|
||||
assert set(params) == {"sessionId", "contentBlocks"}
|
||||
text = input_text(frame)
|
||||
if text == "/plan hold":
|
||||
identity = "plan-" + str(serial)
|
||||
event(params["sessionId"], "command/run", {"commandId": identity})
|
||||
event(params["sessionId"], "command/done", {"commandId": identity, "kind": "success"})
|
||||
send({"id": frame["id"], "result": {"kind": "command", "execution": {
|
||||
"commandId": identity, "result": {"kind": "success", "text": "Plan enabled"}}}})
|
||||
pending_plans.append((frame, identity))
|
||||
elif text == "hold-command":
|
||||
pending_commands.append((frame, serial))
|
||||
elif text in ("/compact", "/plan Describe this image."):
|
||||
command(frame, serial)
|
||||
elif text in ("missing-run", "missing-done", "missing-both", "unrelated-done", "invalid-done"):
|
||||
if text not in ("missing-run", "missing-both"):
|
||||
event(params["sessionId"], "command/run", {"commandId": "silent"})
|
||||
if text == "invalid-done":
|
||||
event(params["sessionId"], "command/done", None)
|
||||
elif text not in ("missing-done", "missing-both"):
|
||||
event(params["sessionId"], "command/done", {
|
||||
"commandId": "another-command" if text == "unrelated-done" else "silent", "kind": "success"})
|
||||
send({"method": "session.query.completed", "params": {"sessionId": params["sessionId"],
|
||||
"commandId": "silent", "finalResponse": "", "finishReason": "completed"}})
|
||||
send({"id": frame["id"], "result": {"kind": "command", "execution": {
|
||||
"commandId": "silent", "result": {"kind": "success"}}}})
|
||||
elif text == "rpc-error":
|
||||
send({"id": frame["id"], "error": {"code": -32602, "message": "query rejected"}})
|
||||
elif text == "custom-response":
|
||||
response = json.loads(os.environ["QUERY_RESPONSE"])
|
||||
execution = response.get("execution") if isinstance(response, dict) else None
|
||||
if isinstance(execution, dict) and isinstance(execution.get("result"), dict):
|
||||
event(params["sessionId"], "command/run", {"commandId": execution.get("commandId")})
|
||||
if os.environ.get("QUERY_LIFECYCLE") != "late-done":
|
||||
event(params["sessionId"], "command/done", {"commandId": execution.get("commandId"), **execution["result"]})
|
||||
send({"method": "session.query.completed", "params": {"sessionId": params["sessionId"],
|
||||
"commandId": execution.get("commandId"), "finalResponse": execution["result"].get("text", ""),
|
||||
"finishReason": "error" if execution["result"].get("kind") == "error" else "completed",
|
||||
**json.loads(os.environ.get("QUERY_COMPLETION", "{}"))}})
|
||||
if os.environ.get("QUERY_LIFECYCLE") == "late-done":
|
||||
event(params["sessionId"], "command/done", {"commandId": execution.get("commandId"), **execution["result"]})
|
||||
send({"id": frame["id"], "result": response})
|
||||
elif text == "bad-route":
|
||||
send({"id": frame["id"], "result": {"kind": "unknown"}})
|
||||
elif text == "bad-execution":
|
||||
send({"id": frame["id"], "result": {"kind": "command", "execution": {"commandId": "c"}}})
|
||||
else:
|
||||
result = {"kind": "prompt", "messageId": "input-" + str(serial)}
|
||||
if text == "fast-prompt":
|
||||
receipt(frame, serial)
|
||||
finish(frame)
|
||||
send({"id": frame["id"], "result": result})
|
||||
else:
|
||||
send({"id": frame["id"], "result": result})
|
||||
receipt(frame, serial)
|
||||
if text == "hold-prompt":
|
||||
pending_prompts.append(frame)
|
||||
else:
|
||||
finish(frame)
|
||||
elif method == "release":
|
||||
for original, identity in pending_plans:
|
||||
finish(original)
|
||||
send({"method": "session.query.completed", "params": {"sessionId": original["params"]["sessionId"],
|
||||
"commandId": identity, "finalResponse": "Generated plan", "finishReason": "completed"}})
|
||||
pending_plans.clear()
|
||||
for original, number in pending_commands:
|
||||
command(original, number)
|
||||
pending_commands.clear()
|
||||
for original in pending_prompts:
|
||||
finish(original)
|
||||
pending_prompts.clear()
|
||||
send({"id": frame["id"], "result": {}})
|
||||
elif method == "barrier":
|
||||
send({"id": frame["id"], "result": {}})
|
||||
elif method == "disconnect":
|
||||
send({"id": frame["id"], "result": {}})
|
||||
break
|
||||
elif method == "shutdown":
|
||||
send({"id": frame["id"], "result": {}})
|
||||
break
|
||||
else:
|
||||
raise AssertionError(method)
|
||||
''', encoding="utf-8")
|
||||
return script
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sdk(peer: Path, tmp_path: Path):
|
||||
with DeepSeekHarness(
|
||||
cwd=str(tmp_path), dsh_home=str(tmp_path / "home"), request_timeout_seconds=5,
|
||||
_launch_args=(sys.executable, str(peer)),
|
||||
env={"REQUEST_LOG": str(tmp_path / "requests.jsonl")},
|
||||
) as harness:
|
||||
yield harness
|
||||
|
||||
|
||||
def release(sdk: DeepSeekHarness) -> None:
|
||||
sdk.client.request("release", {}, response_model=Ack, timeout_seconds=5)
|
||||
|
||||
|
||||
def assert_released(sdk: DeepSeekHarness) -> None:
|
||||
assert sdk.client._responses == {}
|
||||
assert sdk.client._notification_subscribers == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["session", "harness"])
|
||||
def test_run_returns_command_without_waiting_for_idle(sdk: DeepSeekHarness, entry: str) -> None:
|
||||
notifications = []
|
||||
result = (sdk.run("/compact", session_id="main", on_notification=notifications.append)
|
||||
if entry == "harness" else sdk.start_session("main").run("/compact", on_notification=notifications.append))
|
||||
assert isinstance(result, RunResult)
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.session_id == "main"
|
||||
assert result.finish_reason == "completed"
|
||||
assert [notice.payload["event"]["type"] for notice in notifications if notice.method == "session.event"] == ["command/run", "command/done"]
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
def test_top_level_command_results_identify_distinct_sessions_and_allow_continuing_one(
|
||||
sdk: DeepSeekHarness, tmp_path: Path,
|
||||
) -> None:
|
||||
first = sdk.run("/compact")
|
||||
second = sdk.run("/compact")
|
||||
assert isinstance(first, RunResult) and isinstance(second, RunResult)
|
||||
assert first.session_id and second.session_id and first.session_id != second.session_id
|
||||
continued = sdk.run("continue the first session", session_id=first.session_id)
|
||||
assert isinstance(continued, RunResult)
|
||||
assert continued.session_id == first.session_id
|
||||
sent = [json.loads(line) for line in (tmp_path / "requests.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
queries = [frame["params"] for frame in sent if frame["method"] == "session/query"]
|
||||
assert [query["sessionId"] for query in queries] == [first.session_id, second.session_id, first.session_id]
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("result", [
|
||||
{"kind": "success"},
|
||||
{"kind": "success", "text": "Compacted.", "sourceEventSeq": 0},
|
||||
{"kind": "success", "sourceEventSeq": 2**53 - 1},
|
||||
{"kind": "error", "text": "The agent is busy."},
|
||||
])
|
||||
def test_run_adds_session_identity_without_changing_command_results(peer: Path, tmp_path: Path, result: dict) -> None:
|
||||
execution = {"commandId": "custom-command", "result": result}
|
||||
with DeepSeekHarness(
|
||||
cwd=str(tmp_path), dsh_home=str(tmp_path / "home"), request_timeout_seconds=5,
|
||||
_launch_args=(sys.executable, str(peer)),
|
||||
env={"REQUEST_LOG": str(tmp_path / "requests.jsonl"), "QUERY_RESPONSE": json.dumps({
|
||||
"kind": "command", "execution": execution,
|
||||
})},
|
||||
) as sdk:
|
||||
returned = sdk.run("custom-response")
|
||||
assert isinstance(returned, RunResult)
|
||||
assert returned.session_id
|
||||
assert returned.final_response == result.get("text", "")
|
||||
assert returned.finish_reason == ("error" if result["kind"] == "error" else "completed")
|
||||
low_level = sdk.client._query(returned.session_id, "custom-response")
|
||||
assert isinstance(low_level, CommandExecution)
|
||||
assert low_level.model_dump(by_alias=True, exclude_none=True) == execution
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["ordinary", "fast-prompt", "/model", " /compact extra "])
|
||||
def test_run_preserves_ordinary_text_and_result(sdk: DeepSeekHarness, text: str) -> None:
|
||||
notifications = []
|
||||
result = sdk.start_session("main").run(text, on_notification=notifications.append)
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.final_response == "answer for " + text
|
||||
assert result.finish_reason == "completed"
|
||||
assert result.notifications == notifications
|
||||
assert notifications[0].payload["event"]["type"] == "agent/inbox/spliced"
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry,text", [
|
||||
("harness", "/plan Describe this image."),
|
||||
("session", "/plan Describe this image."),
|
||||
("harness", "Explain this image."),
|
||||
("session", "/unknown-slash"),
|
||||
("session", ""),
|
||||
])
|
||||
def test_run_forwards_content_blocks_with_the_routed_result_and_notifications(
|
||||
sdk: DeepSeekHarness, tmp_path: Path, entry: str, text: str,
|
||||
) -> None:
|
||||
blocks = [
|
||||
{"type": "text", "text": text},
|
||||
{"type": "image", "data": "AQ==", "mimeType": "image/png", "name": "diagram.png"},
|
||||
]
|
||||
notifications = []
|
||||
result = (
|
||||
sdk.run(blocks, session_id="main", on_notification=notifications.append)
|
||||
if entry == "harness" else sdk.start_session("main").run(
|
||||
blocks, on_notification=notifications.append,
|
||||
)
|
||||
)
|
||||
if text == "/plan Describe this image.":
|
||||
assert isinstance(result, RunResult)
|
||||
assert [notice.payload["event"]["type"] for notice in notifications if notice.method == "session.event"] == ["command/run", "command/done"]
|
||||
else:
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.final_response == "answer for " + text
|
||||
assert result.notifications == notifications
|
||||
sent = [json.loads(line) for line in (tmp_path / "requests.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
assert sent[-1]["method"] == "session/query"
|
||||
assert sent[-1]["params"] == {
|
||||
"sessionId": "main", "contentBlocks": blocks,
|
||||
}
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
def test_run_normalizes_strings_without_changing_their_text(sdk: DeepSeekHarness, tmp_path: Path) -> None:
|
||||
result = sdk.run(" preserve whitespace ", session_id="main")
|
||||
assert isinstance(result, RunResult)
|
||||
sent = [json.loads(line) for line in (tmp_path / "requests.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
assert sent[-1]["params"] == {
|
||||
"sessionId": "main", "contentBlocks": [{"type": "text", "text": " preserve whitespace "}],
|
||||
}
|
||||
|
||||
|
||||
def test_concurrent_runs_keep_their_own_results(sdk: DeepSeekHarness) -> None:
|
||||
session = sdk.start_session("main")
|
||||
entered = threading.Event()
|
||||
with ThreadPoolExecutor(max_workers=2) as workers:
|
||||
first = workers.submit(session.run, "hold-prompt", on_notification=lambda notice: entered.set())
|
||||
try:
|
||||
assert entered.wait(5), "the first run must observe its receipt"
|
||||
second = workers.submit(session.run, "/compact")
|
||||
assert isinstance(second.result(timeout=5), RunResult)
|
||||
finally:
|
||||
release(sdk)
|
||||
result = first.result(timeout=5)
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.final_response == "answer for hold-prompt"
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["missing-run", "missing-done", "missing-both", "unrelated-done", "invalid-done"])
|
||||
def test_command_result_rejects_incomplete_lifecycle(sdk: DeepSeekHarness, text: str) -> None:
|
||||
session = sdk.start_session("main")
|
||||
with pytest.raises(SdkProtocolError, match="lifecycle"):
|
||||
session.run(text)
|
||||
assert_released(sdk)
|
||||
assert session.run("after").final_response == "answer for after"
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
def test_failure_completion_can_precede_handler_settlement(peer: Path, tmp_path: Path) -> None:
|
||||
with DeepSeekHarness(
|
||||
cwd=str(tmp_path), dsh_home=str(tmp_path / "home"), request_timeout_seconds=5,
|
||||
_launch_args=(sys.executable, str(peer)),
|
||||
env={"REQUEST_LOG": str(tmp_path / "requests.jsonl"), "QUERY_LIFECYCLE": "late-done",
|
||||
"QUERY_RESPONSE": json.dumps({"kind": "command", "execution": {
|
||||
"commandId": "early-failure", "result": {"kind": "success"}}}),
|
||||
"QUERY_COMPLETION": json.dumps({"finalResponse": "goal service unloaded", "finishReason": "error"})},
|
||||
) as sdk:
|
||||
result = sdk.start_session("main").run("custom-response")
|
||||
assert result.final_response == "goal service unloaded" and result.finish_reason == "error"
|
||||
assert [event["type"] for event in result.events] == ["command/run", "command/done"]
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text,error", [("rpc-error", JsonRpcError), ("bad-route", SdkProtocolError), ("bad-execution", SdkProtocolError)])
|
||||
def test_run_reports_routing_failures_without_retry(sdk: DeepSeekHarness, text: str, error: type[Exception]) -> None:
|
||||
session = sdk.start_session("main")
|
||||
with pytest.raises(error):
|
||||
session.run(text)
|
||||
assert_released(sdk)
|
||||
assert isinstance(session.run("/compact"), RunResult)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["ordinary", "/compact"])
|
||||
def test_observer_error_releases_run_subscription(sdk: DeepSeekHarness, text: str) -> None:
|
||||
def reject(_notice: Notification) -> None:
|
||||
raise RuntimeError("observer failed")
|
||||
with pytest.raises(RuntimeError, match="observer failed"):
|
||||
sdk.start_session("main").run(text, on_notification=reject)
|
||||
assert_released(sdk)
|
||||
assert isinstance(sdk.start_session("main").run("after observer"), RunResult)
|
||||
|
||||
|
||||
def test_command_timeout_keeps_runtime_responsive(sdk: DeepSeekHarness) -> None:
|
||||
sdk.client.config.request_timeout_seconds = 0.05
|
||||
with pytest.raises(TimeoutError, match="session/query timed out"):
|
||||
sdk.start_session("main").run("hold-command")
|
||||
sdk.client.config.request_timeout_seconds = 5
|
||||
assert_released(sdk)
|
||||
release(sdk)
|
||||
assert isinstance(sdk.start_session("main").run("/compact"), RunResult)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response", [
|
||||
None, [], {"kind": "unknown"}, {"kind": "prompt"}, {"kind": "prompt", "messageId": ""},
|
||||
{"kind": "prompt", "messageId": 4},
|
||||
{"kind": "command", "execution": {"commandId": "c"}},
|
||||
{"kind": "command", "execution": {"commandId": "c", "result": {"kind": "success", "sourceEventSeq": True}}},
|
||||
])
|
||||
def test_run_rejects_invalid_routing_responses_as_protocol_errors(peer: Path, tmp_path: Path, response: object) -> None:
|
||||
with DeepSeekHarness(
|
||||
cwd=str(tmp_path), dsh_home=str(tmp_path / "home"), request_timeout_seconds=5,
|
||||
_launch_args=(sys.executable, str(peer)),
|
||||
env={"REQUEST_LOG": str(tmp_path / "requests.jsonl"), "QUERY_RESPONSE": json.dumps(response)},
|
||||
) as sdk:
|
||||
with pytest.raises(SdkProtocolError) as raised:
|
||||
sdk.start_session("main").run("custom-response")
|
||||
if isinstance(response, dict) and response.get("kind") == "command":
|
||||
assert isinstance(raised.value.__cause__, ValidationError)
|
||||
assert_released(sdk)
|
||||
assert isinstance(sdk.start_session("main").run("/compact"), RunResult)
|
||||
|
||||
|
||||
def test_plan_run_waits_for_model_output_after_the_command_receipt(sdk: DeepSeekHarness) -> None:
|
||||
admitted = threading.Event()
|
||||
session = sdk.start_session("plan")
|
||||
def observe(notice: Notification) -> None:
|
||||
if notice.method == "session.event" and notice.payload["event"]["type"] == "command/done":
|
||||
admitted.set()
|
||||
with ThreadPoolExecutor(max_workers=1) as workers:
|
||||
running = workers.submit(session.run, "/plan hold", on_notification=observe)
|
||||
try:
|
||||
assert admitted.wait(5)
|
||||
assert not running.done()
|
||||
finally:
|
||||
release(sdk)
|
||||
result = running.result(timeout=5)
|
||||
assert isinstance(result, RunResult)
|
||||
assert result.final_response == "Generated plan" and result.finish_reason == "completed"
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("completion", [{"commandId": None}, {"finalResponse": 1}, {"finishReason": None}, {"finishReason": ""}])
|
||||
def test_run_rejects_malformed_completion_and_releases_its_subscription(
|
||||
peer: Path, tmp_path: Path, completion: dict,
|
||||
) -> None:
|
||||
with DeepSeekHarness(
|
||||
cwd=str(tmp_path), dsh_home=str(tmp_path / "home"),
|
||||
_launch_args=(sys.executable, str(peer)),
|
||||
env={"REQUEST_LOG": str(tmp_path / "requests.jsonl"),
|
||||
"QUERY_COMPLETION": json.dumps(completion),
|
||||
"QUERY_RESPONSE": json.dumps({"kind": "command", "execution": {
|
||||
"commandId": "c", "result": {"kind": "success"}}})},
|
||||
) as sdk:
|
||||
with pytest.raises(SdkProtocolError):
|
||||
sdk.run("custom-response")
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("execution", [
|
||||
{},
|
||||
{"commandId": "command-1"},
|
||||
{"commandId": "", "result": {"kind": "success"}},
|
||||
{"commandId": "command-1", "result": {"kind": "unknown"}},
|
||||
{"commandId": "command-1", "result": {"kind": "error"}},
|
||||
{"commandId": "command-1", "result": {"kind": "error", "text": ""}},
|
||||
{"commandId": "command-1", "result": {"kind": "error", "text": " \t\n"}},
|
||||
{"commandId": "command-1", "result": {"kind": "success", "text": None}},
|
||||
{"commandId": "command-1", "result": {"kind": "success", "sourceEventSeq": None}},
|
||||
{"commandId": "command-1", "result": {"kind": "success", "sourceEventSeq": "4"}},
|
||||
{"commandId": "command-1", "result": {"kind": "success", "sourceEventSeq": True}},
|
||||
{"commandId": "command-1", "result": {"kind": "success", "sourceEventSeq": -1}},
|
||||
{"commandId": "command-1", "result": {"kind": "success", "sourceEventSeq": 2**53}},
|
||||
])
|
||||
def test_run_rejects_malformed_native_results(peer: Path, tmp_path: Path, execution: dict) -> None:
|
||||
with DeepSeekHarness(
|
||||
cwd=str(tmp_path), dsh_home=str(tmp_path / "home"),
|
||||
_launch_args=(sys.executable, str(peer)),
|
||||
env={"REQUEST_LOG": str(tmp_path / "requests.jsonl"), "QUERY_RESPONSE": json.dumps({
|
||||
"kind": "command", "execution": execution,
|
||||
})},
|
||||
) as sdk:
|
||||
with pytest.raises(SdkProtocolError) as raised:
|
||||
sdk.run("custom-response")
|
||||
assert isinstance(raised.value.__cause__, ValidationError)
|
||||
assert_released(sdk)
|
||||
|
||||
|
||||
def test_low_level_prompt_keeps_slash_text_as_an_ordinary_prompt(sdk: DeepSeekHarness, tmp_path: Path) -> None:
|
||||
blocks = [{"type": "text", "text": "/compact"}]
|
||||
assert sdk.client.session_prompt("main", blocks) == "literal-input"
|
||||
sent = [json.loads(line) for line in (tmp_path / "requests.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
assert sent[-1]["method"] == "session/prompt"
|
||||
assert sent[-1]["params"] == {"sessionId": "main", "contentBlocks": blocks}
|
||||
@@ -102,6 +102,15 @@ def test_missing_callback_does_not_advertise_question_support(runtime: Path, tmp
|
||||
assert client.request("stats", {}, response_model=Response).capabilities is None
|
||||
|
||||
|
||||
def test_callback_can_dismiss_a_question_without_a_provider_error(runtime: Path, tmp_path: Path) -> None:
|
||||
with HarnessClient(config(lambda _request, _cancelled: {"cancelled": True}),
|
||||
_launch_args=(sys.executable, str(runtime))) as client:
|
||||
initialize(client, tmp_path)
|
||||
result = client.request("ask", {}, response_model=Response)
|
||||
assert result.answer == {"cancelled": True}
|
||||
assert result.error is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["cancel", "disconnect", "close"])
|
||||
def test_cancellation_stops_cooperative_host_work_and_suppresses_late_answer(runtime: Path, tmp_path: Path, action: str) -> None:
|
||||
entered = threading.Event()
|
||||
|
||||
Reference in new issue
Block a user