mward4/deepseek-harness
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

Merge pull request #5800 from deepseek-harness/worktree/ptc-providers/remove-both

Browse Source 
refactor(tools)!: remove unused mixed tool presentation
This commit is contained in:
Tianyi Cui authored and GitHub committed 2026-10-07 15:34:35 +08:00
2 parents 86b9b9c017 aeb166c42e
commit 36f12b6f6e
103 files changed
+6471 -2409
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/smoke-python-runtime.py
+89 -66
View File
@@ -22,7 +22,7 @@ import time
import zipfile
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import TYPE_CHECKING, Callable
from typing import TYPE_CHECKING, Callable, Literal
if TYPE_CHECKING:
from deepseek_harness import Notification, RunResult
@@ -114,11 +114,10 @@ SNAPSHOT_WORKFLOW_SCRIPT = (
f"const reply = await agent('{SNAPSHOT_WORKFLOW_CHILD_PROMPT}', {{ label: 'workflow-child' }})\n"
"return { reply }"
)
ADVANCED_SNAPSHOT_DIRECTORY = (
Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "advanced"
)
ADVANCED_SNAPSHOT_ROOT = Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe"
ADVANCED_SNAPSHOT_DIRECTORY = ADVANCED_SNAPSHOT_ROOT / "advanced"
ADVANCED_SNAPSHOT_FILENAMES = (
"result.json", "session.v3.jsonl", "session.1.v3.jsonl", "session.2.v3.jsonl",
"result.json", "session.v4.jsonl", "session.1.v4.jsonl", "session.2.v4.jsonl",
)
MINIMAL_SNAPSHOT_DIRECTORY = (
Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "minimal"
@@ -138,7 +137,7 @@ RESTART_SNAPSHOT_DIRECTORY = (
Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "restart"
)
RESTART_SNAPSHOT_FILENAMES = (
"result.json", "requests.json", "session.1.v3.jsonl", "session.2.v3.jsonl",
"result.json", "requests.json", "session.1.v4.jsonl", "session.2.v4.jsonl",
)
RECOVERY_SNAPSHOT_DIRECTORY = (
Path(__file__).resolve().parent / "snapshots" / "python-sdk-single-exe" / "scheduler-recovery"
@@ -241,10 +240,12 @@ def write_profile_patch(
return path
def write_advanced_profile_patch(root: Path, name: str, sessions: Path) -> Path:
def write_advanced_profile_patch(
root: Path, name: str, sessions: Path, mode: Literal["native", "ptc"] = "native",
) -> Path:
"""Write the shared custom, snapshot, and restart profile patch."""
return write_profile_patch(root, name, sessions, [
{"id": "tools", "config": {"mode": "both"}},
{"id": "tools", "config": {"mode": mode}},
{
"id": "system-prompt",
"config": {
@@ -444,13 +445,16 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
if prompt == SNAPSHOT_WORKFLOW_CHILD_PROMPT:
return text_chunks("WORKFLOW_CHILD_OK")
if prompt == SNAPSHOT_PROMPT:
if "run_code" in advertised_tool_names(body):
if advertised_tool_names(body) != {"run_code"}:
raise AssertionError("PTC mode must advertise only run_code")
return tool_call_chunks(
"advanced-code", "run_code",
{"code": "return await tools.snapshot_double({ value: 21 })",
"description": "Call the configured Plugin tool"},
)
assert_advertised_tool(body, "snapshot_double")
assert_advertised_tool(body, "run_code")
return tool_call_chunks(
"advanced-code", "run_code",
{"code": "return await tools.snapshot_double({ value: 21 })",
"description": "Call the configured Plugin tool"},
)
return tool_call_chunks("advanced-native", "snapshot_double", {"value": 21})
if prompt == RESTART_FIRST_PROMPT:
return text_chunks(RESTART_FIRST_TEXT)
if prompt == DYNAMIC_TOOLS_PROMPT:
@@ -619,15 +623,13 @@ def advanced_tool_followup(
"""Advance the executable snapshot's deterministic parent tool chain."""
if not call_id.startswith("advanced-"):
return None
if call_id == "advanced-native" and tool_name == "snapshot_double":
if tool_text != "42":
raise AssertionError(f"native call returned no configured-tool value: {tool_text}")
return tool_call_chunks("advanced-denied-native", "snapshot_double", {"value": -1})
if call_id == "advanced-code" and tool_name == "run_code":
if "42" not in tool_text:
raise AssertionError(f"run_code returned no configured-tool value: {tool_text}")
return tool_call_chunks("advanced-denied-native", "snapshot_double", {"value": -1})
if call_id == "advanced-denied-native" and tool_name == "snapshot_double":
if 'Auto review rejected tool "snapshot_double"; its body was not executed' not in tool_text:
raise AssertionError(f"native denial did not preserve the model result: {tool_text}")
if "transport raw" in tool_text:
raise AssertionError("native denial leaked its user-facing reason to the model")
return tool_call_chunks("advanced-denied-ptc", "run_code", {
"code": "try { await tools.snapshot_double({ value: -1 }) } catch (error) { return error.message }",
"description": "Catch a structured inner tool denial",
@@ -637,6 +639,12 @@ def advanced_tool_followup(
raise AssertionError(f"PTC denial did not preserve the model result: {tool_text}")
if "transport raw" in tool_text:
raise AssertionError("PTC denial leaked its user-facing reason to the model")
return text_chunks(SNAPSHOT_FINAL_TEXT)
if call_id == "advanced-denied-native" and tool_name == "snapshot_double":
if 'Auto review rejected tool "snapshot_double"; its body was not executed' not in tool_text:
raise AssertionError(f"native denial did not preserve the model result: {tool_text}")
if "transport raw" in tool_text:
raise AssertionError("native denial leaked its user-facing reason to the model")
assert_advertised_tool(body, "subagent")
return tool_call_chunks(
"advanced-direct-child",
@@ -1203,33 +1211,35 @@ def smoke_sdk_default(base_url: str) -> None:
def smoke_sdk_custom(base_url: str, executable: Path) -> None:
from deepseek_harness import DeepSeekHarness
with tempfile.TemporaryDirectory(prefix="dsh-sdk-custom-") as temporary:
root = Path(temporary).resolve()
dsh_home = root / "home"
sessions = dsh_home / "sessions"
patch = write_advanced_profile_patch(root, "custom.patch.yml", sessions)
with DeepSeekHarness(
provider="deepseek-official",
model="smoke-model",
cwd=str(root),
dsh_bin=str(executable),
dsh_home=str(dsh_home),
patches=(str(patch),),
env={
"DSH_PERMISSION_MODE": "danger-full-access",
"DSH_TELEMETRY_DISABLED": "1",
},
api_key="sk-keyless-smoke",
base_url=base_url,
request_timeout_seconds=60,
) as harness:
text_result = harness.run("reply with the smoke text", session_id="custom-smoke")
code_result = harness.run(CODE_PROMPT, session_id="custom-smoke")
workflow_result = harness.run(WORKFLOW_PROMPT, session_id="custom-smoke")
assert text_result.final_response == EXPECTED_TEXT, text_result.final_response
assert code_result.final_response == CODE_WORKER_TEXT, code_result.final_response
assert workflow_result.final_response == WORKFLOW_WORKER_TEXT, workflow_result.final_response
assert_session_log(sessions, root, EXPECTED_TEXT, CODE_WORKER_TEXT, WORKFLOW_WORKER_TEXT)
cases: tuple[tuple[Literal["native", "ptc"], tuple[tuple[str, str], ...]], ...] = (
("native", (("reply with the smoke text", EXPECTED_TEXT), (WORKFLOW_PROMPT, WORKFLOW_WORKER_TEXT))),
("ptc", ((CODE_PROMPT, CODE_WORKER_TEXT),)),
)
for mode, turns in cases:
with tempfile.TemporaryDirectory(prefix=f"dsh-sdk-custom-{mode}-") as temporary:
root = Path(temporary).resolve()
dsh_home = root / "home"
sessions = dsh_home / "sessions"
patch = write_advanced_profile_patch(root, "custom.patch.yml", sessions, mode)
with DeepSeekHarness(
provider="deepseek-official",
model="smoke-model",
cwd=str(root),
dsh_bin=str(executable),
dsh_home=str(dsh_home),
patches=(str(patch),),
env={
"DSH_PERMISSION_MODE": "danger-full-access",
"DSH_TELEMETRY_DISABLED": "1",
},
api_key="sk-keyless-smoke",
base_url=base_url,
request_timeout_seconds=60,
) as harness:
for prompt, expected in turns:
result = harness.run(prompt, session_id=f"custom-{mode}")
assert result.final_response == expected, result.final_response
assert_session_log(sessions, root, *(expected for _, expected in turns))
def smoke_sdk_minimal(
@@ -1529,14 +1539,22 @@ def smoke_sdk_profile_plugin(base_url: str) -> None:
def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool) -> None:
"""Drive and compare the advanced SDK/executable behavioral snapshot."""
"""Exercise native and PTC calls in independent packaged-runtime compositions."""
for mode in ("native", "ptc"):
smoke_sdk_snapshot_mode(base_url, executable, update_snapshots, mode)
def smoke_sdk_snapshot_mode(
base_url: str, executable: Path, update_snapshots: bool, mode: Literal["native", "ptc"],
) -> None:
"""Drive one supported presentation through the SDK and persistent Session log."""
from deepseek_harness import DeepSeekHarness
with tempfile.TemporaryDirectory(prefix="dsh-sdk-snapshot-") as temporary:
root = Path(temporary).resolve()
dsh_home = root / "home"
sessions = dsh_home / "sessions"
patch = write_advanced_profile_patch(root, "snapshot.patch.yml", sessions)
patch = write_advanced_profile_patch(root, "snapshot.patch.yml", sessions, mode)
feedback_patch = write_profile_patch(root, "feedback.patch.yml", sessions, [{"insert": [
{"id": "snapshot-tool", "name": (
Path(__file__).resolve().parent / "fixtures/python-snapshot-tool.mjs"
@@ -1600,11 +1618,13 @@ def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool)
if feedback_types != ["feedback/record", "feedback/record", "feedback/message-put", "feedback/message-put", "feedback/message-delete"]:
raise AssertionError(f"advanced snapshot did not exercise all feedback mutations: {feedback_types}")
methods = [notification.method for notification in result.notifications]
if methods.count("subagent.started") != 2 or methods.count("subagent.finished") != 2:
raise AssertionError(f"advanced snapshot emitted unexpected subagent lifecycle: {methods}")
expected_children = 2 if mode == "native" else 0
if methods.count("subagent.started") != expected_children or methods.count("subagent.finished") != expected_children:
raise AssertionError(f"{mode} advanced snapshot emitted unexpected subagent lifecycle: {methods}")
ptc_events = [event for event in result.events
if event.get("type") in ("tool/ptc-dispatch-start", "tool/ptc-dispatch")]
if [event["type"] for event in ptc_events] != ["tool/ptc-dispatch-start", "tool/ptc-dispatch"] * 2:
expected_ptc_calls = 2 if mode == "ptc" else 0
if [event["type"] for event in ptc_events] != ["tool/ptc-dispatch-start", "tool/ptc-dispatch"] * expected_ptc_calls:
raise AssertionError(f"advanced snapshot emitted unexpected PTC dispatch events: {ptc_events}")
for index, event in enumerate(ptc_events):
data = event["data"]
@@ -1618,21 +1638,23 @@ def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool)
if event.get("type") in ("tool/result", "tool/ptc-dispatch") and "error" in event["data"]]
assert errors == [{
"name": "AutoReviewDeniedError", "code": "AUTO_REVIEW_DENIED", "reason": " transport raw\r\nreason ",
}] * 2, errors
}], errors
logs = read_session_logs(sessions)
child_ids = snapshot_child_ids(result)
child_ids = snapshot_child_ids(result, expected_children)
expected_ids = {SNAPSHOT_SESSION_ID, *child_ids}
if set(logs) != expected_ids:
raise AssertionError(f"advanced snapshot expected parent plus two child logs: {sorted(logs)}")
if "DIRECT_CHILD_OK" not in render_jsonl(logs[child_ids[0]]):
raise AssertionError("first advanced child log has no direct-subagent result")
if "WORKFLOW_CHILD_OK" not in render_jsonl(logs[child_ids[1]]):
raise AssertionError("second advanced child log has no workflow-subagent result")
raise AssertionError(f"advanced snapshot expected parent plus {expected_children} child logs: {sorted(logs)}")
if mode == "native":
if "DIRECT_CHILD_OK" not in render_jsonl(logs[child_ids[0]]):
raise AssertionError("first advanced child log has no direct-subagent result")
if "WORKFLOW_CHILD_OK" not in render_jsonl(logs[child_ids[1]]):
raise AssertionError("second advanced child log has no workflow-subagent result")
files = build_snapshot_files(result, logs, child_ids, root)
compare_snapshot_files(
files, update_snapshots, ADVANCED_SNAPSHOT_DIRECTORY, ADVANCED_SNAPSHOT_FILENAMES,
files, update_snapshots, ADVANCED_SNAPSHOT_DIRECTORY if mode == "native" else ADVANCED_SNAPSHOT_ROOT / "advanced-ptc",
ADVANCED_SNAPSHOT_FILENAMES if mode == "native" else ("result.json", "session.v4.jsonl"),
native_writer_output=True,
)
@@ -2168,8 +2190,8 @@ def read_session_logs(sessions: Path) -> dict[str, list[dict[str, object]]]:
return logs
def snapshot_child_ids(result: "RunResult") -> list[str]:
"""Return the two child session ids in their SDK notification order."""
def snapshot_child_ids(result: "RunResult", expected_count: int) -> list[str]:
"""Return the expected child session ids in their SDK notification order."""
child_ids: list[str] = []
for notification in result.notifications:
if notification.method != "subagent.started":
@@ -2180,8 +2202,8 @@ def snapshot_child_ids(result: "RunResult") -> list[str]:
child_id = payload.get("childSessionId")
if isinstance(child_id, str) and child_id not in child_ids:
child_ids.append(child_id)
if len(child_ids) != 2:
raise AssertionError(f"advanced snapshot expected two child session ids: {child_ids}")
if len(child_ids) != expected_count:
raise AssertionError(f"advanced snapshot expected {expected_count} child session ids: {child_ids}")
return child_ids
@@ -2382,9 +2404,10 @@ def build_snapshot_files(
child_ids: list[str],
cwd: Path,
) -> dict[str, str]:
"""Render the SDK result and three persisted logs into stable expected outputs."""
"""Render the SDK result and its persisted parent/child logs into stable expected outputs."""
replacements = snapshot_directory_replacements(cwd, "{{cwd}}") + [(SNAPSHOT_SESSION_ID, "{{parent}}")]
replacements.append((snapshot_workflow_run_id(result), "{{workflow-run}}"))
if child_ids:
replacements.append((snapshot_workflow_run_id(result), "{{workflow-run}}"))
for index, child_id in enumerate(child_ids, start=1):
replacements.append((child_id, f"{{{{child-{index}}}}}"))
agent_id = snapshot_agent_id(result, child_id)