mward4/deepseek-harness
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 31 Wiki Activity

Merge master and integrate image offload with both DeepSeek protocols

Browse Source
This commit is contained in:
creatixchu committed 2026-09-14 10:19:11 +08:00
2 parents 93581008aa 9ccd9b0634
commit a7070008e7
2052 files changed
+727131 -33615
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/smoke-python-runtime.py
+116 -55
View File
@@ -101,6 +101,17 @@ RESTART_FIRST_SESSION_ID = "process-one"
RESTART_SECOND_SESSION_ID = "process-two"
SNAPSHOT_PLUGIN_CODE = """\
return (ctx) => {
ctx.on('tools/pre-execute', (exec, next) => {
if (exec.name !== 'snapshot_double' || exec.arguments.value !== -1) return next()
return {
kind: 'deny',
reason: 'Auto review rejected tool "snapshot_double"; its body was not executed',
info: {
name: 'AutoReviewDeniedError', code: 'AUTO_REVIEW_DENIED',
reason: ' transport raw\\r\\nreason ',
},
}
})
harness.registerTool(ctx, harness.defineTool({
name: 'snapshot_double',
description: 'Double a number for executable snapshot verification.',
@@ -264,7 +275,7 @@ def write_advanced_profile_patch(root: Path, name: str, sessions: Path) -> Path:
},
},
{"insert": [
{"id": "code-runtime", "name": "@deepseek-ai/dsh-code-runtime-worker-thread"},
{"id": "ptc-runtime", "name": "@deepseek-ai/dsh-ptc-runtime-node"},
{"id": "cordis-host-runner", "name": "@deepseek-ai/dsh-cordis-host-runner"},
{"id": "cordis-tool", "name": "@deepseek-ai/dsh-tool-cordis"},
]},
@@ -296,6 +307,9 @@ class MockModelHandler(BaseHTTPRequestHandler):
requests: list[dict[str, object]] = []
def do_POST(self) -> None:
if self.path != "/v1/messages":
self.send_error(404)
return
content_length = int(self.headers.get("content-length", "0"))
body = json.loads(self.rfile.read(content_length))
self.requests.append(body)
@@ -304,8 +318,7 @@ class MockModelHandler(BaseHTTPRequestHandler):
self.end_headers()
chunks = completion_chunks(body)
for chunk in chunks:
self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.write(f"event: {chunk['type']}\ndata: {json.dumps(chunk)}\n\n".encode())
self.wfile.flush()
def log_message(self, _format: str, *_args: object) -> None:
@@ -322,9 +335,14 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
if not isinstance(latest, dict):
raise AssertionError(f"model request has an invalid latest message: {body}")
if latest.get("role") == "tool":
call_id, tool_name = latest_tool_call(messages)
tool_text = message_text(latest.get("content"))
tool_results = [
block for block in latest.get("content", [])
if isinstance(block, dict) and block.get("type") == "tool_result"
]
if tool_results:
result = tool_results[-1]
call_id, tool_name = latest_tool_call(messages, result.get("tool_use_id"))
tool_text = message_text(result.get("content"))
mcp = mcp_tool_followup(call_id, tool_name, tool_text)
if mcp is not None:
return mcp
@@ -349,9 +367,11 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
raise AssertionError(f"unexpected tool follow-up: {tool_name}")
user_prompts = [
message_text(message.get("content"))
block["text"]
for message in reversed(messages)
if isinstance(message, dict) and message.get("role") == "user"
for block in message.get("content", [])
if isinstance(block, dict) and block.get("type") == "text"
]
minimal_prompt = next((prompt for prompt in user_prompts if prompt == MINIMAL_PROMPT), None)
# The minimal composition's assembled system prompt, advertised tool schemas, and
@@ -448,7 +468,7 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
{"a": 19, "b": 23},
)
if prompt == PROFILE_PLUGIN_PROMPT:
system_text = "\n".join(
system_text = message_text(body.get("system")) + "\n" + "\n".join(
message_text(message.get("content"))
for message in messages
if isinstance(message, dict) and message.get("role") == "system"
@@ -585,6 +605,21 @@ def advanced_tool_followup(
if call_id == "advanced-code" and tool_name == "run_code":
if "42" not in tool_text:
raise AssertionError(f"run_code returned no dynamic-tool value: {tool_text}")
return tool_call_chunks("advanced-denied-native", "snapshot_double", {"value": -1})
if call_id == "advanced-denied-native" and tool_name == "snapshot_double":
if 'Auto review rejected tool "snapshot_double"; its body was not executed' not in tool_text:
raise AssertionError(f"native denial did not preserve the model result: {tool_text}")
if "transport raw" in tool_text:
raise AssertionError("native denial leaked its user-facing reason to the model")
return tool_call_chunks("advanced-denied-ptc", "run_code", {
"code": "try { await tools.snapshot_double({ value: -1 }) } catch (error) { return error.message }",
"description": "Catch a structured inner tool denial",
})
if call_id == "advanced-denied-ptc" and tool_name == "run_code":
if 'Auto review rejected tool "snapshot_double"; its body was not executed' not in tool_text:
raise AssertionError(f"PTC denial did not preserve the model result: {tool_text}")
if "transport raw" in tool_text:
raise AssertionError("PTC denial leaked its user-facing reason to the model")
assert_advertised_tool(body, "subagent")
return tool_call_chunks(
"advanced-direct-child",
@@ -628,64 +663,67 @@ def advanced_tool_followup(
def text_chunks(text: str) -> list[dict[str, object]]:
"""Build a complete streaming text response."""
"""Build a complete Messages text response."""
return [
{"choices": [{"delta": {"role": "assistant", "content": None, "reasoning_content": ""}}]},
{"choices": [{"delta": {"content": text}}]},
{
"choices": [{"delta": {"content": ""}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 3},
},
message_start(),
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 3}},
{"type": "message_stop"},
]
def tool_call_chunks(call_id: str, name: str, arguments: dict[str, object]) -> list[dict[str, object]]:
"""Build a complete streaming function-call response."""
"""Build a complete Messages tool-use response."""
return [
{"choices": [{"delta": {"role": "assistant", "content": None, "reasoning_content": ""}}]},
message_start(),
{
"choices": [{
"delta": {
"tool_calls": [{
"index": 0,
"id": call_id,
"type": "function",
"function": {"name": name, "arguments": json.dumps(arguments)},
}],
},
}],
"type": "content_block_start", "index": 0,
"content_block": {"type": "tool_use", "id": call_id, "name": name, "input": {}},
},
{
"choices": [{"delta": {"content": ""}, "finish_reason": "tool_calls"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 3},
"type": "content_block_delta", "index": 0,
"delta": {"type": "input_json_delta", "partial_json": json.dumps(arguments)},
},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 3}},
{"type": "message_stop"},
]
def latest_tool_call(messages: list[object]) -> tuple[str, str]:
def message_start() -> dict[str, object]:
"""Start a Messages response with deterministic token usage."""
return {
"type": "message_start",
"message": {"id": "msg_smoke", "model": "smoke-model", "usage": {"input_tokens": 3, "output_tokens": 0}},
}
def latest_tool_call(messages: list[object], result_id: object) -> tuple[str, str]:
"""Find the assistant call id and name paired with the latest tool result."""
for message in reversed(messages[:-1]):
if not isinstance(message, dict):
continue
calls = message.get("tool_calls")
calls = message.get("content")
if not isinstance(calls, list):
continue
for call in reversed(calls):
if not isinstance(call, dict):
continue
function = call.get("function")
call_id = call.get("id")
if (
isinstance(call_id, str)
and isinstance(function, dict)
and isinstance(function.get("name"), str)
and call_id == result_id
and call.get("type") == "tool_use"
and isinstance(call.get("name"), str)
):
return call_id, function["name"]
return call_id, call["name"]
raise AssertionError(f"tool result has no preceding assistant tool call: {messages}")
def message_text(content: object) -> str:
"""Read OpenAI text content in either string or block-list form."""
"""Read Messages text content in either string or block-list form."""
if isinstance(content, str):
return content
if isinstance(content, list):
@@ -706,9 +744,8 @@ def advertised_tool_names(body: dict[str, object]) -> set[str]:
for tool in tools:
if not isinstance(tool, dict):
continue
function = tool.get("function")
if isinstance(function, dict) and isinstance(function.get("name"), str):
names.add(function["name"])
if isinstance(tool.get("name"), str):
names.add(tool["name"])
return names
@@ -1192,6 +1229,7 @@ def smoke_sdk_mcp(base_url: str, executable: Path | None) -> None:
assert result.final_response == MCP_TEXT, result.final_response
assert discovery_log.read_text().splitlines() == [
"server/discover",
"initialize",
"notifications/initialized",
"tools/list",
@@ -1339,13 +1377,21 @@ def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool)
raise AssertionError(f"advanced snapshot emitted unexpected subagent lifecycle: {methods}")
ptc_events = [event for event in result.events
if event.get("type") in ("tool/ptc-dispatch-start", "tool/ptc-dispatch")]
if [event["type"] for event in ptc_events] != ["tool/ptc-dispatch-start", "tool/ptc-dispatch"]:
if [event["type"] for event in ptc_events] != ["tool/ptc-dispatch-start", "tool/ptc-dispatch"] * 2:
raise AssertionError(f"advanced snapshot emitted unexpected PTC dispatch events: {ptc_events}")
for event in ptc_events:
for index, event in enumerate(ptc_events):
data = event["data"]
identity = (data.get("rootCallId"), data.get("parentCallId"), data.get("subCallId"))
if identity != ("advanced-code", "advanced-code", "advanced-code:ptc:1"):
root_call = "advanced-code" if index < 2 else "advanced-denied-ptc"
if identity != (root_call, root_call, root_call + ":ptc:1"):
raise AssertionError(f"advanced snapshot emitted unexpected PTC dispatch identity: {identity}")
if {"description", "parameters", "schema"}.intersection(data):
raise AssertionError("PTC binding schema entered the packaged SDK wire")
errors = [event["data"]["error"] for event in result.events
if event.get("type") in ("tool/result", "tool/ptc-dispatch") and "error" in event["data"]]
assert errors == [{
"name": "AutoReviewDeniedError", "code": "AUTO_REVIEW_DENIED", "reason": " transport raw\r\nreason ",
}] * 2, errors
logs = read_session_logs(sessions)
child_ids = snapshot_child_ids(result)
@@ -1852,13 +1898,18 @@ def build_in_history_snapshot_files(
request_prompts = []
for index, request in enumerate(requests):
messages = request["messages"]
assert messages[0]["role"] == "system" and message_text(messages[0]["content"]) == prompts[0]
assert message_text(request.get("system")) == prompts[0]
assert request["tools"] == requests[0]["tools"], "prompt update changed tool schemas"
positions = [position for position, message in enumerate(messages) if message["role"] == "system"]
texts = [message_text(messages[position]["content"]) for position in positions]
texts = [message_text(request["system"]), *[
message_text(messages[position]["content"]) for position in positions
]]
assert texts == (prompts[:1] if index == 0 else prompts), texts
if index > 0:
assert messages[positions[1] - 1]["role"] == "tool", messages
previous = messages[positions[0] - 1]
assert previous["role"] == "user" and any(
block.get("type") == "tool_result" for block in previous["content"]
), messages
request_prompts.append(texts)
evidence = {
"requestSystemPrompts": request_prompts,
@@ -1888,6 +1939,7 @@ def build_minimal_snapshot_files(
if not isinstance(messages, list):
raise AssertionError(f"minimal model request has no messages: {body}")
snapshot.append({
"system": minimal_snapshot_text(body.get("system"), cwd),
"tools": minimal_snapshot_text(body.get("tools"), cwd),
"messages": [
minimal_snapshot_message(message, cwd)
@@ -1902,22 +1954,30 @@ def minimal_snapshot_message(message: object, cwd: Path) -> dict[str, object]:
if not isinstance(message, dict):
raise AssertionError(f"minimal model request has an invalid message: {message}")
role = message.get("role")
if role in ("system", "user"):
if role == "system":
return {"role": role, "text": minimal_snapshot_text(message_text(message.get("content")), cwd)}
if role == "user":
content = []
for block in message.get("content", []):
if block.get("type") == "tool_result":
content.append({"type": "tool_result", "tool_use_id": block.get("tool_use_id"), "content": "{{tool-result}}"})
elif block.get("type") == "text":
content.append(minimal_snapshot_text(block, cwd))
else:
raise AssertionError(f"minimal user message has unexpected content: {block}")
return {"role": role, "content": content}
if role == "assistant":
calls = message.get("tool_calls")
calls = message.get("content")
if not isinstance(calls, list):
raise AssertionError(f"minimal assistant message has no tool calls: {message}")
return {
"role": role,
"toolCalls": [
{"id": call.get("id"), "name": (call.get("function") or {}).get("name")}
{"id": call.get("id"), "name": call.get("name")}
for call in calls
if isinstance(call, dict)
if isinstance(call, dict) and call.get("type") == "tool_use"
],
}
if role == "tool":
return {"role": role, "toolCallId": message.get("tool_call_id"), "text": "{{tool-result}}"}
raise AssertionError(f"minimal model request has an unexpected message role: {message}")
@@ -2035,6 +2095,7 @@ def build_restart_snapshot_files(
request_value = [
{
"model": request.get("model"),
"system": "{{system}}" if request.get("system") else None,
"messages": restart_request_messages(request),
"toolNames": sorted(advertised_tool_names(request)),
}
@@ -2213,7 +2274,7 @@ def project_session_snapshot(records: list[dict[str, object]]) -> list[dict[str,
return projected
SESSION_FORMAT_PROVENANCE = "{{sessionFormatVersion}}"
SESSION_FORMAT_TOKEN = "{{sessionFormatVersion}}"
def expand_snapshot_stream_member(member: object) -> list[dict[str, object]]:
@@ -2302,7 +2363,7 @@ def normalize_session_format_comparison(
value: object,
source_session_version: int | None = None,
) -> object:
"""Canonicalize only generation provenance that differs across immutable Session files."""
"""Canonicalize only generation metadata that differs across immutable Session files."""
if isinstance(value, list):
return [
normalize_session_format_comparison(expanded, source_session_version)
@@ -2317,7 +2378,7 @@ def normalize_session_format_comparison(
for key, item in value.items()
}
if normalized.get("type") == "session" and "version" in normalized:
normalized["version"] = SESSION_FORMAT_PROVENANCE
normalized["version"] = SESSION_FORMAT_TOKEN
normalized.setdefault("isSeeded", False)
ordered_header = {
key: normalized[key]
@@ -2337,7 +2398,7 @@ def normalize_session_format_comparison(
def normalize_snapshot_comparison_text(name: str, content: str) -> str:
"""Normalize Session generation provenance only while comparing committed expected outputs."""
"""Normalize Session generation metadata only while comparing committed expected outputs."""
if name.startswith("session") and name.endswith(".jsonl"):
parsed = [json.loads(line) for line in content.splitlines() if line]
header = parsed[0] if parsed else None