Merge origin/master into PR #3440
This commit is contained in:
commit
2025036158
2688 files changed
+57385
-17391
No files matched your search
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write python/development.md
|
||||
development.md: 39c47b7d2a86995eea017c77fb147c63f90412b1
|
||||
development.zh.md: fe7b4dbb2ca2e341dfb9ceb57959b7d9807f54d2
|
||||
development.md: 8e7608ada715e5a0acc6eb597ae9d06ebef3ecbf
|
||||
development.zh.md: 3f6c1f92632d216e540b0dff5c4b72cdf90ae7d1
|
||||
@@ -34,9 +34,9 @@ uv run --project python/sdk python scripts/smoke-python-runtime.py \
|
||||
--scenario sdk-minimal --exe dist-exe/deepseek-harness-sdk-runtime-macos-arm64
|
||||
```
|
||||
|
||||
Three scenarios compare committed expected output under `scripts/snapshots/python-sdk-single-exe/`. `minimal/model-visible.json` pins the Linux/macOS `sdk-minimal` profile's assembled system prompts, advertised tool schemas, and model-visible messages; `minimal/win-x64/model-visible.json` pins its PowerShell counterpart. A plugin that contributes an unintended system section or user message therefore fails the job, and every message the profile emits is compared. `advanced/` pins one complex process's SDK result and parent/child session logs across every target. `restart/` launches two complete SDK runtime processes against one persistence root and snapshots their isolated model histories, high-level results, and separate durable logs across every target. Rerun the owning scenario with `--update-snapshots` and review that diff before committing it.
|
||||
Four scenarios compare committed expected output under `scripts/snapshots/python-sdk-single-exe/`. `minimal/model-visible.json` pins the Linux/macOS `sdk-minimal` profile's assembled system prompts, advertised tool schemas, and model-visible messages; `minimal/win-x64/model-visible.json` pins its PowerShell counterpart. A plugin that contributes an unintended system section or user message therefore fails the job, and every message the profile emits is compared. `advanced/` pins one complex process's SDK result and parent/child session logs across every target. `restart/` launches two complete SDK runtime processes against one persistence root and snapshots their isolated model histories, high-level results, and separate durable logs across every target. `sdk-minimal-in-history` reuses the persistent-shell and editor scenario with a section that changes after the first successful shell call. `minimal-in-history/prompt-history.json` pins both prompt versions, the unchanged leading prompt in later requests, appended SDK system-message events, and `request/context.systemPromptUpdate`; tool schemas stay fixed and the editor file is checked independently. Rerun the owning scenario with `--update-snapshots` and review that diff before committing it.
|
||||
|
||||
Trusted pull requests and master pushes also run `--scenario sdk-live --installed-wheel` on each selected native target. That scenario performs two tool-using turns against `https://api.deepseek.com`, verifies the created file externally, and fails when the repository secret is absent instead of self-skipping. Fork and Dependabot pull requests run the complete keyless installed-wheel path but receive no key.
|
||||
Trusted pull requests and master pushes also run `--scenario sdk-live --installed-wheel` on each selected native target. That scenario performs two tool-using turns against `https://api.deepseek.com`: it checks the created file immediately, replaces its content with a host-only random challenge, and requires the second turn to copy the changed content into a fresh receipt without modifying the source. Both turns must complete with the exact sentinel answer and a model-requested tool call; external byte comparisons check the files. Missing repository secrets fail instead of self-skipping. Fork and Dependabot pull requests run the complete keyless installed-wheel path but receive no key.
|
||||
|
||||
An interactive smoke test needs `DEEPSEEK_API_KEY` in the environment or repository-root `.env`:
|
||||
|
||||
|
||||
@@ -34,9 +34,9 @@ uv run --project python/sdk python scripts/smoke-python-runtime.py \
|
||||
--scenario sdk-minimal --exe dist-exe/deepseek-harness-sdk-runtime-macos-arm64
|
||||
```
|
||||
|
||||
其中三个场景会比对 `scripts/snapshots/python-sdk-single-exe/` 下已提交的期望输出。`minimal/model-visible.json` 固定 Linux/macOS `sdk-minimal` profile 所组装的系统提示词、对外公布的工具 schema 与模型可见消息;`minimal/win-x64/model-visible.json` 固定对应的 PowerShell 版本。因此,插件一旦贡献出计划外的系统分段或 user 消息,该任务即失败,且该 profile 发出的每条消息都会参与比对。`advanced/` 跨所有目标固定一个复杂进程的 SDK 结果及父/子会话日志。`restart/` 针对同一持久化根目录启动两个完整 SDK 运行时进程,并跨所有目标固定其彼此隔离的模型历史、高层结果与独立持久日志。重新运行对应场景时加上 `--update-snapshots`,并在提交前审阅该差异。
|
||||
其中四个场景会比对 `scripts/snapshots/python-sdk-single-exe/` 下已提交的期望输出。`minimal/model-visible.json` 固定 Linux/macOS `sdk-minimal` profile 所组装的系统提示词、对外公布的工具 schema 与模型可见消息;`minimal/win-x64/model-visible.json` 固定对应的 PowerShell 版本。因此,插件一旦贡献出计划外的系统分段或 user 消息,该任务即失败,且该 profile 发出的每条消息都会参与比对。`advanced/` 跨所有目标固定一个复杂进程的 SDK 结果及父/子会话日志。`restart/` 针对同一持久化根目录启动两个完整 SDK 运行时进程,并跨所有目标固定其彼此隔离的模型历史、高层结果与独立持久日志。`sdk-minimal-in-history` 复用持久 shell 与编辑器场景,并在首次 shell 调用成功后更改一个分段。`minimal-in-history/prompt-history.json` 固定两个提示词版本、后续请求中不变的首条提示词、追加的 SDK system-message 事件及 `request/context.systemPromptUpdate`;工具 schema 保持不变,并独立检查编辑器创建的文件。重新运行对应场景时加上 `--update-snapshots`,并在提交前审阅该差异。
|
||||
|
||||
可信拉取请求与 master 推送还会在各自选定的原生目标上运行 `--scenario sdk-live --installed-wheel`。该场景面向 `https://api.deepseek.com` 执行两个使用工具的轮次,从外部验证已创建文件,并在仓库密钥缺失时失败而不是自行 skip。Fork 与 Dependabot 拉取请求会运行完整的 keyless 安装后 wheel 路径,但不会获得密钥。
|
||||
可信拉取请求与 master 推送还会在各自选定的原生目标上运行 `--scenario sdk-live --installed-wheel`。该场景面向 `https://api.deepseek.com` 执行两个使用工具的轮次:立即检查已创建文件,将其内容替换为仅宿主知道的随机挑战值,并要求第二轮将变更后的内容复制到全新的回执文件,且不修改源文件。两个轮次都必须完成、返回精确的哨兵答案并由模型请求调用工具;文件通过外部逐字节比较验证。仓库密钥缺失时失败,而不是自行 skip。Fork 与 Dependabot 拉取请求会运行完整的 keyless 安装后 wheel 路径,但不会获得密钥。
|
||||
|
||||
交互式冒烟测试需要环境变量或仓库根目录 `.env` 中存在 `DEEPSEEK_API_KEY`:
|
||||
|
||||
|
||||
@@ -103,6 +103,7 @@
|
||||
"@deepseek-ai/dsh-tool-bash-persistent": "workspace:^",
|
||||
"@deepseek-ai/dsh-tool-call-timeout-policy": "workspace:^",
|
||||
"@deepseek-ai/dsh-tool-cordis": "workspace:^",
|
||||
"@deepseek-ai/dsh-tool-present": "workspace:^",
|
||||
"@deepseek-ai/dsh-tool-fs": "workspace:^",
|
||||
"@deepseek-ai/dsh-tool-fs-search": "workspace:^",
|
||||
"@deepseek-ai/dsh-tool-goal": "workspace:^",
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write python/sdk/README.md
|
||||
README.md: 77ae34e24f5aec0c767cbe6c367d6d44eefbbfef
|
||||
README.zh.md: 41b343895aa3d03174098ad5f856cc65197f063b
|
||||
README.md: bfca3b0dda6c97bbef255815f33ca81121832f4e
|
||||
README.zh.md: 3824b6d35bcaa0bdec92584c53ce7036130d85c0
|
||||
@@ -59,7 +59,7 @@ with DeepSeekHarness(
|
||||
|
||||
`provider` selects a provider route registered by the chosen Cordis composition; `model` is the model id resolved by that adapter. `reasoning_effort` is an optional non-empty adapter-owned identifier for that exact route; omission preserves the model's own default. `max_tokens` is an optional positive per-request output-token cap for the root agent and its in-process descendants; omission leaves the provider default in control. Initialization rejects a missing adapter, unavailable model, or unsupported effort before a prompt runs. Compaction summaries keep the separate limit configured by their compaction plugin. The bundled default composition registers `deepseek-official`. A custom composition can mount `llm-pi-ai`, configure provider-specific credentials/endpoints there, and select any provider/model present in pi-ai's installed catalog.
|
||||
|
||||
The shipped `sdk-minimal` profile is a standalone explicit tree rather than an overlay on `dsh-base`. Select it with `profile="sdk-minimal"`; the ordinary `model` argument is the sole runtime model selection, including for model ids outside the adapter's advisory catalog. It provides persistent Bash, the string-replace editor, local execution, and JSONL sessions; settings, managed credentials, telemetry, Web tools, and the full default tool roster remain available through the separate full `sdk` and `web` profiles.
|
||||
The shipped `sdk-minimal` profile is a standalone explicit tree rather than an overlay on `dsh-base`. Select it with `profile="sdk-minimal"`; the ordinary `model` argument is the sole runtime model selection, including for model ids outside the adapter's advisory catalog. It provides only a platform-selected persistent shell, local execution, and JSONL sessions; filesystem tools, settings, managed credentials, telemetry, Web tools, and the full default tool roster remain available through the separate full `sdk` and `web` profiles.
|
||||
|
||||
## Results and notifications
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ with DeepSeekHarness(
|
||||
|
||||
`provider` 选择指定 Cordis 组合所注册的提供方路由;`model` 是该适配器解析出的模型 ID。`reasoning_effort` 是该确切路由可选的非空适配器自有标识符;省略时保留模型自身的默认值。`max_tokens` 是一个可选的正整数,用于限制根 agent 及其进程内后代在每次请求中输出的 token 数量;省略该参数时,由提供方的默认行为决定输出上限。缺少适配器、模型不可用或推理强度不受支持时,初始化会在提示词运行前拒绝。压缩摘要继续使用压缩插件单独配置的上限。内置默认组合注册 `deepseek-official`。自定义组合可以挂载 `llm-pi-ai`,在其中配置各提供方专属的凭据和端点,并选择 pi-ai 已安装 catalog 中存在的任意提供方/模型组合。
|
||||
|
||||
随附的 `sdk-minimal` profile 是独立显式配置树,而不是 `dsh-base` 上的 overlay。使用 `profile="sdk-minimal"` 选择它;普通 `model` 参数是唯一运行时模型选择,也适用于不在适配器建议目录中的模型 id。它提供持久 Bash、字符串替换 editor、本地执行与 JSONL 会话;settings、托管凭据、遥测、Web 工具与完整默认工具清单仍由独立的完整 `sdk` 与 `web` profile 提供。
|
||||
随附的 `sdk-minimal` profile 是独立显式配置树,而不是 `dsh-base` 上的 overlay。使用 `profile="sdk-minimal"` 选择它;普通 `model` 参数是唯一运行时模型选择,也适用于不在适配器建议目录中的模型 id。它只提供按平台选择的持久 shell、本地执行与 JSONL 会话;文件系统工具、settings、托管凭据、遥测、Web 工具与完整默认工具清单仍由独立的完整 `sdk` 与 `web` profile 提供。
|
||||
|
||||
## 结果与通知
|
||||
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write python/sdk/examples/README.md
|
||||
README.md: e79cb309318c1aa31d41bb50e52f5517b07ea485
|
||||
README.zh.md: 3208c5c1462a15a50a6990f5c989ef6b81da6ab4
|
||||
README.md: 82715c67cebe5d06decd3a6301cb1a8c245958c6
|
||||
README.zh.md: e241f9d7aa0174ce806e1d67b4183b9824a2f72b
|
||||
@@ -22,11 +22,10 @@ Set `DEEPSEEK_BASE_URL` for a compatible proxy, `DSH_MODEL` for the script's def
|
||||
The shipped [`@deepseek-ai/dsh-sdk-minimal` bundle](../../../packages/bundle/sdk-minimal/README.md) is the complete explicit Cordis tree for this mode. It exposes exactly:
|
||||
|
||||
- owner-scoped persistent `bash` on Linux/macOS or `pwsh` on Windows
|
||||
- `str_replace_editor` with `view`, `create`, `str_replace`, and `insert`
|
||||
|
||||
The bundle does not include `dsh-base`, so every additional row is an explicit profile change. Runtime context, local instruction discovery, compaction, settings, managed credentials, telemetry, Web tools, subagents, and the full default tool roster are absent. The tree retains SDK startup and JSON-RPC serving, one environment-configured DeepSeek adapter, local execution, and JSONL persistence.
|
||||
The bundle does not include `dsh-base`, so every additional row is an explicit profile change. Runtime context, filesystem tools, local instruction discovery, compaction, settings, managed credentials, telemetry, Web tools, subagents, and the full default tool roster are absent. The tree retains SDK startup and JSON-RPC serving, one environment-configured DeepSeek adapter, local execution, and JSONL persistence.
|
||||
|
||||
The persistent PTY and editor can modify any path available to the runtime process, so use a disposable checkout or container.
|
||||
The persistent PTY can modify any path available to the runtime process, so use a disposable checkout or container.
|
||||
|
||||
## Add plugins
|
||||
|
||||
|
||||
@@ -22,11 +22,10 @@ python python/sdk/examples/minimal.py \
|
||||
随附的 [`@deepseek-ai/dsh-sdk-minimal` 组合包](../../../packages/bundle/sdk-minimal/README.zh.md)是该模式完整且显式的 Cordis 配置树。它只暴露:
|
||||
|
||||
- Linux/macOS 上 agent 所有的持久 `bash`,或 Windows 上的 `pwsh`
|
||||
- 支持 `view`、`create`、`str_replace` 与 `insert` 的 `str_replace_editor`
|
||||
|
||||
该组合包不包含 `dsh-base`,因此每一个新增配置项都是显式 profile 变更。运行时上下文、本地指令发现、compaction、settings、托管凭据、遥测、Web 工具、subagent 与完整默认工具清单均不存在。配置树保留 SDK 启动与 JSON-RPC 服务、一个由环境配置的 DeepSeek 适配器、本地执行和 JSONL 持久化。
|
||||
该组合包不包含 `dsh-base`,因此每一个新增配置项都是显式 profile 变更。运行时上下文、文件系统工具、本地指令发现、compaction、settings、托管凭据、遥测、Web 工具、subagent 与完整默认工具清单均不存在。配置树保留 SDK 启动与 JSON-RPC 服务、一个由环境配置的 DeepSeek 适配器、本地执行和 JSONL 持久化。
|
||||
|
||||
持久 PTY 与 editor 可以修改运行时进程可访问的任何路径,因此只应在一次性 checkout 或容器中使用。
|
||||
持久 PTY 可以修改运行时进程可访问的任何路径,因此只应在一次性 checkout 或容器中使用。
|
||||
|
||||
## 添加插件
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import runpy
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
@@ -7,91 +8,156 @@ from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from deepseek_harness import RunResult
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SMOKE = runpy.run_path(ROOT / "scripts" / "smoke-python-runtime.py")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("behavior", "error"),
|
||||
[
|
||||
("read-current", None),
|
||||
("no-verify-tool", "verify turn made no model-requested tool call"),
|
||||
("no-create-tool", "create turn made no model-requested tool call"),
|
||||
("create-error", "create turn ended with.*AUTH.*401"),
|
||||
("stale-answer", "verify turn returned"),
|
||||
("missing-create", "real-model tool turn did not create"),
|
||||
("wrong-create", "real-model tool turn wrote unexpected text"),
|
||||
("modify-verify", "real-model tool turn wrote unexpected text"),
|
||||
],
|
||||
)
|
||||
def test_live_smoke_requires_fresh_file_observation(
|
||||
monkeypatch: pytest.MonkeyPatch, behavior: str, error: str | None,
|
||||
) -> None:
|
||||
def live_result(**overrides: object) -> RunResult:
|
||||
values = {
|
||||
"session_id": "installed-wheel-live-api",
|
||||
"final_response": SMOKE["LIVE_API_SENTINEL"],
|
||||
"finish_reason": "completed",
|
||||
"events": [{"type": "tool/call", "data": {"name": "unrelated_tool"}}],
|
||||
"notifications": [],
|
||||
}
|
||||
values.update(overrides)
|
||||
return RunResult(**values)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_smoke(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
|
||||
import deepseek_harness
|
||||
|
||||
smoke_live = SMOKE["smoke_sdk_live"]
|
||||
sentinel = SMOKE["LIVE_API_SENTINEL"]
|
||||
prompts: list[str] = []
|
||||
session_ids: list[str] = []
|
||||
log_checks: list[Path] = []
|
||||
state = SimpleNamespace(
|
||||
prompts=[], session_ids=[], challenges=[], checked_logs=[], closed=False,
|
||||
create_bytes=SMOKE["LIVE_API_SENTINEL"].encode("utf-8"),
|
||||
create_result=live_result(), verify_result=live_result(), receipt_mode="copy",
|
||||
)
|
||||
globals_ = SMOKE["smoke_sdk_live"].__globals__
|
||||
token_hex = globals_["secrets"].token_hex
|
||||
|
||||
def fresh_challenge(size: int) -> str:
|
||||
assert len(state.prompts) == 1
|
||||
value = token_hex(size)
|
||||
state.challenges.append(value)
|
||||
return value
|
||||
|
||||
class ScriptedHarness:
|
||||
def __init__(self, *, cwd: str, **_kwargs: object) -> None:
|
||||
self.marker = Path(cwd) / "live-api-marker.txt"
|
||||
def __init__(self, **kwargs: object) -> None:
|
||||
state.root = Path(kwargs["cwd"])
|
||||
assert "toolChoice" not in kwargs
|
||||
|
||||
def __enter__(self) -> ScriptedHarness:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
pass
|
||||
def __exit__(self, *args: object) -> None:
|
||||
state.closed = True
|
||||
|
||||
def run(self, prompt: str, *, session_id: str) -> SimpleNamespace:
|
||||
prompts.append(prompt)
|
||||
session_ids.append(session_id)
|
||||
events = [{"type": "tool/call"}]
|
||||
if len(prompts) == 1:
|
||||
assert not self.marker.exists()
|
||||
if behavior == "create-error":
|
||||
return SimpleNamespace(finish_reason="error", final_response="", events=[{
|
||||
"type": "turn/end",
|
||||
"data": {"turn": 1, "reason": {
|
||||
"kind": "error", "error": {"code": "AUTH", "status": 401},
|
||||
}},
|
||||
}])
|
||||
if behavior == "no-create-tool":
|
||||
events = []
|
||||
if behavior != "missing-create":
|
||||
self.marker.write_text(
|
||||
("wrong" if behavior == "wrong-create" else sentinel) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
response = sentinel
|
||||
else:
|
||||
current = self.marker.read_text(encoding="utf-8").strip()
|
||||
assert current != sentinel, "verification must require new world state"
|
||||
assert all(current not in text for text in prompts), "prompts must not reveal the answer"
|
||||
response = sentinel if behavior == "stale-answer" else current
|
||||
if behavior == "no-verify-tool":
|
||||
events = []
|
||||
if behavior == "modify-verify":
|
||||
self.marker.write_text("changed\n", encoding="utf-8")
|
||||
return SimpleNamespace(finish_reason="completed", final_response=response, events=events)
|
||||
def run(self, prompt: str, *, session_id: str) -> RunResult:
|
||||
state.prompts.append(prompt)
|
||||
state.session_ids.append(session_id)
|
||||
if len(state.prompts) == 1:
|
||||
state.marker = Path(prompt.splitlines()[-1])
|
||||
assert state.marker.parent == state.root
|
||||
if state.create_bytes is not None:
|
||||
state.marker.write_bytes(state.create_bytes)
|
||||
return state.create_result
|
||||
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "fixture-key")
|
||||
monkeypatch.setenv("DEEPSEEK_BASE_URL", "https://fixture.invalid")
|
||||
assert len(state.prompts) == 2
|
||||
assert len(state.challenges) == 1
|
||||
challenge = state.challenges[0]
|
||||
assert all(challenge not in sent for sent in state.prompts)
|
||||
assert str(state.marker) not in prompt
|
||||
assert "previous turn" in prompt and "changed externally" in prompt
|
||||
assert state.marker.read_bytes() == challenge.encode("ascii")
|
||||
receipt = Path(prompt.splitlines()[-1])
|
||||
assert receipt != state.marker and not receipt.exists()
|
||||
if state.receipt_mode == "copy":
|
||||
receipt.write_bytes(state.marker.read_bytes())
|
||||
elif state.receipt_mode == "stale":
|
||||
receipt.write_bytes(state.create_bytes)
|
||||
elif state.receipt_mode == "wrong":
|
||||
receipt.write_bytes(b"wrong")
|
||||
elif state.receipt_mode == "newline":
|
||||
receipt.write_bytes(state.marker.read_bytes() + b"\n")
|
||||
elif state.receipt_mode == "changed-source":
|
||||
receipt.write_bytes(state.marker.read_bytes())
|
||||
state.marker.write_bytes(b"changed")
|
||||
elif state.receipt_mode != "missing":
|
||||
raise AssertionError(state.receipt_mode)
|
||||
return state.verify_result
|
||||
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "unit-test-key")
|
||||
monkeypatch.setenv("DEEPSEEK_BASE_URL", "https://api.invalid")
|
||||
monkeypatch.setattr(deepseek_harness, "DeepSeekHarness", ScriptedHarness)
|
||||
monkeypatch.setitem(smoke_live.__globals__, "assert_zstd_session_log", log_checks.append)
|
||||
monkeypatch.setattr(globals_["secrets"], "token_hex", fresh_challenge)
|
||||
monkeypatch.setitem(globals_, "assert_zstd_session_log", state.checked_logs.append)
|
||||
return state
|
||||
|
||||
if error is None:
|
||||
smoke_live()
|
||||
assert len(prompts) == 2
|
||||
assert session_ids[0] == session_ids[1]
|
||||
assert len(log_checks) == 1
|
||||
else:
|
||||
with pytest.raises(AssertionError, match=error):
|
||||
smoke_live()
|
||||
assert not log_checks
|
||||
|
||||
def test_live_smoke_requires_fresh_external_content(live_smoke: SimpleNamespace) -> None:
|
||||
SMOKE["smoke_sdk_live"]()
|
||||
assert len(live_smoke.prompts) == 2
|
||||
assert live_smoke.session_ids == ["installed-wheel-live-api"] * 2
|
||||
assert len(live_smoke.checked_logs) == 1
|
||||
assert live_smoke.closed and not live_smoke.root.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("label", ["create", "verify"])
|
||||
@pytest.mark.parametrize(("overrides", "message"), [
|
||||
({"finish_reason": "error"}, "turn ended with 'error'"),
|
||||
({"finish_reason": "error", "events": [{
|
||||
"type": "turn/end", "data": {"turn": 1, "reason": {
|
||||
"kind": "error", "error": {"code": "AUTH", "status": 401},
|
||||
}},
|
||||
}]}, "turn ended with.*AUTH.*401"),
|
||||
({"events": []}, "turn made no model-requested tool call"),
|
||||
({"final_response": "PYTHON_SDK_LIVE_OK extra"}, "turn returned"),
|
||||
])
|
||||
def test_live_smoke_rejects_invalid_turn_before_continuing(
|
||||
live_smoke: SimpleNamespace, label: str, overrides: dict[str, object], message: str,
|
||||
) -> None:
|
||||
setattr(live_smoke, f"{label}_result", live_result(**overrides))
|
||||
with pytest.raises(AssertionError, match=f"{label} {message}"):
|
||||
SMOKE["smoke_sdk_live"]()
|
||||
assert len(live_smoke.prompts) == (1 if label == "create" else 2)
|
||||
assert not live_smoke.checked_logs
|
||||
assert live_smoke.closed
|
||||
if label == "create":
|
||||
assert not live_smoke.challenges
|
||||
|
||||
|
||||
@pytest.mark.parametrize("content", [None, b"wrong", b"PYTHON_SDK_LIVE_OK\n"])
|
||||
def test_live_smoke_rejects_bad_create_before_host_overwrite(
|
||||
live_smoke: SimpleNamespace, content: bytes | None,
|
||||
) -> None:
|
||||
live_smoke.create_bytes = content
|
||||
with pytest.raises(AssertionError, match="create turn (did not create|wrote unexpected bytes)"):
|
||||
SMOKE["smoke_sdk_live"]()
|
||||
assert len(live_smoke.prompts) == 1
|
||||
assert not live_smoke.challenges and not live_smoke.checked_logs
|
||||
assert live_smoke.closed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("mode", "message"), [
|
||||
("missing", "did not create receipt"),
|
||||
("stale", "wrote unexpected bytes to receipt"),
|
||||
("wrong", "wrote unexpected bytes to receipt"),
|
||||
("newline", "wrote unexpected bytes to receipt"),
|
||||
("changed-source", "changed source file"),
|
||||
])
|
||||
def test_live_smoke_rejects_unrelated_tool_without_exact_receipt(
|
||||
live_smoke: SimpleNamespace, mode: str, message: str,
|
||||
) -> None:
|
||||
live_smoke.receipt_mode = mode
|
||||
with pytest.raises(AssertionError, match=f"verify turn {message}"):
|
||||
SMOKE["smoke_sdk_live"]()
|
||||
assert len(live_smoke.prompts) == 2
|
||||
assert not live_smoke.checked_logs
|
||||
assert live_smoke.closed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -157,7 +223,28 @@ def test_mcp_smoke_accepts_the_external_server_result() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_snapshot_comparison_normalizes_only_session_generation_provenance() -> None:
|
||||
def test_advanced_snapshot_normalizes_catalog_child_creation_time() -> None:
|
||||
value = {
|
||||
"type": "subagent/catalog",
|
||||
"data": {
|
||||
"childCreatedAt": 1788246207176,
|
||||
},
|
||||
}
|
||||
|
||||
assert SMOKE["normalize_snapshot_value"](value, []) == {
|
||||
"type": "subagent/catalog",
|
||||
"data": {
|
||||
"childCreatedAt": 0,
|
||||
},
|
||||
}
|
||||
|
||||
assert SMOKE["normalize_snapshot_value"](
|
||||
{"type": "fixture/event", "data": {"childCreatedAt": 1788246207176}},
|
||||
[],
|
||||
) == {"type": "fixture/event", "data": {"childCreatedAt": 1788246207176}}
|
||||
|
||||
|
||||
def test_snapshot_comparison_preserves_opaque_generation_provenance() -> None:
|
||||
normalize = SMOKE["normalize_session_format_comparison"]
|
||||
expected = {
|
||||
"header": {"type": "session", "version": 0, "otherVersion": 7},
|
||||
@@ -186,8 +273,24 @@ def test_snapshot_comparison_normalizes_only_session_generation_provenance() ->
|
||||
},
|
||||
}
|
||||
|
||||
assert normalize(expected) == normalize(actual)
|
||||
assert normalize(expected) != normalize(actual)
|
||||
assert normalize(expected)["header"] == normalize(actual)["header"]
|
||||
assert normalize(expected)["header"]["otherVersion"] == 7
|
||||
assert normalize(actual)["accepted"] == actual["accepted"]
|
||||
assert normalize(actual)["source"] == actual["source"]
|
||||
|
||||
|
||||
def test_snapshot_value_scrubs_system_nodes_without_erasing_header_fields() -> None:
|
||||
normalize = SMOKE["normalize_snapshot_value"]
|
||||
system = {
|
||||
"type": "system/message",
|
||||
"data": {"message": {"role": "system", "content": [{"type": "text", "text": "prompt"}]}},
|
||||
}
|
||||
header = {"type": "request/header", "data": {"header": {"system": "unexpected"}}}
|
||||
assert normalize(system, [])["data"]["message"]["content"] == [{"type": "text", "text": "{{system}}"}]
|
||||
assert normalize(header, []) == header
|
||||
empty = {"type": "system/message", "data": {"message": {"role": "system", "content": []}}}
|
||||
assert normalize(empty, []) == empty
|
||||
|
||||
|
||||
def test_snapshot_value_normalizes_embedded_assistant_stream_timing() -> None:
|
||||
@@ -339,6 +442,74 @@ def test_snapshot_generation_names_select_highest_role_without_double_counting(
|
||||
}
|
||||
|
||||
|
||||
def test_snapshot_comparison_accepts_v3_output_against_v2_without_rewriting(tmp_path: Path) -> None:
|
||||
predecessor = '{"type":"session","version":2}\n'
|
||||
successor = '{"type":"session","version":3}\n'
|
||||
old_path = tmp_path / "session.v2.jsonl"
|
||||
old_path.write_text(predecessor, encoding="utf-8")
|
||||
files = {"session.v3.jsonl": successor}
|
||||
|
||||
SMOKE["compare_snapshot_files"](files, False, tmp_path, ("session.v2.jsonl",))
|
||||
assert old_path.read_text(encoding="utf-8") == predecessor
|
||||
assert not (tmp_path / "session.v3.jsonl").exists()
|
||||
|
||||
SMOKE["compare_snapshot_files"](files, True, tmp_path, ("session.v2.jsonl",))
|
||||
assert old_path.read_text(encoding="utf-8") == predecessor
|
||||
assert (tmp_path / "session.v3.jsonl").read_text(encoding="utf-8") == successor
|
||||
assert SMOKE["selected_snapshot_session_files"](tmp_path) == {0: tmp_path / "session.v3.jsonl"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("filenames", [
|
||||
("session.1.v2.jsonl", "session.v2.jsonl"),
|
||||
("session.v2.jsonl",),
|
||||
("session.v2.jsonl", "session.2.v2.jsonl"),
|
||||
])
|
||||
def test_snapshot_builder_checks_role_order_and_count_across_generations(
|
||||
tmp_path: Path, filenames: tuple[str, ...],
|
||||
) -> None:
|
||||
files = {"session.v3.jsonl": "", "session.1.v3.jsonl": ""}
|
||||
with pytest.raises(AssertionError, match="snapshot builder produced"):
|
||||
SMOKE["compare_snapshot_files"](files, False, tmp_path, filenames)
|
||||
|
||||
|
||||
def test_snapshot_generation_comparison_rejects_changed_payload(tmp_path: Path) -> None:
|
||||
(tmp_path / "session.v2.jsonl").write_text(
|
||||
'{"type":"session","version":2,"id":"expected"}\n', encoding="utf-8",
|
||||
)
|
||||
with pytest.raises(AssertionError, match="executable snapshot mismatch"):
|
||||
SMOKE["compare_snapshot_files"](
|
||||
{"session.v3.jsonl": '{"type":"session","version":3,"id":"changed"}\n'},
|
||||
False, tmp_path, ("session.v2.jsonl",),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [2, 4])
|
||||
@pytest.mark.parametrize("update", [False, True])
|
||||
def test_snapshot_comparison_rejects_noncurrent_writer(
|
||||
tmp_path: Path, version: int, update: bool,
|
||||
) -> None:
|
||||
golden = '{"type":"session","version":2}\n'
|
||||
(tmp_path / "session.v2.jsonl").write_text(golden, encoding="utf-8")
|
||||
content = json.dumps({"type": "session", "version": version}) + "\n"
|
||||
with pytest.raises(AssertionError, match="expected current Session format v3"):
|
||||
SMOKE["compare_snapshot_files"](
|
||||
{f"session.v{version}.jsonl": content}, update, tmp_path, ("session.v2.jsonl",),
|
||||
)
|
||||
assert (tmp_path / "session.v2.jsonl").read_text(encoding="utf-8") == golden
|
||||
assert not (tmp_path / "session.v4.jsonl").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [2, 3, 4])
|
||||
def test_persisted_session_requires_current_writer(version: int) -> None:
|
||||
content = json.dumps({"type": "session", "version": version}) + "\n"
|
||||
path = Path(f"session.v{version}.jsonl")
|
||||
if version == 3:
|
||||
assert SMOKE["assert_persisted_session_version"](path, content) == version
|
||||
else:
|
||||
with pytest.raises(AssertionError, match="expected current Session format v3"):
|
||||
SMOKE["assert_persisted_session_version"](path, content)
|
||||
|
||||
|
||||
def test_snapshot_generation_filename_must_match_header(tmp_path: Path) -> None:
|
||||
(tmp_path / "session.v1.jsonl").write_text(
|
||||
'{"type":"session","version":0}\n', encoding="utf-8",
|
||||
|
||||
Reference in new issue
Block a user