diff --git a/python/development.i18n.yaml b/python/development.i18n.yaml index 097ae91c3c..f568fedc82 100644 --- a/python/development.i18n.yaml +++ b/python/development.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write python/development.md -development.md: 8e7608ada715e5a0acc6eb597ae9d06ebef3ecbf -development.zh.md: 3f6c1f92632d216e540b0dff5c4b72cdf90ae7d1 +development.md: b36a26b08631ae02532d6f190bab968c1ec24720 +development.zh.md: 81e8015bbcf45424dfb595154bab0ffe1e501251 diff --git a/python/development.md b/python/development.md index 8e7608ada7..b36a26b086 100644 --- a/python/development.md +++ b/python/development.md @@ -36,6 +36,8 @@ uv run --project python/sdk python scripts/smoke-python-runtime.py \ Four scenarios compare committed expected output under `scripts/snapshots/python-sdk-single-exe/`. `minimal/model-visible.json` pins the Linux/macOS `sdk-minimal` profile's assembled system prompts, advertised tool schemas, and model-visible messages; `minimal/win-x64/model-visible.json` pins its PowerShell counterpart. A plugin that contributes an unintended system section or user message therefore fails the job, and every message the profile emits is compared. `advanced/` pins one complex process's SDK result and parent/child session logs across every target. `restart/` launches two complete SDK runtime processes against one persistence root and snapshots their isolated model histories, high-level results, and separate durable logs across every target. `sdk-minimal-in-history` reuses the persistent-shell and editor scenario with a section that changes after the first successful shell call. `minimal-in-history/prompt-history.json` pins both prompt versions, the unchanged leading prompt in later requests, appended SDK system-message events, and `request/context.systemPromptUpdate`; tool schemas stay fixed and the editor file is checked independently. Rerun the owning scenario with `--update-snapshots` and review that diff before committing it. +The `advanced` and `restart` comparisons match native delivery qualifiers against each input's own Session generation, including SDK events and notifications. Captured and other delivery generations retain their numeric values. Each side must use one generation across its Session roles, and fresh logs must identify the current writer. Comparison never rewrites committed recordings. + Trusted pull requests and master pushes also run `--scenario sdk-live --installed-wheel` on each selected native target. That scenario performs two tool-using turns against `https://api.deepseek.com`: it checks the created file immediately, replaces its content with a host-only random challenge, and requires the second turn to copy the changed content into a fresh receipt without modifying the source. Both turns must complete with the exact sentinel answer and a model-requested tool call; external byte comparisons check the files. Missing repository secrets fail instead of self-skipping. Fork and Dependabot pull requests run the complete keyless installed-wheel path but receive no key. An interactive smoke test needs `DEEPSEEK_API_KEY` in the environment or repository-root `.env`: diff --git a/python/development.zh.md b/python/development.zh.md index 3f6c1f9263..81e8015bbc 100644 --- a/python/development.zh.md +++ b/python/development.zh.md @@ -36,6 +36,8 @@ uv run --project python/sdk python scripts/smoke-python-runtime.py \ 其中四个场景会比对 `scripts/snapshots/python-sdk-single-exe/` 下已提交的期望输出。`minimal/model-visible.json` 固定 Linux/macOS `sdk-minimal` profile 所组装的系统提示词、对外公布的工具 schema 与模型可见消息;`minimal/win-x64/model-visible.json` 固定对应的 PowerShell 版本。因此,插件一旦贡献出计划外的系统分段或 user 消息,该任务即失败,且该 profile 发出的每条消息都会参与比对。`advanced/` 跨所有目标固定一个复杂进程的 SDK 结果及父/子会话日志。`restart/` 针对同一持久化根目录启动两个完整 SDK 运行时进程,并跨所有目标固定其彼此隔离的模型历史、高层结果与独立持久日志。`sdk-minimal-in-history` 复用持久 shell 与编辑器场景,并在首次 shell 调用成功后更改一个分段。`minimal-in-history/prompt-history.json` 固定两个提示词版本、后续请求中不变的首条提示词、追加的 SDK system-message 事件及 `request/context.systemPromptUpdate`;工具 schema 保持不变,并独立检查编辑器创建的文件。重新运行对应场景时加上 `--update-snapshots`,并在提交前审阅该差异。 +`advanced` 与 `restart` 的比较按每份输入自身的 Session 代际匹配原生 delivery 限定值,包括 SDK 事件与通知。捕获的代际及其他 delivery 代际保留数值。每一侧的 Session 角色必须使用同一个代际,新日志必须标识当前写入器。比较绝不重写已提交的记录。 + 可信拉取请求与 master 推送还会在各自选定的原生目标上运行 `--scenario sdk-live --installed-wheel`。该场景面向 `https://api.deepseek.com` 执行两个使用工具的轮次:立即检查已创建文件,将其内容替换为仅宿主知道的随机挑战值,并要求第二轮将变更后的内容复制到全新的回执文件,且不修改源文件。两个轮次都必须完成、返回精确的哨兵答案并由模型请求调用工具;文件通过外部逐字节比较验证。仓库密钥缺失时失败,而不是自行 skip。Fork 与 Dependabot 拉取请求会运行完整的 keyless 安装后 wheel 路径,但不会获得密钥。 交互式冒烟测试需要环境变量或仓库根目录 `.env` 中存在 `DEEPSEEK_API_KEY`: diff --git a/python/sdk/tests/test_smoke_model.py b/python/sdk/tests/test_smoke_model.py index 12402c01e3..e972365779 100644 --- a/python/sdk/tests/test_smoke_model.py +++ b/python/sdk/tests/test_smoke_model.py @@ -482,6 +482,143 @@ def test_snapshot_comparison_accepts_current_output_against_v2_without_rewriting assert SMOKE["selected_snapshot_session_files"](tmp_path) == {0: tmp_path / successor_name} +def native_delivery_snapshot(version: int) -> dict[str, str]: + event = { + "type": "session-log-deepseek/delivery-accepted", + "data": {"sessionId": "s", "throughSeq": 0, "sessionFormatVersion": version}, + } + return { + "result.json": json.dumps({ + "events": [event], + "notifications": [{"method": "session.event", "payload": {"sessionId": "s", "event": event}}], + }) + "\n", + f"session.v{version}.jsonl": SMOKE["render_jsonl"]([ + {"type": "session", "version": version, "id": "s"}, + {"type": "turn/start", "data": {"turn": 1}}, + event, + ]), + } + + +def test_native_writer_comparison_requires_opt_in_and_preserves_written_generations(tmp_path: Path) -> None: + golden = native_delivery_snapshot(3) + fresh = native_delivery_snapshot(SESSION_FORMAT_VERSION) + for name, content in golden.items(): + (tmp_path / name).write_text(content, encoding="utf-8") + compare = SMOKE["compare_snapshot_files"] + with pytest.raises(AssertionError, match="executable snapshot mismatch"): + compare(fresh, False, tmp_path, tuple(golden)) + compare(fresh, False, tmp_path, tuple(golden), native_writer_output=True) + assert {path.name: path.read_text(encoding="utf-8") for path in tmp_path.iterdir()} == golden + compare(fresh, True, tmp_path, tuple(golden), native_writer_output=True) + assert (tmp_path / "session.v3.jsonl").read_text(encoding="utf-8") == golden["session.v3.jsonl"] + for name, content in fresh.items(): + assert (tmp_path / name).read_text(encoding="utf-8") == content + assert "{{sourceSessionFormatVersion}}" not in content + + +@pytest.mark.parametrize("surface", ["session", "events", "notifications"]) +def test_native_writer_comparison_rejects_stale_current_delivery(tmp_path: Path, surface: str) -> None: + golden = native_delivery_snapshot(3) + fresh = native_delivery_snapshot(SESSION_FORMAT_VERSION) + for name, content in golden.items(): + (tmp_path / name).write_text(content, encoding="utf-8") + if surface == "session": + name = f"session.v{SESSION_FORMAT_VERSION}.jsonl" + records = [json.loads(line) for line in fresh[name].splitlines()] + records[-1]["data"]["sessionFormatVersion"] = 3 + fresh[name] = SMOKE["render_jsonl"](records) + else: + result = json.loads(fresh["result.json"]) + event = result["events"][0] if surface == "events" else result["notifications"][0]["payload"]["event"] + event["data"]["sessionFormatVersion"] = 3 + fresh["result.json"] = json.dumps(result) + "\n" + with pytest.raises(AssertionError, match="executable snapshot mismatch"): + SMOKE["compare_snapshot_files"](fresh, False, tmp_path, tuple(golden), native_writer_output=True) + + +def test_native_writer_comparison_preserves_captured_and_nested_delivery_values() -> None: + delivery = { + "type": "session-log-deepseek/delivery-accepted", + "data": {"sessionId": "s", "throughSeq": 0, "sessionFormatVersion": 3}, + } + captured = {"type": "custom/event", "data": { + "capturedFormatVersion": 3, "sessionFormatVersion": 3, "nested": delivery, + }} + value = { + "events": [delivery, captured, {**delivery, "data": {**delivery["data"], "sessionFormatVersion": 2}}], + "notifications": [ + {"method": "session.event", "params": {"event": delivery}}, + {"method": "other.event", "payload": {"event": delivery}}, + ], + } + normalized = json.loads(SMOKE["normalize_snapshot_comparison_text"]("result.json", json.dumps(value), 3)) + assert normalized["events"][0]["data"]["sessionFormatVersion"] == "{{sourceSessionFormatVersion}}" + assert normalized["events"][1] == captured + assert normalized["events"][2]["data"]["sessionFormatVersion"] == 2 + assert normalized["notifications"][0]["params"]["event"]["data"]["sessionFormatVersion"] == "{{sourceSessionFormatVersion}}" + assert normalized["notifications"][1] == value["notifications"][1] + assert SMOKE["normalize_session_format_comparison"](delivery) == delivery + + +def test_default_comparison_keeps_a_migrated_delivery_generation_exact() -> None: + source = native_delivery_snapshot(3)["session.v3.jsonl"] + target_records = [json.loads(line) for line in source.splitlines()] + target_records[0]["version"] = SESSION_FORMAT_VERSION + target = SMOKE["render_jsonl"](target_records) + normalize = SMOKE["normalize_snapshot_comparison_text"] + assert normalize("session.v3.jsonl", source) == normalize(f"session.v{SESSION_FORMAT_VERSION}.jsonl", target) + assert '"sessionFormatVersion":3' in normalize(f"session.v{SESSION_FORMAT_VERSION}.jsonl", target) + + +def test_native_writer_comparison_rejects_mixed_expected_role_generations(tmp_path: Path) -> None: + golden = {"session.v2.jsonl": 2, "session.1.v3.jsonl": 3} + fresh = {} + for index, (name, version) in enumerate(golden.items()): + (tmp_path / name).write_text(SMOKE["render_jsonl"]([ + {"type": "session", "version": version}, + ]), encoding="utf-8") + fresh[SMOKE["snapshot_session_filename"](index, SESSION_FORMAT_VERSION)] = SMOKE["render_jsonl"]([ + {"type": "session", "version": SESSION_FORMAT_VERSION}, + ]) + with pytest.raises(AssertionError, match="requires one Session generation across all roles"): + SMOKE["compare_snapshot_files"](fresh, False, tmp_path, tuple(golden), native_writer_output=True) + + +@pytest.mark.parametrize("scenario", ["ADVANCED", "RESTART"]) +def test_committed_python_native_goldens_compare_current_output_without_rewriting(scenario: str) -> None: + directory = SMOKE[f"{scenario}_SNAPSHOT_DIRECTORY"] + filenames = SMOKE[f"{scenario}_SNAPSHOT_FILENAMES"] + before = {path.name: path.read_bytes() for path in directory.iterdir() if path.is_file()} + + def current_writer(value: object) -> object: + if isinstance(value, list): + return [current_writer(item) for item in value] + if not isinstance(value, dict): + return value + result = {key: current_writer(item) for key, item in value.items()} + if result.get("type") == "session": + assert result["version"] == 3 + result["version"] = SESSION_FORMAT_VERSION + if result.get("type") == "session-log-deepseek/delivery-accepted": + assert result["data"]["sessionFormatVersion"] == 3 + result["data"]["sessionFormatVersion"] = SESSION_FORMAT_VERSION + return result + + fresh = {} + for name in filenames: + content = before[name].decode("utf-8") + if name.endswith(".jsonl"): + parsed = SMOKE["parse_snapshot_session_filename"](name) + fresh[SMOKE["snapshot_session_filename"](parsed[0], SESSION_FORMAT_VERSION)] = SMOKE["render_jsonl"]([ + current_writer(json.loads(line)) for line in content.splitlines() if line + ]) + else: + fresh[name] = json.dumps(current_writer(json.loads(content)), indent=2, ensure_ascii=False) + "\n" + SMOKE["compare_snapshot_files"](fresh, False, directory, filenames, native_writer_output=True) + assert {path.name: path.read_bytes() for path in directory.iterdir() if path.is_file()} == before + + @pytest.mark.parametrize("filenames", [ ("session.1.v2.jsonl", "session.v2.jsonl"), ("session.v2.jsonl",), diff --git a/scripts/smoke-python-runtime.py b/scripts/smoke-python-runtime.py index afd1cc4cb7..baec977d4b 100644 --- a/scripts/smoke-python-runtime.py +++ b/scripts/smoke-python-runtime.py @@ -1413,6 +1413,7 @@ def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool) files = build_snapshot_files(result, logs, child_ids, root) compare_snapshot_files( files, update_snapshots, ADVANCED_SNAPSHOT_DIRECTORY, ADVANCED_SNAPSHOT_FILENAMES, + native_writer_output=True, ) @@ -1479,6 +1480,7 @@ def smoke_sdk_restart_snapshot(base_url: str, executable: Path, update_snapshots ) compare_snapshot_files( files, update_snapshots, RESTART_SNAPSHOT_DIRECTORY, RESTART_SNAPSHOT_FILENAMES, + native_writer_output=True, ) @@ -2282,6 +2284,7 @@ def project_session_snapshot(records: list[dict[str, object]]) -> list[dict[str, SESSION_FORMAT_TOKEN = "{{sessionFormatVersion}}" +NATIVE_DELIVERY_FORMAT_TOKEN = "{{sourceSessionFormatVersion}}" def expand_snapshot_stream_member(member: object) -> list[dict[str, object]]: @@ -2404,7 +2407,31 @@ def normalize_session_format_comparison( return normalized -def normalize_snapshot_comparison_text(name: str, content: str) -> str: +def normalize_native_delivery_record(value: object, source_version: int) -> object: + """Tokenize a native delivery qualifier in one Session event or SDK notification.""" + if not isinstance(value, dict): + return value + if value.get("method") == "session.event": + for key in ("payload", "params"): + wrapper = value.get(key) + if isinstance(wrapper, dict) and "event" in wrapper: + return {**value, key: { + **wrapper, + "event": normalize_native_delivery_record(wrapper["event"], source_version), + }} + data = value.get("data") + if value.get("type") != "session-log-deepseek/delivery-accepted" or not isinstance(data, dict): + return value + if data.get("sessionFormatVersion") != source_version: + return value + return {**value, "data": {**data, "sessionFormatVersion": NATIVE_DELIVERY_FORMAT_TOKEN}} + + +def normalize_snapshot_comparison_text( + name: str, + content: str, + native_writer_version: int | None = None, +) -> str: """Normalize Session generation metadata only while comparing committed expected outputs.""" if name.startswith("session") and name.endswith(".jsonl"): parsed = [json.loads(line) for line in content.splitlines() if line] @@ -2412,6 +2439,8 @@ def normalize_snapshot_comparison_text(name: str, content: str) -> str: source_version = header.get("version") if isinstance(header, dict) else None if not isinstance(source_version, int): raise AssertionError(f"{name}: snapshot Session header has no integer format version") + if native_writer_version is not None: + parsed = [normalize_native_delivery_record(record, native_writer_version) for record in parsed] records = [ normalize_session_format_comparison(expanded, source_version) for record in parsed @@ -2419,8 +2448,18 @@ def normalize_snapshot_comparison_text(name: str, content: str) -> str: ] return render_jsonl(records) if name.endswith(".json"): + value = json.loads(content) + if name == "result.json" and native_writer_version is not None and isinstance(value, dict): + value = { + **value, + **{ + key: [normalize_native_delivery_record(record, native_writer_version) for record in value[key]] + for key in ("events", "notifications") + if isinstance(value.get(key), list) + }, + } return json.dumps( - normalize_session_format_comparison(json.loads(content)), + normalize_session_format_comparison(value), indent=2, ensure_ascii=False, ) + "\n" @@ -2432,8 +2471,10 @@ def compare_snapshot_files( update: bool, directory: Path, filenames: tuple[str, ...], + *, + native_writer_output: bool = False, ) -> None: - """Compare ordered artifact roles and Session content across generations, or write generated filenames.""" + """Compare artifact roles and content, optionally matching each side's native delivery generation.""" scenario = directory.name def role_name(name: str) -> str: @@ -2485,12 +2526,24 @@ def compare_snapshot_files( f"{scenario} snapshot Session roles differ: " f"expected={sorted(selected_expected)}, actual={sorted(actual_sessions)}", ) + actual_native_version = expected_native_version = None + if native_writer_output: + def common_generation(contents: list[tuple[str, str]]) -> int: + versions = {session_header_version(content, name) for name, content in contents} + if len(versions) != 1: + raise AssertionError(f"{scenario}: native writer comparison requires one Session generation across all roles") + return versions.pop() + + actual_native_version = common_generation(list(actual_sessions.values())) + expected_native_version = common_generation([ + (path.name, path.read_text(encoding="utf-8")) for path in selected_expected.values() + ]) for name, actual in files.items(): parsed = parse_snapshot_session_filename(name) expected_path = directory / name if parsed is None else selected_expected[parsed[0]] expected_text = expected_path.read_text(encoding="utf-8") - compared_actual = normalize_snapshot_comparison_text(name, actual) - compared_expected = normalize_snapshot_comparison_text(expected_path.name, expected_text) + compared_actual = normalize_snapshot_comparison_text(name, actual, actual_native_version) + compared_expected = normalize_snapshot_comparison_text(expected_path.name, expected_text, expected_native_version) if compared_actual == compared_expected: continue diff = "".join(difflib.unified_diff(