Use string for window id

This commit is contained in:
Huo Yaoyuan committed 2026-11-09 00:41:55 +08:00
1 parent f8f57aa8e0
commit 05f557bbf6
44 files changed
+488 -465

No files matched your search

@@ -41,8 +41,8 @@ struct ForegroundObservation: Sendable {
}
var json: JSONValue {
.object(["frontPid": frontPID.map { .number(Double($0)) } ?? .null,
"focusedWindowId": focusedWindowID.map { .number(Double($0)) } ?? .null,
"frontWindowId": frontWindowID.map { .number(Double($0)) } ?? .null,
"focusedWindowId": focusedWindowID.map { .string(String($0)) } ?? .null,
"frontWindowId": frontWindowID.map { .string(String($0)) } ?? .null,
"frontWindowPid": frontWindowPID.map { .number(Double($0)) } ?? .null])
}
}
@@ -275,7 +275,7 @@ actor AXWorker {
let changed = ForegroundObservation.change(beforePID: beforePID, beforeWindow: beforeWindow,
afterPID: afterPID, afterWindow: afterWindow)
return .result(.object(["dispatched": .bool(true),
"windowId": restored.map { .number(Double($0)) } ?? .null, "foregroundChanged": changed]))
"windowId": restored.map { .string(String($0)) } ?? .null, "foregroundChanged": changed]))
case "activate":
return .foreground(ForegroundPlan(appHandle: handle, identity: identity))
case "getState":
@@ -302,8 +302,8 @@ actor AXWorker {
observedWindowBounds[handle, default: [:]][observed.windowID] = observed.bounds
}
return .result(.object(["appHandle": .string(handle),
"windowId": window.map { .number(Double($0)) } ?? .null,
"focusedWindowId": projection.focusedWindowID.map { .number(Double($0)) } ?? .null,
"windowId": window.map { .string(String($0)) } ?? .null,
"focusedWindowId": projection.focusedWindowID.map { .string(String($0)) } ?? .null,
"snapshotId": .string(id), "tree": .string(projection.tree), "elements": .array(projection.elements),
"windows": .array(observedWindows.map(\.json)), "truncated": .bool(projection.truncated)]))
case "screenshot":
@@ -468,7 +468,7 @@ actor AXWorker {
trace?.mark("postflight.end")
return .object(["mimeType": .string("image/png"), "data": .string(encoded),
"pixelWidth": .number(Double(image.width)), "pixelHeight": .number(Double(image.height)), "bounds": after.bounds.json,
"coordinateSpace": .string("window-points"), "windowId": .number(Double(after.windowID))])
"coordinateSpace": .string("window-points"), "windowId": .string(String(after.windowID))])
}
static func mouseMoveResult(route: String, pointerMoved: Bool) -> JSONValue {
.object(["dispatched": .bool(true), "hoverRoute": .string(route), "systemPointerMoved": .bool(pointerMoved)])
@@ -140,7 +140,7 @@ enum NativePlatform {
return rows.compactMap { row in
let layer = (row[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
guard allLayers || layer == 0,
let id = (row[kCGWindowNumber as String] as? NSNumber)?.uint32Value,
let id = (row[kCGWindowNumber as String] as? NSNumber)?.uint32Value, id != 0,
let pid = (row[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value else { return nil }
let raw = row[kCGWindowBounds as String] as? [String: NSNumber] ?? [:]
let bounds = WindowBounds(x: raw["X"]?.doubleValue ?? 0, y: raw["Y"]?.doubleValue ?? 0,
@@ -141,7 +141,7 @@ struct WindowInfo: Sendable {
var json: JSONValue {
.object([
"windowId": .number(Double(windowID)),
"windowId": .string(String(windowID)),
"appName": .string(appName), "title": .string(title), "bounds": bounds.json,
"isOnScreen": .bool(isOnScreen), "onCurrentSpace": onCurrentSpace.map(JSONValue.bool) ?? .null,
])
@@ -22,6 +22,7 @@ struct RequestParameters: Sendable {
}
}
self.values = values
if contains("windowId") { _ = try windowID() }
}
static func invalid(_ message: String) -> NativeFailure { NativeFailure(code: "INVALID_ARGUMENT", message: message) }
func contains(_ key: String) -> Bool { values[key] != nil }
@@ -54,7 +55,13 @@ struct RequestParameters: Sendable {
}
return integer
}
func windowID() throws -> UInt32 { UInt32(try integer("windowId", in: 1...Int(UInt32.max))) }
func windowID() throws -> UInt32 {
let text = try string("windowId")
guard let id = UInt32(text), id > 0, String(id) == text else {
throw Self.invalid("windowId must be a canonical decimal string in 1...\(UInt32.max)")
}
return id
}
func optionalWindowID() throws -> UInt32? { contains("windowId") ? try windowID() : nil }
/// Element ids are opaque strings on the wire; this helper accepts its decimal form and returns the snapshot index.
func elementIdIndex() throws -> Int {
@@ -168,9 +168,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
}
},
"required": [
@@ -235,9 +234,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"includeMenus": {
"description": "Expand menu contents; omitted or false shows menu-bar entries only.",
@@ -291,9 +289,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -338,9 +335,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -427,9 +423,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -476,9 +471,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -505,9 +499,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -560,9 +553,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -628,9 +620,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -720,9 +711,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -805,9 +795,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -871,9 +860,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"text": {
"type": "string"
@@ -911,9 +899,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"text": {
"type": "string"
@@ -982,9 +969,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"key": {
"type": "string",
@@ -1029,9 +1015,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"key": {
"type": "string",
@@ -1078,9 +1063,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -1141,9 +1125,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"mouseButton": {
"type": "string",
@@ -1235,9 +1218,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
}
},
"required": [
@@ -80,7 +80,7 @@ final class AxlessFixture: NSObject, NSApplicationDelegate, NSWindowDelegate {
let rect = button.convert(button.bounds, to: nil)
let data = try! JSONSerialization.data(withJSONObject: [
"pid": ProcessInfo.processInfo.processIdentifier,
"windowId": window.windowNumber,
"windowId": window.windowNumber > 0 ? String(window.windowNumber) as Any : NSNull(),
"hiddenAX": hiddenAX, "count": count, "resizeCount": resizeCount, "moved": moved,
"hidden": NSApplication.shared.isHidden, "minimized": window.isMiniaturized,
"active": NSApplication.shared.isActive,
@@ -116,7 +116,7 @@ final class CapabilitiesFixture: NSObject, NSApplicationDelegate, NSWindowDelega
}
let frame = pad.convert(pad.bounds, to: nil)
let selection = text.selectedRange()
let result: [String: Any] = ["pid": ProcessInfo.processInfo.processIdentifier, "windowId": window.windowNumber,
let result: [String: Any] = ["pid": ProcessInfo.processInfo.processIdentifier, "windowId": window.windowNumber > 0 ? String(window.windowNumber) as Any : NSNull(),
"active": NSApplication.shared.isActive, "keyWindow": window.isKeyWindow,
"textFocused": window.firstResponder === text, "pasteAttempts": pasteAttempts,
"text": text.string, "selection": [selection.location, selection.length], "runs": runs, "events": events, "axPresses": axPresses,
@@ -209,12 +209,12 @@ final class Fixture: NSObject, NSApplicationDelegate, NSTextFieldDelegate {
? CGWindowListCopyWindowInfo(.optionIncludingWindow, CGWindowID(overlayID)) as? [[String: Any]] : nil
let overlayOnScreen = overlayRows?.contains { ($0[kCGWindowIsOnscreen as String] as? Bool) == true } ?? false
let state: [String: Any] = ["count": count, "secondaryCount": secondaryCount, "text": field.stringValue, "selectedFile": selectedFile,
"rowCount": rowCount, "primaryWindowId": window.windowNumber,
"rowCount": rowCount, "primaryWindowId": window.windowNumber > 0 ? String(window.windowNumber) as Any : NSNull(),
"dragDownCount": dragPad.dragDownCount, "dragCount": dragPad.dragCount, "dragPoint": [dragPad.lastPoint.x, dragPad.lastPoint.y],
"hoverVisible": !dragPad.hoverButton.isHidden, "hoverClicks": dragPad.hoverClicks,
"cursorSamples": cursorSamples, "cursorChanges": cursorChanges,
"overlayMode": overlayMode, "overlayWindowId": inputOverlay?.windowNumber ?? 0,
"overlayOnScreen": overlayOnScreen, "hoverHitWindow": hoverHitWindow,
"overlayMode": overlayMode, "overlayWindowId": overlayID > 0 ? String(overlayID) as Any : NSNull(),
"overlayOnScreen": overlayOnScreen, "hoverHitWindow": hoverHitWindow > 0 ? String(hoverHitWindow) as Any : NSNull(),
"rightDownCount": dragPad.rightDownCount, "rightUpCount": dragPad.rightUpCount,
"scrollEventCount": scroll.eventCount, "scrollY": scroll.contentView.bounds.origin.y, "dragBounds": windowPoints(dragPad), "scrollBounds": windowPoints(scroll),
"primaryWindowSize": [window.frame.width, window.frame.height],
@@ -40,7 +40,7 @@ def main():
nonlocal checked_windows
if not args.desktop:
return
state = wait_for_state(state_path, lambda value: value["pid"] == pid and value["windowId"] > 0)
state = wait_for_state(state_path, lambda value: value["pid"] == pid and bool(value["windowId"]))
owner = {"appHandle": app["appHandle"], "windowId": state["windowId"]}
observed = helper.call("getState", {**owner, "maxElements": 2000, "maxDepth": 25})
pad = next(item for item in observed["elements"] if item["role"] == "AXButton" and item["title"] == "Input receiver")
@@ -125,7 +125,7 @@ def main():
pending = helper.call("getApp", {"selector": str(app)})
process = subprocess.Popen([str(executable)], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
cleanup.callback(stop, process)
wait_for_state(state, lambda value: value["pid"] == process.pid and value["windowId"] > 0)
wait_for_state(state, lambda value: value["pid"] == process.pid and bool(value["windowId"]))
running = helper.call("getApp", {"selector": str(process.pid)})
assert running["authorization"]["id"] == pending["authorization"]["id"] == expected, running["authorization"]
check_window(running, state, process.pid)
@@ -136,7 +136,7 @@ def main():
process = subprocess.Popen([str(binary), str(state), "--background"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
cleanup.callback(stop, process)
try:
wait_for_state(state, lambda value: value["pid"] == process.pid and value["windowId"] > 0)
wait_for_state(state, lambda value: value["pid"] == process.pid and bool(value["windowId"]))
running = helper.call("getApp", {"selector": str(process.pid)})
authorization = running["authorization"]
assert running.get("bundleId") is None and authorization["persistable"] is True, authorization
@@ -155,7 +155,7 @@ def main():
pending = helper.call("getApp", {"selector": str(app)})
launched = helper.call("launchApp", {"appHandle": pending["appHandle"]})
assert launched["running"] is True, launched
pid = wait_for_state(state, lambda value: value["windowId"] > 0)["pid"]
pid = wait_for_state(state, lambda value: bool(value["windowId"]))["pid"]
executable.unlink()
buffer = ctypes.create_string_buffer(4096)
ctypes.set_errno(0)
@@ -152,7 +152,7 @@ def main():
# The Host consent barrier is separately exercised by provider-lifecycle.spec.ts.
app = call("launchApp", owner)
assert app["running"] is True, app
pid = wait_for_state(state_file, lambda value: value["windowId"] > 0)["pid"]
pid = wait_for_state(state_file, lambda value: bool(value["windowId"]))["pid"]
if not args.click_only:
call("activate", owner)
state = wait_for_state(state_file, lambda value: value["pid"] == pid and value["active"] and value["keyWindow"])
@@ -59,7 +59,7 @@ def cases():
for method in ("typeText", "pressKey", "paste"):
identifier = "window-keyboard-" + method
argument = {"key": "Return"} if method == "pressKey" else {"text": "do not dispatch"}
result.append((request(identifier, method, {"appHandle": "never-selected", "windowId": 1, **argument}),
result.append((request(identifier, method, {"appHandle": "never-selected", "windowId": "1", **argument}),
identifier, "INVALID_ARGUMENT", "macOS keyboard input follows application focus; use the App object"))
for index in range(24):
identifier = f"ordered-{index:02d}"
@@ -7,6 +7,14 @@ import Testing
#expect(AXWorker.actionResult == .object(["dispatched": .bool(true)]))
}
@Test func emitsForegroundWindowIDsAsStringsAndPIDsAsNumbers() {
let observed = ForegroundObservation(frontPID: 42, focusedWindowID: 7, frontWindowID: 8, frontWindowPID: 43)
#expect(observed.json == .object([
"frontPid": .number(42), "focusedWindowId": .string("7"),
"frontWindowId": .string("8"), "frontWindowPid": .number(43),
]))
}
@Test func reportsOnlyObservableForegroundChanges() {
#expect(ForegroundObservation.change(beforePID: 1, beforeWindow: nil, afterPID: 2, afterWindow: nil) == .bool(true))
#expect(ForegroundObservation.change(beforePID: nil, beforeWindow: 3, afterPID: nil, afterWindow: 4) == .bool(true))
@@ -48,7 +48,7 @@ func refusesInvalidRequestEnvelopes(_ source: String) {
let encoded = try JSONEncoder().encode(window.json)
let object = try #require(JSONSerialization.jsonObject(with: encoded) as? [String: Any])
#expect(object["onCurrentSpace"] is NSNull)
#expect(window.json["windowId"] == .number(7))
#expect(window.json["windowId"] == .string("7"))
#expect(window.json["bounds"]["x"] == .number(-100))
#expect(window.json["isOnScreen"] == .bool(true))
}
@@ -74,11 +74,24 @@ func refusesInvalidRequestEnvelopes(_ source: String) {
}
}
@Test(arguments: ["true", "1.5", "0", "4294967296", "\"7\""])
func rejectsInvalidWindowIDs(_ value: String) throws {
let request = try NativeRequest(data: Data("{\"id\":\"call\",\"method\":\"screenshot\",\"params\":{\"windowId\":\(value)}}".utf8))
@Test func acceptsWindowIDStrings() throws {
let request = try NativeRequest(data: Data(#"{"id":"call","method":"screenshot","params":{"windowId":"7"}}"#.utf8))
let params = try RequestParameters(request)
#expect(throws: NativeFailure.self) { try params.windowID() }
#expect(try params.windowID() == 7)
#expect(try params.optionalWindowID() == 7)
}
@Test(arguments: ["true", "null", "7"])
func rejectsNonStringWindowIDs(_ value: String) throws {
let request = try NativeRequest(data: Data("{\"id\":\"call\",\"method\":\"screenshot\",\"params\":{\"windowId\":\(value)}}".utf8))
#expect(throws: NativeFailure.self) { try RequestParameters(request) }
}
@Test func validatesOptionalWindowIDsBeforeOperationDispatch() throws {
let absent = try RequestParameters(NativeRequest(data: Data(#"{"id":"call","method":"getState","params":{}}"#.utf8)))
#expect(try absent.optionalWindowID() == nil)
let numeric = try NativeRequest(data: Data(#"{"id":"call","method":"listActions","params":{"windowId":7}}"#.utf8))
#expect(throws: NativeFailure.self) { try RequestParameters(numeric) }
}
@Test func rejectsRoundedIntegerOverflowWithoutTrapping() throws {
@@ -68,7 +68,7 @@ it('defines every non-builtin type name used in method signatures', () => {
expect(COMMON_GUIDANCE).toContain('ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }')
expect(COMMON_GUIDANCE).toContain('JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }')
expect(COMMON_GUIDANCE).toContain('App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level')
expect(COMMON_GUIDANCE).toContain('Window is the window object with readonly window.windowId: number and the methods listed below for window level')
expect(COMMON_GUIDANCE).toContain('Window is the window object with readonly window.windowId: string and the methods listed below for window level')
expect(COMMON_GUIDANCE).toContain('Signatures show resolved return values after await')
})
@@ -108,7 +108,7 @@ it.skipIf(process.platform === 'win32')('delivers a valid PNG larger than 4 MiB
this.requests.push(options)
if (this.requests.length === 1) {
const code = 'const app = await tools.cua({ method: "getApp", params: { selector: "9" } }); '
+ 'const result = await tools.cua({ method: "screenshot", params: { appHandle: app.appHandle, windowId: 7 } }); '
+ 'const result = await tools.cua({ method: "screenshot", params: { appHandle: app.appHandle, windowId: "7" } }); '
+ 'return { mimeType: result.mimeType, encodedLength: result.data.length };'
const call = { type: 'tool-call' as const, id: ToolCallId('large-image-ptc'), name: 'run_code',
arguments: JSON.stringify({ code, description: 'Inspect the generated large screenshot' }),
@@ -33,10 +33,10 @@ const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG
const calls = [
{ method: 'getCapabilities', params: {} },
{ method: 'getApp', params: { selector: 'test.fixture' } },
{ method: 'getState', params: { appHandle: 'fixture-app', windowId: 7 } },
{ method: 'getState', params: { appHandle: 'fixture-app', windowId: '7' } },
{ method: 'click', params: { appHandle: 'fixture-app', snapshotId: 'fixture-snapshot-1', elementId: '0', holdMs: 800 } },
{ method: 'getState', params: { appHandle: 'fixture-app', windowId: 7 } },
{ method: 'screenshot', params: { appHandle: 'fixture-app', windowId: 7 } },
{ method: 'getState', params: { appHandle: 'fixture-app', windowId: '7' } },
{ method: 'screenshot', params: { appHandle: 'fixture-app', windowId: '7' } },
]
class VisualModel extends LlmAdapter {
@@ -139,13 +139,13 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
expect(toolCalls[3]?.data.arguments).toBe(JSON.stringify(calls[3]))
const direct = await executeToolFixture(context, { agent, signal: new AbortController().signal,
callId: ToolCallId('programmatic-window'), name: 'cua', arguments: { method: 'screenshot', params: { appHandle: 'fixture-app', windowId: 7 } },
callId: ToolCallId('programmatic-window'), name: 'cua', arguments: { method: 'screenshot', params: { appHandle: 'fixture-app', windowId: '7' } },
})
expect(direct.isError).toBe(false)
expect(direct.value).toMatchObject({ mimeType: 'image/png', data: png })
expect(context.computerUse.providerName).toBe('dsh-cua')
for (const [callId, activationRequested, beforePid, beforeWindowId] of [
['activate-first', true, 42, 99], ['activate-repeat', false, 9, 7],
['activate-first', true, 42, '99'], ['activate-repeat', false, 9, '7'],
] as const) {
const activated = await executeToolFixture(context, {
agent, signal: new AbortController().signal, callId: ToolCallId(callId), name: 'cua',
@@ -155,8 +155,8 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
expect(activated.value).toEqual({
dispatched: true, foregroundChanged: beforePid !== 9, foreground: {
requestId: callId, targetPid: 9, activationRequested,
before: { frontPid: beforePid, focusedWindowId: 7, frontWindowId: beforeWindowId, frontWindowPid: beforePid },
after: { frontPid: 9, focusedWindowId: 7, frontWindowId: 7, frontWindowPid: 9 },
before: { frontPid: beforePid, focusedWindowId: '7', frontWindowId: beforeWindowId, frontWindowPid: beforePid },
after: { frontPid: 9, focusedWindowId: '7', frontWindowId: '7', frontWindowPid: 9 },
},
})
expect(JSON.stringify(activated.content)).toContain(callId)
@@ -172,14 +172,14 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
return z.json().parse(result.value)
})
const retainedApp = await objectApi(agent).getApp('test.fixture')
const retainedWindow = await retainedApp.getWindow(7)
const retainedWindow = await retainedApp.getWindow('7')
expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 1')
const peer = await context.agents.create({ sessionId: SessionId('native-loader-peer'), agentOptions: { provider: 'native-fixture', model: 'vision' } })
const peerApp = await objectApi(peer.agent).getApp('test.fixture')
const peerWindow = await peerApp.getWindow(7)
const peerWindow = await peerApp.getWindow('7')
expect(peerApp.appHandle).not.toBe(retainedApp.appHandle)
const screenshot = await peerWindow.getScreenshot({ emit: false })
expect(screenshot).toMatchObject({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7 })
expect(screenshot).toMatchObject({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: '7' })
expect(Buffer.from(screenshot.bytes).toString('base64')).toBe(png)
await peerWindow.click(element('0'))
expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 2')
@@ -197,7 +197,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
const next = await context.agents.create({ sessionId: SessionId('native-loader-next'), agentOptions: { provider: 'native-fixture', model: 'vision' } })
const stale = await executeToolFixture(context, { agent: next.agent, signal: new AbortController().signal,
callId: ToolCallId('stale-window'), name: 'cua', arguments: { method: 'screenshot', params: { appHandle: 'fixture-app', windowId: 7 } },
callId: ToolCallId('stale-window'), name: 'cua', arguments: { method: 'screenshot', params: { appHandle: 'fixture-app', windowId: '7' } },
})
expect(stale.isError).toBe(true)
expect(JSON.stringify(stale.content)).toContain('APP_ACCESS_REQUIRED')
@@ -74,9 +74,9 @@ it.skipIf(process.platform === 'win32' || !existsSync(bridge) || !existsSync(boo
const section = (await ctx.systemPrompt.assemble()).sections.find(value => value.name === 'computer-use:dsh-cua')
const initialization = section?.text.match(/Persistent PTC initialization:\n\n```ts\n([\s\S]*?)\n```/u)?.[1]
expect(initialization).toContain('/lib/bridge.js')
const first = await run(`${initialization}\nvar app = await cua.getApp("test.fixture"); var win = await app.getWindow(7); await win.click("0"); var shot = await win.getScreenshot(); ({ mimeType: shot.mimeType, pixelWidth: shot.pixelWidth, pixelHeight: shot.pixelHeight, windowId: shot.windowId, byteLength: shot.bytes.length });`, 'first')
const first = await run(`${initialization}\nvar app = await cua.getApp("test.fixture"); var win = await app.getWindow("7"); await win.click("0"); var shot = await win.getScreenshot(); ({ mimeType: shot.mimeType, pixelWidth: shot.pixelWidth, pixelHeight: shot.pixelHeight, windowId: shot.windowId, byteLength: shot.bytes.length });`, 'first')
expect(first.error).toBeUndefined()
expect(first.value).toEqual({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7, byteLength: 90 })
expect(first.value).toEqual({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: '7', byteLength: 90 })
expect(images).toBe(1)
expect(JSON.stringify(first)).not.toContain('iVBORw0KGgo')
const invalid = await run('await cua.findApps("");', 'invalid')
@@ -23,7 +23,7 @@ describe('native protocol generation', () => {
it.each([
{ method: 'getState', params: { appHandle: 'app', maxElements: 1 } },
{ method: 'drag', params: { appHandle: 'app', windowId: 1, fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } },
{ method: 'drag', params: { appHandle: 'app', windowId: '1', fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } },
{ method: 'paste', params: { appHandle: 'app', text: 'text', maxClipboardBytes: 1024 } },
{ method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } },
{ method: 'findApps', params: { query: 'Editor', maxAppMatches: 1 } },
@@ -107,7 +107,7 @@ class ImageAdapter extends LlmAdapter {
}
const imageData = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
const imageMetadata = { pixelWidth: 1, pixelHeight: 1, bounds: { x: 2, y: 3, width: 1, height: 1 }, coordinateSpace: 'window-points', windowId: 7 }
const imageMetadata = { pixelWidth: 1, pixelHeight: 1, bounds: { x: 2, y: 3, width: 1, height: 1 }, coordinateSpace: 'window-points', windowId: '7' }
async function setup(model?: { provider: string; model: string }) {
native.request.mockReset().mockResolvedValue([])
@@ -377,7 +377,7 @@ it('reports failed cleanup after waiting for other owners and releasing their Ag
it('rejects malformed and unowned operations before acquiring a helper', async () => {
const f = await setup()
expect((await f.execute('screenshot', { pid: 9, windowId: 7 })).isError).toBe(true)
expect((await f.execute('screenshot', { pid: 9, windowId: '7' })).isError).toBe(true)
const unowned = await executeToolFixture(f.ctx, {
signal: new AbortController().signal, callId: ToolCallId('unowned'), name: 'cua', arguments: { method: 'listApps', params: {} },
})
@@ -402,10 +402,10 @@ it('supplies configured resource bounds and clears registrations on unload', asy
const f = await setup()
await selectApplication(f)
native.request.mockResolvedValueOnce({ appHandle: 'app-one', windowId: null, snapshotId: 's', tree: 'Window', truncated: false,
windows: [{ windowId: 7, appName: 'Fixture', title: 'Window', bounds: imageMetadata.bounds, isOnScreen: true, onCurrentSpace: true }] })
windows: [{ windowId: '7', appName: 'Fixture', title: 'Window', bounds: imageMetadata.bounds, isOnScreen: true, onCurrentSpace: true }] })
await f.execute('getState', { appHandle: 'app-one' })
expect(native.request).toHaveBeenLastCalledWith('getState', { appHandle: 'app-one', maxElements: 64, maxDepth: 8 }, expect.any(AbortSignal), ToolCallId('lifecycle-2'))
await f.execute('drag', { appHandle: 'app-one', windowId: 7, fromX: 1, fromY: 2, toX: 3, toY: 4, relative: true })
await f.execute('drag', { appHandle: 'app-one', windowId: '7', fromX: 1, fromY: 2, toX: 3, toY: 4, relative: true })
expect(native.request).toHaveBeenLastCalledWith('drag', expect.objectContaining({ durationMs: 200, steps: 10 }), expect.any(AbortSignal), ToolCallId('lifecycle-3'))
await f.execute('paste', { appHandle: 'app-one', text: 'hello', format: 'text' })
expect(native.request).toHaveBeenLastCalledWith('paste', expect.objectContaining({ maxClipboardBytes: 8192, pasteReadTimeoutMs: 1000, maxElements: 64, maxDepth: 8 }), expect.any(AbortSignal), ToolCallId('lifecycle-4'))
@@ -441,7 +441,7 @@ it('preserves screenshot attachments and invalidates pixel coordinates when wind
const data = imageData
const metadata = imageMetadata
native.request.mockResolvedValue({ ...metadata, mimeType: 'image/png', data })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
expect(result.isError).toBe(false)
const text = result.content.find(block => block.type === 'text')
if (text?.type !== 'text') throw new Error('Screenshot coordinates were not returned')
@@ -451,14 +451,14 @@ it('preserves screenshot attachments and invalidates pixel coordinates when wind
const stored = await f.ctx.attachments.readImage(image.attachment)
expect(Buffer.from(stored.data).toString('base64')).toBe(data)
expect(result.value).toMatchObject({ ...metadata, coordinateSpace: 'screenshot-pixels', mimeType: 'image/png', data })
const windows = [{ windowId: 7, appName: 'Fixture', title: 'Window',
const windows = [{ windowId: '7', appName: 'Fixture', title: 'Window',
bounds: { ...metadata.bounds, width: 2 }, isOnScreen: true, onCurrentSpace: true }]
native.request.mockResolvedValueOnce(windows)
const listed = await f.execute('listWindows', { appHandle: 'app-one' })
expect(listed.isError).toBe(false)
expect(listed.value).toEqual(windows)
native.request.mockClear()
const click = await f.execute('click', { appHandle: 'app-one', windowId: 7, x: 0, y: 0 })
const click = await f.execute('click', { appHandle: 'app-one', windowId: '7', x: 0, y: 0 })
expect(click.isError).toBe(true)
expect(JSON.stringify(click.content)).toContain('SCREENSHOT_REQUIRED')
expect(native.request).not.toHaveBeenCalled()
@@ -480,25 +480,25 @@ it('maps the actual model-request image pixels back to native points without sto
const image = await sharp({ create: { width: 40, height: 20, channels: 3, background: '#4488cc' } }).png().toBuffer()
native.request.mockResolvedValue({ ...imageMetadata, bounds: { x: 10, y: 20, width: 400, height: 200 },
pixelWidth: 40, pixelHeight: 20, mimeType: 'image/png', data: image.toString('base64') })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
expect(result.isError, JSON.stringify(result.content)).toBe(false)
expect(result.value).toMatchObject({ pixelWidth: 10, pixelHeight: 5, coordinateSpace: 'screenshot-pixels' })
const value = result.value as { data: string }
expect(await sharp(Buffer.from(value.data, 'base64')).metadata()).toMatchObject({ width: 10, height: 5 })
expect(save).toHaveBeenCalledTimes(1)
native.request.mockResolvedValue({ dispatched: true })
expect((await f.execute('click', { appHandle: 'app-one', windowId: 7, x: 5, y: 2.5 })).isError).toBe(false)
expect(native.request).toHaveBeenLastCalledWith('click', { appHandle: 'app-one', windowId: 7, x: 200, y: 100 }, expect.any(AbortSignal), expect.any(String))
expect((await f.execute('click', { appHandle: 'app-one', windowId: 7, x: 999, y: 999, relative: true })).isError).toBe(false)
expect(native.request).toHaveBeenLastCalledWith('click', { appHandle: 'app-one', windowId: 7, x: 399, y: 199 }, expect.any(AbortSignal), expect.any(String))
expect((await f.execute('click', { appHandle: 'app-one', windowId: '7', x: 5, y: 2.5 })).isError).toBe(false)
expect(native.request).toHaveBeenLastCalledWith('click', { appHandle: 'app-one', windowId: '7', x: 200, y: 100 }, expect.any(AbortSignal), expect.any(String))
expect((await f.execute('click', { appHandle: 'app-one', windowId: '7', x: 999, y: 999, relative: true })).isError).toBe(false)
expect(native.request).toHaveBeenLastCalledWith('click', { appHandle: 'app-one', windowId: '7', x: 399, y: 199 }, expect.any(AbortSignal), expect.any(String))
})
it.each([{ appHandle: 'foreign', windowId: 7 }, { appHandle: 'app-one', windowId: null }])(
it.each([{ appHandle: 'foreign', windowId: '7' }, { appHandle: 'app-one', windowId: null }])(
'rejects a helper observation for the wrong target before releasing its contents: %j', async (identity) => {
const f = await setup()
await selectApplication(f)
native.request.mockResolvedValueOnce({ ...identity, snapshotId: 'other', tree: 'foreign content', truncated: false, windows: [] })
const result = await f.execute('getState', { appHandle: 'app-one', windowId: 7 })
const result = await f.execute('getState', { appHandle: 'app-one', windowId: '7' })
expect(result.isError).toBe(true)
expect(JSON.stringify(result.content)).toContain('NATIVE_PROTOCOL')
expect(JSON.stringify(result.content)).not.toContain('foreign content')
@@ -513,8 +513,8 @@ it('rejects a screenshot from another window before attachment admission', async
onTestFinished(async () => { try { await f.ctx.fiber.dispose() } finally { await rm(root, { recursive: true, force: true }) } })
await f.ctx.plugin(LocalAttachmentStore, { dshHome: root })
const save = vi.spyOn(f.ctx.attachments, 'saveImages')
native.request.mockResolvedValueOnce({ ...imageMetadata, windowId: 8, mimeType: 'image/png', data: imageData })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
native.request.mockResolvedValueOnce({ ...imageMetadata, windowId: '8', mimeType: 'image/png', data: imageData })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
expect(result.isError).toBe(true)
expect(JSON.stringify(result.content)).toContain('NATIVE_PROTOCOL')
expect(save).not.toHaveBeenCalled()
@@ -528,7 +528,7 @@ it.each([{ pixelWidth: 2 }, { pixelHeight: 2 }])('rejects native screenshot dime
onTestFinished(async () => { await f.ctx.fiber.dispose(); await rm(root, { recursive: true, force: true }) })
await f.ctx.plugin(LocalAttachmentStore, { dshHome: root })
native.request.mockResolvedValue({ ...imageMetadata, ...invalid, mimeType: 'image/png', data: imageData })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
expect(result.isError).toBe(true)
expect(JSON.stringify(result.content)).toContain('dimensions disagree')
expect(result.content.some(block => block.type === 'image')).toBe(false)
@@ -538,7 +538,7 @@ it('refuses screenshot capture when no attachment store is mounted', async () =>
const f = await setup({ provider: 'image-fixture', model: 'vision' })
await selectApplication(f)
native.request.mockClear()
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
expect(result.isError).toBe(true)
expect(JSON.stringify(result.content)).toContain('require an attachment store')
expect(native.request).not.toHaveBeenCalled()
@@ -558,7 +558,7 @@ it.each(['unresolved', 'unknown', 'text-only', 'provider-error'] as const)(
...route === 'text-only' ? { inputModalities: ['text'] } : {},
})
native.request.mockClear()
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
expect(result.isError).toBe(true)
expect(JSON.stringify(result.content)).toContain(route === 'unresolved' ? 'model route could not be resolved'
: route === 'provider-error' ? 'model lookup failed' : 'declare image input')
@@ -579,7 +579,7 @@ it.each([
await f.ctx.plugin(LocalAttachmentStore, { dshHome: root })
vi.spyOn(f.ctx.attachments, 'readImageRequest').mockRejectedValue(failure)
native.request.mockResolvedValue({ ...imageMetadata, mimeType: 'image/png', data: imageData })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
const content = JSON.stringify(result.content)
expect(result.isError).toBe(true)
expect(content).toContain('SCREENSHOT_IMAGE_FAILED: Screenshot model image preparation failed for window 7')
@@ -595,7 +595,7 @@ it('returns an image-admission failure without emitting a successful screenshot'
onTestFinished(async () => { await f.ctx.fiber.dispose(); await rm(root, { recursive: true, force: true }) })
await f.ctx.plugin(LocalAttachmentStore, { dshHome: root })
native.request.mockResolvedValue({ ...imageMetadata, mimeType: 'image/png', data: 'AQ==' })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
const result = await f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
expect(result.isError).toBe(true)
expect(result.content.some(block => block.type === 'image')).toBe(false)
expect(JSON.stringify(result.content)).toContain('INVALID_IMAGE: Screenshot image admission failed for window 7')
@@ -631,7 +631,7 @@ it.each([
if (stage === 'storage') { entered.resolve(); await release.promise }
return saveImages(images)
})
const pending = f.execute('screenshot', { appHandle: 'app-one', windowId: 7 })
const pending = f.execute('screenshot', { appHandle: 'app-one', windowId: '7' })
let closing: Promise<void> | undefined
try {
const reached = await Promise.race([entered.promise.then(() => true), pending.then(() => false)])
@@ -1,4 +1,5 @@
// Private desktop fixture: commands inspect or arrange only the windows owned by this process.
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Security.Cryptography;
@@ -26,7 +27,7 @@ internal sealed class FixtureForm : Form
internal object Inspect() => new
{
text = Editor.Text, clicks, window = Handle.ToInt64(), foreground = GetForegroundWindow().ToInt64(),
text = Editor.Text, clicks, window = Handle.ToString(CultureInfo.InvariantCulture), foreground = GetForegroundWindow().ToString(CultureInfo.InvariantCulture),
cursor = new { x = Cursor.Position.X, y = Cursor.Position.Y },
button = new { x = button.Left + button.Width / 2, y = button.Top + button.Height / 2 },
selection = new { start = Editor.SelectionStart, length = Editor.SelectionLength },
@@ -8,7 +8,7 @@ using DeepSeek.Cua.Protocol;
namespace DeepSeek.Cua.Windows;
internal sealed record ElementReference(AutomationElement Element, WindowState Window);
internal sealed record Snapshot(string Id, uint Scope, WindowState[] Windows, IReadOnlyDictionary<int, ElementReference> Elements);
internal sealed record Snapshot(string Id, nint Scope, WindowState[] Windows, IReadOnlyDictionary<int, ElementReference> Elements);
internal static class Accessibility
{
@@ -25,7 +25,7 @@ internal static class Accessibility
TogglePattern.ToggleStateProperty, ExpandCollapsePattern.ExpandCollapseStateProperty,
];
internal static object State(AppState app, uint selected, int maximumElements, int maximumDepth,
internal static object State(AppState app, nint selected, int maximumElements, int maximumDepth,
bool includeMenus, bool details, RequestContext context)
{
var all = Windows.Refresh(app, context);
@@ -68,8 +68,8 @@ internal static class Accessibility
var snapshot = new Snapshot(Guid.NewGuid().ToString("N"), scope, targets, elements);
var result = new
{
appHandle = app.Handle, pid = 4, windowId = selected,
focusedWindowId = (uint?)Windows.Focused(app, all)?.Id,
appHandle = app.Handle, pid = 4, windowId = selected.ToString(CultureInfo.InvariantCulture),
focusedWindowId = Windows.Focused(app, all)?.Id.ToString(CultureInfo.InvariantCulture),
snapshotId = snapshot.Id, tree = string.Join('\n', rows), windows = all.Select(window => window.Describe(app, context)).ToArray(), truncated,
};
context.Check();
@@ -234,7 +234,7 @@ internal static class Accessibility
return element;
}
internal static void RequireSnapshot(AppState app, string snapshotId, uint windowId)
internal static void RequireSnapshot(AppState app, string snapshotId, nint windowId)
{
var selected = app.SelectedWindow;
var snapshot = app.Snapshots.Values.SingleOrDefault(value => value.Id == snapshotId);
@@ -21,14 +21,8 @@ internal sealed class AppState(string handle, ApplicationIdentity authorization,
internal string Digest { get; } = digest;
internal RunningApplication? Process { get; set; } = process;
internal Dictionary<nint, WindowState> Windows { get; } = [];
internal Dictionary<uint, Snapshot> Snapshots { get; } = [];
private uint nextWindowId;
internal Dictionary<nint, Snapshot> Snapshots { get; } = [];
private System.Diagnostics.Process? owner;
internal uint AllocateWindowId()
{
if (nextWindowId == uint.MaxValue) throw new NativeFailure("NATIVE_LIMIT", "Select the application again to allocate new window identifiers");
return ++nextWindowId;
}
internal RunningApplication RequireProcess()
{
var selected = Process ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Application launch requires Host authorization");
@@ -1,4 +1,5 @@
// The shared Host policy authorizes handles before these platform operations execute.
using System.Globalization;
using System.Text.Json;
using DeepSeek.Cua.Protocol;
@@ -6,7 +7,6 @@ namespace DeepSeek.Cua.Windows;
internal sealed class ClientState
{
internal DesktopWindows Desktop { get; } = new();
internal Dictionary<string, PreparedLaunch> Launches { get; } = new(StringComparer.Ordinal);
internal Dictionary<string, AppState> Apps { get; } = new(StringComparer.Ordinal);
internal AppState App(string handle) => Apps.GetValueOrDefault(handle)
@@ -28,7 +28,9 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
private static string String(JsonElement value, string key) => value.GetProperty(key).GetString()!;
private static string? OptionalString(JsonElement value, string key) => value.TryGetProperty(key, out var item) ? item.GetString() : null;
private static bool Flag(JsonElement value, string key) => value.TryGetProperty(key, out var item) && item.GetBoolean();
private static uint? WindowId(JsonElement value) => value.TryGetProperty("windowId", out var id) ? id.GetUInt32() : null;
private static nint ParseWindowId(JsonElement value) => nint.TryParse(value.GetString(), CultureInfo.InvariantCulture, out var id)
? id : throw new NativeFailure("INVALID_ARGUMENT", "windowId is outside the native handle range");
private static nint? WindowId(JsonElement value) => value.TryGetProperty("windowId", out var id) ? ParseWindowId(id) : null;
private static string[] Modifiers(JsonElement value) => value.TryGetProperty("modifiers", out var items) ? items.EnumerateArray().Select(item => item.GetString()!).ToArray() : [];
// Window-scoped operations act only on the window authorized at getApp; the request must name it.
@@ -36,8 +38,7 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
{
if (!parameters.TryGetProperty("windowId", out var raw))
throw new NativeFailure("UNSUPPORTED", "This operation requires a window target; select a window with getApp and pass its windowId");
if (!raw.TryGetUInt32(out var id))
throw new NativeFailure("INVALID_ARGUMENT", "windowId must be a non-negative 32-bit identifier");
var id = ParseWindowId(raw);
var selected = app.SelectedWindow ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Select a window with getApp before this operation");
if (selected.Id != id)
throw new NativeFailure("WINDOW_CHANGED", "The request targets another window; select that window with getApp");
@@ -87,12 +88,12 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
if (request.Method is "requestPermission" or "openPermissionSettings")
throw new NativeFailure("UNSUPPORTED", "Windows desktop access has no macOS permission switches; inspect the Windows session status");
DesktopSession.RequireInteractive();
if (request.Method == "listApps") return state.Desktop.ListApps(context);
if (request.Method == "listApps") return DesktopWindows.ListApps(context);
if (request.Method == "findApps")
throw new NativeFailure("UNSUPPORTED", "Windows discovers selectable windows with listApps; installed launch entries are not part of this selection");
if (request.Method == "getApp")
{
var selected = state.Desktop.Select(String(parameters, "selector"), context);
var selected = DesktopWindows.Select(String(parameters, "selector"), context);
state.Apps[selected.Handle] = selected;
return new { appHandle = selected.Handle, pid = 4, name = selected.Authorization.Name,
running = true, authorization = selected.Authorization };
@@ -1,18 +1,19 @@
// Discovery IDs are connection-owned and never reused for another HWND lifetime.
// Desktop discovery selects windows by their invariant decimal HWND strings.
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using DeepSeek.Cua.Protocol;
namespace DeepSeek.Cua.Windows;
internal sealed record DiscoveredWindow(WindowState Window, RunningApplication Process, string Selector);
internal sealed class DesktopWindows
internal sealed record DiscoveredWindow(WindowState Window, RunningApplication Process)
{
private readonly Dictionary<nint, DiscoveredWindow> windows = [];
private uint nextId;
internal string Selector => Window.Id.ToString(CultureInfo.InvariantCulture);
}
internal object[] ListApps(RequestContext context)
internal static class DesktopWindows
{
internal static object[] ListApps(RequestContext context)
{
var result = new List<object>();
foreach (var entry in Discover(context))
@@ -26,25 +27,23 @@ internal sealed class DesktopWindows
}
catch (NativeFailure error) when (error.Code is "STALE_APP" or "PERMISSION_DENIED" or "APP_IDENTITY_UNAVAILABLE" or "UNSUPPORTED_SESSION" or "WINDOW_CHANGED")
{
// A disappearing window cannot remain selectable from this connection.
windows.Remove(entry.Window.Handle);
// A disappearing or inaccessible window is omitted from discovery.
}
}
return result.ToArray();
}
internal AppState Select(string selector, RequestContext context)
internal static AppState Select(string selector, RequestContext context)
{
if (string.IsNullOrEmpty(selector)) throw new NativeFailure("INVALID_ARGUMENT", "A window selector from listApps is required");
Discover(context);
var entry = windows.Values.SingleOrDefault(value => string.Equals(value.Selector, selector, StringComparison.Ordinal))
?? throw new NativeFailure("WINDOW_CHANGED", "Discover applications again; this window selector belongs to another connection or an expired window");
var entry = Discover(context).SingleOrDefault(value => string.Equals(value.Selector, selector, StringComparison.Ordinal))
?? throw new NativeFailure("WINDOW_CHANGED", "Discover applications again; this window is no longer available");
var app = Owner(entry, true, context);
try { entry.Window.Geometry(app, context); return app; }
catch { app.Dispose(); throw; }
}
private List<DiscoveredWindow> Discover(RequestContext context)
private static List<DiscoveredWindow> Discover(RequestContext context)
{
var current = new List<nint>();
Win32.EnumWindows((handle, _) =>
@@ -63,25 +62,16 @@ internal sealed class DesktopWindows
{
Win32.GetWindowThreadProcessId(handle, out var pid);
var process = Applications.Running(checked((int)pid));
var epoch = context.Sta.WindowEpoch(handle);
if (!windows.TryGetValue(handle, out var entry) || entry.Process.Pid != process.Pid
|| entry.Process.Started != process.Started || entry.Window.Epoch != epoch)
{
if (nextId == uint.MaxValue) throw new NativeFailure("NATIVE_LIMIT", "Reconnect before discovering more windows");
entry = new(new(++nextId, handle, epoch), process, Guid.NewGuid().ToString("N"));
windows[handle] = entry;
}
var entry = new DiscoveredWindow(new(handle), process);
using var owner = Owner(entry, false, context);
entry.Window.Geometry(owner, context);
result.Add(entry);
}
catch (NativeFailure error) when (error.Code is "STALE_APP" or "PERMISSION_DENIED" or "APP_IDENTITY_UNAVAILABLE" or "UNSUPPORTED_SESSION" or "WINDOW_CHANGED")
{
// A disappearing or inaccessible window cannot be selected from this discovery.
windows.Remove(handle);
// A disappearing or inaccessible window is omitted from discovery.
}
}
foreach (var handle in windows.Keys.Where(handle => !current.Contains(handle)).ToArray()) windows.Remove(handle);
return result;
}
@@ -94,15 +84,14 @@ internal sealed class DesktopWindows
var title = new StringBuilder(length + 1);
Win32.GetWindowText(window.Handle, title, title.Capacity);
var name = $"Window: {title} ({process.Name})";
var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(
$"window\0{process.Pid}\0{process.Started}\0{window.Handle}\0{window.Epoch}")));
// Shared consent IDs require a SHA-256 string, including non-persistable window grants.
var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes($"window\0{entry.Selector}")));
var identity = new ApplicationIdentity(id, name, process.Path, false);
var app = new AppState(Guid.NewGuid().ToString("N"), identity, "", process);
if (authorize)
{
var selected = new WindowState(window.Id, window.Handle, window.Epoch);
app.SelectedWindow = selected;
app.Windows.Add(window.Handle, selected);
app.SelectedWindow = window;
app.Windows.Add(window.Handle, window);
}
return app;
}
@@ -94,43 +94,27 @@ internal sealed class StaDispatcher : IAsyncDisposable
{
private readonly TaskCompletionSource<ServiceControl> ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource<object?> stopped = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly ConcurrentDictionary<nint, long> epochs = new();
private readonly Thread thread;
private Win32.WinEvent? windowEvents;
internal StaDispatcher()
{
thread = new Thread(() =>
{
nint hook = 0;
try
{
Win32.SetThreadDpiAwarenessContext(-4);
using var control = new ServiceControl();
_ = control.Handle;
windowEvents = (_, eventType, window, objectId, childId, _, _) =>
{
if (eventType == 0x8001 && objectId == 0 && childId == 0 && epochs.TryGetValue(window, out var epoch))
epochs.TryUpdate(window, epoch + 1, epoch);
};
hook = Win32.SetWinEventHook(0x8001, 0x8001, 0, windowEvents, 0, 0, 0);
if (hook == 0) throw new NativeFailure("NATIVE_STARTUP", "Window lifecycle observation could not start");
ready.SetResult(control);
Forms.Application.Run();
}
catch (Exception error) { ready.TrySetException(error); }
finally
{
if (hook != 0) Win32.UnhookWinEvent(hook);
stopped.TrySetResult(null);
}
finally { stopped.TrySetResult(null); }
}) { IsBackground = true, Name = "dsh-cua-clipboard" };
thread.SetApartmentState(ApartmentState.STA);
thread.Start();
}
internal long WindowEpoch(nint window) => epochs.GetOrAdd(window, 0);
internal async Task<T> Invoke<T>(Func<ServiceControl, T> work, CancellationToken cancellation)
{
var control = await ready.Task.WaitAsync(cancellation);
@@ -11,7 +11,6 @@ internal static class Win32
[DllImport("user32.dll", EntryPoint = "GetWindowLongPtrW")] internal static extern nint GetWindowLongPtr(nint window, int index);
[DllImport("user32.dll")] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool GetCursorPos(out Point point);
internal delegate bool WindowVisitor(nint window, nint parameter);
internal delegate void WinEvent(nint hook, uint eventType, nint window, int objectId, int childId, uint threadId, uint time);
[StructLayout(LayoutKind.Sequential)] internal struct Point { internal int X, Y; }
[StructLayout(LayoutKind.Sequential)] internal struct Rect
@@ -74,8 +73,6 @@ internal static class Win32
[DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool GetUserObjectInformation(nint handle, int index, StringBuilder value, int length, out int needed);
[DllImport("wtsapi32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool WTSQuerySessionInformation(nint server, int session, int information, out nint buffer, out int bytes);
[DllImport("wtsapi32.dll")] internal static extern void WTSFreeMemory(nint memory);
[DllImport("user32.dll", SetLastError = true)] internal static extern nint SetWinEventHook(uint minimum, uint maximum, nint module, WinEvent callback, uint process, uint thread, uint flags);
[DllImport("user32.dll")] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool UnhookWinEvent(nint hook);
[DllImport("user32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool OpenClipboard(nint owner);
[DllImport("user32.dll")] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool CloseClipboard();
@@ -1,4 +1,5 @@
// Public window IDs never expose or revive a reused HWND.
// Window state retains HWND values; JSON exposes them as invariant decimal strings.
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text;
using DeepSeek.Cua.Protocol;
@@ -11,20 +12,18 @@ internal readonly record struct WindowGeometry(Win32.Rect Pixels, uint Dpi)
internal Bounds Bounds => new(Pixels.Left, Pixels.Top, Pixels.Width, Pixels.Height);
}
internal sealed class WindowState(uint id, nint handle, long epoch)
internal sealed class WindowState(nint handle)
{
internal uint Id { get; } = id;
internal nint Id => Handle;
internal nint Handle { get; } = handle;
internal long Epoch { get; } = epoch;
private WindowGeometry? observation;
internal WindowGeometry Geometry(AppState app, RequestContext context)
{
context.Check();
var process = app.RequireProcess();
Win32.GetWindowThreadProcessId(Handle, out var pid);
if (!Win32.IsWindow(Handle) || pid != process.Pid || context.Sta.WindowEpoch(Handle) != Epoch)
throw new NativeFailure("WINDOW_CHANGED", "The selected window was destroyed or replaced; read fresh application state");
app.RequireProcess();
if (!Win32.IsWindow(Handle))
throw new NativeFailure("WINDOW_CHANGED", "The selected window is no longer available; read fresh application state");
if (Win32.DwmGetWindowAttribute(Handle, 9, out Win32.Rect rectangle, Marshal.SizeOf<Win32.Rect>()) != 0
&& !Win32.GetWindowRect(Handle, out rectangle)) throw new NativeFailure("WINDOW_CHANGED", "The selected window geometry is unavailable");
var dpi = Win32.GetDpiForWindow(Handle);
@@ -64,7 +63,7 @@ internal sealed class WindowState(uint id, nint handle, long epoch)
Win32.GetWindowText(Handle, title, title.Capacity);
return new
{
windowId = Id, pid = 4, appName = app.RequireProcess().Name,
windowId = Id.ToString(CultureInfo.InvariantCulture), pid = 4, appName = app.RequireProcess().Name,
title = title.ToString(), bounds = geometry.Bounds, isOnScreen = IsOnScreen(), onCurrentSpace = (bool?)null,
};
}
@@ -92,7 +91,7 @@ internal static class Windows
}, 0);
foreach (var pair in app.Windows.ToArray())
{
if (handles.Contains(pair.Key) && pair.Value.Epoch == context.Sta.WindowEpoch(pair.Key)) continue;
if (handles.Contains(pair.Key)) continue;
app.Windows.Remove(pair.Key);
foreach (var scope in app.Snapshots.Where(item => item.Value.Windows.Any(window => window.Id == pair.Value.Id)).Select(item => item.Key).ToArray())
app.Snapshots.Remove(scope);
@@ -101,13 +100,13 @@ internal static class Windows
{
if (!app.Windows.ContainsKey(handle))
{
app.Windows.Add(handle, new(app.AllocateWindowId(), handle, context.Sta.WindowEpoch(handle)));
app.Windows.Add(handle, new(handle));
}
}
return handles.Select(handle => app.Windows[handle]).ToArray();
}
internal static WindowState Resolve(AppState app, uint id, RequestContext context) => Refresh(app, context)
internal static WindowState Resolve(AppState app, nint id, RequestContext context) => Refresh(app, context)
.SingleOrDefault(window => window.Id == id) ?? throw new NativeFailure("WINDOW_CHANGED", "Select a current window from getWindows");
internal static WindowState? Focused(AppState app, WindowState[] windows)
@@ -139,7 +138,7 @@ internal static class Windows
throw new NativeFailure("AMBIGUOUS_WINDOW", "Select a window explicitly from getWindows");
}
internal static object Show(AppState app, uint? id, RequestContext context)
internal static object Show(AppState app, nint? id, RequestContext context)
{
var target = id is { } selected ? Resolve(app, selected, context) : Default(app, Refresh(app, context));
var before = Win32.GetForegroundWindow();
@@ -149,7 +148,7 @@ internal static class Windows
if (!Win32.ShowWindowAsync(target.Handle, 4)) throw new NativeFailure("UNSUPPORTED", "Windows refused background restoration; explicitly activate the application if needed");
}
var after = Win32.GetForegroundWindow();
return new { dispatched = true, windowId = (uint?)target?.Id,
return new { dispatched = true, windowId = target?.Id.ToString(CultureInfo.InvariantCulture),
foregroundChanged = before == 0 || after == 0 ? (bool?)null : before != after };
}
@@ -168,9 +168,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
}
},
"required": [
@@ -235,9 +234,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"includeMenus": {
"description": "Expand menu contents; omitted or false shows menu-bar entries only.",
@@ -291,9 +289,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -338,9 +335,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -427,9 +423,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -476,9 +471,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -505,9 +499,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -560,9 +553,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -628,9 +620,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -720,9 +711,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -805,9 +795,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -871,9 +860,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"text": {
"type": "string"
@@ -911,9 +899,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"text": {
"type": "string"
@@ -982,9 +969,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"key": {
"type": "string",
@@ -1029,9 +1015,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"key": {
"type": "string",
@@ -1078,9 +1063,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
@@ -1141,9 +1125,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
},
"mouseButton": {
"type": "string",
@@ -1235,9 +1218,8 @@
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
"type": "string",
"minLength": 1
}
},
"required": [
@@ -16,7 +16,7 @@ void Refuses(Action action, string code)
var request = NativeRequest.Parse(WireJson.Parse("""{"id":"one","method":"getState","params":{"appHandle":"app","maxElements":20,"maxDepth":5}}"""));
Check(request.Method == "getState" && request.Id == "one", "Valid requests retain correlation and method");
Refuses(() => NativeRequest.Parse(WireJson.Parse("""{"id":"one","method":"click","params":{"appHandle":"app","windowId":1,"x":-1,"y":2}}""")), "INVALID_ARGUMENT");
Refuses(() => NativeRequest.Parse(WireJson.Parse("""{"id":"one","method":"click","params":{"appHandle":"app","windowId":"1","x":-1,"y":2}}""")), "INVALID_ARGUMENT");
Refuses(() => NativeRequest.Parse(WireJson.Parse("""{"id":"one","method":"getState","params":{"appHandle":"app","maxElements":0,"maxDepth":5}}""")), "INVALID_ARGUMENT");
Refuses(() => NativeRequest.Parse(WireJson.Parse("""{"id":"one","method":"typeText","params":{"appHandle":"app","text":"hello","unowned":1}}""")), "INVALID_ARGUMENT");
Refuses(() => WireJson.Parse("""{"id":"one","id":"two"}"""), "INVALID_REQUEST");
@@ -58,6 +58,7 @@ it.skipIf(!helperPath || !selector)('admits real application screenshots through
const target = (await app.getWindows({ emit: false })).find(window =>
window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0)
expect(target).toBeDefined()
expect(target!.windowId).toBe(discovered!.id)
const window = await app.getWindow(target!.windowId)
expect(emitted).toEqual([])
await window.getAccessibilityTree()
@@ -10,7 +10,7 @@ it('keeps the C# admission schema identical to the shared generated schema', asy
})
it('preserves Windows physical capture units and observed foreground changes', () => {
const image = { mimeType: 'image/png', data: 'AAAA', pixelWidth: 1200, pixelHeight: 800, windowId: 1,
const image = { mimeType: 'image/png', data: 'AAAA', pixelWidth: 1200, pixelHeight: 800, windowId: '1',
bounds: { x: -1200, y: 0, width: 1200, height: 800 }, coordinateSpace: 'window-pixels' }
expect(NativeScreenshotResult.parse(image).coordinateSpace).toBe('window-pixels')
expect(() => NativeScreenshotResult.parse({ ...image, coordinateSpace: 'screen-pixels' })).toThrow()
+4 -4
View File
@@ -92,19 +92,19 @@ type CuaScopeParams = (params: Record<string, JsonValue>) => Record<string, Json
function createCuaObjects(call: ReturnType<typeof validatedCall>, sink: CuaObservationSink | undefined): {
createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp
} {
const observed = new Map<CuaAppHandle, { snapshotId: string; windowId: number | undefined }>()
const observed = new Map<CuaAppHandle, { snapshotId: string; windowId: string | undefined }>()
function currentSnapshot(appHandle: CuaAppHandle): string {
const current = observed.get(appHandle)
if (current === undefined) throw new Error('Read getAccessibilityTree before using this surface')
return current.snapshotId
}
function coordinateParams(appHandle: CuaAppHandle, windowId: number): Record<string, JsonValue> {
function coordinateParams(appHandle: CuaAppHandle, windowId: string): Record<string, JsonValue> {
const current = observed.get(appHandle)
return current !== undefined && (current.windowId === undefined || current.windowId === windowId)
? { snapshotId: current.snapshotId } : {}
}
function createCuaSurface(
target: { appHandle: CuaAppHandle; windowId?: number },
target: { appHandle: CuaAppHandle; windowId?: string },
paramsFor: CuaScopeParams,
requireAccessibility: () => void,
requireKeyboard: () => void,
@@ -177,7 +177,7 @@ function createCuaObjects(call: ReturnType<typeof validatedCall>, sink: CuaObser
}
return { surface, call: scopedCall, action, elementParams }
}
function createCuaWindow(appHandle: CuaAppHandle, windowId: number, capabilities: CuaCapabilities): CuaWindow {
function createCuaWindow(appHandle: CuaAppHandle, windowId: string, capabilities: CuaCapabilities): CuaWindow {
const base = createCuaSurface({ appHandle, windowId },
params => ({ ...params, appHandle, windowId }), () => undefined, () => {
if (!capabilities.keyboard.includes('window')) {
+4 -4
View File
@@ -12,7 +12,7 @@ interface Frame {
/** Tracks only the current window geometry and last successfully prepared screenshot. */
export class CuaCoordinates {
private readonly applications = new Map<CuaAppHandle, Map<number, Frame>>()
private readonly applications = new Map<CuaAppHandle, Map<string, Frame>>()
/**
* Invalidate screenshot mappings when a window closes or changes size.
@@ -20,9 +20,9 @@ export class CuaCoordinates {
* @param windows - freshly observed windows.
* @param windowId - selected window for a scoped observation; omission means a complete application inventory.
*/
observe(appHandle: CuaAppHandle, windows: readonly CuaWindowInfo[], windowId?: number): void {
observe(appHandle: CuaAppHandle, windows: readonly CuaWindowInfo[], windowId?: string): void {
const previous = this.applications.get(appHandle)
const current = new Map<number, Frame>(windowId === undefined ? [] : previous)
const current = new Map<string, Frame>(windowId === undefined ? [] : previous)
if (windowId !== undefined && !windows.some(window => window.windowId === windowId)) current.delete(windowId)
for (const window of windows) {
const { width, height } = window.bounds
@@ -40,7 +40,7 @@ export class CuaCoordinates {
* @param image - encoded model-request image dimensions.
*/
screenshot(
appHandle: CuaAppHandle, windowId: number, bounds: { width: number; height: number }, image: { width: number; height: number },
appHandle: CuaAppHandle, windowId: string, bounds: { width: number; height: number }, image: { width: number; height: number },
): void {
let frames = this.applications.get(appHandle)
if (!frames) { frames = new Map(); this.applications.set(appHandle, frames) }
+6 -6
View File
@@ -4,7 +4,7 @@ export const COMMON_GUIDANCE = `Use cua to observe and operate native applicatio
In persistent PTC, use the host-provided bridge initialization once and retain the returned objects; cua comes from that bridge and is not a preinstalled global. Use void await for observations that emit, and never stringify screenshot image bytes.
Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level.
Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: string and the methods listed below for window level. Window IDs are opaque provider-issued strings; preserve them verbatim.
Setup and discovery:
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
@@ -12,11 +12,11 @@ Setup and discovery:
- cua.findApps(query: string, options?: { emit?: boolean }) -> { id: string, displayName: string, path: string, isRunning: boolean }[] — search installed applications, including stopped matches.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
- app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
- app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
- app.getWindows(options?: { emit?: boolean }) -> { windowId: string, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
- app.getWindow(windowId: string) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.
- app.show() -> { dispatched: boolean, windowId: number | null, foregroundChanged: boolean | null } — unhide the app and restore its selected, focused, main, or sole window without activation.
- window.show() -> { dispatched: boolean, windowId: number | null, foregroundChanged: boolean | null } — unhide and restore that explicit window without activation.
- app.show() -> { dispatched: boolean, windowId: string | null, foregroundChanged: boolean | null } — unhide the app and restore its selected, focused, main, or sole window without activation.
- window.show() -> { dispatched: boolean, windowId: string | null, foregroundChanged: boolean | null } — unhide and restore that explicit window without activation.
## Accessibility tree
Available at app or window level depending on the platform.
@@ -31,7 +31,7 @@ Element ids are opaque strings from the latest successful tree; the object facad
## Simulated mouse
Use an explicit window for screenshots and pointer input.
- getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG.
- getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: string } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG.
- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — reveal AX alternate UI or move the system pointer without pressing; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration.
- click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press.
- scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left.
+15 -14
View File
@@ -3,7 +3,8 @@ import { Buffer } from 'node:buffer'
import { z } from 'zod'
import type { CuaAppHandle, CuaSnapshotId } from './types.ts'
const id = z.number().int().positive().max(2 ** 32 - 1)
const windowId = z.string().min(1)
const pixelDimension = z.number().int().positive().max(2 ** 32 - 1)
const point = z.number().nonnegative()
const relativePoint = z.number().int().min(0).max(999)
const app = { appHandle: z.string().min(1) }
@@ -12,12 +13,12 @@ const pointer = { mouseButton: z.enum(['left', 'right', 'middle']).optional(),
const click = { ...pointer, clickCount: z.union([z.literal(1), z.literal(2)]).optional(),
holdMs: z.number().int().nonnegative().optional()
.describe('Milliseconds between each mouse press and release; omitted for an ordinary click.') }
const window = { ...app, windowId: id }
const window = { ...app, windowId }
const jsonValue = z.json()
/** Provider-issued element reference from an accessibility observation. */
export const elementId = z.string().min(1)
.describe('Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.')
const snapshot = { ...app, windowId: id.optional(), snapshotId: z.string().min(1), elementId }
const snapshot = { ...app, windowId: windowId.optional(), snapshotId: z.string().min(1), elementId }
const coordinate = { ...window, snapshotId: z.string().min(1).optional(), x: point, y: point }
const absoluteCoordinate = { ...coordinate, relative: z.literal(false).optional().describe('Omitted or false uses pixels in the latest screenshot returned by this tool.') }
const relativeCoordinate = { ...coordinate, relative: z.literal(true).describe('Use integer thousandths of the window width and height, from 0 to 999.'), x: relativePoint, y: relativePoint }
@@ -39,9 +40,9 @@ export const Request = z.discriminatedUnion('method', [
operation('findApps', { query: z.string().min(1, 'Expected a non-empty search string').regex(/\S/, 'Expected a non-empty search string') }),
operation('getApp', { selector: z.string().min(1) }),
operation('activate', app),
operation('show', { ...app, windowId: id.optional() }),
operation('show', { ...app, windowId: windowId.optional() }),
operation('listWindows', app),
operation('getState', { ...app, windowId: id.optional(),
operation('getState', { ...app, windowId: windowId.optional(),
includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'),
details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'),
}),
@@ -59,10 +60,10 @@ export const Request = z.discriminatedUnion('method', [
operation('setValue', { ...snapshot, value: jsonValue }),
operation('selectText', { ...snapshot, text: z.string().min(1), prefix: z.string().optional(), suffix: z.string().optional(),
selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }),
operation('typeText', { ...app, windowId: id.optional(), text: z.string() }),
operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }),
operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }),
operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }),
operation('typeText', { ...app, windowId: windowId.optional(), text: z.string() }),
operation('paste', { ...app, windowId: windowId.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }),
operation('pressKey', { ...app, windowId: windowId.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }),
operation('sendKeyEvent', { ...app, windowId: windowId.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }),
z.object({ method: z.literal('scroll'), params: z.union([
z.object({ ...absoluteCoordinate,
deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(),
@@ -108,14 +109,14 @@ export const FoundAppsResult = z.array(z.object({
}))
/** Current windows and their global native bounds. */
export const WindowsResult = z.array(z.object({
windowId: id, appName: z.string(), title: z.string(), bounds,
windowId, appName: z.string(), title: z.string(), bounds,
isOnScreen: z.boolean(), onCurrentSpace: z.boolean().nullable(),
}))
/** Observed state for the selected application or window. */
export const StateResult = z.object({
appHandle: z.string().min(1).transform(value => value as CuaAppHandle),
windowId: id.nullable(),
focusedWindowId: id.nullable().optional(),
windowId: windowId.nullable(),
focusedWindowId: windowId.nullable().optional(),
snapshotId: z.string().min(1).transform(value => value as CuaSnapshotId),
tree: z.string(), windows: WindowsResult, truncated: z.boolean(),
})
@@ -123,7 +124,7 @@ export const StateResult = z.object({
export const ActionNamesResult = z.array(z.string())
/** Original native PNG and platform window geometry before Host image preparation. */
export const NativeScreenshotResult = z.object({
mimeType: z.literal('image/png'), data: imageBase64, pixelWidth: id, pixelHeight: id, windowId: id,
mimeType: z.literal('image/png'), data: imageBase64, pixelWidth: pixelDimension, pixelHeight: pixelDimension, windowId,
bounds: bounds.extend({ width: z.number().positive(), height: z.number().positive() }),
coordinateSpace: z.enum(['window-points', 'window-pixels']),
})
@@ -151,7 +152,7 @@ export const ActionResult = z.object({
/** Hover reports whether the selected route moved the user's system pointer. */
export const HoverResult = ActionResult.extend({ hoverRoute: z.enum(['accessibility', 'system-pointer']), systemPointerMoved: z.boolean() })
/** Background restoration dispatch and sampled foreground changes; null means the observation was unavailable. */
export const ShowResult = ActionResult.extend({ windowId: id.nullable(), foregroundChanged: z.boolean().nullable() })
export const ShowResult = ActionResult.extend({ windowId: windowId.nullable(), foregroundChanged: z.boolean().nullable() })
/** Bounded native PNG thumbnail for human presentation, excluded from model results and saved grants. */
export const ApplicationIcon = z.string().max(65536).regex(/^data:image\/png;base64,[A-Za-z0-9+/]+={0,2}$/)
+4 -3
View File
@@ -183,7 +183,8 @@ export interface CuaSurface {
* Coordinate input also rejects resizing since the latest successful AX read or screenshot.
*/
export interface CuaWindow extends CuaSurface {
readonly windowId: number
/** Opaque window identifier issued by the platform provider. */
readonly windowId: string
/**
* Capture the selected window.
* @param options - emission options.
@@ -239,10 +240,10 @@ export interface CuaApp extends CuaSurface {
getWindows(options?: CuaObservationOptions): Promise<CuaWindowInfo[]>
/**
* Observe a specific window.
* @param windowId - ID obtained from getWindows.
* @param windowId - Decimal-string ID obtained from getWindows, passed unchanged.
* @returns A window object with its initial accessibility tree read silently; this updates the application's current snapshot.
*/
getWindow(windowId: number): Promise<CuaWindow>
getWindow(windowId: string): Promise<CuaWindow>
}
/** Native app API; its caller owns global bindings and variable persistence. */
export interface CuaApi {
+119 -103
View File
@@ -15,12 +15,12 @@ const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test
const delivered = { dispatched: true }
const capabilities: CuaCapabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
function windowInfo(windowId: number) {
function windowInfo(windowId: string) {
return { windowId, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null }
}
function fixture(withSink = true) {
let windows = [windowInfo(23)]
let windows = [windowInfo('23')]
let currentCapabilities: CuaCapabilities = capabilities
let generation = 0
const sink = { text: vi.fn<(value: string) => void>(), image: vi.fn<(screenshot: CuaScreenshot) => void>() }
@@ -31,7 +31,7 @@ function fixture(withSink = true) {
case 'getApp': return selected
case 'listWindows': return windows
case 'mouseMove': return { ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false }
case 'show': return { ...delivered, windowId: params.windowId ?? 23, foregroundChanged: false }
case 'show': return { ...delivered, windowId: params.windowId ?? '23', foregroundChanged: false }
case 'getState': return {
appHandle: selected.appHandle, windowId: params.windowId ?? null,
snapshotId: `snapshot-${++generation}`, tree: `Tree ${generation}`, truncated: false,
@@ -44,18 +44,18 @@ function fixture(withSink = true) {
})
return {
dispatch, sink, api: createCuaApi(dispatch, withSink ? sink : undefined),
setWindows: (ids: number[]) => { windows = ids.map(windowInfo) },
setWindows: (ids: string[]) => { windows = ids.map(windowInfo) },
setWindowRecords: (records: ReturnType<typeof windowInfo>[]) => { windows = records },
setCapabilities: (value: CuaCapabilities) => { currentCapabilities = value },
}
}
function state(overrides: Record<string, JsonValue> = {}): JsonValue {
return { appHandle: selected.appHandle, windowId: null, snapshotId: 'changed', tree: 'Changed tree', windows: [windowInfo(23)], truncated: false, ...overrides }
return { appHandle: selected.appHandle, windowId: null, snapshotId: 'changed', tree: 'Changed tree', windows: [windowInfo('23')], truncated: false, ...overrides }
}
function screenshot(overrides: Record<string, JsonValue> = {}): JsonValue {
return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 23, bounds, coordinateSpace: 'screenshot-pixels', ...overrides }
return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: '23', bounds, coordinateSpace: 'screenshot-pixels', ...overrides }
}
describe('application discovery', () => {
@@ -93,12 +93,12 @@ describe('application discovery', () => {
expect(app).not.toHaveProperty('pid')
expectTypeOf<CuaApp>().not.toHaveProperty('pid')
expect(app.appHandle).toBe('app-17')
const window = await app.getWindow(23)
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
const window = await app.getWindow('23')
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: '23' })
expect(f.dispatch.mock.calls.map(([operation]) => operation.method)).toEqual([
'getCapabilities', 'listApps', 'getApp', 'getState', 'getState', 'screenshot',
])
expect(await app.getWindows()).toEqual([windowInfo(23)])
expect(await app.getWindows()).toEqual([windowInfo('23')])
expect(f.sink.text).not.toHaveBeenCalled()
expect(f.sink.image).not.toHaveBeenCalled()
})
@@ -138,13 +138,13 @@ describe('background window restoration', () => {
it('shows an app or selected window without dispatching activation or AXRaise', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(24)
const window = await app.getWindow('24')
f.dispatch.mockClear()
expect(await app.show()).toEqual({ ...delivered, windowId: 23, foregroundChanged: false })
expect(await window.show()).toEqual({ ...delivered, windowId: 24, foregroundChanged: false })
expect(await app.show()).toEqual({ ...delivered, windowId: '23', foregroundChanged: false })
expect(await window.show()).toEqual({ ...delivered, windowId: '24', foregroundChanged: false })
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'show', params: { appHandle: 'app-17' } },
{ method: 'show', params: { appHandle: 'app-17', windowId: 24 } },
{ method: 'show', params: { appHandle: 'app-17', windowId: '24' } },
])
})
@@ -171,22 +171,22 @@ describe('mouseMove across observations', () => {
it('moves to an AX element then into a popup without inserting clicks or restoring a point', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
f.dispatch.mockClear()
expect(await window.mouseMove(element('0'))).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
await window.getAccessibilityTree({ emit: false })
await window.getScreenshot({ emit: false })
expect(await window.mouseMove([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'mouseMove', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: 23 } },
{ method: 'getState', params: { appHandle: 'app-17', windowId: 23 } },
{ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } },
{ method: 'mouseMove', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 23, x: 30, y: 40 } },
{ method: 'mouseMove', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: '23' } },
{ method: 'getState', params: { appHandle: 'app-17', windowId: '23' } },
{ method: 'screenshot', params: { appHandle: 'app-17', windowId: '23' } },
{ method: 'mouseMove', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: '23', x: 30, y: 40 } },
])
const other = await app.getWindow(24)
const other = await app.getWindow('24')
await other.mouseMove([5, 6])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'mouseMove', params: {
appHandle: 'app-17', snapshotId: 'snapshot-4', windowId: 24, x: 5, y: 6,
appHandle: 'app-17', snapshotId: 'snapshot-4', windowId: '24', x: 5, y: 6,
} })
f.dispatch.mockRejectedValueOnce(new Error('WINDOW_CHANGED'))
await expect(other.mouseMove([5, 6])).rejects.toThrow('WINDOW_CHANGED')
@@ -198,7 +198,7 @@ describe('pointer buttons and modifiers', () => {
it.each([0, 800])('preserves %d ms for window clicks at elements and coordinates', async (holdMs) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
for (const target of [element('0'), [10, 20] as const]) {
await window.click(target, { holdMs, mouseButton: 'right' })
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ method: 'click', params: { holdMs, mouseButton: 'right' } })
@@ -211,19 +211,19 @@ describe('pointer buttons and modifiers', () => {
it('retains middle-button and modifier intent through click and drag dispatch', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
await window.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: 23, mouseButton: 'middle', modifiers: ['shift'],
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: '23', mouseButton: 'middle', modifiers: ['shift'],
} })
await window.click([10, 20], { modifiers: ['command'] })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20,
appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', x: 10, y: 20,
mouseButton: 'left', clickCount: 1, modifiers: ['command'],
} })
await window.drag([10, 20], [30, 40], { mouseButton: 'middle', modifiers: ['shift', 'option'] })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'drag', params: {
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', fromX: 10, fromY: 20, toX: 30, toY: 40,
appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', fromX: 10, fromY: 20, toX: 30, toY: 40,
mouseButton: 'middle', modifiers: ['shift', 'option'],
} })
const count = f.dispatch.mock.calls.length
@@ -325,30 +325,47 @@ describe('element default invocation', () => {
})
describe('application and window observations', () => {
it('preserves an opaque window ID through discovery, observation and screenshot', async () => {
const windowId = 'window-main'
const f = fixture()
f.setWindows([windowId])
const app = await f.api.getApp('Editor')
expect((await app.getWindows({ emit: false }))[0]!.windowId).toBe(windowId)
const window = await app.getWindow(windowId)
expectTypeOf(window.windowId).toEqualTypeOf<string>()
expect(window.windowId).toBe(windowId)
const screenshot = await window.getScreenshot({ emit: false })
expect(screenshot.windowId).toBe(windowId)
await window.click([10, 20])
for (const [operation] of f.dispatch.mock.calls) {
if ('windowId' in operation.params) expect(operation.params.windowId).toBe(windowId)
}
})
it('shares the latest tree read across an application and each selected window', async () => {
const f = fixture()
f.setWindows([23, 24])
f.setWindows(['23', '24'])
const app = await f.api.getApp('Editor')
expect(await app.getWindows({ emit: false })).toEqual([windowInfo(23), windowInfo(24)])
const first = await app.getWindow(23)
const second = await app.getWindow(24)
expect(await app.getWindows({ emit: false })).toEqual([windowInfo('23'), windowInfo('24')])
const first = await app.getWindow('23')
const second = await app.getWindow('24')
await app.setValue(element('1'), 'First document')
await first.click(element('2'))
await second.click(element('3'))
expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '1', value: 'First document' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2', windowId: 23 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3', windowId: 24 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2', windowId: '23' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3', windowId: '24' } },
])
expect(first.windowId).toBe(23)
expect(second.windowId).toBe(24)
expect(first.windowId).toBe('23')
expect(second.windowId).toBe('24')
expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual([])
await expect(second.getAccessibilityTree({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4')
await second.click(element('3'))
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3', windowId: 24 } })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3', windowId: '24' } })
await app.getAccessibilityTree({ emit: false })
await first.invoke(element('4'))
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4', windowId: 23 } })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4', windowId: '23' } })
expect(f.sink.text).not.toHaveBeenCalled()
})
@@ -382,7 +399,7 @@ describe('application and window observations', () => {
const app = await f.api.getApp('Editor')
expect(app).toHaveProperty('getAccessibilityTree')
for (const method of ['getScreenshot', 'click', 'mouseMove', 'scroll', 'drag']) expect(app).not.toHaveProperty(method)
const window = await app.getWindow(23)
const window = await app.getWindow('23')
for (const method of ['getScreenshot', 'click', 'mouseMove', 'scroll', 'drag']) expect(window).toHaveProperty(method)
expect(window).not.toHaveProperty('hover')
expect(window).toHaveProperty('mouseMove')
@@ -391,36 +408,36 @@ describe('application and window observations', () => {
it('attaches the shared snapshot to coordinates only within its window scope', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const first = await app.getWindow(23)
const first = await app.getWindow('23')
await first.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
const second = await app.getWindow(24)
const second = await app.getWindow('24')
await first.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
appHandle: 'app-17', windowId: '23', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
await second.click([10, 20])
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-3', windowId: 24 } })
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-3', windowId: '24' } })
await app.getAccessibilityTree({ emit: false })
await first.click([10, 20])
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-4', windowId: 23 } })
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-4', windowId: '23' } })
})
it('keeps a validated AXless window usable for screenshots and coordinate input', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockResolvedValueOnce(state({ windowId: 23, snapshotId: 'axless', tree: '', elements: [] }))
const window = await app.getWindow(23)
f.dispatch.mockResolvedValueOnce(state({ windowId: '23', snapshotId: 'axless', tree: '', elements: [] }))
const window = await app.getWindow('23')
expect(f.sink.text).not.toHaveBeenCalled()
await expect(window.getScreenshot()).resolves.toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
await expect(window.getScreenshot()).resolves.toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: '23' })
await window.click([10, 20])
await window.scroll({ x: 10, y: 20, deltaLinesX: 0, deltaLinesY: -1 })
await window.drag([10, 20], [30, 40])
expect(f.dispatch.mock.calls.slice(-4).map(([operation]) => operation.method)).toEqual(['screenshot', 'click', 'scroll', 'drag'])
for (const [operation] of f.dispatch.mock.calls.slice(-3)) {
expect(operation.params).toMatchObject({ appHandle: 'app-17', windowId: 23, snapshotId: 'axless' })
expect(operation.params).toMatchObject({ appHandle: 'app-17', windowId: '23', snapshotId: 'axless' })
}
})
@@ -430,47 +447,47 @@ describe('application and window observations', () => {
const failure = new Error(code)
f.dispatch.mockRejectedValueOnce(failure)
const calls = f.dispatch.mock.calls.length
await expect(app.getWindow(23)).rejects.toBe(failure)
await expect(app.getWindow('23')).rejects.toBe(failure)
expect(f.dispatch.mock.calls.slice(calls).map(([operation]) => operation.method)).toEqual(['getState'])
})
it('captures and clicks the explicitly selected window among several', async () => {
const f = fixture()
const windows = [
windowInfo(23),
{ ...windowInfo(24), title: 'Menu helper', bounds: { ...bounds, width: 0, height: 0 } },
{ ...windowInfo(25), isOnScreen: false },
{ ...windowInfo(26), bounds: { ...bounds, height: 0 } },
windowInfo('23'),
{ ...windowInfo('24'), title: 'Menu helper', bounds: { ...bounds, width: 0, height: 0 } },
{ ...windowInfo('25'), isOnScreen: false },
{ ...windowInfo('26'), bounds: { ...bounds, height: 0 } },
]
f.setWindowRecords(windows)
const app = await f.api.getApp('Editor')
expect(await app.getWindows({ emit: false })).toEqual(windows)
const window = await app.getWindow(23)
const window = await app.getWindow('23')
await window.getScreenshot()
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: '23' } })
await window.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
const hidden = await app.getWindow(25)
const hidden = await app.getWindow('25')
await hidden.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 25, snapshotId: 'snapshot-3', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
appHandle: 'app-17', windowId: '25', snapshotId: 'snapshot-3', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
})
it('captures a window without reading its accessibility tree', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
const calls = f.dispatch.mock.calls.length
await window.getScreenshot()
expect(f.dispatch).toHaveBeenCalledTimes(calls + 1)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: '23' } })
})
const providerIdentities: Record<string, JsonValue>[] = [
{ appHandle: 'another-app' }, { windowId: 24 },
{ appHandle: 'another-app' }, { windowId: '24' },
]
it.each(providerIdentities)('rejects provider state from another target and discards the old snapshot: %j', async (identity) => {
const f = fixture()
@@ -484,7 +501,7 @@ describe('application and window observations', () => {
it('returns provider window records without process filtering', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const observed = windowInfo(24)
const observed = windowInfo('24')
f.dispatch.mockResolvedValueOnce(state({ windows: [observed] }))
await expect(app.getAccessibilityTree()).resolves.toBe('Changed tree')
f.dispatch.mockResolvedValueOnce([observed])
@@ -508,12 +525,12 @@ describe('application and window observations', () => {
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'typeText', params: { appHandle: 'app-17', text: 'keyboard without accessibility' } },
])
const window = await app.getWindow(23)
const window = await app.getWindow('23')
await expect(window.getAccessibilityTree({ emit: false })).resolves.toBe('Tree 2')
await expect(app.getAccessibilityTree()).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
await expect(window.invoke(element('0'))).resolves.toEqual(delivered)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: {
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', elementId: '0',
appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', elementId: '0',
} })
})
@@ -521,7 +538,7 @@ describe('application and window observations', () => {
const f = fixture()
f.setCapabilities({ ...capabilities, keyboard: ['app', 'window'] })
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
f.dispatch.mockClear()
await window.typeText('hello')
await window.paste('<b>bold</b>', { format: 'html' })
@@ -530,9 +547,9 @@ describe('application and window observations', () => {
await app.paste('app paste')
await app.pressKey('Tab', ['shift'])
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'hello' } },
{ method: 'paste', params: { appHandle: 'app-17', windowId: 23, text: '<b>bold</b>', format: 'html' } },
{ method: 'pressKey', params: { appHandle: 'app-17', windowId: 23, key: 'a', modifiers: ['command'] } },
{ method: 'typeText', params: { appHandle: 'app-17', windowId: '23', text: 'hello' } },
{ method: 'paste', params: { appHandle: 'app-17', windowId: '23', text: '<b>bold</b>', format: 'html' } },
{ method: 'pressKey', params: { appHandle: 'app-17', windowId: '23', key: 'a', modifiers: ['command'] } },
{ method: 'typeText', params: { appHandle: 'app-17', text: 'app text' } },
{ method: 'paste', params: { appHandle: 'app-17', text: 'app paste' } },
{ method: 'pressKey', params: { appHandle: 'app-17', key: 'Tab', modifiers: ['shift'] } },
@@ -543,7 +560,7 @@ describe('application and window observations', () => {
const f = fixture()
f.setCapabilities({ ...capabilities, keyboard: ['window'] })
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
f.dispatch.mockClear()
await expect(app.typeText('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
await expect(app.paste('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
@@ -551,25 +568,25 @@ describe('application and window observations', () => {
await expect(app.pressKey('', ['command'])).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
expect(f.dispatch).not.toHaveBeenCalled()
await expect(window.typeText('window text')).resolves.toEqual(delivered)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'window text' } })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'typeText', params: { appHandle: 'app-17', windowId: '23', text: 'window text' } })
})
it('accepts coordinate input without a current snapshot', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
f.dispatch.mockRejectedValueOnce(new Error('Native app is busy'))
await expect(window.getAccessibilityTree()).rejects.toThrow('Native app is busy')
await window.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
appHandle: 'app-17', windowId: '23', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
})
it('clears a failed refresh and requires a new read before element operations', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
const failure = new Error('Native app is busy')
f.dispatch.mockRejectedValueOnce(failure)
await expect(app.getAccessibilityTree()).rejects.toBe(failure)
@@ -590,15 +607,15 @@ describe('application and window observations', () => {
describe('screenshots', () => {
it('returns screenshot metadata independently of input point bounds and supports selective output', async () => {
const f = fixture()
f.setWindows([23, 24])
f.setWindows(['23', '24'])
const app = await f.api.getApp('Editor')
const window = await app.getWindow(24)
const window = await app.getWindow('24')
f.sink.text.mockClear()
const tree = await window.getAccessibilityTree()
const image = await window.getScreenshot()
expect(tree).toBe('Tree 3')
expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 24 })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } })
expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: '24' })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: '24' } })
expect(f.sink.text).toHaveBeenCalledWith('Tree 3')
expect(f.sink.image).toHaveBeenCalledWith(image)
await window.getAccessibilityTree({ emit: false })
@@ -610,8 +627,8 @@ describe('screenshots', () => {
it('rejects a screenshot from another window before emitting its image', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
f.dispatch.mockResolvedValueOnce(screenshot({ windowId: 24 }))
const window = await app.getWindow('23')
f.dispatch.mockResolvedValueOnce(screenshot({ windowId: '24' }))
await expect(window.getScreenshot()).rejects.toThrow('belongs to another window')
expect(f.sink.image).not.toHaveBeenCalled()
})
@@ -619,11 +636,11 @@ describe('screenshots', () => {
it.each(['image/jpeg', 'image/webp', 'image/gif'])('preserves Host-admitted %s bytes and metadata in the return value and image sink', async (mimeType) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
const data = Buffer.from('admitted request image').toString('base64')
f.dispatch.mockResolvedValueOnce(screenshot({ mimeType, data, pixelWidth: 320, pixelHeight: 240 }))
const image = await window.getScreenshot()
expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(data, 'base64')), mimeType, pixelWidth: 320, pixelHeight: 240, windowId: 23 })
expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(data, 'base64')), mimeType, pixelWidth: 320, pixelHeight: 240, windowId: '23' })
expect(f.sink.image).toHaveBeenCalledExactlyOnceWith(image)
})
@@ -633,7 +650,7 @@ describe('input actions', () => {
it('passes explicit relative mode to every coordinate gesture and rejects it on element ids', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
await window.click([500, 250], { relative: true })
await window.mouseMove([500, 250], { relative: true })
await window.drag([0, 0], [999, 999], { relative: true })
@@ -651,32 +668,32 @@ describe('input actions', () => {
it('keeps the selected window when scroll options include another target identity', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
const options = {
x: 10, y: 20, deltaLinesX: 0, deltaLinesY: 1,
appHandle: 'another-app', snapshotId: 'another-snapshot', windowId: 24,
appHandle: 'another-app', snapshotId: 'another-snapshot', windowId: '24',
}
await window.scroll(options)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'scroll', params: {
appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 10, y: 20, deltaLinesX: 0, deltaLinesY: 1,
appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: '23', x: 10, y: 20, deltaLinesX: 0, deltaLinesY: 1,
} })
})
it('dispatches window-point gestures and reports delivery without claiming the action succeeded', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
expect(await window.click([10, 20], { mouseButton: 'right' })).toEqual(delivered)
expect(await window.click([11, 21], { clickCount: 2 })).toEqual(delivered)
expect(await window.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered)
expect(await window.drag([1, 2], [30, 40])).toEqual(delivered)
expect(f.dispatch.mock.calls.slice(-4).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'right', clickCount: 1 } },
{ method: 'click', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 11, y: 21, mouseButton: 'left', clickCount: 2 } },
{ method: 'scroll', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } },
{ method: 'drag', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', fromX: 1, fromY: 2, toX: 30, toY: 40 } },
{ method: 'click', params: { appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'right', clickCount: 1 } },
{ method: 'click', params: { appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', x: 11, y: 21, mouseButton: 'left', clickCount: 2 } },
{ method: 'scroll', params: { appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } },
{ method: 'drag', params: { appHandle: 'app-17', windowId: '23', snapshotId: 'snapshot-2', fromX: 1, fromY: 2, toX: 30, toY: 40 } },
])
})
@@ -698,7 +715,6 @@ describe('input actions', () => {
['empty key', ({ app }) => app.pressKey('')],
]
it.each<[string, (surfaces: Surfaces) => Promise<unknown>]>([
['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)],
['selectText text: Too small: expected string to have >=1 characters', ({ app }) => app.selectText(element('0'), '')],
['mouseMove x: Invalid input: expected number, received NaN', ({ window }) => window.mouseMove([NaN, 0])],
['mouseMove x: Too big: expected number to be <=999', ({ window }) => window.mouseMove([1000, 0], { relative: true })],
@@ -706,7 +722,7 @@ describe('input actions', () => {
])('reports a readable validation failure: %s', async (message, run) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
f.dispatch.mockClear()
await expect(run({ app, window })).rejects.toMatchObject({ name: 'Error', message })
expect(f.dispatch).not.toHaveBeenCalled()
@@ -715,7 +731,7 @@ describe('input actions', () => {
it.each(invalidActions)('rejects %s without invoking native input', async (_label, run) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
const calls = f.dispatch.mock.calls.length
await expect(run({ app, window })).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(calls)
@@ -778,7 +794,7 @@ describe('input actions', () => {
it('rejects keyboard input on a window surface without dispatching an app-level fallback', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
f.dispatch.mockClear()
for (const run of [
() => window.typeText('hello'),
@@ -800,13 +816,13 @@ describe('input actions', () => {
const f = fixture()
f.setCapabilities({ ...capabilities, keyboard: ['app', 'window'] })
const app = await f.api.getApp('Editor')
const surface = scope === 'window' ? await app.getWindow(23) : app
const surface = scope === 'window' ? await app.getWindow('23') : app
f.dispatch.mockClear()
f.dispatch.mockResolvedValueOnce(delivered)
await expect(surface.sendKeyEvent('Shift', direction)).resolves.toEqual(delivered)
expect(f.dispatch).toHaveBeenCalledTimes(1)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'sendKeyEvent', params: {
appHandle: 'app-17', ...scope === 'window' ? { windowId: 23 } : {}, key: 'Shift', direction,
appHandle: 'app-17', ...scope === 'window' ? { windowId: '23' } : {}, key: 'Shift', direction,
} })
})
@@ -833,7 +849,7 @@ describe('input actions', () => {
const windowFixture = fixture()
windowFixture.setCapabilities({ ...capabilities, keyboard: ['app'] })
const windowApp = await windowFixture.api.getApp('Editor')
const window = await windowApp.getWindow(23)
const window = await windowApp.getWindow('23')
windowFixture.dispatch.mockClear()
await expect(window.sendKeyEvent('', 'up')).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.')
expect(windowFixture.dispatch).not.toHaveBeenCalled()
@@ -842,24 +858,24 @@ describe('input actions', () => {
it('retains the snapshot identity for element right and double clicks', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
await window.click(element('12'), { mouseButton: 'right' })
await window.click(element('13'), { clickCount: 2 })
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', windowId: 23, mouseButton: 'right' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', windowId: 23, clickCount: 2 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', windowId: '23', mouseButton: 'right' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', windowId: '23', clickCount: 2 } },
])
})
it('dispatches an opaque provider element id unchanged', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
await window.click(element('42.12345'))
await window.mouseMove(element('42.12345'))
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: 23 } },
{ method: 'mouseMove', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: 23 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: '23' } },
{ method: 'mouseMove', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: '23' } },
])
})
@@ -874,7 +890,7 @@ describe('input actions', () => {
it('propagates native failures once and rejects malformed delivery claims', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
const failure = new Error('Application was restarted; select it again')
f.dispatch.mockRejectedValueOnce(failure)
await expect(app.typeText('hello')).rejects.toBe(failure)
@@ -890,7 +906,7 @@ describe('input actions', () => {
expect(f.dispatch).not.toHaveBeenCalled()
const app = await f.api.getApp('Editor')
const calls = f.dispatch.mock.calls.length
await expect(app.getWindow(0)).rejects.toThrow()
await expect(app.getWindow('0')).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
})
@@ -20,7 +20,7 @@ const selected = { appHandle: 'app-17', name: 'Editor', running: true }
const bounds = { x: 0, y: 0, width: 640, height: 480 }
const snapshot = {
appHandle: 'app-17', windowId: null, snapshotId: 'snapshot-1', tree: '0 button', truncated: false,
windows: [{ windowId: 23, appName: 'Editor', title: 'Document', bounds, isOnScreen: true, onCurrentSpace: null }],
windows: [{ windowId: '23', appName: 'Editor', title: 'Document', bounds, isOnScreen: true, onCurrentSpace: null }],
}
it('uses the current tool binding when a retained app is called by another execution', async () => {
@@ -34,13 +34,13 @@ it('uses the current tool binding when a retained app is called by another execu
const text = vi.fn()
const cua = connectCua(() => current, text)
const app = await cua.getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
const second = vi.fn(async (): Promise<JsonValue> => ({ dispatched: true }))
current = second
await expect(window.click(element('0'))).resolves.toEqual({ dispatched: true })
expect(first).toHaveBeenCalledTimes(4)
expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', windowId: 23 } })
expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', windowId: '23' } })
expect(text).not.toHaveBeenCalled()
current = first
await app.getAccessibilityTree()
@@ -71,15 +71,15 @@ it('returns screenshot bytes and metadata with one image-bearing tool call and n
if (operation.method === 'getApp') return selected
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
return {
mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 23, bounds, coordinateSpace: 'screenshot-pixels',
mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: '23', bounds, coordinateSpace: 'screenshot-pixels',
}
})
const text = vi.fn()
const app = await connectCua(() => tool, text).getApp('Editor')
const window = await app.getWindow(23)
const window = await app.getWindow('23')
text.mockClear()
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: '23' })
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getCapabilities', 'getApp', 'getState', 'getState', 'screenshot'])
expect(text).not.toHaveBeenCalled()
})
@@ -6,14 +6,26 @@ import type { CuaAppHandle, CuaWindowInfo } from '../src/types.ts'
const handle = 'app' as CuaAppHandle
const window = (width = 640, height = 480, x = 100): CuaWindowInfo => ({
windowId: 7, appName: 'Fixture', title: 'Window', bounds: { x, y: 200, width, height }, isOnScreen: true, onCurrentSpace: true,
windowId: '7', appName: 'Fixture', title: 'Window', bounds: { x, y: 200, width, height }, isOnScreen: true, onCurrentSpace: true,
})
const call = (method: string, params: object) => Request.parse({ method, params: { appHandle: handle, windowId: '7', ...params } })
it('keeps distinct windows independently mapped and invalidated', () => {
const coordinates = new CuaCoordinates()
const first = 'window-main', second = 'window-secondary'
coordinates.screenshot(handle, first, window(640, 480).bounds, { width: 640, height: 480 })
coordinates.screenshot(handle, second, window(1280, 960).bounds, { width: 640, height: 480 })
expect(coordinates.parameters(call('click', { windowId: first, x: 10, y: 20 }))).toMatchObject({ windowId: first, x: 10, y: 20 })
expect(coordinates.parameters(call('click', { windowId: second, x: 10, y: 20 }))).toMatchObject({ windowId: second, x: 20, y: 40 })
coordinates.observe(handle, [], first)
expect(() => coordinates.parameters(call('click', { windowId: first, x: 10, y: 20 }))).toThrow('STALE_SNAPSHOT')
expect(coordinates.parameters(call('click', { windowId: second, x: 10, y: 20 }))).toMatchObject({ windowId: second, x: 20, y: 40 })
})
const call = (method: string, params: object) => Request.parse({ method, params: { appHandle: handle, windowId: 7, ...params } })
it('maps Retina and downscaled screenshot pixels to the same native point', () => {
const coordinates = new CuaCoordinates()
for (const pixels of [{ width: 1280, height: 960 }, { width: 320, height: 240 }]) {
coordinates.screenshot(handle, 7, window().bounds, pixels)
coordinates.screenshot(handle, '7', window().bounds, pixels)
for (const method of ['click', 'mouseMove', 'scroll']) {
const request = call(method, { x: pixels.width / 2, y: pixels.height / 4,
...method === 'scroll' ? { deltaLinesX: 0, deltaLinesY: -1 } : {} })
@@ -34,7 +46,7 @@ it.each(['click', 'mouseMove', 'scroll'])('preserves thousandth scaling for %s a
}
assertPoints()
for (const image of [{ width: 2400, height: 1800 }, { width: 300, height: 225 }]) {
coordinates.screenshot(handle, 7, bounds, image)
coordinates.screenshot(handle, '7', bounds, image)
assertPoints()
}
})
@@ -43,7 +55,7 @@ it('maps relative drag endpoints and keeps 999 inside a one-point window', () =>
const coordinates = new CuaCoordinates()
coordinates.observe(handle, [window(1200, 900)])
const params = { fromX: 0, fromY: 250, toX: 999, toY: 500, relative: true }
expect(coordinates.parameters(call('drag', params))).toEqual({ appHandle: handle, windowId: 7,
expect(coordinates.parameters(call('drag', params))).toEqual({ appHandle: handle, windowId: '7',
fromX: 0, fromY: 225, toX: 1198.8, toY: 450 })
coordinates.observe(handle, [window(1, 1)])
expect(coordinates.parameters(call('click', { x: 999, y: 999, relative: true }))).toMatchObject({ x: 0, y: 0 })
@@ -51,15 +63,15 @@ it('maps relative drag endpoints and keeps 999 inside a one-point window', () =>
it('maps both drag endpoints and preserves button, modifiers and snapshot identity', () => {
const coordinates = new CuaCoordinates()
coordinates.screenshot(handle, 7, window().bounds, { width: 1280, height: 960 })
coordinates.screenshot(handle, '7', window().bounds, { width: 1280, height: 960 })
const options = { fromX: 320, fromY: 240, toX: 640, toY: 480, relative: false, snapshotId: 'snapshot', mouseButton: 'right', modifiers: ['shift'] }
expect(coordinates.parameters(call('drag', options))).toEqual({ appHandle: handle, windowId: 7,
expect(coordinates.parameters(call('drag', options))).toEqual({ appHandle: handle, windowId: '7',
fromX: 160, fromY: 120, toX: 320, toY: 240, snapshotId: 'snapshot', mouseButton: 'right', modifiers: ['shift'] })
})
it('preserves pixels after movement but requires a new image after resize or close', () => {
const coordinates = new CuaCoordinates()
coordinates.screenshot(handle, 7, window().bounds, { width: 1280, height: 960 })
coordinates.screenshot(handle, '7', window().bounds, { width: 1280, height: 960 })
const request = call('click', { x: 640, y: 480 })
coordinates.observe(handle, [window(640, 480, -900)])
expect(coordinates.parameters(request)).toMatchObject({ x: 320, y: 240 })
@@ -77,15 +89,15 @@ it('rejects unobserved, empty and out-of-image targets without native dispatch',
expect(() => coordinates.parameters(call('mouseMove', { x: 0, y: 0, relative: true }))).toThrow('STALE_SNAPSHOT')
coordinates.observe(handle, [window(640, 0)])
expect(() => coordinates.parameters(call('mouseMove', { x: 0, y: 0, relative: true }))).toThrow('STALE_SNAPSHOT')
coordinates.screenshot(handle, 7, window().bounds, { width: 320, height: 240 })
coordinates.screenshot(handle, '7', window().bounds, { width: 320, height: 240 })
for (const [x, y] of [[320, 0], [0, 240]]) expect(() => coordinates.parameters(call('click', { x, y }))).toThrow('INVALID_ARGUMENT')
})
it('keeps different application/window mappings independent and leaves element actions unchanged', () => {
const coordinates = new CuaCoordinates()
coordinates.observe(handle, [window()])
coordinates.screenshot(handle, 7, window().bounds, { width: 1280, height: 960 })
coordinates.screenshot('other' as CuaAppHandle, 7, window(100, 100).bounds, { width: 100, height: 100 })
coordinates.screenshot(handle, '7', window().bounds, { width: 1280, height: 960 })
coordinates.screenshot('other' as CuaAppHandle, '7', window(100, 100).bounds, { width: 100, height: 100 })
expect(coordinates.parameters(call('click', { x: 640, y: 480 }))).toMatchObject({ x: 320, y: 240 })
const request = Request.parse({ method: 'mouseMove', params: { appHandle: handle, snapshotId: 's', elementId: '2' } })
expect(coordinates.parameters(request)).toEqual(request.params)
@@ -93,11 +105,11 @@ it('keeps different application/window mappings independent and leaves element a
it('keeps other windows mapped during scoped observations and only removes the scoped closed window', () => {
const coordinates = new CuaCoordinates()
coordinates.screenshot(handle, 7, window().bounds, { width: 1280, height: 960 })
coordinates.observe(handle, [{ ...window(), windowId: 8 }], 8)
coordinates.screenshot(handle, '7', window().bounds, { width: 1280, height: 960 })
coordinates.observe(handle, [{ ...window(), windowId: '8' }], '8')
expect(coordinates.parameters(call('click', { x: 640, y: 480 }))).toMatchObject({ x: 320, y: 240 })
coordinates.observe(handle, [], 8)
coordinates.observe(handle, [], '8')
expect(coordinates.parameters(call('click', { x: 640, y: 480 }))).toMatchObject({ x: 320, y: 240 })
coordinates.observe(handle, [], 7)
coordinates.observe(handle, [], '7')
expect(() => coordinates.parameters(call('click', { x: 0, y: 0 }))).toThrow('STALE_SNAPSHOT')
})
@@ -32,6 +32,17 @@ it('retains optional resource fields and tuple-valued parameters without inventi
expect(generated.enums.get('getState')?.get('choice')).toEqual(new Set(['one']))
})
it('generates opaque string admission for native window targets', () => {
expect(JSON.parse(nativeProtocol().json)).toMatchObject({ oneOf: expect.arrayContaining([
expect.objectContaining({ properties: expect.objectContaining({
method: { const: 'screenshot', type: 'string' },
params: expect.objectContaining({ properties: expect.objectContaining({
windowId: { type: 'string' },
}) }),
}) }),
]) })
})
it('admits only locator fields for default invocation', () => {
expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId', 'windowId']))
})
@@ -1,7 +1,30 @@
/** Model-authored operation JSON cannot supply host configuration or bypass retained identities. */
import { expect, it } from 'vitest'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { CapabilitiesResult, Request, ScreenshotResult } from '../src/protocol.ts'
import { CapabilitiesResult, NativeScreenshotResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from '../src/protocol.ts'
it('preserves opaque window IDs in requests and results', () => {
const windowId = 'window-main'
const request = { method: 'screenshot', params: { appHandle: 'app', windowId } }
expect(Request.parse(request)).toEqual(request)
const windows = [{ windowId, appName: 'Fixture', title: 'Window',
bounds: { x: 0, y: 0, width: 100, height: 80 }, isOnScreen: true, onCurrentSpace: null }]
expect(WindowsResult.parse(windows)).toEqual(windows)
const state = { appHandle: 'app', windowId, focusedWindowId: windowId, snapshotId: 'snapshot', tree: '', windows, truncated: false }
expect(StateResult.parse(state)).toEqual(state)
expect(ShowResult.parse({ dispatched: true, foregroundChanged: false, windowId }).windowId).toBe(windowId)
const nativeImage = { mimeType: 'image/png', data: 'AQ==', pixelWidth: 1, pixelHeight: 1,
windowId, bounds: windows[0]!.bounds, coordinateSpace: 'window-pixels' }
expect(NativeScreenshotResult.parse(nativeImage)).toEqual(nativeImage)
expect(ScreenshotResult.parse({ ...nativeImage, coordinateSpace: 'screenshot-pixels' }).windowId).toBe(windowId)
})
it.each([0, -1, 1.5, '1', 2 ** 32])('keeps screenshot pixel dimensions numeric and bounded: %j', (dimension) => {
const image = { mimeType: 'image/png', data: 'AQ==', pixelWidth: 1, pixelHeight: 1,
windowId: 'window-main', bounds: { x: 0, y: 0, width: 100, height: 80 }, coordinateSpace: 'window-pixels' }
expect(NativeScreenshotResult.safeParse({ ...image, pixelWidth: dimension }).success).toBe(false)
expect(NativeScreenshotResult.safeParse({ ...image, pixelHeight: dimension }).success).toBe(false)
})
const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [
{ reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } },
@@ -11,7 +34,8 @@ const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [
{ reason: 'element click without an observed snapshot', request: { method: 'click', params: { appHandle: 'app', elementId: '0' } } },
{ reason: 'numeric element id', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: 0 } } },
{ reason: 'coordinate click without a window', request: { method: 'click', params: { appHandle: 'app', x: 1, y: 2 } } },
{ reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: 23, x: 1, y: 2 } } },
{ reason: 'numeric window id', request: { method: 'screenshot', params: { appHandle: 'app', windowId: 7 } } },
{ reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: '23', x: 1, y: 2 } } },
{ reason: 'non-string pressKey modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['command', 7] } } },
{ reason: 'sendKeyEvent modifiers', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'down', modifiers: [] } } },
{ reason: 'sendKeyEvent without direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a' } } },
@@ -29,14 +53,14 @@ it('admits arbitrary provider-defined pressKey modifier names', () => {
})
it('admits arbitrary key names with an explicit key-event direction', () => {
const request = { method: 'sendKeyEvent', params: { appHandle: 'app', windowId: 23, key: 'Command', direction: 'down' } }
const request = { method: 'sendKeyEvent', params: { appHandle: 'app', windowId: '23', key: 'Command', direction: 'down' } }
expect(Request.parse(request)).toEqual(request)
})
it.each([
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0' },
{ appHandle: 'app', windowId: 23, x: 1, y: 2 },
{ appHandle: 'app', windowId: 23, x: 1, y: 2, relative: true },
{ appHandle: 'app', windowId: '23', x: 1, y: 2 },
{ appHandle: 'app', windowId: '23', x: 1, y: 2, relative: true },
])('validates optional per-press hold duration for %j', (params) => {
for (const holdMs of [0, 1, 800, 5000, 7000]) {
const request = { method: 'click', params: { ...params, holdMs } }
@@ -51,17 +75,17 @@ it.each([
'', 'not base64!', 'AQ', 'AQ===', 'AQ==\n', 'A Q==', 'AQ-_', 'AB==', 'AAF=',
])('rejects noncanonical screenshot base64 %j', (data) => {
expect(ScreenshotResult.safeParse({
mimeType: 'image/png', data, pixelWidth: 1, pixelHeight: 1, windowId: 23,
mimeType: 'image/png', data, pixelWidth: 1, pixelHeight: 1, windowId: '23',
bounds: { x: 0, y: 0, width: 10, height: 10 }, coordinateSpace: 'screenshot-pixels',
}).success).toBe(false)
})
it.each([
{ mimeType: 'application/octet-stream' }, { pixelWidth: 0 }, { pixelHeight: -1 }, { windowId: 0 },
{ mimeType: 'application/octet-stream' }, { pixelWidth: 0 }, { pixelHeight: -1 }, { windowId: '0' },
{ bounds: { x: 0, y: 0, width: 0, height: 10 } }, { coordinateSpace: 'screen-pixels' },
])('retains screenshot metadata validation for %j', (invalid) => {
expect(ScreenshotResult.safeParse({
mimeType: 'image/png', data: 'AQ==', pixelWidth: 1, pixelHeight: 1, windowId: 23,
mimeType: 'image/png', data: 'AQ==', pixelWidth: 1, pixelHeight: 1, windowId: '23',
bounds: { x: 0, y: 0, width: 10, height: 10 }, coordinateSpace: 'screenshot-pixels', ...invalid,
}).success).toBe(false)
})
@@ -75,7 +99,7 @@ it('accepts disclosure options without accepting host traversal limits', () => {
it.each([
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0' },
{ appHandle: 'app', windowId: 23, x: 1, y: 2 },
{ appHandle: 'app', windowId: '23', x: 1, y: 2 },
])('accepts a mouse move target %j without button input', (params) => {
expect(Request.parse({ method: 'mouseMove', params })).toEqual({ method: 'mouseMove', params })
for (const extra of [{ mouseButton: 'left' }, { clickCount: 1 }, { modifiers: ['shift'] }, { useSystemPointer: 'yes' }]) {
@@ -86,7 +110,7 @@ it.each([
it.each([
{ appHandle: 'app', elementId: '0' },
{ appHandle: 'app', x: 1, y: 2 },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: 23, x: 1, y: 2 },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: '23', x: 1, y: 2 },
])('rejects incomplete or mixed mouse move targets %j', (params) => {
expect(Request.safeParse({ method: 'mouseMove', params }).success).toBe(false)
})
@@ -161,7 +185,7 @@ it('requires a setValue value and rejects unknown parameters', () => {
})
it.each(['click', 'mouseMove', 'scroll', 'drag'])('validates 0–999 integer coordinates for %s', (method) => {
const params = { appHandle: 'app', windowId: 7, relative: true,
const params = { appHandle: 'app', windowId: '7', relative: true,
...method === 'drag' ? { fromX: 0, fromY: 0, toX: 999, toY: 999 } : { x: 0, y: 999 },
...method === 'scroll' ? { deltaLinesX: 0, deltaLinesY: -1 } : {},
}
@@ -80,7 +80,7 @@
"type": "tool-call",
"id": "cua-unauthorized-screenshot",
"name": "cua",
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -179,7 +179,7 @@
"type": "tool-call",
"id": "cua-native-3",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\",\"includeMenus\":true,\"details\":true}}"
}
},
{
@@ -245,7 +245,7 @@
"type": "tool-call",
"id": "cua-native-5",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -278,7 +278,7 @@
"type": "tool-call",
"id": "cua-native-6",
"name": "cua",
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -344,7 +344,7 @@
"type": "tool-call",
"id": "cua-native-8",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"8\"}}"
}
},
{
@@ -377,7 +377,7 @@
"type": "tool-call",
"id": "cua-native-9",
"name": "cua",
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"8\"}}"
}
},
{
@@ -410,7 +410,7 @@
"type": "tool-call",
"id": "cua-native-10",
"name": "cua",
"arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"
"arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"8\",\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"
}
},
{
@@ -443,7 +443,7 @@
"type": "tool-call",
"id": "cua-native-11",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"8\"}}"
}
},
{
@@ -476,7 +476,7 @@
"type": "tool-call",
"id": "cua-native-show",
"name": "cua",
"arguments": "{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -608,7 +608,7 @@
"type": "tool-call",
"id": "cua-native-stale-read",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -674,7 +674,7 @@
"type": "tool-call",
"id": "cua-native-other-window",
"name": "cua",
"arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"
"arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"8\",\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"
}
},
{
@@ -707,7 +707,7 @@
"type": "tool-call",
"id": "cua-native-fresh-read",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -773,7 +773,7 @@
"type": "tool-call",
"id": "cua-native-verify",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -806,7 +806,7 @@
"type": "tool-call",
"id": "cua-native-geometry-observe",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"9\"}}"
}
},
{
@@ -839,7 +839,7 @@
"type": "tool-call",
"id": "cua-native-geometry-capture",
"name": "cua",
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"9\"}}"
}
},
{
@@ -872,7 +872,7 @@
"type": "tool-call",
"id": "cua-native-geometry-click",
"name": "cua",
"arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"
"arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"9\",\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"
}
},
{
@@ -905,7 +905,7 @@
"type": "tool-call",
"id": "cua-native-geometry-scroll",
"name": "cua",
"arguments": "{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"
"arguments": "{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"9\",\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"
}
},
{
@@ -938,7 +938,7 @@
"type": "tool-call",
"id": "cua-native-geometry-drag",
"name": "cua",
"arguments": "{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"
"arguments": "{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"9\",\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"
}
},
{
@@ -1004,7 +1004,7 @@
"type": "tool-call",
"id": "cua-native-geometry-verify",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"9\"}}"
}
},
{
@@ -1070,7 +1070,7 @@
"type": "tool-call",
"id": "cua-capture-busy",
"name": "cua",
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1103,7 +1103,7 @@
"type": "tool-call",
"id": "cua-capture-busy-ax",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1169,7 +1169,7 @@
"type": "tool-call",
"id": "cua-capture-recovered",
"name": "cua",
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1235,7 +1235,7 @@
"type": "tool-call",
"id": "cua-hover-observe",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1268,7 +1268,7 @@
"type": "tool-call",
"id": "cua-hover-capture",
"name": "cua",
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1301,7 +1301,7 @@
"type": "tool-call",
"id": "cua-hover-popup",
"name": "cua",
"arguments": "{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"
"arguments": "{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\",\"x\":100,\"y\":200,\"relative\":true}}"
}
},
{
@@ -1334,7 +1334,7 @@
"type": "tool-call",
"id": "cua-hover-click",
"name": "cua",
"arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"
"arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\",\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"
}
},
{
@@ -1367,7 +1367,7 @@
"type": "tool-call",
"id": "cua-hover-exit",
"name": "cua",
"arguments": "{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"
"arguments": "{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\",\"x\":0,\"y\":200,\"relative\":true}}"
}
},
{
@@ -1400,7 +1400,7 @@
"type": "tool-call",
"id": "cua-hover-verify",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1499,7 +1499,7 @@
"type": "tool-call",
"id": "cua-paste-inspect",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1565,7 +1565,7 @@
"type": "tool-call",
"id": "cua-paste-focus",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1631,7 +1631,7 @@
"type": "tool-call",
"id": "cua-paste-verify",
"name": "cua",
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -1697,7 +1697,7 @@
"type": "tool-call",
"id": "cua-debug-screenshot",
"name": "cua",
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"
"arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":\"7\"}}"
}
},
{
@@ -38,7 +38,7 @@ function serve() {
}
const instanceId = randomUUID()
const sockets = new Set()
const application = { count: 0, value: '', visualCount: 0, hovering: false, frontPid: 42, frontWindowId: 99 }
const application = { count: 0, value: '', visualCount: 0, hovering: false, frontPid: 42, frontWindowId: '99' }
const captureBroker = { pendingCapture: false }
let started = false
let hadClient = false
@@ -111,9 +111,9 @@ function createApplicationState(socket, record, application, captureBroker, conn
const scope = connectionId === 1 ? 'fixture' : `fixture-${connectionId}`
const app = { appHandle: `${scope}-app`, pid: 9, name: 'Fixture', bundleId: 'test.fixture', running: true }
const bounds = { x: 40, y: 60, width: 320, height: 200 }
const window = { windowId: 7, appName: 'Fixture', title: 'CUA fixture', bounds, isOnScreen: true, onCurrentSpace: true }
const axlessWindow = { ...window, windowId: 8, title: 'AXless canvas' }
const transformedWindow = { ...window, windowId: 9, title: 'Transformed window', bounds: { x: 16, y: 512, width: 131, height: 145 } }
const window = { windowId: '7', appName: 'Fixture', title: 'CUA fixture', bounds, isOnScreen: true, onCurrentSpace: true }
const axlessWindow = { ...window, windowId: '8', title: 'AXless canvas' }
const transformedWindow = { ...window, windowId: '9', title: 'Transformed window', bounds: { x: 16, y: 512, width: 131, height: 145 } }
const screenshotConfig = new URL('./fixture-screenshot.json', import.meta.url)
const customScreenshot = existsSync(screenshotConfig) ? JSON.parse(readFileSync(screenshotConfig, 'utf8')) : undefined
const snapshots = new Map()
@@ -165,18 +165,18 @@ function createApplicationState(socket, record, application, captureBroker, conn
refuse('STALE_SNAPSHOT', 'Accessibility tree changed during observation; read fresh getAccessibilityTree')
}
generation++
const axless = params.windowId === 8
const axless = params.windowId === '8'
const snapshotId = `${scope}-snapshot-${generation}`
snapshots.set(params.windowId ?? null, { snapshotId, windowId: params.windowId ?? null, axless })
if (axless) return { appHandle: app.appHandle, windowId: 8, snapshotId,
if (axless) return { appHandle: app.appHandle, windowId: '8', snapshotId,
tree: '', elements: [], windows: [axlessWindow], truncated: false }
return { appHandle: app.appHandle, windowId: params.windowId ?? null,
snapshotId: `${scope}-snapshot-${generation}`, tree: `AX scope=${params.windowId === undefined ? 'application' : 'window'} offsets=UTF-16${params.details ? ' bounds=global-points' : ''}\n- [0] AXButton "Increment"${params.details ? ' id="increment" actions=["AXPress"] bounds=[60,80,100,20]' : ''}\n- [1] AXTextField valueWritable=true selectionWritable=true value=${JSON.stringify(application.value)} placeholder="Fixture text"\n- AXStaticText "Count: ${application.count}"${application.hovering ? '\n- AXStaticText "Hover information"' : ''}\n- [2] AXMenuBarItem "File"${params.includeMenus ? '\n - [3] AXMenuItem "Fixture command"' : '\nMenu contents omitted; request includeMenus=true to expand.'}`,
elements: [{ elementId: '0', role: 'AXButton', title: 'Increment', value: null, description: null,
identifier: null, bounds: [60, 80, 100, 20], actions: ['AXPress'], enabled: true }],
windows: [params.windowId === 9 ? transformedWindow : window], truncated: false }
windows: [params.windowId === '9' ? transformedWindow : window], truncated: false }
}
if (params.windowId === 9 && ['screenshot', 'mouseMove', 'click', 'scroll', 'drag'].includes(method) && params.elementId === undefined) {
if (params.windowId === '9' && ['screenshot', 'mouseMove', 'click', 'scroll', 'drag'].includes(method) && params.elementId === undefined) {
refuse('WINDOW_CHANGED', 'Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input')
}
if (method === 'screenshot' && captureBroker.pendingCapture) {
@@ -200,7 +200,7 @@ function createApplicationState(socket, record, application, captureBroker, conn
if (params.windowId !== undefined && observed.windowId !== null && observed.windowId !== params.windowId) refuse('STALE_SNAPSHOT', 'Snapshot belongs to another window')
if (observed.axless && params.elementId !== undefined) refuse('INVALID_ARGUMENT', 'Element id is absent from the selected snapshot')
if (method === 'mouseMove') application.hovering = params.elementId === '0' || (params.x >= 20 && params.x <= 40)
else if (method === 'click' && params.windowId === 8) application.visualCount++
else if (method === 'click' && params.windowId === '8') application.visualCount++
else if (method === 'click') application.count++
else application.value = params.value
record('state', { count: application.count, value: application.value, visualCount: application.visualCount })
@@ -211,7 +211,7 @@ function createApplicationState(socket, record, application, captureBroker, conn
if (method === 'pressKey' && params.key === 'F11') captureBroker.pendingCapture = true
if (method === 'pressKey' && params.key === 'F10') captureBroker.pendingCapture = false
if (method === 'pressKey' && params.key === 'F12') malformedScreenshot = true
if (method === 'pressKey' && params.key === 'F9') { application.frontPid = 42; application.frontWindowId = 99 }
if (method === 'pressKey' && params.key === 'F9') { application.frontPid = 42; application.frontWindowId = '99' }
if (method === 'paste') {
if (application.frontPid !== app.pid) refuse('PASTE_NOT_CONSUMED', 'Paste command was dispatched, but clipboard data was not requested within the read deadline. Bring the target app to the foreground and focus its text input. Inspect the contents before deciding whether to retry; a delayed paste may still run.')
application.value += params.text
@@ -14,7 +14,7 @@
"type": "tool-call",
"id": "cua-ptc-native-result",
"name": "run_code",
"arguments": "{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst state = await tools.cua({ method: 'getState', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}"
"arguments": "{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst state = await tools.cua({ method: 'getState', params: { appHandle: app.appHandle, windowId: '7' } });\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: '7' } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}"
}
},
{