feat(cua): paste text and HTML through native clipboard commands

This commit is contained in:
creatixchu committed 2026-09-29 17:39:34 +08:00
1 parent d2e8a5be3a
commit d35070bf4b
11 files changed
+232 -16

No files matched your search

@@ -145,6 +145,36 @@ final class AXAccess {
let error = AXUIElementSetAttributeValue(element, "AXValue" as CFString, value as CFString)
guard error == .success else { throw failure(error, "set AXValue") }
}
func paste(_ pid: Int32, maxElements: Int, maxDepth: Int, beforeDispatch: () throws -> Void) throws {
guard let menu = try element(AXUIElementCreateApplication(pid), "AXMenuBar") else {
throw NativeFailure(code: "UNSUPPORTED", message: "The application does not expose a Paste menu command")
}
var pending: [(AXUIElement, Int)] = [(menu, 0)]
var found: AXUIElement?
var seen: [CFHashCode: [AXUIElement]] = [:]
var count = 0
while let (node, depth) = pending.popLast() {
let hash = CFHash(node)
if seen[hash, default: []].contains(where: { CFEqual($0, node) }) { continue }
seen[hash, default: []].append(node)
guard count < maxElements, depth <= maxDepth else {
throw NativeFailure(code: "UNSUPPORTED", message: "Paste menu discovery exceeded its observation limits")
}
count += 1
if try string(node, "AXRole") == "AXMenuItem", try string(node, "AXMenuItemCmdChar")?.lowercased() == "v" {
let raw = try attribute(node, "AXMenuItemCmdModifiers")
if let raw, CFGetTypeID(raw) == CFNumberGetTypeID(), let number = raw as? NSNumber, number.intValue == 0,
try actions(node).contains("AXPress"), try boolean(node, "AXEnabled") != false {
guard found == nil else { throw NativeFailure(code: "UNSUPPORTED", message: "The application exposes ambiguous Paste commands") }
found = node
}
}
for child in try children(node) { pending.append((child, depth + 1)) }
}
guard let found else { throw NativeFailure(code: "UNSUPPORTED", message: "No enabled standard Paste command is available") }
try beforeDispatch()
try perform(found, "AXPress")
}
func selectText(_ element: AXUIElement, text: String, prefix: String?, suffix: String?, mode: String) throws {
try requireEnabled(element)
guard try writable(element, "AXSelectedTextRange"), let original = try string(element, "AXValue") else {
@@ -224,6 +224,16 @@ actor AXWorker {
try NativePlatform.requireScreenRecording()
let window = try coordinateWindow(identity, params.windowID())
return .capture(CaptureTarget(appHandle: handle, process: identity, window: window))
case "paste":
try ClipboardPaste().perform(text: params.string("text"), format: params.optionalString("format") ?? "text",
maxBytes: params.integer("maxClipboardBytes", in: 1024...Int.max)) { requireClipboard in
try ax().paste(identity.pid, maxElements: params.integer("maxElements", in: 1...10000),
maxDepth: params.integer("maxDepth", in: 1...100)) {
try cancellation.check()
_ = try validateIdentity(handle)
try requireClipboard()
}
}
case "typeText", "pressKey":
let recipient = try ax().keyboardRecipient(identity, catalog: catalog)
try NativePlatform.verify(recipient)
@@ -0,0 +1,83 @@
// Temporary clipboard contents are restored only while this transaction still owns them.
import AppKit
import Foundation
final class ClipboardPaste {
private struct SavedItem { let values: [(NSPasteboard.PasteboardType, Data)] }
private static let ownerType = NSPasteboard.PasteboardType("org.deepseek-harness.cua.clipboard-owner")
private let pasteboard: NSPasteboard
init(pasteboard: NSPasteboard = .general) { self.pasteboard = pasteboard }
func perform(text: String, format: String, maxBytes: Int, operation: (() throws -> Void) throws -> Void) throws {
let type: NSPasteboard.PasteboardType
switch format {
case "text": type = .string
case "html": type = .html
default: throw RequestParameters.invalid("Paste format must be text or html")
}
let data = Data(text.utf8)
guard data.count <= maxBytes else { throw NativeFailure(code: "CLIPBOARD_LIMIT", message: "Paste data exceeds the configured clipboard byte limit") }
let originalCount = pasteboard.changeCount
let saved = try snapshot(maxBytes: maxBytes)
let owner = Data(UUID().uuidString.utf8)
let item = NSPasteboardItem()
guard item.setData(data, forType: type), item.setData(owner, forType: Self.ownerType) else {
throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Could not prepare paste contents")
}
guard pasteboard.changeCount == originalCount else {
throw NativeFailure(code: "CLIPBOARD_CHANGED", message: "Clipboard changed while preserving it; paste was not dispatched")
}
let cleared = pasteboard.clearContents()
guard pasteboard.writeObjects([item]) else {
if pasteboard.changeCount == cleared { try restore(saved) }
throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Could not publish paste contents")
}
let expectedCount = pasteboard.changeCount
func owned() -> Bool { pasteboard.changeCount == expectedCount && pasteboard.data(forType: Self.ownerType) == owner }
func requireOwnership() throws {
guard owned() else { throw NativeFailure(code: "CLIPBOARD_CHANGED", message: "Clipboard changed before paste; input was not dispatched") }
}
do {
try requireOwnership()
try operation(requireOwnership)
} catch {
if owned() { try restore(saved) }
throw error
}
if owned() { try restore(saved) }
}
private func snapshot(maxBytes: Int) throws -> [SavedItem] {
var total = 0
return try (pasteboard.pasteboardItems ?? []).map { item in
let values = try item.types.map { type -> (NSPasteboard.PasteboardType, Data) in
guard let data = item.data(forType: type) else {
throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Existing clipboard formats could not be preserved")
}
guard data.count <= maxBytes - total else {
throw NativeFailure(code: "CLIPBOARD_LIMIT", message: "Existing clipboard exceeds the configured preservation byte limit")
}
total += data.count
return (type, data)
}
return SavedItem(values: values)
}
}
private func restore(_ saved: [SavedItem]) throws {
let items = try saved.map { original -> NSPasteboardItem in
let item = NSPasteboardItem()
for (type, data) in original.values {
guard item.setData(data, forType: type) else {
throw NativeFailure(code: "CLIPBOARD_RESTORE_FAILED", message: "Could not reconstruct the original clipboard formats")
}
}
return item
}
pasteboard.clearContents()
if !items.isEmpty && !pasteboard.writeObjects(items) {
throw NativeFailure(code: "CLIPBOARD_RESTORE_FAILED", message: "Could not restore the original clipboard contents")
}
}
}
@@ -10,6 +10,7 @@ struct RequestParameters: Sendable {
"setValue": ["appHandle", "snapshotId", "elementIndex", "value"],
"selectText": ["appHandle", "snapshotId", "elementIndex", "text", "prefix", "suffix", "selectionType"],
"performSecondaryAction": ["appHandle", "snapshotId", "elementIndex", "action"],
"paste": ["appHandle", "text", "format", "maxClipboardBytes", "maxElements", "maxDepth"],
"typeText": ["appHandle", "text"], "pressKey": ["appHandle", "key", "modifiers"],
"scroll": ["appHandle", "windowId", "snapshotId", "x", "y", "deltaLinesX", "deltaLinesY"],
"drag": ["appHandle", "windowId", "snapshotId", "fromX", "fromY", "toX", "toY", "durationMs", "steps", "mouseButton", "modifiers"],
@@ -0,0 +1,71 @@
// Each clipboard case owns a named pasteboard rather than touching the user's clipboard.
import AppKit
import Foundation
import Testing
@testable import CuaNative
@Test func clipboardPasteRestoresEveryOriginalFormatAfterDispatch() throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
let item = NSPasteboardItem()
let custom = NSPasteboard.PasteboardType("org.deepseek-harness.test.original")
#expect(item.setString("original", forType: .string))
#expect(item.setData(Data([1, 2, 3]), forType: custom))
#expect(board.writeObjects([item]))
try ClipboardPaste(pasteboard: board).perform(text: "line 1\nline 2\tvalue", format: "text", maxBytes: 1024) { current in
try current()
#expect(board.string(forType: .string) == "line 1\nline 2\tvalue")
}
#expect(board.string(forType: .string) == "original")
#expect(board.data(forType: custom) == Data([1, 2, 3]))
}
@Test func clipboardPastePublishesHTMLWithoutTypingItsMarkup() throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
let html = "<h1>Title</h1><p><b>Bold</b> <a href=\"https://example.com/\">link</a></p>"
try ClipboardPaste(pasteboard: board).perform(text: html, format: "html", maxBytes: 1024) { current in
try current()
#expect(board.data(forType: .html) == Data(html.utf8))
#expect(board.string(forType: .string) == nil)
}
#expect(board.pasteboardItems?.isEmpty ?? true)
}
@Test func clipboardPastePreservesANewerCopyInsteadOfRestoringOverIt() throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
#expect(board.setString("original", forType: .string))
do {
try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { current in
board.clearContents()
#expect(board.setString("new user copy", forType: .string))
try current()
}
Issue.record("Changed clipboard was accepted")
} catch let error as NativeFailure { #expect(error.code == "CLIPBOARD_CHANGED") }
#expect(board.string(forType: .string) == "new user copy")
}
@Test func clipboardPasteRestoresOnCancellationAndRejectsOversizedPreservationBeforeChangingData() throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
#expect(board.setString("original", forType: .string))
do {
try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in
throw NativeFailure(code: "NATIVE_CANCELLED", message: "fixture cancellation")
}
Issue.record("Cancelled operation succeeded")
} catch let error as NativeFailure { #expect(error.code == "NATIVE_CANCELLED") }
#expect(board.string(forType: .string) == "original")
board.clearContents()
let large = String(repeating: "x", count: 2048)
#expect(board.setString(large, forType: .string))
do {
try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in
Issue.record("Oversized clipboard reached dispatch")
}
Issue.record("Oversized clipboard was accepted")
} catch let error as NativeFailure { #expect(error.code == "CLIPBOARD_LIMIT") }
#expect(board.string(forType: .string) == large)
}
+3 -1
View File
@@ -161,7 +161,9 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
return window
},
async typeText(text) { return action('typeText', { appHandle, text }) },
async paste(text) { return action('typeText', { appHandle, text }) },
async paste(text, options) { return action('paste', { appHandle, text,
...options?.format === undefined ? {} : { format: options.format },
}) },
async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) },
}
await app.getAXState()
+19 -8
View File
@@ -1,5 +1,7 @@
/** macOS computer use over a managed native helper. */
import { isAbsolute, resolve } from 'node:path'
import { isAbsolute, join, resolve } from 'node:path'
import { tmpdir } from 'node:os'
import { withFileLock } from '@deepseek-ai/dsh-util-atomic-write'
import { fileURLToPath } from 'node:url'
import type { Context, Volatile } from '@deepseek-ai/cordis'
import Schema from '@deepseek-ai/schemastery'
@@ -43,6 +45,8 @@ export interface Config {
axTimeoutMs: number
/** Maximum bytes retained for one helper response frame. */
maxResponseBytes: number
/** Maximum bytes preserved from the system clipboard during one paste. */
maxClipboardBytes: number
/** Maximum UTF-8 bytes admitted for one helper request frame. */
maxRequestBytes: number
/** Duration in milliseconds of one same-window drag. */
@@ -69,6 +73,7 @@ export const Config = Schema.object({
buildTimeoutMs: Schema.number().step(1).min(1).max(2 ** 31 - 1).default(300_000),
timeoutMs: Schema.number().step(1).min(1).max(2 ** 31 - 1).default(30_000),
axTimeoutMs: Schema.number().step(1).min(1).max(2 ** 31 - 1).default(3000),
maxClipboardBytes: Schema.number().step(1).min(1024).default(32 * 1024 * 1024),
maxRequestBytes: Schema.number().step(1).min(1024).default(1024 * 1024),
dragDurationMs: Schema.number().step(1).min(1).max(5000).default(400),
dragSteps: Schema.number().step(1).min(1).max(240).default(20),
@@ -95,7 +100,8 @@ Object API signatures (options default to { emit: true }):
- app.activate() -> { dispatched: true, verified: false }
- app.getWindows(options?) -> Array<{ windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace }>
- app.getWindow(windowId: number) -> Window
- app.typeText(text: string), app.paste(text: string), app.pressKey(chord: string) -> action result
- app.typeText(text: string), app.pressKey(chord: string) -> action result
- app.paste(text: string, options?: { format?: "text" | "html" }) -> action result
App and Window methods:
@@ -113,7 +119,7 @@ Raw operation tools use { method, params }, including params: {} for parameterle
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.`
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.`
function executionAgent(execution: Readonly<ToolExecution>): Agent {
if (!execution.agent) throw new Error('dsh-cua requires an agent-owned execution')
@@ -229,11 +235,16 @@ function mount(ctx: Context, config: Config, helperPath: string): void {
const signal = execution.signal
signal.throwIfAborted()
const client = (execution as Readonly<CuaExecution>).nativeClient
const nativeParams = method === 'getState'
? { ...params, maxElements: config.maxElements, maxDepth: config.maxDepth }
: method === 'drag' ? { ...params, durationMs: config.dragDurationMs, steps: config.dragSteps }
: params
const result = await client.request(method, nativeParams, signal, execution.callId)
const nativeParams = method === 'paste'
? { ...params, maxClipboardBytes: config.maxClipboardBytes, maxElements: config.maxElements, maxDepth: config.maxDepth }
: method === 'getState'
? { ...params, maxElements: config.maxElements, maxDepth: config.maxDepth }
: method === 'drag' ? { ...params, durationMs: config.dragDurationMs, steps: config.dragSteps }
: params
const requestNative = () => client.request(method, nativeParams, signal, execution.callId)
const result = method === 'paste'
? await withFileLock(join(tmpdir(), 'dsh-cua-system-clipboard'), requestNative, { waitMs: config.timeoutMs, signal })
: await requestNative()
if (method === 'getApp') {
const selected = AppResult.parse(result)
const { authorization } = z.object({ authorization: ApplicationAuthorizationResult }).parse(result)
@@ -39,6 +39,7 @@ export const Request = z.discriminatedUnion('method', [
selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }),
operation('performSecondaryAction', { ...snapshot, action: z.string().min(1) }),
operation('typeText', { ...app, text: z.string() }),
operation('paste', { ...app, text: z.string(), format: z.enum(['text', 'html']).optional() }),
operation('pressKey', { ...app, key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }),
operation('scroll', { ...coordinate, deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }),
operation('drag', { ...window, ...pointer, snapshotId: z.string().min(1).optional(), fromX: point, fromY: point, toX: point, toY: point }),
+7 -4
View File
@@ -61,6 +61,8 @@ export type CuaMouseButton = 'left' | 'right' | 'middle'
export interface CuaClickOptions { mouseButton?: CuaMouseButton; clickCount?: 1 | 2; modifiers?: readonly CuaPointerModifier[] }
/** One same-window drag; omitted button is left and omitted modifiers are empty. */
export interface CuaDragOptions { mouseButton?: CuaMouseButton; modifiers?: readonly CuaPointerModifier[] }
/** Clipboard representation delivered through the application's Paste command. */
export interface CuaPasteOptions { format?: 'text' | 'html' }
/** Adjacent literal context disambiguates repeated text; cursor modes collapse the matched range. */
export interface CuaSelectTextOptions { prefix?: string; suffix?: string; selectionType?: 'text' | 'cursor_before' | 'cursor_after' }
/** Positive deltas reveal content above/left; each axis is an integer from -10 to 10. */
@@ -170,11 +172,12 @@ export interface CuaApp extends CuaSurface {
*/
typeText(text: string): Promise<CuaActionResult>
/**
* Insert plain text at current focus through native Unicode input, preserving the system clipboard.
* @param text - Literal text; rich-text clipboard formats are unsupported.
* @returns Dispatch evidence; cancellation may leave a prefix entered.
* Paste text or HTML through the application's standard Paste menu command.
* @param text - Clipboard contents in the selected format; defaults to plain text.
* @param options - Clipboard format. Original contents are restored unless another writer changes them.
* @returns Dispatch evidence; unsupported Paste commands fail without typing fallback.
*/
paste(text: string): Promise<CuaActionResult>
paste(text: string, options?: CuaPasteOptions): Promise<CuaActionResult>
/**
* Send a key/chord at current focus.
* @param key - xdotool-style chord such as super+c, shift+Tab, + or cmd++; plus is also a key name.
+4 -2
View File
@@ -402,11 +402,13 @@ describe('input actions', () => {
const app = await f.api.getApp('Editor')
await app.typeText('中文\nline two')
await app.paste('粘贴\nplain text')
await app.paste('<b>bold</b>', { format: 'html' })
await app.pressKey('Control_L+shift+a')
await app.pressKey('Return')
expect(f.dispatch.mock.calls.slice(-4).map(([operation]) => operation)).toEqual([
expect(f.dispatch.mock.calls.slice(-5).map(([operation]) => operation)).toEqual([
{ method: 'typeText', params: { appHandle: 'app-17', text: '中文\nline two' } },
{ method: 'typeText', params: { appHandle: 'app-17', text: '粘贴\nplain text' } },
{ method: 'paste', params: { appHandle: 'app-17', text: '粘贴\nplain text' } },
{ method: 'paste', params: { appHandle: 'app-17', text: '<b>bold</b>', format: 'html' } },
{ method: 'pressKey', params: { appHandle: 'app-17', key: 'a', modifiers: ['control', 'shift'] } },
{ method: 'pressKey', params: { appHandle: 'app-17', key: 'Return', modifiers: [] } },
])
@@ -48,7 +48,7 @@ vi.mock('../src/native-client.ts', async importOriginal => ({
}))
const config = {
helperPath: resolve('fixture-helper'), buildTimeoutMs: 300_000, timeoutMs: 10_000, axTimeoutMs: 3000, maxRequestBytes: 1024, maxResponseBytes: 4096,
helperPath: resolve('fixture-helper'), buildTimeoutMs: 300_000, timeoutMs: 10_000, axTimeoutMs: 3000, maxRequestBytes: 1024, maxResponseBytes: 4096, maxClipboardBytes: 8192,
graceMs: 10, textEventIntervalMs: 8, maxElements: 64, maxDepth: 8, dragDurationMs: 200, dragSteps: 10,
} satisfies Omit<NativeProvider.Config, 'allowedApps'>
@@ -275,6 +275,8 @@ it('supplies configured resource bounds and clears registrations on unload', asy
expect(native.request).toHaveBeenLastCalledWith('getState', { appHandle: 'app-one', maxElements: 64, maxDepth: 8 }, expect.any(AbortSignal), ToolCallId('lifecycle-2'))
await f.execute('drag', { appHandle: 'app-one', windowId: 7, fromX: 1, fromY: 2, toX: 3, toY: 4 })
expect(native.request).toHaveBeenLastCalledWith('drag', expect.objectContaining({ durationMs: 200, steps: 10 }), expect.any(AbortSignal), ToolCallId('lifecycle-3'))
await f.execute('paste', { appHandle: 'app-one', text: 'hello', format: 'text' })
expect(native.request).toHaveBeenLastCalledWith('paste', expect.objectContaining({ maxClipboardBytes: 8192, maxElements: 64, maxDepth: 8 }), expect.any(AbortSignal), ToolCallId('lifecycle-4'))
await f.provider.dispose()
expect(f.ctx.tools.schemas()).toEqual([])
expect(native.dispose).toHaveBeenCalledTimes(1)