mward4/deepseek-harness
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 29 Wiki Activity

fix(cua): own clipboard transactions on the AppKit main actor

Browse Source
This commit is contained in:
creatixchu committed 2026-09-29 18:35:13 +08:00
1 parent e312d6fdf6
commit e12fe33aaa
5 files changed
+71 -43
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/computer-use/cua/native/Sources/CuaNative/AXAccess.swift
+2 -3
View File
@@ -145,7 +145,7 @@ final class AXAccess {
let error = AXUIElementSetAttributeValue(element, "AXValue" as CFString, value as CFString)
guard error == .success else { throw failure(error, "set AXValue") }
}
func paste(_ pid: Int32, maxElements: Int, maxDepth: Int, beforeDispatch: () throws -> Void) throws {
func pasteCommand(_ pid: Int32, maxElements: Int, maxDepth: Int) throws -> AXUIElement {
guard let menu = try element(AXUIElementCreateApplication(pid), "AXMenuBar") else {
throw NativeFailure(code: "UNSUPPORTED", message: "The application does not expose a Paste menu command")
}
@@ -172,8 +172,7 @@ final class AXAccess {
for child in try children(node) { pending.append((child, depth + 1)) }
}
guard let found else { throw NativeFailure(code: "UNSUPPORTED", message: "No standard Paste command is available") }
try beforeDispatch()
try perform(found, "AXPress")
return found
}
func selectText(_ element: AXUIElement, text: String, prefix: String?, suffix: String?, mode: String) throws {
try requireEnabled(element)
packages/computer-use/cua/native/Sources/CuaNative/AXWorker.swift
+22 -11
View File
@@ -47,6 +47,14 @@ struct ForegroundObservation: Sendable {
"frontWindowPid": frontWindowPID.map { .number(Double($0)) } ?? .null])
}
}
struct PastePlan: Sendable {
let handle: String
let text: String
let format: String
let maxBytes: Int
let maxElements: Int
let maxDepth: Int
}
struct ApplicationLaunch: Sendable {
let handle: String
let application: InstalledApplication
@@ -56,7 +64,7 @@ private enum ApplicationTarget {
case running(ProcessIdentity)
case pending(InstalledApplication, JSONValue)
}
enum DriverOutcome: Sendable { case result(JSONValue), capture(CaptureTarget), foreground(ForegroundPlan), launch(ApplicationLaunch) }
enum DriverOutcome: Sendable { case result(JSONValue), capture(CaptureTarget), foreground(ForegroundPlan), launch(ApplicationLaunch), paste(PastePlan) }
actor AXWorker {
nonisolated let executor = NativeSerialExecutor()
@@ -253,16 +261,9 @@ actor AXWorker {
let window = try coordinateWindow(identity, params.windowID())
return .capture(CaptureTarget(appHandle: handle, process: identity, window: window))
case "paste":
try ClipboardPaste().perform(text: params.string("text"), format: params.optionalString("format") ?? "text",
maxBytes: params.integer("maxClipboardBytes", in: 1024...Int.max),
checkCancellation: { try cancellation.check() }) { requireClipboard in
try ax().paste(identity.pid, maxElements: params.integer("maxElements", in: 1...10000),
maxDepth: params.integer("maxDepth", in: 1...100)) {
try cancellation.check()
_ = try validateIdentity(handle)
try requireClipboard()
}
}
return .paste(PastePlan(handle: handle, text: try params.string("text"), format: try params.optionalString("format") ?? "text",
maxBytes: try params.integer("maxClipboardBytes", in: 1024...Int.max),
maxElements: try params.integer("maxElements", in: 1...10000), maxDepth: try params.integer("maxDepth", in: 1...100)))
case "typeText", "pressKey":
let recipient = try ax().keyboardRecipient(identity, catalog: catalog)
try NativePlatform.verify(recipient)
@@ -329,6 +330,16 @@ actor AXWorker {
return .result(Self.actionResult)
}
func performPaste(_ plan: PastePlan, beforeDispatch: @MainActor @Sendable () throws -> Void) async throws {
let identity = try validateIdentity(plan.handle)
let command = try ax().pasteCommand(identity.pid, maxElements: plan.maxElements, maxDepth: plan.maxDepth)
try await beforeDispatch()
try cancellation.check()
_ = try validateIdentity(plan.handle)
try NativePlatform.requireUnlocked(observeSession())
try ax().perform(command, "AXPress")
}
private func pendingApplication(_ app: InstalledApplication, handle: String, authorization: JSONValue) -> JSONValue {
var value: [String: JSONValue] = ["appHandle": .string(handle), "pid": .null, "name": .string(app.name), "authorization": authorization]
if let bundleID = app.bundleID { value["bundleId"] = .string(bundleID) }
packages/computer-use/cua/native/Sources/CuaNative/ClipboardPaste.swift
+27 -18
View File
@@ -17,32 +17,31 @@ private final class PasteDataProvider: NSObject, NSPasteboardItemDataProvider, @
condition.broadcast()
condition.unlock()
}
func wait(check: () throws -> Void) throws {
@MainActor func wait(check: () throws -> Void) async throws {
while true {
try check()
condition.lock()
switch delivery {
case .supplied: condition.unlock(); return
case .failed:
condition.unlock()
throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Requested paste data could not be supplied")
case .pending:
// A bounded wait lets helper cancellation and a newer clipboard owner interrupt delivery.
_ = condition.wait(until: Date(timeIntervalSinceNow: 0.025))
condition.unlock()
let state = condition.withLock { delivery }
switch state {
case .supplied: return
case .failed: throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Requested paste data could not be supplied")
case .pending: try await Task.sleep(for: .milliseconds(25))
}
}
}
}
final class ClipboardPaste {
private struct SavedItem { let values: [(NSPasteboard.PasteboardType, Data)] }
@MainActor final class ClipboardPaste {
private struct SavedItem: Sendable { let values: [(NSPasteboard.PasteboardType, Data)] }
private static let ownerType = NSPasteboard.PasteboardType("org.deepseek-harness.cua.clipboard-owner")
private let pasteboard: NSPasteboard
private var restorePending: (() throws -> Void)?
init(pasteboard: NSPasteboard = .general) { self.pasteboard = pasteboard }
func perform(text: String, format: String, maxBytes: Int, checkCancellation: () throws -> Void = {}, operation: (() throws -> Void) throws -> Void) throws {
func perform(text: String, format: String, maxBytes: Int, checkCancellation: () throws -> Void = {},
operation: (@escaping @MainActor @Sendable () throws -> Void) async throws -> Void) async throws {
try checkCancellation()
let type: NSPasteboard.PasteboardType
switch format {
case "text": type = .string
@@ -62,20 +61,25 @@ final class ClipboardPaste {
guard pasteboard.changeCount == originalCount else {
throw NativeFailure(code: "CLIPBOARD_CHANGED", message: "Clipboard changed while preserving it; paste was not dispatched")
}
try checkCancellation()
let cleared = pasteboard.clearContents()
guard pasteboard.writeObjects([item]) else {
if pasteboard.changeCount == cleared { try restore(saved) }
throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Could not publish paste contents")
}
let expectedCount = pasteboard.changeCount
func owned() -> Bool { pasteboard.changeCount == expectedCount && pasteboard.data(forType: Self.ownerType) == owner }
func requireOwnership() throws {
let owned: @MainActor @Sendable () -> Bool = { [self] in
pasteboard.changeCount == expectedCount && pasteboard.data(forType: Self.ownerType) == owner
}
let requireOwnership: @MainActor @Sendable () throws -> Void = {
guard owned() else { throw NativeFailure(code: "CLIPBOARD_CHANGED", message: "Clipboard changed before paste; input was not dispatched") }
}
restorePending = { [self] in if owned() { try restore(saved) } }
defer { restorePending = nil }
do {
try requireOwnership()
try operation(requireOwnership)
try provider.wait { try checkCancellation(); try requireOwnership() }
try await operation(requireOwnership)
try await provider.wait { try checkCancellation(); try requireOwnership() }
} catch {
if owned() { try restore(saved) }
throw error
@@ -83,6 +87,11 @@ final class ClipboardPaste {
if owned() { try restore(saved) }
}
func cancel() throws {
try restorePending?()
restorePending = nil
}
private func snapshot(maxBytes: Int) throws -> [SavedItem] {
var total = 0
return try (pasteboard.pasteboardItems ?? []).map { item in
packages/computer-use/cua/native/Sources/CuaNative/RequestCoordinator.swift
+9
View File
@@ -9,6 +9,7 @@ import Darwin
private let cancellation: CancellationLatch
private let worker: AXWorker
private let capture = WindowCapture()
private let clipboard = ClipboardPaste()
private let input: JSONLInput
private let writer: JSONLWriter
private let signalQueue = DispatchQueue(label: "dsh.cua.signals")
@@ -85,6 +86,12 @@ import Darwin
let image = try await self.capture.capture(target, cancellation: self.cancellation)
result = try await self.worker.finishCapture(target, image: image)
case .foreground(let plan): result = try await self.performForeground(plan, requestID: request.id)
case .paste(let plan):
try await self.clipboard.perform(text: plan.text, format: plan.format, maxBytes: plan.maxBytes,
checkCancellation: { try self.cancellation.check() }) { requireClipboard in
try await self.worker.performPaste(plan, beforeDispatch: requireClipboard)
}
result = AXWorker.actionResult
case .launch(let plan):
let pid = try await ApplicationDiscovery.launch(plan.application, cancellation: self.cancellation)
result = try await self.worker.finishLaunch(plan, pid: pid)
@@ -144,6 +151,8 @@ import Darwin
guard !stopping else { return }
stopping = true
cancellation.cancel()
do { try clipboard.cancel() }
catch { fputs("Clipboard restoration failed during shutdown: \(error.localizedDescription)\n", stderr) }
active = nil
operation?.cancel()
input.stop()
packages/computer-use/cua/native/tests/swift/ClipboardPasteTests.swift
+11 -11
View File
@@ -4,7 +4,7 @@ import Foundation
import Testing
@testable import CuaNative
@Test func clipboardPasteRestoresEveryOriginalFormatAfterDispatch() throws {
@Test @MainActor func clipboardPasteRestoresEveryOriginalFormatAfterDispatch() async throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
let item = NSPasteboardItem()
@@ -12,7 +12,7 @@ import Testing
#expect(item.setString("original", forType: .string))
#expect(item.setData(Data([1, 2, 3]), forType: custom))
#expect(board.writeObjects([item]))
try ClipboardPaste(pasteboard: board).perform(text: "line 1\nline 2\tvalue", format: "text", maxBytes: 1024) { current in
try await ClipboardPaste(pasteboard: board).perform(text: "line 1\nline 2\tvalue", format: "text", maxBytes: 1024) { current in
try current()
#expect(board.string(forType: .string) == "line 1\nline 2\tvalue")
}
@@ -20,11 +20,11 @@ import Testing
#expect(board.data(forType: custom) == Data([1, 2, 3]))
}
@Test func clipboardPastePublishesHTMLWithoutTypingItsMarkup() throws {
@Test @MainActor func clipboardPastePublishesHTMLWithoutTypingItsMarkup() async throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
let html = "<h1>Title</h1><p><b>Bold</b> <a href=\"https://example.com/\">link</a></p>"
try ClipboardPaste(pasteboard: board).perform(text: html, format: "html", maxBytes: 1024) { current in
try await ClipboardPaste(pasteboard: board).perform(text: html, format: "html", maxBytes: 1024) { current in
try current()
#expect(board.data(forType: .html) == Data(html.utf8))
#expect(board.string(forType: .string) == nil)
@@ -32,12 +32,12 @@ import Testing
#expect(board.pasteboardItems?.isEmpty ?? true)
}
@Test func clipboardPastePreservesANewerCopyInsteadOfRestoringOverIt() throws {
@Test @MainActor func clipboardPastePreservesANewerCopyInsteadOfRestoringOverIt() async throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
#expect(board.setString("original", forType: .string))
do {
try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { current in
try await ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { current in
board.clearContents()
#expect(board.setString("new user copy", forType: .string))
try current()
@@ -47,12 +47,12 @@ import Testing
#expect(board.string(forType: .string) == "new user copy")
}
@Test func clipboardPasteRestoresOnCancellationAndRejectsOversizedPreservationBeforeChangingData() throws {
@Test @MainActor func clipboardPasteRestoresOnCancellationAndRejectsOversizedPreservationBeforeChangingData() async throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
#expect(board.setString("original", forType: .string))
do {
try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in
try await ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in
throw NativeFailure(code: "NATIVE_CANCELLED", message: "fixture cancellation")
}
Issue.record("Cancelled operation succeeded")
@@ -62,7 +62,7 @@ import Testing
let large = String(repeating: "x", count: 2048)
#expect(board.setString(large, forType: .string))
do {
try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in
try await ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in
Issue.record("Oversized clipboard reached dispatch")
}
Issue.record("Oversized clipboard was accepted")
@@ -70,12 +70,12 @@ import Testing
#expect(board.string(forType: .string) == large)
}
@Test func clipboardPasteAwaitsDataConsumptionAndCanCancelBeforeItIsRequested() throws {
@Test @MainActor func clipboardPasteAwaitsDataConsumptionAndCanCancelBeforeItIsRequested() async throws {
let board = NSPasteboard.withUniqueName()
defer { board.releaseGlobally() }
#expect(board.setString("original", forType: .string))
do {
try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024,
try await ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024,
checkCancellation: { throw NativeFailure(code: "NATIVE_CANCELLED", message: "delivery cancelled") }) { _ in }
Issue.record("Unread clipboard data completed without cancellation")
} catch let error as NativeFailure { #expect(error.code == "NATIVE_CANCELLED") }