diff --git a/packages/computer-use/cua/native/Sources/CuaNative/AXAccess.swift b/packages/computer-use/cua/native/Sources/CuaNative/AXAccess.swift index 472d8b79bf..2d7f61de41 100644 --- a/packages/computer-use/cua/native/Sources/CuaNative/AXAccess.swift +++ b/packages/computer-use/cua/native/Sources/CuaNative/AXAccess.swift @@ -145,7 +145,7 @@ final class AXAccess { let error = AXUIElementSetAttributeValue(element, "AXValue" as CFString, value as CFString) guard error == .success else { throw failure(error, "set AXValue") } } - func paste(_ pid: Int32, maxElements: Int, maxDepth: Int, beforeDispatch: () throws -> Void) throws { + func pasteCommand(_ pid: Int32, maxElements: Int, maxDepth: Int) throws -> AXUIElement { guard let menu = try element(AXUIElementCreateApplication(pid), "AXMenuBar") else { throw NativeFailure(code: "UNSUPPORTED", message: "The application does not expose a Paste menu command") } @@ -172,8 +172,7 @@ final class AXAccess { for child in try children(node) { pending.append((child, depth + 1)) } } guard let found else { throw NativeFailure(code: "UNSUPPORTED", message: "No standard Paste command is available") } - try beforeDispatch() - try perform(found, "AXPress") + return found } func selectText(_ element: AXUIElement, text: String, prefix: String?, suffix: String?, mode: String) throws { try requireEnabled(element) diff --git a/packages/computer-use/cua/native/Sources/CuaNative/AXWorker.swift b/packages/computer-use/cua/native/Sources/CuaNative/AXWorker.swift index 37d94b05f1..61b4aa7366 100644 --- a/packages/computer-use/cua/native/Sources/CuaNative/AXWorker.swift +++ b/packages/computer-use/cua/native/Sources/CuaNative/AXWorker.swift @@ -47,6 +47,14 @@ struct ForegroundObservation: Sendable { "frontWindowPid": frontWindowPID.map { .number(Double($0)) } ?? .null]) } } +struct PastePlan: Sendable { + let handle: String + let text: String + let format: String + let maxBytes: Int + let maxElements: Int + let maxDepth: Int +} struct ApplicationLaunch: Sendable { let handle: String let application: InstalledApplication @@ -56,7 +64,7 @@ private enum ApplicationTarget { case running(ProcessIdentity) case pending(InstalledApplication, JSONValue) } -enum DriverOutcome: Sendable { case result(JSONValue), capture(CaptureTarget), foreground(ForegroundPlan), launch(ApplicationLaunch) } +enum DriverOutcome: Sendable { case result(JSONValue), capture(CaptureTarget), foreground(ForegroundPlan), launch(ApplicationLaunch), paste(PastePlan) } actor AXWorker { nonisolated let executor = NativeSerialExecutor() @@ -253,16 +261,9 @@ actor AXWorker { let window = try coordinateWindow(identity, params.windowID()) return .capture(CaptureTarget(appHandle: handle, process: identity, window: window)) case "paste": - try ClipboardPaste().perform(text: params.string("text"), format: params.optionalString("format") ?? "text", - maxBytes: params.integer("maxClipboardBytes", in: 1024...Int.max), - checkCancellation: { try cancellation.check() }) { requireClipboard in - try ax().paste(identity.pid, maxElements: params.integer("maxElements", in: 1...10000), - maxDepth: params.integer("maxDepth", in: 1...100)) { - try cancellation.check() - _ = try validateIdentity(handle) - try requireClipboard() - } - } + return .paste(PastePlan(handle: handle, text: try params.string("text"), format: try params.optionalString("format") ?? "text", + maxBytes: try params.integer("maxClipboardBytes", in: 1024...Int.max), + maxElements: try params.integer("maxElements", in: 1...10000), maxDepth: try params.integer("maxDepth", in: 1...100))) case "typeText", "pressKey": let recipient = try ax().keyboardRecipient(identity, catalog: catalog) try NativePlatform.verify(recipient) @@ -329,6 +330,16 @@ actor AXWorker { return .result(Self.actionResult) } + func performPaste(_ plan: PastePlan, beforeDispatch: @MainActor @Sendable () throws -> Void) async throws { + let identity = try validateIdentity(plan.handle) + let command = try ax().pasteCommand(identity.pid, maxElements: plan.maxElements, maxDepth: plan.maxDepth) + try await beforeDispatch() + try cancellation.check() + _ = try validateIdentity(plan.handle) + try NativePlatform.requireUnlocked(observeSession()) + try ax().perform(command, "AXPress") + } + private func pendingApplication(_ app: InstalledApplication, handle: String, authorization: JSONValue) -> JSONValue { var value: [String: JSONValue] = ["appHandle": .string(handle), "pid": .null, "name": .string(app.name), "authorization": authorization] if let bundleID = app.bundleID { value["bundleId"] = .string(bundleID) } diff --git a/packages/computer-use/cua/native/Sources/CuaNative/ClipboardPaste.swift b/packages/computer-use/cua/native/Sources/CuaNative/ClipboardPaste.swift index 936f253900..27b6193fce 100644 --- a/packages/computer-use/cua/native/Sources/CuaNative/ClipboardPaste.swift +++ b/packages/computer-use/cua/native/Sources/CuaNative/ClipboardPaste.swift @@ -17,32 +17,31 @@ private final class PasteDataProvider: NSObject, NSPasteboardItemDataProvider, @ condition.broadcast() condition.unlock() } - func wait(check: () throws -> Void) throws { + @MainActor func wait(check: () throws -> Void) async throws { while true { try check() - condition.lock() - switch delivery { - case .supplied: condition.unlock(); return - case .failed: - condition.unlock() - throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Requested paste data could not be supplied") - case .pending: - // A bounded wait lets helper cancellation and a newer clipboard owner interrupt delivery. - _ = condition.wait(until: Date(timeIntervalSinceNow: 0.025)) - condition.unlock() + let state = condition.withLock { delivery } + switch state { + case .supplied: return + case .failed: throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Requested paste data could not be supplied") + case .pending: try await Task.sleep(for: .milliseconds(25)) } } } + } -final class ClipboardPaste { - private struct SavedItem { let values: [(NSPasteboard.PasteboardType, Data)] } +@MainActor final class ClipboardPaste { + private struct SavedItem: Sendable { let values: [(NSPasteboard.PasteboardType, Data)] } private static let ownerType = NSPasteboard.PasteboardType("org.deepseek-harness.cua.clipboard-owner") private let pasteboard: NSPasteboard + private var restorePending: (() throws -> Void)? init(pasteboard: NSPasteboard = .general) { self.pasteboard = pasteboard } - func perform(text: String, format: String, maxBytes: Int, checkCancellation: () throws -> Void = {}, operation: (() throws -> Void) throws -> Void) throws { + func perform(text: String, format: String, maxBytes: Int, checkCancellation: () throws -> Void = {}, + operation: (@escaping @MainActor @Sendable () throws -> Void) async throws -> Void) async throws { + try checkCancellation() let type: NSPasteboard.PasteboardType switch format { case "text": type = .string @@ -62,20 +61,25 @@ final class ClipboardPaste { guard pasteboard.changeCount == originalCount else { throw NativeFailure(code: "CLIPBOARD_CHANGED", message: "Clipboard changed while preserving it; paste was not dispatched") } + try checkCancellation() let cleared = pasteboard.clearContents() guard pasteboard.writeObjects([item]) else { if pasteboard.changeCount == cleared { try restore(saved) } throw NativeFailure(code: "CLIPBOARD_UNAVAILABLE", message: "Could not publish paste contents") } let expectedCount = pasteboard.changeCount - func owned() -> Bool { pasteboard.changeCount == expectedCount && pasteboard.data(forType: Self.ownerType) == owner } - func requireOwnership() throws { + let owned: @MainActor @Sendable () -> Bool = { [self] in + pasteboard.changeCount == expectedCount && pasteboard.data(forType: Self.ownerType) == owner + } + let requireOwnership: @MainActor @Sendable () throws -> Void = { guard owned() else { throw NativeFailure(code: "CLIPBOARD_CHANGED", message: "Clipboard changed before paste; input was not dispatched") } } + restorePending = { [self] in if owned() { try restore(saved) } } + defer { restorePending = nil } do { try requireOwnership() - try operation(requireOwnership) - try provider.wait { try checkCancellation(); try requireOwnership() } + try await operation(requireOwnership) + try await provider.wait { try checkCancellation(); try requireOwnership() } } catch { if owned() { try restore(saved) } throw error @@ -83,6 +87,11 @@ final class ClipboardPaste { if owned() { try restore(saved) } } + func cancel() throws { + try restorePending?() + restorePending = nil + } + private func snapshot(maxBytes: Int) throws -> [SavedItem] { var total = 0 return try (pasteboard.pasteboardItems ?? []).map { item in diff --git a/packages/computer-use/cua/native/Sources/CuaNative/RequestCoordinator.swift b/packages/computer-use/cua/native/Sources/CuaNative/RequestCoordinator.swift index 54d161aac9..59533e7ef1 100644 --- a/packages/computer-use/cua/native/Sources/CuaNative/RequestCoordinator.swift +++ b/packages/computer-use/cua/native/Sources/CuaNative/RequestCoordinator.swift @@ -9,6 +9,7 @@ import Darwin private let cancellation: CancellationLatch private let worker: AXWorker private let capture = WindowCapture() + private let clipboard = ClipboardPaste() private let input: JSONLInput private let writer: JSONLWriter private let signalQueue = DispatchQueue(label: "dsh.cua.signals") @@ -85,6 +86,12 @@ import Darwin let image = try await self.capture.capture(target, cancellation: self.cancellation) result = try await self.worker.finishCapture(target, image: image) case .foreground(let plan): result = try await self.performForeground(plan, requestID: request.id) + case .paste(let plan): + try await self.clipboard.perform(text: plan.text, format: plan.format, maxBytes: plan.maxBytes, + checkCancellation: { try self.cancellation.check() }) { requireClipboard in + try await self.worker.performPaste(plan, beforeDispatch: requireClipboard) + } + result = AXWorker.actionResult case .launch(let plan): let pid = try await ApplicationDiscovery.launch(plan.application, cancellation: self.cancellation) result = try await self.worker.finishLaunch(plan, pid: pid) @@ -144,6 +151,8 @@ import Darwin guard !stopping else { return } stopping = true cancellation.cancel() + do { try clipboard.cancel() } + catch { fputs("Clipboard restoration failed during shutdown: \(error.localizedDescription)\n", stderr) } active = nil operation?.cancel() input.stop() diff --git a/packages/computer-use/cua/native/tests/swift/ClipboardPasteTests.swift b/packages/computer-use/cua/native/tests/swift/ClipboardPasteTests.swift index 024dfd429b..d0c0d26276 100644 --- a/packages/computer-use/cua/native/tests/swift/ClipboardPasteTests.swift +++ b/packages/computer-use/cua/native/tests/swift/ClipboardPasteTests.swift @@ -4,7 +4,7 @@ import Foundation import Testing @testable import CuaNative -@Test func clipboardPasteRestoresEveryOriginalFormatAfterDispatch() throws { +@Test @MainActor func clipboardPasteRestoresEveryOriginalFormatAfterDispatch() async throws { let board = NSPasteboard.withUniqueName() defer { board.releaseGlobally() } let item = NSPasteboardItem() @@ -12,7 +12,7 @@ import Testing #expect(item.setString("original", forType: .string)) #expect(item.setData(Data([1, 2, 3]), forType: custom)) #expect(board.writeObjects([item])) - try ClipboardPaste(pasteboard: board).perform(text: "line 1\nline 2\tvalue", format: "text", maxBytes: 1024) { current in + try await ClipboardPaste(pasteboard: board).perform(text: "line 1\nline 2\tvalue", format: "text", maxBytes: 1024) { current in try current() #expect(board.string(forType: .string) == "line 1\nline 2\tvalue") } @@ -20,11 +20,11 @@ import Testing #expect(board.data(forType: custom) == Data([1, 2, 3])) } -@Test func clipboardPastePublishesHTMLWithoutTypingItsMarkup() throws { +@Test @MainActor func clipboardPastePublishesHTMLWithoutTypingItsMarkup() async throws { let board = NSPasteboard.withUniqueName() defer { board.releaseGlobally() } let html = "

Title

Bold link

" - try ClipboardPaste(pasteboard: board).perform(text: html, format: "html", maxBytes: 1024) { current in + try await ClipboardPaste(pasteboard: board).perform(text: html, format: "html", maxBytes: 1024) { current in try current() #expect(board.data(forType: .html) == Data(html.utf8)) #expect(board.string(forType: .string) == nil) @@ -32,12 +32,12 @@ import Testing #expect(board.pasteboardItems?.isEmpty ?? true) } -@Test func clipboardPastePreservesANewerCopyInsteadOfRestoringOverIt() throws { +@Test @MainActor func clipboardPastePreservesANewerCopyInsteadOfRestoringOverIt() async throws { let board = NSPasteboard.withUniqueName() defer { board.releaseGlobally() } #expect(board.setString("original", forType: .string)) do { - try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { current in + try await ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { current in board.clearContents() #expect(board.setString("new user copy", forType: .string)) try current() @@ -47,12 +47,12 @@ import Testing #expect(board.string(forType: .string) == "new user copy") } -@Test func clipboardPasteRestoresOnCancellationAndRejectsOversizedPreservationBeforeChangingData() throws { +@Test @MainActor func clipboardPasteRestoresOnCancellationAndRejectsOversizedPreservationBeforeChangingData() async throws { let board = NSPasteboard.withUniqueName() defer { board.releaseGlobally() } #expect(board.setString("original", forType: .string)) do { - try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in + try await ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in throw NativeFailure(code: "NATIVE_CANCELLED", message: "fixture cancellation") } Issue.record("Cancelled operation succeeded") @@ -62,7 +62,7 @@ import Testing let large = String(repeating: "x", count: 2048) #expect(board.setString(large, forType: .string)) do { - try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in + try await ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024) { _ in Issue.record("Oversized clipboard reached dispatch") } Issue.record("Oversized clipboard was accepted") @@ -70,12 +70,12 @@ import Testing #expect(board.string(forType: .string) == large) } -@Test func clipboardPasteAwaitsDataConsumptionAndCanCancelBeforeItIsRequested() throws { +@Test @MainActor func clipboardPasteAwaitsDataConsumptionAndCanCancelBeforeItIsRequested() async throws { let board = NSPasteboard.withUniqueName() defer { board.releaseGlobally() } #expect(board.setString("original", forType: .string)) do { - try ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024, + try await ClipboardPaste(pasteboard: board).perform(text: "paste", format: "text", maxBytes: 1024, checkCancellation: { throw NativeFailure(code: "NATIVE_CANCELLED", message: "delivery cancelled") }) { _ in } Issue.record("Unread clipboard data completed without cancellation") } catch let error as NativeFailure { #expect(error.code == "NATIVE_CANCELLED") }