feat(cua-windows): select applications by AUMID

This commit is contained in:
Huo Yaoyuan committed 2026-11-10 12:55:39 +08:00
1 parent a0f326da1d
commit d4785e69c0
24 files changed
+676 -574

No files matched your search

@@ -73,7 +73,7 @@ it('inlines discovery, restoration, screenshot, and mouse-move return records',
expect(lines.find(line => line.startsWith(`- ${name}(`))).toMatch(/ -> \(?\{ /)
}
expect(COMMON_GUIDANCE).toContain("{ platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false }")
expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, isRunning: true }[]')
expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, isRunning: boolean }[]')
expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, path: string, isRunning: boolean }[]')
expect(COMMON_GUIDANCE).toContain('bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null')
})
@@ -29,8 +29,9 @@ internal static class Accessibility
bool includeMenus, bool details, RequestContext context)
{
var all = Windows.Refresh(app, context);
var targets = new[] { all.SingleOrDefault(window => window.Id == selected)
?? throw new NativeFailure("WINDOW_CHANGED", "Select a current window") };
var target = Windows.Resolve(app, selected, context);
if (!all.Any(window => window.Id == target.Id)) all = [.. all, target];
var targets = new[] { target };
var rows = new List<string> { "AX scope=window offsets=UTF-16" };
var elements = new Dictionary<int, ElementReference>();
var observations = new List<(WindowState Window, WindowGeometry Geometry)>();
@@ -64,16 +65,18 @@ internal static class Accessibility
}
if (truncated) Add($"[truncated: maxElements={maximumElements}, maxDepth={maximumDepth}]");
DesktopSession.RequireInteractive();
var scope = selected;
var snapshot = new Snapshot(Guid.NewGuid().ToString("N"), scope, targets, elements);
var descriptions = Windows.DescribeAll(app, all, context);
if (!descriptions.Windows.Any(window => window.Id == selected))
throw new NativeFailure("WINDOW_CHANGED", "The selected window disappeared during observation; read fresh application state");
var snapshot = new Snapshot(Guid.NewGuid().ToString("N"), selected, targets, elements);
var result = new
{
appHandle = app.Handle, pid = 4, windowId = selected.ToString(CultureInfo.InvariantCulture),
focusedWindowId = Windows.Focused(app, all)?.Id.ToString(CultureInfo.InvariantCulture),
snapshotId = snapshot.Id, tree = string.Join('\n', rows), windows = all.Select(window => window.Describe(app, context)).ToArray(), truncated,
focusedWindowId = Windows.Focused(app, descriptions.Windows)?.Id.ToString(CultureInfo.InvariantCulture),
snapshotId = snapshot.Id, tree = string.Join('\n', rows), windows = descriptions.Values, truncated,
};
context.Check();
app.Snapshots[scope] = snapshot;
app.Snapshots[selected] = snapshot;
foreach (var observation in observations) observation.Window.Observe(observation.Geometry);
return result;
@@ -218,28 +221,26 @@ internal static class Accessibility
return "AXGroup";
}
internal static ElementReference Element(AppState app, string snapshotId, string elementId, RequestContext context)
internal static ElementReference Element(AppState app, WindowState window, string snapshotId, string elementId,
RequestContext context)
{
if (!int.TryParse(elementId, NumberStyles.None, CultureInfo.InvariantCulture, out var index))
throw new NativeFailure("INVALID_ELEMENT", "Use an elementId from this snapshot");
Windows.Refresh(app, context);
var selected = app.SelectedWindow ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Select a window with getApp before using element identifiers");
var snapshot = app.Snapshots.Values.SingleOrDefault(value => value.Id == snapshotId)
?? throw new NativeFailure("STALE_SNAPSHOT", "Read fresh AX state before using element identifiers");
if (snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != selected.Id)
throw new NativeFailure("STALE_SNAPSHOT", "Use an observation from the selected window");
window.Geometry(app, context);
if (!app.Snapshots.TryGetValue(window.Id, out var snapshot) || snapshot.Id != snapshotId
|| snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != window.Id)
throw new NativeFailure("STALE_SNAPSHOT", "Read fresh AX state before using element identifiers");
if (!snapshot.Elements.TryGetValue(index, out var element)) throw new NativeFailure("INVALID_ELEMENT", "Use an elementId from this snapshot");
element.Window.Geometry(app, context);
if (element.Window.Id != window.Id)
throw new NativeFailure("STALE_SNAPSHOT", "The element belongs to another window; read fresh AX state");
if (!context.Ax(() => element.Element.Current.IsEnabled)) throw new NativeFailure("ELEMENT_DISABLED", "The element is disabled");
return element;
}
internal static void RequireSnapshot(AppState app, string snapshotId, nint windowId)
{
var selected = app.SelectedWindow;
var snapshot = app.Snapshots.Values.SingleOrDefault(value => value.Id == snapshotId);
if (snapshot is null || selected is null || selected.Id != windowId
if (!app.Snapshots.TryGetValue(windowId, out var snapshot) || snapshot.Id != snapshotId
|| snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != windowId)
throw new NativeFailure("STALE_SNAPSHOT", "Use a fresh observation from the selected window");
throw new NativeFailure("STALE_SNAPSHOT", "Use a fresh observation from this window");
}
}
@@ -1,45 +1,48 @@
// Application consent identifies the executable before any launch or UI observation.
// Process identity, executable metadata, and the authorized application's own window state.
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Win32;
using System.Text.Json.Serialization;
using DeepSeek.Cua.Protocol;
namespace DeepSeek.Cua.Windows;
internal sealed record ApplicationIdentity(string Id, string Name, string ExecutablePath, bool Persistable);
internal sealed record RunningApplication(int Pid, long Started, string Path, string Name);
internal sealed record InstalledApplication(string Name, string Path);
internal sealed class AppState(string handle, ApplicationIdentity authorization, string digest, RunningApplication? process) : IDisposable
internal sealed record ApplicationAuthorization(string Id, string Name,
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ExecutablePath)
{
/// <summary>Not eligible for a permanent grant; the Host still persists conversation grants and denials.</summary>
public bool Persistable => false;
internal static ApplicationAuthorization Create(string appId, string name, string? executablePath)
=> new(Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("windows-aumid\0" + appId))), name, executablePath);
}
internal sealed class AppState(string handle, string appId, string name, ApplicationAuthorization authorization,
bool isRunning) : IDisposable
{
internal string Handle { get; } = handle;
internal WindowState? SelectedWindow { get; set; }
internal ApplicationIdentity Authorization { get; } = authorization;
internal string Digest { get; } = digest;
internal RunningApplication? Process { get; set; } = process;
internal string AppId { get; } = appId;
internal string Name { get; } = name;
internal bool IsRunning { get; } = isRunning;
internal ApplicationAuthorization Authorization { get; } = authorization;
internal Dictionary<nint, WindowState> Windows { get; } = [];
internal Dictionary<nint, Snapshot> Snapshots { get; } = [];
private System.Diagnostics.Process? owner;
internal RunningApplication RequireProcess()
public void Dispose()
{
var selected = Process ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Application launch requires Host authorization");
try
{
owner ??= System.Diagnostics.Process.GetProcessById(selected.Pid);
if (owner.HasExited || Applications.Started(owner) != selected.Started)
throw new NativeFailure("STALE_APP", "Select the application again after its process changes");
}
catch (ArgumentException) { throw new NativeFailure("STALE_APP", "Select the application again after it exits"); }
return selected;
Snapshots.Clear();
Windows.Clear();
}
public void Dispose() { owner?.Dispose(); Snapshots.Clear(); Windows.Clear(); }
}
internal static class Applications
{
internal static int SessionId { get; } = CurrentSession();
private static int CurrentSession() { using var process = System.Diagnostics.Process.GetCurrentProcess(); return process.SessionId; }
internal static object Icons(JsonElement parameters, RequestContext context)
{
var paths = parameters.GetProperty("paths");
@@ -71,14 +74,13 @@ internal static class Applications
return result;
}
private static readonly int SessionId = CurrentSession();
private static int CurrentSession() { using var process = System.Diagnostics.Process.GetCurrentProcess(); return process.SessionId; }
internal static long Started(System.Diagnostics.Process process)
{
if (!Win32.GetProcessTimes(process.SafeHandle, out var created, out _, out _, out _))
throw new NativeFailure("STALE_APP", "The process lifetime is unavailable; select the application again");
return created;
}
internal static string CanonicalPath(string path)
{
using var handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
@@ -98,54 +100,6 @@ internal static class Applications
return Convert.ToHexStringLower(SHA256.HashData(file));
}
private static string? VerifiedSigner(string path)
{
var file = new Win32.TrustFile { Size = (uint)Marshal.SizeOf<Win32.TrustFile>(), Path = path };
var memory = Marshal.AllocHGlobal(Marshal.SizeOf<Win32.TrustFile>());
Marshal.StructureToPtr(file, memory, false);
var data = new Win32.TrustData
{
Size = (uint)Marshal.SizeOf<Win32.TrustData>(), UiChoice = 2, UnionChoice = 1,
File = memory, StateAction = 1, ProviderFlags = 0x1000 | 0x80,
};
var action = new Guid("00AAC56B-CD44-11D0-8CC2-00C04FC295EE");
try
{
if (Win32.WinVerifyTrust(0, ref action, ref data) != 0) return null;
var provider = Win32.WTHelperProvDataFromStateData(data.StateData);
var signerPointer = provider == 0 ? 0 : Win32.WTHelperGetProvSignerFromChain(provider, 0, false, 0);
if (signerPointer == 0) return null;
var signer = Marshal.PtrToStructure<Win32.ProviderSigner>(signerPointer);
if (signer.CertificateCount == 0 || signer.Certificates == 0) return null;
var certificate = Marshal.PtrToStructure<Win32.ProviderCertificate>(signer.Certificates);
if (certificate.Certificate == 0) return null;
var context = Marshal.PtrToStructure<Win32.CertificateContext>(certificate.Certificate);
if (context.Encoded == 0 || context.EncodedSize == 0 || context.EncodedSize > int.MaxValue) return null;
var bytes = new byte[(int)context.EncodedSize];
Marshal.Copy(context.Encoded, bytes, 0, bytes.Length);
return Convert.ToHexStringLower(SHA256.HashData(bytes));
}
catch (CryptographicException) { return null; }
finally
{
data.StateAction = 2;
Win32.WinVerifyTrust(0, ref action, ref data);
Marshal.DestroyStructure<Win32.TrustFile>(memory);
Marshal.FreeHGlobal(memory);
}
}
internal static (ApplicationIdentity Identity, string Digest) Identify(string path, string name, RunningApplication? process)
{
var canonical = CanonicalPath(path);
var digest = Digest(canonical);
var signer = VerifiedSigner(canonical);
var requirement = signer is not null ? "signed:" + signer
: process is null ? "pending:" + digest : $"process:{process.Pid}:{process.Started}:{digest}";
var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("win32\0" + canonical + "\0" + requirement)));
return (new(id, name, canonical, signer is not null), digest);
}
internal static RunningApplication Running(int pid)
{
try
@@ -163,123 +117,4 @@ internal static class Applications
}
catch (ArgumentException) { throw new NativeFailure("STALE_APP", "The selected process has exited"); }
}
internal static IReadOnlyList<RunningApplication> List()
{
var ids = new HashSet<int>();
Win32.EnumWindows((window, _) =>
{
if (Win32.IsWindowVisible(window))
{
Win32.GetWindowThreadProcessId(window, out var pid);
if (pid != 0 && pid <= int.MaxValue) ids.Add((int)pid);
}
return true;
}, 0);
var result = new List<RunningApplication>();
foreach (var pid in ids.Order())
{
try { result.Add(Running(pid)); }
catch (NativeFailure) { /* Processes can exit or deny metadata while the desktop is enumerated. */ }
catch (IOException) { /* A disappearing executable is not a selectable application. */ }
}
return result;
}
internal static IReadOnlyList<InstalledApplication> Find(string query, int maximum, bool exact = false)
{
var found = new Dictionary<string, InstalledApplication>(StringComparer.Ordinal);
void Add(string name, string path)
{
if (exact ? !string.Equals(name, query, StringComparison.OrdinalIgnoreCase) && !string.Equals(Path.GetFileName(path), query, StringComparison.OrdinalIgnoreCase)
: !name.Contains(query, StringComparison.OrdinalIgnoreCase) && !Path.GetFileName(path).Contains(query, StringComparison.OrdinalIgnoreCase)) return;
if (!Path.IsPathFullyQualified(path) || !path.EndsWith(".exe", StringComparison.OrdinalIgnoreCase) || !File.Exists(path)) return;
found.TryAdd(CanonicalPath(path), new(name, CanonicalPath(path)));
if (found.Count > maximum) throw new NativeFailure("TOO_MANY_APPS", "Use a more specific application name or executable path");
}
foreach (var app in List()) Add(app.Name, app.Path);
foreach (var hive in new[] { RegistryHive.CurrentUser, RegistryHive.LocalMachine })
foreach (var view in new[] { RegistryView.Registry64, RegistryView.Registry32 })
{
using var registry = RegistryKey.OpenBaseKey(hive, view);
using var paths = registry.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths");
if (paths is null) continue;
foreach (var key in paths.GetSubKeyNames())
{
using var app = paths.OpenSubKey(key);
if (app?.GetValue(null) is string path) Add(Path.GetFileNameWithoutExtension(key), Environment.ExpandEnvironmentVariables(path.Trim('"')));
}
}
foreach (var directory in new[] { Environment.SpecialFolder.Programs, Environment.SpecialFolder.CommonPrograms })
{
var root = Environment.GetFolderPath(directory);
if (!Directory.Exists(root)) continue;
foreach (var link in Directory.EnumerateFiles(root, "*.lnk", new EnumerationOptions
{ RecurseSubdirectories = true, IgnoreInaccessible = true, AttributesToSkip = FileAttributes.ReparsePoint }))
{
var name = Path.GetFileNameWithoutExtension(link);
if (!name.Contains(query, StringComparison.OrdinalIgnoreCase)) continue;
var target = ShellBroker.ShortcutTarget(link);
if (target is not null) Add(name, target);
}
}
return found.Values.OrderBy(app => app.Name, StringComparer.OrdinalIgnoreCase).ToArray();
}
internal static AppState Select(string selector, IEnumerable<AppState> retained)
{
RunningApplication? running = null;
string path, name;
if (int.TryParse(selector, out var pid))
{
running = Running(pid); path = running.Path; name = running.Name;
}
else
{
var absolute = Path.IsPathFullyQualified(selector) ? CanonicalPath(selector) : null;
var matches = List().Where(app => absolute is not null ? string.Equals(app.Path, absolute, StringComparison.Ordinal) : string.Equals(app.Name, selector, StringComparison.OrdinalIgnoreCase)
|| string.Equals(Path.GetFileName(app.Path), selector, StringComparison.OrdinalIgnoreCase)
).ToArray();
if (matches.Length > 1) throw new NativeFailure("AMBIGUOUS_APP", "Select one of the observed process IDs");
if (matches.Length == 1) { running = matches[0]; path = running.Path; name = running.Name; }
else if (absolute is not null) { path = absolute; name = Path.GetFileNameWithoutExtension(path); }
else
{
var installed = Find(selector, 1, true).Where(app => string.Equals(app.Name, selector, StringComparison.OrdinalIgnoreCase)
|| string.Equals(Path.GetFileName(app.Path), selector, StringComparison.OrdinalIgnoreCase)).ToArray();
if (installed.Length != 1) throw new NativeFailure(installed.Length == 0 ? "APP_NOT_FOUND" : "AMBIGUOUS_APP", "Use findApps, an executable path, or an observed PID");
path = installed[0].Path; name = installed[0].Name;
}
}
if (!path.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) throw new NativeFailure("UNSUPPORTED", "Select a Windows application executable");
if (running is not null)
{
var previous = retained.FirstOrDefault(app => app.Process?.Pid == running.Pid && app.Process.Started == running.Started);
if (previous is not null) return previous;
}
var identity = Identify(path, name, running);
return new(Guid.NewGuid().ToString("N"), identity.Identity, identity.Digest, running);
}
internal static async Task Launch(AppState app, RequestContext context)
{
if (app.Process is not null) { app.RequireProcess(); return; }
var identity = app.Authorization;
if (Digest(identity.ExecutablePath) != app.Digest) throw new NativeFailure("STALE_APP", "The selected executable changed before launch; select it again");
context.Check();
await context.Sta.Invoke(() => ShellBroker.Launch(identity.ExecutablePath), context.Cancellation);
while (true)
{
context.Check();
var matches = List().Where(candidate => string.Equals(candidate.Path, identity.ExecutablePath, StringComparison.Ordinal)).ToArray();
if (matches.Length > 1) throw new NativeFailure("AMBIGUOUS_APP", "Several matching processes started; select an observed PID");
if (matches.Length == 1)
{
if (Digest(matches[0].Path) != app.Digest) throw new NativeFailure("STALE_APP", "The launched executable differs from the approved file");
app.Process = matches[0];
return;
}
await Task.Delay(context.Config.PollIntervalMs, context.Cancellation);
}
}
}
@@ -154,15 +154,16 @@ internal sealed class ClipboardPaste
finally { Win32.CloseClipboard(); }
}
internal async Task Paste(AppState app, string text, string format, int maximumBytes, int readTimeoutMs, RequestContext context)
internal async Task Paste(AppState app, WindowState window, string text, string format, int maximumBytes, int readTimeoutMs, RequestContext context)
{
Windows.RequireForeground(app, null, context);
var targetPid = app.RequireProcess().Pid;
Windows.RequireForeground(app, window, context);
// Clipboard readers are matched against this window's real process, not a removal-only app identity.
var targetPid = window.ProcessId;
var transaction = await context.Sta.Invoke(control => Prepare(control, targetPid, text, format, maximumBytes), context.Cancellation);
try
{
await context.Sta.Invoke(() => transaction.Begin(), context.Cancellation);
Input.Press(app, "v", ["control"], context);
Input.Press(app, window, "v", ["control"], context);
try { await transaction.Consumed.Task.WaitAsync(TimeSpan.FromMilliseconds(readTimeoutMs), context.Cancellation); }
catch (TimeoutException)
{
@@ -7,11 +7,10 @@ namespace DeepSeek.Cua.Windows;
internal sealed class ClientState
{
internal Dictionary<string, PreparedLaunch> Launches { get; } = new(StringComparer.Ordinal);
internal Dictionary<string, AppState> Apps { get; } = new(StringComparer.Ordinal);
internal AppState App(string handle) => Apps.GetValueOrDefault(handle)
?? throw new NativeFailure("STALE_APP", "Select the application again on this connection");
internal void Clear() { foreach (var app in Apps.Values) app.Dispose(); Apps.Clear(); Launches.Clear(); }
internal void Clear() { foreach (var app in Apps.Values) app.Dispose(); Apps.Clear(); }
}
internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IAsyncDisposable
@@ -33,17 +32,12 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
private static nint? WindowId(JsonElement value) => value.TryGetProperty("windowId", out var id) ? ParseWindowId(id) : null;
private static string[] Modifiers(JsonElement value) => value.TryGetProperty("modifiers", out var items) ? items.EnumerateArray().Select(item => item.GetString()!).ToArray() : [];
// Window-scoped operations act only on the window authorized at getApp; the request must name it.
// Window-scoped operations revalidate the explicit target against the authorized exact AUMID.
private static WindowState Selected(AppState app, JsonElement parameters, RequestContext context)
{
if (!parameters.TryGetProperty("windowId", out var raw))
throw new NativeFailure("UNSUPPORTED", "This operation requires a window target; select a window with getApp and pass its windowId");
var id = ParseWindowId(raw);
var selected = app.SelectedWindow ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Select a window with getApp before this operation");
if (selected.Id != id)
throw new NativeFailure("WINDOW_CHANGED", "The request targets another window; select that window with getApp");
selected.Geometry(app, context);
return selected;
throw new NativeFailure("UNSUPPORTED", "This operation requires a window target; select a window with getWindows and pass its windowId");
return Windows.Resolve(app, ParseWindowId(raw), context);
}
private static (WindowState Window, double X, double Y) Coordinate(AppState app, JsonElement parameters, RequestContext context)
@@ -88,24 +82,24 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
if (request.Method is "requestPermission" or "openPermissionSettings")
throw new NativeFailure("UNSUPPORTED", "Windows desktop access has no macOS permission switches; inspect the Windows session status");
DesktopSession.RequireInteractive();
if (request.Method == "listApps") return DesktopWindows.ListApps(context);
if (request.Method == "listApps") return await DesktopWindows.ListApps(context);
if (request.Method == "findApps")
throw new NativeFailure("UNSUPPORTED", "Windows discovers selectable windows with listApps; installed launch entries are not part of this selection");
throw new NativeFailure("UNSUPPORTED", "Windows discovers selectable applications with listApps");
if (request.Method == "getApp")
{
var selected = DesktopWindows.Select(String(parameters, "selector"), context);
var selected = await DesktopWindows.Select(String(parameters, "selector"), context);
state.Apps[selected.Handle] = selected;
return new { appHandle = selected.Handle, pid = 4, name = selected.Authorization.Name,
running = true, authorization = selected.Authorization };
return new { appHandle = selected.Handle, pid = 4, name = selected.Name,
running = selected.IsRunning, authorization = selected.Authorization };
}
if (request.Method == "launchApp")
throw new NativeFailure("UNSUPPORTED", "A selected Windows window cannot be launched; select an observed window from listApps");
throw new NativeFailure("UNSUPPORTED", "Windows application selection does not launch applications");
var app = state.App(String(parameters, "appHandle"));
app.RequireProcess();
var foreground = Win32.GetForegroundWindow();
switch (request.Method)
{
case "listWindows": return Windows.Refresh(app, context).Select(window => window.Describe(app, context)).ToArray();
case "listWindows":
return Windows.DescribeAll(app, Windows.Refresh(app, context), context).Values;
case "getState":
{
var target = Selected(app, parameters, context);
@@ -115,29 +109,34 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
case "activate": Windows.Activate(app, null, context); return ActionResult(foreground);
case "show": return Windows.Show(app, WindowId(parameters), context);
case "typeText":
Selected(app, parameters, context);
await Input.Type(app, String(parameters, "text"), context);
{
var target = Selected(app, parameters, context);
await Input.Type(app, target, String(parameters, "text"), context);
return ActionResult(foreground);
}
case "pressKey":
Selected(app, parameters, context);
Input.Press(app, String(parameters, "key"), Modifiers(parameters), context);
{
var target = Selected(app, parameters, context);
Input.Press(app, target, String(parameters, "key"), Modifiers(parameters), context);
return ActionResult(foreground);
}
case "sendKeyEvent":
throw new NativeFailure("UNSUPPORTED", "Held key events are not supported on current platform.");
case "paste":
Selected(app, parameters, context);
await clipboard.Paste(app, String(parameters, "text"), OptionalString(parameters, "format") ?? "text",
{
var target = Selected(app, parameters, context);
await clipboard.Paste(app, target, String(parameters, "text"), OptionalString(parameters, "format") ?? "text",
parameters.GetProperty("maxClipboardBytes").GetInt32(), parameters.GetProperty("pasteReadTimeoutMs").GetInt32(), context);
return ActionResult(foreground);
}
case "mouseMove":
{
var target = Selected(app, parameters, context);
if (parameters.TryGetProperty("elementId", out _))
{
var target = Selected(app, parameters, context);
return await mta.Invoke(() =>
{
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
if (element.Window.Id != target.Id) throw new NativeFailure("WINDOW_CHANGED", "The element belongs to another window");
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
var bounds = context.Ax(() => element.Element.Current.BoundingRectangle);
if (bounds.IsEmpty) throw new NativeFailure("UNSUPPORTED", "The element has no pointer bounds");
var geometry = element.Window.RequireObservation(app, context);
@@ -183,8 +182,7 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
{
await mta.Invoke(() =>
{
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
if (element.Window.Id != target.Id) throw new NativeFailure("WINDOW_CHANGED", "The element belongs to another window");
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
if (!hold.HasValue && button == "left" && count == 1 && modifiers.Length == 0 && AccessibilityActions.Click(element, context)) return true;
var bounds = context.Ax(() => element.Element.Current.BoundingRectangle);
if (bounds.IsEmpty) throw new NativeFailure("UNSUPPORTED", "The element has no pointer bounds");
@@ -204,17 +202,17 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
}
case "listActions":
{
Selected(app, parameters, context);
var target = Selected(app, parameters, context);
return await mta.Invoke(() => AccessibilityActions.Actions(
Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation);
Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation);
}
case "performAction":
{
Selected(app, parameters, context);
var target = Selected(app, parameters, context);
var argument = parameters.TryGetProperty("argument", out var raw) ? raw : (JsonElement?)null;
await mta.Invoke(() =>
{
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
AccessibilityActions.Perform(app, element, String(parameters, "action"), argument, context);
return true;
}, cancellation);
@@ -222,17 +220,17 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
}
case "invoke":
{
Selected(app, parameters, context);
var target = Selected(app, parameters, context);
var dispatched = await mta.Invoke(() => AccessibilityActions.Invoke(
Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation);
Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation);
return new { dispatched };
}
case "setValue":
{
Selected(app, parameters, context);
var target = Selected(app, parameters, context);
await mta.Invoke(() =>
{
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
AccessibilityActions.SetValue(element, parameters.GetProperty("value"), context);
return true;
}, cancellation);
@@ -240,10 +238,10 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
}
case "selectText":
{
Selected(app, parameters, context);
var target = Selected(app, parameters, context);
await mta.Invoke(() =>
{
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
AccessibilityActions.SelectText(element, String(parameters, "text"), OptionalString(parameters, "prefix"), OptionalString(parameters, "suffix"),
OptionalString(parameters, "selectionType") ?? "text", context);
return true;
@@ -1,98 +1,378 @@
// Desktop discovery selects windows by their invariant decimal HWND strings.
using System.Globalization;
using System.Security.Cryptography;
// AppsFolder AUMID enumeration, exact-ordinal merge, and exact window identity resolution.
using System.ComponentModel;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Text;
using DeepSeek.Cua.Protocol;
using Microsoft.Win32.SafeHandles;
namespace DeepSeek.Cua.Windows;
internal sealed record DiscoveredWindow(WindowState Window, RunningApplication Process)
{
internal string Selector => Window.Id.ToString(CultureInfo.InvariantCulture);
}
internal sealed record ShellApplication(string AppId, string DisplayName, bool HasShellName);
internal sealed record ShellCatalog(ShellApplication[] Applications, int Omitted);
internal sealed record ResolvedWindow(nint Handle, string AppId, string? ExecutablePath, string ProcessName,
int ProcessId, long ProcessStarted);
internal sealed record WindowObservation(ResolvedWindow[] Windows, int Denied, int Gone);
internal sealed record CatalogApplication(string AppId, string DisplayName, string? ExecutablePath, bool IsRunning);
internal static class DesktopWindows
{
internal static object[] ListApps(RequestContext context)
{
var result = new List<object>();
foreach (var entry in Discover(context))
{
context.Check();
try
{
using var owner = Owner(entry, false, context);
entry.Window.Geometry(owner, context);
result.Add(new { pid = 4, name = owner.Authorization.Name, id = entry.Selector });
}
catch (NativeFailure error) when (error.Code is "STALE_APP" or "PERMISSION_DENIED" or "APP_IDENTITY_UNAVAILABLE" or "UNSUPPORTED_SESSION" or "WINDOW_CHANGED")
{
// A disappearing or inaccessible window is omitted from discovery.
}
}
return result.ToArray();
}
private const uint ProcessQueryLimitedInformation = 0x1000;
private const int ErrorAccessDenied = 5;
private const int ErrorInvalidParameter = 87;
private const int ErrorInsufficientBuffer = 122;
private const int AppModelErrorNoApplication = 15703;
private const int EAccessDenied = unchecked((int)0x80070005);
private const int EHandle = unchecked((int)0x80070006);
private const int EInvalidArgument = unchecked((int)0x80070057);
private const int EInvalidWindowHandle = unchecked((int)0x80070578);
private const int MaximumIdentityBuffer = 1_048_576;
private static readonly Guid AppUserModelIdFormat = new("9F4C2855-9F79-4B39-A8D0-E1D42DE1D5F3");
internal static AppState Select(string selector, RequestContext context)
internal static async Task<object[]> ListApps(RequestContext context)
{
if (string.IsNullOrEmpty(selector)) throw new NativeFailure("INVALID_ARGUMENT", "A window selector from listApps is required");
var entry = Discover(context).SingleOrDefault(value => string.Equals(value.Selector, selector, StringComparison.Ordinal))
?? throw new NativeFailure("WINDOW_CHANGED", "Discover applications again; this window is no longer available");
var app = Owner(entry, true, context);
try { entry.Window.Geometry(app, context); return app; }
catch { app.Dispose(); throw; }
}
private static List<DiscoveredWindow> Discover(RequestContext context)
{
var current = new List<nint>();
Win32.EnumWindows((handle, _) =>
var applications = await Discover(context);
var result = applications.Select(application => new
{
var extended = Win32.GetWindowLongPtr(handle, -20).ToInt64();
if (Win32.IsWindowVisible(handle) && (extended & 0x80) == 0
&& Win32.GetWindowRect(handle, out var bounds) && bounds.Width > 0 && bounds.Height > 0)
current.Add(handle);
return true;
}, 0);
var result = new List<DiscoveredWindow>();
foreach (var handle in current)
{
context.Check();
try
{
Win32.GetWindowThreadProcessId(handle, out var pid);
var process = Applications.Running(checked((int)pid));
var entry = new DiscoveredWindow(new(handle), process);
using var owner = Owner(entry, false, context);
entry.Window.Geometry(owner, context);
result.Add(entry);
}
catch (NativeFailure error) when (error.Code is "STALE_APP" or "PERMISSION_DENIED" or "APP_IDENTITY_UNAVAILABLE" or "UNSUPPORTED_SESSION" or "WINDOW_CHANGED")
{
// A disappearing or inaccessible window is omitted from discovery.
}
}
pid = 4, id = application.AppId, name = application.DisplayName, isRunning = application.IsRunning,
}).ToArray();
context.Check();
return result;
}
private static AppState Owner(DiscoveredWindow entry, bool authorize, RequestContext context)
internal static async Task<AppState> Select(string selector, RequestContext context)
{
var window = entry.Window;
var process = entry.Process;
var length = Win32.GetWindowTextLength(window.Handle);
if (length > context.Config.MaxResponseBytes) throw new NativeFailure("NATIVE_RESPONSE_LIMIT", "Window title exceeds the response limit");
var title = new StringBuilder(length + 1);
Win32.GetWindowText(window.Handle, title, title.Capacity);
var name = $"Window: {title} ({process.Name})";
// Shared consent IDs require a SHA-256 string, including non-persistable window grants.
var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes($"window\0{entry.Selector}")));
var identity = new ApplicationIdentity(id, name, process.Path, false);
var app = new AppState(Guid.NewGuid().ToString("N"), identity, "", process);
if (authorize)
{
app.SelectedWindow = window;
app.Windows.Add(window.Handle, window);
}
return app;
var applications = await Discover(context);
var application = SelectApplication(applications, selector);
context.Check();
return new AppState(Guid.NewGuid().ToString("N"), application.AppId, application.DisplayName,
ApplicationAuthorization.Create(application.AppId, application.DisplayName, application.ExecutablePath),
application.IsRunning);
}
private static async Task<CatalogApplication[]> Discover(RequestContext context)
{
var catalog = await context.Sta.Invoke(_ => ReadAppsFolder(context), context.Cancellation);
var observation = Observe(context);
if (catalog.Omitted > 0)
Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {catalog.Omitted} AppsFolder items without an application identity");
if (observation.Denied > 0)
Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {observation.Denied} windows whose process or window access was denied");
if (observation.Gone > 0)
Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {observation.Gone} disappeared or identity-less windows");
var applications = Merge(catalog.Applications, observation.Windows);
context.Check();
return applications;
}
private static ShellCatalog ReadAppsFolder(RequestContext context)
{
var owned = new List<object>();
object Own(object value) { owned.Add(value); return value; }
try
{
dynamic shell = Own(Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("13709620-C279-11CE-A49E-444553540000"), true)!)!);
dynamic folder = Own((object)shell.NameSpace("shell:AppsFolder"));
if (folder is null) throw new COMException("AppsFolder is unavailable");
dynamic items = Own((object)folder.Items());
var applications = new List<ShellApplication>();
var omitted = 0;
for (var index = 0; index < (int)items.Count; index++)
{
context.Check();
dynamic item = Own((object)items.Item(index));
// A failed property read is a catalog failure; only an absent or non-string property is missing.
object? property = item.ExtendedProperty("System.AppUserModel.ID");
object? displayName = item.Name;
string? aumid = property as string;
string? name = displayName as string;
if (string.IsNullOrWhiteSpace(aumid)) { omitted++; continue; }
var hasName = !string.IsNullOrWhiteSpace(name);
applications.Add(new(aumid, hasName ? name! : aumid, hasName));
}
return new(applications.ToArray(), omitted);
}
catch (Exception error) when (error is COMException or Microsoft.CSharp.RuntimeBinder.RuntimeBinderException)
{
throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows Shell could not enumerate AppsFolder; retry discovery after Explorer is available");
}
finally
{
var released = new HashSet<object>(ReferenceEqualityComparer.Instance);
foreach (var value in owned.AsEnumerable().Reverse())
if (released.Add(value) && Marshal.IsComObject(value)) Marshal.FinalReleaseComObject(value);
}
}
private static CatalogApplication SelectApplication(IReadOnlyList<CatalogApplication> applications, string selector)
{
if (string.IsNullOrEmpty(selector))
throw new NativeFailure("INVALID_ARGUMENT", "An exact application id from listApps is required");
foreach (var application in applications)
if (string.Equals(application.AppId, selector, StringComparison.Ordinal)) return application;
throw new NativeFailure("APP_NOT_FOUND", "Use an exact application id from listApps");
}
private static CatalogApplication[] Merge(ShellApplication[] shellApplications, ResolvedWindow[] windows)
{
var applications = new Dictionary<string, Builder>(StringComparer.Ordinal);
foreach (var shell in shellApplications)
{
if (string.IsNullOrEmpty(shell.AppId)) continue;
if (!applications.TryGetValue(shell.AppId, out var existing))
applications.Add(shell.AppId, new(shell.AppId, shell.DisplayName, shell.HasShellName, shellNamed: true));
else existing.PreferShellName(shell.DisplayName, shell.HasShellName);
}
foreach (var window in windows.OrderBy(window => window.AppId, StringComparer.Ordinal)
.ThenBy(window => (long)window.Handle))
{
if (string.IsNullOrEmpty(window.AppId) || string.IsNullOrEmpty(window.ProcessName)) continue;
if (!applications.TryGetValue(window.AppId, out var builder))
{
builder = new(window.AppId, window.ProcessName, hasShellName: false, shellNamed: false);
applications.Add(window.AppId, builder);
}
builder.Add(window);
}
return applications.Values.Select(builder => builder.Build())
.OrderBy(application => application.AppId, StringComparer.Ordinal).ToArray();
}
private sealed class Builder(string appId, string displayName, bool hasShellName, bool shellNamed)
{
private readonly HashSet<string> executablePaths = new(StringComparer.Ordinal);
private readonly HashSet<string> processNames = new(StringComparer.Ordinal);
private string shellDisplayName = displayName;
private bool shellNameProvided = hasShellName;
internal void PreferShellName(string candidate, bool provided)
{
if (provided && !shellNameProvided) { shellDisplayName = candidate; shellNameProvided = true; }
else if (provided == shellNameProvided && string.CompareOrdinal(candidate, shellDisplayName) < 0)
shellDisplayName = candidate;
}
internal void Add(ResolvedWindow window)
{
if (!string.IsNullOrEmpty(window.ExecutablePath)) executablePaths.Add(window.ExecutablePath);
processNames.Add(window.ProcessName);
}
internal CatalogApplication Build()
{
var name = shellNamed ? shellDisplayName
: processNames.Count == 0 ? shellDisplayName : processNames.Min(StringComparer.Ordinal)!;
var executablePath = executablePaths.Count == 1 ? executablePaths.First() : null;
return new(appId, name, executablePath, processNames.Count != 0);
}
}
internal static WindowObservation Observe(RequestContext context)
{
var windows = new List<ResolvedWindow>();
var denied = 0;
var gone = 0;
foreach (var handle in EnumerateHandles(context))
{
context.Check();
try
{
var window = ResolveWindow(handle, context);
if (window is null) gone++;
else windows.Add(window);
}
catch (NativeFailure error) when (error.Code == "PERMISSION_DENIED") { denied++; }
}
return new(windows.ToArray(), denied, gone);
}
internal static bool Eligible(nint window)
{
// EnumWindows yields top-level windows; the explicit Resolve path also checks the root ancestor.
if (Win32.GetAncestor(window, 2) != window) return false;
var extended = Win32.GetWindowLongPtr(window, -20).ToInt64();
if ((extended & 0x80) != 0) return false;
if (!Win32.IsWindowVisible(window)) return false;
if (!Win32.GetWindowRect(window, out var bounds) || bounds.Width <= 0 || bounds.Height <= 0) return false;
Win32.GetWindowThreadProcessId(window, out var processId);
if (processId == 0 || processId > int.MaxValue) return false;
return Win32.ProcessIdToSessionId(processId, out var session) && session == Applications.SessionId;
}
private static IReadOnlyList<nint> EnumerateHandles(RequestContext context)
{
var handles = new List<nint>();
var enumerated = Win32.EnumWindows((window, _) =>
{
// Cancellation cannot throw across the unmanaged callback; stop, then rethrow outside it.
if (context.Cancellation.IsCancellationRequested) return false;
if (Eligible(window)) handles.Add(window);
return true;
}, 0);
context.Check();
if (!enumerated) throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows could not enumerate top-level windows; retry discovery after the desktop is available");
return handles;
}
internal static ResolvedWindow? ResolveWindow(nint handle, RequestContext context)
{
context.Check();
Win32.GetWindowThreadProcessId(handle, out var rawProcessId);
if (rawProcessId == 0 || rawProcessId > int.MaxValue) return null;
var processId = (int)rawProcessId;
if (!Win32.ProcessIdToSessionId(rawProcessId, out var session) || session != Applications.SessionId) return null;
var process = Win32.OpenProcess(ProcessQueryLimitedInformation, false, rawProcessId);
if (process == 0) return OpenFailure();
try
{
using var native = new SafeProcessHandle(process, ownsHandle: false);
if (!Win32.GetProcessTimes(native, out var started, out _, out _, out _)) return null;
var processName = ReadProcessName(processId);
if (string.IsNullOrEmpty(processName)) return null;
context.Check();
var windowAumid = ReadWindowAumid(handle, context);
string? executablePath = null;
string? appId = null;
if (!string.IsNullOrEmpty(windowAumid)) appId = windowAumid;
else
{
appId = ReadProcessAumid(process, context);
if (string.IsNullOrEmpty(appId)) executablePath = ReadExecutablePath(process, context);
appId ??= executablePath;
}
if (appId is null) return null;
// A recycled handle must not inherit the identity resolved for its former process.
Win32.GetWindowThreadProcessId(handle, out var currentProcessId);
if (currentProcessId != rawProcessId) return null;
using var current = new SafeProcessHandle(process, ownsHandle: false);
if (!Win32.GetProcessTimes(current, out var currentStarted, out _, out _, out _) || currentStarted != started) return null;
context.Check();
return new(handle, appId, executablePath, processName, processId, started);
}
finally { Win32.CloseHandle(process); }
}
private static ResolvedWindow? OpenFailure() => Marshal.GetLastPInvokeError() switch
{
ErrorAccessDenied => throw PermissionDenied(),
ErrorInvalidParameter => null,
_ => throw Unavailable(),
};
private static string? ReadProcessName(int processId)
{
try
{
using var process = Process.GetProcessById(processId);
return process.ProcessName;
}
catch (ArgumentException) { return null; }
catch (InvalidOperationException) { return null; }
catch (Win32Exception error) when (error.NativeErrorCode == ErrorAccessDenied) { throw PermissionDenied(); }
catch (Win32Exception) { throw Unavailable(); }
}
private static string? ReadWindowAumid(nint handle, RequestContext context)
{
context.Check();
var read = context.Sta.InvokeSync(_ => ReadWindowAumidCore(handle, context), context.Cancellation);
context.Check();
return read;
}
private static string? ReadWindowAumidCore(nint handle, RequestContext context)
{
var id = typeof(Win32.IPropertyStore).GUID;
var status = Win32.SHGetPropertyStoreForWindow(handle, ref id, out var store);
if (status != 0)
{
if (status == EAccessDenied) throw PermissionDenied();
if (status is EHandle or EInvalidArgument or EInvalidWindowHandle) return null;
throw Unavailable();
}
if (store is null) throw Unavailable();
try
{
var key = new Win32.PropertyKey { FormatId = AppUserModelIdFormat, PropertyId = 5 };
var value = default(Win32.PropVariant);
var result = store.GetValue(ref key, out value);
try
{
if (result != 0)
{
if (result == EAccessDenied) throw PermissionDenied();
throw Unavailable();
}
context.Check();
if (value.ValueType == 0) return null;
if (value.ValueType == 31) return value.Pointer == 0 ? null : Marshal.PtrToStringUni(value.Pointer);
if (value.ValueType == 8) return value.Pointer == 0 ? null : Marshal.PtrToStringBSTR(value.Pointer);
throw Unavailable();
}
finally { Win32.PropVariantClear(ref value); }
}
finally { Marshal.FinalReleaseComObject(store); }
}
private static string? ReadProcessAumid(nint process, RequestContext context)
{
context.Check();
uint length = 0;
var status = Win32.GetApplicationUserModelId(process, ref length, null);
if (status == 0) throw Unavailable();
if (status != ErrorInsufficientBuffer) return ClassifyAumidStatus(status);
for (var attempt = 0; attempt < 4; attempt++)
{
context.Check();
if (length == 0 || length > MaximumIdentityBuffer) throw Unavailable();
var buffer = new StringBuilder(checked((int)length));
uint size = length;
status = Win32.GetApplicationUserModelId(process, ref size, buffer);
if (status == 0)
{
var value = buffer.ToString();
if (string.IsNullOrEmpty(value)) throw Unavailable();
return value;
}
if (status != ErrorInsufficientBuffer) return ClassifyAumidStatus(status);
length = size;
}
throw Unavailable();
}
private static string? ClassifyAumidStatus(int status)
{
if (status == AppModelErrorNoApplication) return null;
if (status == ErrorAccessDenied) throw PermissionDenied();
throw Unavailable();
}
private static string? ReadExecutablePath(nint process, RequestContext context)
{
for (var capacity = 1024; capacity <= 65536; capacity *= 2)
{
context.Check();
var buffer = new StringBuilder(capacity);
uint size = (uint)capacity;
if (Win32.QueryFullProcessImageName(process, 0, buffer, ref size))
{
context.Check();
var path = buffer.ToString(0, (int)Math.Min(size, (uint)buffer.Length));
if (path.Length == 0) throw Unavailable();
return path;
}
var error = Marshal.GetLastPInvokeError();
if (error == ErrorInsufficientBuffer) continue;
if (error == ErrorAccessDenied) throw PermissionDenied();
if (error == ErrorInvalidParameter) return null;
throw Unavailable();
}
throw Unavailable();
}
private static NativeFailure PermissionDenied()
=> new("PERMISSION_DENIED", "Windows refused the window application identity; its privilege level may exceed the helper's");
private static NativeFailure Unavailable()
=> new("APP_IDENTITY_UNAVAILABLE", "The window application identity could not be resolved; retry after its owner is available");
}
@@ -95,6 +95,7 @@ internal sealed class StaDispatcher : IAsyncDisposable
private readonly TaskCompletionSource<ServiceControl> ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource<object?> stopped = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly Thread thread;
private ServiceControl? service;
internal StaDispatcher()
{
@@ -105,6 +106,7 @@ internal sealed class StaDispatcher : IAsyncDisposable
Win32.SetThreadDpiAwarenessContext(-4);
using var control = new ServiceControl();
_ = control.Handle;
service = control;
ready.SetResult(control);
Forms.Application.Run();
}
@@ -118,16 +120,37 @@ internal sealed class StaDispatcher : IAsyncDisposable
internal async Task<T> Invoke<T>(Func<ServiceControl, T> work, CancellationToken cancellation)
{
var control = await ready.Task.WaitAsync(cancellation);
cancellation.ThrowIfCancellationRequested();
// A caller already on the STA must not enqueue work for the message loop it is blocking.
if (Environment.CurrentManagedThreadId == thread.ManagedThreadId)
{
var inline = work(control);
cancellation.ThrowIfCancellationRequested();
return inline;
}
var result = new TaskCompletionSource<T>(TaskCreationOptions.RunContinuationsAsynchronously);
control.BeginInvoke(() =>
{
try { cancellation.ThrowIfCancellationRequested(); var value = work(control); result.TrySetResult(value); }
try { cancellation.ThrowIfCancellationRequested(); var value = work(control); cancellation.ThrowIfCancellationRequested(); result.TrySetResult(value); }
catch (OperationCanceledException) { result.TrySetCanceled(cancellation); }
catch (Exception error) { result.TrySetException(error); }
});
return await result.Task;
}
internal T InvokeSync<T>(Func<ServiceControl, T> work, CancellationToken cancellation)
{
cancellation.ThrowIfCancellationRequested();
if (Environment.CurrentManagedThreadId == thread.ManagedThreadId)
{
var control = service ?? throw new InvalidOperationException("The STA dispatcher has not started");
var inline = work(control);
cancellation.ThrowIfCancellationRequested();
return inline;
}
return Invoke(work, cancellation).GetAwaiter().GetResult();
}
internal Task Invoke(Action work, CancellationToken cancellation) => Invoke(_ => { work(); return true; }, cancellation);
public async ValueTask DisposeAsync()
@@ -46,7 +46,7 @@ internal static class Input
finally { foreach (var key in attempted.AsEnumerable().Reverse()) Release(Keyboard(key, flags: 2u | (key == 0x5B ? 1u : 0))); }
}
private static (ushort Key, bool Extended, string[] Modifiers) ResolveKey(string name, AppState app)
private static (ushort Key, bool Extended, string[] Modifiers) ResolveKey(string name)
{
var key = name.ToLowerInvariant();
if (key.StartsWith("kp_", StringComparison.Ordinal))
@@ -85,38 +85,38 @@ internal static class Input
return ((ushort)(mapped & 0xFF), false, modifiers.ToArray());
}
internal static void Press(AppState app, string key, string[] modifiers, RequestContext context)
internal static void Press(AppState app, WindowState window, string key, string[] modifiers, RequestContext context)
{
Windows.RequireForeground(app, null, context);
Windows.RequireForeground(app, window, context);
var chord = new KeyChord(key, modifiers.Select(KeyChord.NormalizeModifier).Distinct().ToArray());
var name = chord.Key.ToLowerInvariant();
if (name == "printscreen" || chord.Modifiers.Contains("option") && name is "tab" or "escape" or "esc"
|| chord.Modifiers.Contains("control") && name is "escape" or "esc"
|| chord.Modifiers.Contains("control") && chord.Modifiers.Contains("option") && name == "delete")
throw new NativeFailure("UNSUPPORTED", "This shortcut acts on the system rather than the selected application");
var resolved = ResolveKey(chord.Key, app);
var resolved = ResolveKey(chord.Key);
WithModifiers(chord.Modifiers.Concat(resolved.Modifiers), () =>
{
Windows.RequireForeground(app, null, context);
Windows.RequireForeground(app, window, context);
var flags = resolved.Extended ? 1u : 0;
try { Send(Keyboard(resolved.Key, flags: flags), Keyboard(resolved.Key, flags: flags | 2)); }
catch { Release(Keyboard(resolved.Key, flags: flags | 2)); throw; }
});
}
internal static async Task Type(AppState app, string text, RequestContext context)
internal static async Task Type(AppState app, WindowState window, string text, RequestContext context)
{
if (text.Contains('\0')) throw new NativeFailure("INVALID_ARGUMENT", "Text input cannot contain NUL");
RequireIdleModifiers();
for (var index = 0; index < text.Length; index++)
{
Windows.RequireForeground(app, null, context);
Windows.RequireForeground(app, window, context);
RequireIdleModifiers();
var character = text[index];
if (character is '\r' or '\n' or '\t')
{
if (character == '\r' && index + 1 < text.Length && text[index + 1] == '\n') index++;
Press(app, character == '\t' ? "Tab" : "Enter", [], context);
Press(app, window, character == '\t' ? "Tab" : "Enter", [], context);
}
else
{
@@ -1,88 +0,0 @@
// Shell namespace enumeration and activation run on the existing STA dispatcher.
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using DeepSeek.Cua.Protocol;
namespace DeepSeek.Cua.Windows;
internal sealed record LaunchEntry(string Id, string DisplayName, string Kind, string? Path);
internal sealed class PreparedLaunch(LaunchEntry entry, ApplicationIdentity authorization, string? digest)
{
internal string Handle { get; } = Guid.NewGuid().ToString("N");
internal ApplicationIdentity Authorization { get; } = authorization;
private bool dispatched;
internal void Dispatch()
{
if (dispatched) throw new NativeFailure("LAUNCH_ALREADY_DISPATCHED", "Discover windows before deciding whether another launch is needed");
if (entry.Kind == "executable")
{
if (Applications.CanonicalPath(entry.Id) != entry.Id || Applications.Digest(entry.Id) != digest)
throw new NativeFailure("STALE_APP", "The approved executable changed; select it again");
}
else if (!LaunchEntries.All().Any(current => current.Id == entry.Id))
throw new NativeFailure("STALE_APP", "The Shell entry disappeared; discover launch entries again");
// Dispatch may succeed even if Explorer subsequently reports a failure. Never replay this selection.
dispatched = true;
ShellBroker.Launch(entry.Id, show: 1);
}
}
internal static class LaunchEntries
{
internal static LaunchEntry[] All()
{
var owned = new List<object>();
object Own(object value) { owned.Add(value); return value; }
try
{
dynamic shell = Own(Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("13709620-C279-11CE-A49E-444553540000"), true)!)!);
dynamic folder = Own((object)shell.NameSpace("shell:AppsFolder"));
dynamic items = Own((object)folder.Items());
var result = new List<LaunchEntry>();
for (var index = 0; index < (int)items.Count; index++)
{
dynamic item = Own((object)items.Item(index));
string parsingName = item.Path;
string name = item.Name;
if (string.IsNullOrWhiteSpace(parsingName) || string.IsNullOrWhiteSpace(name)) continue;
var target = parsingName.StartsWith("::", StringComparison.Ordinal) ? parsingName : "shell:AppsFolder\\" + parsingName;
result.Add(new(target, name, "shell", null));
}
return result.DistinctBy(entry => entry.Id, StringComparer.OrdinalIgnoreCase).ToArray();
}
catch (Exception error) when (error is COMException or Microsoft.CSharp.RuntimeBinder.RuntimeBinderException)
{
throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows Shell could not enumerate AppsFolder; retry discovery after Explorer is available");
}
finally
{
foreach (var item in owned.AsEnumerable().Reverse())
if (Marshal.IsComObject(item)) Marshal.ReleaseComObject(item);
}
}
internal static LaunchEntry[] Find(string query, int maximum)
{
var matches = All().Where(entry => entry.DisplayName.Contains(query, StringComparison.OrdinalIgnoreCase)
|| entry.Id.Contains(query, StringComparison.OrdinalIgnoreCase)).OrderBy(entry => entry.DisplayName, StringComparer.OrdinalIgnoreCase).ToArray();
if (matches.Length > maximum) throw new NativeFailure("TOO_MANY_APPS", "Use a more specific launch entry name");
return matches;
}
internal static PreparedLaunch Prepare(string target)
{
if (Path.IsPathFullyQualified(target) && target.EndsWith(".exe", StringComparison.OrdinalIgnoreCase))
{
var path = Applications.CanonicalPath(target);
var name = Path.GetFileNameWithoutExtension(path);
var identity = Applications.Identify(path, "Launch: " + name, null);
return new(new(path, name, "executable", path), identity.Identity with { Persistable = false }, identity.Digest);
}
var shell = All().SingleOrDefault(entry => string.Equals(entry.Id, target, StringComparison.OrdinalIgnoreCase))
?? throw new NativeFailure("APP_NOT_FOUND", "Use an ID from findApps or a user-specified absolute executable path");
var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("launch\0" + shell.Id)));
return new(shell, new(id, "Launch: " + shell.DisplayName, shell.Id, false), null);
}
}
@@ -1,8 +1,6 @@
// Explorer owns user-launched applications, independently of the helper's managed process Job.
using System.Reflection;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.ComTypes;
using System.Text;
using DeepSeek.Cua.Protocol;
namespace DeepSeek.Cua.Windows;
@@ -44,21 +42,6 @@ internal static class ShellBroker
}
}
internal static string? ShortcutTarget(string path)
{
object? instance = null;
try
{
instance = Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("00021401-0000-0000-C000-000000000046"), true)!);
((IPersistFile)instance!).Load(path, 0);
var target = new StringBuilder(32768);
((IShellLink)instance).GetPath(target, target.Capacity, 0, 4);
return target.Length == 0 ? null : Environment.ExpandEnvironmentVariables(target.ToString());
}
catch (COMException) { return null; }
finally { if (instance is not null) Marshal.FinalReleaseComObject(instance); }
}
[ComImport, Guid("6D5140C1-7436-11CE-8034-00AA006009FA"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
private interface IServiceProvider
{
@@ -103,9 +86,4 @@ internal static class ShellBroker
[PreserveSig] int GetItemObject(uint item, ref Guid id, [MarshalAs(UnmanagedType.Interface)] out object value);
}
[ComImport, Guid("000214F9-0000-0000-C000-000000000046"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
private interface IShellLink
{
void GetPath([Out, MarshalAs(UnmanagedType.LPWStr)] StringBuilder path, int capacity, nint findData, uint flags);
}
}
@@ -92,42 +92,42 @@ internal static class Win32
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] internal static extern uint GetFinalPathNameByHandle(Microsoft.Win32.SafeHandles.SafeFileHandle file, StringBuilder path, uint length, uint flags);
[DllImport("kernel32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool GetProcessTimes(Microsoft.Win32.SafeHandles.SafeProcessHandle process, out long created, out long exited, out long kernel, out long user);
[DllImport("wintrust.dll", CharSet = CharSet.Unicode, ExactSpelling = true)] internal static extern int WinVerifyTrust(nint window, ref Guid action, ref TrustData data);
[DllImport("wintrust.dll")] internal static extern nint WTHelperProvDataFromStateData(nint state);
[DllImport("wintrust.dll")] internal static extern nint WTHelperGetProvSignerFromChain(nint provider, uint signer, [MarshalAs(UnmanagedType.Bool)] bool counterSigner, uint counter);
[StructLayout(LayoutKind.Sequential)] internal struct ProviderSigner
[DllImport("shell32.dll", PreserveSig = true)]
internal static extern int SHGetPropertyStoreForWindow(nint window, ref Guid id,
[MarshalAs(UnmanagedType.Interface)] out IPropertyStore? store);
[DllImport("ole32.dll")] internal static extern int PropVariantClear(ref PropVariant value);
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, ExactSpelling = true)]
internal static extern int GetApplicationUserModelId(nint process, ref uint length, StringBuilder? value);
[DllImport("kernel32.dll", EntryPoint = "QueryFullProcessImageNameW", CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static extern bool QueryFullProcessImageName(nint process, uint flags, StringBuilder name, ref uint length);
[DllImport("kernel32.dll", SetLastError = true)]
internal static extern nint OpenProcess(uint access, [MarshalAs(UnmanagedType.Bool)] bool inherit, uint processId);
[DllImport("kernel32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static extern bool CloseHandle(nint handle);
[DllImport("kernel32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static extern bool ProcessIdToSessionId(uint processId, out uint sessionId);
[StructLayout(LayoutKind.Sequential)] internal struct PropertyKey { internal Guid FormatId; internal uint PropertyId; }
// PROPVARIANT is 24 bytes on supported x64/arm64 targets; the trailing field reserves its union.
[StructLayout(LayoutKind.Sequential)] internal struct PropVariant
{
internal uint Size, VerifyTimeLow, VerifyTimeHigh, CertificateCount;
internal nint Certificates;
}
[StructLayout(LayoutKind.Sequential)] internal struct ProviderCertificate
{
internal uint Size;
internal nint Certificate;
}
[StructLayout(LayoutKind.Sequential)] internal struct CertificateContext
{
internal uint Encoding;
internal nint Encoded;
internal uint EncodedSize;
internal ushort ValueType;
internal ushort Reserved1, Reserved2, Reserved3;
internal nint Pointer;
internal nint Pointer2;
}
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] internal struct TrustFile
[ComImport, Guid("886D8EEB-8CF2-4446-8D02-CDBA1DBDCF99"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface IPropertyStore
{
internal uint Size;
[MarshalAs(UnmanagedType.LPWStr)] internal string Path;
internal nint Handle, Subject;
}
[StructLayout(LayoutKind.Sequential)] internal struct TrustData
{
internal uint Size;
internal nint PolicyCallback, SipClient;
internal uint UiChoice, RevocationChecks, UnionChoice;
internal nint File;
internal uint StateAction;
internal nint StateData, UrlReference;
internal uint ProviderFlags, UiContext;
internal nint SignatureSettings;
[PreserveSig] int GetCount(out uint count);
[PreserveSig] int GetAt(uint index, out PropertyKey key);
[PreserveSig] int GetValue(ref PropertyKey key, out PropVariant value);
[PreserveSig] int SetValue(ref PropertyKey key, ref PropVariant value);
[PreserveSig] int Commit();
}
}
@@ -1,4 +1,4 @@
// Window state retains HWND values; JSON exposes them as invariant decimal strings.
// Window state belongs to one exact application identity and revalidates its own process and AUMID.
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text;
@@ -12,24 +12,56 @@ internal readonly record struct WindowGeometry(Win32.Rect Pixels, uint Dpi)
internal Bounds Bounds => new(Pixels.Left, Pixels.Top, Pixels.Width, Pixels.Height);
}
internal sealed class WindowState(nint handle)
internal sealed class WindowState(nint handle, int processId, long processStarted)
{
internal nint Id => Handle;
internal nint Handle { get; } = handle;
internal int ProcessId { get; } = processId;
internal long ProcessStarted { get; } = processStarted;
private WindowGeometry? observation;
internal void RequireCurrent(AppState app, RequestContext context)
{
context.Check();
if (!Win32.IsWindow(Handle))
throw Remove(app, "The selected window is no longer available; read fresh application state");
var current = DesktopWindows.ResolveWindow(Handle, context);
if (current is null)
throw Remove(app, "The selected window is no longer available; read fresh application state");
if (!string.Equals(app.AppId, current.AppId, StringComparison.Ordinal)
|| ProcessId != current.ProcessId || ProcessStarted != current.ProcessStarted)
throw Remove(app, "The selected window no longer belongs to the authorized application; read fresh application state");
}
// A stale window and its snapshots must not remain selectable after a direct operation detects the change.
private NativeFailure Remove(AppState app, string message)
{
if (app.Windows.TryGetValue(Handle, out var current) && ReferenceEquals(current, this))
{
app.Windows.Remove(Handle);
app.Snapshots.Remove(Handle);
}
return new NativeFailure("WINDOW_CHANGED", message);
}
internal WindowGeometry Geometry(AppState app, RequestContext context)
{
context.Check();
app.RequireProcess();
if (!Win32.IsWindow(Handle))
throw new NativeFailure("WINDOW_CHANGED", "The selected window is no longer available; read fresh application state");
RequireCurrent(app, context);
return TryGeometry(out var geometry)
? geometry
: throw new NativeFailure("WINDOW_CHANGED", "The selected window geometry is unavailable; read fresh application state");
}
private bool TryGeometry(out WindowGeometry geometry)
{
geometry = default;
if (Win32.DwmGetWindowAttribute(Handle, 9, out Win32.Rect rectangle, Marshal.SizeOf<Win32.Rect>()) != 0
&& !Win32.GetWindowRect(Handle, out rectangle)) throw new NativeFailure("WINDOW_CHANGED", "The selected window geometry is unavailable");
&& !Win32.GetWindowRect(Handle, out rectangle)) return false;
var dpi = Win32.GetDpiForWindow(Handle);
if (dpi == 0 || rectangle.Width <= 0 || rectangle.Height <= 0)
throw new NativeFailure("WINDOW_CHANGED", "The selected window has no usable bounds");
return new(rectangle, dpi);
if (dpi == 0 || rectangle.Width <= 0 || rectangle.Height <= 0) return false;
geometry = new(rectangle, dpi);
return true;
}
internal void Observe(WindowGeometry value) => observation = value;
@@ -56,58 +88,101 @@ internal sealed class WindowState(nint handle)
internal object Describe(AppState app, RequestContext context)
{
var geometry = Geometry(app, context);
RequireCurrent(app, context);
// A minimized or cloaked window may lack usable capture geometry; keep listing its association.
var bounds = TryGeometry(out var geometry) ? geometry.Bounds : observation?.Bounds ?? new Bounds(0, 0, 0, 0);
var length = Win32.GetWindowTextLength(Handle);
if (length > context.Config.MaxResponseBytes) throw new NativeFailure("NATIVE_RESPONSE_LIMIT", "The window title exceeds the response limit");
var title = new StringBuilder(length + 1);
Win32.GetWindowText(Handle, title, title.Capacity);
return new
{
windowId = Id.ToString(CultureInfo.InvariantCulture), pid = 4, appName = app.RequireProcess().Name,
title = title.ToString(), bounds = geometry.Bounds, isOnScreen = IsOnScreen(), onCurrentSpace = (bool?)null,
windowId = Id.ToString(CultureInfo.InvariantCulture), pid = 4, appName = app.Name,
title = title.ToString(), bounds, isOnScreen = IsOnScreen(Handle), onCurrentSpace = (bool?)null,
};
}
internal bool IsOnScreen() => Win32.IsWindowVisible(Handle) && !Win32.IsIconic(Handle)
&& (Win32.DwmGetWindowAttribute(Handle, 14, out int cloaked, sizeof(int)) != 0 || cloaked == 0);
internal static bool IsOnScreen(nint handle) => Win32.IsWindowVisible(handle) && !Win32.IsIconic(handle)
&& (Win32.DwmGetWindowAttribute(handle, 14, out int cloaked, sizeof(int)) != 0 || cloaked == 0);
}
internal static class Windows
{
internal static WindowState[] Refresh(AppState app, RequestContext context)
{
var process = app.RequireProcess();
if (app.SelectedWindow is { } selected) { selected.Geometry(app, context); return [selected]; }
var handles = new List<nint>();
Win32.EnumWindows((window, _) =>
var observation = DesktopWindows.Observe(context);
var current = new Dictionary<nint, WindowState>();
var added = new List<WindowState>();
foreach (var window in observation.Windows)
{
Win32.GetWindowThreadProcessId(window, out var pid);
var style = Win32.GetWindowLongPtr(window, -16).ToInt64();
var extended = Win32.GetWindowLongPtr(window, -20).ToInt64();
if (pid == process.Pid && (extended & 0x80) == 0
&& (Win32.IsWindowVisible(window) || (style & 0x00C00000) == 0x00C00000)
&& Win32.GetWindowRect(window, out var bounds) && bounds.Width > 0 && bounds.Height > 0) handles.Add(window);
return true;
}, 0);
foreach (var pair in app.Windows.ToArray())
{
if (handles.Contains(pair.Key)) continue;
app.Windows.Remove(pair.Key);
foreach (var scope in app.Snapshots.Where(item => item.Value.Windows.Any(window => window.Id == pair.Value.Id)).Select(item => item.Key).ToArray())
app.Snapshots.Remove(scope);
}
foreach (var handle in handles)
{
if (!app.Windows.ContainsKey(handle))
if (!string.Equals(window.AppId, app.AppId, StringComparison.Ordinal)) continue;
if (app.Windows.TryGetValue(window.Handle, out var existing)
&& existing.ProcessId == window.ProcessId && existing.ProcessStarted == window.ProcessStarted)
current[window.Handle] = existing;
else
{
app.Windows.Add(handle, new(handle));
var state = new WindowState(window.Handle, window.ProcessId, window.ProcessStarted);
current[window.Handle] = state;
added.Add(state);
}
}
return handles.Select(handle => app.Windows[handle]).ToArray();
foreach (var pair in app.Windows.ToArray())
{
if (current.TryGetValue(pair.Key, out var kept) && ReferenceEquals(kept, pair.Value)) continue;
app.Windows.Remove(pair.Key);
app.Snapshots.Remove(pair.Key);
}
foreach (var window in added) app.Windows[window.Handle] = window;
return current.Values.OrderBy(window => (long)window.Handle).ToArray();
}
internal static WindowState Resolve(AppState app, nint id, RequestContext context) => Refresh(app, context)
.SingleOrDefault(window => window.Id == id) ?? throw new NativeFailure("WINDOW_CHANGED", "Select a current window from getWindows");
internal static WindowState Resolve(AppState app, nint id, RequestContext context)
{
if (app.Windows.TryGetValue(id, out var known))
{
// A known target is revalidated in place; a changed or closed one is removed, never switched.
known.Geometry(app, context);
return known;
}
context.Check();
if (!DesktopWindows.Eligible(id))
throw new NativeFailure("WINDOW_CHANGED", "Select a current window from getWindows");
var observed = DesktopWindows.ResolveWindow(id, context);
if (observed is null || !string.Equals(observed.AppId, app.AppId, StringComparison.Ordinal))
throw new NativeFailure("WINDOW_CHANGED", "The window does not belong to the authorized application; read fresh application state");
// A newly appeared explicit window may join the application; no default target is inferred.
app.Snapshots.Remove(id);
var window = new WindowState(observed.Handle, observed.ProcessId, observed.ProcessStarted);
app.Windows[id] = window;
return window;
}
// A sibling window that disappears during enumeration is omitted with its snapshots; a direct
// selected target still fails through Resolve with a structured error.
internal static (object[] Values, WindowState[] Windows) DescribeAll(AppState app, WindowState[] windows,
RequestContext context)
{
var described = new List<object>();
var surviving = new List<WindowState>();
foreach (var window in windows)
{
context.Check();
try
{
described.Add(window.Describe(app, context));
surviving.Add(window);
}
catch (NativeFailure error) when (error.Code is "WINDOW_CHANGED" or "PERMISSION_DENIED")
{
if (app.Windows.TryGetValue(window.Handle, out var current) && ReferenceEquals(current, window))
{
app.Windows.Remove(window.Handle);
app.Snapshots.Remove(window.Handle);
}
}
}
return (described.ToArray(), surviving.ToArray());
}
internal static WindowState? Focused(AppState app, WindowState[] windows)
{
@@ -130,25 +205,24 @@ internal static class Windows
internal static WindowState? Default(AppState app, WindowState[] windows)
{
var visible = windows.Where(window => window.IsOnScreen()).ToArray();
var focused = Focused(app, visible);
var onScreen = windows.Where(window => WindowState.IsOnScreen(window.Handle)).ToArray();
var focused = Focused(app, onScreen);
if (focused is not null) return focused;
if (visible.Length == 1) return visible[0];
if (visible.Length == 0 && windows.Length <= 1) return windows.SingleOrDefault();
if (onScreen.Length == 1) return onScreen[0];
if (onScreen.Length == 0 && windows.Length <= 1) return windows.SingleOrDefault();
throw new NativeFailure("AMBIGUOUS_WINDOW", "Select a window explicitly from getWindows");
}
internal static object Show(AppState app, nint? id, RequestContext context)
{
var target = id is { } selected ? Resolve(app, selected, context) : Default(app, Refresh(app, context));
var target = id is { } selected
? Resolve(app, selected, context)
: (Default(app, Refresh(app, context)) ?? throw new NativeFailure("WINDOW_NOT_FOUND", "The application has no window to show"));
var before = Win32.GetForegroundWindow();
if (target is not null)
{
target.Geometry(app, context);
if (!Win32.ShowWindowAsync(target.Handle, 4)) throw new NativeFailure("UNSUPPORTED", "Windows refused background restoration; explicitly activate the application if needed");
}
target.Geometry(app, context);
if (!Win32.ShowWindowAsync(target.Handle, 4)) throw new NativeFailure("UNSUPPORTED", "Windows refused background restoration; explicitly activate the application if needed");
var after = Win32.GetForegroundWindow();
return new { dispatched = true, windowId = target?.Id.ToString(CultureInfo.InvariantCulture),
return new { dispatched = true, windowId = target.Id.ToString(CultureInfo.InvariantCulture),
foregroundChanged = before == 0 || after == 0 ? (bool?)null : before != after };
}
@@ -161,15 +235,14 @@ internal static class Windows
throw new NativeFailure("FOREGROUND_DENIED", "Windows refused foreground access; ask the user to focus the target window");
}
internal static void RequireForeground(AppState app, WindowState? target, RequestContext context)
internal static void RequireForeground(AppState app, WindowState target, RequestContext context)
{
context.Check();
DesktopSession.RequireInteractive();
target ??= app.SelectedWindow;
target?.Geometry(app, context);
target.Geometry(app, context);
var foreground = Win32.GetForegroundWindow();
Win32.GetWindowThreadProcessId(foreground, out var pid);
if (pid != app.RequireProcess().Pid || target is not null && Win32.GetAncestor(foreground, 2) != target.Handle)
if (pid != (uint)target.ProcessId || Win32.GetAncestor(foreground, 2) != target.Handle)
throw new NativeFailure("FOREGROUND_REQUIRED", "This input needs the target in front; explain why, explicitly activate or raise it, then observe again");
}
}
@@ -52,13 +52,11 @@ it.skipIf(!helperPath || !selector)('admits real application screenshots through
image: (image) => { expect(image.bytes.byteLength).toBeGreaterThan(0) },
})
const discovered = (await api.listApps({ emit: false })).find(app =>
app.id === selector || app.displayName.toLowerCase() === selector?.toLowerCase())
expect(discovered).toBeDefined()
const app = await api.getApp(discovered!.id)
app.id === selector || app.displayName === selector)
const app = await api.getApp(discovered?.id ?? selector!)
const target = (await app.getWindows({ emit: false })).find(window =>
window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0)
expect(target).toBeDefined()
expect(target!.windowId).toBe(discovered!.id)
const window = await app.getWindow(target!.windowId)
expect(emitted).toEqual([])
await window.getAccessibilityTree()
+1 -1
View File
@@ -65,7 +65,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
},
async listApps(options) {
const result = AppsResult.parse(await call('listApps', {})).map(app => ({
id: app.id ?? (app.bundleId || app.name), displayName: app.name, isRunning: true as const,
id: app.id ?? (app.bundleId || app.name), displayName: app.name, isRunning: app.isRunning,
}))
await emit(sink, result, options)
return result
+3 -3
View File
@@ -10,10 +10,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched:
Setup and discovery:
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true }[] — enumerate running applications.
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications.
- cua.findApps(query: string, options?: { emit?: boolean }) -> { id: string, displayName: string, path: string, isRunning: boolean }[] — search installed applications, including stopped matches.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app.
- app.getWindows(options?: { emit?: boolean }) -> { windowId: string, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
- app.getWindow(windowId: string) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.
+1
View File
@@ -273,6 +273,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend)
}).parse(result)
await access.select(agent, client, prepared.appHandle, authorization, signal, applicationIcon)
signal.throwIfAborted()
// TODO: launchApp on Windows
const selected = prepared.running
? prepared
: AppResult.parse(await access.run(client, prepared.appHandle, signal,
+4 -2
View File
@@ -113,8 +113,10 @@ export const AppResult = z.object({
appHandle: z.string().min(1).transform(value => value as CuaAppHandle),
name: z.string(), bundleId: z.string().optional(), running: z.boolean(),
})
/** Running application metadata used for selection; other native fields are ignored. */
export const AppsResult = z.array(z.object({ id: z.string().min(1).optional(), name: z.string(), bundleId: z.string().optional() }))
/** Application discovery entries with the provider-reported running state. */
export const AppsResult = z.array(z.object({
id: z.string().min(1).optional(), name: z.string(), bundleId: z.string().optional(), isRunning: z.boolean(),
}))
/** Installed application matches and their current running state. */
export const FoundAppsResult = z.array(z.object({
id: z.string().min(1), displayName: z.string(), path: z.string().min(1), isRunning: z.boolean(),
+8 -8
View File
@@ -50,8 +50,8 @@ export type CuaStartupCapabilities = Pick<
CuaCapabilities,
'platform' | 'methods' | 'appLevelAccessibility' | 'keyboard' | 'browser' | 'coordinateSpace'
> & { systemPointer?: boolean }
/** App discovery result accepted by getApp. */
export type CuaAppInfo = Pick<CuaFoundAppInfo, 'id' | 'displayName'> & { isRunning: true }
/** App discovery result accepted by getApp, including the provider-reported running state. */
export type CuaAppInfo = Pick<CuaFoundAppInfo, 'id' | 'displayName' | 'isRunning'>
/** An installed application match with its current running state. */
export type CuaFoundAppInfo = z.output<typeof FoundAppsResult>[number]
/** Window bounds in the provider's global native units. Local operations use an origin of [0,0]. */
@@ -256,9 +256,9 @@ export interface CuaApi {
*/
getCapabilities(options?: CuaObservationOptions): Promise<CuaCapabilities>
/**
* Enumerate running applications.
* List applications with their current running state. Providers may enumerate only running applications.
* @param options - output options.
* @returns identifiers accepted by getApp.
* @returns identifiers accepted by getApp and each application's running state.
*/
listApps(options?: CuaObservationOptions): Promise<CuaAppInfo[]>
/**
@@ -269,14 +269,14 @@ export interface CuaApi {
*/
findApps(query: string, options?: CuaObservationOptions): Promise<CuaFoundAppInfo[]>
/**
* Discover apps.
* Summarize provider application discovery.
* @param options - output options.
* @returns native apps and an empty browser list.
* @returns native apps with their running state and an empty browser list.
*/
getState(options?: CuaObservationOptions): Promise<{ apps: CuaAppInfo[]; browsers: [] }>
/**
* Select an app, obtain consent, launch it in the background if necessary, and silently retain initial AX state.
* @param selector - Provider-issued identifier, exact name, bundle ID or absolute application path; ambiguous selectors reject.
* Select an app, obtain consent, and launch it only where the provider supports launch; selection never activates it.
* @param selector - Provider-issued ID, or a supported exact name, bundle ID or absolute path; ambiguous selectors reject.
* @returns An application handle tied to this process lifetime; its initial tree read becomes the current snapshot.
*/
getApp(selector: string): Promise<CuaApp>
+4 -4
View File
@@ -11,7 +11,7 @@ const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
const bounds = { x: -640, y: 100, width: 640, height: 480 }
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', running: true }
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', isRunning: true, running: true }
const delivered = { dispatched: true }
const capabilities: CuaCapabilities = { platform: 'macos', systemPointer: true, sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
@@ -27,7 +27,7 @@ function fixture(withSink = true) {
const dispatch = vi.fn(async ({ method, params }: CuaOperation): Promise<JsonValue> => {
switch (method) {
case 'getCapabilities': return currentCapabilities
case 'listApps': return [selected, { id: 'utility-selector', pid: 4, name: 'Utility' }]
case 'listApps': return [selected, { id: 'utility-selector', pid: 4, name: 'Utility', isRunning: true }]
case 'getApp': return selected
case 'listWindows': return windows
case 'mouseMove': return { ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false }
@@ -81,8 +81,8 @@ describe('application discovery', () => {
it('uses the application name when discovery has no identifier', async () => {
const f = fixture()
f.dispatch.mockResolvedValueOnce([{ pid: 18, name: 'Utility' }])
expect(await f.api.listApps({ emit: false })).toEqual([{ id: 'Utility', displayName: 'Utility', isRunning: true }])
f.dispatch.mockResolvedValueOnce([{ pid: 18, name: 'Utility', isRunning: false }])
expect(await f.api.listApps({ emit: false })).toEqual([{ id: 'Utility', displayName: 'Utility', isRunning: false }])
})
it('returns observations without requiring an output sink', async () => {
@@ -56,7 +56,7 @@ it('propagates a denied action without retrying it', async () => {
})
it('supports callers that retain observations without a text callback', async () => {
const cua = connectCua(() => async () => [{ id: 'editor-selector', name: 'Editor' }])
const cua = connectCua(() => async () => [{ id: 'editor-selector', name: 'Editor', isRunning: true }])
await expect(cua.listApps()).resolves.toEqual([{ id: 'editor-selector', displayName: 'Editor', isRunning: true }])
})
@@ -43,10 +43,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched:
Setup and discovery:
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications.
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications.
- cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app.
- app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
- app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.
@@ -131,7 +131,7 @@ function createApplicationState(socket, record, application, captureBroker, conn
return { platform: 'darwin', ...state, helper: { pid: process.pid }, parent: { pid: process.ppid } }
}
if (method === 'getCapabilities') return { sessionLocked: false, permissions: { accessibility: true, screenRecording: true } }
if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId }]
if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId, isRunning: true }]
if (method === 'getApp') {
if (params.selector === 'Denied Fixture' || params.selector === 'Revocable Fixture') {
const denied = params.selector === 'Denied Fixture'
@@ -45,10 +45,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched:
Setup and discovery:
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications.
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications.
- cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app.
- app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
- app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.
@@ -45,10 +45,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched:
Setup and discovery:
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications.
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications.
- cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation.
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app.
- app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
- app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.