diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index 266ec10141..aff433344e 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -73,7 +73,7 @@ it('inlines discovery, restoration, screenshot, and mouse-move return records', expect(lines.find(line => line.startsWith(`- ${name}(`))).toMatch(/ -> \(?\{ /) } expect(COMMON_GUIDANCE).toContain("{ platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false }") - expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, isRunning: true }[]') + expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, isRunning: boolean }[]') expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, path: string, isRunning: boolean }[]') expect(COMMON_GUIDANCE).toContain('bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null') }) diff --git a/packages/experimental/cua-windows/native/entry/Accessibility.cs b/packages/experimental/cua-windows/native/entry/Accessibility.cs index 92327237a2..b4846ff0f9 100644 --- a/packages/experimental/cua-windows/native/entry/Accessibility.cs +++ b/packages/experimental/cua-windows/native/entry/Accessibility.cs @@ -29,8 +29,9 @@ internal static class Accessibility bool includeMenus, bool details, RequestContext context) { var all = Windows.Refresh(app, context); - var targets = new[] { all.SingleOrDefault(window => window.Id == selected) - ?? throw new NativeFailure("WINDOW_CHANGED", "Select a current window") }; + var target = Windows.Resolve(app, selected, context); + if (!all.Any(window => window.Id == target.Id)) all = [.. all, target]; + var targets = new[] { target }; var rows = new List { "AX scope=window offsets=UTF-16" }; var elements = new Dictionary(); var observations = new List<(WindowState Window, WindowGeometry Geometry)>(); @@ -64,16 +65,18 @@ internal static class Accessibility } if (truncated) Add($"[truncated: maxElements={maximumElements}, maxDepth={maximumDepth}]"); DesktopSession.RequireInteractive(); - var scope = selected; - var snapshot = new Snapshot(Guid.NewGuid().ToString("N"), scope, targets, elements); + var descriptions = Windows.DescribeAll(app, all, context); + if (!descriptions.Windows.Any(window => window.Id == selected)) + throw new NativeFailure("WINDOW_CHANGED", "The selected window disappeared during observation; read fresh application state"); + var snapshot = new Snapshot(Guid.NewGuid().ToString("N"), selected, targets, elements); var result = new { appHandle = app.Handle, pid = 4, windowId = selected.ToString(CultureInfo.InvariantCulture), - focusedWindowId = Windows.Focused(app, all)?.Id.ToString(CultureInfo.InvariantCulture), - snapshotId = snapshot.Id, tree = string.Join('\n', rows), windows = all.Select(window => window.Describe(app, context)).ToArray(), truncated, + focusedWindowId = Windows.Focused(app, descriptions.Windows)?.Id.ToString(CultureInfo.InvariantCulture), + snapshotId = snapshot.Id, tree = string.Join('\n', rows), windows = descriptions.Values, truncated, }; context.Check(); - app.Snapshots[scope] = snapshot; + app.Snapshots[selected] = snapshot; foreach (var observation in observations) observation.Window.Observe(observation.Geometry); return result; @@ -218,28 +221,26 @@ internal static class Accessibility return "AXGroup"; } - internal static ElementReference Element(AppState app, string snapshotId, string elementId, RequestContext context) + internal static ElementReference Element(AppState app, WindowState window, string snapshotId, string elementId, + RequestContext context) { if (!int.TryParse(elementId, NumberStyles.None, CultureInfo.InvariantCulture, out var index)) throw new NativeFailure("INVALID_ELEMENT", "Use an elementId from this snapshot"); - Windows.Refresh(app, context); - var selected = app.SelectedWindow ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Select a window with getApp before using element identifiers"); - var snapshot = app.Snapshots.Values.SingleOrDefault(value => value.Id == snapshotId) - ?? throw new NativeFailure("STALE_SNAPSHOT", "Read fresh AX state before using element identifiers"); - if (snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != selected.Id) - throw new NativeFailure("STALE_SNAPSHOT", "Use an observation from the selected window"); + window.Geometry(app, context); + if (!app.Snapshots.TryGetValue(window.Id, out var snapshot) || snapshot.Id != snapshotId + || snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != window.Id) + throw new NativeFailure("STALE_SNAPSHOT", "Read fresh AX state before using element identifiers"); if (!snapshot.Elements.TryGetValue(index, out var element)) throw new NativeFailure("INVALID_ELEMENT", "Use an elementId from this snapshot"); - element.Window.Geometry(app, context); + if (element.Window.Id != window.Id) + throw new NativeFailure("STALE_SNAPSHOT", "The element belongs to another window; read fresh AX state"); if (!context.Ax(() => element.Element.Current.IsEnabled)) throw new NativeFailure("ELEMENT_DISABLED", "The element is disabled"); return element; } internal static void RequireSnapshot(AppState app, string snapshotId, nint windowId) { - var selected = app.SelectedWindow; - var snapshot = app.Snapshots.Values.SingleOrDefault(value => value.Id == snapshotId); - if (snapshot is null || selected is null || selected.Id != windowId + if (!app.Snapshots.TryGetValue(windowId, out var snapshot) || snapshot.Id != snapshotId || snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != windowId) - throw new NativeFailure("STALE_SNAPSHOT", "Use a fresh observation from the selected window"); + throw new NativeFailure("STALE_SNAPSHOT", "Use a fresh observation from this window"); } } diff --git a/packages/experimental/cua-windows/native/entry/Applications.cs b/packages/experimental/cua-windows/native/entry/Applications.cs index bad7a32470..aaf794d9d0 100644 --- a/packages/experimental/cua-windows/native/entry/Applications.cs +++ b/packages/experimental/cua-windows/native/entry/Applications.cs @@ -1,45 +1,48 @@ -// Application consent identifies the executable before any launch or UI observation. +// Process identity, executable metadata, and the authorized application's own window state. using System.Diagnostics; -using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Text; using System.Text.Json; -using Microsoft.Win32; +using System.Text.Json.Serialization; using DeepSeek.Cua.Protocol; namespace DeepSeek.Cua.Windows; -internal sealed record ApplicationIdentity(string Id, string Name, string ExecutablePath, bool Persistable); internal sealed record RunningApplication(int Pid, long Started, string Path, string Name); -internal sealed record InstalledApplication(string Name, string Path); -internal sealed class AppState(string handle, ApplicationIdentity authorization, string digest, RunningApplication? process) : IDisposable +internal sealed record ApplicationAuthorization(string Id, string Name, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ExecutablePath) +{ + /// Not eligible for a permanent grant; the Host still persists conversation grants and denials. + public bool Persistable => false; + + internal static ApplicationAuthorization Create(string appId, string name, string? executablePath) + => new(Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("windows-aumid\0" + appId))), name, executablePath); +} + +internal sealed class AppState(string handle, string appId, string name, ApplicationAuthorization authorization, + bool isRunning) : IDisposable { internal string Handle { get; } = handle; - internal WindowState? SelectedWindow { get; set; } - internal ApplicationIdentity Authorization { get; } = authorization; - internal string Digest { get; } = digest; - internal RunningApplication? Process { get; set; } = process; + internal string AppId { get; } = appId; + internal string Name { get; } = name; + internal bool IsRunning { get; } = isRunning; + internal ApplicationAuthorization Authorization { get; } = authorization; internal Dictionary Windows { get; } = []; internal Dictionary Snapshots { get; } = []; - private System.Diagnostics.Process? owner; - internal RunningApplication RequireProcess() + + public void Dispose() { - var selected = Process ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Application launch requires Host authorization"); - try - { - owner ??= System.Diagnostics.Process.GetProcessById(selected.Pid); - if (owner.HasExited || Applications.Started(owner) != selected.Started) - throw new NativeFailure("STALE_APP", "Select the application again after its process changes"); - } - catch (ArgumentException) { throw new NativeFailure("STALE_APP", "Select the application again after it exits"); } - return selected; + Snapshots.Clear(); + Windows.Clear(); } - public void Dispose() { owner?.Dispose(); Snapshots.Clear(); Windows.Clear(); } } internal static class Applications { + internal static int SessionId { get; } = CurrentSession(); + private static int CurrentSession() { using var process = System.Diagnostics.Process.GetCurrentProcess(); return process.SessionId; } + internal static object Icons(JsonElement parameters, RequestContext context) { var paths = parameters.GetProperty("paths"); @@ -71,14 +74,13 @@ internal static class Applications return result; } - private static readonly int SessionId = CurrentSession(); - private static int CurrentSession() { using var process = System.Diagnostics.Process.GetCurrentProcess(); return process.SessionId; } internal static long Started(System.Diagnostics.Process process) { if (!Win32.GetProcessTimes(process.SafeHandle, out var created, out _, out _, out _)) throw new NativeFailure("STALE_APP", "The process lifetime is unavailable; select the application again"); return created; } + internal static string CanonicalPath(string path) { using var handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); @@ -98,54 +100,6 @@ internal static class Applications return Convert.ToHexStringLower(SHA256.HashData(file)); } - private static string? VerifiedSigner(string path) - { - var file = new Win32.TrustFile { Size = (uint)Marshal.SizeOf(), Path = path }; - var memory = Marshal.AllocHGlobal(Marshal.SizeOf()); - Marshal.StructureToPtr(file, memory, false); - var data = new Win32.TrustData - { - Size = (uint)Marshal.SizeOf(), UiChoice = 2, UnionChoice = 1, - File = memory, StateAction = 1, ProviderFlags = 0x1000 | 0x80, - }; - var action = new Guid("00AAC56B-CD44-11D0-8CC2-00C04FC295EE"); - try - { - if (Win32.WinVerifyTrust(0, ref action, ref data) != 0) return null; - var provider = Win32.WTHelperProvDataFromStateData(data.StateData); - var signerPointer = provider == 0 ? 0 : Win32.WTHelperGetProvSignerFromChain(provider, 0, false, 0); - if (signerPointer == 0) return null; - var signer = Marshal.PtrToStructure(signerPointer); - if (signer.CertificateCount == 0 || signer.Certificates == 0) return null; - var certificate = Marshal.PtrToStructure(signer.Certificates); - if (certificate.Certificate == 0) return null; - var context = Marshal.PtrToStructure(certificate.Certificate); - if (context.Encoded == 0 || context.EncodedSize == 0 || context.EncodedSize > int.MaxValue) return null; - var bytes = new byte[(int)context.EncodedSize]; - Marshal.Copy(context.Encoded, bytes, 0, bytes.Length); - return Convert.ToHexStringLower(SHA256.HashData(bytes)); - } - catch (CryptographicException) { return null; } - finally - { - data.StateAction = 2; - Win32.WinVerifyTrust(0, ref action, ref data); - Marshal.DestroyStructure(memory); - Marshal.FreeHGlobal(memory); - } - } - - internal static (ApplicationIdentity Identity, string Digest) Identify(string path, string name, RunningApplication? process) - { - var canonical = CanonicalPath(path); - var digest = Digest(canonical); - var signer = VerifiedSigner(canonical); - var requirement = signer is not null ? "signed:" + signer - : process is null ? "pending:" + digest : $"process:{process.Pid}:{process.Started}:{digest}"; - var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("win32\0" + canonical + "\0" + requirement))); - return (new(id, name, canonical, signer is not null), digest); - } - internal static RunningApplication Running(int pid) { try @@ -163,123 +117,4 @@ internal static class Applications } catch (ArgumentException) { throw new NativeFailure("STALE_APP", "The selected process has exited"); } } - - internal static IReadOnlyList List() - { - var ids = new HashSet(); - Win32.EnumWindows((window, _) => - { - if (Win32.IsWindowVisible(window)) - { - Win32.GetWindowThreadProcessId(window, out var pid); - if (pid != 0 && pid <= int.MaxValue) ids.Add((int)pid); - } - return true; - }, 0); - var result = new List(); - foreach (var pid in ids.Order()) - { - try { result.Add(Running(pid)); } - catch (NativeFailure) { /* Processes can exit or deny metadata while the desktop is enumerated. */ } - catch (IOException) { /* A disappearing executable is not a selectable application. */ } - } - return result; - } - - internal static IReadOnlyList Find(string query, int maximum, bool exact = false) - { - var found = new Dictionary(StringComparer.Ordinal); - void Add(string name, string path) - { - if (exact ? !string.Equals(name, query, StringComparison.OrdinalIgnoreCase) && !string.Equals(Path.GetFileName(path), query, StringComparison.OrdinalIgnoreCase) - : !name.Contains(query, StringComparison.OrdinalIgnoreCase) && !Path.GetFileName(path).Contains(query, StringComparison.OrdinalIgnoreCase)) return; - if (!Path.IsPathFullyQualified(path) || !path.EndsWith(".exe", StringComparison.OrdinalIgnoreCase) || !File.Exists(path)) return; - found.TryAdd(CanonicalPath(path), new(name, CanonicalPath(path))); - if (found.Count > maximum) throw new NativeFailure("TOO_MANY_APPS", "Use a more specific application name or executable path"); - } - foreach (var app in List()) Add(app.Name, app.Path); - foreach (var hive in new[] { RegistryHive.CurrentUser, RegistryHive.LocalMachine }) - foreach (var view in new[] { RegistryView.Registry64, RegistryView.Registry32 }) - { - using var registry = RegistryKey.OpenBaseKey(hive, view); - using var paths = registry.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths"); - if (paths is null) continue; - foreach (var key in paths.GetSubKeyNames()) - { - using var app = paths.OpenSubKey(key); - if (app?.GetValue(null) is string path) Add(Path.GetFileNameWithoutExtension(key), Environment.ExpandEnvironmentVariables(path.Trim('"'))); - } - } - foreach (var directory in new[] { Environment.SpecialFolder.Programs, Environment.SpecialFolder.CommonPrograms }) - { - var root = Environment.GetFolderPath(directory); - if (!Directory.Exists(root)) continue; - foreach (var link in Directory.EnumerateFiles(root, "*.lnk", new EnumerationOptions - { RecurseSubdirectories = true, IgnoreInaccessible = true, AttributesToSkip = FileAttributes.ReparsePoint })) - { - var name = Path.GetFileNameWithoutExtension(link); - if (!name.Contains(query, StringComparison.OrdinalIgnoreCase)) continue; - var target = ShellBroker.ShortcutTarget(link); - if (target is not null) Add(name, target); - } - } - return found.Values.OrderBy(app => app.Name, StringComparer.OrdinalIgnoreCase).ToArray(); - } - - internal static AppState Select(string selector, IEnumerable retained) - { - RunningApplication? running = null; - string path, name; - if (int.TryParse(selector, out var pid)) - { - running = Running(pid); path = running.Path; name = running.Name; - } - else - { - var absolute = Path.IsPathFullyQualified(selector) ? CanonicalPath(selector) : null; - var matches = List().Where(app => absolute is not null ? string.Equals(app.Path, absolute, StringComparison.Ordinal) : string.Equals(app.Name, selector, StringComparison.OrdinalIgnoreCase) - || string.Equals(Path.GetFileName(app.Path), selector, StringComparison.OrdinalIgnoreCase) - ).ToArray(); - if (matches.Length > 1) throw new NativeFailure("AMBIGUOUS_APP", "Select one of the observed process IDs"); - if (matches.Length == 1) { running = matches[0]; path = running.Path; name = running.Name; } - else if (absolute is not null) { path = absolute; name = Path.GetFileNameWithoutExtension(path); } - else - { - var installed = Find(selector, 1, true).Where(app => string.Equals(app.Name, selector, StringComparison.OrdinalIgnoreCase) - || string.Equals(Path.GetFileName(app.Path), selector, StringComparison.OrdinalIgnoreCase)).ToArray(); - if (installed.Length != 1) throw new NativeFailure(installed.Length == 0 ? "APP_NOT_FOUND" : "AMBIGUOUS_APP", "Use findApps, an executable path, or an observed PID"); - path = installed[0].Path; name = installed[0].Name; - } - } - if (!path.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) throw new NativeFailure("UNSUPPORTED", "Select a Windows application executable"); - if (running is not null) - { - var previous = retained.FirstOrDefault(app => app.Process?.Pid == running.Pid && app.Process.Started == running.Started); - if (previous is not null) return previous; - } - var identity = Identify(path, name, running); - return new(Guid.NewGuid().ToString("N"), identity.Identity, identity.Digest, running); - } - - internal static async Task Launch(AppState app, RequestContext context) - { - if (app.Process is not null) { app.RequireProcess(); return; } - var identity = app.Authorization; - if (Digest(identity.ExecutablePath) != app.Digest) throw new NativeFailure("STALE_APP", "The selected executable changed before launch; select it again"); - context.Check(); - await context.Sta.Invoke(() => ShellBroker.Launch(identity.ExecutablePath), context.Cancellation); - while (true) - { - context.Check(); - var matches = List().Where(candidate => string.Equals(candidate.Path, identity.ExecutablePath, StringComparison.Ordinal)).ToArray(); - if (matches.Length > 1) throw new NativeFailure("AMBIGUOUS_APP", "Several matching processes started; select an observed PID"); - if (matches.Length == 1) - { - if (Digest(matches[0].Path) != app.Digest) throw new NativeFailure("STALE_APP", "The launched executable differs from the approved file"); - app.Process = matches[0]; - return; - } - await Task.Delay(context.Config.PollIntervalMs, context.Cancellation); - } - } } diff --git a/packages/experimental/cua-windows/native/entry/ClipboardPaste.cs b/packages/experimental/cua-windows/native/entry/ClipboardPaste.cs index ad166b5bd2..59a4249752 100644 --- a/packages/experimental/cua-windows/native/entry/ClipboardPaste.cs +++ b/packages/experimental/cua-windows/native/entry/ClipboardPaste.cs @@ -154,15 +154,16 @@ internal sealed class ClipboardPaste finally { Win32.CloseClipboard(); } } - internal async Task Paste(AppState app, string text, string format, int maximumBytes, int readTimeoutMs, RequestContext context) + internal async Task Paste(AppState app, WindowState window, string text, string format, int maximumBytes, int readTimeoutMs, RequestContext context) { - Windows.RequireForeground(app, null, context); - var targetPid = app.RequireProcess().Pid; + Windows.RequireForeground(app, window, context); + // Clipboard readers are matched against this window's real process, not a removal-only app identity. + var targetPid = window.ProcessId; var transaction = await context.Sta.Invoke(control => Prepare(control, targetPid, text, format, maximumBytes), context.Cancellation); try { await context.Sta.Invoke(() => transaction.Begin(), context.Cancellation); - Input.Press(app, "v", ["control"], context); + Input.Press(app, window, "v", ["control"], context); try { await transaction.Consumed.Task.WaitAsync(TimeSpan.FromMilliseconds(readTimeoutMs), context.Cancellation); } catch (TimeoutException) { diff --git a/packages/experimental/cua-windows/native/entry/DesktopExecutor.cs b/packages/experimental/cua-windows/native/entry/DesktopExecutor.cs index 31739fa778..c4db2601c3 100644 --- a/packages/experimental/cua-windows/native/entry/DesktopExecutor.cs +++ b/packages/experimental/cua-windows/native/entry/DesktopExecutor.cs @@ -7,11 +7,10 @@ namespace DeepSeek.Cua.Windows; internal sealed class ClientState { - internal Dictionary Launches { get; } = new(StringComparer.Ordinal); internal Dictionary Apps { get; } = new(StringComparer.Ordinal); internal AppState App(string handle) => Apps.GetValueOrDefault(handle) ?? throw new NativeFailure("STALE_APP", "Select the application again on this connection"); - internal void Clear() { foreach (var app in Apps.Values) app.Dispose(); Apps.Clear(); Launches.Clear(); } + internal void Clear() { foreach (var app in Apps.Values) app.Dispose(); Apps.Clear(); } } internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IAsyncDisposable @@ -33,17 +32,12 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA private static nint? WindowId(JsonElement value) => value.TryGetProperty("windowId", out var id) ? ParseWindowId(id) : null; private static string[] Modifiers(JsonElement value) => value.TryGetProperty("modifiers", out var items) ? items.EnumerateArray().Select(item => item.GetString()!).ToArray() : []; - // Window-scoped operations act only on the window authorized at getApp; the request must name it. + // Window-scoped operations revalidate the explicit target against the authorized exact AUMID. private static WindowState Selected(AppState app, JsonElement parameters, RequestContext context) { if (!parameters.TryGetProperty("windowId", out var raw)) - throw new NativeFailure("UNSUPPORTED", "This operation requires a window target; select a window with getApp and pass its windowId"); - var id = ParseWindowId(raw); - var selected = app.SelectedWindow ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Select a window with getApp before this operation"); - if (selected.Id != id) - throw new NativeFailure("WINDOW_CHANGED", "The request targets another window; select that window with getApp"); - selected.Geometry(app, context); - return selected; + throw new NativeFailure("UNSUPPORTED", "This operation requires a window target; select a window with getWindows and pass its windowId"); + return Windows.Resolve(app, ParseWindowId(raw), context); } private static (WindowState Window, double X, double Y) Coordinate(AppState app, JsonElement parameters, RequestContext context) @@ -88,24 +82,24 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA if (request.Method is "requestPermission" or "openPermissionSettings") throw new NativeFailure("UNSUPPORTED", "Windows desktop access has no macOS permission switches; inspect the Windows session status"); DesktopSession.RequireInteractive(); - if (request.Method == "listApps") return DesktopWindows.ListApps(context); + if (request.Method == "listApps") return await DesktopWindows.ListApps(context); if (request.Method == "findApps") - throw new NativeFailure("UNSUPPORTED", "Windows discovers selectable windows with listApps; installed launch entries are not part of this selection"); + throw new NativeFailure("UNSUPPORTED", "Windows discovers selectable applications with listApps"); if (request.Method == "getApp") { - var selected = DesktopWindows.Select(String(parameters, "selector"), context); + var selected = await DesktopWindows.Select(String(parameters, "selector"), context); state.Apps[selected.Handle] = selected; - return new { appHandle = selected.Handle, pid = 4, name = selected.Authorization.Name, - running = true, authorization = selected.Authorization }; + return new { appHandle = selected.Handle, pid = 4, name = selected.Name, + running = selected.IsRunning, authorization = selected.Authorization }; } if (request.Method == "launchApp") - throw new NativeFailure("UNSUPPORTED", "A selected Windows window cannot be launched; select an observed window from listApps"); + throw new NativeFailure("UNSUPPORTED", "Windows application selection does not launch applications"); var app = state.App(String(parameters, "appHandle")); - app.RequireProcess(); var foreground = Win32.GetForegroundWindow(); switch (request.Method) { - case "listWindows": return Windows.Refresh(app, context).Select(window => window.Describe(app, context)).ToArray(); + case "listWindows": + return Windows.DescribeAll(app, Windows.Refresh(app, context), context).Values; case "getState": { var target = Selected(app, parameters, context); @@ -115,29 +109,34 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA case "activate": Windows.Activate(app, null, context); return ActionResult(foreground); case "show": return Windows.Show(app, WindowId(parameters), context); case "typeText": - Selected(app, parameters, context); - await Input.Type(app, String(parameters, "text"), context); + { + var target = Selected(app, parameters, context); + await Input.Type(app, target, String(parameters, "text"), context); return ActionResult(foreground); + } case "pressKey": - Selected(app, parameters, context); - Input.Press(app, String(parameters, "key"), Modifiers(parameters), context); + { + var target = Selected(app, parameters, context); + Input.Press(app, target, String(parameters, "key"), Modifiers(parameters), context); return ActionResult(foreground); + } case "sendKeyEvent": throw new NativeFailure("UNSUPPORTED", "Held key events are not supported on current platform."); case "paste": - Selected(app, parameters, context); - await clipboard.Paste(app, String(parameters, "text"), OptionalString(parameters, "format") ?? "text", + { + var target = Selected(app, parameters, context); + await clipboard.Paste(app, target, String(parameters, "text"), OptionalString(parameters, "format") ?? "text", parameters.GetProperty("maxClipboardBytes").GetInt32(), parameters.GetProperty("pasteReadTimeoutMs").GetInt32(), context); return ActionResult(foreground); + } case "mouseMove": { + var target = Selected(app, parameters, context); if (parameters.TryGetProperty("elementId", out _)) { - var target = Selected(app, parameters, context); return await mta.Invoke(() => { - var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context); - if (element.Window.Id != target.Id) throw new NativeFailure("WINDOW_CHANGED", "The element belongs to another window"); + var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context); var bounds = context.Ax(() => element.Element.Current.BoundingRectangle); if (bounds.IsEmpty) throw new NativeFailure("UNSUPPORTED", "The element has no pointer bounds"); var geometry = element.Window.RequireObservation(app, context); @@ -183,8 +182,7 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA { await mta.Invoke(() => { - var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context); - if (element.Window.Id != target.Id) throw new NativeFailure("WINDOW_CHANGED", "The element belongs to another window"); + var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context); if (!hold.HasValue && button == "left" && count == 1 && modifiers.Length == 0 && AccessibilityActions.Click(element, context)) return true; var bounds = context.Ax(() => element.Element.Current.BoundingRectangle); if (bounds.IsEmpty) throw new NativeFailure("UNSUPPORTED", "The element has no pointer bounds"); @@ -204,17 +202,17 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA } case "listActions": { - Selected(app, parameters, context); + var target = Selected(app, parameters, context); return await mta.Invoke(() => AccessibilityActions.Actions( - Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation); + Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation); } case "performAction": { - Selected(app, parameters, context); + var target = Selected(app, parameters, context); var argument = parameters.TryGetProperty("argument", out var raw) ? raw : (JsonElement?)null; await mta.Invoke(() => { - var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context); + var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context); AccessibilityActions.Perform(app, element, String(parameters, "action"), argument, context); return true; }, cancellation); @@ -222,17 +220,17 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA } case "invoke": { - Selected(app, parameters, context); + var target = Selected(app, parameters, context); var dispatched = await mta.Invoke(() => AccessibilityActions.Invoke( - Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation); + Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation); return new { dispatched }; } case "setValue": { - Selected(app, parameters, context); + var target = Selected(app, parameters, context); await mta.Invoke(() => { - var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context); + var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context); AccessibilityActions.SetValue(element, parameters.GetProperty("value"), context); return true; }, cancellation); @@ -240,10 +238,10 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA } case "selectText": { - Selected(app, parameters, context); + var target = Selected(app, parameters, context); await mta.Invoke(() => { - var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context); + var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context); AccessibilityActions.SelectText(element, String(parameters, "text"), OptionalString(parameters, "prefix"), OptionalString(parameters, "suffix"), OptionalString(parameters, "selectionType") ?? "text", context); return true; diff --git a/packages/experimental/cua-windows/native/entry/DesktopWindows.cs b/packages/experimental/cua-windows/native/entry/DesktopWindows.cs index 0189021692..06b1dd68c7 100644 --- a/packages/experimental/cua-windows/native/entry/DesktopWindows.cs +++ b/packages/experimental/cua-windows/native/entry/DesktopWindows.cs @@ -1,98 +1,378 @@ -// Desktop discovery selects windows by their invariant decimal HWND strings. -using System.Globalization; -using System.Security.Cryptography; +// AppsFolder AUMID enumeration, exact-ordinal merge, and exact window identity resolution. +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.InteropServices; using System.Text; using DeepSeek.Cua.Protocol; +using Microsoft.Win32.SafeHandles; namespace DeepSeek.Cua.Windows; -internal sealed record DiscoveredWindow(WindowState Window, RunningApplication Process) -{ - internal string Selector => Window.Id.ToString(CultureInfo.InvariantCulture); -} +internal sealed record ShellApplication(string AppId, string DisplayName, bool HasShellName); +internal sealed record ShellCatalog(ShellApplication[] Applications, int Omitted); +internal sealed record ResolvedWindow(nint Handle, string AppId, string? ExecutablePath, string ProcessName, + int ProcessId, long ProcessStarted); +internal sealed record WindowObservation(ResolvedWindow[] Windows, int Denied, int Gone); +internal sealed record CatalogApplication(string AppId, string DisplayName, string? ExecutablePath, bool IsRunning); internal static class DesktopWindows { - internal static object[] ListApps(RequestContext context) - { - var result = new List(); - foreach (var entry in Discover(context)) - { - context.Check(); - try - { - using var owner = Owner(entry, false, context); - entry.Window.Geometry(owner, context); - result.Add(new { pid = 4, name = owner.Authorization.Name, id = entry.Selector }); - } - catch (NativeFailure error) when (error.Code is "STALE_APP" or "PERMISSION_DENIED" or "APP_IDENTITY_UNAVAILABLE" or "UNSUPPORTED_SESSION" or "WINDOW_CHANGED") - { - // A disappearing or inaccessible window is omitted from discovery. - } - } - return result.ToArray(); - } + private const uint ProcessQueryLimitedInformation = 0x1000; + private const int ErrorAccessDenied = 5; + private const int ErrorInvalidParameter = 87; + private const int ErrorInsufficientBuffer = 122; + private const int AppModelErrorNoApplication = 15703; + private const int EAccessDenied = unchecked((int)0x80070005); + private const int EHandle = unchecked((int)0x80070006); + private const int EInvalidArgument = unchecked((int)0x80070057); + private const int EInvalidWindowHandle = unchecked((int)0x80070578); + private const int MaximumIdentityBuffer = 1_048_576; + private static readonly Guid AppUserModelIdFormat = new("9F4C2855-9F79-4B39-A8D0-E1D42DE1D5F3"); - internal static AppState Select(string selector, RequestContext context) + internal static async Task ListApps(RequestContext context) { - if (string.IsNullOrEmpty(selector)) throw new NativeFailure("INVALID_ARGUMENT", "A window selector from listApps is required"); - var entry = Discover(context).SingleOrDefault(value => string.Equals(value.Selector, selector, StringComparison.Ordinal)) - ?? throw new NativeFailure("WINDOW_CHANGED", "Discover applications again; this window is no longer available"); - var app = Owner(entry, true, context); - try { entry.Window.Geometry(app, context); return app; } - catch { app.Dispose(); throw; } - } - - private static List Discover(RequestContext context) - { - var current = new List(); - Win32.EnumWindows((handle, _) => + var applications = await Discover(context); + var result = applications.Select(application => new { - var extended = Win32.GetWindowLongPtr(handle, -20).ToInt64(); - if (Win32.IsWindowVisible(handle) && (extended & 0x80) == 0 - && Win32.GetWindowRect(handle, out var bounds) && bounds.Width > 0 && bounds.Height > 0) - current.Add(handle); - return true; - }, 0); - var result = new List(); - foreach (var handle in current) - { - context.Check(); - try - { - Win32.GetWindowThreadProcessId(handle, out var pid); - var process = Applications.Running(checked((int)pid)); - var entry = new DiscoveredWindow(new(handle), process); - using var owner = Owner(entry, false, context); - entry.Window.Geometry(owner, context); - result.Add(entry); - } - catch (NativeFailure error) when (error.Code is "STALE_APP" or "PERMISSION_DENIED" or "APP_IDENTITY_UNAVAILABLE" or "UNSUPPORTED_SESSION" or "WINDOW_CHANGED") - { - // A disappearing or inaccessible window is omitted from discovery. - } - } + pid = 4, id = application.AppId, name = application.DisplayName, isRunning = application.IsRunning, + }).ToArray(); + context.Check(); return result; } - private static AppState Owner(DiscoveredWindow entry, bool authorize, RequestContext context) + internal static async Task Select(string selector, RequestContext context) { - var window = entry.Window; - var process = entry.Process; - var length = Win32.GetWindowTextLength(window.Handle); - if (length > context.Config.MaxResponseBytes) throw new NativeFailure("NATIVE_RESPONSE_LIMIT", "Window title exceeds the response limit"); - var title = new StringBuilder(length + 1); - Win32.GetWindowText(window.Handle, title, title.Capacity); - var name = $"Window: {title} ({process.Name})"; - // Shared consent IDs require a SHA-256 string, including non-persistable window grants. - var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes($"window\0{entry.Selector}"))); - var identity = new ApplicationIdentity(id, name, process.Path, false); - var app = new AppState(Guid.NewGuid().ToString("N"), identity, "", process); - if (authorize) - { - app.SelectedWindow = window; - app.Windows.Add(window.Handle, window); - } - return app; + var applications = await Discover(context); + var application = SelectApplication(applications, selector); + context.Check(); + return new AppState(Guid.NewGuid().ToString("N"), application.AppId, application.DisplayName, + ApplicationAuthorization.Create(application.AppId, application.DisplayName, application.ExecutablePath), + application.IsRunning); } + + private static async Task Discover(RequestContext context) + { + var catalog = await context.Sta.Invoke(_ => ReadAppsFolder(context), context.Cancellation); + var observation = Observe(context); + if (catalog.Omitted > 0) + Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {catalog.Omitted} AppsFolder items without an application identity"); + if (observation.Denied > 0) + Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {observation.Denied} windows whose process or window access was denied"); + if (observation.Gone > 0) + Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {observation.Gone} disappeared or identity-less windows"); + var applications = Merge(catalog.Applications, observation.Windows); + context.Check(); + return applications; + } + + private static ShellCatalog ReadAppsFolder(RequestContext context) + { + var owned = new List(); + object Own(object value) { owned.Add(value); return value; } + try + { + dynamic shell = Own(Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("13709620-C279-11CE-A49E-444553540000"), true)!)!); + dynamic folder = Own((object)shell.NameSpace("shell:AppsFolder")); + if (folder is null) throw new COMException("AppsFolder is unavailable"); + dynamic items = Own((object)folder.Items()); + var applications = new List(); + var omitted = 0; + for (var index = 0; index < (int)items.Count; index++) + { + context.Check(); + dynamic item = Own((object)items.Item(index)); + // A failed property read is a catalog failure; only an absent or non-string property is missing. + object? property = item.ExtendedProperty("System.AppUserModel.ID"); + object? displayName = item.Name; + string? aumid = property as string; + string? name = displayName as string; + if (string.IsNullOrWhiteSpace(aumid)) { omitted++; continue; } + var hasName = !string.IsNullOrWhiteSpace(name); + applications.Add(new(aumid, hasName ? name! : aumid, hasName)); + } + return new(applications.ToArray(), omitted); + } + catch (Exception error) when (error is COMException or Microsoft.CSharp.RuntimeBinder.RuntimeBinderException) + { + throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows Shell could not enumerate AppsFolder; retry discovery after Explorer is available"); + } + finally + { + var released = new HashSet(ReferenceEqualityComparer.Instance); + foreach (var value in owned.AsEnumerable().Reverse()) + if (released.Add(value) && Marshal.IsComObject(value)) Marshal.FinalReleaseComObject(value); + } + } + + private static CatalogApplication SelectApplication(IReadOnlyList applications, string selector) + { + if (string.IsNullOrEmpty(selector)) + throw new NativeFailure("INVALID_ARGUMENT", "An exact application id from listApps is required"); + foreach (var application in applications) + if (string.Equals(application.AppId, selector, StringComparison.Ordinal)) return application; + throw new NativeFailure("APP_NOT_FOUND", "Use an exact application id from listApps"); + } + + private static CatalogApplication[] Merge(ShellApplication[] shellApplications, ResolvedWindow[] windows) + { + var applications = new Dictionary(StringComparer.Ordinal); + foreach (var shell in shellApplications) + { + if (string.IsNullOrEmpty(shell.AppId)) continue; + if (!applications.TryGetValue(shell.AppId, out var existing)) + applications.Add(shell.AppId, new(shell.AppId, shell.DisplayName, shell.HasShellName, shellNamed: true)); + else existing.PreferShellName(shell.DisplayName, shell.HasShellName); + } + foreach (var window in windows.OrderBy(window => window.AppId, StringComparer.Ordinal) + .ThenBy(window => (long)window.Handle)) + { + if (string.IsNullOrEmpty(window.AppId) || string.IsNullOrEmpty(window.ProcessName)) continue; + if (!applications.TryGetValue(window.AppId, out var builder)) + { + builder = new(window.AppId, window.ProcessName, hasShellName: false, shellNamed: false); + applications.Add(window.AppId, builder); + } + builder.Add(window); + } + return applications.Values.Select(builder => builder.Build()) + .OrderBy(application => application.AppId, StringComparer.Ordinal).ToArray(); + } + + private sealed class Builder(string appId, string displayName, bool hasShellName, bool shellNamed) + { + private readonly HashSet executablePaths = new(StringComparer.Ordinal); + private readonly HashSet processNames = new(StringComparer.Ordinal); + private string shellDisplayName = displayName; + private bool shellNameProvided = hasShellName; + + internal void PreferShellName(string candidate, bool provided) + { + if (provided && !shellNameProvided) { shellDisplayName = candidate; shellNameProvided = true; } + else if (provided == shellNameProvided && string.CompareOrdinal(candidate, shellDisplayName) < 0) + shellDisplayName = candidate; + } + + internal void Add(ResolvedWindow window) + { + if (!string.IsNullOrEmpty(window.ExecutablePath)) executablePaths.Add(window.ExecutablePath); + processNames.Add(window.ProcessName); + } + + internal CatalogApplication Build() + { + var name = shellNamed ? shellDisplayName + : processNames.Count == 0 ? shellDisplayName : processNames.Min(StringComparer.Ordinal)!; + var executablePath = executablePaths.Count == 1 ? executablePaths.First() : null; + return new(appId, name, executablePath, processNames.Count != 0); + } + } + + internal static WindowObservation Observe(RequestContext context) + { + var windows = new List(); + var denied = 0; + var gone = 0; + foreach (var handle in EnumerateHandles(context)) + { + context.Check(); + try + { + var window = ResolveWindow(handle, context); + if (window is null) gone++; + else windows.Add(window); + } + catch (NativeFailure error) when (error.Code == "PERMISSION_DENIED") { denied++; } + } + return new(windows.ToArray(), denied, gone); + } + + internal static bool Eligible(nint window) + { + // EnumWindows yields top-level windows; the explicit Resolve path also checks the root ancestor. + if (Win32.GetAncestor(window, 2) != window) return false; + var extended = Win32.GetWindowLongPtr(window, -20).ToInt64(); + if ((extended & 0x80) != 0) return false; + if (!Win32.IsWindowVisible(window)) return false; + if (!Win32.GetWindowRect(window, out var bounds) || bounds.Width <= 0 || bounds.Height <= 0) return false; + Win32.GetWindowThreadProcessId(window, out var processId); + if (processId == 0 || processId > int.MaxValue) return false; + return Win32.ProcessIdToSessionId(processId, out var session) && session == Applications.SessionId; + } + + private static IReadOnlyList EnumerateHandles(RequestContext context) + { + var handles = new List(); + var enumerated = Win32.EnumWindows((window, _) => + { + // Cancellation cannot throw across the unmanaged callback; stop, then rethrow outside it. + if (context.Cancellation.IsCancellationRequested) return false; + if (Eligible(window)) handles.Add(window); + return true; + }, 0); + context.Check(); + if (!enumerated) throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows could not enumerate top-level windows; retry discovery after the desktop is available"); + return handles; + } + + internal static ResolvedWindow? ResolveWindow(nint handle, RequestContext context) + { + context.Check(); + Win32.GetWindowThreadProcessId(handle, out var rawProcessId); + if (rawProcessId == 0 || rawProcessId > int.MaxValue) return null; + var processId = (int)rawProcessId; + if (!Win32.ProcessIdToSessionId(rawProcessId, out var session) || session != Applications.SessionId) return null; + var process = Win32.OpenProcess(ProcessQueryLimitedInformation, false, rawProcessId); + if (process == 0) return OpenFailure(); + try + { + using var native = new SafeProcessHandle(process, ownsHandle: false); + if (!Win32.GetProcessTimes(native, out var started, out _, out _, out _)) return null; + var processName = ReadProcessName(processId); + if (string.IsNullOrEmpty(processName)) return null; + context.Check(); + var windowAumid = ReadWindowAumid(handle, context); + string? executablePath = null; + string? appId = null; + if (!string.IsNullOrEmpty(windowAumid)) appId = windowAumid; + else + { + appId = ReadProcessAumid(process, context); + if (string.IsNullOrEmpty(appId)) executablePath = ReadExecutablePath(process, context); + appId ??= executablePath; + } + if (appId is null) return null; + // A recycled handle must not inherit the identity resolved for its former process. + Win32.GetWindowThreadProcessId(handle, out var currentProcessId); + if (currentProcessId != rawProcessId) return null; + using var current = new SafeProcessHandle(process, ownsHandle: false); + if (!Win32.GetProcessTimes(current, out var currentStarted, out _, out _, out _) || currentStarted != started) return null; + context.Check(); + return new(handle, appId, executablePath, processName, processId, started); + } + finally { Win32.CloseHandle(process); } + } + + private static ResolvedWindow? OpenFailure() => Marshal.GetLastPInvokeError() switch + { + ErrorAccessDenied => throw PermissionDenied(), + ErrorInvalidParameter => null, + _ => throw Unavailable(), + }; + + private static string? ReadProcessName(int processId) + { + try + { + using var process = Process.GetProcessById(processId); + return process.ProcessName; + } + catch (ArgumentException) { return null; } + catch (InvalidOperationException) { return null; } + catch (Win32Exception error) when (error.NativeErrorCode == ErrorAccessDenied) { throw PermissionDenied(); } + catch (Win32Exception) { throw Unavailable(); } + } + + private static string? ReadWindowAumid(nint handle, RequestContext context) + { + context.Check(); + var read = context.Sta.InvokeSync(_ => ReadWindowAumidCore(handle, context), context.Cancellation); + context.Check(); + return read; + } + + private static string? ReadWindowAumidCore(nint handle, RequestContext context) + { + var id = typeof(Win32.IPropertyStore).GUID; + var status = Win32.SHGetPropertyStoreForWindow(handle, ref id, out var store); + if (status != 0) + { + if (status == EAccessDenied) throw PermissionDenied(); + if (status is EHandle or EInvalidArgument or EInvalidWindowHandle) return null; + throw Unavailable(); + } + if (store is null) throw Unavailable(); + try + { + var key = new Win32.PropertyKey { FormatId = AppUserModelIdFormat, PropertyId = 5 }; + var value = default(Win32.PropVariant); + var result = store.GetValue(ref key, out value); + try + { + if (result != 0) + { + if (result == EAccessDenied) throw PermissionDenied(); + throw Unavailable(); + } + context.Check(); + if (value.ValueType == 0) return null; + if (value.ValueType == 31) return value.Pointer == 0 ? null : Marshal.PtrToStringUni(value.Pointer); + if (value.ValueType == 8) return value.Pointer == 0 ? null : Marshal.PtrToStringBSTR(value.Pointer); + throw Unavailable(); + } + finally { Win32.PropVariantClear(ref value); } + } + finally { Marshal.FinalReleaseComObject(store); } + } + + private static string? ReadProcessAumid(nint process, RequestContext context) + { + context.Check(); + uint length = 0; + var status = Win32.GetApplicationUserModelId(process, ref length, null); + if (status == 0) throw Unavailable(); + if (status != ErrorInsufficientBuffer) return ClassifyAumidStatus(status); + for (var attempt = 0; attempt < 4; attempt++) + { + context.Check(); + if (length == 0 || length > MaximumIdentityBuffer) throw Unavailable(); + var buffer = new StringBuilder(checked((int)length)); + uint size = length; + status = Win32.GetApplicationUserModelId(process, ref size, buffer); + if (status == 0) + { + var value = buffer.ToString(); + if (string.IsNullOrEmpty(value)) throw Unavailable(); + return value; + } + if (status != ErrorInsufficientBuffer) return ClassifyAumidStatus(status); + length = size; + } + throw Unavailable(); + } + + private static string? ClassifyAumidStatus(int status) + { + if (status == AppModelErrorNoApplication) return null; + if (status == ErrorAccessDenied) throw PermissionDenied(); + throw Unavailable(); + } + + private static string? ReadExecutablePath(nint process, RequestContext context) + { + for (var capacity = 1024; capacity <= 65536; capacity *= 2) + { + context.Check(); + var buffer = new StringBuilder(capacity); + uint size = (uint)capacity; + if (Win32.QueryFullProcessImageName(process, 0, buffer, ref size)) + { + context.Check(); + var path = buffer.ToString(0, (int)Math.Min(size, (uint)buffer.Length)); + if (path.Length == 0) throw Unavailable(); + return path; + } + var error = Marshal.GetLastPInvokeError(); + if (error == ErrorInsufficientBuffer) continue; + if (error == ErrorAccessDenied) throw PermissionDenied(); + if (error == ErrorInvalidParameter) return null; + throw Unavailable(); + } + throw Unavailable(); + } + + private static NativeFailure PermissionDenied() + => new("PERMISSION_DENIED", "Windows refused the window application identity; its privilege level may exceed the helper's"); + + private static NativeFailure Unavailable() + => new("APP_IDENTITY_UNAVAILABLE", "The window application identity could not be resolved; retry after its owner is available"); } diff --git a/packages/experimental/cua-windows/native/entry/Dispatchers.cs b/packages/experimental/cua-windows/native/entry/Dispatchers.cs index 758414ed73..14a895eed6 100644 --- a/packages/experimental/cua-windows/native/entry/Dispatchers.cs +++ b/packages/experimental/cua-windows/native/entry/Dispatchers.cs @@ -95,6 +95,7 @@ internal sealed class StaDispatcher : IAsyncDisposable private readonly TaskCompletionSource ready = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource stopped = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly Thread thread; + private ServiceControl? service; internal StaDispatcher() { @@ -105,6 +106,7 @@ internal sealed class StaDispatcher : IAsyncDisposable Win32.SetThreadDpiAwarenessContext(-4); using var control = new ServiceControl(); _ = control.Handle; + service = control; ready.SetResult(control); Forms.Application.Run(); } @@ -118,16 +120,37 @@ internal sealed class StaDispatcher : IAsyncDisposable internal async Task Invoke(Func work, CancellationToken cancellation) { var control = await ready.Task.WaitAsync(cancellation); + cancellation.ThrowIfCancellationRequested(); + // A caller already on the STA must not enqueue work for the message loop it is blocking. + if (Environment.CurrentManagedThreadId == thread.ManagedThreadId) + { + var inline = work(control); + cancellation.ThrowIfCancellationRequested(); + return inline; + } var result = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); control.BeginInvoke(() => { - try { cancellation.ThrowIfCancellationRequested(); var value = work(control); result.TrySetResult(value); } + try { cancellation.ThrowIfCancellationRequested(); var value = work(control); cancellation.ThrowIfCancellationRequested(); result.TrySetResult(value); } catch (OperationCanceledException) { result.TrySetCanceled(cancellation); } catch (Exception error) { result.TrySetException(error); } }); return await result.Task; } + internal T InvokeSync(Func work, CancellationToken cancellation) + { + cancellation.ThrowIfCancellationRequested(); + if (Environment.CurrentManagedThreadId == thread.ManagedThreadId) + { + var control = service ?? throw new InvalidOperationException("The STA dispatcher has not started"); + var inline = work(control); + cancellation.ThrowIfCancellationRequested(); + return inline; + } + return Invoke(work, cancellation).GetAwaiter().GetResult(); + } + internal Task Invoke(Action work, CancellationToken cancellation) => Invoke(_ => { work(); return true; }, cancellation); public async ValueTask DisposeAsync() diff --git a/packages/experimental/cua-windows/native/entry/Input.cs b/packages/experimental/cua-windows/native/entry/Input.cs index ec6570591d..f75d435510 100644 --- a/packages/experimental/cua-windows/native/entry/Input.cs +++ b/packages/experimental/cua-windows/native/entry/Input.cs @@ -46,7 +46,7 @@ internal static class Input finally { foreach (var key in attempted.AsEnumerable().Reverse()) Release(Keyboard(key, flags: 2u | (key == 0x5B ? 1u : 0))); } } - private static (ushort Key, bool Extended, string[] Modifiers) ResolveKey(string name, AppState app) + private static (ushort Key, bool Extended, string[] Modifiers) ResolveKey(string name) { var key = name.ToLowerInvariant(); if (key.StartsWith("kp_", StringComparison.Ordinal)) @@ -85,38 +85,38 @@ internal static class Input return ((ushort)(mapped & 0xFF), false, modifiers.ToArray()); } - internal static void Press(AppState app, string key, string[] modifiers, RequestContext context) + internal static void Press(AppState app, WindowState window, string key, string[] modifiers, RequestContext context) { - Windows.RequireForeground(app, null, context); + Windows.RequireForeground(app, window, context); var chord = new KeyChord(key, modifiers.Select(KeyChord.NormalizeModifier).Distinct().ToArray()); var name = chord.Key.ToLowerInvariant(); if (name == "printscreen" || chord.Modifiers.Contains("option") && name is "tab" or "escape" or "esc" || chord.Modifiers.Contains("control") && name is "escape" or "esc" || chord.Modifiers.Contains("control") && chord.Modifiers.Contains("option") && name == "delete") throw new NativeFailure("UNSUPPORTED", "This shortcut acts on the system rather than the selected application"); - var resolved = ResolveKey(chord.Key, app); + var resolved = ResolveKey(chord.Key); WithModifiers(chord.Modifiers.Concat(resolved.Modifiers), () => { - Windows.RequireForeground(app, null, context); + Windows.RequireForeground(app, window, context); var flags = resolved.Extended ? 1u : 0; try { Send(Keyboard(resolved.Key, flags: flags), Keyboard(resolved.Key, flags: flags | 2)); } catch { Release(Keyboard(resolved.Key, flags: flags | 2)); throw; } }); } - internal static async Task Type(AppState app, string text, RequestContext context) + internal static async Task Type(AppState app, WindowState window, string text, RequestContext context) { if (text.Contains('\0')) throw new NativeFailure("INVALID_ARGUMENT", "Text input cannot contain NUL"); RequireIdleModifiers(); for (var index = 0; index < text.Length; index++) { - Windows.RequireForeground(app, null, context); + Windows.RequireForeground(app, window, context); RequireIdleModifiers(); var character = text[index]; if (character is '\r' or '\n' or '\t') { if (character == '\r' && index + 1 < text.Length && text[index + 1] == '\n') index++; - Press(app, character == '\t' ? "Tab" : "Enter", [], context); + Press(app, window, character == '\t' ? "Tab" : "Enter", [], context); } else { diff --git a/packages/experimental/cua-windows/native/entry/LaunchEntries.cs b/packages/experimental/cua-windows/native/entry/LaunchEntries.cs deleted file mode 100644 index cc97dfd8bf..0000000000 --- a/packages/experimental/cua-windows/native/entry/LaunchEntries.cs +++ /dev/null @@ -1,88 +0,0 @@ -// Shell namespace enumeration and activation run on the existing STA dispatcher. -using System.Runtime.InteropServices; -using System.Security.Cryptography; -using System.Text; -using DeepSeek.Cua.Protocol; - -namespace DeepSeek.Cua.Windows; - -internal sealed record LaunchEntry(string Id, string DisplayName, string Kind, string? Path); -internal sealed class PreparedLaunch(LaunchEntry entry, ApplicationIdentity authorization, string? digest) -{ - internal string Handle { get; } = Guid.NewGuid().ToString("N"); - internal ApplicationIdentity Authorization { get; } = authorization; - private bool dispatched; - - internal void Dispatch() - { - if (dispatched) throw new NativeFailure("LAUNCH_ALREADY_DISPATCHED", "Discover windows before deciding whether another launch is needed"); - if (entry.Kind == "executable") - { - if (Applications.CanonicalPath(entry.Id) != entry.Id || Applications.Digest(entry.Id) != digest) - throw new NativeFailure("STALE_APP", "The approved executable changed; select it again"); - } - else if (!LaunchEntries.All().Any(current => current.Id == entry.Id)) - throw new NativeFailure("STALE_APP", "The Shell entry disappeared; discover launch entries again"); - // Dispatch may succeed even if Explorer subsequently reports a failure. Never replay this selection. - dispatched = true; - ShellBroker.Launch(entry.Id, show: 1); - } -} - -internal static class LaunchEntries -{ - internal static LaunchEntry[] All() - { - var owned = new List(); - object Own(object value) { owned.Add(value); return value; } - try - { - dynamic shell = Own(Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("13709620-C279-11CE-A49E-444553540000"), true)!)!); - dynamic folder = Own((object)shell.NameSpace("shell:AppsFolder")); - dynamic items = Own((object)folder.Items()); - var result = new List(); - for (var index = 0; index < (int)items.Count; index++) - { - dynamic item = Own((object)items.Item(index)); - string parsingName = item.Path; - string name = item.Name; - if (string.IsNullOrWhiteSpace(parsingName) || string.IsNullOrWhiteSpace(name)) continue; - var target = parsingName.StartsWith("::", StringComparison.Ordinal) ? parsingName : "shell:AppsFolder\\" + parsingName; - result.Add(new(target, name, "shell", null)); - } - return result.DistinctBy(entry => entry.Id, StringComparer.OrdinalIgnoreCase).ToArray(); - } - catch (Exception error) when (error is COMException or Microsoft.CSharp.RuntimeBinder.RuntimeBinderException) - { - throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows Shell could not enumerate AppsFolder; retry discovery after Explorer is available"); - } - finally - { - foreach (var item in owned.AsEnumerable().Reverse()) - if (Marshal.IsComObject(item)) Marshal.ReleaseComObject(item); - } - } - - internal static LaunchEntry[] Find(string query, int maximum) - { - var matches = All().Where(entry => entry.DisplayName.Contains(query, StringComparison.OrdinalIgnoreCase) - || entry.Id.Contains(query, StringComparison.OrdinalIgnoreCase)).OrderBy(entry => entry.DisplayName, StringComparer.OrdinalIgnoreCase).ToArray(); - if (matches.Length > maximum) throw new NativeFailure("TOO_MANY_APPS", "Use a more specific launch entry name"); - return matches; - } - - internal static PreparedLaunch Prepare(string target) - { - if (Path.IsPathFullyQualified(target) && target.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) - { - var path = Applications.CanonicalPath(target); - var name = Path.GetFileNameWithoutExtension(path); - var identity = Applications.Identify(path, "Launch: " + name, null); - return new(new(path, name, "executable", path), identity.Identity with { Persistable = false }, identity.Digest); - } - var shell = All().SingleOrDefault(entry => string.Equals(entry.Id, target, StringComparison.OrdinalIgnoreCase)) - ?? throw new NativeFailure("APP_NOT_FOUND", "Use an ID from findApps or a user-specified absolute executable path"); - var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("launch\0" + shell.Id))); - return new(shell, new(id, "Launch: " + shell.DisplayName, shell.Id, false), null); - } -} diff --git a/packages/experimental/cua-windows/native/entry/ShellBroker.cs b/packages/experimental/cua-windows/native/entry/ShellBroker.cs index 291833ac1c..cf3d654344 100644 --- a/packages/experimental/cua-windows/native/entry/ShellBroker.cs +++ b/packages/experimental/cua-windows/native/entry/ShellBroker.cs @@ -1,8 +1,6 @@ // Explorer owns user-launched applications, independently of the helper's managed process Job. using System.Reflection; using System.Runtime.InteropServices; -using System.Runtime.InteropServices.ComTypes; -using System.Text; using DeepSeek.Cua.Protocol; namespace DeepSeek.Cua.Windows; @@ -44,21 +42,6 @@ internal static class ShellBroker } } - internal static string? ShortcutTarget(string path) - { - object? instance = null; - try - { - instance = Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("00021401-0000-0000-C000-000000000046"), true)!); - ((IPersistFile)instance!).Load(path, 0); - var target = new StringBuilder(32768); - ((IShellLink)instance).GetPath(target, target.Capacity, 0, 4); - return target.Length == 0 ? null : Environment.ExpandEnvironmentVariables(target.ToString()); - } - catch (COMException) { return null; } - finally { if (instance is not null) Marshal.FinalReleaseComObject(instance); } - } - [ComImport, Guid("6D5140C1-7436-11CE-8034-00AA006009FA"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] private interface IServiceProvider { @@ -103,9 +86,4 @@ internal static class ShellBroker [PreserveSig] int GetItemObject(uint item, ref Guid id, [MarshalAs(UnmanagedType.Interface)] out object value); } - [ComImport, Guid("000214F9-0000-0000-C000-000000000046"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - private interface IShellLink - { - void GetPath([Out, MarshalAs(UnmanagedType.LPWStr)] StringBuilder path, int capacity, nint findData, uint flags); - } } diff --git a/packages/experimental/cua-windows/native/entry/Win32.cs b/packages/experimental/cua-windows/native/entry/Win32.cs index 128123b667..65ce6f8b58 100644 --- a/packages/experimental/cua-windows/native/entry/Win32.cs +++ b/packages/experimental/cua-windows/native/entry/Win32.cs @@ -92,42 +92,42 @@ internal static class Win32 [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] internal static extern uint GetFinalPathNameByHandle(Microsoft.Win32.SafeHandles.SafeFileHandle file, StringBuilder path, uint length, uint flags); [DllImport("kernel32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool GetProcessTimes(Microsoft.Win32.SafeHandles.SafeProcessHandle process, out long created, out long exited, out long kernel, out long user); - [DllImport("wintrust.dll", CharSet = CharSet.Unicode, ExactSpelling = true)] internal static extern int WinVerifyTrust(nint window, ref Guid action, ref TrustData data); - [DllImport("wintrust.dll")] internal static extern nint WTHelperProvDataFromStateData(nint state); - [DllImport("wintrust.dll")] internal static extern nint WTHelperGetProvSignerFromChain(nint provider, uint signer, [MarshalAs(UnmanagedType.Bool)] bool counterSigner, uint counter); - [StructLayout(LayoutKind.Sequential)] internal struct ProviderSigner + [DllImport("shell32.dll", PreserveSig = true)] + internal static extern int SHGetPropertyStoreForWindow(nint window, ref Guid id, + [MarshalAs(UnmanagedType.Interface)] out IPropertyStore? store); + [DllImport("ole32.dll")] internal static extern int PropVariantClear(ref PropVariant value); + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, ExactSpelling = true)] + internal static extern int GetApplicationUserModelId(nint process, ref uint length, StringBuilder? value); + [DllImport("kernel32.dll", EntryPoint = "QueryFullProcessImageNameW", CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static extern bool QueryFullProcessImageName(nint process, uint flags, StringBuilder name, ref uint length); + [DllImport("kernel32.dll", SetLastError = true)] + internal static extern nint OpenProcess(uint access, [MarshalAs(UnmanagedType.Bool)] bool inherit, uint processId); + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static extern bool CloseHandle(nint handle); + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static extern bool ProcessIdToSessionId(uint processId, out uint sessionId); + + [StructLayout(LayoutKind.Sequential)] internal struct PropertyKey { internal Guid FormatId; internal uint PropertyId; } + // PROPVARIANT is 24 bytes on supported x64/arm64 targets; the trailing field reserves its union. + [StructLayout(LayoutKind.Sequential)] internal struct PropVariant { - internal uint Size, VerifyTimeLow, VerifyTimeHigh, CertificateCount; - internal nint Certificates; - } - [StructLayout(LayoutKind.Sequential)] internal struct ProviderCertificate - { - internal uint Size; - internal nint Certificate; - } - [StructLayout(LayoutKind.Sequential)] internal struct CertificateContext - { - internal uint Encoding; - internal nint Encoded; - internal uint EncodedSize; + internal ushort ValueType; + internal ushort Reserved1, Reserved2, Reserved3; + internal nint Pointer; + internal nint Pointer2; } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] internal struct TrustFile + [ComImport, Guid("886D8EEB-8CF2-4446-8D02-CDBA1DBDCF99"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IPropertyStore { - internal uint Size; - [MarshalAs(UnmanagedType.LPWStr)] internal string Path; - internal nint Handle, Subject; - } - [StructLayout(LayoutKind.Sequential)] internal struct TrustData - { - internal uint Size; - internal nint PolicyCallback, SipClient; - internal uint UiChoice, RevocationChecks, UnionChoice; - internal nint File; - internal uint StateAction; - internal nint StateData, UrlReference; - internal uint ProviderFlags, UiContext; - internal nint SignatureSettings; + [PreserveSig] int GetCount(out uint count); + [PreserveSig] int GetAt(uint index, out PropertyKey key); + [PreserveSig] int GetValue(ref PropertyKey key, out PropVariant value); + [PreserveSig] int SetValue(ref PropertyKey key, ref PropVariant value); + [PreserveSig] int Commit(); } } diff --git a/packages/experimental/cua-windows/native/entry/Windows.cs b/packages/experimental/cua-windows/native/entry/Windows.cs index dbc21a5614..40d0b2b2d5 100644 --- a/packages/experimental/cua-windows/native/entry/Windows.cs +++ b/packages/experimental/cua-windows/native/entry/Windows.cs @@ -1,4 +1,4 @@ -// Window state retains HWND values; JSON exposes them as invariant decimal strings. +// Window state belongs to one exact application identity and revalidates its own process and AUMID. using System.Globalization; using System.Runtime.InteropServices; using System.Text; @@ -12,24 +12,56 @@ internal readonly record struct WindowGeometry(Win32.Rect Pixels, uint Dpi) internal Bounds Bounds => new(Pixels.Left, Pixels.Top, Pixels.Width, Pixels.Height); } -internal sealed class WindowState(nint handle) +internal sealed class WindowState(nint handle, int processId, long processStarted) { internal nint Id => Handle; internal nint Handle { get; } = handle; + internal int ProcessId { get; } = processId; + internal long ProcessStarted { get; } = processStarted; private WindowGeometry? observation; + internal void RequireCurrent(AppState app, RequestContext context) + { + context.Check(); + if (!Win32.IsWindow(Handle)) + throw Remove(app, "The selected window is no longer available; read fresh application state"); + var current = DesktopWindows.ResolveWindow(Handle, context); + if (current is null) + throw Remove(app, "The selected window is no longer available; read fresh application state"); + if (!string.Equals(app.AppId, current.AppId, StringComparison.Ordinal) + || ProcessId != current.ProcessId || ProcessStarted != current.ProcessStarted) + throw Remove(app, "The selected window no longer belongs to the authorized application; read fresh application state"); + } + + // A stale window and its snapshots must not remain selectable after a direct operation detects the change. + private NativeFailure Remove(AppState app, string message) + { + if (app.Windows.TryGetValue(Handle, out var current) && ReferenceEquals(current, this)) + { + app.Windows.Remove(Handle); + app.Snapshots.Remove(Handle); + } + return new NativeFailure("WINDOW_CHANGED", message); + } + internal WindowGeometry Geometry(AppState app, RequestContext context) { context.Check(); - app.RequireProcess(); - if (!Win32.IsWindow(Handle)) - throw new NativeFailure("WINDOW_CHANGED", "The selected window is no longer available; read fresh application state"); + RequireCurrent(app, context); + return TryGeometry(out var geometry) + ? geometry + : throw new NativeFailure("WINDOW_CHANGED", "The selected window geometry is unavailable; read fresh application state"); + } + + private bool TryGeometry(out WindowGeometry geometry) + { + geometry = default; if (Win32.DwmGetWindowAttribute(Handle, 9, out Win32.Rect rectangle, Marshal.SizeOf()) != 0 - && !Win32.GetWindowRect(Handle, out rectangle)) throw new NativeFailure("WINDOW_CHANGED", "The selected window geometry is unavailable"); + && !Win32.GetWindowRect(Handle, out rectangle)) return false; var dpi = Win32.GetDpiForWindow(Handle); - if (dpi == 0 || rectangle.Width <= 0 || rectangle.Height <= 0) - throw new NativeFailure("WINDOW_CHANGED", "The selected window has no usable bounds"); - return new(rectangle, dpi); + if (dpi == 0 || rectangle.Width <= 0 || rectangle.Height <= 0) return false; + geometry = new(rectangle, dpi); + return true; } internal void Observe(WindowGeometry value) => observation = value; @@ -56,58 +88,101 @@ internal sealed class WindowState(nint handle) internal object Describe(AppState app, RequestContext context) { - var geometry = Geometry(app, context); + RequireCurrent(app, context); + // A minimized or cloaked window may lack usable capture geometry; keep listing its association. + var bounds = TryGeometry(out var geometry) ? geometry.Bounds : observation?.Bounds ?? new Bounds(0, 0, 0, 0); var length = Win32.GetWindowTextLength(Handle); if (length > context.Config.MaxResponseBytes) throw new NativeFailure("NATIVE_RESPONSE_LIMIT", "The window title exceeds the response limit"); var title = new StringBuilder(length + 1); Win32.GetWindowText(Handle, title, title.Capacity); return new { - windowId = Id.ToString(CultureInfo.InvariantCulture), pid = 4, appName = app.RequireProcess().Name, - title = title.ToString(), bounds = geometry.Bounds, isOnScreen = IsOnScreen(), onCurrentSpace = (bool?)null, + windowId = Id.ToString(CultureInfo.InvariantCulture), pid = 4, appName = app.Name, + title = title.ToString(), bounds, isOnScreen = IsOnScreen(Handle), onCurrentSpace = (bool?)null, }; } - internal bool IsOnScreen() => Win32.IsWindowVisible(Handle) && !Win32.IsIconic(Handle) - && (Win32.DwmGetWindowAttribute(Handle, 14, out int cloaked, sizeof(int)) != 0 || cloaked == 0); + internal static bool IsOnScreen(nint handle) => Win32.IsWindowVisible(handle) && !Win32.IsIconic(handle) + && (Win32.DwmGetWindowAttribute(handle, 14, out int cloaked, sizeof(int)) != 0 || cloaked == 0); } internal static class Windows { internal static WindowState[] Refresh(AppState app, RequestContext context) { - var process = app.RequireProcess(); - if (app.SelectedWindow is { } selected) { selected.Geometry(app, context); return [selected]; } - var handles = new List(); - Win32.EnumWindows((window, _) => + var observation = DesktopWindows.Observe(context); + var current = new Dictionary(); + var added = new List(); + foreach (var window in observation.Windows) { - Win32.GetWindowThreadProcessId(window, out var pid); - var style = Win32.GetWindowLongPtr(window, -16).ToInt64(); - var extended = Win32.GetWindowLongPtr(window, -20).ToInt64(); - if (pid == process.Pid && (extended & 0x80) == 0 - && (Win32.IsWindowVisible(window) || (style & 0x00C00000) == 0x00C00000) - && Win32.GetWindowRect(window, out var bounds) && bounds.Width > 0 && bounds.Height > 0) handles.Add(window); - return true; - }, 0); - foreach (var pair in app.Windows.ToArray()) - { - if (handles.Contains(pair.Key)) continue; - app.Windows.Remove(pair.Key); - foreach (var scope in app.Snapshots.Where(item => item.Value.Windows.Any(window => window.Id == pair.Value.Id)).Select(item => item.Key).ToArray()) - app.Snapshots.Remove(scope); - } - foreach (var handle in handles) - { - if (!app.Windows.ContainsKey(handle)) + if (!string.Equals(window.AppId, app.AppId, StringComparison.Ordinal)) continue; + if (app.Windows.TryGetValue(window.Handle, out var existing) + && existing.ProcessId == window.ProcessId && existing.ProcessStarted == window.ProcessStarted) + current[window.Handle] = existing; + else { - app.Windows.Add(handle, new(handle)); + var state = new WindowState(window.Handle, window.ProcessId, window.ProcessStarted); + current[window.Handle] = state; + added.Add(state); } } - return handles.Select(handle => app.Windows[handle]).ToArray(); + foreach (var pair in app.Windows.ToArray()) + { + if (current.TryGetValue(pair.Key, out var kept) && ReferenceEquals(kept, pair.Value)) continue; + app.Windows.Remove(pair.Key); + app.Snapshots.Remove(pair.Key); + } + foreach (var window in added) app.Windows[window.Handle] = window; + return current.Values.OrderBy(window => (long)window.Handle).ToArray(); } - internal static WindowState Resolve(AppState app, nint id, RequestContext context) => Refresh(app, context) - .SingleOrDefault(window => window.Id == id) ?? throw new NativeFailure("WINDOW_CHANGED", "Select a current window from getWindows"); + internal static WindowState Resolve(AppState app, nint id, RequestContext context) + { + if (app.Windows.TryGetValue(id, out var known)) + { + // A known target is revalidated in place; a changed or closed one is removed, never switched. + known.Geometry(app, context); + return known; + } + context.Check(); + if (!DesktopWindows.Eligible(id)) + throw new NativeFailure("WINDOW_CHANGED", "Select a current window from getWindows"); + var observed = DesktopWindows.ResolveWindow(id, context); + if (observed is null || !string.Equals(observed.AppId, app.AppId, StringComparison.Ordinal)) + throw new NativeFailure("WINDOW_CHANGED", "The window does not belong to the authorized application; read fresh application state"); + // A newly appeared explicit window may join the application; no default target is inferred. + app.Snapshots.Remove(id); + var window = new WindowState(observed.Handle, observed.ProcessId, observed.ProcessStarted); + app.Windows[id] = window; + return window; + } + + // A sibling window that disappears during enumeration is omitted with its snapshots; a direct + // selected target still fails through Resolve with a structured error. + internal static (object[] Values, WindowState[] Windows) DescribeAll(AppState app, WindowState[] windows, + RequestContext context) + { + var described = new List(); + var surviving = new List(); + foreach (var window in windows) + { + context.Check(); + try + { + described.Add(window.Describe(app, context)); + surviving.Add(window); + } + catch (NativeFailure error) when (error.Code is "WINDOW_CHANGED" or "PERMISSION_DENIED") + { + if (app.Windows.TryGetValue(window.Handle, out var current) && ReferenceEquals(current, window)) + { + app.Windows.Remove(window.Handle); + app.Snapshots.Remove(window.Handle); + } + } + } + return (described.ToArray(), surviving.ToArray()); + } internal static WindowState? Focused(AppState app, WindowState[] windows) { @@ -130,25 +205,24 @@ internal static class Windows internal static WindowState? Default(AppState app, WindowState[] windows) { - var visible = windows.Where(window => window.IsOnScreen()).ToArray(); - var focused = Focused(app, visible); + var onScreen = windows.Where(window => WindowState.IsOnScreen(window.Handle)).ToArray(); + var focused = Focused(app, onScreen); if (focused is not null) return focused; - if (visible.Length == 1) return visible[0]; - if (visible.Length == 0 && windows.Length <= 1) return windows.SingleOrDefault(); + if (onScreen.Length == 1) return onScreen[0]; + if (onScreen.Length == 0 && windows.Length <= 1) return windows.SingleOrDefault(); throw new NativeFailure("AMBIGUOUS_WINDOW", "Select a window explicitly from getWindows"); } internal static object Show(AppState app, nint? id, RequestContext context) { - var target = id is { } selected ? Resolve(app, selected, context) : Default(app, Refresh(app, context)); + var target = id is { } selected + ? Resolve(app, selected, context) + : (Default(app, Refresh(app, context)) ?? throw new NativeFailure("WINDOW_NOT_FOUND", "The application has no window to show")); var before = Win32.GetForegroundWindow(); - if (target is not null) - { - target.Geometry(app, context); - if (!Win32.ShowWindowAsync(target.Handle, 4)) throw new NativeFailure("UNSUPPORTED", "Windows refused background restoration; explicitly activate the application if needed"); - } + target.Geometry(app, context); + if (!Win32.ShowWindowAsync(target.Handle, 4)) throw new NativeFailure("UNSUPPORTED", "Windows refused background restoration; explicitly activate the application if needed"); var after = Win32.GetForegroundWindow(); - return new { dispatched = true, windowId = target?.Id.ToString(CultureInfo.InvariantCulture), + return new { dispatched = true, windowId = target.Id.ToString(CultureInfo.InvariantCulture), foregroundChanged = before == 0 || after == 0 ? (bool?)null : before != after }; } @@ -161,15 +235,14 @@ internal static class Windows throw new NativeFailure("FOREGROUND_DENIED", "Windows refused foreground access; ask the user to focus the target window"); } - internal static void RequireForeground(AppState app, WindowState? target, RequestContext context) + internal static void RequireForeground(AppState app, WindowState target, RequestContext context) { context.Check(); DesktopSession.RequireInteractive(); - target ??= app.SelectedWindow; - target?.Geometry(app, context); + target.Geometry(app, context); var foreground = Win32.GetForegroundWindow(); Win32.GetWindowThreadProcessId(foreground, out var pid); - if (pid != app.RequireProcess().Pid || target is not null && Win32.GetAncestor(foreground, 2) != target.Handle) + if (pid != (uint)target.ProcessId || Win32.GetAncestor(foreground, 2) != target.Handle) throw new NativeFailure("FOREGROUND_REQUIRED", "This input needs the target in front; explain why, explicitly activate or raise it, then observe again"); } } diff --git a/packages/experimental/cua-windows/tests/desktop-tool.spec.ts b/packages/experimental/cua-windows/tests/desktop-tool.spec.ts index 338ef5a9a5..e390c63432 100644 --- a/packages/experimental/cua-windows/tests/desktop-tool.spec.ts +++ b/packages/experimental/cua-windows/tests/desktop-tool.spec.ts @@ -52,13 +52,11 @@ it.skipIf(!helperPath || !selector)('admits real application screenshots through image: (image) => { expect(image.bytes.byteLength).toBeGreaterThan(0) }, }) const discovered = (await api.listApps({ emit: false })).find(app => - app.id === selector || app.displayName.toLowerCase() === selector?.toLowerCase()) - expect(discovered).toBeDefined() - const app = await api.getApp(discovered!.id) + app.id === selector || app.displayName === selector) + const app = await api.getApp(discovered?.id ?? selector!) const target = (await app.getWindows({ emit: false })).find(window => window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0) expect(target).toBeDefined() - expect(target!.windowId).toBe(discovered!.id) const window = await app.getWindow(target!.windowId) expect(emitted).toEqual([]) await window.getAccessibilityTree() diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index 2dae07225f..87a23f06a7 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -65,7 +65,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): }, async listApps(options) { const result = AppsResult.parse(await call('listApps', {})).map(app => ({ - id: app.id ?? (app.bundleId || app.name), displayName: app.name, isRunning: true as const, + id: app.id ?? (app.bundleId || app.name), displayName: app.name, isRunning: app.isRunning, })) await emit(sink, result, options) return result diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index b72e418b2b..cdb45981f3 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -10,10 +10,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true }[] — enumerate running applications. +- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> { id: string, displayName: string, path: string, isRunning: boolean }[] — search installed applications, including stopped matches. -- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true }[], browsers: [] } — summarize running applications; this is not an accessibility observation. -- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app. +- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation. +- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app. - app.getWindows(options?: { emit?: boolean }) -> { windowId: string, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership. - app.getWindow(windowId: string) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input. - app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required. diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 43ca13308e..662c4c92d5 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -273,6 +273,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) }).parse(result) await access.select(agent, client, prepared.appHandle, authorization, signal, applicationIcon) signal.throwIfAborted() + // TODO: launchApp on Windows const selected = prepared.running ? prepared : AppResult.parse(await access.run(client, prepared.appHandle, signal, diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 9b3303a787..a6d1a80798 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -113,8 +113,10 @@ export const AppResult = z.object({ appHandle: z.string().min(1).transform(value => value as CuaAppHandle), name: z.string(), bundleId: z.string().optional(), running: z.boolean(), }) -/** Running application metadata used for selection; other native fields are ignored. */ -export const AppsResult = z.array(z.object({ id: z.string().min(1).optional(), name: z.string(), bundleId: z.string().optional() })) +/** Application discovery entries with the provider-reported running state. */ +export const AppsResult = z.array(z.object({ + id: z.string().min(1).optional(), name: z.string(), bundleId: z.string().optional(), isRunning: z.boolean(), +})) /** Installed application matches and their current running state. */ export const FoundAppsResult = z.array(z.object({ id: z.string().min(1), displayName: z.string(), path: z.string().min(1), isRunning: z.boolean(), diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index d665c663ec..be79121ff6 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -50,8 +50,8 @@ export type CuaStartupCapabilities = Pick< CuaCapabilities, 'platform' | 'methods' | 'appLevelAccessibility' | 'keyboard' | 'browser' | 'coordinateSpace' > & { systemPointer?: boolean } -/** App discovery result accepted by getApp. */ -export type CuaAppInfo = Pick & { isRunning: true } +/** App discovery result accepted by getApp, including the provider-reported running state. */ +export type CuaAppInfo = Pick /** An installed application match with its current running state. */ export type CuaFoundAppInfo = z.output[number] /** Window bounds in the provider's global native units. Local operations use an origin of [0,0]. */ @@ -256,9 +256,9 @@ export interface CuaApi { */ getCapabilities(options?: CuaObservationOptions): Promise /** - * Enumerate running applications. + * List applications with their current running state. Providers may enumerate only running applications. * @param options - output options. - * @returns identifiers accepted by getApp. + * @returns identifiers accepted by getApp and each application's running state. */ listApps(options?: CuaObservationOptions): Promise /** @@ -269,14 +269,14 @@ export interface CuaApi { */ findApps(query: string, options?: CuaObservationOptions): Promise /** - * Discover apps. + * Summarize provider application discovery. * @param options - output options. - * @returns native apps and an empty browser list. + * @returns native apps with their running state and an empty browser list. */ getState(options?: CuaObservationOptions): Promise<{ apps: CuaAppInfo[]; browsers: [] }> /** - * Select an app, obtain consent, launch it in the background if necessary, and silently retain initial AX state. - * @param selector - Provider-issued identifier, exact name, bundle ID or absolute application path; ambiguous selectors reject. + * Select an app, obtain consent, and launch it only where the provider supports launch; selection never activates it. + * @param selector - Provider-issued ID, or a supported exact name, bundle ID or absolute path; ambiguous selectors reject. * @returns An application handle tied to this process lifetime; its initial tree read becomes the current snapshot. */ getApp(selector: string): Promise diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 00077068e2..800cea9736 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -11,7 +11,7 @@ const element = (id: string): CuaElementId => brandString(id) const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC' const bounds = { x: -640, y: 100, width: 640, height: 480 } -const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', running: true } +const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', isRunning: true, running: true } const delivered = { dispatched: true } const capabilities: CuaCapabilities = { platform: 'macos', systemPointer: true, sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false } @@ -27,7 +27,7 @@ function fixture(withSink = true) { const dispatch = vi.fn(async ({ method, params }: CuaOperation): Promise => { switch (method) { case 'getCapabilities': return currentCapabilities - case 'listApps': return [selected, { id: 'utility-selector', pid: 4, name: 'Utility' }] + case 'listApps': return [selected, { id: 'utility-selector', pid: 4, name: 'Utility', isRunning: true }] case 'getApp': return selected case 'listWindows': return windows case 'mouseMove': return { ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false } @@ -81,8 +81,8 @@ describe('application discovery', () => { it('uses the application name when discovery has no identifier', async () => { const f = fixture() - f.dispatch.mockResolvedValueOnce([{ pid: 18, name: 'Utility' }]) - expect(await f.api.listApps({ emit: false })).toEqual([{ id: 'Utility', displayName: 'Utility', isRunning: true }]) + f.dispatch.mockResolvedValueOnce([{ pid: 18, name: 'Utility', isRunning: false }]) + expect(await f.api.listApps({ emit: false })).toEqual([{ id: 'Utility', displayName: 'Utility', isRunning: false }]) }) it('returns observations without requiring an output sink', async () => { diff --git a/packages/experimental/cua/tests/bridge.spec.ts b/packages/experimental/cua/tests/bridge.spec.ts index 8f7a1e5da4..44fc33e971 100644 --- a/packages/experimental/cua/tests/bridge.spec.ts +++ b/packages/experimental/cua/tests/bridge.spec.ts @@ -56,7 +56,7 @@ it('propagates a denied action without retrying it', async () => { }) it('supports callers that retain observations without a text callback', async () => { - const cua = connectCua(() => async () => [{ id: 'editor-selector', name: 'Editor' }]) + const cua = connectCua(() => async () => [{ id: 'editor-selector', name: 'Editor', isRunning: true }]) await expect(cua.listApps()).resolves.toEqual([{ id: 'editor-selector', displayName: 'Editor', isRunning: true }]) }) diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 9589c20d8e..83ac0b31e7 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -43,10 +43,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. +- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. -- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. -- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app. +- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation. +- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app. - app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership. - app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input. - app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required. diff --git a/snapshots/session/cua-native/workspace/native-helper.mjs b/snapshots/session/cua-native/workspace/native-helper.mjs index 9895510aba..ad8369c110 100755 --- a/snapshots/session/cua-native/workspace/native-helper.mjs +++ b/snapshots/session/cua-native/workspace/native-helper.mjs @@ -131,7 +131,7 @@ function createApplicationState(socket, record, application, captureBroker, conn return { platform: 'darwin', ...state, helper: { pid: process.pid }, parent: { pid: process.ppid } } } if (method === 'getCapabilities') return { sessionLocked: false, permissions: { accessibility: true, screenRecording: true } } - if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId }] + if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId, isRunning: true }] if (method === 'getApp') { if (params.selector === 'Denied Fixture' || params.selector === 'Revocable Fixture') { const denied = params.selector === 'Denied Fixture' diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 6888ad0481..aa5e6045d6 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -45,10 +45,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. +- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. -- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. -- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app. +- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation. +- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app. - app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership. - app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input. - app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index c670f443b9..fa1d25770f 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -45,10 +45,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. +- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. -- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. -- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app. +- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation. +- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app. - app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership. - app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input. - app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.