feat(cua-windows): select applications by AUMID
This commit is contained in:
1 parent
a0f326da1d
commit
d4785e69c0
24 files changed
+676
-574
No files matched your search
@@ -73,7 +73,7 @@ it('inlines discovery, restoration, screenshot, and mouse-move return records',
|
||||
expect(lines.find(line => line.startsWith(`- ${name}(`))).toMatch(/ -> \(?\{ /)
|
||||
}
|
||||
expect(COMMON_GUIDANCE).toContain("{ platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false }")
|
||||
expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, isRunning: true }[]')
|
||||
expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, isRunning: boolean }[]')
|
||||
expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, path: string, isRunning: boolean }[]')
|
||||
expect(COMMON_GUIDANCE).toContain('bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null')
|
||||
})
|
||||
|
||||
@@ -29,8 +29,9 @@ internal static class Accessibility
|
||||
bool includeMenus, bool details, RequestContext context)
|
||||
{
|
||||
var all = Windows.Refresh(app, context);
|
||||
var targets = new[] { all.SingleOrDefault(window => window.Id == selected)
|
||||
?? throw new NativeFailure("WINDOW_CHANGED", "Select a current window") };
|
||||
var target = Windows.Resolve(app, selected, context);
|
||||
if (!all.Any(window => window.Id == target.Id)) all = [.. all, target];
|
||||
var targets = new[] { target };
|
||||
var rows = new List<string> { "AX scope=window offsets=UTF-16" };
|
||||
var elements = new Dictionary<int, ElementReference>();
|
||||
var observations = new List<(WindowState Window, WindowGeometry Geometry)>();
|
||||
@@ -64,16 +65,18 @@ internal static class Accessibility
|
||||
}
|
||||
if (truncated) Add($"[truncated: maxElements={maximumElements}, maxDepth={maximumDepth}]");
|
||||
DesktopSession.RequireInteractive();
|
||||
var scope = selected;
|
||||
var snapshot = new Snapshot(Guid.NewGuid().ToString("N"), scope, targets, elements);
|
||||
var descriptions = Windows.DescribeAll(app, all, context);
|
||||
if (!descriptions.Windows.Any(window => window.Id == selected))
|
||||
throw new NativeFailure("WINDOW_CHANGED", "The selected window disappeared during observation; read fresh application state");
|
||||
var snapshot = new Snapshot(Guid.NewGuid().ToString("N"), selected, targets, elements);
|
||||
var result = new
|
||||
{
|
||||
appHandle = app.Handle, pid = 4, windowId = selected.ToString(CultureInfo.InvariantCulture),
|
||||
focusedWindowId = Windows.Focused(app, all)?.Id.ToString(CultureInfo.InvariantCulture),
|
||||
snapshotId = snapshot.Id, tree = string.Join('\n', rows), windows = all.Select(window => window.Describe(app, context)).ToArray(), truncated,
|
||||
focusedWindowId = Windows.Focused(app, descriptions.Windows)?.Id.ToString(CultureInfo.InvariantCulture),
|
||||
snapshotId = snapshot.Id, tree = string.Join('\n', rows), windows = descriptions.Values, truncated,
|
||||
};
|
||||
context.Check();
|
||||
app.Snapshots[scope] = snapshot;
|
||||
app.Snapshots[selected] = snapshot;
|
||||
foreach (var observation in observations) observation.Window.Observe(observation.Geometry);
|
||||
return result;
|
||||
|
||||
@@ -218,28 +221,26 @@ internal static class Accessibility
|
||||
return "AXGroup";
|
||||
}
|
||||
|
||||
internal static ElementReference Element(AppState app, string snapshotId, string elementId, RequestContext context)
|
||||
internal static ElementReference Element(AppState app, WindowState window, string snapshotId, string elementId,
|
||||
RequestContext context)
|
||||
{
|
||||
if (!int.TryParse(elementId, NumberStyles.None, CultureInfo.InvariantCulture, out var index))
|
||||
throw new NativeFailure("INVALID_ELEMENT", "Use an elementId from this snapshot");
|
||||
Windows.Refresh(app, context);
|
||||
var selected = app.SelectedWindow ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Select a window with getApp before using element identifiers");
|
||||
var snapshot = app.Snapshots.Values.SingleOrDefault(value => value.Id == snapshotId)
|
||||
?? throw new NativeFailure("STALE_SNAPSHOT", "Read fresh AX state before using element identifiers");
|
||||
if (snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != selected.Id)
|
||||
throw new NativeFailure("STALE_SNAPSHOT", "Use an observation from the selected window");
|
||||
window.Geometry(app, context);
|
||||
if (!app.Snapshots.TryGetValue(window.Id, out var snapshot) || snapshot.Id != snapshotId
|
||||
|| snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != window.Id)
|
||||
throw new NativeFailure("STALE_SNAPSHOT", "Read fresh AX state before using element identifiers");
|
||||
if (!snapshot.Elements.TryGetValue(index, out var element)) throw new NativeFailure("INVALID_ELEMENT", "Use an elementId from this snapshot");
|
||||
element.Window.Geometry(app, context);
|
||||
if (element.Window.Id != window.Id)
|
||||
throw new NativeFailure("STALE_SNAPSHOT", "The element belongs to another window; read fresh AX state");
|
||||
if (!context.Ax(() => element.Element.Current.IsEnabled)) throw new NativeFailure("ELEMENT_DISABLED", "The element is disabled");
|
||||
return element;
|
||||
}
|
||||
|
||||
internal static void RequireSnapshot(AppState app, string snapshotId, nint windowId)
|
||||
{
|
||||
var selected = app.SelectedWindow;
|
||||
var snapshot = app.Snapshots.Values.SingleOrDefault(value => value.Id == snapshotId);
|
||||
if (snapshot is null || selected is null || selected.Id != windowId
|
||||
if (!app.Snapshots.TryGetValue(windowId, out var snapshot) || snapshot.Id != snapshotId
|
||||
|| snapshot.Windows.Length != 1 || snapshot.Windows[0].Id != windowId)
|
||||
throw new NativeFailure("STALE_SNAPSHOT", "Use a fresh observation from the selected window");
|
||||
throw new NativeFailure("STALE_SNAPSHOT", "Use a fresh observation from this window");
|
||||
}
|
||||
}
|
||||
@@ -1,45 +1,48 @@
|
||||
// Application consent identifies the executable before any launch or UI observation.
|
||||
// Process identity, executable metadata, and the authorized application's own window state.
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Win32;
|
||||
using System.Text.Json.Serialization;
|
||||
using DeepSeek.Cua.Protocol;
|
||||
|
||||
namespace DeepSeek.Cua.Windows;
|
||||
|
||||
internal sealed record ApplicationIdentity(string Id, string Name, string ExecutablePath, bool Persistable);
|
||||
internal sealed record RunningApplication(int Pid, long Started, string Path, string Name);
|
||||
internal sealed record InstalledApplication(string Name, string Path);
|
||||
|
||||
internal sealed class AppState(string handle, ApplicationIdentity authorization, string digest, RunningApplication? process) : IDisposable
|
||||
internal sealed record ApplicationAuthorization(string Id, string Name,
|
||||
[property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ExecutablePath)
|
||||
{
|
||||
/// <summary>Not eligible for a permanent grant; the Host still persists conversation grants and denials.</summary>
|
||||
public bool Persistable => false;
|
||||
|
||||
internal static ApplicationAuthorization Create(string appId, string name, string? executablePath)
|
||||
=> new(Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("windows-aumid\0" + appId))), name, executablePath);
|
||||
}
|
||||
|
||||
internal sealed class AppState(string handle, string appId, string name, ApplicationAuthorization authorization,
|
||||
bool isRunning) : IDisposable
|
||||
{
|
||||
internal string Handle { get; } = handle;
|
||||
internal WindowState? SelectedWindow { get; set; }
|
||||
internal ApplicationIdentity Authorization { get; } = authorization;
|
||||
internal string Digest { get; } = digest;
|
||||
internal RunningApplication? Process { get; set; } = process;
|
||||
internal string AppId { get; } = appId;
|
||||
internal string Name { get; } = name;
|
||||
internal bool IsRunning { get; } = isRunning;
|
||||
internal ApplicationAuthorization Authorization { get; } = authorization;
|
||||
internal Dictionary<nint, WindowState> Windows { get; } = [];
|
||||
internal Dictionary<nint, Snapshot> Snapshots { get; } = [];
|
||||
private System.Diagnostics.Process? owner;
|
||||
internal RunningApplication RequireProcess()
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
var selected = Process ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Application launch requires Host authorization");
|
||||
try
|
||||
{
|
||||
owner ??= System.Diagnostics.Process.GetProcessById(selected.Pid);
|
||||
if (owner.HasExited || Applications.Started(owner) != selected.Started)
|
||||
throw new NativeFailure("STALE_APP", "Select the application again after its process changes");
|
||||
}
|
||||
catch (ArgumentException) { throw new NativeFailure("STALE_APP", "Select the application again after it exits"); }
|
||||
return selected;
|
||||
Snapshots.Clear();
|
||||
Windows.Clear();
|
||||
}
|
||||
public void Dispose() { owner?.Dispose(); Snapshots.Clear(); Windows.Clear(); }
|
||||
}
|
||||
|
||||
internal static class Applications
|
||||
{
|
||||
internal static int SessionId { get; } = CurrentSession();
|
||||
private static int CurrentSession() { using var process = System.Diagnostics.Process.GetCurrentProcess(); return process.SessionId; }
|
||||
|
||||
internal static object Icons(JsonElement parameters, RequestContext context)
|
||||
{
|
||||
var paths = parameters.GetProperty("paths");
|
||||
@@ -71,14 +74,13 @@ internal static class Applications
|
||||
return result;
|
||||
}
|
||||
|
||||
private static readonly int SessionId = CurrentSession();
|
||||
private static int CurrentSession() { using var process = System.Diagnostics.Process.GetCurrentProcess(); return process.SessionId; }
|
||||
internal static long Started(System.Diagnostics.Process process)
|
||||
{
|
||||
if (!Win32.GetProcessTimes(process.SafeHandle, out var created, out _, out _, out _))
|
||||
throw new NativeFailure("STALE_APP", "The process lifetime is unavailable; select the application again");
|
||||
return created;
|
||||
}
|
||||
|
||||
internal static string CanonicalPath(string path)
|
||||
{
|
||||
using var handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
|
||||
@@ -98,54 +100,6 @@ internal static class Applications
|
||||
return Convert.ToHexStringLower(SHA256.HashData(file));
|
||||
}
|
||||
|
||||
private static string? VerifiedSigner(string path)
|
||||
{
|
||||
var file = new Win32.TrustFile { Size = (uint)Marshal.SizeOf<Win32.TrustFile>(), Path = path };
|
||||
var memory = Marshal.AllocHGlobal(Marshal.SizeOf<Win32.TrustFile>());
|
||||
Marshal.StructureToPtr(file, memory, false);
|
||||
var data = new Win32.TrustData
|
||||
{
|
||||
Size = (uint)Marshal.SizeOf<Win32.TrustData>(), UiChoice = 2, UnionChoice = 1,
|
||||
File = memory, StateAction = 1, ProviderFlags = 0x1000 | 0x80,
|
||||
};
|
||||
var action = new Guid("00AAC56B-CD44-11D0-8CC2-00C04FC295EE");
|
||||
try
|
||||
{
|
||||
if (Win32.WinVerifyTrust(0, ref action, ref data) != 0) return null;
|
||||
var provider = Win32.WTHelperProvDataFromStateData(data.StateData);
|
||||
var signerPointer = provider == 0 ? 0 : Win32.WTHelperGetProvSignerFromChain(provider, 0, false, 0);
|
||||
if (signerPointer == 0) return null;
|
||||
var signer = Marshal.PtrToStructure<Win32.ProviderSigner>(signerPointer);
|
||||
if (signer.CertificateCount == 0 || signer.Certificates == 0) return null;
|
||||
var certificate = Marshal.PtrToStructure<Win32.ProviderCertificate>(signer.Certificates);
|
||||
if (certificate.Certificate == 0) return null;
|
||||
var context = Marshal.PtrToStructure<Win32.CertificateContext>(certificate.Certificate);
|
||||
if (context.Encoded == 0 || context.EncodedSize == 0 || context.EncodedSize > int.MaxValue) return null;
|
||||
var bytes = new byte[(int)context.EncodedSize];
|
||||
Marshal.Copy(context.Encoded, bytes, 0, bytes.Length);
|
||||
return Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
}
|
||||
catch (CryptographicException) { return null; }
|
||||
finally
|
||||
{
|
||||
data.StateAction = 2;
|
||||
Win32.WinVerifyTrust(0, ref action, ref data);
|
||||
Marshal.DestroyStructure<Win32.TrustFile>(memory);
|
||||
Marshal.FreeHGlobal(memory);
|
||||
}
|
||||
}
|
||||
|
||||
internal static (ApplicationIdentity Identity, string Digest) Identify(string path, string name, RunningApplication? process)
|
||||
{
|
||||
var canonical = CanonicalPath(path);
|
||||
var digest = Digest(canonical);
|
||||
var signer = VerifiedSigner(canonical);
|
||||
var requirement = signer is not null ? "signed:" + signer
|
||||
: process is null ? "pending:" + digest : $"process:{process.Pid}:{process.Started}:{digest}";
|
||||
var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("win32\0" + canonical + "\0" + requirement)));
|
||||
return (new(id, name, canonical, signer is not null), digest);
|
||||
}
|
||||
|
||||
internal static RunningApplication Running(int pid)
|
||||
{
|
||||
try
|
||||
@@ -163,123 +117,4 @@ internal static class Applications
|
||||
}
|
||||
catch (ArgumentException) { throw new NativeFailure("STALE_APP", "The selected process has exited"); }
|
||||
}
|
||||
|
||||
internal static IReadOnlyList<RunningApplication> List()
|
||||
{
|
||||
var ids = new HashSet<int>();
|
||||
Win32.EnumWindows((window, _) =>
|
||||
{
|
||||
if (Win32.IsWindowVisible(window))
|
||||
{
|
||||
Win32.GetWindowThreadProcessId(window, out var pid);
|
||||
if (pid != 0 && pid <= int.MaxValue) ids.Add((int)pid);
|
||||
}
|
||||
return true;
|
||||
}, 0);
|
||||
var result = new List<RunningApplication>();
|
||||
foreach (var pid in ids.Order())
|
||||
{
|
||||
try { result.Add(Running(pid)); }
|
||||
catch (NativeFailure) { /* Processes can exit or deny metadata while the desktop is enumerated. */ }
|
||||
catch (IOException) { /* A disappearing executable is not a selectable application. */ }
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
internal static IReadOnlyList<InstalledApplication> Find(string query, int maximum, bool exact = false)
|
||||
{
|
||||
var found = new Dictionary<string, InstalledApplication>(StringComparer.Ordinal);
|
||||
void Add(string name, string path)
|
||||
{
|
||||
if (exact ? !string.Equals(name, query, StringComparison.OrdinalIgnoreCase) && !string.Equals(Path.GetFileName(path), query, StringComparison.OrdinalIgnoreCase)
|
||||
: !name.Contains(query, StringComparison.OrdinalIgnoreCase) && !Path.GetFileName(path).Contains(query, StringComparison.OrdinalIgnoreCase)) return;
|
||||
if (!Path.IsPathFullyQualified(path) || !path.EndsWith(".exe", StringComparison.OrdinalIgnoreCase) || !File.Exists(path)) return;
|
||||
found.TryAdd(CanonicalPath(path), new(name, CanonicalPath(path)));
|
||||
if (found.Count > maximum) throw new NativeFailure("TOO_MANY_APPS", "Use a more specific application name or executable path");
|
||||
}
|
||||
foreach (var app in List()) Add(app.Name, app.Path);
|
||||
foreach (var hive in new[] { RegistryHive.CurrentUser, RegistryHive.LocalMachine })
|
||||
foreach (var view in new[] { RegistryView.Registry64, RegistryView.Registry32 })
|
||||
{
|
||||
using var registry = RegistryKey.OpenBaseKey(hive, view);
|
||||
using var paths = registry.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths");
|
||||
if (paths is null) continue;
|
||||
foreach (var key in paths.GetSubKeyNames())
|
||||
{
|
||||
using var app = paths.OpenSubKey(key);
|
||||
if (app?.GetValue(null) is string path) Add(Path.GetFileNameWithoutExtension(key), Environment.ExpandEnvironmentVariables(path.Trim('"')));
|
||||
}
|
||||
}
|
||||
foreach (var directory in new[] { Environment.SpecialFolder.Programs, Environment.SpecialFolder.CommonPrograms })
|
||||
{
|
||||
var root = Environment.GetFolderPath(directory);
|
||||
if (!Directory.Exists(root)) continue;
|
||||
foreach (var link in Directory.EnumerateFiles(root, "*.lnk", new EnumerationOptions
|
||||
{ RecurseSubdirectories = true, IgnoreInaccessible = true, AttributesToSkip = FileAttributes.ReparsePoint }))
|
||||
{
|
||||
var name = Path.GetFileNameWithoutExtension(link);
|
||||
if (!name.Contains(query, StringComparison.OrdinalIgnoreCase)) continue;
|
||||
var target = ShellBroker.ShortcutTarget(link);
|
||||
if (target is not null) Add(name, target);
|
||||
}
|
||||
}
|
||||
return found.Values.OrderBy(app => app.Name, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
|
||||
internal static AppState Select(string selector, IEnumerable<AppState> retained)
|
||||
{
|
||||
RunningApplication? running = null;
|
||||
string path, name;
|
||||
if (int.TryParse(selector, out var pid))
|
||||
{
|
||||
running = Running(pid); path = running.Path; name = running.Name;
|
||||
}
|
||||
else
|
||||
{
|
||||
var absolute = Path.IsPathFullyQualified(selector) ? CanonicalPath(selector) : null;
|
||||
var matches = List().Where(app => absolute is not null ? string.Equals(app.Path, absolute, StringComparison.Ordinal) : string.Equals(app.Name, selector, StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(Path.GetFileName(app.Path), selector, StringComparison.OrdinalIgnoreCase)
|
||||
).ToArray();
|
||||
if (matches.Length > 1) throw new NativeFailure("AMBIGUOUS_APP", "Select one of the observed process IDs");
|
||||
if (matches.Length == 1) { running = matches[0]; path = running.Path; name = running.Name; }
|
||||
else if (absolute is not null) { path = absolute; name = Path.GetFileNameWithoutExtension(path); }
|
||||
else
|
||||
{
|
||||
var installed = Find(selector, 1, true).Where(app => string.Equals(app.Name, selector, StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(Path.GetFileName(app.Path), selector, StringComparison.OrdinalIgnoreCase)).ToArray();
|
||||
if (installed.Length != 1) throw new NativeFailure(installed.Length == 0 ? "APP_NOT_FOUND" : "AMBIGUOUS_APP", "Use findApps, an executable path, or an observed PID");
|
||||
path = installed[0].Path; name = installed[0].Name;
|
||||
}
|
||||
}
|
||||
if (!path.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) throw new NativeFailure("UNSUPPORTED", "Select a Windows application executable");
|
||||
if (running is not null)
|
||||
{
|
||||
var previous = retained.FirstOrDefault(app => app.Process?.Pid == running.Pid && app.Process.Started == running.Started);
|
||||
if (previous is not null) return previous;
|
||||
}
|
||||
var identity = Identify(path, name, running);
|
||||
return new(Guid.NewGuid().ToString("N"), identity.Identity, identity.Digest, running);
|
||||
}
|
||||
|
||||
internal static async Task Launch(AppState app, RequestContext context)
|
||||
{
|
||||
if (app.Process is not null) { app.RequireProcess(); return; }
|
||||
var identity = app.Authorization;
|
||||
if (Digest(identity.ExecutablePath) != app.Digest) throw new NativeFailure("STALE_APP", "The selected executable changed before launch; select it again");
|
||||
context.Check();
|
||||
await context.Sta.Invoke(() => ShellBroker.Launch(identity.ExecutablePath), context.Cancellation);
|
||||
while (true)
|
||||
{
|
||||
context.Check();
|
||||
var matches = List().Where(candidate => string.Equals(candidate.Path, identity.ExecutablePath, StringComparison.Ordinal)).ToArray();
|
||||
if (matches.Length > 1) throw new NativeFailure("AMBIGUOUS_APP", "Several matching processes started; select an observed PID");
|
||||
if (matches.Length == 1)
|
||||
{
|
||||
if (Digest(matches[0].Path) != app.Digest) throw new NativeFailure("STALE_APP", "The launched executable differs from the approved file");
|
||||
app.Process = matches[0];
|
||||
return;
|
||||
}
|
||||
await Task.Delay(context.Config.PollIntervalMs, context.Cancellation);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -154,15 +154,16 @@ internal sealed class ClipboardPaste
|
||||
finally { Win32.CloseClipboard(); }
|
||||
}
|
||||
|
||||
internal async Task Paste(AppState app, string text, string format, int maximumBytes, int readTimeoutMs, RequestContext context)
|
||||
internal async Task Paste(AppState app, WindowState window, string text, string format, int maximumBytes, int readTimeoutMs, RequestContext context)
|
||||
{
|
||||
Windows.RequireForeground(app, null, context);
|
||||
var targetPid = app.RequireProcess().Pid;
|
||||
Windows.RequireForeground(app, window, context);
|
||||
// Clipboard readers are matched against this window's real process, not a removal-only app identity.
|
||||
var targetPid = window.ProcessId;
|
||||
var transaction = await context.Sta.Invoke(control => Prepare(control, targetPid, text, format, maximumBytes), context.Cancellation);
|
||||
try
|
||||
{
|
||||
await context.Sta.Invoke(() => transaction.Begin(), context.Cancellation);
|
||||
Input.Press(app, "v", ["control"], context);
|
||||
Input.Press(app, window, "v", ["control"], context);
|
||||
try { await transaction.Consumed.Task.WaitAsync(TimeSpan.FromMilliseconds(readTimeoutMs), context.Cancellation); }
|
||||
catch (TimeoutException)
|
||||
{
|
||||
|
||||
@@ -7,11 +7,10 @@ namespace DeepSeek.Cua.Windows;
|
||||
|
||||
internal sealed class ClientState
|
||||
{
|
||||
internal Dictionary<string, PreparedLaunch> Launches { get; } = new(StringComparer.Ordinal);
|
||||
internal Dictionary<string, AppState> Apps { get; } = new(StringComparer.Ordinal);
|
||||
internal AppState App(string handle) => Apps.GetValueOrDefault(handle)
|
||||
?? throw new NativeFailure("STALE_APP", "Select the application again on this connection");
|
||||
internal void Clear() { foreach (var app in Apps.Values) app.Dispose(); Apps.Clear(); Launches.Clear(); }
|
||||
internal void Clear() { foreach (var app in Apps.Values) app.Dispose(); Apps.Clear(); }
|
||||
}
|
||||
|
||||
internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IAsyncDisposable
|
||||
@@ -33,17 +32,12 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
|
||||
private static nint? WindowId(JsonElement value) => value.TryGetProperty("windowId", out var id) ? ParseWindowId(id) : null;
|
||||
private static string[] Modifiers(JsonElement value) => value.TryGetProperty("modifiers", out var items) ? items.EnumerateArray().Select(item => item.GetString()!).ToArray() : [];
|
||||
|
||||
// Window-scoped operations act only on the window authorized at getApp; the request must name it.
|
||||
// Window-scoped operations revalidate the explicit target against the authorized exact AUMID.
|
||||
private static WindowState Selected(AppState app, JsonElement parameters, RequestContext context)
|
||||
{
|
||||
if (!parameters.TryGetProperty("windowId", out var raw))
|
||||
throw new NativeFailure("UNSUPPORTED", "This operation requires a window target; select a window with getApp and pass its windowId");
|
||||
var id = ParseWindowId(raw);
|
||||
var selected = app.SelectedWindow ?? throw new NativeFailure("APP_ACCESS_REQUIRED", "Select a window with getApp before this operation");
|
||||
if (selected.Id != id)
|
||||
throw new NativeFailure("WINDOW_CHANGED", "The request targets another window; select that window with getApp");
|
||||
selected.Geometry(app, context);
|
||||
return selected;
|
||||
throw new NativeFailure("UNSUPPORTED", "This operation requires a window target; select a window with getWindows and pass its windowId");
|
||||
return Windows.Resolve(app, ParseWindowId(raw), context);
|
||||
}
|
||||
|
||||
private static (WindowState Window, double X, double Y) Coordinate(AppState app, JsonElement parameters, RequestContext context)
|
||||
@@ -88,24 +82,24 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
|
||||
if (request.Method is "requestPermission" or "openPermissionSettings")
|
||||
throw new NativeFailure("UNSUPPORTED", "Windows desktop access has no macOS permission switches; inspect the Windows session status");
|
||||
DesktopSession.RequireInteractive();
|
||||
if (request.Method == "listApps") return DesktopWindows.ListApps(context);
|
||||
if (request.Method == "listApps") return await DesktopWindows.ListApps(context);
|
||||
if (request.Method == "findApps")
|
||||
throw new NativeFailure("UNSUPPORTED", "Windows discovers selectable windows with listApps; installed launch entries are not part of this selection");
|
||||
throw new NativeFailure("UNSUPPORTED", "Windows discovers selectable applications with listApps");
|
||||
if (request.Method == "getApp")
|
||||
{
|
||||
var selected = DesktopWindows.Select(String(parameters, "selector"), context);
|
||||
var selected = await DesktopWindows.Select(String(parameters, "selector"), context);
|
||||
state.Apps[selected.Handle] = selected;
|
||||
return new { appHandle = selected.Handle, pid = 4, name = selected.Authorization.Name,
|
||||
running = true, authorization = selected.Authorization };
|
||||
return new { appHandle = selected.Handle, pid = 4, name = selected.Name,
|
||||
running = selected.IsRunning, authorization = selected.Authorization };
|
||||
}
|
||||
if (request.Method == "launchApp")
|
||||
throw new NativeFailure("UNSUPPORTED", "A selected Windows window cannot be launched; select an observed window from listApps");
|
||||
throw new NativeFailure("UNSUPPORTED", "Windows application selection does not launch applications");
|
||||
var app = state.App(String(parameters, "appHandle"));
|
||||
app.RequireProcess();
|
||||
var foreground = Win32.GetForegroundWindow();
|
||||
switch (request.Method)
|
||||
{
|
||||
case "listWindows": return Windows.Refresh(app, context).Select(window => window.Describe(app, context)).ToArray();
|
||||
case "listWindows":
|
||||
return Windows.DescribeAll(app, Windows.Refresh(app, context), context).Values;
|
||||
case "getState":
|
||||
{
|
||||
var target = Selected(app, parameters, context);
|
||||
@@ -115,29 +109,34 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
|
||||
case "activate": Windows.Activate(app, null, context); return ActionResult(foreground);
|
||||
case "show": return Windows.Show(app, WindowId(parameters), context);
|
||||
case "typeText":
|
||||
Selected(app, parameters, context);
|
||||
await Input.Type(app, String(parameters, "text"), context);
|
||||
{
|
||||
var target = Selected(app, parameters, context);
|
||||
await Input.Type(app, target, String(parameters, "text"), context);
|
||||
return ActionResult(foreground);
|
||||
}
|
||||
case "pressKey":
|
||||
Selected(app, parameters, context);
|
||||
Input.Press(app, String(parameters, "key"), Modifiers(parameters), context);
|
||||
{
|
||||
var target = Selected(app, parameters, context);
|
||||
Input.Press(app, target, String(parameters, "key"), Modifiers(parameters), context);
|
||||
return ActionResult(foreground);
|
||||
}
|
||||
case "sendKeyEvent":
|
||||
throw new NativeFailure("UNSUPPORTED", "Held key events are not supported on current platform.");
|
||||
case "paste":
|
||||
Selected(app, parameters, context);
|
||||
await clipboard.Paste(app, String(parameters, "text"), OptionalString(parameters, "format") ?? "text",
|
||||
{
|
||||
var target = Selected(app, parameters, context);
|
||||
await clipboard.Paste(app, target, String(parameters, "text"), OptionalString(parameters, "format") ?? "text",
|
||||
parameters.GetProperty("maxClipboardBytes").GetInt32(), parameters.GetProperty("pasteReadTimeoutMs").GetInt32(), context);
|
||||
return ActionResult(foreground);
|
||||
}
|
||||
case "mouseMove":
|
||||
{
|
||||
var target = Selected(app, parameters, context);
|
||||
if (parameters.TryGetProperty("elementId", out _))
|
||||
{
|
||||
var target = Selected(app, parameters, context);
|
||||
return await mta.Invoke(() =>
|
||||
{
|
||||
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
if (element.Window.Id != target.Id) throw new NativeFailure("WINDOW_CHANGED", "The element belongs to another window");
|
||||
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
var bounds = context.Ax(() => element.Element.Current.BoundingRectangle);
|
||||
if (bounds.IsEmpty) throw new NativeFailure("UNSUPPORTED", "The element has no pointer bounds");
|
||||
var geometry = element.Window.RequireObservation(app, context);
|
||||
@@ -183,8 +182,7 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
|
||||
{
|
||||
await mta.Invoke(() =>
|
||||
{
|
||||
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
if (element.Window.Id != target.Id) throw new NativeFailure("WINDOW_CHANGED", "The element belongs to another window");
|
||||
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
if (!hold.HasValue && button == "left" && count == 1 && modifiers.Length == 0 && AccessibilityActions.Click(element, context)) return true;
|
||||
var bounds = context.Ax(() => element.Element.Current.BoundingRectangle);
|
||||
if (bounds.IsEmpty) throw new NativeFailure("UNSUPPORTED", "The element has no pointer bounds");
|
||||
@@ -204,17 +202,17 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
|
||||
}
|
||||
case "listActions":
|
||||
{
|
||||
Selected(app, parameters, context);
|
||||
var target = Selected(app, parameters, context);
|
||||
return await mta.Invoke(() => AccessibilityActions.Actions(
|
||||
Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation);
|
||||
Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation);
|
||||
}
|
||||
case "performAction":
|
||||
{
|
||||
Selected(app, parameters, context);
|
||||
var target = Selected(app, parameters, context);
|
||||
var argument = parameters.TryGetProperty("argument", out var raw) ? raw : (JsonElement?)null;
|
||||
await mta.Invoke(() =>
|
||||
{
|
||||
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
AccessibilityActions.Perform(app, element, String(parameters, "action"), argument, context);
|
||||
return true;
|
||||
}, cancellation);
|
||||
@@ -222,17 +220,17 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
|
||||
}
|
||||
case "invoke":
|
||||
{
|
||||
Selected(app, parameters, context);
|
||||
var target = Selected(app, parameters, context);
|
||||
var dispatched = await mta.Invoke(() => AccessibilityActions.Invoke(
|
||||
Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation);
|
||||
Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context), context), cancellation);
|
||||
return new { dispatched };
|
||||
}
|
||||
case "setValue":
|
||||
{
|
||||
Selected(app, parameters, context);
|
||||
var target = Selected(app, parameters, context);
|
||||
await mta.Invoke(() =>
|
||||
{
|
||||
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
AccessibilityActions.SetValue(element, parameters.GetProperty("value"), context);
|
||||
return true;
|
||||
}, cancellation);
|
||||
@@ -240,10 +238,10 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
|
||||
}
|
||||
case "selectText":
|
||||
{
|
||||
Selected(app, parameters, context);
|
||||
var target = Selected(app, parameters, context);
|
||||
await mta.Invoke(() =>
|
||||
{
|
||||
var element = Accessibility.Element(app, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
|
||||
AccessibilityActions.SelectText(element, String(parameters, "text"), OptionalString(parameters, "prefix"), OptionalString(parameters, "suffix"),
|
||||
OptionalString(parameters, "selectionType") ?? "text", context);
|
||||
return true;
|
||||
|
||||
@@ -1,98 +1,378 @@
|
||||
// Desktop discovery selects windows by their invariant decimal HWND strings.
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
// AppsFolder AUMID enumeration, exact-ordinal merge, and exact window identity resolution.
|
||||
using System.ComponentModel;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using DeepSeek.Cua.Protocol;
|
||||
using Microsoft.Win32.SafeHandles;
|
||||
|
||||
namespace DeepSeek.Cua.Windows;
|
||||
|
||||
internal sealed record DiscoveredWindow(WindowState Window, RunningApplication Process)
|
||||
{
|
||||
internal string Selector => Window.Id.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
internal sealed record ShellApplication(string AppId, string DisplayName, bool HasShellName);
|
||||
internal sealed record ShellCatalog(ShellApplication[] Applications, int Omitted);
|
||||
internal sealed record ResolvedWindow(nint Handle, string AppId, string? ExecutablePath, string ProcessName,
|
||||
int ProcessId, long ProcessStarted);
|
||||
internal sealed record WindowObservation(ResolvedWindow[] Windows, int Denied, int Gone);
|
||||
internal sealed record CatalogApplication(string AppId, string DisplayName, string? ExecutablePath, bool IsRunning);
|
||||
|
||||
internal static class DesktopWindows
|
||||
{
|
||||
internal static object[] ListApps(RequestContext context)
|
||||
{
|
||||
var result = new List<object>();
|
||||
foreach (var entry in Discover(context))
|
||||
{
|
||||
context.Check();
|
||||
try
|
||||
{
|
||||
using var owner = Owner(entry, false, context);
|
||||
entry.Window.Geometry(owner, context);
|
||||
result.Add(new { pid = 4, name = owner.Authorization.Name, id = entry.Selector });
|
||||
}
|
||||
catch (NativeFailure error) when (error.Code is "STALE_APP" or "PERMISSION_DENIED" or "APP_IDENTITY_UNAVAILABLE" or "UNSUPPORTED_SESSION" or "WINDOW_CHANGED")
|
||||
{
|
||||
// A disappearing or inaccessible window is omitted from discovery.
|
||||
}
|
||||
}
|
||||
return result.ToArray();
|
||||
}
|
||||
private const uint ProcessQueryLimitedInformation = 0x1000;
|
||||
private const int ErrorAccessDenied = 5;
|
||||
private const int ErrorInvalidParameter = 87;
|
||||
private const int ErrorInsufficientBuffer = 122;
|
||||
private const int AppModelErrorNoApplication = 15703;
|
||||
private const int EAccessDenied = unchecked((int)0x80070005);
|
||||
private const int EHandle = unchecked((int)0x80070006);
|
||||
private const int EInvalidArgument = unchecked((int)0x80070057);
|
||||
private const int EInvalidWindowHandle = unchecked((int)0x80070578);
|
||||
private const int MaximumIdentityBuffer = 1_048_576;
|
||||
private static readonly Guid AppUserModelIdFormat = new("9F4C2855-9F79-4B39-A8D0-E1D42DE1D5F3");
|
||||
|
||||
internal static AppState Select(string selector, RequestContext context)
|
||||
internal static async Task<object[]> ListApps(RequestContext context)
|
||||
{
|
||||
if (string.IsNullOrEmpty(selector)) throw new NativeFailure("INVALID_ARGUMENT", "A window selector from listApps is required");
|
||||
var entry = Discover(context).SingleOrDefault(value => string.Equals(value.Selector, selector, StringComparison.Ordinal))
|
||||
?? throw new NativeFailure("WINDOW_CHANGED", "Discover applications again; this window is no longer available");
|
||||
var app = Owner(entry, true, context);
|
||||
try { entry.Window.Geometry(app, context); return app; }
|
||||
catch { app.Dispose(); throw; }
|
||||
}
|
||||
|
||||
private static List<DiscoveredWindow> Discover(RequestContext context)
|
||||
{
|
||||
var current = new List<nint>();
|
||||
Win32.EnumWindows((handle, _) =>
|
||||
var applications = await Discover(context);
|
||||
var result = applications.Select(application => new
|
||||
{
|
||||
var extended = Win32.GetWindowLongPtr(handle, -20).ToInt64();
|
||||
if (Win32.IsWindowVisible(handle) && (extended & 0x80) == 0
|
||||
&& Win32.GetWindowRect(handle, out var bounds) && bounds.Width > 0 && bounds.Height > 0)
|
||||
current.Add(handle);
|
||||
return true;
|
||||
}, 0);
|
||||
var result = new List<DiscoveredWindow>();
|
||||
foreach (var handle in current)
|
||||
{
|
||||
context.Check();
|
||||
try
|
||||
{
|
||||
Win32.GetWindowThreadProcessId(handle, out var pid);
|
||||
var process = Applications.Running(checked((int)pid));
|
||||
var entry = new DiscoveredWindow(new(handle), process);
|
||||
using var owner = Owner(entry, false, context);
|
||||
entry.Window.Geometry(owner, context);
|
||||
result.Add(entry);
|
||||
}
|
||||
catch (NativeFailure error) when (error.Code is "STALE_APP" or "PERMISSION_DENIED" or "APP_IDENTITY_UNAVAILABLE" or "UNSUPPORTED_SESSION" or "WINDOW_CHANGED")
|
||||
{
|
||||
// A disappearing or inaccessible window is omitted from discovery.
|
||||
}
|
||||
}
|
||||
pid = 4, id = application.AppId, name = application.DisplayName, isRunning = application.IsRunning,
|
||||
}).ToArray();
|
||||
context.Check();
|
||||
return result;
|
||||
}
|
||||
|
||||
private static AppState Owner(DiscoveredWindow entry, bool authorize, RequestContext context)
|
||||
internal static async Task<AppState> Select(string selector, RequestContext context)
|
||||
{
|
||||
var window = entry.Window;
|
||||
var process = entry.Process;
|
||||
var length = Win32.GetWindowTextLength(window.Handle);
|
||||
if (length > context.Config.MaxResponseBytes) throw new NativeFailure("NATIVE_RESPONSE_LIMIT", "Window title exceeds the response limit");
|
||||
var title = new StringBuilder(length + 1);
|
||||
Win32.GetWindowText(window.Handle, title, title.Capacity);
|
||||
var name = $"Window: {title} ({process.Name})";
|
||||
// Shared consent IDs require a SHA-256 string, including non-persistable window grants.
|
||||
var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes($"window\0{entry.Selector}")));
|
||||
var identity = new ApplicationIdentity(id, name, process.Path, false);
|
||||
var app = new AppState(Guid.NewGuid().ToString("N"), identity, "", process);
|
||||
if (authorize)
|
||||
{
|
||||
app.SelectedWindow = window;
|
||||
app.Windows.Add(window.Handle, window);
|
||||
}
|
||||
return app;
|
||||
var applications = await Discover(context);
|
||||
var application = SelectApplication(applications, selector);
|
||||
context.Check();
|
||||
return new AppState(Guid.NewGuid().ToString("N"), application.AppId, application.DisplayName,
|
||||
ApplicationAuthorization.Create(application.AppId, application.DisplayName, application.ExecutablePath),
|
||||
application.IsRunning);
|
||||
}
|
||||
|
||||
private static async Task<CatalogApplication[]> Discover(RequestContext context)
|
||||
{
|
||||
var catalog = await context.Sta.Invoke(_ => ReadAppsFolder(context), context.Cancellation);
|
||||
var observation = Observe(context);
|
||||
if (catalog.Omitted > 0)
|
||||
Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {catalog.Omitted} AppsFolder items without an application identity");
|
||||
if (observation.Denied > 0)
|
||||
Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {observation.Denied} windows whose process or window access was denied");
|
||||
if (observation.Gone > 0)
|
||||
Console.Error.WriteLine($"NATIVE_DISCOVERY: omitted {observation.Gone} disappeared or identity-less windows");
|
||||
var applications = Merge(catalog.Applications, observation.Windows);
|
||||
context.Check();
|
||||
return applications;
|
||||
}
|
||||
|
||||
private static ShellCatalog ReadAppsFolder(RequestContext context)
|
||||
{
|
||||
var owned = new List<object>();
|
||||
object Own(object value) { owned.Add(value); return value; }
|
||||
try
|
||||
{
|
||||
dynamic shell = Own(Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("13709620-C279-11CE-A49E-444553540000"), true)!)!);
|
||||
dynamic folder = Own((object)shell.NameSpace("shell:AppsFolder"));
|
||||
if (folder is null) throw new COMException("AppsFolder is unavailable");
|
||||
dynamic items = Own((object)folder.Items());
|
||||
var applications = new List<ShellApplication>();
|
||||
var omitted = 0;
|
||||
for (var index = 0; index < (int)items.Count; index++)
|
||||
{
|
||||
context.Check();
|
||||
dynamic item = Own((object)items.Item(index));
|
||||
// A failed property read is a catalog failure; only an absent or non-string property is missing.
|
||||
object? property = item.ExtendedProperty("System.AppUserModel.ID");
|
||||
object? displayName = item.Name;
|
||||
string? aumid = property as string;
|
||||
string? name = displayName as string;
|
||||
if (string.IsNullOrWhiteSpace(aumid)) { omitted++; continue; }
|
||||
var hasName = !string.IsNullOrWhiteSpace(name);
|
||||
applications.Add(new(aumid, hasName ? name! : aumid, hasName));
|
||||
}
|
||||
return new(applications.ToArray(), omitted);
|
||||
}
|
||||
catch (Exception error) when (error is COMException or Microsoft.CSharp.RuntimeBinder.RuntimeBinderException)
|
||||
{
|
||||
throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows Shell could not enumerate AppsFolder; retry discovery after Explorer is available");
|
||||
}
|
||||
finally
|
||||
{
|
||||
var released = new HashSet<object>(ReferenceEqualityComparer.Instance);
|
||||
foreach (var value in owned.AsEnumerable().Reverse())
|
||||
if (released.Add(value) && Marshal.IsComObject(value)) Marshal.FinalReleaseComObject(value);
|
||||
}
|
||||
}
|
||||
|
||||
private static CatalogApplication SelectApplication(IReadOnlyList<CatalogApplication> applications, string selector)
|
||||
{
|
||||
if (string.IsNullOrEmpty(selector))
|
||||
throw new NativeFailure("INVALID_ARGUMENT", "An exact application id from listApps is required");
|
||||
foreach (var application in applications)
|
||||
if (string.Equals(application.AppId, selector, StringComparison.Ordinal)) return application;
|
||||
throw new NativeFailure("APP_NOT_FOUND", "Use an exact application id from listApps");
|
||||
}
|
||||
|
||||
private static CatalogApplication[] Merge(ShellApplication[] shellApplications, ResolvedWindow[] windows)
|
||||
{
|
||||
var applications = new Dictionary<string, Builder>(StringComparer.Ordinal);
|
||||
foreach (var shell in shellApplications)
|
||||
{
|
||||
if (string.IsNullOrEmpty(shell.AppId)) continue;
|
||||
if (!applications.TryGetValue(shell.AppId, out var existing))
|
||||
applications.Add(shell.AppId, new(shell.AppId, shell.DisplayName, shell.HasShellName, shellNamed: true));
|
||||
else existing.PreferShellName(shell.DisplayName, shell.HasShellName);
|
||||
}
|
||||
foreach (var window in windows.OrderBy(window => window.AppId, StringComparer.Ordinal)
|
||||
.ThenBy(window => (long)window.Handle))
|
||||
{
|
||||
if (string.IsNullOrEmpty(window.AppId) || string.IsNullOrEmpty(window.ProcessName)) continue;
|
||||
if (!applications.TryGetValue(window.AppId, out var builder))
|
||||
{
|
||||
builder = new(window.AppId, window.ProcessName, hasShellName: false, shellNamed: false);
|
||||
applications.Add(window.AppId, builder);
|
||||
}
|
||||
builder.Add(window);
|
||||
}
|
||||
return applications.Values.Select(builder => builder.Build())
|
||||
.OrderBy(application => application.AppId, StringComparer.Ordinal).ToArray();
|
||||
}
|
||||
|
||||
private sealed class Builder(string appId, string displayName, bool hasShellName, bool shellNamed)
|
||||
{
|
||||
private readonly HashSet<string> executablePaths = new(StringComparer.Ordinal);
|
||||
private readonly HashSet<string> processNames = new(StringComparer.Ordinal);
|
||||
private string shellDisplayName = displayName;
|
||||
private bool shellNameProvided = hasShellName;
|
||||
|
||||
internal void PreferShellName(string candidate, bool provided)
|
||||
{
|
||||
if (provided && !shellNameProvided) { shellDisplayName = candidate; shellNameProvided = true; }
|
||||
else if (provided == shellNameProvided && string.CompareOrdinal(candidate, shellDisplayName) < 0)
|
||||
shellDisplayName = candidate;
|
||||
}
|
||||
|
||||
internal void Add(ResolvedWindow window)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(window.ExecutablePath)) executablePaths.Add(window.ExecutablePath);
|
||||
processNames.Add(window.ProcessName);
|
||||
}
|
||||
|
||||
internal CatalogApplication Build()
|
||||
{
|
||||
var name = shellNamed ? shellDisplayName
|
||||
: processNames.Count == 0 ? shellDisplayName : processNames.Min(StringComparer.Ordinal)!;
|
||||
var executablePath = executablePaths.Count == 1 ? executablePaths.First() : null;
|
||||
return new(appId, name, executablePath, processNames.Count != 0);
|
||||
}
|
||||
}
|
||||
|
||||
internal static WindowObservation Observe(RequestContext context)
|
||||
{
|
||||
var windows = new List<ResolvedWindow>();
|
||||
var denied = 0;
|
||||
var gone = 0;
|
||||
foreach (var handle in EnumerateHandles(context))
|
||||
{
|
||||
context.Check();
|
||||
try
|
||||
{
|
||||
var window = ResolveWindow(handle, context);
|
||||
if (window is null) gone++;
|
||||
else windows.Add(window);
|
||||
}
|
||||
catch (NativeFailure error) when (error.Code == "PERMISSION_DENIED") { denied++; }
|
||||
}
|
||||
return new(windows.ToArray(), denied, gone);
|
||||
}
|
||||
|
||||
internal static bool Eligible(nint window)
|
||||
{
|
||||
// EnumWindows yields top-level windows; the explicit Resolve path also checks the root ancestor.
|
||||
if (Win32.GetAncestor(window, 2) != window) return false;
|
||||
var extended = Win32.GetWindowLongPtr(window, -20).ToInt64();
|
||||
if ((extended & 0x80) != 0) return false;
|
||||
if (!Win32.IsWindowVisible(window)) return false;
|
||||
if (!Win32.GetWindowRect(window, out var bounds) || bounds.Width <= 0 || bounds.Height <= 0) return false;
|
||||
Win32.GetWindowThreadProcessId(window, out var processId);
|
||||
if (processId == 0 || processId > int.MaxValue) return false;
|
||||
return Win32.ProcessIdToSessionId(processId, out var session) && session == Applications.SessionId;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<nint> EnumerateHandles(RequestContext context)
|
||||
{
|
||||
var handles = new List<nint>();
|
||||
var enumerated = Win32.EnumWindows((window, _) =>
|
||||
{
|
||||
// Cancellation cannot throw across the unmanaged callback; stop, then rethrow outside it.
|
||||
if (context.Cancellation.IsCancellationRequested) return false;
|
||||
if (Eligible(window)) handles.Add(window);
|
||||
return true;
|
||||
}, 0);
|
||||
context.Check();
|
||||
if (!enumerated) throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows could not enumerate top-level windows; retry discovery after the desktop is available");
|
||||
return handles;
|
||||
}
|
||||
|
||||
internal static ResolvedWindow? ResolveWindow(nint handle, RequestContext context)
|
||||
{
|
||||
context.Check();
|
||||
Win32.GetWindowThreadProcessId(handle, out var rawProcessId);
|
||||
if (rawProcessId == 0 || rawProcessId > int.MaxValue) return null;
|
||||
var processId = (int)rawProcessId;
|
||||
if (!Win32.ProcessIdToSessionId(rawProcessId, out var session) || session != Applications.SessionId) return null;
|
||||
var process = Win32.OpenProcess(ProcessQueryLimitedInformation, false, rawProcessId);
|
||||
if (process == 0) return OpenFailure();
|
||||
try
|
||||
{
|
||||
using var native = new SafeProcessHandle(process, ownsHandle: false);
|
||||
if (!Win32.GetProcessTimes(native, out var started, out _, out _, out _)) return null;
|
||||
var processName = ReadProcessName(processId);
|
||||
if (string.IsNullOrEmpty(processName)) return null;
|
||||
context.Check();
|
||||
var windowAumid = ReadWindowAumid(handle, context);
|
||||
string? executablePath = null;
|
||||
string? appId = null;
|
||||
if (!string.IsNullOrEmpty(windowAumid)) appId = windowAumid;
|
||||
else
|
||||
{
|
||||
appId = ReadProcessAumid(process, context);
|
||||
if (string.IsNullOrEmpty(appId)) executablePath = ReadExecutablePath(process, context);
|
||||
appId ??= executablePath;
|
||||
}
|
||||
if (appId is null) return null;
|
||||
// A recycled handle must not inherit the identity resolved for its former process.
|
||||
Win32.GetWindowThreadProcessId(handle, out var currentProcessId);
|
||||
if (currentProcessId != rawProcessId) return null;
|
||||
using var current = new SafeProcessHandle(process, ownsHandle: false);
|
||||
if (!Win32.GetProcessTimes(current, out var currentStarted, out _, out _, out _) || currentStarted != started) return null;
|
||||
context.Check();
|
||||
return new(handle, appId, executablePath, processName, processId, started);
|
||||
}
|
||||
finally { Win32.CloseHandle(process); }
|
||||
}
|
||||
|
||||
private static ResolvedWindow? OpenFailure() => Marshal.GetLastPInvokeError() switch
|
||||
{
|
||||
ErrorAccessDenied => throw PermissionDenied(),
|
||||
ErrorInvalidParameter => null,
|
||||
_ => throw Unavailable(),
|
||||
};
|
||||
|
||||
private static string? ReadProcessName(int processId)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var process = Process.GetProcessById(processId);
|
||||
return process.ProcessName;
|
||||
}
|
||||
catch (ArgumentException) { return null; }
|
||||
catch (InvalidOperationException) { return null; }
|
||||
catch (Win32Exception error) when (error.NativeErrorCode == ErrorAccessDenied) { throw PermissionDenied(); }
|
||||
catch (Win32Exception) { throw Unavailable(); }
|
||||
}
|
||||
|
||||
private static string? ReadWindowAumid(nint handle, RequestContext context)
|
||||
{
|
||||
context.Check();
|
||||
var read = context.Sta.InvokeSync(_ => ReadWindowAumidCore(handle, context), context.Cancellation);
|
||||
context.Check();
|
||||
return read;
|
||||
}
|
||||
|
||||
private static string? ReadWindowAumidCore(nint handle, RequestContext context)
|
||||
{
|
||||
var id = typeof(Win32.IPropertyStore).GUID;
|
||||
var status = Win32.SHGetPropertyStoreForWindow(handle, ref id, out var store);
|
||||
if (status != 0)
|
||||
{
|
||||
if (status == EAccessDenied) throw PermissionDenied();
|
||||
if (status is EHandle or EInvalidArgument or EInvalidWindowHandle) return null;
|
||||
throw Unavailable();
|
||||
}
|
||||
if (store is null) throw Unavailable();
|
||||
try
|
||||
{
|
||||
var key = new Win32.PropertyKey { FormatId = AppUserModelIdFormat, PropertyId = 5 };
|
||||
var value = default(Win32.PropVariant);
|
||||
var result = store.GetValue(ref key, out value);
|
||||
try
|
||||
{
|
||||
if (result != 0)
|
||||
{
|
||||
if (result == EAccessDenied) throw PermissionDenied();
|
||||
throw Unavailable();
|
||||
}
|
||||
context.Check();
|
||||
if (value.ValueType == 0) return null;
|
||||
if (value.ValueType == 31) return value.Pointer == 0 ? null : Marshal.PtrToStringUni(value.Pointer);
|
||||
if (value.ValueType == 8) return value.Pointer == 0 ? null : Marshal.PtrToStringBSTR(value.Pointer);
|
||||
throw Unavailable();
|
||||
}
|
||||
finally { Win32.PropVariantClear(ref value); }
|
||||
}
|
||||
finally { Marshal.FinalReleaseComObject(store); }
|
||||
}
|
||||
|
||||
private static string? ReadProcessAumid(nint process, RequestContext context)
|
||||
{
|
||||
context.Check();
|
||||
uint length = 0;
|
||||
var status = Win32.GetApplicationUserModelId(process, ref length, null);
|
||||
if (status == 0) throw Unavailable();
|
||||
if (status != ErrorInsufficientBuffer) return ClassifyAumidStatus(status);
|
||||
for (var attempt = 0; attempt < 4; attempt++)
|
||||
{
|
||||
context.Check();
|
||||
if (length == 0 || length > MaximumIdentityBuffer) throw Unavailable();
|
||||
var buffer = new StringBuilder(checked((int)length));
|
||||
uint size = length;
|
||||
status = Win32.GetApplicationUserModelId(process, ref size, buffer);
|
||||
if (status == 0)
|
||||
{
|
||||
var value = buffer.ToString();
|
||||
if (string.IsNullOrEmpty(value)) throw Unavailable();
|
||||
return value;
|
||||
}
|
||||
if (status != ErrorInsufficientBuffer) return ClassifyAumidStatus(status);
|
||||
length = size;
|
||||
}
|
||||
throw Unavailable();
|
||||
}
|
||||
|
||||
private static string? ClassifyAumidStatus(int status)
|
||||
{
|
||||
if (status == AppModelErrorNoApplication) return null;
|
||||
if (status == ErrorAccessDenied) throw PermissionDenied();
|
||||
throw Unavailable();
|
||||
}
|
||||
|
||||
private static string? ReadExecutablePath(nint process, RequestContext context)
|
||||
{
|
||||
for (var capacity = 1024; capacity <= 65536; capacity *= 2)
|
||||
{
|
||||
context.Check();
|
||||
var buffer = new StringBuilder(capacity);
|
||||
uint size = (uint)capacity;
|
||||
if (Win32.QueryFullProcessImageName(process, 0, buffer, ref size))
|
||||
{
|
||||
context.Check();
|
||||
var path = buffer.ToString(0, (int)Math.Min(size, (uint)buffer.Length));
|
||||
if (path.Length == 0) throw Unavailable();
|
||||
return path;
|
||||
}
|
||||
var error = Marshal.GetLastPInvokeError();
|
||||
if (error == ErrorInsufficientBuffer) continue;
|
||||
if (error == ErrorAccessDenied) throw PermissionDenied();
|
||||
if (error == ErrorInvalidParameter) return null;
|
||||
throw Unavailable();
|
||||
}
|
||||
throw Unavailable();
|
||||
}
|
||||
|
||||
private static NativeFailure PermissionDenied()
|
||||
=> new("PERMISSION_DENIED", "Windows refused the window application identity; its privilege level may exceed the helper's");
|
||||
|
||||
private static NativeFailure Unavailable()
|
||||
=> new("APP_IDENTITY_UNAVAILABLE", "The window application identity could not be resolved; retry after its owner is available");
|
||||
}
|
||||
@@ -95,6 +95,7 @@ internal sealed class StaDispatcher : IAsyncDisposable
|
||||
private readonly TaskCompletionSource<ServiceControl> ready = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource<object?> stopped = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly Thread thread;
|
||||
private ServiceControl? service;
|
||||
|
||||
internal StaDispatcher()
|
||||
{
|
||||
@@ -105,6 +106,7 @@ internal sealed class StaDispatcher : IAsyncDisposable
|
||||
Win32.SetThreadDpiAwarenessContext(-4);
|
||||
using var control = new ServiceControl();
|
||||
_ = control.Handle;
|
||||
service = control;
|
||||
ready.SetResult(control);
|
||||
Forms.Application.Run();
|
||||
}
|
||||
@@ -118,16 +120,37 @@ internal sealed class StaDispatcher : IAsyncDisposable
|
||||
internal async Task<T> Invoke<T>(Func<ServiceControl, T> work, CancellationToken cancellation)
|
||||
{
|
||||
var control = await ready.Task.WaitAsync(cancellation);
|
||||
cancellation.ThrowIfCancellationRequested();
|
||||
// A caller already on the STA must not enqueue work for the message loop it is blocking.
|
||||
if (Environment.CurrentManagedThreadId == thread.ManagedThreadId)
|
||||
{
|
||||
var inline = work(control);
|
||||
cancellation.ThrowIfCancellationRequested();
|
||||
return inline;
|
||||
}
|
||||
var result = new TaskCompletionSource<T>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
control.BeginInvoke(() =>
|
||||
{
|
||||
try { cancellation.ThrowIfCancellationRequested(); var value = work(control); result.TrySetResult(value); }
|
||||
try { cancellation.ThrowIfCancellationRequested(); var value = work(control); cancellation.ThrowIfCancellationRequested(); result.TrySetResult(value); }
|
||||
catch (OperationCanceledException) { result.TrySetCanceled(cancellation); }
|
||||
catch (Exception error) { result.TrySetException(error); }
|
||||
});
|
||||
return await result.Task;
|
||||
}
|
||||
|
||||
internal T InvokeSync<T>(Func<ServiceControl, T> work, CancellationToken cancellation)
|
||||
{
|
||||
cancellation.ThrowIfCancellationRequested();
|
||||
if (Environment.CurrentManagedThreadId == thread.ManagedThreadId)
|
||||
{
|
||||
var control = service ?? throw new InvalidOperationException("The STA dispatcher has not started");
|
||||
var inline = work(control);
|
||||
cancellation.ThrowIfCancellationRequested();
|
||||
return inline;
|
||||
}
|
||||
return Invoke(work, cancellation).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
internal Task Invoke(Action work, CancellationToken cancellation) => Invoke(_ => { work(); return true; }, cancellation);
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
|
||||
@@ -46,7 +46,7 @@ internal static class Input
|
||||
finally { foreach (var key in attempted.AsEnumerable().Reverse()) Release(Keyboard(key, flags: 2u | (key == 0x5B ? 1u : 0))); }
|
||||
}
|
||||
|
||||
private static (ushort Key, bool Extended, string[] Modifiers) ResolveKey(string name, AppState app)
|
||||
private static (ushort Key, bool Extended, string[] Modifiers) ResolveKey(string name)
|
||||
{
|
||||
var key = name.ToLowerInvariant();
|
||||
if (key.StartsWith("kp_", StringComparison.Ordinal))
|
||||
@@ -85,38 +85,38 @@ internal static class Input
|
||||
return ((ushort)(mapped & 0xFF), false, modifiers.ToArray());
|
||||
}
|
||||
|
||||
internal static void Press(AppState app, string key, string[] modifiers, RequestContext context)
|
||||
internal static void Press(AppState app, WindowState window, string key, string[] modifiers, RequestContext context)
|
||||
{
|
||||
Windows.RequireForeground(app, null, context);
|
||||
Windows.RequireForeground(app, window, context);
|
||||
var chord = new KeyChord(key, modifiers.Select(KeyChord.NormalizeModifier).Distinct().ToArray());
|
||||
var name = chord.Key.ToLowerInvariant();
|
||||
if (name == "printscreen" || chord.Modifiers.Contains("option") && name is "tab" or "escape" or "esc"
|
||||
|| chord.Modifiers.Contains("control") && name is "escape" or "esc"
|
||||
|| chord.Modifiers.Contains("control") && chord.Modifiers.Contains("option") && name == "delete")
|
||||
throw new NativeFailure("UNSUPPORTED", "This shortcut acts on the system rather than the selected application");
|
||||
var resolved = ResolveKey(chord.Key, app);
|
||||
var resolved = ResolveKey(chord.Key);
|
||||
WithModifiers(chord.Modifiers.Concat(resolved.Modifiers), () =>
|
||||
{
|
||||
Windows.RequireForeground(app, null, context);
|
||||
Windows.RequireForeground(app, window, context);
|
||||
var flags = resolved.Extended ? 1u : 0;
|
||||
try { Send(Keyboard(resolved.Key, flags: flags), Keyboard(resolved.Key, flags: flags | 2)); }
|
||||
catch { Release(Keyboard(resolved.Key, flags: flags | 2)); throw; }
|
||||
});
|
||||
}
|
||||
|
||||
internal static async Task Type(AppState app, string text, RequestContext context)
|
||||
internal static async Task Type(AppState app, WindowState window, string text, RequestContext context)
|
||||
{
|
||||
if (text.Contains('\0')) throw new NativeFailure("INVALID_ARGUMENT", "Text input cannot contain NUL");
|
||||
RequireIdleModifiers();
|
||||
for (var index = 0; index < text.Length; index++)
|
||||
{
|
||||
Windows.RequireForeground(app, null, context);
|
||||
Windows.RequireForeground(app, window, context);
|
||||
RequireIdleModifiers();
|
||||
var character = text[index];
|
||||
if (character is '\r' or '\n' or '\t')
|
||||
{
|
||||
if (character == '\r' && index + 1 < text.Length && text[index + 1] == '\n') index++;
|
||||
Press(app, character == '\t' ? "Tab" : "Enter", [], context);
|
||||
Press(app, window, character == '\t' ? "Tab" : "Enter", [], context);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
// Shell namespace enumeration and activation run on the existing STA dispatcher.
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using DeepSeek.Cua.Protocol;
|
||||
|
||||
namespace DeepSeek.Cua.Windows;
|
||||
|
||||
internal sealed record LaunchEntry(string Id, string DisplayName, string Kind, string? Path);
|
||||
internal sealed class PreparedLaunch(LaunchEntry entry, ApplicationIdentity authorization, string? digest)
|
||||
{
|
||||
internal string Handle { get; } = Guid.NewGuid().ToString("N");
|
||||
internal ApplicationIdentity Authorization { get; } = authorization;
|
||||
private bool dispatched;
|
||||
|
||||
internal void Dispatch()
|
||||
{
|
||||
if (dispatched) throw new NativeFailure("LAUNCH_ALREADY_DISPATCHED", "Discover windows before deciding whether another launch is needed");
|
||||
if (entry.Kind == "executable")
|
||||
{
|
||||
if (Applications.CanonicalPath(entry.Id) != entry.Id || Applications.Digest(entry.Id) != digest)
|
||||
throw new NativeFailure("STALE_APP", "The approved executable changed; select it again");
|
||||
}
|
||||
else if (!LaunchEntries.All().Any(current => current.Id == entry.Id))
|
||||
throw new NativeFailure("STALE_APP", "The Shell entry disappeared; discover launch entries again");
|
||||
// Dispatch may succeed even if Explorer subsequently reports a failure. Never replay this selection.
|
||||
dispatched = true;
|
||||
ShellBroker.Launch(entry.Id, show: 1);
|
||||
}
|
||||
}
|
||||
|
||||
internal static class LaunchEntries
|
||||
{
|
||||
internal static LaunchEntry[] All()
|
||||
{
|
||||
var owned = new List<object>();
|
||||
object Own(object value) { owned.Add(value); return value; }
|
||||
try
|
||||
{
|
||||
dynamic shell = Own(Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("13709620-C279-11CE-A49E-444553540000"), true)!)!);
|
||||
dynamic folder = Own((object)shell.NameSpace("shell:AppsFolder"));
|
||||
dynamic items = Own((object)folder.Items());
|
||||
var result = new List<LaunchEntry>();
|
||||
for (var index = 0; index < (int)items.Count; index++)
|
||||
{
|
||||
dynamic item = Own((object)items.Item(index));
|
||||
string parsingName = item.Path;
|
||||
string name = item.Name;
|
||||
if (string.IsNullOrWhiteSpace(parsingName) || string.IsNullOrWhiteSpace(name)) continue;
|
||||
var target = parsingName.StartsWith("::", StringComparison.Ordinal) ? parsingName : "shell:AppsFolder\\" + parsingName;
|
||||
result.Add(new(target, name, "shell", null));
|
||||
}
|
||||
return result.DistinctBy(entry => entry.Id, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
catch (Exception error) when (error is COMException or Microsoft.CSharp.RuntimeBinder.RuntimeBinderException)
|
||||
{
|
||||
throw new NativeFailure("APP_DISCOVERY_FAILED", "Windows Shell could not enumerate AppsFolder; retry discovery after Explorer is available");
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var item in owned.AsEnumerable().Reverse())
|
||||
if (Marshal.IsComObject(item)) Marshal.ReleaseComObject(item);
|
||||
}
|
||||
}
|
||||
|
||||
internal static LaunchEntry[] Find(string query, int maximum)
|
||||
{
|
||||
var matches = All().Where(entry => entry.DisplayName.Contains(query, StringComparison.OrdinalIgnoreCase)
|
||||
|| entry.Id.Contains(query, StringComparison.OrdinalIgnoreCase)).OrderBy(entry => entry.DisplayName, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
if (matches.Length > maximum) throw new NativeFailure("TOO_MANY_APPS", "Use a more specific launch entry name");
|
||||
return matches;
|
||||
}
|
||||
|
||||
internal static PreparedLaunch Prepare(string target)
|
||||
{
|
||||
if (Path.IsPathFullyQualified(target) && target.EndsWith(".exe", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var path = Applications.CanonicalPath(target);
|
||||
var name = Path.GetFileNameWithoutExtension(path);
|
||||
var identity = Applications.Identify(path, "Launch: " + name, null);
|
||||
return new(new(path, name, "executable", path), identity.Identity with { Persistable = false }, identity.Digest);
|
||||
}
|
||||
var shell = All().SingleOrDefault(entry => string.Equals(entry.Id, target, StringComparison.OrdinalIgnoreCase))
|
||||
?? throw new NativeFailure("APP_NOT_FOUND", "Use an ID from findApps or a user-specified absolute executable path");
|
||||
var id = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("launch\0" + shell.Id)));
|
||||
return new(shell, new(id, "Launch: " + shell.DisplayName, shell.Id, false), null);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
// Explorer owns user-launched applications, independently of the helper's managed process Job.
|
||||
using System.Reflection;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.InteropServices.ComTypes;
|
||||
using System.Text;
|
||||
using DeepSeek.Cua.Protocol;
|
||||
|
||||
namespace DeepSeek.Cua.Windows;
|
||||
@@ -44,21 +42,6 @@ internal static class ShellBroker
|
||||
}
|
||||
}
|
||||
|
||||
internal static string? ShortcutTarget(string path)
|
||||
{
|
||||
object? instance = null;
|
||||
try
|
||||
{
|
||||
instance = Activator.CreateInstance(Type.GetTypeFromCLSID(new Guid("00021401-0000-0000-C000-000000000046"), true)!);
|
||||
((IPersistFile)instance!).Load(path, 0);
|
||||
var target = new StringBuilder(32768);
|
||||
((IShellLink)instance).GetPath(target, target.Capacity, 0, 4);
|
||||
return target.Length == 0 ? null : Environment.ExpandEnvironmentVariables(target.ToString());
|
||||
}
|
||||
catch (COMException) { return null; }
|
||||
finally { if (instance is not null) Marshal.FinalReleaseComObject(instance); }
|
||||
}
|
||||
|
||||
[ComImport, Guid("6D5140C1-7436-11CE-8034-00AA006009FA"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
|
||||
private interface IServiceProvider
|
||||
{
|
||||
@@ -103,9 +86,4 @@ internal static class ShellBroker
|
||||
[PreserveSig] int GetItemObject(uint item, ref Guid id, [MarshalAs(UnmanagedType.Interface)] out object value);
|
||||
}
|
||||
|
||||
[ComImport, Guid("000214F9-0000-0000-C000-000000000046"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
|
||||
private interface IShellLink
|
||||
{
|
||||
void GetPath([Out, MarshalAs(UnmanagedType.LPWStr)] StringBuilder path, int capacity, nint findData, uint flags);
|
||||
}
|
||||
}
|
||||
@@ -92,42 +92,42 @@ internal static class Win32
|
||||
|
||||
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] internal static extern uint GetFinalPathNameByHandle(Microsoft.Win32.SafeHandles.SafeFileHandle file, StringBuilder path, uint length, uint flags);
|
||||
[DllImport("kernel32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool GetProcessTimes(Microsoft.Win32.SafeHandles.SafeProcessHandle process, out long created, out long exited, out long kernel, out long user);
|
||||
[DllImport("wintrust.dll", CharSet = CharSet.Unicode, ExactSpelling = true)] internal static extern int WinVerifyTrust(nint window, ref Guid action, ref TrustData data);
|
||||
[DllImport("wintrust.dll")] internal static extern nint WTHelperProvDataFromStateData(nint state);
|
||||
[DllImport("wintrust.dll")] internal static extern nint WTHelperGetProvSignerFromChain(nint provider, uint signer, [MarshalAs(UnmanagedType.Bool)] bool counterSigner, uint counter);
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)] internal struct ProviderSigner
|
||||
[DllImport("shell32.dll", PreserveSig = true)]
|
||||
internal static extern int SHGetPropertyStoreForWindow(nint window, ref Guid id,
|
||||
[MarshalAs(UnmanagedType.Interface)] out IPropertyStore? store);
|
||||
[DllImport("ole32.dll")] internal static extern int PropVariantClear(ref PropVariant value);
|
||||
[DllImport("kernel32.dll", CharSet = CharSet.Unicode, ExactSpelling = true)]
|
||||
internal static extern int GetApplicationUserModelId(nint process, ref uint length, StringBuilder? value);
|
||||
[DllImport("kernel32.dll", EntryPoint = "QueryFullProcessImageNameW", CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
internal static extern bool QueryFullProcessImageName(nint process, uint flags, StringBuilder name, ref uint length);
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
internal static extern nint OpenProcess(uint access, [MarshalAs(UnmanagedType.Bool)] bool inherit, uint processId);
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
internal static extern bool CloseHandle(nint handle);
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
internal static extern bool ProcessIdToSessionId(uint processId, out uint sessionId);
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)] internal struct PropertyKey { internal Guid FormatId; internal uint PropertyId; }
|
||||
// PROPVARIANT is 24 bytes on supported x64/arm64 targets; the trailing field reserves its union.
|
||||
[StructLayout(LayoutKind.Sequential)] internal struct PropVariant
|
||||
{
|
||||
internal uint Size, VerifyTimeLow, VerifyTimeHigh, CertificateCount;
|
||||
internal nint Certificates;
|
||||
}
|
||||
[StructLayout(LayoutKind.Sequential)] internal struct ProviderCertificate
|
||||
{
|
||||
internal uint Size;
|
||||
internal nint Certificate;
|
||||
}
|
||||
[StructLayout(LayoutKind.Sequential)] internal struct CertificateContext
|
||||
{
|
||||
internal uint Encoding;
|
||||
internal nint Encoded;
|
||||
internal uint EncodedSize;
|
||||
internal ushort ValueType;
|
||||
internal ushort Reserved1, Reserved2, Reserved3;
|
||||
internal nint Pointer;
|
||||
internal nint Pointer2;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] internal struct TrustFile
|
||||
[ComImport, Guid("886D8EEB-8CF2-4446-8D02-CDBA1DBDCF99"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
|
||||
internal interface IPropertyStore
|
||||
{
|
||||
internal uint Size;
|
||||
[MarshalAs(UnmanagedType.LPWStr)] internal string Path;
|
||||
internal nint Handle, Subject;
|
||||
}
|
||||
[StructLayout(LayoutKind.Sequential)] internal struct TrustData
|
||||
{
|
||||
internal uint Size;
|
||||
internal nint PolicyCallback, SipClient;
|
||||
internal uint UiChoice, RevocationChecks, UnionChoice;
|
||||
internal nint File;
|
||||
internal uint StateAction;
|
||||
internal nint StateData, UrlReference;
|
||||
internal uint ProviderFlags, UiContext;
|
||||
internal nint SignatureSettings;
|
||||
[PreserveSig] int GetCount(out uint count);
|
||||
[PreserveSig] int GetAt(uint index, out PropertyKey key);
|
||||
[PreserveSig] int GetValue(ref PropertyKey key, out PropVariant value);
|
||||
[PreserveSig] int SetValue(ref PropertyKey key, ref PropVariant value);
|
||||
[PreserveSig] int Commit();
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
// Window state retains HWND values; JSON exposes them as invariant decimal strings.
|
||||
// Window state belongs to one exact application identity and revalidates its own process and AUMID.
|
||||
using System.Globalization;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
@@ -12,24 +12,56 @@ internal readonly record struct WindowGeometry(Win32.Rect Pixels, uint Dpi)
|
||||
internal Bounds Bounds => new(Pixels.Left, Pixels.Top, Pixels.Width, Pixels.Height);
|
||||
}
|
||||
|
||||
internal sealed class WindowState(nint handle)
|
||||
internal sealed class WindowState(nint handle, int processId, long processStarted)
|
||||
{
|
||||
internal nint Id => Handle;
|
||||
internal nint Handle { get; } = handle;
|
||||
internal int ProcessId { get; } = processId;
|
||||
internal long ProcessStarted { get; } = processStarted;
|
||||
private WindowGeometry? observation;
|
||||
|
||||
internal void RequireCurrent(AppState app, RequestContext context)
|
||||
{
|
||||
context.Check();
|
||||
if (!Win32.IsWindow(Handle))
|
||||
throw Remove(app, "The selected window is no longer available; read fresh application state");
|
||||
var current = DesktopWindows.ResolveWindow(Handle, context);
|
||||
if (current is null)
|
||||
throw Remove(app, "The selected window is no longer available; read fresh application state");
|
||||
if (!string.Equals(app.AppId, current.AppId, StringComparison.Ordinal)
|
||||
|| ProcessId != current.ProcessId || ProcessStarted != current.ProcessStarted)
|
||||
throw Remove(app, "The selected window no longer belongs to the authorized application; read fresh application state");
|
||||
}
|
||||
|
||||
// A stale window and its snapshots must not remain selectable after a direct operation detects the change.
|
||||
private NativeFailure Remove(AppState app, string message)
|
||||
{
|
||||
if (app.Windows.TryGetValue(Handle, out var current) && ReferenceEquals(current, this))
|
||||
{
|
||||
app.Windows.Remove(Handle);
|
||||
app.Snapshots.Remove(Handle);
|
||||
}
|
||||
return new NativeFailure("WINDOW_CHANGED", message);
|
||||
}
|
||||
|
||||
internal WindowGeometry Geometry(AppState app, RequestContext context)
|
||||
{
|
||||
context.Check();
|
||||
app.RequireProcess();
|
||||
if (!Win32.IsWindow(Handle))
|
||||
throw new NativeFailure("WINDOW_CHANGED", "The selected window is no longer available; read fresh application state");
|
||||
RequireCurrent(app, context);
|
||||
return TryGeometry(out var geometry)
|
||||
? geometry
|
||||
: throw new NativeFailure("WINDOW_CHANGED", "The selected window geometry is unavailable; read fresh application state");
|
||||
}
|
||||
|
||||
private bool TryGeometry(out WindowGeometry geometry)
|
||||
{
|
||||
geometry = default;
|
||||
if (Win32.DwmGetWindowAttribute(Handle, 9, out Win32.Rect rectangle, Marshal.SizeOf<Win32.Rect>()) != 0
|
||||
&& !Win32.GetWindowRect(Handle, out rectangle)) throw new NativeFailure("WINDOW_CHANGED", "The selected window geometry is unavailable");
|
||||
&& !Win32.GetWindowRect(Handle, out rectangle)) return false;
|
||||
var dpi = Win32.GetDpiForWindow(Handle);
|
||||
if (dpi == 0 || rectangle.Width <= 0 || rectangle.Height <= 0)
|
||||
throw new NativeFailure("WINDOW_CHANGED", "The selected window has no usable bounds");
|
||||
return new(rectangle, dpi);
|
||||
if (dpi == 0 || rectangle.Width <= 0 || rectangle.Height <= 0) return false;
|
||||
geometry = new(rectangle, dpi);
|
||||
return true;
|
||||
}
|
||||
|
||||
internal void Observe(WindowGeometry value) => observation = value;
|
||||
@@ -56,58 +88,101 @@ internal sealed class WindowState(nint handle)
|
||||
|
||||
internal object Describe(AppState app, RequestContext context)
|
||||
{
|
||||
var geometry = Geometry(app, context);
|
||||
RequireCurrent(app, context);
|
||||
// A minimized or cloaked window may lack usable capture geometry; keep listing its association.
|
||||
var bounds = TryGeometry(out var geometry) ? geometry.Bounds : observation?.Bounds ?? new Bounds(0, 0, 0, 0);
|
||||
var length = Win32.GetWindowTextLength(Handle);
|
||||
if (length > context.Config.MaxResponseBytes) throw new NativeFailure("NATIVE_RESPONSE_LIMIT", "The window title exceeds the response limit");
|
||||
var title = new StringBuilder(length + 1);
|
||||
Win32.GetWindowText(Handle, title, title.Capacity);
|
||||
return new
|
||||
{
|
||||
windowId = Id.ToString(CultureInfo.InvariantCulture), pid = 4, appName = app.RequireProcess().Name,
|
||||
title = title.ToString(), bounds = geometry.Bounds, isOnScreen = IsOnScreen(), onCurrentSpace = (bool?)null,
|
||||
windowId = Id.ToString(CultureInfo.InvariantCulture), pid = 4, appName = app.Name,
|
||||
title = title.ToString(), bounds, isOnScreen = IsOnScreen(Handle), onCurrentSpace = (bool?)null,
|
||||
};
|
||||
}
|
||||
|
||||
internal bool IsOnScreen() => Win32.IsWindowVisible(Handle) && !Win32.IsIconic(Handle)
|
||||
&& (Win32.DwmGetWindowAttribute(Handle, 14, out int cloaked, sizeof(int)) != 0 || cloaked == 0);
|
||||
internal static bool IsOnScreen(nint handle) => Win32.IsWindowVisible(handle) && !Win32.IsIconic(handle)
|
||||
&& (Win32.DwmGetWindowAttribute(handle, 14, out int cloaked, sizeof(int)) != 0 || cloaked == 0);
|
||||
}
|
||||
|
||||
internal static class Windows
|
||||
{
|
||||
internal static WindowState[] Refresh(AppState app, RequestContext context)
|
||||
{
|
||||
var process = app.RequireProcess();
|
||||
if (app.SelectedWindow is { } selected) { selected.Geometry(app, context); return [selected]; }
|
||||
var handles = new List<nint>();
|
||||
Win32.EnumWindows((window, _) =>
|
||||
var observation = DesktopWindows.Observe(context);
|
||||
var current = new Dictionary<nint, WindowState>();
|
||||
var added = new List<WindowState>();
|
||||
foreach (var window in observation.Windows)
|
||||
{
|
||||
Win32.GetWindowThreadProcessId(window, out var pid);
|
||||
var style = Win32.GetWindowLongPtr(window, -16).ToInt64();
|
||||
var extended = Win32.GetWindowLongPtr(window, -20).ToInt64();
|
||||
if (pid == process.Pid && (extended & 0x80) == 0
|
||||
&& (Win32.IsWindowVisible(window) || (style & 0x00C00000) == 0x00C00000)
|
||||
&& Win32.GetWindowRect(window, out var bounds) && bounds.Width > 0 && bounds.Height > 0) handles.Add(window);
|
||||
return true;
|
||||
}, 0);
|
||||
foreach (var pair in app.Windows.ToArray())
|
||||
{
|
||||
if (handles.Contains(pair.Key)) continue;
|
||||
app.Windows.Remove(pair.Key);
|
||||
foreach (var scope in app.Snapshots.Where(item => item.Value.Windows.Any(window => window.Id == pair.Value.Id)).Select(item => item.Key).ToArray())
|
||||
app.Snapshots.Remove(scope);
|
||||
}
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
if (!app.Windows.ContainsKey(handle))
|
||||
if (!string.Equals(window.AppId, app.AppId, StringComparison.Ordinal)) continue;
|
||||
if (app.Windows.TryGetValue(window.Handle, out var existing)
|
||||
&& existing.ProcessId == window.ProcessId && existing.ProcessStarted == window.ProcessStarted)
|
||||
current[window.Handle] = existing;
|
||||
else
|
||||
{
|
||||
app.Windows.Add(handle, new(handle));
|
||||
var state = new WindowState(window.Handle, window.ProcessId, window.ProcessStarted);
|
||||
current[window.Handle] = state;
|
||||
added.Add(state);
|
||||
}
|
||||
}
|
||||
return handles.Select(handle => app.Windows[handle]).ToArray();
|
||||
foreach (var pair in app.Windows.ToArray())
|
||||
{
|
||||
if (current.TryGetValue(pair.Key, out var kept) && ReferenceEquals(kept, pair.Value)) continue;
|
||||
app.Windows.Remove(pair.Key);
|
||||
app.Snapshots.Remove(pair.Key);
|
||||
}
|
||||
foreach (var window in added) app.Windows[window.Handle] = window;
|
||||
return current.Values.OrderBy(window => (long)window.Handle).ToArray();
|
||||
}
|
||||
|
||||
internal static WindowState Resolve(AppState app, nint id, RequestContext context) => Refresh(app, context)
|
||||
.SingleOrDefault(window => window.Id == id) ?? throw new NativeFailure("WINDOW_CHANGED", "Select a current window from getWindows");
|
||||
internal static WindowState Resolve(AppState app, nint id, RequestContext context)
|
||||
{
|
||||
if (app.Windows.TryGetValue(id, out var known))
|
||||
{
|
||||
// A known target is revalidated in place; a changed or closed one is removed, never switched.
|
||||
known.Geometry(app, context);
|
||||
return known;
|
||||
}
|
||||
context.Check();
|
||||
if (!DesktopWindows.Eligible(id))
|
||||
throw new NativeFailure("WINDOW_CHANGED", "Select a current window from getWindows");
|
||||
var observed = DesktopWindows.ResolveWindow(id, context);
|
||||
if (observed is null || !string.Equals(observed.AppId, app.AppId, StringComparison.Ordinal))
|
||||
throw new NativeFailure("WINDOW_CHANGED", "The window does not belong to the authorized application; read fresh application state");
|
||||
// A newly appeared explicit window may join the application; no default target is inferred.
|
||||
app.Snapshots.Remove(id);
|
||||
var window = new WindowState(observed.Handle, observed.ProcessId, observed.ProcessStarted);
|
||||
app.Windows[id] = window;
|
||||
return window;
|
||||
}
|
||||
|
||||
// A sibling window that disappears during enumeration is omitted with its snapshots; a direct
|
||||
// selected target still fails through Resolve with a structured error.
|
||||
internal static (object[] Values, WindowState[] Windows) DescribeAll(AppState app, WindowState[] windows,
|
||||
RequestContext context)
|
||||
{
|
||||
var described = new List<object>();
|
||||
var surviving = new List<WindowState>();
|
||||
foreach (var window in windows)
|
||||
{
|
||||
context.Check();
|
||||
try
|
||||
{
|
||||
described.Add(window.Describe(app, context));
|
||||
surviving.Add(window);
|
||||
}
|
||||
catch (NativeFailure error) when (error.Code is "WINDOW_CHANGED" or "PERMISSION_DENIED")
|
||||
{
|
||||
if (app.Windows.TryGetValue(window.Handle, out var current) && ReferenceEquals(current, window))
|
||||
{
|
||||
app.Windows.Remove(window.Handle);
|
||||
app.Snapshots.Remove(window.Handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
return (described.ToArray(), surviving.ToArray());
|
||||
}
|
||||
|
||||
internal static WindowState? Focused(AppState app, WindowState[] windows)
|
||||
{
|
||||
@@ -130,25 +205,24 @@ internal static class Windows
|
||||
|
||||
internal static WindowState? Default(AppState app, WindowState[] windows)
|
||||
{
|
||||
var visible = windows.Where(window => window.IsOnScreen()).ToArray();
|
||||
var focused = Focused(app, visible);
|
||||
var onScreen = windows.Where(window => WindowState.IsOnScreen(window.Handle)).ToArray();
|
||||
var focused = Focused(app, onScreen);
|
||||
if (focused is not null) return focused;
|
||||
if (visible.Length == 1) return visible[0];
|
||||
if (visible.Length == 0 && windows.Length <= 1) return windows.SingleOrDefault();
|
||||
if (onScreen.Length == 1) return onScreen[0];
|
||||
if (onScreen.Length == 0 && windows.Length <= 1) return windows.SingleOrDefault();
|
||||
throw new NativeFailure("AMBIGUOUS_WINDOW", "Select a window explicitly from getWindows");
|
||||
}
|
||||
|
||||
internal static object Show(AppState app, nint? id, RequestContext context)
|
||||
{
|
||||
var target = id is { } selected ? Resolve(app, selected, context) : Default(app, Refresh(app, context));
|
||||
var target = id is { } selected
|
||||
? Resolve(app, selected, context)
|
||||
: (Default(app, Refresh(app, context)) ?? throw new NativeFailure("WINDOW_NOT_FOUND", "The application has no window to show"));
|
||||
var before = Win32.GetForegroundWindow();
|
||||
if (target is not null)
|
||||
{
|
||||
target.Geometry(app, context);
|
||||
if (!Win32.ShowWindowAsync(target.Handle, 4)) throw new NativeFailure("UNSUPPORTED", "Windows refused background restoration; explicitly activate the application if needed");
|
||||
}
|
||||
target.Geometry(app, context);
|
||||
if (!Win32.ShowWindowAsync(target.Handle, 4)) throw new NativeFailure("UNSUPPORTED", "Windows refused background restoration; explicitly activate the application if needed");
|
||||
var after = Win32.GetForegroundWindow();
|
||||
return new { dispatched = true, windowId = target?.Id.ToString(CultureInfo.InvariantCulture),
|
||||
return new { dispatched = true, windowId = target.Id.ToString(CultureInfo.InvariantCulture),
|
||||
foregroundChanged = before == 0 || after == 0 ? (bool?)null : before != after };
|
||||
}
|
||||
|
||||
@@ -161,15 +235,14 @@ internal static class Windows
|
||||
throw new NativeFailure("FOREGROUND_DENIED", "Windows refused foreground access; ask the user to focus the target window");
|
||||
}
|
||||
|
||||
internal static void RequireForeground(AppState app, WindowState? target, RequestContext context)
|
||||
internal static void RequireForeground(AppState app, WindowState target, RequestContext context)
|
||||
{
|
||||
context.Check();
|
||||
DesktopSession.RequireInteractive();
|
||||
target ??= app.SelectedWindow;
|
||||
target?.Geometry(app, context);
|
||||
target.Geometry(app, context);
|
||||
var foreground = Win32.GetForegroundWindow();
|
||||
Win32.GetWindowThreadProcessId(foreground, out var pid);
|
||||
if (pid != app.RequireProcess().Pid || target is not null && Win32.GetAncestor(foreground, 2) != target.Handle)
|
||||
if (pid != (uint)target.ProcessId || Win32.GetAncestor(foreground, 2) != target.Handle)
|
||||
throw new NativeFailure("FOREGROUND_REQUIRED", "This input needs the target in front; explain why, explicitly activate or raise it, then observe again");
|
||||
}
|
||||
}
|
||||
@@ -52,13 +52,11 @@ it.skipIf(!helperPath || !selector)('admits real application screenshots through
|
||||
image: (image) => { expect(image.bytes.byteLength).toBeGreaterThan(0) },
|
||||
})
|
||||
const discovered = (await api.listApps({ emit: false })).find(app =>
|
||||
app.id === selector || app.displayName.toLowerCase() === selector?.toLowerCase())
|
||||
expect(discovered).toBeDefined()
|
||||
const app = await api.getApp(discovered!.id)
|
||||
app.id === selector || app.displayName === selector)
|
||||
const app = await api.getApp(discovered?.id ?? selector!)
|
||||
const target = (await app.getWindows({ emit: false })).find(window =>
|
||||
window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0)
|
||||
expect(target).toBeDefined()
|
||||
expect(target!.windowId).toBe(discovered!.id)
|
||||
const window = await app.getWindow(target!.windowId)
|
||||
expect(emitted).toEqual([])
|
||||
await window.getAccessibilityTree()
|
||||
|
||||
@@ -65,7 +65,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
|
||||
},
|
||||
async listApps(options) {
|
||||
const result = AppsResult.parse(await call('listApps', {})).map(app => ({
|
||||
id: app.id ?? (app.bundleId || app.name), displayName: app.name, isRunning: true as const,
|
||||
id: app.id ?? (app.bundleId || app.name), displayName: app.name, isRunning: app.isRunning,
|
||||
}))
|
||||
await emit(sink, result, options)
|
||||
return result
|
||||
|
||||
@@ -10,10 +10,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched:
|
||||
|
||||
Setup and discovery:
|
||||
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
|
||||
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true }[] — enumerate running applications.
|
||||
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications.
|
||||
- cua.findApps(query: string, options?: { emit?: boolean }) -> { id: string, displayName: string, path: string, isRunning: boolean }[] — search installed applications, including stopped matches.
|
||||
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
|
||||
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
|
||||
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation.
|
||||
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app.
|
||||
- app.getWindows(options?: { emit?: boolean }) -> { windowId: string, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
|
||||
- app.getWindow(windowId: string) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
|
||||
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.
|
||||
|
||||
@@ -273,6 +273,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend)
|
||||
}).parse(result)
|
||||
await access.select(agent, client, prepared.appHandle, authorization, signal, applicationIcon)
|
||||
signal.throwIfAborted()
|
||||
// TODO: launchApp on Windows
|
||||
const selected = prepared.running
|
||||
? prepared
|
||||
: AppResult.parse(await access.run(client, prepared.appHandle, signal,
|
||||
|
||||
@@ -113,8 +113,10 @@ export const AppResult = z.object({
|
||||
appHandle: z.string().min(1).transform(value => value as CuaAppHandle),
|
||||
name: z.string(), bundleId: z.string().optional(), running: z.boolean(),
|
||||
})
|
||||
/** Running application metadata used for selection; other native fields are ignored. */
|
||||
export const AppsResult = z.array(z.object({ id: z.string().min(1).optional(), name: z.string(), bundleId: z.string().optional() }))
|
||||
/** Application discovery entries with the provider-reported running state. */
|
||||
export const AppsResult = z.array(z.object({
|
||||
id: z.string().min(1).optional(), name: z.string(), bundleId: z.string().optional(), isRunning: z.boolean(),
|
||||
}))
|
||||
/** Installed application matches and their current running state. */
|
||||
export const FoundAppsResult = z.array(z.object({
|
||||
id: z.string().min(1), displayName: z.string(), path: z.string().min(1), isRunning: z.boolean(),
|
||||
|
||||
@@ -50,8 +50,8 @@ export type CuaStartupCapabilities = Pick<
|
||||
CuaCapabilities,
|
||||
'platform' | 'methods' | 'appLevelAccessibility' | 'keyboard' | 'browser' | 'coordinateSpace'
|
||||
> & { systemPointer?: boolean }
|
||||
/** App discovery result accepted by getApp. */
|
||||
export type CuaAppInfo = Pick<CuaFoundAppInfo, 'id' | 'displayName'> & { isRunning: true }
|
||||
/** App discovery result accepted by getApp, including the provider-reported running state. */
|
||||
export type CuaAppInfo = Pick<CuaFoundAppInfo, 'id' | 'displayName' | 'isRunning'>
|
||||
/** An installed application match with its current running state. */
|
||||
export type CuaFoundAppInfo = z.output<typeof FoundAppsResult>[number]
|
||||
/** Window bounds in the provider's global native units. Local operations use an origin of [0,0]. */
|
||||
@@ -256,9 +256,9 @@ export interface CuaApi {
|
||||
*/
|
||||
getCapabilities(options?: CuaObservationOptions): Promise<CuaCapabilities>
|
||||
/**
|
||||
* Enumerate running applications.
|
||||
* List applications with their current running state. Providers may enumerate only running applications.
|
||||
* @param options - output options.
|
||||
* @returns identifiers accepted by getApp.
|
||||
* @returns identifiers accepted by getApp and each application's running state.
|
||||
*/
|
||||
listApps(options?: CuaObservationOptions): Promise<CuaAppInfo[]>
|
||||
/**
|
||||
@@ -269,14 +269,14 @@ export interface CuaApi {
|
||||
*/
|
||||
findApps(query: string, options?: CuaObservationOptions): Promise<CuaFoundAppInfo[]>
|
||||
/**
|
||||
* Discover apps.
|
||||
* Summarize provider application discovery.
|
||||
* @param options - output options.
|
||||
* @returns native apps and an empty browser list.
|
||||
* @returns native apps with their running state and an empty browser list.
|
||||
*/
|
||||
getState(options?: CuaObservationOptions): Promise<{ apps: CuaAppInfo[]; browsers: [] }>
|
||||
/**
|
||||
* Select an app, obtain consent, launch it in the background if necessary, and silently retain initial AX state.
|
||||
* @param selector - Provider-issued identifier, exact name, bundle ID or absolute application path; ambiguous selectors reject.
|
||||
* Select an app, obtain consent, and launch it only where the provider supports launch; selection never activates it.
|
||||
* @param selector - Provider-issued ID, or a supported exact name, bundle ID or absolute path; ambiguous selectors reject.
|
||||
* @returns An application handle tied to this process lifetime; its initial tree read becomes the current snapshot.
|
||||
*/
|
||||
getApp(selector: string): Promise<CuaApp>
|
||||
|
||||
@@ -11,7 +11,7 @@ const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
|
||||
|
||||
const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
|
||||
const bounds = { x: -640, y: 100, width: 640, height: 480 }
|
||||
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', running: true }
|
||||
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', isRunning: true, running: true }
|
||||
const delivered = { dispatched: true }
|
||||
const capabilities: CuaCapabilities = { platform: 'macos', systemPointer: true, sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
|
||||
|
||||
@@ -27,7 +27,7 @@ function fixture(withSink = true) {
|
||||
const dispatch = vi.fn(async ({ method, params }: CuaOperation): Promise<JsonValue> => {
|
||||
switch (method) {
|
||||
case 'getCapabilities': return currentCapabilities
|
||||
case 'listApps': return [selected, { id: 'utility-selector', pid: 4, name: 'Utility' }]
|
||||
case 'listApps': return [selected, { id: 'utility-selector', pid: 4, name: 'Utility', isRunning: true }]
|
||||
case 'getApp': return selected
|
||||
case 'listWindows': return windows
|
||||
case 'mouseMove': return { ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false }
|
||||
@@ -81,8 +81,8 @@ describe('application discovery', () => {
|
||||
|
||||
it('uses the application name when discovery has no identifier', async () => {
|
||||
const f = fixture()
|
||||
f.dispatch.mockResolvedValueOnce([{ pid: 18, name: 'Utility' }])
|
||||
expect(await f.api.listApps({ emit: false })).toEqual([{ id: 'Utility', displayName: 'Utility', isRunning: true }])
|
||||
f.dispatch.mockResolvedValueOnce([{ pid: 18, name: 'Utility', isRunning: false }])
|
||||
expect(await f.api.listApps({ emit: false })).toEqual([{ id: 'Utility', displayName: 'Utility', isRunning: false }])
|
||||
})
|
||||
|
||||
it('returns observations without requiring an output sink', async () => {
|
||||
|
||||
@@ -56,7 +56,7 @@ it('propagates a denied action without retrying it', async () => {
|
||||
})
|
||||
|
||||
it('supports callers that retain observations without a text callback', async () => {
|
||||
const cua = connectCua(() => async () => [{ id: 'editor-selector', name: 'Editor' }])
|
||||
const cua = connectCua(() => async () => [{ id: 'editor-selector', name: 'Editor', isRunning: true }])
|
||||
await expect(cua.listApps()).resolves.toEqual([{ id: 'editor-selector', displayName: 'Editor', isRunning: true }])
|
||||
})
|
||||
|
||||
|
||||
@@ -43,10 +43,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched:
|
||||
|
||||
Setup and discovery:
|
||||
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
|
||||
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications.
|
||||
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications.
|
||||
- cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid.
|
||||
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
|
||||
- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
|
||||
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation.
|
||||
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app.
|
||||
- app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
|
||||
- app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
|
||||
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.
|
||||
|
||||
@@ -131,7 +131,7 @@ function createApplicationState(socket, record, application, captureBroker, conn
|
||||
return { platform: 'darwin', ...state, helper: { pid: process.pid }, parent: { pid: process.ppid } }
|
||||
}
|
||||
if (method === 'getCapabilities') return { sessionLocked: false, permissions: { accessibility: true, screenRecording: true } }
|
||||
if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId }]
|
||||
if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId, isRunning: true }]
|
||||
if (method === 'getApp') {
|
||||
if (params.selector === 'Denied Fixture' || params.selector === 'Revocable Fixture') {
|
||||
const denied = params.selector === 'Denied Fixture'
|
||||
|
||||
@@ -45,10 +45,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched:
|
||||
|
||||
Setup and discovery:
|
||||
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
|
||||
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications.
|
||||
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications.
|
||||
- cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid.
|
||||
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
|
||||
- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
|
||||
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation.
|
||||
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app.
|
||||
- app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
|
||||
- app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
|
||||
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.
|
||||
|
||||
@@ -45,10 +45,10 @@ Signatures show resolved return values after await. ActionResult = { dispatched:
|
||||
|
||||
Setup and discovery:
|
||||
- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions.
|
||||
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications.
|
||||
- cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: boolean }[] — list applications the provider exposes for selection with their current running state; some providers enumerate only running applications.
|
||||
- cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid.
|
||||
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation.
|
||||
- cua.getApp(selector: string) -> App — select an application by name, bundle ID, absolute path, or running PID string, obtain consent, launch in the background if needed, and read initial AX state silently; it does not activate the app.
|
||||
- cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: boolean }[], browsers: [] } — summarize the same provider discovery; this is not an accessibility observation.
|
||||
- cua.getApp(selector: string) -> App — select an application by provider-issued discovery ID, name, bundle ID, or absolute path where supported, obtain consent, start a stopped application in the background only where the provider supports launch, and read initial AX state silently when that scope is supported; it does not activate the app.
|
||||
- app.getWindows(options?: { emit?: boolean }) -> { windowId: number, appName: string, title: string, bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null }[] — enumerate windows with global native bounds; null onCurrentSpace means unknown Space membership.
|
||||
- app.getWindow(windowId: number) -> Window — select one window and read its initial AX state silently; do this before any window screenshot or coordinate input.
|
||||
- app.activate() -> ActionResult — request explicit foreground activation; use only when foreground access is required.
|
||||
|
||||
Reference in new issue
Block a user