diff --git a/packages/experimental/cua-windows/native/entry/Accessibility.cs b/packages/experimental/cua-windows/native/entry/Accessibility.cs index 5d86f8bf89..cc47d8acac 100644 --- a/packages/experimental/cua-windows/native/entry/Accessibility.cs +++ b/packages/experimental/cua-windows/native/entry/Accessibility.cs @@ -15,12 +15,9 @@ internal static class Accessibility private static readonly AutomationProperty[] Properties = [ AutomationElement.ControlTypeProperty, AutomationElement.NameProperty, AutomationElement.HelpTextProperty, AutomationElement.AutomationIdProperty, AutomationElement.IsEnabledProperty, - AutomationElement.HasKeyboardFocusProperty, AutomationElement.IsKeyboardFocusableProperty, AutomationElement.IsPasswordProperty, - AutomationElement.BoundingRectangleProperty, AutomationElement.IsInvokePatternAvailableProperty, - AutomationElement.IsValuePatternAvailableProperty, AutomationElement.IsRangeValuePatternAvailableProperty, - AutomationElement.IsTextPatternAvailableProperty, AutomationElement.IsSelectionItemPatternAvailableProperty, - AutomationElement.IsTogglePatternAvailableProperty, AutomationElement.IsExpandCollapsePatternAvailableProperty, - AutomationElement.IsScrollItemPatternAvailableProperty, ValuePattern.ValueProperty, ValuePattern.IsReadOnlyProperty, + AutomationElement.HasKeyboardFocusProperty, AutomationElement.IsPasswordProperty, + AutomationElement.BoundingRectangleProperty, AutomationElement.IsTextPatternAvailableProperty, + ValuePattern.ValueProperty, ValuePattern.IsReadOnlyProperty, RangeValuePattern.ValueProperty, RangeValuePattern.IsReadOnlyProperty, SelectionItemPattern.IsSelectedProperty, TogglePattern.ToggleStateProperty, ExpandCollapsePattern.ExpandCollapseStateProperty, ]; @@ -37,7 +34,7 @@ internal static class Accessibility var observations = new List<(WindowState Window, WindowGeometry Geometry)>(); var visited = new HashSet(StringComparer.Ordinal); var cache = new CacheRequest { AutomationElementMode = AutomationElementMode.Full, TreeScope = TreeScope.Element }; - foreach (var property in Properties) cache.Add(property); + foreach (var property in Properties.Concat(PatternActions.Patterns.Select(pattern => pattern.Available))) cache.Add(property); var truncated = false; var count = 0; var nextIndex = 0; @@ -125,14 +122,7 @@ internal static class Accessibility private static AccessibilityNode Read(AutomationElement element, WindowState window, WindowGeometry geometry, RequestContext context) { var type = Property(element, AutomationElement.ControlTypeProperty) as ControlType; - var actions = new List(); - if (Flag(element, AutomationElement.IsInvokePatternAvailableProperty)) actions.Add("Press"); - if (Flag(element, AutomationElement.IsTogglePatternAvailableProperty)) actions.Add("Toggle"); - if (Flag(element, AutomationElement.IsSelectionItemPatternAvailableProperty)) actions.Add("Pick"); - if (Flag(element, AutomationElement.IsExpandCollapsePatternAvailableProperty)) { actions.Add("Expand"); actions.Add("Collapse"); } - if (Flag(element, AutomationElement.IsScrollItemPatternAvailableProperty)) actions.Add("ScrollToVisible"); - if (Flag(element, AutomationElement.IsKeyboardFocusableProperty)) actions.Add("Focus"); - if (type == ControlType.Window) actions.Add("Raise"); + var actions = PatternActions.Names(pattern => Flag(element, pattern.Available)); var password = Flag(element, AutomationElement.IsPasswordProperty); string? value = null; var hasValue = false; @@ -187,7 +177,7 @@ internal static class Accessibility Enabled = Property(element, AutomationElement.IsEnabledProperty) as bool?, Focused = Property(element, AutomationElement.HasKeyboardFocusProperty) as bool?, Selected = Property(element, SelectionItemPattern.IsSelectedProperty) as bool?, Expanded = expanded, HasSelection = hasSelection, Selection = selection, SelectedText = selectedText, SelectionWritable = selectionWritable, - Actions = actions.ToArray(), Bounds = bounds, + Actions = actions, Bounds = bounds, }; } diff --git a/packages/experimental/cua-windows/native/entry/AccessibilityActions.cs b/packages/experimental/cua-windows/native/entry/AccessibilityActions.cs index ae21c9610f..4d41b456ae 100644 --- a/packages/experimental/cua-windows/native/entry/AccessibilityActions.cs +++ b/packages/experimental/cua-windows/native/entry/AccessibilityActions.cs @@ -19,23 +19,23 @@ internal static class AccessibilityActions internal static string[] Actions(ElementReference reference, RequestContext context) { - var element = reference.Element; - var actions = new List(); - if (context.Ax(() => element.TryGetCurrentPattern(InvokePattern.Pattern, out _))) actions.Add("Press"); - if (context.Ax(() => element.TryGetCurrentPattern(TogglePattern.Pattern, out _))) actions.Add("Toggle"); - if (context.Ax(() => element.TryGetCurrentPattern(SelectionItemPattern.Pattern, out _))) actions.Add("Pick"); - if (context.Ax(() => element.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out _))) { actions.Add("Expand"); actions.Add("Collapse"); } - if (context.Ax(() => element.TryGetCurrentPattern(ScrollItemPattern.Pattern, out _))) actions.Add("ScrollToVisible"); - if (context.Ax(() => element.Current.IsKeyboardFocusable)) actions.Add("Focus"); - if (context.Ax(() => element.Current.ControlType) == ControlType.Window) actions.Add("Raise"); - return actions.ToArray(); + var supported = context.Ax(reference.Element.GetSupportedPatterns).ToHashSet(); + return PatternActions.Names(pattern => supported.Contains(pattern.Pattern)); } - internal static void Perform(AppState app, ElementReference reference, string action, JsonElement? argument, RequestContext context) + internal static void Perform(ElementReference reference, string action, JsonElement? argument, RequestContext context) { - if (argument is { } value && value.ValueKind is not (JsonValueKind.Null or JsonValueKind.Undefined)) - throw new NativeFailure("INVALID_ARGUMENT", "This accessibility action does not accept an argument"); - Secondary(app, reference, action, context); + var operation = PatternActions.Resolve(action, argument); + var pattern = Pattern(reference.Element, operation.Pattern, context); + try { context.Ax(() => { operation.Invoke(pattern); return true; }); } + catch (ArgumentException) + { + throw new NativeFailure("INVALID_ARGUMENT", $"The control rejected the argument for {action}; check its accepted values before retrying"); + } + catch (InvalidOperationException error) when (error is not ElementNotEnabledException) + { + throw new NativeFailure("UNSUPPORTED", $"The control could not complete {action} in its current state; read fresh state before further input"); + } } internal static bool Invoke(ElementReference reference, RequestContext context) @@ -90,40 +90,6 @@ internal static class AccessibilityActions context.Ax(() => { range.SetValue(number); return true; }); } - internal static void Secondary(AppState app, ElementReference reference, string action, RequestContext context) - { - switch (action.ToLowerInvariant()) - { - case "press": - context.Ax(() => { Pattern(reference.Element, InvokePattern.Pattern, context).Invoke(); return true; }); - break; - case "pick": - context.Ax(() => { Pattern(reference.Element, SelectionItemPattern.Pattern, context).Select(); return true; }); - break; - case "toggle": - context.Ax(() => { Pattern(reference.Element, TogglePattern.Pattern, context).Toggle(); return true; }); - break; - case "expand": - context.Ax(() => { Pattern(reference.Element, ExpandCollapsePattern.Pattern, context).Expand(); return true; }); - break; - case "collapse": - context.Ax(() => { Pattern(reference.Element, ExpandCollapsePattern.Pattern, context).Collapse(); return true; }); - break; - case "scrolltovisible": - context.Ax(() => { Pattern(reference.Element, ScrollItemPattern.Pattern, context).ScrollIntoView(); return true; }); - break; - case "focus": - context.Ax(() => { reference.Element.SetFocus(); return true; }); - break; - case "raise": - if (context.Ax(() => reference.Element.Current.ControlType) != ControlType.Window) - throw new NativeFailure("UNSUPPORTED", "Raise is available only on the observed window element"); - Windows.Activate(app, reference.Window, context); - break; - default: throw new NativeFailure("UNSUPPORTED", "Use an action name from a fresh details:true observation"); - } - } - internal static void SelectText(ElementReference reference, string text, string? prefix, string? suffix, string selectionType, RequestContext context) { diff --git a/packages/experimental/cua-windows/native/entry/DesktopExecutor.cs b/packages/experimental/cua-windows/native/entry/DesktopExecutor.cs index c4db2601c3..20b2bac160 100644 --- a/packages/experimental/cua-windows/native/entry/DesktopExecutor.cs +++ b/packages/experimental/cua-windows/native/entry/DesktopExecutor.cs @@ -213,7 +213,7 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA await mta.Invoke(() => { var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context); - AccessibilityActions.Perform(app, element, String(parameters, "action"), argument, context); + AccessibilityActions.Perform(element, String(parameters, "action"), argument, context); return true; }, cancellation); return ActionResult(foreground); diff --git a/packages/experimental/cua-windows/native/entry/PatternActions.cs b/packages/experimental/cua-windows/native/entry/PatternActions.cs new file mode 100644 index 0000000000..771476c05a --- /dev/null +++ b/packages/experimental/cua-windows/native/entry/PatternActions.cs @@ -0,0 +1,150 @@ +// Cached observations and live dispatch share the same UIA operation definitions. +using System.Text.Json; +using System.Windows.Automation; +using DeepSeek.Cua.Protocol; + +namespace DeepSeek.Cua.Windows; + +internal sealed record PatternAction(string Name, Func> Bind); +internal sealed record PatternOperations(AutomationPattern Pattern, AutomationProperty Available, PatternAction[] Actions); + +internal static class PatternActions +{ + internal static readonly PatternOperations[] Patterns = [ + new(InvokePattern.Pattern, AutomationElement.IsInvokePatternAvailableProperty, [ + NoArgument(nameof(InvokePattern.Invoke), pattern => pattern.Invoke()), + ]), + new(TogglePattern.Pattern, AutomationElement.IsTogglePatternAvailableProperty, [ + NoArgument(nameof(TogglePattern.Toggle), pattern => pattern.Toggle()), + ]), + new(SelectionItemPattern.Pattern, AutomationElement.IsSelectionItemPatternAvailableProperty, [ + NoArgument(nameof(SelectionItemPattern.Select), pattern => pattern.Select()), + NoArgument(nameof(SelectionItemPattern.AddToSelection), pattern => pattern.AddToSelection()), + NoArgument(nameof(SelectionItemPattern.RemoveFromSelection), pattern => pattern.RemoveFromSelection()), + ]), + new(ExpandCollapsePattern.Pattern, AutomationElement.IsExpandCollapsePatternAvailableProperty, [ + NoArgument(nameof(ExpandCollapsePattern.Expand), pattern => pattern.Expand()), + NoArgument(nameof(ExpandCollapsePattern.Collapse), pattern => pattern.Collapse()), + ]), + new(ScrollItemPattern.Pattern, AutomationElement.IsScrollItemPatternAvailableProperty, [ + NoArgument(nameof(ScrollItemPattern.ScrollIntoView), pattern => pattern.ScrollIntoView()), + ]), + new(ValuePattern.Pattern, AutomationElement.IsValuePatternAvailableProperty, [ + WithArgument(nameof(ValuePattern.SetValue), String, (pattern, value) => pattern.SetValue(value)), + ]), + new(RangeValuePattern.Pattern, AutomationElement.IsRangeValuePatternAvailableProperty, [ + WithArgument(nameof(RangeValuePattern.SetValue), Number, (pattern, value) => pattern.SetValue(value)), + ]), + new(ScrollPattern.Pattern, AutomationElement.IsScrollPatternAvailableProperty, [ + WithArgument(nameof(ScrollPattern.Scroll), + value => Pair(value, "horizontalAmount", "verticalAmount", EnumValue), + (pattern, amount) => pattern.Scroll(amount.X, amount.Y)), + WithArgument(nameof(ScrollPattern.ScrollHorizontal), EnumValue, + (pattern, amount) => pattern.ScrollHorizontal(amount)), + WithArgument(nameof(ScrollPattern.ScrollVertical), EnumValue, + (pattern, amount) => pattern.ScrollVertical(amount)), + WithArgument(nameof(ScrollPattern.SetScrollPercent), + value => Pair(value, "horizontalPercent", "verticalPercent", Percent), + (pattern, percent) => pattern.SetScrollPercent(percent.X, percent.Y)), + ]), + new(TransformPattern.Pattern, AutomationElement.IsTransformPatternAvailableProperty, [ + WithArgument(nameof(TransformPattern.Move), + value => Pair(value, "x", "y", Number), (pattern, point) => pattern.Move(point.X, point.Y)), + WithArgument(nameof(TransformPattern.Resize), + value => Pair(value, "width", "height", PositiveNumber), (pattern, size) => pattern.Resize(size.Width, size.Height)), + WithArgument(nameof(TransformPattern.Rotate), Number, (pattern, degrees) => pattern.Rotate(degrees)), + ]), + new(WindowPattern.Pattern, AutomationElement.IsWindowPatternAvailableProperty, [ + NoArgument(nameof(WindowPattern.Close), pattern => pattern.Close()), + WithArgument(nameof(WindowPattern.SetWindowVisualState), EnumValue, + (pattern, state) => pattern.SetWindowVisualState(state)), + ]), + new(DockPattern.Pattern, AutomationElement.IsDockPatternAvailableProperty, [ + WithArgument(nameof(DockPattern.SetDockPosition), EnumValue, + (pattern, position) => pattern.SetDockPosition(position)), + ]), + new(MultipleViewPattern.Pattern, AutomationElement.IsMultipleViewPatternAvailableProperty, [ + WithArgument(nameof(MultipleViewPattern.SetCurrentView), Integer, (pattern, view) => pattern.SetCurrentView(view)), + ]), + new(VirtualizedItemPattern.Pattern, AutomationElement.IsVirtualizedItemPatternAvailableProperty, [ + NoArgument(nameof(VirtualizedItemPattern.Realize), pattern => pattern.Realize()), + ]), + new(SynchronizedInputPattern.Pattern, AutomationElement.IsSynchronizedInputPatternAvailableProperty, [ + WithArgument(nameof(SynchronizedInputPattern.StartListening), EnumValue, + (pattern, inputType) => pattern.StartListening(inputType)), + NoArgument(nameof(SynchronizedInputPattern.Cancel), pattern => pattern.Cancel()), + ]), + ]; + + internal static string[] Names(Func supported) => Patterns + .Where(supported).SelectMany(pattern => pattern.Actions.Select(action => action.Name)).ToArray(); + + // Bind validates the complete JSON argument before any UIA operation is attempted. + internal static (AutomationPattern Pattern, Action Invoke) Resolve(string name, JsonElement? argument) + { + foreach (var pattern in Patterns) + foreach (var action in pattern.Actions) + if (action.Name == name) return (pattern.Pattern, action.Bind(argument)); + throw new NativeFailure("UNSUPPORTED", "Use an exact Pattern.Action name from a fresh observation or listActions"); + } + + private static PatternAction NoArgument(string method, Action invoke) where T : BasePattern => + WithArgument(method, value => + { + if (value is { ValueKind: not (JsonValueKind.Null or JsonValueKind.Undefined) }) + throw Invalid("omit argument or pass null for this no-argument action"); + return true; + }, (pattern, _) => invoke(pattern)); + + // Single parameters are direct JSON values; multiple parameters use their native parameter names. + private static PatternAction WithArgument(string method, Func parse, Action invoke) + where T : BasePattern + { + var name = typeof(T).Name[..^"Pattern".Length] + "." + method; + return new(name, argument => + { + var value = parse(argument); + return pattern => invoke((T)pattern, value); + }); + } + + private static NativeFailure Invalid(string expected) => new("INVALID_ARGUMENT", "Pattern action argument: " + expected); + + private static string String(JsonElement? value) => value is { ValueKind: JsonValueKind.String } text + ? text.GetString()! : throw Invalid("pass a string"); + + private static double Number(JsonElement? value) => value is { ValueKind: JsonValueKind.Number } number + && number.TryGetDouble(out var result) && double.IsFinite(result) ? result : throw Invalid("pass a finite number"); + + private static int Integer(JsonElement? value) => value is { ValueKind: JsonValueKind.Number } number + && number.TryGetInt32(out var result) ? result : throw Invalid("pass a 32-bit integer viewId"); + + private static double PositiveNumber(JsonElement? value) + { + var number = Number(value); + return number > 0 ? number : throw Invalid("width and height must be positive"); + } + + private static double Percent(JsonElement? value) + { + var number = Number(value); + return number == ScrollPattern.NoScroll || number is >= 0 and <= 100 + ? number : throw Invalid("pass a scroll percentage from 0 to 100, or -1 to leave that axis unchanged"); + } + + private static T EnumValue(JsonElement? value) where T : struct, Enum + { + if (value is { ValueKind: JsonValueKind.String } text && Enum.TryParse(text.GetString(), out var result) + && Enum.IsDefined(result) && result.ToString() == text.GetString()) return result; + throw Invalid("pass one of: " + string.Join(", ", Enum.GetNames())); + } + + private static (T, T) Pair(JsonElement? value, string first, string second, Func parse) + { + if (value is not { ValueKind: JsonValueKind.Object } fields + || !fields.TryGetProperty(first, out var a) || !fields.TryGetProperty(second, out var b) + || fields.EnumerateObject().Count() != 2) + throw Invalid($"pass an object containing exactly {first} and {second}"); + return (parse(a), parse(b)); + } +} diff --git a/packages/experimental/cua-windows/native/protocol/AccessibilityText.cs b/packages/experimental/cua-windows/native/protocol/AccessibilityText.cs index 82884d1b22..39605ef2e6 100644 --- a/packages/experimental/cua-windows/native/protocol/AccessibilityText.cs +++ b/packages/experimental/cua-windows/native/protocol/AccessibilityText.cs @@ -58,8 +58,7 @@ public static class AccessibilityText else text.Append(" selection=unavailable"); if (node.SelectedText is { Length: > 0 } selectedText) text.Append(" selectedText=").Append(Quote(selectedText)); } - var actions = details ? node.Actions : node.Actions.Where(action => action is not ("Press" or "Pick" or "ScrollToVisible")).ToArray(); - if (actions.Length != 0) text.Append(" actions=[").Append(string.Join(", ", actions.Select(Quote))).Append(']'); + if (node.Actions.Length != 0) text.Append(" actions=[").Append(string.Join(", ", node.Actions.Select(Quote))).Append(']'); if (details) { if (node.Identifier is { Length: > 0 } identifier) text.Append(" id=").Append(Quote(identifier)); diff --git a/packages/experimental/cua-windows/native/tests/Program.cs b/packages/experimental/cua-windows/native/tests/Program.cs index 602e733cf1..41031c2f6c 100644 --- a/packages/experimental/cua-windows/native/tests/Program.cs +++ b/packages/experimental/cua-windows/native/tests/Program.cs @@ -65,9 +65,9 @@ Check(KeyChord.Parse("Control+a", ["ctrl"]).Modifiers.SequenceEqual(["control"]) Refuses(() => KeyChord.Parse("ctrl+", []), "INVALID_ARGUMENT"); Refuses(() => KeyChord.Parse("fn+a", []), "UNSUPPORTED"); -var node = new AccessibilityNode { ControlType = "Edit", Name = "Title", Value = "", HasValue = true, Placeholder = "Title", ValueWritable = "true", Actions = ["Press", "Focus"] }; +var node = new AccessibilityNode { ControlType = "Edit", Name = "Title", Value = "", HasValue = true, Placeholder = "Title", ValueWritable = "true", Actions = ["Invoke.Invoke", "Value.SetValue"] }; var text = AccessibilityText.Format(node, 3, false); -Check(text == "- [3] Edit valueWritable=true \"Title\" = \"\" placeholder=\"Title\" actions=[\"Focus\"]", +Check(text == "- [3] Edit valueWritable=true \"Title\" = \"\" placeholder=\"Title\" actions=[\"Invoke.Invoke\", \"Value.SetValue\"]", "Rows expose the native ControlType name without a subrole"); foreach (var controlType in new[] { "Group", "Other", "ListItem", "TreeItem", "TabItem", "Table", "DataGrid" }) Check(AccessibilityText.Format(new() { ControlType = controlType }, null, false) == "- " + controlType, @@ -75,10 +75,16 @@ foreach (var controlType in new[] { "Group", "Other", "ListItem", "TreeItem", "T Check(!AccessibilityText.Retain(new() { ControlType = "Other" }), "Empty fallback containers can collapse"); Check(AccessibilityText.Retain(new() { ControlType = "Other", Name = "Custom content" }), "Named fallback containers retain their row"); Check(text.Contains("= \"\" placeholder=\"Title\""), "Empty values and placeholders remain distinct"); -Check(!text.Contains("\"Press\"") && text.Contains("\"Focus\""), "Common actions compact without hiding secondary actions"); -Check(AccessibilityText.Format(node, 3, true).Contains("\"Press\""), "Details retain complete action names"); +Check(AccessibilityText.Format(node, 3, true) == text, "Both detail modes expose every action"); +var selectionActions = new AccessibilityNode { ControlType = "ListItem", Actions = [ + "SelectionItem.Select", "SelectionItem.AddToSelection", "SelectionItem.RemoveFromSelection", + "ExpandCollapse.Expand", "ExpandCollapse.Collapse", "ScrollItem.ScrollIntoView", +] }; +foreach (var details in new[] { false, true }) + Check(AccessibilityText.Format(selectionActions, 0, details) == "- [0] ListItem actions=[\"SelectionItem.Select\", \"SelectionItem.AddToSelection\", \"SelectionItem.RemoveFromSelection\", \"ExpandCollapse.Expand\", \"ExpandCollapse.Collapse\", \"ScrollItem.ScrollIntoView\"]", + "Multi-action patterns retain all operations in tree output"); Check(!AccessibilityText.Retain(new() { ControlType = "Group" }), "Empty structural containers can collapse"); -Check(AccessibilityText.Retain(new() { ControlType = "Group", Actions = ["Copy"] }), "Actionable containers retain their row"); +Check(AccessibilityText.Retain(new() { ControlType = "Group", Actions = ["Value.SetValue"] }), "Actionable containers retain their row"); Check(AccessibilityText.Format(new() { ControlType = "Document", Value = "abc", HasValue = true, HasSelection = true, Selection = new(2, 5) }, 0, false) .Contains("selection=unavailable"), "Out-of-range selections do not describe valid text"); Console.WriteLine($"{checks} portable CUA checks passed");