feat(cua-windows): expose native pattern actions

packages/experimental/cua-windows/native/entry/Accessibility.cs

@@ -15,12 +15,9 @@ internal static class Accessibility
     private static readonly AutomationProperty[] Properties = [
         AutomationElement.ControlTypeProperty, AutomationElement.NameProperty, AutomationElement.HelpTextProperty,
         AutomationElement.AutomationIdProperty, AutomationElement.IsEnabledProperty,
-        AutomationElement.HasKeyboardFocusProperty, AutomationElement.IsKeyboardFocusableProperty, AutomationElement.IsPasswordProperty,
-        AutomationElement.BoundingRectangleProperty, AutomationElement.IsInvokePatternAvailableProperty,
-        AutomationElement.IsValuePatternAvailableProperty, AutomationElement.IsRangeValuePatternAvailableProperty,
-        AutomationElement.IsTextPatternAvailableProperty, AutomationElement.IsSelectionItemPatternAvailableProperty,
-        AutomationElement.IsTogglePatternAvailableProperty, AutomationElement.IsExpandCollapsePatternAvailableProperty,
-        AutomationElement.IsScrollItemPatternAvailableProperty, ValuePattern.ValueProperty, ValuePattern.IsReadOnlyProperty,
+        AutomationElement.HasKeyboardFocusProperty, AutomationElement.IsPasswordProperty,
+        AutomationElement.BoundingRectangleProperty, AutomationElement.IsTextPatternAvailableProperty,
+        ValuePattern.ValueProperty, ValuePattern.IsReadOnlyProperty,
         RangeValuePattern.ValueProperty, RangeValuePattern.IsReadOnlyProperty, SelectionItemPattern.IsSelectedProperty,
         TogglePattern.ToggleStateProperty, ExpandCollapsePattern.ExpandCollapseStateProperty,
     ];
@@ -37,7 +34,7 @@ internal static class Accessibility
         var observations = new List<(WindowState Window, WindowGeometry Geometry)>();
         var visited = new HashSet<string>(StringComparer.Ordinal);
         var cache = new CacheRequest { AutomationElementMode = AutomationElementMode.Full, TreeScope = TreeScope.Element };
-        foreach (var property in Properties) cache.Add(property);
+        foreach (var property in Properties.Concat(PatternActions.Patterns.Select(pattern => pattern.Available))) cache.Add(property);
         var truncated = false;
         var count = 0;
         var nextIndex = 0;
@@ -125,14 +122,7 @@ internal static class Accessibility
     private static AccessibilityNode Read(AutomationElement element, WindowState window, WindowGeometry geometry, RequestContext context)
     {
         var type = Property(element, AutomationElement.ControlTypeProperty) as ControlType;
-        var actions = new List<string>();
-        if (Flag(element, AutomationElement.IsInvokePatternAvailableProperty)) actions.Add("Press");
-        if (Flag(element, AutomationElement.IsTogglePatternAvailableProperty)) actions.Add("Toggle");
-        if (Flag(element, AutomationElement.IsSelectionItemPatternAvailableProperty)) actions.Add("Pick");
-        if (Flag(element, AutomationElement.IsExpandCollapsePatternAvailableProperty)) { actions.Add("Expand"); actions.Add("Collapse"); }
-        if (Flag(element, AutomationElement.IsScrollItemPatternAvailableProperty)) actions.Add("ScrollToVisible");
-        if (Flag(element, AutomationElement.IsKeyboardFocusableProperty)) actions.Add("Focus");
-        if (type == ControlType.Window) actions.Add("Raise");
+        var actions = PatternActions.Names(pattern => Flag(element, pattern.Available));
         var password = Flag(element, AutomationElement.IsPasswordProperty);
         string? value = null;
         var hasValue = false;
@@ -187,7 +177,7 @@ internal static class Accessibility
             Enabled = Property(element, AutomationElement.IsEnabledProperty) as bool?, Focused = Property(element, AutomationElement.HasKeyboardFocusProperty) as bool?,
             Selected = Property(element, SelectionItemPattern.IsSelectedProperty) as bool?, Expanded = expanded,
             HasSelection = hasSelection, Selection = selection, SelectedText = selectedText, SelectionWritable = selectionWritable,
-            Actions = actions.ToArray(), Bounds = bounds,
+            Actions = actions, Bounds = bounds,
         };
     }
 

packages/experimental/cua-windows/native/entry/AccessibilityActions.cs

@@ -19,23 +19,23 @@ internal static class AccessibilityActions
 
     internal static string[] Actions(ElementReference reference, RequestContext context)
     {
-        var element = reference.Element;
-        var actions = new List<string>();
-        if (context.Ax(() => element.TryGetCurrentPattern(InvokePattern.Pattern, out _))) actions.Add("Press");
-        if (context.Ax(() => element.TryGetCurrentPattern(TogglePattern.Pattern, out _))) actions.Add("Toggle");
-        if (context.Ax(() => element.TryGetCurrentPattern(SelectionItemPattern.Pattern, out _))) actions.Add("Pick");
-        if (context.Ax(() => element.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out _))) { actions.Add("Expand"); actions.Add("Collapse"); }
-        if (context.Ax(() => element.TryGetCurrentPattern(ScrollItemPattern.Pattern, out _))) actions.Add("ScrollToVisible");
-        if (context.Ax(() => element.Current.IsKeyboardFocusable)) actions.Add("Focus");
-        if (context.Ax(() => element.Current.ControlType) == ControlType.Window) actions.Add("Raise");
-        return actions.ToArray();
+        var supported = context.Ax(reference.Element.GetSupportedPatterns).ToHashSet();
+        return PatternActions.Names(pattern => supported.Contains(pattern.Pattern));
     }
 
-    internal static void Perform(AppState app, ElementReference reference, string action, JsonElement? argument, RequestContext context)
+    internal static void Perform(ElementReference reference, string action, JsonElement? argument, RequestContext context)
     {
-        if (argument is { } value && value.ValueKind is not (JsonValueKind.Null or JsonValueKind.Undefined))
-            throw new NativeFailure("INVALID_ARGUMENT", "This accessibility action does not accept an argument");
-        Secondary(app, reference, action, context);
+        var operation = PatternActions.Resolve(action, argument);
+        var pattern = Pattern<BasePattern>(reference.Element, operation.Pattern, context);
+        try { context.Ax(() => { operation.Invoke(pattern); return true; }); }
+        catch (ArgumentException)
+        {
+            throw new NativeFailure("INVALID_ARGUMENT", $"The control rejected the argument for {action}; check its accepted values before retrying");
+        }
+        catch (InvalidOperationException error) when (error is not ElementNotEnabledException)
+        {
+            throw new NativeFailure("UNSUPPORTED", $"The control could not complete {action} in its current state; read fresh state before further input");
+        }
     }
 
     internal static bool Invoke(ElementReference reference, RequestContext context)
@@ -90,40 +90,6 @@ internal static class AccessibilityActions
         context.Ax(() => { range.SetValue(number); return true; });
     }
 
-    internal static void Secondary(AppState app, ElementReference reference, string action, RequestContext context)
-    {
-        switch (action.ToLowerInvariant())
-        {
-            case "press":
-                context.Ax(() => { Pattern<InvokePattern>(reference.Element, InvokePattern.Pattern, context).Invoke(); return true; });
-                break;
-            case "pick":
-                context.Ax(() => { Pattern<SelectionItemPattern>(reference.Element, SelectionItemPattern.Pattern, context).Select(); return true; });
-                break;
-            case "toggle":
-                context.Ax(() => { Pattern<TogglePattern>(reference.Element, TogglePattern.Pattern, context).Toggle(); return true; });
-                break;
-            case "expand":
-                context.Ax(() => { Pattern<ExpandCollapsePattern>(reference.Element, ExpandCollapsePattern.Pattern, context).Expand(); return true; });
-                break;
-            case "collapse":
-                context.Ax(() => { Pattern<ExpandCollapsePattern>(reference.Element, ExpandCollapsePattern.Pattern, context).Collapse(); return true; });
-                break;
-            case "scrolltovisible":
-                context.Ax(() => { Pattern<ScrollItemPattern>(reference.Element, ScrollItemPattern.Pattern, context).ScrollIntoView(); return true; });
-                break;
-            case "focus":
-                context.Ax(() => { reference.Element.SetFocus(); return true; });
-                break;
-            case "raise":
-                if (context.Ax(() => reference.Element.Current.ControlType) != ControlType.Window)
-                    throw new NativeFailure("UNSUPPORTED", "Raise is available only on the observed window element");
-                Windows.Activate(app, reference.Window, context);
-                break;
-            default: throw new NativeFailure("UNSUPPORTED", "Use an action name from a fresh details:true observation");
-        }
-    }
-
     internal static void SelectText(ElementReference reference, string text, string? prefix, string? suffix,
         string selectionType, RequestContext context)
     {

packages/experimental/cua-windows/native/entry/DesktopExecutor.cs

@@ -213,7 +213,7 @@ internal sealed class DesktopExecutor(MtaDispatcher mta, StaDispatcher sta) : IA
                     await mta.Invoke(() =>
                     {
                         var element = Accessibility.Element(app, target, String(parameters, "snapshotId"), String(parameters, "elementId"), context);
-                        AccessibilityActions.Perform(app, element, String(parameters, "action"), argument, context);
+                        AccessibilityActions.Perform(element, String(parameters, "action"), argument, context);
                         return true;
                     }, cancellation);
                     return ActionResult(foreground);

packages/experimental/cua-windows/native/entry/PatternActions.cs

@@ -0,0 +1,150 @@
+// Cached observations and live dispatch share the same UIA operation definitions.
+using System.Text.Json;
+using System.Windows.Automation;
+using DeepSeek.Cua.Protocol;
+
+namespace DeepSeek.Cua.Windows;
+
+internal sealed record PatternAction(string Name, Func<JsonElement?, Action<BasePattern>> Bind);
+internal sealed record PatternOperations(AutomationPattern Pattern, AutomationProperty Available, PatternAction[] Actions);
+
+internal static class PatternActions
+{
+    internal static readonly PatternOperations[] Patterns = [
+        new(InvokePattern.Pattern, AutomationElement.IsInvokePatternAvailableProperty, [
+            NoArgument<InvokePattern>(nameof(InvokePattern.Invoke), pattern => pattern.Invoke()),
+        ]),
+        new(TogglePattern.Pattern, AutomationElement.IsTogglePatternAvailableProperty, [
+            NoArgument<TogglePattern>(nameof(TogglePattern.Toggle), pattern => pattern.Toggle()),
+        ]),
+        new(SelectionItemPattern.Pattern, AutomationElement.IsSelectionItemPatternAvailableProperty, [
+            NoArgument<SelectionItemPattern>(nameof(SelectionItemPattern.Select), pattern => pattern.Select()),
+            NoArgument<SelectionItemPattern>(nameof(SelectionItemPattern.AddToSelection), pattern => pattern.AddToSelection()),
+            NoArgument<SelectionItemPattern>(nameof(SelectionItemPattern.RemoveFromSelection), pattern => pattern.RemoveFromSelection()),
+        ]),
+        new(ExpandCollapsePattern.Pattern, AutomationElement.IsExpandCollapsePatternAvailableProperty, [
+            NoArgument<ExpandCollapsePattern>(nameof(ExpandCollapsePattern.Expand), pattern => pattern.Expand()),
+            NoArgument<ExpandCollapsePattern>(nameof(ExpandCollapsePattern.Collapse), pattern => pattern.Collapse()),
+        ]),
+        new(ScrollItemPattern.Pattern, AutomationElement.IsScrollItemPatternAvailableProperty, [
+            NoArgument<ScrollItemPattern>(nameof(ScrollItemPattern.ScrollIntoView), pattern => pattern.ScrollIntoView()),
+        ]),
+        new(ValuePattern.Pattern, AutomationElement.IsValuePatternAvailableProperty, [
+            WithArgument<ValuePattern, string>(nameof(ValuePattern.SetValue), String, (pattern, value) => pattern.SetValue(value)),
+        ]),
+        new(RangeValuePattern.Pattern, AutomationElement.IsRangeValuePatternAvailableProperty, [
+            WithArgument<RangeValuePattern, double>(nameof(RangeValuePattern.SetValue), Number, (pattern, value) => pattern.SetValue(value)),
+        ]),
+        new(ScrollPattern.Pattern, AutomationElement.IsScrollPatternAvailableProperty, [
+            WithArgument<ScrollPattern, (ScrollAmount X, ScrollAmount Y)>(nameof(ScrollPattern.Scroll),
+                value => Pair(value, "horizontalAmount", "verticalAmount", EnumValue<ScrollAmount>),
+                (pattern, amount) => pattern.Scroll(amount.X, amount.Y)),
+            WithArgument<ScrollPattern, ScrollAmount>(nameof(ScrollPattern.ScrollHorizontal), EnumValue<ScrollAmount>,
+                (pattern, amount) => pattern.ScrollHorizontal(amount)),
+            WithArgument<ScrollPattern, ScrollAmount>(nameof(ScrollPattern.ScrollVertical), EnumValue<ScrollAmount>,
+                (pattern, amount) => pattern.ScrollVertical(amount)),
+            WithArgument<ScrollPattern, (double X, double Y)>(nameof(ScrollPattern.SetScrollPercent),
+                value => Pair(value, "horizontalPercent", "verticalPercent", Percent),
+                (pattern, percent) => pattern.SetScrollPercent(percent.X, percent.Y)),
+        ]),
+        new(TransformPattern.Pattern, AutomationElement.IsTransformPatternAvailableProperty, [
+            WithArgument<TransformPattern, (double X, double Y)>(nameof(TransformPattern.Move),
+                value => Pair(value, "x", "y", Number), (pattern, point) => pattern.Move(point.X, point.Y)),
+            WithArgument<TransformPattern, (double Width, double Height)>(nameof(TransformPattern.Resize),
+                value => Pair(value, "width", "height", PositiveNumber), (pattern, size) => pattern.Resize(size.Width, size.Height)),
+            WithArgument<TransformPattern, double>(nameof(TransformPattern.Rotate), Number, (pattern, degrees) => pattern.Rotate(degrees)),
+        ]),
+        new(WindowPattern.Pattern, AutomationElement.IsWindowPatternAvailableProperty, [
+            NoArgument<WindowPattern>(nameof(WindowPattern.Close), pattern => pattern.Close()),
+            WithArgument<WindowPattern, WindowVisualState>(nameof(WindowPattern.SetWindowVisualState), EnumValue<WindowVisualState>,
+                (pattern, state) => pattern.SetWindowVisualState(state)),
+        ]),
+        new(DockPattern.Pattern, AutomationElement.IsDockPatternAvailableProperty, [
+            WithArgument<DockPattern, DockPosition>(nameof(DockPattern.SetDockPosition), EnumValue<DockPosition>,
+                (pattern, position) => pattern.SetDockPosition(position)),
+        ]),
+        new(MultipleViewPattern.Pattern, AutomationElement.IsMultipleViewPatternAvailableProperty, [
+            WithArgument<MultipleViewPattern, int>(nameof(MultipleViewPattern.SetCurrentView), Integer, (pattern, view) => pattern.SetCurrentView(view)),
+        ]),
+        new(VirtualizedItemPattern.Pattern, AutomationElement.IsVirtualizedItemPatternAvailableProperty, [
+            NoArgument<VirtualizedItemPattern>(nameof(VirtualizedItemPattern.Realize), pattern => pattern.Realize()),
+        ]),
+        new(SynchronizedInputPattern.Pattern, AutomationElement.IsSynchronizedInputPatternAvailableProperty, [
+            WithArgument<SynchronizedInputPattern, SynchronizedInputType>(nameof(SynchronizedInputPattern.StartListening), EnumValue<SynchronizedInputType>,
+                (pattern, inputType) => pattern.StartListening(inputType)),
+            NoArgument<SynchronizedInputPattern>(nameof(SynchronizedInputPattern.Cancel), pattern => pattern.Cancel()),
+        ]),
+    ];
+
+    internal static string[] Names(Func<PatternOperations, bool> supported) => Patterns
+        .Where(supported).SelectMany(pattern => pattern.Actions.Select(action => action.Name)).ToArray();
+
+    // Bind validates the complete JSON argument before any UIA operation is attempted.
+    internal static (AutomationPattern Pattern, Action<BasePattern> Invoke) Resolve(string name, JsonElement? argument)
+    {
+        foreach (var pattern in Patterns)
+            foreach (var action in pattern.Actions)
+                if (action.Name == name) return (pattern.Pattern, action.Bind(argument));
+        throw new NativeFailure("UNSUPPORTED", "Use an exact Pattern.Action name from a fresh observation or listActions");
+    }
+
+    private static PatternAction NoArgument<T>(string method, Action<T> invoke) where T : BasePattern =>
+        WithArgument<T, bool>(method, value =>
+        {
+            if (value is { ValueKind: not (JsonValueKind.Null or JsonValueKind.Undefined) })
+                throw Invalid("omit argument or pass null for this no-argument action");
+            return true;
+        }, (pattern, _) => invoke(pattern));
+
+    // Single parameters are direct JSON values; multiple parameters use their native parameter names.
+    private static PatternAction WithArgument<T, TValue>(string method, Func<JsonElement?, TValue> parse, Action<T, TValue> invoke)
+        where T : BasePattern
+    {
+        var name = typeof(T).Name[..^"Pattern".Length] + "." + method;
+        return new(name, argument =>
+        {
+            var value = parse(argument);
+            return pattern => invoke((T)pattern, value);
+        });
+    }
+
+    private static NativeFailure Invalid(string expected) => new("INVALID_ARGUMENT", "Pattern action argument: " + expected);
+
+    private static string String(JsonElement? value) => value is { ValueKind: JsonValueKind.String } text
+        ? text.GetString()! : throw Invalid("pass a string");
+
+    private static double Number(JsonElement? value) => value is { ValueKind: JsonValueKind.Number } number
+        && number.TryGetDouble(out var result) && double.IsFinite(result) ? result : throw Invalid("pass a finite number");
+
+    private static int Integer(JsonElement? value) => value is { ValueKind: JsonValueKind.Number } number
+        && number.TryGetInt32(out var result) ? result : throw Invalid("pass a 32-bit integer viewId");
+
+    private static double PositiveNumber(JsonElement? value)
+    {
+        var number = Number(value);
+        return number > 0 ? number : throw Invalid("width and height must be positive");
+    }
+
+    private static double Percent(JsonElement? value)
+    {
+        var number = Number(value);
+        return number == ScrollPattern.NoScroll || number is >= 0 and <= 100
+            ? number : throw Invalid("pass a scroll percentage from 0 to 100, or -1 to leave that axis unchanged");
+    }
+
+    private static T EnumValue<T>(JsonElement? value) where T : struct, Enum
+    {
+        if (value is { ValueKind: JsonValueKind.String } text && Enum.TryParse<T>(text.GetString(), out var result)
+            && Enum.IsDefined(result) && result.ToString() == text.GetString()) return result;
+        throw Invalid("pass one of: " + string.Join(", ", Enum.GetNames<T>()));
+    }
+
+    private static (T, T) Pair<T>(JsonElement? value, string first, string second, Func<JsonElement?, T> parse)
+    {
+        if (value is not { ValueKind: JsonValueKind.Object } fields
+            || !fields.TryGetProperty(first, out var a) || !fields.TryGetProperty(second, out var b)
+            || fields.EnumerateObject().Count() != 2)
+            throw Invalid($"pass an object containing exactly {first} and {second}");
+        return (parse(a), parse(b));
+    }
+}

packages/experimental/cua-windows/native/protocol/AccessibilityText.cs

@@ -58,8 +58,7 @@ public static class AccessibilityText
             else text.Append(" selection=unavailable");
             if (node.SelectedText is { Length: > 0 } selectedText) text.Append(" selectedText=").Append(Quote(selectedText));
         }
-        var actions = details ? node.Actions : node.Actions.Where(action => action is not ("Press" or "Pick" or "ScrollToVisible")).ToArray();
-        if (actions.Length != 0) text.Append(" actions=[").Append(string.Join(", ", actions.Select(Quote))).Append(']');
+        if (node.Actions.Length != 0) text.Append(" actions=[").Append(string.Join(", ", node.Actions.Select(Quote))).Append(']');
         if (details)
         {
             if (node.Identifier is { Length: > 0 } identifier) text.Append(" id=").Append(Quote(identifier));

packages/experimental/cua-windows/native/tests/Program.cs

@@ -65,9 +65,9 @@ Check(KeyChord.Parse("Control+a", ["ctrl"]).Modifiers.SequenceEqual(["control"])
 Refuses(() => KeyChord.Parse("ctrl+", []), "INVALID_ARGUMENT");
 Refuses(() => KeyChord.Parse("fn+a", []), "UNSUPPORTED");
 
-var node = new AccessibilityNode { ControlType = "Edit", Name = "Title", Value = "", HasValue = true, Placeholder = "Title", ValueWritable = "true", Actions = ["Press", "Focus"] };
+var node = new AccessibilityNode { ControlType = "Edit", Name = "Title", Value = "", HasValue = true, Placeholder = "Title", ValueWritable = "true", Actions = ["Invoke.Invoke", "Value.SetValue"] };
 var text = AccessibilityText.Format(node, 3, false);
-Check(text == "- [3] Edit valueWritable=true \"Title\" = \"\" placeholder=\"Title\" actions=[\"Focus\"]",
+Check(text == "- [3] Edit valueWritable=true \"Title\" = \"\" placeholder=\"Title\" actions=[\"Invoke.Invoke\", \"Value.SetValue\"]",
     "Rows expose the native ControlType name without a subrole");
 foreach (var controlType in new[] { "Group", "Other", "ListItem", "TreeItem", "TabItem", "Table", "DataGrid" })
     Check(AccessibilityText.Format(new() { ControlType = controlType }, null, false) == "- " + controlType,
@@ -75,10 +75,16 @@ foreach (var controlType in new[] { "Group", "Other", "ListItem", "TreeItem", "T
 Check(!AccessibilityText.Retain(new() { ControlType = "Other" }), "Empty fallback containers can collapse");
 Check(AccessibilityText.Retain(new() { ControlType = "Other", Name = "Custom content" }), "Named fallback containers retain their row");
 Check(text.Contains("= \"\" placeholder=\"Title\""), "Empty values and placeholders remain distinct");
-Check(!text.Contains("\"Press\"") && text.Contains("\"Focus\""), "Common actions compact without hiding secondary actions");
-Check(AccessibilityText.Format(node, 3, true).Contains("\"Press\""), "Details retain complete action names");
+Check(AccessibilityText.Format(node, 3, true) == text, "Both detail modes expose every action");
+var selectionActions = new AccessibilityNode { ControlType = "ListItem", Actions = [
+    "SelectionItem.Select", "SelectionItem.AddToSelection", "SelectionItem.RemoveFromSelection",
+    "ExpandCollapse.Expand", "ExpandCollapse.Collapse", "ScrollItem.ScrollIntoView",
+] };
+foreach (var details in new[] { false, true })
+    Check(AccessibilityText.Format(selectionActions, 0, details) == "- [0] ListItem actions=[\"SelectionItem.Select\", \"SelectionItem.AddToSelection\", \"SelectionItem.RemoveFromSelection\", \"ExpandCollapse.Expand\", \"ExpandCollapse.Collapse\", \"ScrollItem.ScrollIntoView\"]",
+        "Multi-action patterns retain all operations in tree output");
 Check(!AccessibilityText.Retain(new() { ControlType = "Group" }), "Empty structural containers can collapse");
-Check(AccessibilityText.Retain(new() { ControlType = "Group", Actions = ["Copy"] }), "Actionable containers retain their row");
+Check(AccessibilityText.Retain(new() { ControlType = "Group", Actions = ["Value.SetValue"] }), "Actionable containers retain their row");
 Check(AccessibilityText.Format(new() { ControlType = "Document", Value = "abc", HasValue = true, HasSelection = true, Selection = new(2, 5) }, 0, false)
     .Contains("selection=unavailable"), "Out-of-range selections do not describe valid text");
 Console.WriteLine($"{checks} portable CUA checks passed");