feat(ai-assistant): 重构页面操作为统一执行器(Web DOM floor + Electron CDP 双后端)

页面操作(doo page → /ws → 浮窗 operation-module → action-executor)原用手写 iView-only 选择器 + 朴素 value= 赋值,在 React+Radix 同源 iframe 业务表单上几乎填不进。重构为「复用 Playwright 注入脚本的描述层 + 按环境选后端的执行层」: 描述层(共用): - 新增 aria/aria-bundle.js,vendoring Playwright ariaSnapshot/roleUtils/domUtils 子集预构建为 ESM(Apache-2.0 NOTICE + 固定上游 commit),产出 YAML 快照与活的 ref→Element Map。 - page-context-collector 改用 buildSnapshot 采集,operation-module 经 setRefMap 以 Map 直接持有元素取代 selector 反查。执行层(按环境选后端,失败回退): - web-backend:原生 prototype value setter + beforeinput/input/change/blur 序列 + 回读校验(不符报 value_not_applied,绝不假报成功);自定义下拉/日期 open→click。 - electron-backend + electron/lib/page-input.js:经 webContents.debugger 走 CDP Input 域产生 isTrusted=true 可信输入(insertText/dispatchMouseEvent/KeyEvent), 任一步失败回退 Web 后端。 - 复杂控件(动态明细表/成员选择器/文件上传)如实返回 unsupported_widget。协议与链路(doo page、active-context 失效守卫、导航类 action)保持不变。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-22 14:08:24 +00:00 · 2026-06-17 02:03:59 +00:00 · 2026-06-17 02:03:59 +00:00 · a3d5bdf93c
commit a3d5bdf93c
parent b0a4fe6646
12 changed files with 3010 additions and 257 deletions
--- a/electron/electron.js
+++ b/electron/electron.js
@ -43,6 +43,7 @@ const { startMCPServer, stopMCPServer } = require("./lib/mcp");
 const {onRenderer, renderer} = require("./lib/renderer");
 const {onExport} = require("./lib/pdf-export");
 const {allowedCalls, isWin} = require("./lib/other");
+const {registerPageInput} = require("./lib/page-input");
 const webTabManager = require("./lib/web-tab-manager");
 const faviconCache = require("./lib/favicon-cache");

@ -624,6 +625,9 @@ app.on("will-quit", () => {
    globalShortcut.unregisterAll();
 })

+// AI 助手页面操作 · CDP 可信输入（渲染端经 sendAsync('pageInput', ...) 调用）
+registerPageInput();
+
 /**
 * 设置菜单语言包
 * @param args {path}
--- a/electron/lib/page-input.js
+++ b/electron/lib/page-input.js
@ -0,0 +1,57 @@
+// AI 助手页面操作 · CDP 可信输入（主进程）
+//
+// 渲染端经 window.electron.sendAsync('pageInput', {op, ...}) 调用（preload 已有的
+// invoke 通道）；这里用 webContents.debugger 走 CDP Input 域，产生 isTrusted=true 的
+// 可信输入，等价 Playwright 真正可靠的那一半，但第一方、在用户会话内，且同源 iframe 可达。
+//
+// 目标 webContents = event.sender（发起的渲染进程；同源 iframe 在其内，视口坐标可达）。
+// 任一步骤失败抛错 → 渲染端 electron-backend 捕获后回退 Web 后端。
+
+const { ipcMain } = require('electron')
+
+const attached = new WeakSet()
+
+function ensureAttached(wc) {
+    if (attached.has(wc)) return
+    if (!wc.debugger.isAttached()) {
+        wc.debugger.attach('1.3') // 抛错则上抛，渲染端回退
+    }
+    attached.add(wc)
+    wc.once('destroyed', () => attached.delete(wc))
+}
+
+async function dispatchClick(wc, x, y) {
+    const base = { x, y, button: 'left', buttons: 1, clickCount: 1 }
+    await wc.debugger.sendCommand('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y })
+    await wc.debugger.sendCommand('Input.dispatchMouseEvent', Object.assign({ type: 'mousePressed' }, base))
+    await wc.debugger.sendCommand('Input.dispatchMouseEvent', Object.assign({ type: 'mouseReleased' }, base))
+}
+
+async function handle(event, args) {
+    const wc = event.sender
+    const op = args && args.op
+    ensureAttached(wc)
+    switch (op) {
+        case 'insertText':
+            // 覆盖当前选区（渲染端已 selectAll）
+            await wc.debugger.sendCommand('Input.insertText', { text: String(args.text == null ? '' : args.text) })
+            return { ok: true }
+        case 'click':
+            await dispatchClick(wc, Math.round(args.x), Math.round(args.y))
+            return { ok: true }
+        case 'key': {
+            const key = String(args.key || '')
+            await wc.debugger.sendCommand('Input.dispatchKeyEvent', { type: 'keyDown', key })
+            await wc.debugger.sendCommand('Input.dispatchKeyEvent', { type: 'keyUp', key })
+            return { ok: true }
+        }
+        default:
+            throw new Error('unknown_op: ' + op)
+    }
+}
+
+function registerPageInput() {
+    ipcMain.handle('pageInput', handle)
+}
+
+module.exports = { registerPageInput }
--- a/resources/assets/js/components/AIAssistant/action-executor.js
+++ b/resources/assets/js/components/AIAssistant/action-executor.js
@ -9,8 +9,8 @@
 * 本模块只负责前端导航和 UI 操作。
 */

-import { findElementByRef } from './page-context-collector';
 import { resolveActiveContext } from './active-context';
+import { selectBackend } from './input-backends';

 /**
 * 创建操作执行器
@ -219,8 +219,9 @@ class ActionExecutor {
    /**
     * 设置当前的 refMap 与活动上下文（由 operation-module 在获取上下文后调用）
     */
-    setRefMap(refMap, context = null) {
-        this.currentRefMap = refMap;
+    setRefMap(refElements, context = null) {
+        // refElements: Map<ref, Element>（描述层产出，直接持有元素）；容错旧的 plain object
+        this.currentRefElements = refElements instanceof Map ? refElements : null;
        this.currentContext = context;
    }

@ -254,65 +255,17 @@ class ActionExecutor {
        const doc = this.resolveContextDoc();
        const element = this.findElement(elementUid, doc);
        if (!element) {
-            throw new Error(`找不到元素: ${elementUid}`);
+            throw new Error(`element_not_found: 找不到元素 ${elementUid}（可能页面已变更，请重新获取页面上下文）`);
        }

-        // 元素所在 window（主文档或微应用 iframe），事件需用它的构造器才被框架信任
+        // 元素所在 window（主文档或微应用 iframe），合成事件需用它的构造器才被框架信任
        const win = element.ownerDocument.defaultView || window;

-        switch (action) {
-            case 'click':
-                element.click();
-                return { success: true, action: 'click', element: elementUid };
+        // 选择输入后端：Electron CDP 可信输入优先，否则页面内合成事件（地板）
+        if (!this.backend) this.backend = selectBackend();

-            case 'type':
-                if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA' || element.contentEditable === 'true') {
-                    element.focus();
-                    if (element.contentEditable === 'true') {
-                        element.textContent = value || '';
-                    } else {
-                        element.value = value || '';
-                    }
-                    element.dispatchEvent(new win.Event('input', { bubbles: true }));
-                    element.dispatchEvent(new win.Event('change', { bubbles: true }));
-                    return { success: true, action: 'type', value, element: elementUid };
-                }
-                throw new Error('元素不支持输入操作');
-
-            case 'select':
-                if (element.tagName === 'SELECT') {
-                    element.value = value;
-                    element.dispatchEvent(new win.Event('change', { bubbles: true }));
-                    return { success: true, action: 'select', value, element: elementUid };
-                }
-                // iView Select 组件 - 先点击打开下拉
-                element.click();
-                await this.delay(200);
-                const options = element.ownerDocument.querySelectorAll('.ivu-select-dropdown-list .ivu-select-item');
-                for (const option of options) {
-                    if (option.textContent.trim().includes(value)) {
-                        option.click();
-                        return { success: true, action: 'select', value, element: elementUid };
-                    }
-                }
-                throw new Error(`找不到选项: ${value}`);
-
-            case 'focus':
-                element.focus();
-                return { success: true, action: 'focus', element: elementUid };
-
-            case 'scroll':
-                element.scrollIntoView({ behavior: 'smooth', block: 'center' });
-                return { success: true, action: 'scroll', element: elementUid };
-
-            case 'hover':
-                element.dispatchEvent(new win.MouseEvent('mouseenter', { bubbles: true }));
-                element.dispatchEvent(new win.MouseEvent('mouseover', { bubbles: true }));
-                return { success: true, action: 'hover', element: elementUid };
-
-            default:
-                throw new Error(`不支持的元素操作: ${action}`);
-        }
+        const result = await this.backend.perform(element, action, value, { doc, win, elementUid });
+        return Object.assign({ element: elementUid }, result);
    }

    /**
@ -329,13 +282,14 @@ class ActionExecutor {
            ref = identifier;
        }

-        // 如果是 ref 格式，使用 refMap 查找
-        if (ref && this.currentRefMap) {
-            const element = findElementByRef(ref, this.currentRefMap, doc);
-            if (element) return element;
+        // ref 格式：从描述层产出的 ref→Element 实时 Map 直接取（最准）
+        if (ref && this.currentRefElements) {
+            const element = this.currentRefElements.get(ref);
+            if (element && element.isConnected) return element;
+            if (element && !element.isConnected) return null; // 失联（DOM 已变更）
        }

-        // 尝试作为 CSS 选择器
+        // 尝试作为 CSS 选择器（兜底）
        try {
            const element = doc.querySelector(identifier);
            if (element) return element;
--- a/resources/assets/js/components/AIAssistant/aria/NOTICE.txt
+++ b/resources/assets/js/components/AIAssistant/aria/NOTICE.txt
@ -0,0 +1,19 @@
+aria-bundle.js is a vendored, esbuild-bundled subset of Playwright's injected
+ariaSnapshot code.
+
+  Playwright
+  Copyright (c) Microsoft Corporation
+  Licensed under the Apache License, Version 2.0
+  https://github.com/microsoft/playwright/blob/main/LICENSE
+
+Included source (from packages/injected/src and packages/isomorphic):
+  ariaSnapshot.ts, roleUtils.ts, domUtils.ts, and @isomorphic helpers
+  (stringUtils, yaml, cssTokenizer, ariaSnapshot).
+
+Regenerate:
+  1. git clone github.com/microsoft/playwright (pin the commit in aria-bundle.js banner)
+  2. esbuild esm-entry.ts --bundle --format=esm --platform=browser \
+       --tsconfig=spike-tsconfig.json --outfile=aria-bundle.js
+  esm-entry.ts and spike-tsconfig.json in this folder are the build inputs.
+
+This file is provided to comply with the Apache-2.0 NOTICE/attribution requirement.
--- a/resources/assets/js/components/AIAssistant/aria/aria-bundle.js
+++ b/resources/assets/js/components/AIAssistant/aria/aria-bundle.js
--- a/resources/assets/js/components/AIAssistant/aria/esm-entry.ts
+++ b/resources/assets/js/components/AIAssistant/aria/esm-entry.ts
@ -0,0 +1,16 @@
+import { generateAriaTree, renderAriaTree } from '@injected/ariaSnapshot';
+
+// dootask AI 助手页面操作 · 描述层封装。
+// buildSnapshot(root) → { text: ariaSnapshot YAML, elements: Map<ref,Element>, refs: Map<Element,ref> }
+export function buildSnapshot(root?: Element, options?: any) {
+  const o = Object.assign({ mode: 'ai' }, options || {});
+  const tree = generateAriaTree((root || document.body) as Element, o);
+  const rendered = renderAriaTree(tree, o);
+  return {
+    text: rendered.text,
+    elements: tree.elements, // Map<string ref, Element>
+    refs: tree.refs,         // Map<Element, string ref>
+  };
+}
+
+export { generateAriaTree, renderAriaTree };
--- a/resources/assets/js/components/AIAssistant/aria/spike-tsconfig.json
+++ b/resources/assets/js/components/AIAssistant/aria/spike-tsconfig.json
@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "@injected/*": ["./packages/injected/src/*"],
+      "@isomorphic/*": ["./packages/isomorphic/*"]
+    }
+  }
+}
--- a/resources/assets/js/components/AIAssistant/input-backends/electron-backend.js
+++ b/resources/assets/js/components/AIAssistant/input-backends/electron-backend.js
@ -0,0 +1,75 @@
+/**
+ * Electron 输入后端（CDP 可信输入，渲染侧）
+ *
+ * 桌面端经 preload 暴露的 window.electron.sendAsync('pageInput', ...) 把输入意图
+ * 转给主进程，由 webContents.debugger 走 CDP Input 域产生 isTrusted=true 的可信输入
+ * （等价 Playwright 真正可靠的那一半，但第一方、在用户会话内，且能到达同源 iframe）。
+ *
+ * 仅覆盖最受益于可信输入的 type/click；其余操作（select 的 open→click、check、scroll
+ * 等）页面内合成事件已足够，交回 Web 后端。任一 CDP 调用失败 → 回退 Web 后端。
+ *
+ * 注意：主进程 ipcMain.handle('pageInput') 见 electron/lib/page-input.js。
+ */
+
+export function isElectronInputAvailable() {
+    return !!(typeof window !== 'undefined'
+        && window.$A && window.$A.isElectron
+        && window.electron && typeof window.electron.sendAsync === 'function');
+}
+
+function centerOf(el) {
+    const r = el.getBoundingClientRect();
+    return { x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2) };
+}
+
+// 选中元素现有内容，便于 insertText 覆盖
+function selectAll(el, win) {
+    try {
+        if (typeof el.select === 'function') { el.select(); return; }
+        if (el.isContentEditable) {
+            const sel = win.getSelection && win.getSelection();
+            if (sel) {
+                const range = el.ownerDocument.createRange();
+                range.selectNodeContents(el);
+                sel.removeAllRanges();
+                sel.addRange(range);
+            }
+        }
+    } catch (e) { /* ignore */ }
+}
+
+export function createElectronBackend(getFallback) {
+    const cdp = (op, args) => window.electron.sendAsync('pageInput', Object.assign({ op }, args));
+
+    return {
+        name: 'electron',
+        async perform(element, action, value, ctx) {
+            const win = (ctx && ctx.win) || element.ownerDocument.defaultView || window;
+            try {
+                if (action === 'type' || action === 'fill' || action === 'clear') {
+                    const v = action === 'clear' ? '' : (value == null ? '' : String(value));
+                    if (typeof element.focus === 'function') element.focus();
+                    selectAll(element, win);
+                    await cdp('insertText', { text: v }); // 覆盖当前选区
+                    const got = element.value != null ? element.value : (element.textContent || '');
+                    if (v && got !== v && !(got || '').includes(v)) {
+                        throw new Error('value_not_applied: 可信输入回读不一致');
+                    }
+                    return { success: true, action: 'type', value: v, backend: 'electron' };
+                }
+                if (action === 'click') {
+                    const { x, y } = centerOf(element);
+                    await cdp('click', { x, y });
+                    return { success: true, action: 'click', backend: 'electron' };
+                }
+            } catch (e) {
+                // 可信输入失败 → 回退 Web 后端
+                return getFallback().perform(element, action, value, ctx);
+            }
+            // 其余操作页面内合成已足够，交回 Web 后端
+            return getFallback().perform(element, action, value, ctx);
+        },
+    };
+}
+
+export default createElectronBackend;
--- a/resources/assets/js/components/AIAssistant/input-backends/index.js
+++ b/resources/assets/js/components/AIAssistant/input-backends/index.js
@ -0,0 +1,28 @@
+/**
+ * 输入后端选择
+ *
+ * - Electron 桌面端且 CDP 桥可用 → ElectronInputBackend（可信输入），失败回退 Web。
+ * - 其他环境 → WebInputBackend（页面内合成事件，地板）。
+ *
+ * 描述/定位层（ariaSnapshot + ref→Element）与后端无关，由 collector/executor 负责；
+ * 后端只接收已定位的 element + action + value。
+ */
+
+import { createWebBackend } from './web-backend';
+import { createElectronBackend, isElectronInputAvailable } from './electron-backend';
+
+let _web = null;
+let _electron = null;
+
+export function selectBackend() {
+    if (isElectronInputAvailable()) {
+        if (!_electron) _electron = createElectronBackend(() => getWebBackend());
+        return _electron;
+    }
+    return getWebBackend();
+}
+
+export function getWebBackend() {
+    if (!_web) _web = createWebBackend();
+    return _web;
+}
--- a/resources/assets/js/components/AIAssistant/input-backends/web-backend.js
+++ b/resources/assets/js/components/AIAssistant/input-backends/web-backend.js
@ -0,0 +1,167 @@
+/**
+ * Web 输入后端（DOM 兜底）
+ *
+ * 在页面内用合成事件操作元素：原生 prototype value setter（绕过 React/Vue 值追踪）
+ * + 完整事件序列 + 写后回读校验；自定义下拉/日期等用 open→click。无法处理的控件
+ * （文件上传、动态明细表等）抛 unsupported_widget，由上层（app call/回问）兜底。
+ *
+ * 已知边界：合成事件 isTrusted=false，部分组件（拖拽/指针捕获/原生文件框）填不动。
+ * 桌面端优先走 Electron CDP 可信输入后端（见 electron-backend.js）。
+ */
+
+const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+
+// 结构化错误：消息以 `<code>: ` 前缀，便于上层与模型识别
+function fail(code, msg) {
+    return new Error(`${code}: ${msg}`);
+}
+
+// 组件壳 → 内层可写控件
+function resolveWritable(el) {
+    if (!el) return el;
+    if (el.matches && el.matches('input, textarea')) return el;
+    if (el.isContentEditable) return el;
+    const inner = el.querySelector && el.querySelector('input, textarea, [contenteditable="true"], [contenteditable=""]');
+    return inner || el;
+}
+
+function fireInput(el, win) {
+    const InputEvt = win.InputEvent || win.Event;
+    el.dispatchEvent(new InputEvt('input', { bubbles: true }));
+    el.dispatchEvent(new win.Event('change', { bubbles: true }));
+}
+
+// 原生 setter + 事件序列 + 回读
+function setValue(rawEl, value, win) {
+    const el = resolveWritable(rawEl);
+    const v = value == null ? '' : String(value);
+    const tag = (el.tagName || '').toLowerCase();
+
+    if (el.isContentEditable) {
+        el.focus();
+        const sel = win.getSelection && win.getSelection();
+        if (sel) {
+            const range = el.ownerDocument.createRange();
+            range.selectNodeContents(el);
+            sel.removeAllRanges();
+            sel.addRange(range);
+        }
+        const ok = el.ownerDocument.execCommand && el.ownerDocument.execCommand('insertText', false, v);
+        if (!ok) { el.textContent = v; }
+        fireInput(el, win);
+        if (v && !(el.textContent || '').includes(v)) throw fail('value_not_applied', '富文本写入未生效');
+        return { success: true, action: 'type', value: v };
+    }
+
+    if (tag !== 'input' && tag !== 'textarea') {
+        throw fail('unsupported_widget', '元素不可输入文本');
+    }
+
+    el.focus();
+    const proto = tag === 'textarea' ? win.HTMLTextAreaElement.prototype : win.HTMLInputElement.prototype;
+    const desc = Object.getOwnPropertyDescriptor(proto, 'value');
+    const setter = desc && desc.set;
+    if (setter) setter.call(el, v); else el.value = v; // 兜底
+    fireInput(el, win);
+    el.dispatchEvent(new win.Event('blur', { bubbles: true }));
+    if (el.value !== v) throw fail('value_not_applied', '写入后回读不一致');
+    return { success: true, action: 'type', value: v };
+}
+
+function clickEl(el) {
+    if (typeof el.focus === 'function') { try { el.focus(); } catch (e) {} }
+    el.click();
+    return { success: true, action: 'click' };
+}
+
+// 原生 select 直接设值；自定义下拉/日期 open→click 匹配项
+async function selectOption(el, value, doc, win) {
+    const v = value == null ? '' : String(value);
+    const tag = (el.tagName || '').toLowerCase();
+
+    if (tag === 'select') {
+        const opt = [...el.options].find((o) =>
+            o.value === v || (o.textContent || '').trim() === v || (o.textContent || '').includes(v));
+        if (!opt) throw fail('unsupported_widget', `原生 select 无选项 ${v}`);
+        el.value = opt.value;
+        fireInput(el, win);
+        return { success: true, action: 'select', value: v };
+    }
+
+    // 自定义下拉：点击展开 → 在选项/日历单元格里按文本匹配 → 点击
+    el.click();
+    await delay(180);
+    const optSelectors = '[role="option"], [role="gridcell"], [role="menuitemradio"], .ivu-select-item, li[role]';
+    const opts = [...doc.querySelectorAll(optSelectors)].filter((o) => o.offsetParent !== null || o.getClientRects().length);
+    const exact = opts.find((o) => (o.textContent || '').trim() === v);
+    const fuzzy = opts.find((o) => (o.textContent || '').trim().includes(v));
+    const match = exact || fuzzy;
+    if (!match) throw fail('unsupported_widget', `下拉未渲染匹配项 ${v}`);
+    match.click();
+    await delay(80);
+    return { success: true, action: 'select', value: v };
+}
+
+function isChecked(el) {
+    const t = el.matches && el.matches('input[type="checkbox"], input[type="radio"]')
+        ? el : (el.querySelector && el.querySelector('input[type="checkbox"], input[type="radio"]'));
+    if (t) return !!t.checked;
+    const a = el.getAttribute && el.getAttribute('aria-checked');
+    return a === 'true';
+}
+
+function setChecked(el, want) {
+    const t = el.matches && el.matches('input[type="checkbox"], input[type="radio"]')
+        ? el : (el.querySelector && el.querySelector('input[type="checkbox"], input[type="radio"]'));
+    if (t) {
+        if (!!t.checked !== want) t.click();
+        return { success: true, action: want ? 'check' : 'uncheck', checked: !!t.checked };
+    }
+    const a = el.getAttribute && el.getAttribute('aria-checked');
+    if (a != null) {
+        if ((a === 'true') !== want) el.click();
+        return { success: true, action: want ? 'check' : 'uncheck' };
+    }
+    throw fail('unsupported_widget', '非复选/单选元素');
+}
+
+export function createWebBackend() {
+    return {
+        name: 'web',
+        async perform(element, action, value, ctx) {
+            const win = (ctx && ctx.win) || element.ownerDocument.defaultView || window;
+            const doc = (ctx && ctx.doc) || element.ownerDocument;
+            switch (action) {
+                case 'click':
+                    return clickEl(element);
+                case 'type':
+                case 'fill':
+                    return setValue(element, value, win);
+                case 'clear':
+                    return setValue(element, '', win);
+                case 'select':
+                    return selectOption(element, value, doc, win);
+                case 'check':
+                    return setChecked(element, true);
+                case 'uncheck':
+                    return setChecked(element, false);
+                case 'toggle':
+                    return setChecked(element, !isChecked(element));
+                case 'focus':
+                    element.focus();
+                    return { success: true, action: 'focus' };
+                case 'scroll':
+                    element.scrollIntoView({ behavior: 'smooth', block: 'center' });
+                    return { success: true, action: 'scroll' };
+                case 'hover':
+                    element.dispatchEvent(new win.MouseEvent('mouseenter', { bubbles: true }));
+                    element.dispatchEvent(new win.MouseEvent('mouseover', { bubbles: true }));
+                    return { success: true, action: 'hover' };
+                default:
+                    throw fail('not_actionable', `不支持的操作: ${action}`);
+            }
+        },
+    };
+}
+
+export default createWebBackend;
--- a/resources/assets/js/components/AIAssistant/operation-module.js
+++ b/resources/assets/js/components/AIAssistant/operation-module.js
@ -97,7 +97,7 @@ class OperationModule {
                reason: active.reason,
            };
            base.hint = `${reasonText}。可改用主界面（scope=main）操作，或改用数据命令完成。`;
-            this.executor.setRefMap({}, null);
+            this.executor.setRefMap(new Map(), null);
            return base;
        }

@ -132,15 +132,11 @@ class OperationModule {
                    context.total_count = vectorMatches.length;
                    context.has_more = false;
                    context.vector_matched = true;
+                    context.refElements = allContext.refElements; // 全量 Map 覆盖向量命中的 ref
                    context.ref_map = {};
                    for (const el of vectorMatches) {
                        if (el.ref) {
-                            context.ref_map[el.ref] = {
-                                role: el.role,
-                                name: el.name,
-                                selector: el.selector,
-                                nth: el.nth,
-                            };
+                            context.ref_map[el.ref] = { role: el.role, name: el.name };
                        }
                    }
                }
@ -154,10 +150,11 @@ class OperationModule {
            operable: true,
        };

-        // 将 refMap 与活动上下文一并存入 executor，供后续元素操作使用（含失效守卫）
-        if (context.ref_map && this.executor) {
-            this.executor.setRefMap(context.ref_map, active);
+        // 将 ref→Element 实时 Map 与活动上下文存入 executor，供后续元素操作解析（含失效守卫）
+        if (context.refElements && this.executor) {
+            this.executor.setRefMap(context.refElements, active);
        }
+        delete context.refElements; // Map 不参与序列化回包

        return context;
    }
--- a/resources/assets/js/components/AIAssistant/page-context-collector.js
+++ b/resources/assets/js/components/AIAssistant/page-context-collector.js
@ -3,8 +3,14 @@
 *
 * 借鉴 agent-browser 项目的设计思想，基于 ARIA 角色收集页面元素。
 * 提供结构化的页面快照，包括可交互元素和内容元素。
+ *
+ * 描述层（元素发现 + 可访问名/角色 + ariaSnapshot 文本 + ref）复用 Playwright
+ * 注入脚本的 ariaSnapshot 子集（见 ./aria/aria-bundle.js）；本文件负责分页、
+ * 关键词/向量过滤、available_actions 等业务包装，并持有 ref→Element 实时 Map。
 */

+import { buildSnapshot } from './aria/aria-bundle';
+
 // ========== ARIA 角色分类 ==========

 // 可交互角色 - 这些元素可以被点击、输入等
@ -111,6 +117,8 @@ export function collectPageContext(store, options = {}) {
        context.total_count = result.totalCount;
        context.has_more = result.hasMore;
        context.ref_map = result.refMap;
+        context.snapshot = result.snapshot;        // ariaSnapshot YAML（模型主消费）
+        context.refElements = result.refElements;  // Map<ref,Element>（实时，供执行器；序列化前删除）
        // 标记是否经过关键词过滤
        if (query) {
            context.query = query;
@ -227,225 +235,81 @@ function getAvailableActions(routeName, store) {
 }

 /**
- * 收集页面元素
+ * 收集页面元素（基于 Playwright ariaSnapshot 子集发现可交互元素）
+ *
+ * 元素发现 / 可访问名·角色 / ref 分配全部来自 buildSnapshot；本函数只做关键词
+ * 过滤、分页与结构化包装，并返回全量 ref→Element 实时 Map 供执行器解析。
 * @param {Object} options
- * @param {boolean} options.interactiveOnly - 仅返回可交互元素
 * @param {number} options.maxElements - 每页最大元素数量
 * @param {number} options.offset - 跳过前 N 个元素
 * @param {string} options.container - 容器选择器
 * @param {string} options.query - 搜索关键词
- * @returns {Object} { elements, refMap, totalCount, hasMore, keywordMatched }
+ * @returns {Object} { elements, refMap, refElements, snapshot, totalCount, hasMore, keywordMatched }
 */
 export function collectElements(options = {}) {
    const {
        doc = document,
-        interactiveOnly = false,
        maxElements = 50,
        offset = 0,
        container = null,
        query = '',
    } = options;

-    // 确定查询的根元素
-    let rootElement = doc;
+    const empty = { elements: [], refMap: {}, refElements: new Map(), snapshot: '', totalCount: 0, hasMore: false, keywordMatched: false };
+
+    // 确定快照根元素
+    let rootElement = doc.body || doc.documentElement || doc;
    if (container) {
        rootElement = doc.querySelector(container);
-        if (!rootElement) {
-            return { elements: [], refMap: {}, totalCount: 0, hasMore: false };
-        }
+        if (!rootElement) return empty;
    }

-    // 角色+名称计数器，用于处理重复元素
-    const roleNameCounter = new Map();
-
-    // 获取所有可能的交互元素选择器
-    const selectors = [
-        // 按钮类
-        'button',
-        '[role="button"]',
-        'input[type="submit"]',
-        'input[type="button"]',
-        'input[type="reset"]',
-        '.ivu-btn',
-        // 链接
-        'a[href]',
-        '[role="link"]',
-        // 输入框
-        'input:not([type="hidden"])',
-        'textarea',
-        '[role="textbox"]',
-        '[contenteditable="true"]',
-        // 选择器
-        'select',
-        '[role="combobox"]',
-        '[role="listbox"]',
-        '.ivu-select',
-        // 复选框和单选框
-        'input[type="checkbox"]',
-        'input[type="radio"]',
-        '[role="checkbox"]',
-        '[role="radio"]',
-        '.ivu-checkbox',
-        '.ivu-radio',
-        // 菜单项
-        '[role="menuitem"]',
-        '[role="tab"]',
-        '.ivu-menu-item',
-        '.ivu-tabs-tab',
-        // 可点击的图标和操作按钮
-        '[class*="click"]',
-        '[class*="btn"]',
-        '[class*="action"]',
-        '.taskfont[title]',
-        // 表格单元格（可能可点击）
-        'td[onclick]',
-        'tr[onclick]',
-    ];
-
-    // 基于 INTERACTIVE_ROLES 补全所有 ARIA 可交互角色（与上面重复的选择器会被 querySelectorAll 按元素自动去重）
-    for (const role of INTERACTIVE_ROLES) {
-        selectors.push(`[role="${role}"]`);
+    // 描述层：Playwright ariaSnapshot 子集 → YAML 文本 + 全量 ref→Element Map（仅可交互元素分配 ref）
+    let snap;
+    try {
+        snap = buildSnapshot(rootElement, { mode: 'ai' });
+    } catch (e) {
+        return empty;
    }
-    // 可聚焦 / 自定义可交互元素（显式 tabindex 且非 -1）
-    selectors.push('[tabindex]:not([tabindex="-1"])');
+    const refElements = snap.elements || new Map(); // Map<ref, Element>，覆盖全部可交互元素

-    // 如果不仅限交互元素，添加内容元素选择器
-    if (!interactiveOnly) {
-        selectors.push(
-            'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
-            '[role="heading"]',
-            'img[alt]',
-            'nav',
-            'main',
-        );
+    // 基于 ref→Element 构建结构化列表（顺序即 DOM 顺序，ref 与 YAML 一致）
+    const all = [];
+    for (const [ref, el] of refElements.entries()) {
+        if (!el || !el.isConnected) continue;
+        const info = extractElementInfo(el);
+        if (!info) continue;
+        info.ref = ref;
+        all.push(info);
    }

-    // 第一阶段：收集所有有效元素（不限数量）
-    const allValidElements = [];
-    const processedElements = new Set();
-
-    // 查询所有匹配的元素
-    const candidateElements = rootElement.querySelectorAll(selectors.join(', '));
-
-    for (const el of candidateElements) {
-        processedElements.add(el);
-
-        // 跳过不可见元素
-        if (!isElementVisible(el)) continue;
-
-        // 跳过禁用元素
-        if (isElementDisabled(el)) continue;
-
-        allValidElements.push({ el, fromPointerScan: false });
-    }
-
-    // 第二遍扫描：查找具有 cursor: pointer 但未被选择器匹配的元素
-    const clickableSelectors = 'div, span, li, td, tr, section, article, aside, header, footer, label, i, svg';
-    const potentialClickables = rootElement.querySelectorAll(clickableSelectors);
-
-    for (const el of potentialClickables) {
-        if (processedElements.has(el)) continue;
-
-        // label[for] 或包裹表单控件的 label：点击等价于其控件（已单独采集），跳过避免重复
-        if (el.tagName === 'LABEL' && (el.htmlFor || el.querySelector('input, textarea, select'))) continue;
-
-        // 检查是否有 cursor: pointer 样式
-        const computedStyle = (el.ownerDocument.defaultView || window).getComputedStyle(el);
-        if (computedStyle.cursor !== 'pointer') continue;
-
-        // 跳过不可见或禁用元素
-        if (!isElementVisible(el)) continue;
-        if (isElementDisabled(el)) continue;
-
-        processedElements.add(el);
-        allValidElements.push({ el, fromPointerScan: true });
-    }
-
-    // 第二阶段：应用关键词过滤（如果有 query）
-    let filteredElements = allValidElements;
+    // 关键词过滤（无匹配则回退到全部）
+    let filtered = all;
    let keywordMatched = false;
-
    if (query) {
-        const lowerQuery = query.toLowerCase();
-        filteredElements = allValidElements.filter(({ el }) => {
-            const name = getElementName(el).toLowerCase();
-            const ariaLabel = (el.getAttribute('aria-label') || '').toLowerCase();
-            const placeholder = (el.placeholder || '').toLowerCase();
-            const title = (el.title || '').toLowerCase();
-
-            return name.includes(lowerQuery)
-                || ariaLabel.includes(lowerQuery)
-                || placeholder.includes(lowerQuery)
-                || title.includes(lowerQuery);
-        });
-        keywordMatched = filteredElements.length > 0;
-
-        // 如果关键词匹配不到任何元素，回退到全部元素
-        if (filteredElements.length === 0) {
-            filteredElements = allValidElements;
-        }
+        const q = query.toLowerCase();
+        filtered = all.filter((info) => (info.name || '').toLowerCase().includes(q)
+            || (info.placeholder || '').toLowerCase().includes(q)
+            || (info.aria_label || '').toLowerCase().includes(q)
+            || (info.title || '').toLowerCase().includes(q));
+        keywordMatched = filtered.length > 0;
+        if (filtered.length === 0) filtered = all;
    }

-    // 第三阶段：应用分页，生成最终结果
-    const totalCount = filteredElements.length;
-    const startIndex = offset;
+    // 分页（结构化列表分页；refElements 始终为全量，执行器可解析任意 ref）
+    const totalCount = filtered.length;
    const endIndex = Math.min(offset + maxElements, totalCount);
    const hasMore = endIndex < totalCount;

    const elements = [];
    const refMap = {};
-    let refCounter = offset + 1; // ref 从 offset+1 开始，保持全局唯一
-
-    for (let i = startIndex; i < endIndex; i++) {
-        const { el, fromPointerScan } = filteredElements[i];
-
-        // 获取元素信息
-        const elementInfo = extractElementInfo(el, refCounter);
-        if (!elementInfo) continue;
-
-        // 如果是从 pointer 扫描来的，强制设为 button 角色
-        if (fromPointerScan && elementInfo.role === 'generic') {
-            elementInfo.role = 'button';
-        }
-
-        // 处理重复的角色+名称组合
-        const key = `${elementInfo.role}:${elementInfo.name || ''}`;
-        const count = roleNameCounter.get(key) || 0;
-        roleNameCounter.set(key, count + 1);
-
-        if (count > 0) {
-            elementInfo.nth = count;
-        }
-
-        // 生成 ref
-        const ref = `e${refCounter++}`;
-        elementInfo.ref = ref;
-
-        // 存储到 refMap
-        refMap[ref] = {
-            role: elementInfo.role,
-            name: elementInfo.name,
-            selector: elementInfo.selector,
-            nth: elementInfo.nth,
-        };
-
-        elements.push(elementInfo);
+    for (let i = offset; i < endIndex; i++) {
+        const info = filtered[i];
+        elements.push(info);
+        refMap[info.ref] = { role: info.role, name: info.name };
    }

-    // 为重复元素添加 nth 标记
-    for (const element of elements) {
-        const key = `${element.role}:${element.name || ''}`;
-        const roleCount = roleNameCounter.get(key);
-        // 只有真正重复的才保留 nth
-        if (roleCount <= 1) {
-            delete element.nth;
-            if (refMap[element.ref]) {
-                delete refMap[element.ref].nth;
-            }
-        }
-    }
-
-    return { elements, refMap, totalCount, hasMore, keywordMatched };
+    return { elements, refMap, refElements, snapshot: snap.text, totalCount, hasMore, keywordMatched };
 }

 /**