kuaifan a3d5bdf93c feat(ai-assistant): 重构页面操作为统一执行器(Web DOM floor + Electron CDP 双后端)
页面操作(doo page → /ws → 浮窗 operation-module → action-executor)原用手写
iView-only 选择器 + 朴素 value= 赋值,在 React+Radix 同源 iframe 业务表单上几乎
填不进。重构为「复用 Playwright 注入脚本的描述层 + 按环境选后端的执行层」:

描述层(共用):
- 新增 aria/aria-bundle.js,vendoring Playwright ariaSnapshot/roleUtils/domUtils
  子集预构建为 ESM(Apache-2.0 NOTICE + 固定上游 commit),产出 YAML 快照与活的
  ref→Element Map。
- page-context-collector 改用 buildSnapshot 采集,operation-module 经 setRefMap
  以 Map 直接持有元素取代 selector 反查。

执行层(按环境选后端,失败回退):
- web-backend:原生 prototype value setter + beforeinput/input/change/blur 序列
  + 回读校验(不符报 value_not_applied,绝不假报成功);自定义下拉/日期 open→click。
- electron-backend + electron/lib/page-input.js:经 webContents.debugger 走 CDP
  Input 域产生 isTrusted=true 可信输入(insertText/dispatchMouseEvent/KeyEvent),
  任一步失败回退 Web 后端。
- 复杂控件(动态明细表/成员选择器/文件上传)如实返回 unsupported_widget。

协议与链路(doo page、active-context 失效守卫、导航类 action)保持不变。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-17 02:03:59 +00:00

168 lines
7.0 KiB
JavaScript
Vendored
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* Web 输入后端DOM 兜底)
*
* 在页面内用合成事件操作元素:原生 prototype value setter绕过 React/Vue 值追踪)
* + 完整事件序列 + 写后回读校验;自定义下拉/日期等用 open→click。无法处理的控件
* (文件上传、动态明细表等)抛 unsupported_widget由上层app call/回问)兜底。
*
* 已知边界:合成事件 isTrusted=false部分组件拖拽/指针捕获/原生文件框)填不动。
* 桌面端优先走 Electron CDP 可信输入后端(见 electron-backend.js
*/
const delay = (ms) => new Promise((r) => setTimeout(r, ms));
// 结构化错误:消息以 `<code>: ` 前缀,便于上层与模型识别
function fail(code, msg) {
return new Error(`${code}: ${msg}`);
}
// 组件壳 → 内层可写控件
function resolveWritable(el) {
if (!el) return el;
if (el.matches && el.matches('input, textarea')) return el;
if (el.isContentEditable) return el;
const inner = el.querySelector && el.querySelector('input, textarea, [contenteditable="true"], [contenteditable=""]');
return inner || el;
}
function fireInput(el, win) {
const InputEvt = win.InputEvent || win.Event;
el.dispatchEvent(new InputEvt('input', { bubbles: true }));
el.dispatchEvent(new win.Event('change', { bubbles: true }));
}
// 原生 setter + 事件序列 + 回读
function setValue(rawEl, value, win) {
const el = resolveWritable(rawEl);
const v = value == null ? '' : String(value);
const tag = (el.tagName || '').toLowerCase();
if (el.isContentEditable) {
el.focus();
const sel = win.getSelection && win.getSelection();
if (sel) {
const range = el.ownerDocument.createRange();
range.selectNodeContents(el);
sel.removeAllRanges();
sel.addRange(range);
}
const ok = el.ownerDocument.execCommand && el.ownerDocument.execCommand('insertText', false, v);
if (!ok) { el.textContent = v; }
fireInput(el, win);
if (v && !(el.textContent || '').includes(v)) throw fail('value_not_applied', '富文本写入未生效');
return { success: true, action: 'type', value: v };
}
if (tag !== 'input' && tag !== 'textarea') {
throw fail('unsupported_widget', '元素不可输入文本');
}
el.focus();
const proto = tag === 'textarea' ? win.HTMLTextAreaElement.prototype : win.HTMLInputElement.prototype;
const desc = Object.getOwnPropertyDescriptor(proto, 'value');
const setter = desc && desc.set;
if (setter) setter.call(el, v); else el.value = v; // 兜底
fireInput(el, win);
el.dispatchEvent(new win.Event('blur', { bubbles: true }));
if (el.value !== v) throw fail('value_not_applied', '写入后回读不一致');
return { success: true, action: 'type', value: v };
}
function clickEl(el) {
if (typeof el.focus === 'function') { try { el.focus(); } catch (e) {} }
el.click();
return { success: true, action: 'click' };
}
// 原生 select 直接设值;自定义下拉/日期 open→click 匹配项
async function selectOption(el, value, doc, win) {
const v = value == null ? '' : String(value);
const tag = (el.tagName || '').toLowerCase();
if (tag === 'select') {
const opt = [...el.options].find((o) =>
o.value === v || (o.textContent || '').trim() === v || (o.textContent || '').includes(v));
if (!opt) throw fail('unsupported_widget', `原生 select 无选项 ${v}`);
el.value = opt.value;
fireInput(el, win);
return { success: true, action: 'select', value: v };
}
// 自定义下拉:点击展开 → 在选项/日历单元格里按文本匹配 → 点击
el.click();
await delay(180);
const optSelectors = '[role="option"], [role="gridcell"], [role="menuitemradio"], .ivu-select-item, li[role]';
const opts = [...doc.querySelectorAll(optSelectors)].filter((o) => o.offsetParent !== null || o.getClientRects().length);
const exact = opts.find((o) => (o.textContent || '').trim() === v);
const fuzzy = opts.find((o) => (o.textContent || '').trim().includes(v));
const match = exact || fuzzy;
if (!match) throw fail('unsupported_widget', `下拉未渲染匹配项 ${v}`);
match.click();
await delay(80);
return { success: true, action: 'select', value: v };
}
function isChecked(el) {
const t = el.matches && el.matches('input[type="checkbox"], input[type="radio"]')
? el : (el.querySelector && el.querySelector('input[type="checkbox"], input[type="radio"]'));
if (t) return !!t.checked;
const a = el.getAttribute && el.getAttribute('aria-checked');
return a === 'true';
}
function setChecked(el, want) {
const t = el.matches && el.matches('input[type="checkbox"], input[type="radio"]')
? el : (el.querySelector && el.querySelector('input[type="checkbox"], input[type="radio"]'));
if (t) {
if (!!t.checked !== want) t.click();
return { success: true, action: want ? 'check' : 'uncheck', checked: !!t.checked };
}
const a = el.getAttribute && el.getAttribute('aria-checked');
if (a != null) {
if ((a === 'true') !== want) el.click();
return { success: true, action: want ? 'check' : 'uncheck' };
}
throw fail('unsupported_widget', '非复选/单选元素');
}
export function createWebBackend() {
return {
name: 'web',
async perform(element, action, value, ctx) {
const win = (ctx && ctx.win) || element.ownerDocument.defaultView || window;
const doc = (ctx && ctx.doc) || element.ownerDocument;
switch (action) {
case 'click':
return clickEl(element);
case 'type':
case 'fill':
return setValue(element, value, win);
case 'clear':
return setValue(element, '', win);
case 'select':
return selectOption(element, value, doc, win);
case 'check':
return setChecked(element, true);
case 'uncheck':
return setChecked(element, false);
case 'toggle':
return setChecked(element, !isChecked(element));
case 'focus':
element.focus();
return { success: true, action: 'focus' };
case 'scroll':
element.scrollIntoView({ behavior: 'smooth', block: 'center' });
return { success: true, action: 'scroll' };
case 'hover':
element.dispatchEvent(new win.MouseEvent('mouseenter', { bubbles: true }));
element.dispatchEvent(new win.MouseEvent('mouseover', { bubbles: true }));
return { success: true, action: 'hover' };
default:
throw fail('not_actionable', `不支持的操作: ${action}`);
}
},
};
}
export default createWebBackend;