mirror of
https://github.com/kuaifan/dootask.git
synced 2026-07-31 02:15:57 +00:00
页面操作(doo page → /ws → 浮窗 operation-module → action-executor)原用手写 iView-only 选择器 + 朴素 value= 赋值,在 React+Radix 同源 iframe 业务表单上几乎 填不进。重构为「复用 Playwright 注入脚本的描述层 + 按环境选后端的执行层」: 描述层(共用): - 新增 aria/aria-bundle.js,vendoring Playwright ariaSnapshot/roleUtils/domUtils 子集预构建为 ESM(Apache-2.0 NOTICE + 固定上游 commit),产出 YAML 快照与活的 ref→Element Map。 - page-context-collector 改用 buildSnapshot 采集,operation-module 经 setRefMap 以 Map 直接持有元素取代 selector 反查。 执行层(按环境选后端,失败回退): - web-backend:原生 prototype value setter + beforeinput/input/change/blur 序列 + 回读校验(不符报 value_not_applied,绝不假报成功);自定义下拉/日期 open→click。 - electron-backend + electron/lib/page-input.js:经 webContents.debugger 走 CDP Input 域产生 isTrusted=true 可信输入(insertText/dispatchMouseEvent/KeyEvent), 任一步失败回退 Web 后端。 - 复杂控件(动态明细表/成员选择器/文件上传)如实返回 unsupported_widget。 协议与链路(doo page、active-context 失效守卫、导航类 action)保持不变。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
197 lines
6.6 KiB
JavaScript
Vendored
197 lines
6.6 KiB
JavaScript
Vendored
/**
|
||
* AI 助手前端操作模块
|
||
*
|
||
* 集成页面上下文收集器和操作执行器,供 AI 助手组件执行页面操作。
|
||
* 传输层已并入主程序常驻 WebSocket(/ws):后端经 assistant/operation/dispatch
|
||
* 推送 type=operation 消息,浮窗组件收到后调用本模块 handleRequest 执行并回包,
|
||
* 不再单独连接 MCP 的 operation WebSocket。
|
||
*/
|
||
|
||
import { collectPageContext, searchByVector } from './page-context-collector';
|
||
import { createActionExecutor } from './action-executor';
|
||
import { resolveActiveContext } from './active-context';
|
||
|
||
/**
|
||
* 创建操作模块实例
|
||
* @param {Object} options
|
||
* @param {Object} options.store - Vuex store 实例
|
||
* @param {Object} options.router - Vue Router 实例
|
||
* @returns {Object} 操作模块实例
|
||
*/
|
||
export function createOperationModule(options = {}) {
|
||
return new OperationModule(options);
|
||
}
|
||
|
||
class OperationModule {
|
||
constructor(options) {
|
||
this.store = options.store;
|
||
this.router = options.router;
|
||
this.executor = null;
|
||
}
|
||
|
||
/**
|
||
* 确保操作执行器已创建(惰性初始化)
|
||
*/
|
||
ensureExecutor() {
|
||
if (!this.executor) {
|
||
this.executor = createActionExecutor(this.store, this.router);
|
||
}
|
||
return this.executor;
|
||
}
|
||
|
||
/**
|
||
* 处理一次页面操作请求
|
||
* @param {string} action 操作类型
|
||
* @param {Object} payload 操作参数
|
||
*/
|
||
async handleRequest(action, payload) {
|
||
switch (action) {
|
||
case 'get_page_context':
|
||
return this.getPageContext(payload);
|
||
|
||
case 'execute_action':
|
||
return this.executeAction(payload);
|
||
|
||
case 'execute_element_action':
|
||
return this.executeElementAction(payload);
|
||
|
||
default:
|
||
throw new Error(`未知的操作类型: ${action}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 获取页面上下文
|
||
*/
|
||
async getPageContext(payload) {
|
||
this.ensureExecutor();
|
||
|
||
const includeElements = payload?.include_elements !== false;
|
||
const interactiveOnly = payload?.interactive_only || false;
|
||
const maxElements = payload?.max_elements || 100;
|
||
const query = payload?.query || '';
|
||
const offset = payload?.offset || 0;
|
||
const container = payload?.container || null;
|
||
const scope = payload?.scope || 'auto';
|
||
|
||
// 解析当前活动上下文:主界面,或最前打开的同源微应用 iframe
|
||
const active = resolveActiveContext(this.store, scope);
|
||
|
||
// 微应用打开但不可读(跨源 / 未就绪 / 指定 app 但无应用):优雅降级,不采集
|
||
if (active.kind === 'app' && !active.reachable) {
|
||
const reasonText = active.reason === 'cross_origin'
|
||
? '当前应用为跨源页面,无法读取其内部元素'
|
||
: active.reason === 'no_app'
|
||
? '当前没有打开任何微应用'
|
||
: '当前应用尚未加载完成';
|
||
const base = collectPageContext(this.store, { include_elements: false });
|
||
base.elements = [];
|
||
base.element_count = 0;
|
||
base.total_count = 0;
|
||
base.has_more = false;
|
||
base.frame = {
|
||
scope: 'app',
|
||
app_name: active.appName || null,
|
||
operable: false,
|
||
reachable: false,
|
||
reason: active.reason,
|
||
};
|
||
base.hint = `${reasonText}。可改用主界面(scope=main)操作,或改用数据命令完成。`;
|
||
this.executor.setRefMap(new Map(), null);
|
||
return base;
|
||
}
|
||
|
||
const doc = active.doc || document;
|
||
|
||
let context = collectPageContext(this.store, {
|
||
doc,
|
||
include_elements: includeElements,
|
||
interactive_only: interactiveOnly,
|
||
max_elements: maxElements,
|
||
offset,
|
||
container,
|
||
query,
|
||
});
|
||
|
||
// 如果有 query 且关键词匹配失败,尝试向量搜索
|
||
if (query && !context.keyword_matched) {
|
||
const allContext = collectPageContext(this.store, {
|
||
doc,
|
||
include_elements: true,
|
||
interactive_only: interactiveOnly,
|
||
max_elements: 200,
|
||
offset: 0,
|
||
container,
|
||
});
|
||
|
||
if (allContext.elements.length > 0) {
|
||
const vectorMatches = await searchByVector(this.store, query, allContext.elements, 10);
|
||
if (vectorMatches.length > 0) {
|
||
context.elements = vectorMatches;
|
||
context.element_count = vectorMatches.length;
|
||
context.total_count = vectorMatches.length;
|
||
context.has_more = false;
|
||
context.vector_matched = true;
|
||
context.refElements = allContext.refElements; // 全量 Map 覆盖向量命中的 ref
|
||
context.ref_map = {};
|
||
for (const el of vectorMatches) {
|
||
if (el.ref) {
|
||
context.ref_map[el.ref] = { role: el.role, name: el.name };
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 标注本次采集所在的上下文(主界面 / 微应用),让模型清楚在操作谁
|
||
context.frame = {
|
||
scope: active.scope,
|
||
app_name: active.appName || null,
|
||
operable: true,
|
||
};
|
||
|
||
// 将 ref→Element 实时 Map 与活动上下文存入 executor,供后续元素操作解析(含失效守卫)
|
||
if (context.refElements && this.executor) {
|
||
this.executor.setRefMap(context.refElements, active);
|
||
}
|
||
delete context.refElements; // Map 不参与序列化回包
|
||
|
||
return context;
|
||
}
|
||
|
||
/**
|
||
* 执行业务操作
|
||
*/
|
||
async executeAction(payload) {
|
||
this.ensureExecutor();
|
||
|
||
const actionName = payload?.name;
|
||
const params = payload?.params || {};
|
||
|
||
if (!actionName) {
|
||
throw new Error('缺少操作名称');
|
||
}
|
||
|
||
return this.executor.executeAction(actionName, params);
|
||
}
|
||
|
||
/**
|
||
* 执行元素操作
|
||
*/
|
||
async executeElementAction(payload) {
|
||
this.ensureExecutor();
|
||
|
||
const elementUid = payload?.element_uid;
|
||
const action = payload?.action;
|
||
const value = payload?.value;
|
||
|
||
if (!elementUid || !action) {
|
||
throw new Error('缺少必要参数');
|
||
}
|
||
|
||
return this.executor.executeElementAction(elementUid, action, value);
|
||
}
|
||
}
|
||
|
||
export default createOperationModule;
|