diff --git a/resources/assets/js/components/AIAssistant/action-executor.js b/resources/assets/js/components/AIAssistant/action-executor.js new file mode 100644 index 000000000..c19e4ba80 --- /dev/null +++ b/resources/assets/js/components/AIAssistant/action-executor.js @@ -0,0 +1,332 @@ +/** + * 操作执行器 + * + * 执行来自 MCP 工具的操作请求,包括: + * - 导航操作(打开任务、切换项目、跳转页面等) + * - 元素级操作(点击、输入等,作为兜底) + * + * 注意:数据操作(创建任务、发送消息等)应使用 MCP DooTask 工具直接调用 API, + * 本模块只负责前端导航和 UI 操作。 + */ + +import { findElementByRef } from './page-context-collector'; + +/** + * 创建操作执行器 + * @param {Object} store - Vuex store 实例 + * @param {Object} router - Vue Router 实例 + * @returns {Object} 执行器实例 + */ +export function createActionExecutor(store, router) { + return new ActionExecutor(store, router); +} + +class ActionExecutor { + constructor(store, router) { + this.store = store; + this.router = router; + + // 导航操作注册表 + this.actionHandlers = { + // 打开资源详情(通过 Vuex action) + open_task: this.openTask.bind(this), + open_dialog: this.openDialog.bind(this), + + // 页面导航(通过 goForward) + open_project: this.openProject.bind(this), + open_file: this.openFile.bind(this), + open_folder: this.openFolder.bind(this), + + // 功能页面导航 + navigate_to_dashboard: this.navigateToDashboard.bind(this), + navigate_to_messenger: this.navigateToMessenger.bind(this), + navigate_to_calendar: this.navigateToCalendar.bind(this), + navigate_to_files: this.navigateToFiles.bind(this), + + // 别名支持 + goto_task: this.openTask.bind(this), + goto_project: this.openProject.bind(this), + goto_dialog: this.openDialog.bind(this), + navigate_to_task: this.openTask.bind(this), + navigate_to_project: this.openProject.bind(this), + navigate_to_dialog: this.openDialog.bind(this), + }; + } + + /** + * 执行导航操作 + * @param {string} actionName - 操作名称 + * @param {Object} params - 操作参数 + * @returns {Promise} 执行结果 + */ + async executeAction(actionName, params = {}) { + // 智能解析操作名,支持 open_task_358 这样的格式 + const { normalizedAction, extractedParams } = this.parseActionName(actionName); + const mergedParams = { ...extractedParams, ...params }; + + const handler = this.actionHandlers[normalizedAction]; + if (!handler) { + throw new Error(`不支持的操作: ${actionName}。支持的操作: ${Object.keys(this.actionHandlers).join(', ')}`); + } + + try { + const result = await handler(mergedParams); + return { + success: true, + action: normalizedAction, + result, + }; + } catch (error) { + throw new Error(`执行操作失败: ${error.message}`); + } + } + + /** + * 解析操作名,提取嵌入的参数 + * 支持格式: open_task_358 -> { normalizedAction: 'open_task', extractedParams: { task_id: 358 } } + */ + parseActionName(actionName) { + const patterns = [ + { regex: /^(open_task|goto_task|navigate_to_task)_(\d+)$/, paramName: 'task_id' }, + { regex: /^(open_project|goto_project|navigate_to_project)_(\d+)$/, paramName: 'project_id' }, + { regex: /^(open_dialog|goto_dialog|navigate_to_dialog)_(\d+)$/, paramName: 'dialog_id' }, + { regex: /^(open_file)_(\d+)$/, paramName: 'file_id' }, + { regex: /^(open_folder)_(\d+)$/, paramName: 'folder_id' }, + ]; + + for (const { regex, paramName } of patterns) { + const match = actionName.match(regex); + if (match) { + return { + normalizedAction: match[1], + extractedParams: { [paramName]: parseInt(match[2], 10) }, + }; + } + } + + return { normalizedAction: actionName, extractedParams: {} }; + } + + // ========== 打开资源详情 ========== + + /** + * 打开任务详情 + */ + async openTask(params) { + const taskId = params.task_id; + if (!taskId) { + throw new Error('缺少 task_id 参数'); + } + + this.store.dispatch('openTask', taskId); + return { opened: true, task_id: taskId }; + } + + /** + * 打开对话 + */ + async openDialog(params) { + const dialogId = params.dialog_id; + if (!dialogId) { + throw new Error('缺少 dialog_id 参数'); + } + + // 支持高级参数:跳转到特定消息 + const dialogParams = params.msg_id + ? { dialog_id: dialogId, search_msg_id: params.msg_id } + : dialogId; + + this.store.dispatch('openDialog', dialogParams); + return { opened: true, dialog_id: dialogId }; + } + + // ========== 页面导航 ========== + + /** + * 打开/切换到项目 + */ + async openProject(params) { + const projectId = params.project_id; + if (!projectId) { + throw new Error('缺少 project_id 参数'); + } + + window.$A.goForward({ name: 'manage-project', params: { projectId } }); + return { navigated: true, project_id: projectId }; + } + + /** + * 打开文件预览 + */ + async openFile(params) { + const fileId = params.file_id; + if (!fileId) { + throw new Error('缺少 file_id 参数'); + } + + window.$A.goForward({ name: 'manage-file', params: { fileId } }); + return { navigated: true, file_id: fileId }; + } + + /** + * 打开文件夹 + */ + async openFolder(params) { + const folderId = params.folder_id; + if (!folderId) { + throw new Error('缺少 folder_id 参数'); + } + + window.$A.goForward({ name: 'manage-file', params: { folderId, fileId: null } }); + return { navigated: true, folder_id: folderId }; + } + + /** + * 导航到仪表盘 + */ + async navigateToDashboard() { + window.$A.goForward({ name: 'manage-dashboard' }); + return { navigated: true, page: 'dashboard' }; + } + + /** + * 导航到消息页面 + */ + async navigateToMessenger() { + window.$A.goForward({ name: 'manage-messenger' }); + return { navigated: true, page: 'messenger' }; + } + + /** + * 导航到日历页面 + */ + async navigateToCalendar() { + window.$A.goForward({ name: 'manage-calendar' }); + return { navigated: true, page: 'calendar' }; + } + + /** + * 导航到文件管理页面 + */ + async navigateToFiles() { + window.$A.goForward({ name: 'manage-file' }); + return { navigated: true, page: 'files' }; + } + + // ========== 元素级操作 ========== + + /** + * 设置当前的 refMap(由 operation-module 在获取上下文后调用) + */ + setRefMap(refMap) { + this.currentRefMap = refMap; + } + + /** + * 执行元素级操作 + * @param {string} elementUid - 元素标识 (e1, e2, ... 或选择器) + * @param {string} action - 操作类型 + * @param {string} value - 操作值 + * @returns {Promise} 执行结果 + */ + async executeElementAction(elementUid, action, value) { + const element = this.findElement(elementUid); + if (!element) { + throw new Error(`找不到元素: ${elementUid}`); + } + + switch (action) { + case 'click': + element.click(); + return { success: true, action: 'click', element: elementUid }; + + case 'type': + if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA' || element.contentEditable === 'true') { + element.focus(); + if (element.contentEditable === 'true') { + element.textContent = value || ''; + } else { + element.value = value || ''; + } + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + return { success: true, action: 'type', value, element: elementUid }; + } + throw new Error('元素不支持输入操作'); + + case 'select': + if (element.tagName === 'SELECT') { + element.value = value; + element.dispatchEvent(new Event('change', { bubbles: true })); + return { success: true, action: 'select', value, element: elementUid }; + } + // iView Select 组件 - 先点击打开下拉 + element.click(); + await this.delay(200); + const options = document.querySelectorAll('.ivu-select-dropdown-list .ivu-select-item'); + for (const option of options) { + if (option.textContent.trim().includes(value)) { + option.click(); + return { success: true, action: 'select', value, element: elementUid }; + } + } + throw new Error(`找不到选项: ${value}`); + + case 'focus': + element.focus(); + return { success: true, action: 'focus', element: elementUid }; + + case 'scroll': + element.scrollIntoView({ behavior: 'smooth', block: 'center' }); + return { success: true, action: 'scroll', element: elementUid }; + + case 'hover': + element.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true })); + element.dispatchEvent(new MouseEvent('mouseover', { bubbles: true })); + return { success: true, action: 'hover', element: elementUid }; + + default: + throw new Error(`不支持的元素操作: ${action}`); + } + } + + /** + * 查找元素 + * 支持多种格式:e1, @e1, ref=e1, CSS选择器 + */ + findElement(identifier) { + let ref = null; + if (identifier.startsWith('@')) { + ref = identifier.slice(1); + } else if (identifier.startsWith('ref=')) { + ref = identifier.slice(4); + } else if (/^e\d+$/.test(identifier)) { + ref = identifier; + } + + // 如果是 ref 格式,使用 refMap 查找 + if (ref && this.currentRefMap) { + const element = findElementByRef(ref, this.currentRefMap); + if (element) return element; + } + + // 尝试作为 CSS 选择器 + try { + const element = document.querySelector(identifier); + if (element) return element; + } catch (e) { + // 选择器无效,忽略 + } + + return null; + } + + /** + * 延迟工具方法 + */ + delay(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} + +export default createActionExecutor; diff --git a/resources/assets/js/components/AIAssistant/float-button.vue b/resources/assets/js/components/AIAssistant/float-button.vue index f79591669..8394253bf 100644 --- a/resources/assets/js/components/AIAssistant/float-button.vue +++ b/resources/assets/js/components/AIAssistant/float-button.vue @@ -27,6 +27,7 @@ import {mapState} from "vuex"; import emitter from "../../store/events"; import {withLanguagePreferencePrompt} from "../../utils/ai"; import {getPageContext, getSceneKey} from "./page-context"; +import {createOperationModule} from "./operation-module"; export default { name: 'AIAssistantFloatButton', @@ -50,6 +51,9 @@ export default { collapseDelay: 1000, // 收起延迟(毫秒) collapseTimer: null, // 收起定时器 record: {}, + // 前端操作模块 + operationModule: null, + operationSessionId: null, }; }, @@ -147,15 +151,19 @@ export default { this.loadPosition(); window.addEventListener('resize', this.onResize); emitter.on('openAIAssistantGlobal', this.onClick); + emitter.on('aiAssistantClosed', this.onAssistantClosed); + this.initOperationModule(); }, beforeDestroy() { window.removeEventListener('resize', this.onResize); emitter.off('openAIAssistantGlobal', this.onClick); + emitter.off('aiAssistantClosed', this.onAssistantClosed); document.removeEventListener('mousemove', this.onMouseMove); document.removeEventListener('mouseup', this.onMouseUp); document.removeEventListener('contextmenu', this.onContextMenu); this.clearCollapseTimer(); + this.destroyOperationModule(); }, methods: { @@ -363,6 +371,9 @@ export default { const routeParams = this.$route?.params || {}; const sceneKey = getSceneKey(this.$store, routeParams); + // 启用前端操作模块 + this.enableOperationModule(); + emitter.emit('openAIAssistant', { displayMode: 'chat', sessionKey: 'global', @@ -373,6 +384,13 @@ export default { }); }, + /** + * AI 助手关闭事件 + */ + onAssistantClosed() { + this.disableOperationModule(); + }, + /** * 处理发送前的上下文准备 * 在发送时动态获取当前页面上下文,确保上下文与用户当前所在页面一致 @@ -383,8 +401,14 @@ export default { const routeParams = this.$route?.params || {}; const {systemPrompt} = getPageContext(this.$store, routeParams); + // 添加操作会话信息 + let operationContext = ''; + if (this.operationSessionId) { + operationContext = `\n\n前端操作会话已建立,session_id: ${this.operationSessionId}。你可以使用 get_page_context、execute_action、execute_element_action 工具直接操作用户的页面。`; + } + const prepared = [ - ['system', withLanguagePreferencePrompt(systemPrompt)], + ['system', withLanguagePreferencePrompt(systemPrompt + operationContext)], ]; if (context.length > 0) { @@ -392,7 +416,57 @@ export default { } return prepared; - } + }, + + /** + * 初始化操作模块 + */ + initOperationModule() { + if (this.operationModule) { + return; + } + + this.operationModule = createOperationModule({ + store: this.$store, + router: this.$router, + onSessionReady: (sessionId) => { + this.operationSessionId = sessionId; + }, + onSessionLost: () => { + this.operationSessionId = null; + }, + }); + }, + + /** + * 启用操作模块 + */ + enableOperationModule() { + if (this.operationModule) { + this.operationModule.enable(); + } + }, + + /** + * 禁用操作模块 + */ + disableOperationModule() { + if (this.operationModule) { + this.operationModule.disable(); + this.operationSessionId = null; + } + }, + + /** + * 销毁操作模块 + */ + destroyOperationModule() { + if (this.operationModule) { + this.operationModule.disable(); + this.operationModule = null; + this.operationSessionId = null; + } + }, }, }; diff --git a/resources/assets/js/components/AIAssistant/index.vue b/resources/assets/js/components/AIAssistant/index.vue index 5b79ab588..b65113c2f 100644 --- a/resources/assets/js/components/AIAssistant/index.vue +++ b/resources/assets/js/components/AIAssistant/index.vue @@ -273,10 +273,16 @@ export default { }, welcomePromptsKey: { handler() { - this.refreshWelcomePromptsDebounced(); + this.refreshWelcomePromptsDebounced?.(); }, immediate: true, }, + showModal(value) { + if (!value) { + // 弹窗关闭时通知操作模块 + emitter.emit('aiAssistantClosed'); + } + }, }, methods: { /** diff --git a/resources/assets/js/components/AIAssistant/operation-client.js b/resources/assets/js/components/AIAssistant/operation-client.js new file mode 100644 index 000000000..0dc3a3eaf --- /dev/null +++ b/resources/assets/js/components/AIAssistant/operation-client.js @@ -0,0 +1,231 @@ +/** + * AI 助手前端操作 WebSocket 客户端 + * + * 负责与 MCP Server 建立 WebSocket 连接, + * 接收来自 MCP 工具的请求并返回响应。 + */ + +const WS_PATH = '/apps/mcp_server/mcp/operation'; +const RECONNECT_DELAY = 3000; +const MAX_RECONNECT_ATTEMPTS = 5; + +/** + * 前端操作客户端 + */ +export class OperationClient { + /** + * @param {Object} options + * @param {Function} options.getToken - 获取用户 token 的函数 + * @param {Function} options.onRequest - 处理请求的回调函数 + * @param {Function} options.onConnected - 连接成功回调 + * @param {Function} options.onDisconnected - 断开连接回调 + * @param {Function} options.onError - 错误回调 + */ + constructor(options = {}) { + this.getToken = options.getToken; + this.onRequest = options.onRequest; + this.onConnected = options.onConnected; + this.onDisconnected = options.onDisconnected; + this.onError = options.onError; + + this.ws = null; + this.sessionId = null; + this.expiresAt = null; + this.reconnectAttempts = 0; + this.reconnectTimer = null; + this.isConnecting = false; + this.isManualClose = false; + } + + /** + * 建立 WebSocket 连接 + */ + connect() { + if (this.ws && (this.ws.readyState === WebSocket.CONNECTING || this.ws.readyState === WebSocket.OPEN)) { + return; + } + + if (this.isConnecting) { + return; + } + + this.isConnecting = true; + this.isManualClose = false; + + const token = this.getToken?.(); + if (!token) { + this.isConnecting = false; + this.onError?.('未登录或 token 不可用'); + return; + } + + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const host = window.location.host; + const url = `${protocol}//${host}${WS_PATH}?token=${encodeURIComponent(token)}`; + + try { + this.ws = new WebSocket(url); + this.setupEventHandlers(); + } catch (error) { + this.isConnecting = false; + this.onError?.(error.message); + } + } + + /** + * 设置 WebSocket 事件处理器 + */ + setupEventHandlers() { + this.ws.onopen = () => { + this.isConnecting = false; + this.reconnectAttempts = 0; + }; + + this.ws.onmessage = (event) => { + this.handleMessage(event.data); + }; + + this.ws.onclose = (event) => { + this.isConnecting = false; + this.sessionId = null; + this.onDisconnected?.(); + + if (!this.isManualClose && this.reconnectAttempts < MAX_RECONNECT_ATTEMPTS) { + this.scheduleReconnect(); + } + }; + + this.ws.onerror = () => { + this.isConnecting = false; + this.onError?.('WebSocket 连接错误'); + }; + } + + /** + * 处理收到的消息 + */ + handleMessage(data) { + let msg; + try { + msg = JSON.parse(data); + } catch { + return; + } + + switch (msg.type) { + case 'connected': + this.sessionId = msg.session_id; + this.expiresAt = msg.expires_at; + this.onConnected?.(this.sessionId); + break; + + case 'request': + this.handleRequest(msg); + break; + + case 'pong': + // 心跳响应 + break; + } + } + + /** + * 处理来自 MCP 的请求 + */ + async handleRequest(msg) { + const { id, action, payload } = msg; + + if (!this.onRequest) { + this.sendResponse(id, false, null, '请求处理器未配置'); + return; + } + + try { + const result = await this.onRequest(action, payload); + this.sendResponse(id, true, result, null); + } catch (error) { + const errorMsg = error.message || '操作执行失败'; + this.sendResponse(id, false, null, errorMsg); + } + } + + /** + * 发送响应 + */ + sendResponse(id, success, data, error) { + if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { + return; + } + + const response = { + type: 'response', + id, + success, + data, + error, + }; + + this.ws.send(JSON.stringify(response)); + } + + /** + * 发送心跳 + */ + ping() { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify({ type: 'ping' })); + } + } + + /** + * 安排重连 + */ + scheduleReconnect() { + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + } + + this.reconnectAttempts++; + const delay = RECONNECT_DELAY * this.reconnectAttempts; + + this.reconnectTimer = setTimeout(() => { + this.connect(); + }, delay); + } + + /** + * 断开连接 + */ + disconnect() { + this.isManualClose = true; + + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + + if (this.ws) { + this.ws.close(); + this.ws = null; + } + + this.sessionId = null; + this.reconnectAttempts = 0; + } + + /** + * 获取当前 session ID + */ + getSessionId() { + return this.sessionId; + } + + /** + * 检查是否已连接 + */ + isConnected() { + return this.ws && this.ws.readyState === WebSocket.OPEN && this.sessionId; + } +} + +export default OperationClient; diff --git a/resources/assets/js/components/AIAssistant/operation-module.js b/resources/assets/js/components/AIAssistant/operation-module.js new file mode 100644 index 000000000..ba04ae11a --- /dev/null +++ b/resources/assets/js/components/AIAssistant/operation-module.js @@ -0,0 +1,220 @@ +/** + * AI 助手前端操作模块 + * + * 集成 WebSocket 客户端、页面上下文收集器和操作执行器, + * 提供给 AI 助手组件使用。 + */ + +import { OperationClient } from './operation-client'; +import { collectPageContext } from './page-context-collector'; +import { createActionExecutor } from './action-executor'; + +/** + * 创建操作模块实例 + * @param {Object} options + * @param {Object} options.store - Vuex store 实例 + * @param {Object} options.router - Vue Router 实例 + * @returns {Object} 操作模块实例 + */ +export function createOperationModule(options = {}) { + return new OperationModule(options); +} + +class OperationModule { + constructor(options) { + this.store = options.store; + this.router = options.router; + this.enabled = false; + this.client = null; + this.executor = null; + this.sessionId = null; + + // 回调函数 + this.onSessionReady = options.onSessionReady; + this.onSessionLost = options.onSessionLost; + this.onError = options.onError; + } + + /** + * 启用操作模块 + */ + enable() { + if (this.enabled) { + return; + } + + this.enabled = true; + + // 创建操作执行器 + this.executor = createActionExecutor(this.store, this.router); + + // 创建 WebSocket 客户端 + this.client = new OperationClient({ + getToken: () => this.store.state.userToken, + onRequest: this.handleRequest.bind(this), + onConnected: this.handleConnected.bind(this), + onDisconnected: this.handleDisconnected.bind(this), + onError: this.handleError.bind(this), + }); + + // 建立连接 + this.client.connect(); + + // 设置心跳 + this.heartbeatTimer = setInterval(() => { + if (this.client) { + this.client.ping(); + } + }, 30000); + } + + /** + * 禁用操作模块 + */ + disable() { + if (!this.enabled) { + return; + } + + this.enabled = false; + + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + + if (this.client) { + this.client.disconnect(); + this.client = null; + } + + this.executor = null; + this.sessionId = null; + } + + /** + * 处理来自 MCP 的请求 + */ + async handleRequest(action, payload) { + switch (action) { + case 'get_page_context': + return this.getPageContext(payload); + + case 'execute_action': + return this.executeAction(payload); + + case 'execute_element_action': + return this.executeElementAction(payload); + + default: + throw new Error(`未知的操作类型: ${action}`); + } + } + + /** + * 获取页面上下文 + */ + getPageContext(payload) { + const includeElements = payload?.include_elements !== false; + const interactiveOnly = payload?.interactive_only || false; + const maxElements = payload?.max_elements || 100; + + const context = collectPageContext(this.store, { + include_elements: includeElements, + interactive_only: interactiveOnly, + max_elements: maxElements, + }); + + // 将 refMap 存储到 executor,供后续元素操作使用 + if (context.ref_map && this.executor) { + this.executor.setRefMap(context.ref_map); + } + + return context; + } + + /** + * 执行业务操作 + */ + async executeAction(payload) { + if (!this.executor) { + throw new Error('操作执行器未初始化'); + } + + const actionName = payload?.name; + const params = payload?.params || {}; + + if (!actionName) { + throw new Error('缺少操作名称'); + } + + return this.executor.executeAction(actionName, params); + } + + /** + * 执行元素操作 + */ + async executeElementAction(payload) { + if (!this.executor) { + throw new Error('操作执行器未初始化'); + } + + const elementUid = payload?.element_uid; + const action = payload?.action; + const value = payload?.value; + + if (!elementUid || !action) { + throw new Error('缺少必要参数'); + } + + return this.executor.executeElementAction(elementUid, action, value); + } + + /** + * 处理连接成功 + */ + handleConnected(sessionId) { + this.sessionId = sessionId; + this.onSessionReady?.(sessionId); + } + + /** + * 处理连接断开 + */ + handleDisconnected() { + this.sessionId = null; + this.onSessionLost?.(); + } + + /** + * 处理错误 + */ + handleError(error) { + this.onError?.(error); + } + + /** + * 获取当前 session ID + */ + getSessionId() { + return this.sessionId; + } + + /** + * 检查是否已连接 + */ + isConnected() { + return this.client?.isConnected() || false; + } + + /** + * 重新连接 + */ + reconnect() { + if (this.client) { + this.client.connect(); + } + } +} + +export default createOperationModule; diff --git a/resources/assets/js/components/AIAssistant/page-context-collector.js b/resources/assets/js/components/AIAssistant/page-context-collector.js new file mode 100644 index 000000000..aeb39a67d --- /dev/null +++ b/resources/assets/js/components/AIAssistant/page-context-collector.js @@ -0,0 +1,737 @@ +/** + * 页面上下文收集器 + * + * 借鉴 agent-browser 项目的设计思想,基于 ARIA 角色收集页面元素。 + * 提供结构化的页面快照,包括可交互元素和内容元素。 + */ + +// ========== ARIA 角色分类 ========== + +// 可交互角色 - 这些元素可以被点击、输入等 +const INTERACTIVE_ROLES = new Set([ + 'button', 'link', 'textbox', 'checkbox', 'radio', + 'combobox', 'listbox', 'menuitem', 'menuitemcheckbox', + 'menuitemradio', 'option', 'searchbox', 'slider', + 'spinbutton', 'switch', 'tab', 'treeitem', 'gridcell', +]); + +// 内容角色 - 这些元素包含重要内容 +const CONTENT_ROLES = new Set([ + 'heading', 'cell', 'columnheader', 'rowheader', + 'listitem', 'article', 'region', 'main', 'navigation', + 'img', 'figure', +]); + +// HTML 元素到 ARIA 角色的映射 +const ELEMENT_ROLE_MAP = { + 'button': 'button', + 'a': 'link', + 'input': (el) => { + const type = el.type?.toLowerCase() || 'text'; + switch (type) { + case 'checkbox': return 'checkbox'; + case 'radio': return 'radio'; + case 'submit': + case 'reset': + case 'button': return 'button'; + case 'search': return 'searchbox'; + case 'range': return 'slider'; + case 'number': return 'spinbutton'; + default: return 'textbox'; + } + }, + 'textarea': 'textbox', + 'select': 'combobox', + 'option': 'option', + 'h1': 'heading', + 'h2': 'heading', + 'h3': 'heading', + 'h4': 'heading', + 'h5': 'heading', + 'h6': 'heading', + 'img': 'img', + 'nav': 'navigation', + 'main': 'main', + 'article': 'article', + 'li': 'listitem', + 'td': 'cell', + 'th': 'columnheader', +}; + +// ========== 元素收集器 ========== + +/** + * 收集当前页面上下文 + * @param {Object} store - Vuex store 实例 + * @param {Object} options - 收集选项 + * @param {boolean} options.include_elements - 是否包含可交互元素 + * @param {boolean} options.interactive_only - 仅返回可交互元素 + * @param {number} options.max_elements - 每页最大元素数量,默认 50 + * @param {number} options.offset - 跳过前 N 个元素(分页用),默认 0 + * @param {string} options.container - 容器选择器,只扫描该容器内的元素 + * @returns {Object} 页面上下文 + */ +export function collectPageContext(store, options = {}) { + const routeName = store?.state?.routeName; + const includeElements = options.include_elements !== false; + const interactiveOnly = options.interactive_only || false; + const maxElements = options.max_elements || 50; + const offset = options.offset || 0; + const container = options.container || null; + + // 基础上下文 + const context = { + page_type: routeName || 'unknown', + page_url: window.location.href, + page_title: document.title, + timestamp: Date.now(), + elements: [], + element_count: 0, + total_count: 0, + offset: offset, + has_more: false, + available_actions: getAvailableActions(routeName, store), + }; + + // 收集可交互元素 + if (includeElements) { + const result = collectElements({ + interactiveOnly, + maxElements, + offset, + container, + }); + context.elements = result.elements; + context.element_count = result.elements.length; + context.total_count = result.totalCount; + context.has_more = result.hasMore; + context.ref_map = result.refMap; + } + + return context; +} + +/** + * 根据页面类型获取可用的导航操作 + * @param {string} routeName - 路由名称 + * @param {Object} store - Vuex store 实例 + * @returns {Array} 可用操作列表 + */ +function getAvailableActions(routeName, store) { + // 通用导航操作 - 在所有页面都可用 + const commonActions = [ + { + name: 'navigate_to_dashboard', + description: '跳转到仪表盘', + }, + { + name: 'navigate_to_messenger', + description: '跳转到消息页面', + }, + { + name: 'navigate_to_calendar', + description: '跳转到日历页面', + }, + { + name: 'navigate_to_files', + description: '跳转到文件管理页面', + }, + ]; + + // 根据页面类型添加特定操作 + const pageSpecificActions = []; + + switch (routeName) { + case 'manage-project': + // 项目页面:可以打开任务 + pageSpecificActions.push({ + name: 'open_task', + description: '打开任务详情', + params: { task_id: '任务ID' }, + }); + break; + + case 'manage-messenger': + // 消息页面:可以打开特定对话 + pageSpecificActions.push({ + name: 'open_dialog', + description: '打开/切换对话', + params: { dialog_id: '对话ID', msg_id: '(可选)跳转到指定消息' }, + }); + break; + + case 'manage-file': + // 文件页面:可以打开文件夹或文件 + pageSpecificActions.push( + { + name: 'open_folder', + description: '打开文件夹', + params: { folder_id: '文件夹ID' }, + }, + { + name: 'open_file', + description: '打开文件预览', + params: { file_id: '文件ID' }, + } + ); + break; + + case 'manage-dashboard': + // 仪表盘:可以快速跳转到项目或打开任务 + pageSpecificActions.push( + { + name: 'open_project', + description: '打开/切换到项目', + params: { project_id: '项目ID' }, + }, + { + name: 'open_task', + description: '打开任务详情', + params: { task_id: '任务ID' }, + } + ); + break; + + default: + // 其他页面:提供基础的打开操作 + pageSpecificActions.push( + { + name: 'open_project', + description: '打开/切换到项目', + params: { project_id: '项目ID' }, + }, + { + name: 'open_task', + description: '打开任务详情', + params: { task_id: '任务ID' }, + }, + { + name: 'open_dialog', + description: '打开对话', + params: { dialog_id: '对话ID' }, + } + ); + } + + return [...pageSpecificActions, ...commonActions]; +} + +/** + * 收集页面元素 + * @param {Object} options + * @param {boolean} options.interactiveOnly - 仅返回可交互元素 + * @param {number} options.maxElements - 每页最大元素数量 + * @param {number} options.offset - 跳过前 N 个元素 + * @param {string} options.container - 容器选择器 + * @returns {Object} { elements, refMap, totalCount, hasMore } + */ +function collectElements(options = {}) { + const { + interactiveOnly = false, + maxElements = 50, + offset = 0, + container = null, + } = options; + + // 确定查询的根元素 + let rootElement = document; + if (container) { + rootElement = document.querySelector(container); + if (!rootElement) { + return { elements: [], refMap: {}, totalCount: 0, hasMore: false }; + } + } + + // 角色+名称计数器,用于处理重复元素 + const roleNameCounter = new Map(); + + // 获取所有可能的交互元素选择器 + const selectors = [ + // 按钮类 + 'button', + '[role="button"]', + 'input[type="submit"]', + 'input[type="button"]', + 'input[type="reset"]', + '.ivu-btn', + // 链接 + 'a[href]', + '[role="link"]', + // 输入框 + 'input:not([type="hidden"])', + 'textarea', + '[role="textbox"]', + '[contenteditable="true"]', + // 选择器 + 'select', + '[role="combobox"]', + '[role="listbox"]', + '.ivu-select', + // 复选框和单选框 + 'input[type="checkbox"]', + 'input[type="radio"]', + '[role="checkbox"]', + '[role="radio"]', + '.ivu-checkbox', + '.ivu-radio', + // 菜单项 + '[role="menuitem"]', + '[role="tab"]', + '.ivu-menu-item', + '.ivu-tabs-tab', + // 可点击的图标和操作按钮 + '[class*="click"]', + '[class*="btn"]', + '[class*="action"]', + '.taskfont[title]', + // 表格单元格(可能可点击) + 'td[onclick]', + 'tr[onclick]', + ]; + + // 如果不仅限交互元素,添加内容元素选择器 + if (!interactiveOnly) { + selectors.push( + 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', + '[role="heading"]', + 'img[alt]', + 'nav', + 'main', + ); + } + + // 第一阶段:收集所有有效元素(不限数量) + const allValidElements = []; + const processedElements = new Set(); + + // 查询所有匹配的元素 + const candidateElements = rootElement.querySelectorAll(selectors.join(', ')); + + for (const el of candidateElements) { + processedElements.add(el); + + // 跳过不可见元素 + if (!isElementVisible(el)) continue; + + // 跳过禁用元素 + if (isElementDisabled(el)) continue; + + allValidElements.push({ el, fromPointerScan: false }); + } + + // 第二遍扫描:查找具有 cursor: pointer 但未被选择器匹配的元素 + const clickableSelectors = 'div, span, li, td, tr, section, article, aside, header, footer, label, i, svg'; + const potentialClickables = rootElement.querySelectorAll(clickableSelectors); + + for (const el of potentialClickables) { + if (processedElements.has(el)) continue; + + // 检查是否有 cursor: pointer 样式 + const computedStyle = window.getComputedStyle(el); + if (computedStyle.cursor !== 'pointer') continue; + + // 跳过不可见或禁用元素 + if (!isElementVisible(el)) continue; + if (isElementDisabled(el)) continue; + + processedElements.add(el); + allValidElements.push({ el, fromPointerScan: true }); + } + + // 第二阶段:应用分页,生成最终结果 + const totalCount = allValidElements.length; + const startIndex = offset; + const endIndex = Math.min(offset + maxElements, totalCount); + const hasMore = endIndex < totalCount; + + const elements = []; + const refMap = {}; + let refCounter = offset + 1; // ref 从 offset+1 开始,保持全局唯一 + + for (let i = startIndex; i < endIndex; i++) { + const { el, fromPointerScan } = allValidElements[i]; + + // 获取元素信息 + const elementInfo = extractElementInfo(el, refCounter); + if (!elementInfo) continue; + + // 如果是从 pointer 扫描来的,强制设为 button 角色 + if (fromPointerScan && elementInfo.role === 'generic') { + elementInfo.role = 'button'; + } + + // 处理重复的角色+名称组合 + const key = `${elementInfo.role}:${elementInfo.name || ''}`; + const count = roleNameCounter.get(key) || 0; + roleNameCounter.set(key, count + 1); + + if (count > 0) { + elementInfo.nth = count; + } + + // 生成 ref + const ref = `e${refCounter++}`; + elementInfo.ref = ref; + + // 存储到 refMap + refMap[ref] = { + role: elementInfo.role, + name: elementInfo.name, + selector: elementInfo.selector, + nth: elementInfo.nth, + }; + + elements.push(elementInfo); + } + + // 为重复元素添加 nth 标记 + for (const element of elements) { + const key = `${element.role}:${element.name || ''}`; + const roleCount = roleNameCounter.get(key); + // 只有真正重复的才保留 nth + if (roleCount <= 1) { + delete element.nth; + if (refMap[element.ref]) { + delete refMap[element.ref].nth; + } + } + } + + return { elements, refMap, totalCount, hasMore }; +} + +/** + * 提取元素信息 + * @param {Element} el + * @param {number} index + * @returns {Object|null} + */ +function extractElementInfo(el, index) { + const tagName = el.tagName.toLowerCase(); + const role = getElementRole(el); + + // 获取元素名称/文本 + const name = getElementName(el); + + // 生成选择器 + const selector = generateSelector(el); + + // 基础信息 + const info = { + role, + tag: tagName, + name: name || undefined, + selector, + }; + + // 添加特定属性 + if (el.id) { + info.id = el.id; + } + + if (el.type && (tagName === 'input' || tagName === 'button')) { + info.input_type = el.type; + } + + if (el.placeholder) { + info.placeholder = el.placeholder; + } + + if (el.value && (tagName === 'input' || tagName === 'textarea')) { + info.value = el.value.substring(0, 50); + } + + if (el.href && tagName === 'a') { + info.href = el.href; + } + + if (el.checked !== undefined) { + info.checked = el.checked; + } + + if (el.title) { + info.title = el.title; + } + + // 添加 aria 属性 + const ariaLabel = el.getAttribute('aria-label'); + if (ariaLabel) { + info.aria_label = ariaLabel; + } + + return info; +} + +/** + * 获取元素的 ARIA 角色 + * @param {Element} el + * @returns {string} + */ +function getElementRole(el) { + // 首先检查显式的 role 属性 + const explicitRole = el.getAttribute('role'); + if (explicitRole) { + return explicitRole; + } + + // 使用映射表 + const tagName = el.tagName.toLowerCase(); + const roleMapping = ELEMENT_ROLE_MAP[tagName]; + + if (typeof roleMapping === 'function') { + return roleMapping(el); + } + + if (typeof roleMapping === 'string') { + return roleMapping; + } + + // 检查是否可点击 + if (el.onclick || el.hasAttribute('onclick') || + el.style.cursor === 'pointer' || + window.getComputedStyle(el).cursor === 'pointer') { + return 'button'; + } + + return 'generic'; +} + +/** + * 获取元素的可访问名称 + * @param {Element} el + * @returns {string} + */ +function getElementName(el) { + // 优先级:aria-label > aria-labelledby > 内容文本 > title > placeholder > alt + + const ariaLabel = el.getAttribute('aria-label'); + if (ariaLabel) return ariaLabel.trim().substring(0, 100); + + const ariaLabelledBy = el.getAttribute('aria-labelledby'); + if (ariaLabelledBy) { + const labelEl = document.getElementById(ariaLabelledBy); + if (labelEl) { + return getTextContent(labelEl).substring(0, 100); + } + } + + // 对于输入元素,查找关联的 label + if (el.id) { + const label = document.querySelector(`label[for="${el.id}"]`); + if (label) { + return getTextContent(label).substring(0, 100); + } + } + + // 获取元素内的文本内容 + const text = getTextContent(el); + if (text) return text.substring(0, 100); + + // 其他属性 + if (el.title) return el.title.substring(0, 100); + if (el.placeholder) return el.placeholder.substring(0, 100); + if (el.alt) return el.alt.substring(0, 100); + if (el.value && (el.tagName === 'INPUT' || el.tagName === 'BUTTON')) { + return el.value.substring(0, 100); + } + + return ''; +} + +/** + * 获取元素的文本内容(排除子元素中隐藏的文本) + * @param {Element} el + * @returns {string} + */ +function getTextContent(el) { + // 克隆元素以避免修改原始 DOM + const clone = el.cloneNode(true); + + // 移除脚本和样式 + clone.querySelectorAll('script, style, [hidden], [aria-hidden="true"]').forEach(e => e.remove()); + + // 获取文本并清理 + let text = clone.textContent || clone.innerText || ''; + text = text.replace(/\s+/g, ' ').trim(); + + return text; +} + +/** + * 检查元素是否可见 + * @param {Element} el + * @returns {boolean} + */ +function isElementVisible(el) { + if (!el) return false; + + // 检查元素本身 + const style = window.getComputedStyle(el); + + if (style.display === 'none') return false; + if (style.visibility === 'hidden') return false; + if (style.opacity === '0') return false; + + // 检查边界框 + const rect = el.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) return false; + + // 检查是否在视口内或附近(允许稍微超出) + const viewportHeight = window.innerHeight; + const viewportWidth = window.innerWidth; + + // 元素完全在视口外 + if (rect.bottom < -100 || rect.top > viewportHeight + 100) return false; + if (rect.right < -100 || rect.left > viewportWidth + 100) return false; + + // 检查父元素的可见性 + let parent = el.parentElement; + while (parent) { + const parentStyle = window.getComputedStyle(parent); + if (parentStyle.display === 'none') return false; + if (parentStyle.visibility === 'hidden') return false; + parent = parent.parentElement; + } + + return true; +} + +/** + * 检查元素是否禁用 + * @param {Element} el + * @returns {boolean} + */ +function isElementDisabled(el) { + if (el.disabled) return true; + if (el.getAttribute('aria-disabled') === 'true') return true; + + // 检查是否在禁用的 fieldset 中 + const fieldset = el.closest('fieldset'); + if (fieldset && fieldset.disabled) { + // legend 中的元素不受影响 + const legend = fieldset.querySelector('legend'); + if (legend && legend.contains(el)) return false; + return true; + } + + // 检查 iView 组件的禁用状态 + if (el.classList.contains('ivu-btn-disabled')) return true; + if (el.classList.contains('ivu-input-disabled')) return true; + if (el.classList.contains('ivu-select-disabled')) return true; + + return false; +} + +/** + * 生成元素选择器 + * @param {Element} el + * @returns {string} + */ +function generateSelector(el) { + // 如果有 ID,直接使用 + if (el.id) { + return `#${el.id}`; + } + + // 尝试生成唯一选择器 + const parts = []; + let current = el; + let depth = 0; + + while (current && current !== document.body && depth < 5) { + let selector = current.tagName.toLowerCase(); + + // 添加重要的类名(排除动态类) + if (current.className && typeof current.className === 'string') { + const classes = current.className + .split(' ') + .filter(c => c && !c.startsWith('ivu-') && !c.includes('--') && !c.includes('active') && !c.includes('hover')) + .slice(0, 2); + if (classes.length) { + selector += '.' + classes.join('.'); + } + } + + // 添加有用的属性 + if (current.getAttribute('role')) { + selector += `[role="${current.getAttribute('role')}"]`; + } else if (current.getAttribute('data-id')) { + selector += `[data-id="${current.getAttribute('data-id')}"]`; + } else if (current.getAttribute('name')) { + selector += `[name="${current.getAttribute('name')}"]`; + } + + parts.unshift(selector); + current = current.parentElement; + depth++; + } + + return parts.join(' > '); +} + +/** + * 根据 ref 查找元素 + * @param {string} ref - 元素引用 (e1, e2, ...) + * @param {Object} refMap - 引用映射表 + * @returns {Element|null} + */ +export function findElementByRef(ref, refMap) { + const refData = refMap[ref]; + if (!refData) return null; + + // 首先尝试使用选择器 + if (refData.selector) { + const elements = document.querySelectorAll(refData.selector); + if (refData.nth !== undefined && elements.length > refData.nth) { + return elements[refData.nth]; + } + if (elements.length > 0) { + return elements[0]; + } + } + + // 回退到角色+名称匹配 + const roleSelector = `[role="${refData.role}"]`; + const candidates = document.querySelectorAll(roleSelector); + + for (const candidate of candidates) { + if (refData.name) { + const candidateName = getElementName(candidate); + if (candidateName === refData.name) { + return candidate; + } + } + } + + return null; +} + +export default collectPageContext; + +// 暴露到 window 供调试使用 +if (typeof window !== 'undefined') { + window.__testPageContext = (options = {}) => { + // 简化版,不需要 store + const context = { + page_url: window.location.href, + page_title: document.title, + timestamp: Date.now(), + }; + + const result = collectElements({ + interactiveOnly: options.interactive_only || false, + maxElements: options.max_elements || 50, + offset: options.offset || 0, + container: options.container || null, + }); + + context.elements = result.elements; + context.element_count = result.elements.length; + context.total_count = result.totalCount; + context.offset = options.offset || 0; + context.has_more = result.hasMore; + context.ref_map = result.refMap; + + return context; + }; +}