From 21f9c7e45385ad5b76cb469a65360241ad0eb0bd Mon Sep 17 00:00:00 2001 From: ssurui Date: Thu, 3 Sep 2026 14:50:52 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=E6=8B=8D=E7=85=A7?= =?UTF-8?q?=E6=8F=90=E9=97=AE=E6=A8=A1=E5=BC=8F=20MVP=EF=BC=88=E9=98=B6?= =?UTF-8?q?=E6=AE=B51+2=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 docs/feature-pic-design.md 的方案实现： - AppConfig 新增 vision 字段（enabled 总开关、cameraDeviceId 记住上次 选用的摄像头），主进程/渲染层类型两处同步 - generator.ts 抽出共享的 streamChatCompletion()，新增 streamGenerateVision() 支持 OpenAI 视觉 API 的 image_url content-part， 文字问答路径不受影响 - 新增 visionHandler.ts（vision-query IPC，跳过向量检索，直接转发 图片+问题给 er_backend 代理），注册进 ipc/index.ts - 新增 CameraCapture.vue：getUserMedia 取景/拍照/重拍/多摄像头切换， 纯交互组件不掺业务逻辑 - ChatPanel.vue 接入拍照入口（仅 vision.enabled 时显示）、待发送图片 缩略图、问题气泡里的图片展示；抽出 beginQuery() 复用文字/拍照两条 路径共同的状态重置逻辑 - SettingsModal.vue 加拍照提问开关 - index.ts 权限处理器注释说明 getUserMedia 摄像头请求也走 'media' 不做检索、不做照片持久化、不做多摄像头预配置界面（MVP 范围见设计文档 §3）。 本地未跑 vue-tsc，接下来在真机上验证类型检查 + 构建 + 端到端联调。 Co-Authored-By: Claude Sonnet 5 --- src/main/config.ts | 12 ++ src/main/index.ts | 3 +- src/main/ipc/index.ts | 2 + src/main/ipc/visionHandler.ts | 31 ++++ src/main/rag/generator.ts | 54 +++++- src/preload/index.ts | 4 + src/renderer/src/components/CameraCapture.vue | 170 +++++++++++++++++ src/renderer/src/components/ChatPanel.vue | 173 ++++++++++++++++-- src/renderer/src/components/SettingsModal.vue | 20 +- src/renderer/src/types/index.ts | 5 + 10 files changed, 445 insertions(+), 29 deletions(-) create mode 100644 src/main/ipc/visionHandler.ts create mode 100644 src/renderer/src/components/CameraCapture.vue diff --git a/src/main/config.ts b/src/main/config.ts index a6b3d76..933aafa 100644 --- a/src/main/config.ts +++ b/src/main/config.ts @@ -58,6 +58,10 @@ export interface AppConfig { volume: number } } + vision: { + enabled: boolean // 拍照提问入口总开关，默认关闭（依赖 er_backend 是否加载了支持视觉的模型） + cameraDeviceId: string // 上次选用的摄像头 deviceId，空字符串 = 使用第一个可用设备 + } } export const DEFAULT_CONFIG: AppConfig = { @@ -106,6 +110,10 @@ export const DEFAULT_CONFIG: AppConfig = { volume: 1 } }, + vision: { + enabled: false, + cameraDeviceId: '' + }, rag: { chunkSize: 480, chunkOverlap: 0, @@ -198,6 +206,10 @@ export function getConfig(): AppConfig { rate: typeof raw.voice?.tts?.rate === 'number' ? raw.voice.tts.rate : DEFAULT_CONFIG.voice.tts.rate, volume: typeof raw.voice?.tts?.volume === 'number' ? raw.voice.tts.volume : DEFAULT_CONFIG.voice.tts.volume } + }, + vision: { + enabled: raw.vision?.enabled === true, + cameraDeviceId: typeof raw.vision?.cameraDeviceId === 'string' ? raw.vision.cameraDeviceId : '' } } applyPlatformOverrides(config) diff --git a/src/main/index.ts b/src/main/index.ts index 193c586..248e9a0 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -80,7 +80,8 @@ function getMimeType(filePath: string): string { } app.whenReady().then(() => { - // 允许渲染进程请求麦克风权限（语音输入所需） + // 允许渲染进程请求麦克风权限（语音输入所需）；Electron 的 getUserMedia 摄像头请求 + // （拍照提问用）在权限系统里也归为 'media'，跟麦克风共用同一个判断，不需要单独加 'camera' session.defaultSession.setPermissionRequestHandler((webContents, permission, callback) => { const allowed = permission === 'media' || permission === 'microphone' console.log(`[permission] ${permission} -> ${allowed ? 'granted' : 'denied'}`) diff --git a/src/main/ipc/index.ts b/src/main/ipc/index.ts index 24d2288..55863dc 100644 --- a/src/main/ipc/index.ts +++ b/src/main/ipc/index.ts @@ -1,5 +1,6 @@ import { registerIndexHandlers } from './indexHandler' import { registerQueryHandlers } from './queryHandler' +import { registerVisionHandlers } from './visionHandler' import { registerStatusHandlers } from './statusHandler' import { registerAsrHandlers } from './asrHandler' import { registerTtsHandlers } from './ttsHandler' @@ -11,6 +12,7 @@ import type { BrowserWindow } from 'electron' export function registerIpcHandlers(getWin: () => BrowserWindow | null): void { registerIndexHandlers(getWin) registerQueryHandlers(getWin) + registerVisionHandlers(getWin) registerStatusHandlers(getWin) registerAsrHandlers(getWin) registerTtsHandlers(getWin) diff --git a/src/main/ipc/visionHandler.ts b/src/main/ipc/visionHandler.ts new file mode 100644 index 0000000..9ed0f31 --- /dev/null +++ b/src/main/ipc/visionHandler.ts @@ -0,0 +1,31 @@ +import { ipcMain, BrowserWindow } from 'electron' +import { streamGenerateVision } from '../rag/generator' + +// 拍照提问：跳过向量检索，图片 + 问题直接发给 er_backend 代理的多模态生成服务 +export function registerVisionHandlers(getWin: () => BrowserWindow | null): void { + ipcMain.handle('vision-query', async (_event, payload: { + question: string + imageDataUrl: string + }) => { + console.log(`\n${'='.repeat(60)}`) + console.log('[vision-query] 问题：', payload.question) + + const send = (channel: string, data: unknown): void => { + getWin()?.webContents.send(channel, data) + } + + try { + await streamGenerateVision( + payload.question, + payload.imageDataUrl, + { + onDelta: (delta) => send('vision-query:delta', { delta }), + onDone: (stats) => send('vision-query:done', { stats: stats ?? null }), + onError: (message) => send('vision-query:error', { message }) + } + ) + } catch (e) { + send('vision-query:error', { message: String(e) }) + } + }) +} diff --git a/src/main/rag/generator.ts b/src/main/rag/generator.ts index c8e4a6a..fac2fdf 100644 --- a/src/main/rag/generator.ts +++ b/src/main/rag/generator.ts @@ -14,6 +14,17 @@ export interface GeneratorCallbacks { onError: (message: string) => void } +// OpenAI 视觉 API 的 content-part 格式，multimodal 模型（如 Qwen-VL + mmproj）用这种数组形式 +// 代替纯字符串 content 来同时携带文字与图片 +type ContentPart = + | { type: 'text'; text: string } + | { type: 'image_url'; image_url: { url: string } } + +interface ChatMessage { + role: 'system' | 'user' + content: string | ContentPart[] +} + // 生成服务经 er_backend 代理访问其管理的 llama-server（OpenAI 兼容接口，SSE 流式） export async function streamGenerate( question: string, @@ -37,13 +48,43 @@ ${context} console.log('[generator] user message：\n', userMessage) console.log('[generator] ---- 消息结束 ----\n') - await streamGenerateLlamaCpp(userMessage, systemPrompt, callbacks) + await streamChatCompletion( + [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userMessage } + ], + callbacks + ) +} + +// 拍照提问：不检索、不拼接知识库上下文，图片 + 问题直接发给多模态生成服务 +export async function streamGenerateVision( + question: string, + imageDataUrl: string, + callbacks: GeneratorCallbacks +): Promise { + console.log('\n[generator] ---- 拍照提问 ----') + console.log(`[generator] 服务地址：${ER_BACKEND_URL}（er_backend 代理）`) + console.log('[generator] 问题：', question) + console.log('[generator] ---- 消息结束 ----\n') + + await streamChatCompletion( + [ + { + role: 'user', + content: [ + { type: 'text', text: question }, + { type: 'image_url', image_url: { url: imageDataUrl } } + ] + } + ], + callbacks + ) } // llama.cpp server OpenAI 兼容接口（/v1/chat/completions），SSE 格式 -async function streamGenerateLlamaCpp( - userMessage: string, - systemPrompt: string, +async function streamChatCompletion( + messages: ChatMessage[], callbacks: GeneratorCallbacks ): Promise { const baseUrl = ER_BACKEND_URL @@ -60,10 +101,7 @@ async function streamGenerateLlamaCpp( headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: 'default', - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: userMessage } - ], + messages, stream: true, stream_options: { include_usage: true } }) diff --git a/src/preload/index.ts b/src/preload/index.ts index 72c4d99..d43ab59 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -17,6 +17,10 @@ contextBridge.exposeInMainWorld('electronAPI', { query: (payload: { question: string; systemPrompt: string; topK: number }): Promise => ipcRenderer.invoke('query', payload), + // 拍照提问（不走检索，直接把图片+问题发给多模态生成服务） + visionQuery: (payload: { question: string; imageDataUrl: string }): Promise => + ipcRenderer.invoke('vision-query', payload), + // 状态 & 配置 getDbStatus: (): Promise<{ count: number; ready: boolean }> => ipcRenderer.invoke('get-db-status'), diff --git a/src/renderer/src/components/CameraCapture.vue b/src/renderer/src/components/CameraCapture.vue new file mode 100644 index 0000000..051960d --- /dev/null +++ b/src/renderer/src/components/CameraCapture.vue @@ -0,0 +1,170 @@ + + + + + diff --git a/src/renderer/src/components/ChatPanel.vue b/src/renderer/src/components/ChatPanel.vue index 7ddebc7..6f9f915 100644 --- a/src/renderer/src/components/ChatPanel.vue +++ b/src/renderer/src/components/ChatPanel.vue @@ -6,6 +6,7 @@ import { useRagStore } from '../stores/rag' import { resolveImagePaths } from '../utils/markdown' import { AudioRecorder } from '../utils/audio' import { ttsPlayer } from '../utils/tts' +import CameraCapture from './CameraCapture.vue' import type { Chunk } from '../types' interface InferStats { @@ -153,19 +154,13 @@ function onAnswerClick(e: MouseEvent): void { popupVisible.value = true } -async function sendQuery(): Promise { - const q = question.value.trim() - if (!q) return - if (!store.dbStatus.ready) { message.warning('向量库为空，请先入库文档'); return } - if (store.querying) return - +// 两种提问方式（文字检索 / 拍照）共用的准备工作：重置状态、打断正在进行的语音输入、停止旧的 TTS 播报 +function beginQuery(): void { store.querying = true store.resetAnswer() inferStats.value = null readyCitations.value = new Set() if (hoverTimer) { clearTimeout(hoverTimer); hoverTimer = null } - sentQuestion.value = q - question.value = '' // 立即解除 ASR 监听并清空状态，防止延迟到达的 asr:stopped 通过 syncAsrText() 恢复输入框 cleanupAsrListeners() asrBaseText = '' @@ -185,6 +180,18 @@ async function sendQuery(): Promise { startWaitTimer() ttsPlayer.stop() syncTtsConfig() +} + +async function sendQuery(): Promise { + const q = question.value.trim() + if (!q) return + if (!store.dbStatus.ready) { message.warning('向量库为空，请先入库文档'); return } + if (store.querying) return + + beginQuery() + sentQuestion.value = q + sentImage.value = '' + question.value = '' let firstDelta = true @@ -228,8 +235,84 @@ async function sendQuery(): Promise { await window.electronAPI.query({ question: q, systemPrompt: store.config?.rag.systemPrompt ?? '', topK: topK.value }) } +const VISION_DEFAULT_QUESTION = '请描述这张图片的内容' + +// 拍照提问：不查向量库，图片 + 问题直接发给多模态生成服务 +async function sendVisionQuery(): Promise { + if (!pendingImage.value) return + if (store.querying) return + + const q = question.value.trim() || VISION_DEFAULT_QUESTION + const img = pendingImage.value + + beginQuery() + sentQuestion.value = q + sentImage.value = img + question.value = '' + pendingImage.value = '' + + let firstDelta = true + + const onDelta = (data: unknown) => { + if (firstDelta) { firstDelta = false; stopWaitTimer() } + store.rawAnswer += (data as { delta: string }).delta + scrollToBottom() + } + const onDone = (data: unknown) => { + stopWaitTimer() + store.querying = false + queryStage.value = '' + const stats = (data as { stats?: InferStats | null }).stats + if (stats) inferStats.value = stats + cleanup() + } + const onError = (data: unknown) => { + stopWaitTimer() + store.querying = false + queryStage.value = '' + message.error((data as { message: string }).message) + cleanup() + } + + function cleanup(): void { + window.electronAPI.off('vision-query:delta') + window.electronAPI.off('vision-query:done') + window.electronAPI.off('vision-query:error') + } + + window.electronAPI.on('vision-query:delta', onDelta) + window.electronAPI.on('vision-query:done', onDone) + window.electronAPI.on('vision-query:error', onError) + + await window.electronAPI.visionQuery({ question: q, imageDataUrl: img }) +} + +function handleSend(): void { + if (pendingImage.value) { sendVisionQuery(); return } + sendQuery() +} + function handleEnter(e: KeyboardEvent): void { - if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); sendQuery() } + if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); handleSend() } +} + +// ── 拍照提问 ── +const cameraOpen = ref(false) +const pendingImage = ref('') +const sentImage = ref('') + +function onCaptured(dataUrl: string): void { + pendingImage.value = dataUrl +} + +function removePendingImage(): void { + pendingImage.value = '' +} + +async function onCameraDeviceChange(deviceId: string): Promise { + if (!store.config) return + if (store.config.vision.cameraDeviceId === deviceId) return + await store.saveConfig({ ...store.config, vision: { ...store.config.vision, cameraDeviceId: deviceId } }) } // ── 语音输入 ── @@ -498,6 +581,7 @@ onUnmounted(() => {
问
提问
+ 提问附带的照片
{{ sentQuestion }}
@@ -571,22 +655,38 @@ onUnmounted(() => { 语音输出 + +
+ +
-