diff --git a/src/main/ipc/visionHandler.ts b/src/main/ipc/visionHandler.ts index 9ed0f31..7aa31e3 100644 --- a/src/main/ipc/visionHandler.ts +++ b/src/main/ipc/visionHandler.ts @@ -1,23 +1,57 @@ import { ipcMain, BrowserWindow } from 'electron' -import { streamGenerateVision } from '../rag/generator' +import { getConfig, getDbPath } from '../config' +import { embedOne } from '../rag/embedder' +import { search } from '../db/lancedb' +import { describeImage, streamGenerate } from '../rag/generator' -// 拍照提问:跳过向量检索,图片 + 问题直接发给 er_backend 代理的多模态生成服务 +// 拍照提问:先用视觉模型快速生成一句图像描述(不查库),描述+用户问题拼接去检索本地 +// 知识库,再用用户原始问题(没打字就用描述本身)+ 检索到的上下文做纯文字生成—— +// 跟 queryHandler.ts 的文字问答共用 embedOne/search/streamGenerate,只是多了第①步。 export function registerVisionHandlers(getWin: () => BrowserWindow | null): void { ipcMain.handle('vision-query', async (_event, payload: { question: string imageDataUrl: string }) => { + const config = getConfig() + const dbPath = getDbPath() + console.log(`\n${'='.repeat(60)}`) - console.log('[vision-query] 问题:', payload.question) + console.log('[vision-query] 用户问题:', payload.question || '(未输入)') const send = (channel: string, data: unknown): void => { getWin()?.webContents.send(channel, data) } try { - await streamGenerateVision( - payload.question, - payload.imageDataUrl, + // ① 图像理解:固定提示词,快速拿一句话描述 + send('vision-query:stage', { stage: '正在识别图片...' }) + const caption = await describeImage(payload.imageDataUrl) + send('vision-query:caption', { caption }) + + // ② 检索:描述 + 用户问题拼接,扩大召回面 + const retrievalQuery = payload.question ? `${caption}\n${payload.question}` : caption + send('vision-query:stage', { stage: '正在检索...' }) + const queryVector = await embedOne(retrievalQuery, config.embed) + const chunks = await search(dbPath, queryVector, config.rag.topK) + console.log(`[vision-query] 检索到 ${chunks.length} 个片段:`) + chunks.forEach((c, i) => { + console.log(` 片段${i + 1} 相关度=${c.score} 来源=${c.source}`) + }) + send('vision-query:context', { chunks }) + + if (chunks.length === 0) { + send('vision-query:delta', { delta: '我不知道。' }) + send('vision-query:done', {}) + return + } + + // ③ 生成:用用户原始问题保持聚焦,没打字就用描述本身 + const genQuestion = payload.question || caption + send('vision-query:stage', { stage: '正在生成...' }) + await streamGenerate( + genQuestion, + chunks, + config.rag.systemPrompt, { onDelta: (delta) => send('vision-query:delta', { delta }), onDone: (stats) => send('vision-query:done', { stats: stats ?? null }), diff --git a/src/main/rag/generator.ts b/src/main/rag/generator.ts index fac2fdf..9aec557 100644 --- a/src/main/rag/generator.ts +++ b/src/main/rag/generator.ts @@ -57,29 +57,47 @@ ${context} ) } -// 拍照提问:不检索、不拼接知识库上下文,图片 + 问题直接发给多模态生成服务 -export async function streamGenerateVision( - question: string, - imageDataUrl: string, - callbacks: GeneratorCallbacks -): Promise { - console.log('\n[generator] ---- 拍照提问 ----') - console.log(`[generator] 服务地址:${ER_BACKEND_URL}(er_backend 代理)`) - console.log('[generator] 问题:', question) - console.log('[generator] ---- 消息结束 ----\n') +// 拍照提问第一步:用固定的默认提示词(不是用户问题)让视觉模型快速给一句话描述, +// 用于后续拼检索查询。单次非流式请求——只要一句简短结果,不需要逐字显示, +// 也不需要每次都重新走一遍长回答的 SSE 解析。 +const IMAGE_DESCRIBE_PROMPT = + '请用一句话简要描述这张图片中的部件、故障现象或异常情况,不要展开解释、不要客套。' - await streamChatCompletion( - [ - { - role: 'user', - content: [ - { type: 'text', text: question }, - { type: 'image_url', image_url: { url: imageDataUrl } } - ] - } - ], - callbacks - ) +export async function describeImage(imageDataUrl: string): Promise { + const baseUrl = ER_BACKEND_URL + const messages: ChatMessage[] = [ + { + role: 'user', + content: [ + { type: 'text', text: IMAGE_DESCRIBE_PROMPT }, + { type: 'image_url', image_url: { url: imageDataUrl } } + ] + } + ] + + console.log('\n[generator] ---- 图像理解(拍照提问第①步)----') + console.log(`[generator] 服务地址:${baseUrl}(er_backend 代理)`) + + const start = Date.now() + let response: Response + try { + response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: 'default', messages, stream: false }) + }) + } catch (e) { + throw new Error(`无法连接视觉理解服务(${baseUrl},er_backend 代理):${String(e)}`) + } + + if (!response.ok) { + throw new Error(`图像理解请求失败:${response.status}`) + } + + const data = await response.json() + const caption: string = data.choices?.[0]?.message?.content?.trim() ?? '' + console.log(`[generator] 图像描述(耗时 ${((Date.now() - start) / 1000).toFixed(1)}s):${caption}`) + return caption } // llama.cpp server OpenAI 兼容接口(/v1/chat/completions),SSE 格式 diff --git a/src/renderer/src/components/CameraCapture.vue b/src/renderer/src/components/CameraCapture.vue index 3165284..66f8291 100644 --- a/src/renderer/src/components/CameraCapture.vue +++ b/src/renderer/src/components/CameraCapture.vue @@ -29,18 +29,36 @@ function stopStream(): void { stream = null } +/** + * 优先按指定 deviceId 打开摄像头;该 id 打不开时退回默认摄像头。 + * deviceId 在 Web 标准里不保证跨会话稳定,配置里存的是上次运行时的 id, + * 换一次会话后经常已经失效——此时 exact 约束会抛 OverconstrainedError, + * 不兜底的话整个拍照功能都用不了。 + */ +async function openStream(deviceId: string): Promise { + if (!deviceId) return navigator.mediaDevices.getUserMedia({ video: true }) + try { + return await navigator.mediaDevices.getUserMedia({ video: { deviceId: { exact: deviceId } } }) + } catch (e) { + console.warn('[camera] 指定的摄像头不可用,回退到默认摄像头:', e) + return navigator.mediaDevices.getUserMedia({ video: true }) + } +} + async function startStream(deviceId: string): Promise { stopStream() errorMessage.value = '' starting.value = true try { - stream = await navigator.mediaDevices.getUserMedia({ - video: deviceId ? { deviceId: { exact: deviceId } } : true - }) + stream = await openStream(deviceId) if (videoRef.value) { videoRef.value.srcObject = stream await videoRef.value.play() } + // 记录实际拿到的设备(回退时跟请求的 deviceId 不是同一个), + // 后续切换/匹配后置摄像头都以这个真实值为准 + const actualId = stream.getVideoTracks()[0]?.getSettings().deviceId + if (actualId) selectedDeviceId.value = actualId // 拿到权限后 label 才会有值,借机刷新一次设备列表 await refreshDevices() } catch (e) { @@ -119,9 +137,13 @@ watch( selectedDeviceId.value = props.initialDeviceId // 摄像头 label 在拿到权限前是空的(浏览器隐私限制),没法在这一步就按标签 // 找后置摄像头,先随便开一个拿到权限;startStream 成功后 devices 里才有真实 label。 - await startStream(selectedDeviceId.value) - // 用户/配置没有指定具体摄像头时,权限到手后尝试切到确认过的后置摄像头 - if (!props.initialDeviceId) { + await startStream(props.initialDeviceId) + if (!stream) return // 打开失败,错误信息已经显示在弹层里 + // 配置里没指定过摄像头,或指定的那个已失效(startStream 回退后 selectedDeviceId + // 会被改写成实际设备),两种情况都尝试切到实测确认的后置摄像头 + const usingRequested = + !!props.initialDeviceId && selectedDeviceId.value === props.initialDeviceId + if (!usingRequested) { const rear = findRearCamera() if (rear && rear.deviceId !== selectedDeviceId.value) { await switchDevice(rear.deviceId) diff --git a/src/renderer/src/components/ChatPanel.vue b/src/renderer/src/components/ChatPanel.vue index 25ab552..aa22d5d 100644 --- a/src/renderer/src/components/ChatPanel.vue +++ b/src/renderer/src/components/ChatPanel.vue @@ -159,6 +159,7 @@ function beginQuery(): void { store.querying = true store.resetAnswer() inferStats.value = null + visionCaption.value = '' readyCitations.value = new Set() if (hoverTimer) { clearTimeout(hoverTimer); hoverTimer = null } // 立即解除 ASR 监听并清空状态,防止延迟到达的 asr:stopped 通过 syncAsrText() 恢复输入框 @@ -240,14 +241,13 @@ async function sendQuery(): Promise { await window.electronAPI.query({ question: q, systemPrompt: store.config?.rag.systemPrompt ?? '', topK: topK.value }) } -const VISION_DEFAULT_QUESTION = '请描述这张图片的内容' - -// 拍照提问:不查向量库,图片 + 问题直接发给多模态生成服务 +// 拍照提问:先让视觉模型快速生成一句图像描述(跟用户问题拼接后去检索知识库), +// 再走一遍跟文字问答一样的检索+生成,图片本身不会传给最终生成模型 async function sendVisionQuery(): Promise { if (!pendingImage.value) return if (store.querying) return - const q = question.value.trim() || VISION_DEFAULT_QUESTION + const q = question.value.trim() const img = pendingImage.value beginQuery() @@ -258,6 +258,9 @@ async function sendVisionQuery(): Promise { let firstDelta = true + const onStage = (data: unknown) => { queryStage.value = (data as { stage: string }).stage } + const onCaption = (data: unknown) => { visionCaption.value = (data as { caption: string }).caption } + const onContext = (data: unknown) => { store.chunks = (data as { chunks: Chunk[] }).chunks } const onDelta = (data: unknown) => { if (firstDelta) { firstDelta = false; stopWaitTimer() } const delta = (data as { delta: string }).delta @@ -284,11 +287,17 @@ async function sendVisionQuery(): Promise { } function cleanup(): void { + window.electronAPI.off('vision-query:stage') + window.electronAPI.off('vision-query:caption') + window.electronAPI.off('vision-query:context') window.electronAPI.off('vision-query:delta') window.electronAPI.off('vision-query:done') window.electronAPI.off('vision-query:error') } + window.electronAPI.on('vision-query:stage', onStage) + window.electronAPI.on('vision-query:caption', onCaption) + window.electronAPI.on('vision-query:context', onContext) window.electronAPI.on('vision-query:delta', onDelta) window.electronAPI.on('vision-query:done', onDone) window.electronAPI.on('vision-query:error', onError) @@ -309,6 +318,7 @@ function handleEnter(e: KeyboardEvent): void { const cameraOpen = ref(false) const pendingImage = ref('') const sentImage = ref('') +const visionCaption = ref('') function onCaptured(dataUrl: string): void { pendingImage.value = dataUrl @@ -584,10 +594,13 @@ onUnmounted(() => {
提问
提问附带的照片 -
{{ sentQuestion }}
+
{{ sentQuestion || (sentImage ? '(未输入文字,根据图片自动识别提问)' : '') }}
+ +
识别到:{{ visionCaption }}
+
@@ -779,6 +792,13 @@ onUnmounted(() => { border: 2px solid var(--line); } +/* Vision caption */ +.vision-caption { + font-size: 15px; color: var(--ink-3); + background: var(--panel-2); border-radius: var(--radius-sm); + padding: 8px 14px; margin-bottom: 14px; +} + /* Stage indicator */ .stage-row { display: flex; align-items: center; gap: 10px;