feat: 拍照提问接入本地知识库检索；修复摄像头 deviceId 失效导致打不开
架构调整（按 docs/feature-pic-design.md §11）：拍照提问从"图+文直接 问视觉模型"改成三段式——① 用固定提示词让视觉模型快一句话描述图片 （describeImage，非流式，实测 6.7s / 45 tokens）② 描述+用户问题拼接 去检索本地知识库 ③ 用户原始问题+检索到的上下文做纯文字生成。②③ 完全 复用文字问答的 embedOne/search/streamGenerate，图片不再传给最终生成。 前端补 vision-query:stage / :caption / :context 三个事件，识别结果以 "识别到：xxx"常驻显示。 同时修一个真机暴露的 bug：CameraCapture 用配置里持久化的 deviceId 做 exact 约束打开摄像头，但 deviceId 在 Web 标准里不保证跨会话稳定，重启 后旧 id 失效直接抛 OverconstrainedError，整个拍照功能不可用（真机截图 确认弹层报错、预览全黑）。改为失效时自动回退默认摄像头，并把实际拿到的 设备 id 回写，让"切到后置摄像头"的逻辑在回退场景下同样生效。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QeoAy5KmkHuKWHAktpEsaN
This commit is contained in:
ssurui
parent
87b9ffbab5
commit
687e0ca9e2
@ -1,23 +1,57 @@
|
||||
import { ipcMain, BrowserWindow } from 'electron'
|
||||
import { streamGenerateVision } from '../rag/generator'
|
||||
import { getConfig, getDbPath } from '../config'
|
||||
import { embedOne } from '../rag/embedder'
|
||||
import { search } from '../db/lancedb'
|
||||
import { describeImage, streamGenerate } from '../rag/generator'
|
||||
|
||||
// 拍照提问：跳过向量检索，图片 + 问题直接发给 er_backend 代理的多模态生成服务
|
||||
// 拍照提问：先用视觉模型快速生成一句图像描述（不查库），描述+用户问题拼接去检索本地
|
||||
// 知识库，再用用户原始问题（没打字就用描述本身）+ 检索到的上下文做纯文字生成——
|
||||
// 跟 queryHandler.ts 的文字问答共用 embedOne/search/streamGenerate，只是多了第①步。
|
||||
export function registerVisionHandlers(getWin: () => BrowserWindow | null): void {
|
||||
ipcMain.handle('vision-query', async (_event, payload: {
|
||||
question: string
|
||||
imageDataUrl: string
|
||||
}) => {
|
||||
const config = getConfig()
|
||||
const dbPath = getDbPath()
|
||||
|
||||
console.log(`\n${'='.repeat(60)}`)
|
||||
console.log('[vision-query] 问题：', payload.question)
|
||||
console.log('[vision-query] 用户问题：', payload.question || '(未输入)')
|
||||
|
||||
const send = (channel: string, data: unknown): void => {
|
||||
getWin()?.webContents.send(channel, data)
|
||||
}
|
||||
|
||||
try {
|
||||
await streamGenerateVision(
|
||||
payload.question,
|
||||
payload.imageDataUrl,
|
||||
// ① 图像理解：固定提示词，快速拿一句话描述
|
||||
send('vision-query:stage', { stage: '正在识别图片...' })
|
||||
const caption = await describeImage(payload.imageDataUrl)
|
||||
send('vision-query:caption', { caption })
|
||||
|
||||
// ② 检索：描述 + 用户问题拼接，扩大召回面
|
||||
const retrievalQuery = payload.question ? `${caption}\n${payload.question}` : caption
|
||||
send('vision-query:stage', { stage: '正在检索...' })
|
||||
const queryVector = await embedOne(retrievalQuery, config.embed)
|
||||
const chunks = await search(dbPath, queryVector, config.rag.topK)
|
||||
console.log(`[vision-query] 检索到 ${chunks.length} 个片段：`)
|
||||
chunks.forEach((c, i) => {
|
||||
console.log(` 片段${i + 1} 相关度=${c.score} 来源=${c.source}`)
|
||||
})
|
||||
send('vision-query:context', { chunks })
|
||||
|
||||
if (chunks.length === 0) {
|
||||
send('vision-query:delta', { delta: '我不知道。' })
|
||||
send('vision-query:done', {})
|
||||
return
|
||||
}
|
||||
|
||||
// ③ 生成：用用户原始问题保持聚焦，没打字就用描述本身
|
||||
const genQuestion = payload.question || caption
|
||||
send('vision-query:stage', { stage: '正在生成...' })
|
||||
await streamGenerate(
|
||||
genQuestion,
|
||||
chunks,
|
||||
config.rag.systemPrompt,
|
||||
{
|
||||
onDelta: (delta) => send('vision-query:delta', { delta }),
|
||||
onDone: (stats) => send('vision-query:done', { stats: stats ?? null }),
|
||||
|
||||
@ -57,29 +57,47 @@ ${context}
|
||||
)
|
||||
}
|
||||
|
||||
// 拍照提问：不检索、不拼接知识库上下文，图片 + 问题直接发给多模态生成服务
|
||||
export async function streamGenerateVision(
|
||||
question: string,
|
||||
imageDataUrl: string,
|
||||
callbacks: GeneratorCallbacks
|
||||
): Promise<void> {
|
||||
console.log('\n[generator] ---- 拍照提问 ----')
|
||||
console.log(`[generator] 服务地址：${ER_BACKEND_URL}（er_backend 代理）`)
|
||||
console.log('[generator] 问题：', question)
|
||||
console.log('[generator] ---- 消息结束 ----\n')
|
||||
// 拍照提问第一步：用固定的默认提示词（不是用户问题）让视觉模型快速给一句话描述，
|
||||
// 用于后续拼检索查询。单次非流式请求——只要一句简短结果，不需要逐字显示，
|
||||
// 也不需要每次都重新走一遍长回答的 SSE 解析。
|
||||
const IMAGE_DESCRIBE_PROMPT =
|
||||
'请用一句话简要描述这张图片中的部件、故障现象或异常情况，不要展开解释、不要客套。'
|
||||
|
||||
await streamChatCompletion(
|
||||
[
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: question },
|
||||
{ type: 'image_url', image_url: { url: imageDataUrl } }
|
||||
]
|
||||
}
|
||||
],
|
||||
callbacks
|
||||
)
|
||||
export async function describeImage(imageDataUrl: string): Promise<string> {
|
||||
const baseUrl = ER_BACKEND_URL
|
||||
const messages: ChatMessage[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: IMAGE_DESCRIBE_PROMPT },
|
||||
{ type: 'image_url', image_url: { url: imageDataUrl } }
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
console.log('\n[generator] ---- 图像理解（拍照提问第①步）----')
|
||||
console.log(`[generator] 服务地址：${baseUrl}（er_backend 代理）`)
|
||||
|
||||
const start = Date.now()
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetch(`${baseUrl}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model: 'default', messages, stream: false })
|
||||
})
|
||||
} catch (e) {
|
||||
throw new Error(`无法连接视觉理解服务（${baseUrl}，er_backend 代理）：${String(e)}`)
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`图像理解请求失败：${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const caption: string = data.choices?.[0]?.message?.content?.trim() ?? ''
|
||||
console.log(`[generator] 图像描述（耗时 ${((Date.now() - start) / 1000).toFixed(1)}s）：${caption}`)
|
||||
return caption
|
||||
}
|
||||
|
||||
// llama.cpp server OpenAI 兼容接口（/v1/chat/completions），SSE 格式
|
||||
|
||||
@ -29,18 +29,36 @@ function stopStream(): void {
|
||||
stream = null
|
||||
}
|
||||
|
||||
/**
|
||||
* 优先按指定 deviceId 打开摄像头；该 id 打不开时退回默认摄像头。
|
||||
* deviceId 在 Web 标准里不保证跨会话稳定，配置里存的是上次运行时的 id，
|
||||
* 换一次会话后经常已经失效——此时 exact 约束会抛 OverconstrainedError，
|
||||
* 不兜底的话整个拍照功能都用不了。
|
||||
*/
|
||||
async function openStream(deviceId: string): Promise<MediaStream> {
|
||||
if (!deviceId) return navigator.mediaDevices.getUserMedia({ video: true })
|
||||
try {
|
||||
return await navigator.mediaDevices.getUserMedia({ video: { deviceId: { exact: deviceId } } })
|
||||
} catch (e) {
|
||||
console.warn('[camera] 指定的摄像头不可用，回退到默认摄像头：', e)
|
||||
return navigator.mediaDevices.getUserMedia({ video: true })
|
||||
}
|
||||
}
|
||||
|
||||
async function startStream(deviceId: string): Promise<void> {
|
||||
stopStream()
|
||||
errorMessage.value = ''
|
||||
starting.value = true
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: deviceId ? { deviceId: { exact: deviceId } } : true
|
||||
})
|
||||
stream = await openStream(deviceId)
|
||||
if (videoRef.value) {
|
||||
videoRef.value.srcObject = stream
|
||||
await videoRef.value.play()
|
||||
}
|
||||
// 记录实际拿到的设备（回退时跟请求的 deviceId 不是同一个），
|
||||
// 后续切换/匹配后置摄像头都以这个真实值为准
|
||||
const actualId = stream.getVideoTracks()[0]?.getSettings().deviceId
|
||||
if (actualId) selectedDeviceId.value = actualId
|
||||
// 拿到权限后 label 才会有值，借机刷新一次设备列表
|
||||
await refreshDevices()
|
||||
} catch (e) {
|
||||
@ -119,9 +137,13 @@ watch(
|
||||
selectedDeviceId.value = props.initialDeviceId
|
||||
// 摄像头 label 在拿到权限前是空的（浏览器隐私限制），没法在这一步就按标签
|
||||
// 找后置摄像头，先随便开一个拿到权限；startStream 成功后 devices 里才有真实 label。
|
||||
await startStream(selectedDeviceId.value)
|
||||
// 用户/配置没有指定具体摄像头时，权限到手后尝试切到确认过的后置摄像头
|
||||
if (!props.initialDeviceId) {
|
||||
await startStream(props.initialDeviceId)
|
||||
if (!stream) return // 打开失败，错误信息已经显示在弹层里
|
||||
// 配置里没指定过摄像头，或指定的那个已失效（startStream 回退后 selectedDeviceId
|
||||
// 会被改写成实际设备），两种情况都尝试切到实测确认的后置摄像头
|
||||
const usingRequested =
|
||||
!!props.initialDeviceId && selectedDeviceId.value === props.initialDeviceId
|
||||
if (!usingRequested) {
|
||||
const rear = findRearCamera()
|
||||
if (rear && rear.deviceId !== selectedDeviceId.value) {
|
||||
await switchDevice(rear.deviceId)
|
||||
|
||||
@ -159,6 +159,7 @@ function beginQuery(): void {
|
||||
store.querying = true
|
||||
store.resetAnswer()
|
||||
inferStats.value = null
|
||||
visionCaption.value = ''
|
||||
readyCitations.value = new Set()
|
||||
if (hoverTimer) { clearTimeout(hoverTimer); hoverTimer = null }
|
||||
// 立即解除 ASR 监听并清空状态，防止延迟到达的 asr:stopped 通过 syncAsrText() 恢复输入框
|
||||
@ -240,14 +241,13 @@ async function sendQuery(): Promise<void> {
|
||||
await window.electronAPI.query({ question: q, systemPrompt: store.config?.rag.systemPrompt ?? '', topK: topK.value })
|
||||
}
|
||||
|
||||
const VISION_DEFAULT_QUESTION = '请描述这张图片的内容'
|
||||
|
||||
// 拍照提问：不查向量库，图片 + 问题直接发给多模态生成服务
|
||||
// 拍照提问：先让视觉模型快速生成一句图像描述（跟用户问题拼接后去检索知识库），
|
||||
// 再走一遍跟文字问答一样的检索+生成，图片本身不会传给最终生成模型
|
||||
async function sendVisionQuery(): Promise<void> {
|
||||
if (!pendingImage.value) return
|
||||
if (store.querying) return
|
||||
|
||||
const q = question.value.trim() || VISION_DEFAULT_QUESTION
|
||||
const q = question.value.trim()
|
||||
const img = pendingImage.value
|
||||
|
||||
beginQuery()
|
||||
@ -258,6 +258,9 @@ async function sendVisionQuery(): Promise<void> {
|
||||
|
||||
let firstDelta = true
|
||||
|
||||
const onStage = (data: unknown) => { queryStage.value = (data as { stage: string }).stage }
|
||||
const onCaption = (data: unknown) => { visionCaption.value = (data as { caption: string }).caption }
|
||||
const onContext = (data: unknown) => { store.chunks = (data as { chunks: Chunk[] }).chunks }
|
||||
const onDelta = (data: unknown) => {
|
||||
if (firstDelta) { firstDelta = false; stopWaitTimer() }
|
||||
const delta = (data as { delta: string }).delta
|
||||
@ -284,11 +287,17 @@ async function sendVisionQuery(): Promise<void> {
|
||||
}
|
||||
|
||||
function cleanup(): void {
|
||||
window.electronAPI.off('vision-query:stage')
|
||||
window.electronAPI.off('vision-query:caption')
|
||||
window.electronAPI.off('vision-query:context')
|
||||
window.electronAPI.off('vision-query:delta')
|
||||
window.electronAPI.off('vision-query:done')
|
||||
window.electronAPI.off('vision-query:error')
|
||||
}
|
||||
|
||||
window.electronAPI.on('vision-query:stage', onStage)
|
||||
window.electronAPI.on('vision-query:caption', onCaption)
|
||||
window.electronAPI.on('vision-query:context', onContext)
|
||||
window.electronAPI.on('vision-query:delta', onDelta)
|
||||
window.electronAPI.on('vision-query:done', onDone)
|
||||
window.electronAPI.on('vision-query:error', onError)
|
||||
@ -309,6 +318,7 @@ function handleEnter(e: KeyboardEvent): void {
|
||||
const cameraOpen = ref(false)
|
||||
const pendingImage = ref('')
|
||||
const sentImage = ref('')
|
||||
const visionCaption = ref('')
|
||||
|
||||
function onCaptured(dataUrl: string): void {
|
||||
pendingImage.value = dataUrl
|
||||
@ -584,10 +594,13 @@ onUnmounted(() => {
|
||||
<div>
|
||||
<div class="q-label">提问</div>
|
||||
<img v-if="sentImage" :src="sentImage" class="q-image" alt="提问附带的照片" />
|
||||
<div class="q-text">{{ sentQuestion }}</div>
|
||||
<div class="q-text">{{ sentQuestion || (sentImage ? '（未输入文字，根据图片自动识别提问）' : '') }}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Vision caption：拍照提问识别到的图像描述，识别完就常驻显示 -->
|
||||
<div v-if="visionCaption" class="vision-caption">识别到：{{ visionCaption }}</div>
|
||||
|
||||
<!-- Stage indicator -->
|
||||
<div v-if="store.querying && !store.rawAnswer" class="stage-row">
|
||||
<span class="stage-spinner" />
|
||||
@ -779,6 +792,13 @@ onUnmounted(() => {
|
||||
border: 2px solid var(--line);
|
||||
}
|
||||
|
||||
/* Vision caption */
|
||||
.vision-caption {
|
||||
font-size: 15px; color: var(--ink-3);
|
||||
background: var(--panel-2); border-radius: var(--radius-sm);
|
||||
padding: 8px 14px; margin-bottom: 14px;
|
||||
}
|
||||
|
||||
/* Stage indicator */
|
||||
.stage-row {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user