feat: 实现拍照提问模式 MVP(阶段1+2)
按 docs/feature-pic-design.md 的方案实现: - AppConfig 新增 vision 字段(enabled 总开关、cameraDeviceId 记住上次 选用的摄像头),主进程/渲染层类型两处同步 - generator.ts 抽出共享的 streamChatCompletion(),新增 streamGenerateVision() 支持 OpenAI 视觉 API 的 image_url content-part, 文字问答路径不受影响 - 新增 visionHandler.ts(vision-query IPC,跳过向量检索,直接转发 图片+问题给 er_backend 代理),注册进 ipc/index.ts - 新增 CameraCapture.vue:getUserMedia 取景/拍照/重拍/多摄像头切换, 纯交互组件不掺业务逻辑 - ChatPanel.vue 接入拍照入口(仅 vision.enabled 时显示)、待发送图片 缩略图、问题气泡里的图片展示;抽出 beginQuery() 复用文字/拍照两条 路径共同的状态重置逻辑 - SettingsModal.vue 加拍照提问开关 - index.ts 权限处理器注释说明 getUserMedia 摄像头请求也走 'media' 不做检索、不做照片持久化、不做多摄像头预配置界面(MVP 范围见设计文档 §3)。 本地未跑 vue-tsc,接下来在真机上验证类型检查 + 构建 + 端到端联调。 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
2ae01c2050
commit
21f9c7e453
@ -58,6 +58,10 @@ export interface AppConfig {
|
|||||||
volume: number
|
volume: number
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
vision: {
|
||||||
|
enabled: boolean // 拍照提问入口总开关,默认关闭(依赖 er_backend 是否加载了支持视觉的模型)
|
||||||
|
cameraDeviceId: string // 上次选用的摄像头 deviceId,空字符串 = 使用第一个可用设备
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const DEFAULT_CONFIG: AppConfig = {
|
export const DEFAULT_CONFIG: AppConfig = {
|
||||||
@ -106,6 +110,10 @@ export const DEFAULT_CONFIG: AppConfig = {
|
|||||||
volume: 1
|
volume: 1
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
vision: {
|
||||||
|
enabled: false,
|
||||||
|
cameraDeviceId: ''
|
||||||
|
},
|
||||||
rag: {
|
rag: {
|
||||||
chunkSize: 480,
|
chunkSize: 480,
|
||||||
chunkOverlap: 0,
|
chunkOverlap: 0,
|
||||||
@ -198,6 +206,10 @@ export function getConfig(): AppConfig {
|
|||||||
rate: typeof raw.voice?.tts?.rate === 'number' ? raw.voice.tts.rate : DEFAULT_CONFIG.voice.tts.rate,
|
rate: typeof raw.voice?.tts?.rate === 'number' ? raw.voice.tts.rate : DEFAULT_CONFIG.voice.tts.rate,
|
||||||
volume: typeof raw.voice?.tts?.volume === 'number' ? raw.voice.tts.volume : DEFAULT_CONFIG.voice.tts.volume
|
volume: typeof raw.voice?.tts?.volume === 'number' ? raw.voice.tts.volume : DEFAULT_CONFIG.voice.tts.volume
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
vision: {
|
||||||
|
enabled: raw.vision?.enabled === true,
|
||||||
|
cameraDeviceId: typeof raw.vision?.cameraDeviceId === 'string' ? raw.vision.cameraDeviceId : ''
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
applyPlatformOverrides(config)
|
applyPlatformOverrides(config)
|
||||||
|
|||||||
@ -80,7 +80,8 @@ function getMimeType(filePath: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
app.whenReady().then(() => {
|
app.whenReady().then(() => {
|
||||||
// 允许渲染进程请求麦克风权限(语音输入所需)
|
// 允许渲染进程请求麦克风权限(语音输入所需);Electron 的 getUserMedia 摄像头请求
|
||||||
|
// (拍照提问用)在权限系统里也归为 'media',跟麦克风共用同一个判断,不需要单独加 'camera'
|
||||||
session.defaultSession.setPermissionRequestHandler((webContents, permission, callback) => {
|
session.defaultSession.setPermissionRequestHandler((webContents, permission, callback) => {
|
||||||
const allowed = permission === 'media' || permission === 'microphone'
|
const allowed = permission === 'media' || permission === 'microphone'
|
||||||
console.log(`[permission] ${permission} -> ${allowed ? 'granted' : 'denied'}`)
|
console.log(`[permission] ${permission} -> ${allowed ? 'granted' : 'denied'}`)
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
import { registerIndexHandlers } from './indexHandler'
|
import { registerIndexHandlers } from './indexHandler'
|
||||||
import { registerQueryHandlers } from './queryHandler'
|
import { registerQueryHandlers } from './queryHandler'
|
||||||
|
import { registerVisionHandlers } from './visionHandler'
|
||||||
import { registerStatusHandlers } from './statusHandler'
|
import { registerStatusHandlers } from './statusHandler'
|
||||||
import { registerAsrHandlers } from './asrHandler'
|
import { registerAsrHandlers } from './asrHandler'
|
||||||
import { registerTtsHandlers } from './ttsHandler'
|
import { registerTtsHandlers } from './ttsHandler'
|
||||||
@ -11,6 +12,7 @@ import type { BrowserWindow } from 'electron'
|
|||||||
export function registerIpcHandlers(getWin: () => BrowserWindow | null): void {
|
export function registerIpcHandlers(getWin: () => BrowserWindow | null): void {
|
||||||
registerIndexHandlers(getWin)
|
registerIndexHandlers(getWin)
|
||||||
registerQueryHandlers(getWin)
|
registerQueryHandlers(getWin)
|
||||||
|
registerVisionHandlers(getWin)
|
||||||
registerStatusHandlers(getWin)
|
registerStatusHandlers(getWin)
|
||||||
registerAsrHandlers(getWin)
|
registerAsrHandlers(getWin)
|
||||||
registerTtsHandlers(getWin)
|
registerTtsHandlers(getWin)
|
||||||
|
|||||||
31
src/main/ipc/visionHandler.ts
Normal file
31
src/main/ipc/visionHandler.ts
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
import { ipcMain, BrowserWindow } from 'electron'
|
||||||
|
import { streamGenerateVision } from '../rag/generator'
|
||||||
|
|
||||||
|
// 拍照提问:跳过向量检索,图片 + 问题直接发给 er_backend 代理的多模态生成服务
|
||||||
|
export function registerVisionHandlers(getWin: () => BrowserWindow | null): void {
|
||||||
|
ipcMain.handle('vision-query', async (_event, payload: {
|
||||||
|
question: string
|
||||||
|
imageDataUrl: string
|
||||||
|
}) => {
|
||||||
|
console.log(`\n${'='.repeat(60)}`)
|
||||||
|
console.log('[vision-query] 问题:', payload.question)
|
||||||
|
|
||||||
|
const send = (channel: string, data: unknown): void => {
|
||||||
|
getWin()?.webContents.send(channel, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await streamGenerateVision(
|
||||||
|
payload.question,
|
||||||
|
payload.imageDataUrl,
|
||||||
|
{
|
||||||
|
onDelta: (delta) => send('vision-query:delta', { delta }),
|
||||||
|
onDone: (stats) => send('vision-query:done', { stats: stats ?? null }),
|
||||||
|
onError: (message) => send('vision-query:error', { message })
|
||||||
|
}
|
||||||
|
)
|
||||||
|
} catch (e) {
|
||||||
|
send('vision-query:error', { message: String(e) })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@ -14,6 +14,17 @@ export interface GeneratorCallbacks {
|
|||||||
onError: (message: string) => void
|
onError: (message: string) => void
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OpenAI 视觉 API 的 content-part 格式,multimodal 模型(如 Qwen-VL + mmproj)用这种数组形式
|
||||||
|
// 代替纯字符串 content 来同时携带文字与图片
|
||||||
|
type ContentPart =
|
||||||
|
| { type: 'text'; text: string }
|
||||||
|
| { type: 'image_url'; image_url: { url: string } }
|
||||||
|
|
||||||
|
interface ChatMessage {
|
||||||
|
role: 'system' | 'user'
|
||||||
|
content: string | ContentPart[]
|
||||||
|
}
|
||||||
|
|
||||||
// 生成服务经 er_backend 代理访问其管理的 llama-server(OpenAI 兼容接口,SSE 流式)
|
// 生成服务经 er_backend 代理访问其管理的 llama-server(OpenAI 兼容接口,SSE 流式)
|
||||||
export async function streamGenerate(
|
export async function streamGenerate(
|
||||||
question: string,
|
question: string,
|
||||||
@ -37,13 +48,43 @@ ${context}
|
|||||||
console.log('[generator] user message:\n', userMessage)
|
console.log('[generator] user message:\n', userMessage)
|
||||||
console.log('[generator] ---- 消息结束 ----\n')
|
console.log('[generator] ---- 消息结束 ----\n')
|
||||||
|
|
||||||
await streamGenerateLlamaCpp(userMessage, systemPrompt, callbacks)
|
await streamChatCompletion(
|
||||||
|
[
|
||||||
|
{ role: 'system', content: systemPrompt },
|
||||||
|
{ role: 'user', content: userMessage }
|
||||||
|
],
|
||||||
|
callbacks
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 拍照提问:不检索、不拼接知识库上下文,图片 + 问题直接发给多模态生成服务
|
||||||
|
export async function streamGenerateVision(
|
||||||
|
question: string,
|
||||||
|
imageDataUrl: string,
|
||||||
|
callbacks: GeneratorCallbacks
|
||||||
|
): Promise<void> {
|
||||||
|
console.log('\n[generator] ---- 拍照提问 ----')
|
||||||
|
console.log(`[generator] 服务地址:${ER_BACKEND_URL}(er_backend 代理)`)
|
||||||
|
console.log('[generator] 问题:', question)
|
||||||
|
console.log('[generator] ---- 消息结束 ----\n')
|
||||||
|
|
||||||
|
await streamChatCompletion(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
role: 'user',
|
||||||
|
content: [
|
||||||
|
{ type: 'text', text: question },
|
||||||
|
{ type: 'image_url', image_url: { url: imageDataUrl } }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
callbacks
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// llama.cpp server OpenAI 兼容接口(/v1/chat/completions),SSE 格式
|
// llama.cpp server OpenAI 兼容接口(/v1/chat/completions),SSE 格式
|
||||||
async function streamGenerateLlamaCpp(
|
async function streamChatCompletion(
|
||||||
userMessage: string,
|
messages: ChatMessage[],
|
||||||
systemPrompt: string,
|
|
||||||
callbacks: GeneratorCallbacks
|
callbacks: GeneratorCallbacks
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const baseUrl = ER_BACKEND_URL
|
const baseUrl = ER_BACKEND_URL
|
||||||
@ -60,10 +101,7 @@ async function streamGenerateLlamaCpp(
|
|||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
model: 'default',
|
model: 'default',
|
||||||
messages: [
|
messages,
|
||||||
{ role: 'system', content: systemPrompt },
|
|
||||||
{ role: 'user', content: userMessage }
|
|
||||||
],
|
|
||||||
stream: true,
|
stream: true,
|
||||||
stream_options: { include_usage: true }
|
stream_options: { include_usage: true }
|
||||||
})
|
})
|
||||||
|
|||||||
@ -17,6 +17,10 @@ contextBridge.exposeInMainWorld('electronAPI', {
|
|||||||
query: (payload: { question: string; systemPrompt: string; topK: number }): Promise<void> =>
|
query: (payload: { question: string; systemPrompt: string; topK: number }): Promise<void> =>
|
||||||
ipcRenderer.invoke('query', payload),
|
ipcRenderer.invoke('query', payload),
|
||||||
|
|
||||||
|
// 拍照提问(不走检索,直接把图片+问题发给多模态生成服务)
|
||||||
|
visionQuery: (payload: { question: string; imageDataUrl: string }): Promise<void> =>
|
||||||
|
ipcRenderer.invoke('vision-query', payload),
|
||||||
|
|
||||||
// 状态 & 配置
|
// 状态 & 配置
|
||||||
getDbStatus: (): Promise<{ count: number; ready: boolean }> =>
|
getDbStatus: (): Promise<{ count: number; ready: boolean }> =>
|
||||||
ipcRenderer.invoke('get-db-status'),
|
ipcRenderer.invoke('get-db-status'),
|
||||||
|
|||||||
170
src/renderer/src/components/CameraCapture.vue
Normal file
170
src/renderer/src/components/CameraCapture.vue
Normal file
@ -0,0 +1,170 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { ref, watch, onUnmounted } from 'vue'
|
||||||
|
import { NModal, NCard } from 'naive-ui'
|
||||||
|
|
||||||
|
const props = defineProps<{ show: boolean; initialDeviceId: string }>()
|
||||||
|
const emit = defineEmits<{
|
||||||
|
'update:show': [value: boolean]
|
||||||
|
captured: [dataUrl: string]
|
||||||
|
deviceChange: [deviceId: string]
|
||||||
|
}>()
|
||||||
|
|
||||||
|
const videoRef = ref<HTMLVideoElement | null>(null)
|
||||||
|
const canvasRef = ref<HTMLCanvasElement | null>(null)
|
||||||
|
|
||||||
|
const devices = ref<MediaDeviceInfo[]>([])
|
||||||
|
const selectedDeviceId = ref('')
|
||||||
|
const capturedDataUrl = ref('')
|
||||||
|
const starting = ref(false)
|
||||||
|
const errorMessage = ref('')
|
||||||
|
|
||||||
|
let stream: MediaStream | null = null
|
||||||
|
|
||||||
|
function stopStream(): void {
|
||||||
|
stream?.getTracks().forEach((t) => t.stop())
|
||||||
|
stream = null
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startStream(deviceId: string): Promise<void> {
|
||||||
|
stopStream()
|
||||||
|
errorMessage.value = ''
|
||||||
|
starting.value = true
|
||||||
|
try {
|
||||||
|
stream = await navigator.mediaDevices.getUserMedia({
|
||||||
|
video: deviceId ? { deviceId: { exact: deviceId } } : true
|
||||||
|
})
|
||||||
|
if (videoRef.value) {
|
||||||
|
videoRef.value.srcObject = stream
|
||||||
|
await videoRef.value.play()
|
||||||
|
}
|
||||||
|
// 拿到权限后 label 才会有值,借机刷新一次设备列表
|
||||||
|
await refreshDevices()
|
||||||
|
} catch (e) {
|
||||||
|
errorMessage.value = `无法打开摄像头:${String(e)}`
|
||||||
|
} finally {
|
||||||
|
starting.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshDevices(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const list = await navigator.mediaDevices.enumerateDevices()
|
||||||
|
devices.value = list.filter((d) => d.kind === 'videoinput')
|
||||||
|
if (!selectedDeviceId.value && devices.value.length > 0) {
|
||||||
|
selectedDeviceId.value = devices.value[0].deviceId
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// enumerateDevices 在少数环境下可能不可用,静默忽略,不影响已经拿到的 stream
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function switchDevice(deviceId: string): Promise<void> {
|
||||||
|
selectedDeviceId.value = deviceId
|
||||||
|
emit('deviceChange', deviceId)
|
||||||
|
await startStream(deviceId)
|
||||||
|
}
|
||||||
|
|
||||||
|
function capture(): void {
|
||||||
|
const video = videoRef.value
|
||||||
|
const canvas = canvasRef.value
|
||||||
|
if (!video || !canvas || video.videoWidth === 0) return
|
||||||
|
canvas.width = video.videoWidth
|
||||||
|
canvas.height = video.videoHeight
|
||||||
|
const ctx = canvas.getContext('2d')
|
||||||
|
if (!ctx) return
|
||||||
|
ctx.drawImage(video, 0, 0, canvas.width, canvas.height)
|
||||||
|
capturedDataUrl.value = canvas.toDataURL('image/jpeg', 0.85)
|
||||||
|
}
|
||||||
|
|
||||||
|
function retake(): void {
|
||||||
|
capturedDataUrl.value = ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function confirm(): void {
|
||||||
|
if (!capturedDataUrl.value) return
|
||||||
|
emit('captured', capturedDataUrl.value)
|
||||||
|
close()
|
||||||
|
}
|
||||||
|
|
||||||
|
function close(): void {
|
||||||
|
emit('update:show', false)
|
||||||
|
}
|
||||||
|
|
||||||
|
watch(
|
||||||
|
() => props.show,
|
||||||
|
async (val) => {
|
||||||
|
if (val) {
|
||||||
|
capturedDataUrl.value = ''
|
||||||
|
selectedDeviceId.value = props.initialDeviceId
|
||||||
|
await refreshDevices()
|
||||||
|
await startStream(selectedDeviceId.value)
|
||||||
|
} else {
|
||||||
|
stopStream()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
onUnmounted(() => stopStream())
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<n-modal :show="show" :mask-closable="false" style="width: 720px; max-width: 92vw;" @update:show="close">
|
||||||
|
<n-card title="拍照提问" closable @close="close">
|
||||||
|
<div class="cam-body">
|
||||||
|
<div v-if="errorMessage" class="cam-error">{{ errorMessage }}</div>
|
||||||
|
|
||||||
|
<div v-show="!capturedDataUrl" class="cam-preview-wrap">
|
||||||
|
<video ref="videoRef" class="cam-preview" autoplay muted playsinline />
|
||||||
|
<div v-if="starting" class="cam-loading">正在打开摄像头...</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<img v-if="capturedDataUrl" :src="capturedDataUrl" class="cam-preview" alt="拍摄预览" />
|
||||||
|
|
||||||
|
<canvas ref="canvasRef" style="display:none" />
|
||||||
|
|
||||||
|
<div v-if="devices.length > 1 && !capturedDataUrl" class="cam-device-row">
|
||||||
|
<label class="fl">摄像头</label>
|
||||||
|
<select class="s-inp" :value="selectedDeviceId" @change="switchDevice(($event.target as HTMLSelectElement).value)">
|
||||||
|
<option v-for="d in devices" :key="d.deviceId" :value="d.deviceId">
|
||||||
|
{{ d.label || `摄像头 ${d.deviceId.slice(0, 6)}` }}
|
||||||
|
</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<template #footer>
|
||||||
|
<div class="cam-actions">
|
||||||
|
<template v-if="!capturedDataUrl">
|
||||||
|
<button class="i-btn i-btn-primary i-btn-lg" :disabled="starting || !!errorMessage" @click="capture">拍照</button>
|
||||||
|
</template>
|
||||||
|
<template v-else>
|
||||||
|
<button class="i-btn i-btn-ghost" @click="retake">重拍</button>
|
||||||
|
<button class="i-btn i-btn-primary i-btn-lg" @click="confirm">使用这张</button>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
</n-card>
|
||||||
|
</n-modal>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.cam-body { display: flex; flex-direction: column; gap: 14px; }
|
||||||
|
.cam-preview-wrap { position: relative; }
|
||||||
|
.cam-preview {
|
||||||
|
width: 100%; max-height: 60vh; object-fit: contain;
|
||||||
|
background: #000; border-radius: var(--radius-sm);
|
||||||
|
}
|
||||||
|
.cam-loading {
|
||||||
|
position: absolute; inset: 0; display: flex;
|
||||||
|
align-items: center; justify-content: center;
|
||||||
|
color: #fff; font-size: 15px; background: rgba(0,0,0,.35);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
}
|
||||||
|
.cam-error {
|
||||||
|
color: var(--red); background: #FBEAE8;
|
||||||
|
border-radius: var(--radius-sm); padding: 12px 14px; font-size: 15px;
|
||||||
|
}
|
||||||
|
.cam-device-row { display: flex; align-items: center; gap: 12px; }
|
||||||
|
.cam-device-row .fl { flex-shrink: 0; font-size: 15px; color: var(--ink-2); }
|
||||||
|
.cam-actions { display: flex; justify-content: flex-end; gap: 12px; }
|
||||||
|
</style>
|
||||||
@ -6,6 +6,7 @@ import { useRagStore } from '../stores/rag'
|
|||||||
import { resolveImagePaths } from '../utils/markdown'
|
import { resolveImagePaths } from '../utils/markdown'
|
||||||
import { AudioRecorder } from '../utils/audio'
|
import { AudioRecorder } from '../utils/audio'
|
||||||
import { ttsPlayer } from '../utils/tts'
|
import { ttsPlayer } from '../utils/tts'
|
||||||
|
import CameraCapture from './CameraCapture.vue'
|
||||||
import type { Chunk } from '../types'
|
import type { Chunk } from '../types'
|
||||||
|
|
||||||
interface InferStats {
|
interface InferStats {
|
||||||
@ -153,19 +154,13 @@ function onAnswerClick(e: MouseEvent): void {
|
|||||||
popupVisible.value = true
|
popupVisible.value = true
|
||||||
}
|
}
|
||||||
|
|
||||||
async function sendQuery(): Promise<void> {
|
// 两种提问方式(文字检索 / 拍照)共用的准备工作:重置状态、打断正在进行的语音输入、停止旧的 TTS 播报
|
||||||
const q = question.value.trim()
|
function beginQuery(): void {
|
||||||
if (!q) return
|
|
||||||
if (!store.dbStatus.ready) { message.warning('向量库为空,请先入库文档'); return }
|
|
||||||
if (store.querying) return
|
|
||||||
|
|
||||||
store.querying = true
|
store.querying = true
|
||||||
store.resetAnswer()
|
store.resetAnswer()
|
||||||
inferStats.value = null
|
inferStats.value = null
|
||||||
readyCitations.value = new Set()
|
readyCitations.value = new Set()
|
||||||
if (hoverTimer) { clearTimeout(hoverTimer); hoverTimer = null }
|
if (hoverTimer) { clearTimeout(hoverTimer); hoverTimer = null }
|
||||||
sentQuestion.value = q
|
|
||||||
question.value = ''
|
|
||||||
// 立即解除 ASR 监听并清空状态,防止延迟到达的 asr:stopped 通过 syncAsrText() 恢复输入框
|
// 立即解除 ASR 监听并清空状态,防止延迟到达的 asr:stopped 通过 syncAsrText() 恢复输入框
|
||||||
cleanupAsrListeners()
|
cleanupAsrListeners()
|
||||||
asrBaseText = ''
|
asrBaseText = ''
|
||||||
@ -185,6 +180,18 @@ async function sendQuery(): Promise<void> {
|
|||||||
startWaitTimer()
|
startWaitTimer()
|
||||||
ttsPlayer.stop()
|
ttsPlayer.stop()
|
||||||
syncTtsConfig()
|
syncTtsConfig()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendQuery(): Promise<void> {
|
||||||
|
const q = question.value.trim()
|
||||||
|
if (!q) return
|
||||||
|
if (!store.dbStatus.ready) { message.warning('向量库为空,请先入库文档'); return }
|
||||||
|
if (store.querying) return
|
||||||
|
|
||||||
|
beginQuery()
|
||||||
|
sentQuestion.value = q
|
||||||
|
sentImage.value = ''
|
||||||
|
question.value = ''
|
||||||
|
|
||||||
let firstDelta = true
|
let firstDelta = true
|
||||||
|
|
||||||
@ -228,8 +235,84 @@ async function sendQuery(): Promise<void> {
|
|||||||
await window.electronAPI.query({ question: q, systemPrompt: store.config?.rag.systemPrompt ?? '', topK: topK.value })
|
await window.electronAPI.query({ question: q, systemPrompt: store.config?.rag.systemPrompt ?? '', topK: topK.value })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const VISION_DEFAULT_QUESTION = '请描述这张图片的内容'
|
||||||
|
|
||||||
|
// 拍照提问:不查向量库,图片 + 问题直接发给多模态生成服务
|
||||||
|
async function sendVisionQuery(): Promise<void> {
|
||||||
|
if (!pendingImage.value) return
|
||||||
|
if (store.querying) return
|
||||||
|
|
||||||
|
const q = question.value.trim() || VISION_DEFAULT_QUESTION
|
||||||
|
const img = pendingImage.value
|
||||||
|
|
||||||
|
beginQuery()
|
||||||
|
sentQuestion.value = q
|
||||||
|
sentImage.value = img
|
||||||
|
question.value = ''
|
||||||
|
pendingImage.value = ''
|
||||||
|
|
||||||
|
let firstDelta = true
|
||||||
|
|
||||||
|
const onDelta = (data: unknown) => {
|
||||||
|
if (firstDelta) { firstDelta = false; stopWaitTimer() }
|
||||||
|
store.rawAnswer += (data as { delta: string }).delta
|
||||||
|
scrollToBottom()
|
||||||
|
}
|
||||||
|
const onDone = (data: unknown) => {
|
||||||
|
stopWaitTimer()
|
||||||
|
store.querying = false
|
||||||
|
queryStage.value = ''
|
||||||
|
const stats = (data as { stats?: InferStats | null }).stats
|
||||||
|
if (stats) inferStats.value = stats
|
||||||
|
cleanup()
|
||||||
|
}
|
||||||
|
const onError = (data: unknown) => {
|
||||||
|
stopWaitTimer()
|
||||||
|
store.querying = false
|
||||||
|
queryStage.value = ''
|
||||||
|
message.error((data as { message: string }).message)
|
||||||
|
cleanup()
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanup(): void {
|
||||||
|
window.electronAPI.off('vision-query:delta')
|
||||||
|
window.electronAPI.off('vision-query:done')
|
||||||
|
window.electronAPI.off('vision-query:error')
|
||||||
|
}
|
||||||
|
|
||||||
|
window.electronAPI.on('vision-query:delta', onDelta)
|
||||||
|
window.electronAPI.on('vision-query:done', onDone)
|
||||||
|
window.electronAPI.on('vision-query:error', onError)
|
||||||
|
|
||||||
|
await window.electronAPI.visionQuery({ question: q, imageDataUrl: img })
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleSend(): void {
|
||||||
|
if (pendingImage.value) { sendVisionQuery(); return }
|
||||||
|
sendQuery()
|
||||||
|
}
|
||||||
|
|
||||||
function handleEnter(e: KeyboardEvent): void {
|
function handleEnter(e: KeyboardEvent): void {
|
||||||
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); sendQuery() }
|
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); handleSend() }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 拍照提问 ──
|
||||||
|
const cameraOpen = ref(false)
|
||||||
|
const pendingImage = ref('')
|
||||||
|
const sentImage = ref('')
|
||||||
|
|
||||||
|
function onCaptured(dataUrl: string): void {
|
||||||
|
pendingImage.value = dataUrl
|
||||||
|
}
|
||||||
|
|
||||||
|
function removePendingImage(): void {
|
||||||
|
pendingImage.value = ''
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCameraDeviceChange(deviceId: string): Promise<void> {
|
||||||
|
if (!store.config) return
|
||||||
|
if (store.config.vision.cameraDeviceId === deviceId) return
|
||||||
|
await store.saveConfig({ ...store.config, vision: { ...store.config.vision, cameraDeviceId: deviceId } })
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 语音输入 ──
|
// ── 语音输入 ──
|
||||||
@ -498,6 +581,7 @@ onUnmounted(() => {
|
|||||||
<div class="q-ic">问</div>
|
<div class="q-ic">问</div>
|
||||||
<div>
|
<div>
|
||||||
<div class="q-label">提问</div>
|
<div class="q-label">提问</div>
|
||||||
|
<img v-if="sentImage" :src="sentImage" class="q-image" alt="提问附带的照片" />
|
||||||
<div class="q-text">{{ sentQuestion }}</div>
|
<div class="q-text">{{ sentQuestion }}</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@ -571,22 +655,38 @@ onUnmounted(() => {
|
|||||||
语音输出
|
语音输出
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
<!-- Camera button -->
|
||||||
|
<div v-if="store.config?.vision.enabled" class="voice-btn-wrap">
|
||||||
|
<button class="voice-btn" @click="cameraOpen = true">
|
||||||
|
<svg width="26" height="26" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round">
|
||||||
|
<path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z"/>
|
||||||
|
<circle cx="12" cy="13" r="4"/>
|
||||||
|
</svg>
|
||||||
|
拍照提问
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Question textarea -->
|
<!-- Question textarea -->
|
||||||
|
<div class="q-input-wrap">
|
||||||
|
<div v-if="pendingImage" class="pending-image-chip">
|
||||||
|
<img :src="pendingImage" alt="待发送的照片" />
|
||||||
|
<button class="pending-image-remove" title="移除照片" @click="removePendingImage">×</button>
|
||||||
|
</div>
|
||||||
<textarea
|
<textarea
|
||||||
v-model="question"
|
v-model="question"
|
||||||
class="q-input"
|
class="q-input"
|
||||||
:disabled="batchRunning"
|
:disabled="batchRunning"
|
||||||
placeholder="输入问题,Enter 发送,Shift+Enter 换行…"
|
:placeholder="pendingImage ? '针对这张照片提问,留空则默认描述图片…' : '输入问题,Enter 发送,Shift+Enter 换行…'"
|
||||||
@keydown="handleEnter"
|
@keydown="handleEnter"
|
||||||
/>
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Send -->
|
<!-- Send -->
|
||||||
<button
|
<button
|
||||||
class="i-btn i-btn-primary i-btn-lg send-btn"
|
class="i-btn i-btn-primary i-btn-lg send-btn"
|
||||||
:disabled="store.querying || !question.trim() || batchRunning"
|
:disabled="store.querying || (!question.trim() && !pendingImage) || batchRunning"
|
||||||
@click="sendQuery"
|
@click="handleSend"
|
||||||
>
|
>
|
||||||
<svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round">
|
<svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round">
|
||||||
<path d="M22 2L11 13"/><path d="M22 2l-7 20-4-9-9-4 20-7z"/>
|
<path d="M22 2L11 13"/><path d="M22 2l-7 20-4-9-9-4 20-7z"/>
|
||||||
@ -619,6 +719,14 @@ onUnmounted(() => {
|
|||||||
</n-scrollbar>
|
</n-scrollbar>
|
||||||
</n-card>
|
</n-card>
|
||||||
</n-modal>
|
</n-modal>
|
||||||
|
|
||||||
|
<!-- Camera capture -->
|
||||||
|
<CameraCapture
|
||||||
|
v-model:show="cameraOpen"
|
||||||
|
:initial-device-id="store.config?.vision.cameraDeviceId ?? ''"
|
||||||
|
@captured="onCaptured"
|
||||||
|
@device-change="onCameraDeviceChange"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@ -662,6 +770,11 @@ onUnmounted(() => {
|
|||||||
}
|
}
|
||||||
.q-label { font-size: 13px; color: var(--blue-d); font-weight: 700; letter-spacing: .8px; margin-bottom: 3px; }
|
.q-label { font-size: 13px; color: var(--blue-d); font-weight: 700; letter-spacing: .8px; margin-bottom: 3px; }
|
||||||
.q-text { font-size: 20px; font-weight: 700; line-height: 1.4; color: var(--ink); }
|
.q-text { font-size: 20px; font-weight: 700; line-height: 1.4; color: var(--ink); }
|
||||||
|
.q-image {
|
||||||
|
max-width: 240px; max-height: 240px; object-fit: contain;
|
||||||
|
border-radius: 8px; margin-bottom: 10px; display: block;
|
||||||
|
border: 2px solid var(--line);
|
||||||
|
}
|
||||||
|
|
||||||
/* Stage indicator */
|
/* Stage indicator */
|
||||||
.stage-row {
|
.stage-row {
|
||||||
@ -750,8 +863,10 @@ onUnmounted(() => {
|
|||||||
display: flex; align-items: flex-end; gap: 14px;
|
display: flex; align-items: flex-end; gap: 14px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.q-input-wrap { flex: 1; min-width: 0; position: relative; }
|
||||||
|
|
||||||
.q-input {
|
.q-input {
|
||||||
flex: 1; min-width: 0;
|
width: 100%; box-sizing: border-box;
|
||||||
height: 96px;
|
height: 96px;
|
||||||
border: 2px solid var(--line-2); border-radius: var(--radius);
|
border: 2px solid var(--line-2); border-radius: var(--radius);
|
||||||
padding: 16px 20px; font-family: inherit;
|
padding: 16px 20px; font-family: inherit;
|
||||||
@ -762,6 +877,26 @@ onUnmounted(() => {
|
|||||||
.q-input:focus { outline: none; border-color: var(--blue); }
|
.q-input:focus { outline: none; border-color: var(--blue); }
|
||||||
.q-input:disabled { background: var(--panel-2); color: var(--ink-3); }
|
.q-input:disabled { background: var(--panel-2); color: var(--ink-3); }
|
||||||
|
|
||||||
|
.pending-image-chip {
|
||||||
|
position: absolute; left: 14px; bottom: 100%;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
display: flex; align-items: center;
|
||||||
|
border: 2px solid var(--line-2); border-radius: var(--radius-sm);
|
||||||
|
background: var(--panel); padding: 4px;
|
||||||
|
box-shadow: 0 6px 20px rgba(0,0,0,.12);
|
||||||
|
}
|
||||||
|
.pending-image-chip img {
|
||||||
|
width: 64px; height: 64px; object-fit: cover;
|
||||||
|
border-radius: 6px;
|
||||||
|
}
|
||||||
|
.pending-image-remove {
|
||||||
|
position: absolute; top: -8px; right: -8px;
|
||||||
|
width: 22px; height: 22px; border-radius: 50%;
|
||||||
|
border: none; background: var(--red); color: #fff;
|
||||||
|
font-size: 15px; line-height: 1; cursor: pointer;
|
||||||
|
display: flex; align-items: center; justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
.send-btn { height: 96px; min-width: 150px; }
|
.send-btn { height: 96px; min-width: 150px; }
|
||||||
.batch-btn { height: 96px; min-width: 130px; }
|
.batch-btn { height: 96px; min-width: 130px; }
|
||||||
.i-btn-success { background: var(--green); color: #fff; box-shadow: 0 2px 0 #0a6238; }
|
.i-btn-success { background: var(--green); color: #fff; box-shadow: 0 2px 0 #0a6238; }
|
||||||
|
|||||||
@ -25,7 +25,8 @@ const form = ref<AppConfig>({
|
|||||||
voice: {
|
voice: {
|
||||||
asr: { enabled: false, executablePath: '', modelDir: '', numThreads: 2, vadSilenceTimeoutMs: 5000, vadEnergyThreshold: 0.005, inputGain: 10, punctuationModelDir: '' },
|
asr: { enabled: false, executablePath: '', modelDir: '', numThreads: 2, vadSilenceTimeoutMs: 5000, vadEnergyThreshold: 0.005, inputGain: 10, punctuationModelDir: '' },
|
||||||
tts: { enabled: false, voiceURI: '', pitch: 1, rate: 1, volume: 1 }
|
tts: { enabled: false, voiceURI: '', pitch: 1, rate: 1, volume: 1 }
|
||||||
}
|
},
|
||||||
|
vision: { enabled: false, cameraDeviceId: '' }
|
||||||
})
|
})
|
||||||
|
|
||||||
const activeTab = ref<'index' | 'basic' | 'voice' | 'batch' | 'log'>('basic')
|
const activeTab = ref<'index' | 'basic' | 'voice' | 'batch' | 'log'>('basic')
|
||||||
@ -175,6 +176,23 @@ async function save(): Promise<void> {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- ── 拍照提问 ── -->
|
||||||
|
<div class="sgroup-title">拍照提问</div>
|
||||||
|
<div class="field">
|
||||||
|
<div class="fl">启用拍照提问</div>
|
||||||
|
<div class="fc">
|
||||||
|
<label class="s-radio" :class="{ on: form.vision.enabled }" @click="form.vision.enabled = true">
|
||||||
|
<span class="rk"></span>开启
|
||||||
|
</label>
|
||||||
|
<label class="s-radio" :class="{ on: !form.vision.enabled }" @click="form.vision.enabled = false">
|
||||||
|
<span class="rk"></span>关闭
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p class="s-note" style="padding-left:120px;margin:-6px 0 0;">
|
||||||
|
依赖 er_backend 当前加载的是支持视觉输入的模型(能看到 allow_image=1),否则开启后拍照提问会请求失败。
|
||||||
|
</p>
|
||||||
|
|
||||||
</div><!-- /基本设置 -->
|
</div><!-- /基本设置 -->
|
||||||
|
|
||||||
<!-- ── 语音输入输出 ── -->
|
<!-- ── 语音输入输出 ── -->
|
||||||
|
|||||||
@ -50,6 +50,10 @@ export interface AppConfig {
|
|||||||
volume: number
|
volume: number
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
vision: {
|
||||||
|
enabled: boolean
|
||||||
|
cameraDeviceId: string
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export type DetectedCard = 'er160' | 'er25' | 'none'
|
export type DetectedCard = 'er160' | 'er25' | 'none'
|
||||||
@ -113,6 +117,7 @@ declare global {
|
|||||||
selectDirectory: () => Promise<string | null>
|
selectDirectory: () => Promise<string | null>
|
||||||
indexDocuments: (payload: { paths: string[]; mode: 'append' | 'rebuild' }) => Promise<void>
|
indexDocuments: (payload: { paths: string[]; mode: 'append' | 'rebuild' }) => Promise<void>
|
||||||
query: (payload: { question: string; systemPrompt: string; topK: number }) => Promise<void>
|
query: (payload: { question: string; systemPrompt: string; topK: number }) => Promise<void>
|
||||||
|
visionQuery: (payload: { question: string; imageDataUrl: string }) => Promise<void>
|
||||||
getDbStatus: () => Promise<DbStatus>
|
getDbStatus: () => Promise<DbStatus>
|
||||||
getConfig: () => Promise<AppConfig>
|
getConfig: () => Promise<AppConfig>
|
||||||
saveConfig: (config: AppConfig) => Promise<void>
|
saveConfig: (config: AppConfig) => Promise<void>
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user