feat: 实现拍照提问模式 MVP（阶段1+2）
按 docs/feature-pic-design.md 的方案实现： - AppConfig 新增 vision 字段（enabled 总开关、cameraDeviceId 记住上次 选用的摄像头），主进程/渲染层类型两处同步 - generator.ts 抽出共享的 streamChatCompletion()，新增 streamGenerateVision() 支持 OpenAI 视觉 API 的 image_url content-part， 文字问答路径不受影响 - 新增 visionHandler.ts（vision-query IPC，跳过向量检索，直接转发 图片+问题给 er_backend 代理），注册进 ipc/index.ts - 新增 CameraCapture.vue：getUserMedia 取景/拍照/重拍/多摄像头切换， 纯交互组件不掺业务逻辑 - ChatPanel.vue 接入拍照入口（仅 vision.enabled 时显示）、待发送图片 缩略图、问题气泡里的图片展示；抽出 beginQuery() 复用文字/拍照两条 路径共同的状态重置逻辑 - SettingsModal.vue 加拍照提问开关 - index.ts 权限处理器注释说明 getUserMedia 摄像头请求也走 'media' 不做检索、不做照片持久化、不做多摄像头预配置界面（MVP 范围见设计文档 §3）。 本地未跑 vue-tsc，接下来在真机上验证类型检查 + 构建 + 端到端联调。 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ssurui
parent
2ae01c2050
commit
21f9c7e453
@ -58,6 +58,10 @@ export interface AppConfig {
|
||||
volume: number
|
||||
}
|
||||
}
|
||||
vision: {
|
||||
enabled: boolean // 拍照提问入口总开关，默认关闭（依赖 er_backend 是否加载了支持视觉的模型）
|
||||
cameraDeviceId: string // 上次选用的摄像头 deviceId，空字符串 = 使用第一个可用设备
|
||||
}
|
||||
}
|
||||
|
||||
export const DEFAULT_CONFIG: AppConfig = {
|
||||
@ -106,6 +110,10 @@ export const DEFAULT_CONFIG: AppConfig = {
|
||||
volume: 1
|
||||
}
|
||||
},
|
||||
vision: {
|
||||
enabled: false,
|
||||
cameraDeviceId: ''
|
||||
},
|
||||
rag: {
|
||||
chunkSize: 480,
|
||||
chunkOverlap: 0,
|
||||
@ -198,6 +206,10 @@ export function getConfig(): AppConfig {
|
||||
rate: typeof raw.voice?.tts?.rate === 'number' ? raw.voice.tts.rate : DEFAULT_CONFIG.voice.tts.rate,
|
||||
volume: typeof raw.voice?.tts?.volume === 'number' ? raw.voice.tts.volume : DEFAULT_CONFIG.voice.tts.volume
|
||||
}
|
||||
},
|
||||
vision: {
|
||||
enabled: raw.vision?.enabled === true,
|
||||
cameraDeviceId: typeof raw.vision?.cameraDeviceId === 'string' ? raw.vision.cameraDeviceId : ''
|
||||
}
|
||||
}
|
||||
applyPlatformOverrides(config)
|
||||
|
||||
@ -80,7 +80,8 @@ function getMimeType(filePath: string): string {
|
||||
}
|
||||
|
||||
app.whenReady().then(() => {
|
||||
// 允许渲染进程请求麦克风权限（语音输入所需）
|
||||
// 允许渲染进程请求麦克风权限（语音输入所需）；Electron 的 getUserMedia 摄像头请求
|
||||
// （拍照提问用）在权限系统里也归为 'media'，跟麦克风共用同一个判断，不需要单独加 'camera'
|
||||
session.defaultSession.setPermissionRequestHandler((webContents, permission, callback) => {
|
||||
const allowed = permission === 'media' || permission === 'microphone'
|
||||
console.log(`[permission] ${permission} -> ${allowed ? 'granted' : 'denied'}`)
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { registerIndexHandlers } from './indexHandler'
|
||||
import { registerQueryHandlers } from './queryHandler'
|
||||
import { registerVisionHandlers } from './visionHandler'
|
||||
import { registerStatusHandlers } from './statusHandler'
|
||||
import { registerAsrHandlers } from './asrHandler'
|
||||
import { registerTtsHandlers } from './ttsHandler'
|
||||
@ -11,6 +12,7 @@ import type { BrowserWindow } from 'electron'
|
||||
export function registerIpcHandlers(getWin: () => BrowserWindow | null): void {
|
||||
registerIndexHandlers(getWin)
|
||||
registerQueryHandlers(getWin)
|
||||
registerVisionHandlers(getWin)
|
||||
registerStatusHandlers(getWin)
|
||||
registerAsrHandlers(getWin)
|
||||
registerTtsHandlers(getWin)
|
||||
|
||||
31
src/main/ipc/visionHandler.ts
Normal file
31
src/main/ipc/visionHandler.ts
Normal file
@ -0,0 +1,31 @@
|
||||
import { ipcMain, BrowserWindow } from 'electron'
|
||||
import { streamGenerateVision } from '../rag/generator'
|
||||
|
||||
// 拍照提问：跳过向量检索，图片 + 问题直接发给 er_backend 代理的多模态生成服务
|
||||
export function registerVisionHandlers(getWin: () => BrowserWindow | null): void {
|
||||
ipcMain.handle('vision-query', async (_event, payload: {
|
||||
question: string
|
||||
imageDataUrl: string
|
||||
}) => {
|
||||
console.log(`\n${'='.repeat(60)}`)
|
||||
console.log('[vision-query] 问题：', payload.question)
|
||||
|
||||
const send = (channel: string, data: unknown): void => {
|
||||
getWin()?.webContents.send(channel, data)
|
||||
}
|
||||
|
||||
try {
|
||||
await streamGenerateVision(
|
||||
payload.question,
|
||||
payload.imageDataUrl,
|
||||
{
|
||||
onDelta: (delta) => send('vision-query:delta', { delta }),
|
||||
onDone: (stats) => send('vision-query:done', { stats: stats ?? null }),
|
||||
onError: (message) => send('vision-query:error', { message })
|
||||
}
|
||||
)
|
||||
} catch (e) {
|
||||
send('vision-query:error', { message: String(e) })
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -14,6 +14,17 @@ export interface GeneratorCallbacks {
|
||||
onError: (message: string) => void
|
||||
}
|
||||
|
||||
// OpenAI 视觉 API 的 content-part 格式，multimodal 模型（如 Qwen-VL + mmproj）用这种数组形式
|
||||
// 代替纯字符串 content 来同时携带文字与图片
|
||||
type ContentPart =
|
||||
| { type: 'text'; text: string }
|
||||
| { type: 'image_url'; image_url: { url: string } }
|
||||
|
||||
interface ChatMessage {
|
||||
role: 'system' | 'user'
|
||||
content: string | ContentPart[]
|
||||
}
|
||||
|
||||
// 生成服务经 er_backend 代理访问其管理的 llama-server（OpenAI 兼容接口，SSE 流式）
|
||||
export async function streamGenerate(
|
||||
question: string,
|
||||
@ -37,13 +48,43 @@ ${context}
|
||||
console.log('[generator] user message：\n', userMessage)
|
||||
console.log('[generator] ---- 消息结束 ----\n')
|
||||
|
||||
await streamGenerateLlamaCpp(userMessage, systemPrompt, callbacks)
|
||||
await streamChatCompletion(
|
||||
[
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userMessage }
|
||||
],
|
||||
callbacks
|
||||
)
|
||||
}
|
||||
|
||||
// 拍照提问：不检索、不拼接知识库上下文，图片 + 问题直接发给多模态生成服务
|
||||
export async function streamGenerateVision(
|
||||
question: string,
|
||||
imageDataUrl: string,
|
||||
callbacks: GeneratorCallbacks
|
||||
): Promise<void> {
|
||||
console.log('\n[generator] ---- 拍照提问 ----')
|
||||
console.log(`[generator] 服务地址：${ER_BACKEND_URL}（er_backend 代理）`)
|
||||
console.log('[generator] 问题：', question)
|
||||
console.log('[generator] ---- 消息结束 ----\n')
|
||||
|
||||
await streamChatCompletion(
|
||||
[
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: question },
|
||||
{ type: 'image_url', image_url: { url: imageDataUrl } }
|
||||
]
|
||||
}
|
||||
],
|
||||
callbacks
|
||||
)
|
||||
}
|
||||
|
||||
// llama.cpp server OpenAI 兼容接口（/v1/chat/completions），SSE 格式
|
||||
async function streamGenerateLlamaCpp(
|
||||
userMessage: string,
|
||||
systemPrompt: string,
|
||||
async function streamChatCompletion(
|
||||
messages: ChatMessage[],
|
||||
callbacks: GeneratorCallbacks
|
||||
): Promise<void> {
|
||||
const baseUrl = ER_BACKEND_URL
|
||||
@ -60,10 +101,7 @@ async function streamGenerateLlamaCpp(
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'default',
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userMessage }
|
||||
],
|
||||
messages,
|
||||
stream: true,
|
||||
stream_options: { include_usage: true }
|
||||
})
|
||||
|
||||
@ -17,6 +17,10 @@ contextBridge.exposeInMainWorld('electronAPI', {
|
||||
query: (payload: { question: string; systemPrompt: string; topK: number }): Promise<void> =>
|
||||
ipcRenderer.invoke('query', payload),
|
||||
|
||||
// 拍照提问（不走检索，直接把图片+问题发给多模态生成服务）
|
||||
visionQuery: (payload: { question: string; imageDataUrl: string }): Promise<void> =>
|
||||
ipcRenderer.invoke('vision-query', payload),
|
||||
|
||||
// 状态 & 配置
|
||||
getDbStatus: (): Promise<{ count: number; ready: boolean }> =>
|
||||
ipcRenderer.invoke('get-db-status'),
|
||||
|
||||
170
src/renderer/src/components/CameraCapture.vue
Normal file
170
src/renderer/src/components/CameraCapture.vue
Normal file
@ -0,0 +1,170 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, watch, onUnmounted } from 'vue'
|
||||
import { NModal, NCard } from 'naive-ui'
|
||||
|
||||
const props = defineProps<{ show: boolean; initialDeviceId: string }>()
|
||||
const emit = defineEmits<{
|
||||
'update:show': [value: boolean]
|
||||
captured: [dataUrl: string]
|
||||
deviceChange: [deviceId: string]
|
||||
}>()
|
||||
|
||||
const videoRef = ref<HTMLVideoElement | null>(null)
|
||||
const canvasRef = ref<HTMLCanvasElement | null>(null)
|
||||
|
||||
const devices = ref<MediaDeviceInfo[]>([])
|
||||
const selectedDeviceId = ref('')
|
||||
const capturedDataUrl = ref('')
|
||||
const starting = ref(false)
|
||||
const errorMessage = ref('')
|
||||
|
||||
let stream: MediaStream | null = null
|
||||
|
||||
function stopStream(): void {
|
||||
stream?.getTracks().forEach((t) => t.stop())
|
||||
stream = null
|
||||
}
|
||||
|
||||
async function startStream(deviceId: string): Promise<void> {
|
||||
stopStream()
|
||||
errorMessage.value = ''
|
||||
starting.value = true
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: deviceId ? { deviceId: { exact: deviceId } } : true
|
||||
})
|
||||
if (videoRef.value) {
|
||||
videoRef.value.srcObject = stream
|
||||
await videoRef.value.play()
|
||||
}
|
||||
// 拿到权限后 label 才会有值，借机刷新一次设备列表
|
||||
await refreshDevices()
|
||||
} catch (e) {
|
||||
errorMessage.value = `无法打开摄像头：${String(e)}`
|
||||
} finally {
|
||||
starting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshDevices(): Promise<void> {
|
||||
try {
|
||||
const list = await navigator.mediaDevices.enumerateDevices()
|
||||
devices.value = list.filter((d) => d.kind === 'videoinput')
|
||||
if (!selectedDeviceId.value && devices.value.length > 0) {
|
||||
selectedDeviceId.value = devices.value[0].deviceId
|
||||
}
|
||||
} catch {
|
||||
// enumerateDevices 在少数环境下可能不可用，静默忽略，不影响已经拿到的 stream
|
||||
}
|
||||
}
|
||||
|
||||
async function switchDevice(deviceId: string): Promise<void> {
|
||||
selectedDeviceId.value = deviceId
|
||||
emit('deviceChange', deviceId)
|
||||
await startStream(deviceId)
|
||||
}
|
||||
|
||||
function capture(): void {
|
||||
const video = videoRef.value
|
||||
const canvas = canvasRef.value
|
||||
if (!video || !canvas || video.videoWidth === 0) return
|
||||
canvas.width = video.videoWidth
|
||||
canvas.height = video.videoHeight
|
||||
const ctx = canvas.getContext('2d')
|
||||
if (!ctx) return
|
||||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height)
|
||||
capturedDataUrl.value = canvas.toDataURL('image/jpeg', 0.85)
|
||||
}
|
||||
|
||||
function retake(): void {
|
||||
capturedDataUrl.value = ''
|
||||
}
|
||||
|
||||
function confirm(): void {
|
||||
if (!capturedDataUrl.value) return
|
||||
emit('captured', capturedDataUrl.value)
|
||||
close()
|
||||
}
|
||||
|
||||
function close(): void {
|
||||
emit('update:show', false)
|
||||
}
|
||||
|
||||
watch(
|
||||
() => props.show,
|
||||
async (val) => {
|
||||
if (val) {
|
||||
capturedDataUrl.value = ''
|
||||
selectedDeviceId.value = props.initialDeviceId
|
||||
await refreshDevices()
|
||||
await startStream(selectedDeviceId.value)
|
||||
} else {
|
||||
stopStream()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
onUnmounted(() => stopStream())
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<n-modal :show="show" :mask-closable="false" style="width: 720px; max-width: 92vw;" @update:show="close">
|
||||
<n-card title="拍照提问" closable @close="close">
|
||||
<div class="cam-body">
|
||||
<div v-if="errorMessage" class="cam-error">{{ errorMessage }}</div>
|
||||
|
||||
<div v-show="!capturedDataUrl" class="cam-preview-wrap">
|
||||
<video ref="videoRef" class="cam-preview" autoplay muted playsinline />
|
||||
<div v-if="starting" class="cam-loading">正在打开摄像头...</div>
|
||||
</div>
|
||||
|
||||
<img v-if="capturedDataUrl" :src="capturedDataUrl" class="cam-preview" alt="拍摄预览" />
|
||||
|
||||
<canvas ref="canvasRef" style="display:none" />
|
||||
|
||||
<div v-if="devices.length > 1 && !capturedDataUrl" class="cam-device-row">
|
||||
<label class="fl">摄像头</label>
|
||||
<select class="s-inp" :value="selectedDeviceId" @change="switchDevice(($event.target as HTMLSelectElement).value)">
|
||||
<option v-for="d in devices" :key="d.deviceId" :value="d.deviceId">
|
||||
{{ d.label || `摄像头 ${d.deviceId.slice(0, 6)}` }}
|
||||
</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<template #footer>
|
||||
<div class="cam-actions">
|
||||
<template v-if="!capturedDataUrl">
|
||||
<button class="i-btn i-btn-primary i-btn-lg" :disabled="starting || !!errorMessage" @click="capture">拍照</button>
|
||||
</template>
|
||||
<template v-else>
|
||||
<button class="i-btn i-btn-ghost" @click="retake">重拍</button>
|
||||
<button class="i-btn i-btn-primary i-btn-lg" @click="confirm">使用这张</button>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
</n-card>
|
||||
</n-modal>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.cam-body { display: flex; flex-direction: column; gap: 14px; }
|
||||
.cam-preview-wrap { position: relative; }
|
||||
.cam-preview {
|
||||
width: 100%; max-height: 60vh; object-fit: contain;
|
||||
background: #000; border-radius: var(--radius-sm);
|
||||
}
|
||||
.cam-loading {
|
||||
position: absolute; inset: 0; display: flex;
|
||||
align-items: center; justify-content: center;
|
||||
color: #fff; font-size: 15px; background: rgba(0,0,0,.35);
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
.cam-error {
|
||||
color: var(--red); background: #FBEAE8;
|
||||
border-radius: var(--radius-sm); padding: 12px 14px; font-size: 15px;
|
||||
}
|
||||
.cam-device-row { display: flex; align-items: center; gap: 12px; }
|
||||
.cam-device-row .fl { flex-shrink: 0; font-size: 15px; color: var(--ink-2); }
|
||||
.cam-actions { display: flex; justify-content: flex-end; gap: 12px; }
|
||||
</style>
|
||||
@ -6,6 +6,7 @@ import { useRagStore } from '../stores/rag'
|
||||
import { resolveImagePaths } from '../utils/markdown'
|
||||
import { AudioRecorder } from '../utils/audio'
|
||||
import { ttsPlayer } from '../utils/tts'
|
||||
import CameraCapture from './CameraCapture.vue'
|
||||
import type { Chunk } from '../types'
|
||||
|
||||
interface InferStats {
|
||||
@ -153,19 +154,13 @@ function onAnswerClick(e: MouseEvent): void {
|
||||
popupVisible.value = true
|
||||
}
|
||||
|
||||
async function sendQuery(): Promise<void> {
|
||||
const q = question.value.trim()
|
||||
if (!q) return
|
||||
if (!store.dbStatus.ready) { message.warning('向量库为空，请先入库文档'); return }
|
||||
if (store.querying) return
|
||||
|
||||
// 两种提问方式（文字检索 / 拍照）共用的准备工作：重置状态、打断正在进行的语音输入、停止旧的 TTS 播报
|
||||
function beginQuery(): void {
|
||||
store.querying = true
|
||||
store.resetAnswer()
|
||||
inferStats.value = null
|
||||
readyCitations.value = new Set()
|
||||
if (hoverTimer) { clearTimeout(hoverTimer); hoverTimer = null }
|
||||
sentQuestion.value = q
|
||||
question.value = ''
|
||||
// 立即解除 ASR 监听并清空状态，防止延迟到达的 asr:stopped 通过 syncAsrText() 恢复输入框
|
||||
cleanupAsrListeners()
|
||||
asrBaseText = ''
|
||||
@ -185,6 +180,18 @@ async function sendQuery(): Promise<void> {
|
||||
startWaitTimer()
|
||||
ttsPlayer.stop()
|
||||
syncTtsConfig()
|
||||
}
|
||||
|
||||
async function sendQuery(): Promise<void> {
|
||||
const q = question.value.trim()
|
||||
if (!q) return
|
||||
if (!store.dbStatus.ready) { message.warning('向量库为空，请先入库文档'); return }
|
||||
if (store.querying) return
|
||||
|
||||
beginQuery()
|
||||
sentQuestion.value = q
|
||||
sentImage.value = ''
|
||||
question.value = ''
|
||||
|
||||
let firstDelta = true
|
||||
|
||||
@ -228,8 +235,84 @@ async function sendQuery(): Promise<void> {
|
||||
await window.electronAPI.query({ question: q, systemPrompt: store.config?.rag.systemPrompt ?? '', topK: topK.value })
|
||||
}
|
||||
|
||||
const VISION_DEFAULT_QUESTION = '请描述这张图片的内容'
|
||||
|
||||
// 拍照提问：不查向量库，图片 + 问题直接发给多模态生成服务
|
||||
async function sendVisionQuery(): Promise<void> {
|
||||
if (!pendingImage.value) return
|
||||
if (store.querying) return
|
||||
|
||||
const q = question.value.trim() || VISION_DEFAULT_QUESTION
|
||||
const img = pendingImage.value
|
||||
|
||||
beginQuery()
|
||||
sentQuestion.value = q
|
||||
sentImage.value = img
|
||||
question.value = ''
|
||||
pendingImage.value = ''
|
||||
|
||||
let firstDelta = true
|
||||
|
||||
const onDelta = (data: unknown) => {
|
||||
if (firstDelta) { firstDelta = false; stopWaitTimer() }
|
||||
store.rawAnswer += (data as { delta: string }).delta
|
||||
scrollToBottom()
|
||||
}
|
||||
const onDone = (data: unknown) => {
|
||||
stopWaitTimer()
|
||||
store.querying = false
|
||||
queryStage.value = ''
|
||||
const stats = (data as { stats?: InferStats | null }).stats
|
||||
if (stats) inferStats.value = stats
|
||||
cleanup()
|
||||
}
|
||||
const onError = (data: unknown) => {
|
||||
stopWaitTimer()
|
||||
store.querying = false
|
||||
queryStage.value = ''
|
||||
message.error((data as { message: string }).message)
|
||||
cleanup()
|
||||
}
|
||||
|
||||
function cleanup(): void {
|
||||
window.electronAPI.off('vision-query:delta')
|
||||
window.electronAPI.off('vision-query:done')
|
||||
window.electronAPI.off('vision-query:error')
|
||||
}
|
||||
|
||||
window.electronAPI.on('vision-query:delta', onDelta)
|
||||
window.electronAPI.on('vision-query:done', onDone)
|
||||
window.electronAPI.on('vision-query:error', onError)
|
||||
|
||||
await window.electronAPI.visionQuery({ question: q, imageDataUrl: img })
|
||||
}
|
||||
|
||||
function handleSend(): void {
|
||||
if (pendingImage.value) { sendVisionQuery(); return }
|
||||
sendQuery()
|
||||
}
|
||||
|
||||
function handleEnter(e: KeyboardEvent): void {
|
||||
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); sendQuery() }
|
||||
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); handleSend() }
|
||||
}
|
||||
|
||||
// ── 拍照提问 ──
|
||||
const cameraOpen = ref(false)
|
||||
const pendingImage = ref('')
|
||||
const sentImage = ref('')
|
||||
|
||||
function onCaptured(dataUrl: string): void {
|
||||
pendingImage.value = dataUrl
|
||||
}
|
||||
|
||||
function removePendingImage(): void {
|
||||
pendingImage.value = ''
|
||||
}
|
||||
|
||||
async function onCameraDeviceChange(deviceId: string): Promise<void> {
|
||||
if (!store.config) return
|
||||
if (store.config.vision.cameraDeviceId === deviceId) return
|
||||
await store.saveConfig({ ...store.config, vision: { ...store.config.vision, cameraDeviceId: deviceId } })
|
||||
}
|
||||
|
||||
// ── 语音输入 ──
|
||||
@ -498,6 +581,7 @@ onUnmounted(() => {
|
||||
<div class="q-ic">问</div>
|
||||
<div>
|
||||
<div class="q-label">提问</div>
|
||||
<img v-if="sentImage" :src="sentImage" class="q-image" alt="提问附带的照片" />
|
||||
<div class="q-text">{{ sentQuestion }}</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -571,22 +655,38 @@ onUnmounted(() => {
|
||||
语音输出
|
||||
</button>
|
||||
</div>
|
||||
<!-- Camera button -->
|
||||
<div v-if="store.config?.vision.enabled" class="voice-btn-wrap">
|
||||
<button class="voice-btn" @click="cameraOpen = true">
|
||||
<svg width="26" height="26" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z"/>
|
||||
<circle cx="12" cy="13" r="4"/>
|
||||
</svg>
|
||||
拍照提问
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Question textarea -->
|
||||
<textarea
|
||||
v-model="question"
|
||||
class="q-input"
|
||||
:disabled="batchRunning"
|
||||
placeholder="输入问题，Enter 发送，Shift+Enter 换行…"
|
||||
@keydown="handleEnter"
|
||||
/>
|
||||
<div class="q-input-wrap">
|
||||
<div v-if="pendingImage" class="pending-image-chip">
|
||||
<img :src="pendingImage" alt="待发送的照片" />
|
||||
<button class="pending-image-remove" title="移除照片" @click="removePendingImage">×</button>
|
||||
</div>
|
||||
<textarea
|
||||
v-model="question"
|
||||
class="q-input"
|
||||
:disabled="batchRunning"
|
||||
:placeholder="pendingImage ? '针对这张照片提问，留空则默认描述图片…' : '输入问题，Enter 发送，Shift+Enter 换行…'"
|
||||
@keydown="handleEnter"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- Send -->
|
||||
<button
|
||||
class="i-btn i-btn-primary i-btn-lg send-btn"
|
||||
:disabled="store.querying || !question.trim() || batchRunning"
|
||||
@click="sendQuery"
|
||||
:disabled="store.querying || (!question.trim() && !pendingImage) || batchRunning"
|
||||
@click="handleSend"
|
||||
>
|
||||
<svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M22 2L11 13"/><path d="M22 2l-7 20-4-9-9-4 20-7z"/>
|
||||
@ -619,6 +719,14 @@ onUnmounted(() => {
|
||||
</n-scrollbar>
|
||||
</n-card>
|
||||
</n-modal>
|
||||
|
||||
<!-- Camera capture -->
|
||||
<CameraCapture
|
||||
v-model:show="cameraOpen"
|
||||
:initial-device-id="store.config?.vision.cameraDeviceId ?? ''"
|
||||
@captured="onCaptured"
|
||||
@device-change="onCameraDeviceChange"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@ -662,6 +770,11 @@ onUnmounted(() => {
|
||||
}
|
||||
.q-label { font-size: 13px; color: var(--blue-d); font-weight: 700; letter-spacing: .8px; margin-bottom: 3px; }
|
||||
.q-text { font-size: 20px; font-weight: 700; line-height: 1.4; color: var(--ink); }
|
||||
.q-image {
|
||||
max-width: 240px; max-height: 240px; object-fit: contain;
|
||||
border-radius: 8px; margin-bottom: 10px; display: block;
|
||||
border: 2px solid var(--line);
|
||||
}
|
||||
|
||||
/* Stage indicator */
|
||||
.stage-row {
|
||||
@ -750,8 +863,10 @@ onUnmounted(() => {
|
||||
display: flex; align-items: flex-end; gap: 14px;
|
||||
}
|
||||
|
||||
.q-input-wrap { flex: 1; min-width: 0; position: relative; }
|
||||
|
||||
.q-input {
|
||||
flex: 1; min-width: 0;
|
||||
width: 100%; box-sizing: border-box;
|
||||
height: 96px;
|
||||
border: 2px solid var(--line-2); border-radius: var(--radius);
|
||||
padding: 16px 20px; font-family: inherit;
|
||||
@ -762,6 +877,26 @@ onUnmounted(() => {
|
||||
.q-input:focus { outline: none; border-color: var(--blue); }
|
||||
.q-input:disabled { background: var(--panel-2); color: var(--ink-3); }
|
||||
|
||||
.pending-image-chip {
|
||||
position: absolute; left: 14px; bottom: 100%;
|
||||
margin-bottom: 8px;
|
||||
display: flex; align-items: center;
|
||||
border: 2px solid var(--line-2); border-radius: var(--radius-sm);
|
||||
background: var(--panel); padding: 4px;
|
||||
box-shadow: 0 6px 20px rgba(0,0,0,.12);
|
||||
}
|
||||
.pending-image-chip img {
|
||||
width: 64px; height: 64px; object-fit: cover;
|
||||
border-radius: 6px;
|
||||
}
|
||||
.pending-image-remove {
|
||||
position: absolute; top: -8px; right: -8px;
|
||||
width: 22px; height: 22px; border-radius: 50%;
|
||||
border: none; background: var(--red); color: #fff;
|
||||
font-size: 15px; line-height: 1; cursor: pointer;
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
}
|
||||
|
||||
.send-btn { height: 96px; min-width: 150px; }
|
||||
.batch-btn { height: 96px; min-width: 130px; }
|
||||
.i-btn-success { background: var(--green); color: #fff; box-shadow: 0 2px 0 #0a6238; }
|
||||
|
||||
@ -25,7 +25,8 @@ const form = ref<AppConfig>({
|
||||
voice: {
|
||||
asr: { enabled: false, executablePath: '', modelDir: '', numThreads: 2, vadSilenceTimeoutMs: 5000, vadEnergyThreshold: 0.005, inputGain: 10, punctuationModelDir: '' },
|
||||
tts: { enabled: false, voiceURI: '', pitch: 1, rate: 1, volume: 1 }
|
||||
}
|
||||
},
|
||||
vision: { enabled: false, cameraDeviceId: '' }
|
||||
})
|
||||
|
||||
const activeTab = ref<'index' | 'basic' | 'voice' | 'batch' | 'log'>('basic')
|
||||
@ -175,6 +176,23 @@ async function save(): Promise<void> {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── 拍照提问 ── -->
|
||||
<div class="sgroup-title">拍照提问</div>
|
||||
<div class="field">
|
||||
<div class="fl">启用拍照提问</div>
|
||||
<div class="fc">
|
||||
<label class="s-radio" :class="{ on: form.vision.enabled }" @click="form.vision.enabled = true">
|
||||
<span class="rk"></span>开启
|
||||
</label>
|
||||
<label class="s-radio" :class="{ on: !form.vision.enabled }" @click="form.vision.enabled = false">
|
||||
<span class="rk"></span>关闭
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
<p class="s-note" style="padding-left:120px;margin:-6px 0 0;">
|
||||
依赖 er_backend 当前加载的是支持视觉输入的模型（能看到 allow_image=1），否则开启后拍照提问会请求失败。
|
||||
</p>
|
||||
|
||||
</div><!-- /基本设置 -->
|
||||
|
||||
<!-- ── 语音输入输出 ── -->
|
||||
|
||||
@ -50,6 +50,10 @@ export interface AppConfig {
|
||||
volume: number
|
||||
}
|
||||
}
|
||||
vision: {
|
||||
enabled: boolean
|
||||
cameraDeviceId: string
|
||||
}
|
||||
}
|
||||
|
||||
export type DetectedCard = 'er160' | 'er25' | 'none'
|
||||
@ -113,6 +117,7 @@ declare global {
|
||||
selectDirectory: () => Promise<string | null>
|
||||
indexDocuments: (payload: { paths: string[]; mode: 'append' | 'rebuild' }) => Promise<void>
|
||||
query: (payload: { question: string; systemPrompt: string; topK: number }) => Promise<void>
|
||||
visionQuery: (payload: { question: string; imageDataUrl: string }) => Promise<void>
|
||||
getDbStatus: () => Promise<DbStatus>
|
||||
getConfig: () => Promise<AppConfig>
|
||||
saveConfig: (config: AppConfig) => Promise<void>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user