refactor: 推理请求统一走 er_backend 代理(127.0.0.1:17700)
- 嵌入/生成服务地址不再由前端按 cardType 推导端口,改为经 er_backend 的 /v1/* 代理转发,服务端口以后端配置为唯一真相来源 - config.ts: 新增 ER_BACKEND_URL 统一常量,删除端口推导逻辑 - generator/statusHandler/queryHandler/erBackendHandler 跟随改造 - fix: package-kylin-arm64-deb.sh 从 .app-version 读版本号时剥离 v 前缀, 修复无 git 环境下 dpkg-deb 报"版本号没有以数字开头"
This commit is contained in:
parent
a319caeff6
commit
836d25cad2
@ -29,7 +29,7 @@ fi
|
|||||||
# 的解析逻辑保持一致,确保三处版本号(deb 文件名、界面显示、控制文件)永远一致。
|
# 的解析逻辑保持一致,确保三处版本号(deb 文件名、界面显示、控制文件)永远一致。
|
||||||
VERSION=$(git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || true
|
VERSION=$(git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || true
|
||||||
if [ -z "$VERSION" ] && [ -f .app-version ]; then
|
if [ -z "$VERSION" ] && [ -f .app-version ]; then
|
||||||
VERSION=$(cat .app-version | tr -d '[:space:]')
|
VERSION=$(cat .app-version | tr -d '[:space:]' | sed 's/^v//')
|
||||||
fi
|
fi
|
||||||
if [ -z "$VERSION" ]; then
|
if [ -z "$VERSION" ]; then
|
||||||
VERSION=$(node -p "require('./package.json').version")
|
VERSION=$(node -p "require('./package.json').version")
|
||||||
|
|||||||
@ -2,11 +2,14 @@ import { app } from 'electron'
|
|||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { existsSync, readFileSync, writeFileSync } from 'fs'
|
import { existsSync, readFileSync, writeFileSync } from 'fs'
|
||||||
|
|
||||||
|
// 推理请求统一走 er_backend 代理(管理面端口,/v1/* 由后端转发到其实际管理的 llama-server)
|
||||||
|
export const ER_BACKEND_URL = 'http://127.0.0.1:17700'
|
||||||
|
|
||||||
export interface AppConfig {
|
export interface AppConfig {
|
||||||
embed: {
|
embed: {
|
||||||
provider: 'local' | 'llamacpp'
|
provider: 'local' | 'llamacpp'
|
||||||
localModelPath: string
|
localModelPath: string
|
||||||
baseUrl: string // 由 applyPlatformOverrides 自动推导,不对用户暴露
|
baseUrl: string // 由 applyPlatformOverrides 固定为 er_backend 代理地址,不对用户暴露
|
||||||
}
|
}
|
||||||
rag: {
|
rag: {
|
||||||
chunkSize: number
|
chunkSize: number
|
||||||
@ -14,6 +17,7 @@ export interface AppConfig {
|
|||||||
topK: number
|
topK: number
|
||||||
systemPrompt: string
|
systemPrompt: string
|
||||||
}
|
}
|
||||||
|
// 服务的启动参数/端口由 er_backend 统一管理,以下字段仅为历史配置保留,前端不再用于路由
|
||||||
llamacppServer: {
|
llamacppServer: {
|
||||||
executablePath: string
|
executablePath: string
|
||||||
modelPath: string
|
modelPath: string
|
||||||
@ -60,7 +64,7 @@ export const DEFAULT_CONFIG: AppConfig = {
|
|||||||
embed: {
|
embed: {
|
||||||
provider: 'llamacpp',
|
provider: 'llamacpp',
|
||||||
localModelPath: '',
|
localModelPath: '',
|
||||||
baseUrl: 'http://127.0.0.1:17701'
|
baseUrl: ER_BACKEND_URL
|
||||||
},
|
},
|
||||||
llamacppServer: {
|
llamacppServer: {
|
||||||
executablePath: '',
|
executablePath: '',
|
||||||
@ -206,19 +210,18 @@ export function getConfig(): AppConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 按计算卡型号覆盖 embed provider 和默认启动参数
|
// 按计算卡型号覆盖 embed provider 和默认启动参数
|
||||||
|
// 推理请求统一走 er_backend 代理(ER_BACKEND_URL),服务端口以后端配置为准,前端不再维护
|
||||||
function applyPlatformOverrides(config: AppConfig): void {
|
function applyPlatformOverrides(config: AppConfig): void {
|
||||||
const card = config.hardware.cardType
|
const card = config.hardware.cardType
|
||||||
if (card === 'er25') {
|
if (card === 'er25') {
|
||||||
// ER25:嵌入固定本地 CPU,无嵌入服务进程
|
// ER25:嵌入固定本地 CPU,无嵌入服务进程
|
||||||
config.embed.provider = 'local'
|
config.embed.provider = 'local'
|
||||||
config.llamacppServer.port = 8080
|
|
||||||
if (!config.llamacppServer.extraArgs.trim()) {
|
if (!config.llamacppServer.extraArgs.trim()) {
|
||||||
config.llamacppServer.extraArgs = '--port 8080 --ctk bf16 --ctv bf16 --no-warmup -ub 128'
|
config.llamacppServer.extraArgs = '--port 8080 --ctk bf16 --ctv bf16 --no-warmup -ub 128'
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// ER160:嵌入默认 llamacpp NPU(local 保持不变)
|
// ER160:嵌入默认 llamacpp NPU(local 保持不变)
|
||||||
if (config.embed.provider !== 'local') config.embed.provider = 'llamacpp'
|
if (config.embed.provider !== 'local') config.embed.provider = 'llamacpp'
|
||||||
config.llamacppServer.port = 8081
|
|
||||||
if (!config.llamacppServer.extraArgs.trim()) {
|
if (!config.llamacppServer.extraArgs.trim()) {
|
||||||
config.llamacppServer.extraArgs = '--port 8081 --presence_penalty 1.5'
|
config.llamacppServer.extraArgs = '--port 8081 --presence_penalty 1.5'
|
||||||
}
|
}
|
||||||
@ -226,8 +229,8 @@ function applyPlatformOverrides(config: AppConfig): void {
|
|||||||
config.llamacppEmbed.extraArgs = DEFAULT_CONFIG.llamacppEmbed.extraArgs
|
config.llamacppEmbed.extraArgs = DEFAULT_CONFIG.llamacppEmbed.extraArgs
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// 嵌入服务地址跟随端口
|
// 嵌入/生成服务地址固定为 er_backend 代理
|
||||||
config.embed.baseUrl = `http://127.0.0.1:${config.llamacppEmbed.port}`
|
config.embed.baseUrl = ER_BACKEND_URL
|
||||||
}
|
}
|
||||||
|
|
||||||
export function saveConfig(config: AppConfig): void {
|
export function saveConfig(config: AppConfig): void {
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
import { ipcMain } from 'electron'
|
import { ipcMain } from 'electron'
|
||||||
import { execFile } from 'child_process'
|
import { execFile } from 'child_process'
|
||||||
|
import { ER_BACKEND_URL } from '../config'
|
||||||
const ER_BACKEND_URL = 'http://localhost:17700'
|
|
||||||
|
|
||||||
async function isRunning(): Promise<boolean> {
|
async function isRunning(): Promise<boolean> {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@ -16,7 +16,7 @@ export function registerQueryHandlers(getWin: () => BrowserWindow | null): void
|
|||||||
console.log(`\n${'='.repeat(60)}`)
|
console.log(`\n${'='.repeat(60)}`)
|
||||||
console.log('[query] 问题:', payload.question)
|
console.log('[query] 问题:', payload.question)
|
||||||
console.log('[query] topK:', payload.topK)
|
console.log('[query] topK:', payload.topK)
|
||||||
console.log('[query] 向量化方式:', config.embed.provider, '生成服务端口:', config.llamacppServer.port)
|
console.log('[query] 向量化方式:', config.embed.provider, '生成/嵌入服务:er_backend 代理(127.0.0.1:17700)')
|
||||||
|
|
||||||
const send = (channel: string, data: unknown): void => {
|
const send = (channel: string, data: unknown): void => {
|
||||||
getWin()?.webContents.send(channel, data)
|
getWin()?.webContents.send(channel, data)
|
||||||
@ -50,7 +50,6 @@ export function registerQueryHandlers(getWin: () => BrowserWindow | null): void
|
|||||||
payload.question,
|
payload.question,
|
||||||
chunks,
|
chunks,
|
||||||
payload.systemPrompt,
|
payload.systemPrompt,
|
||||||
config.llamacppServer,
|
|
||||||
{
|
{
|
||||||
onDelta: (delta) => send('query:delta', { delta }),
|
onDelta: (delta) => send('query:delta', { delta }),
|
||||||
onDone: (stats) => send('query:done', { stats: stats ?? null }),
|
onDone: (stats) => send('query:done', { stats: stats ?? null }),
|
||||||
|
|||||||
@ -2,7 +2,7 @@ import { ipcMain, dialog } from 'electron'
|
|||||||
import type { BrowserWindow } from 'electron'
|
import type { BrowserWindow } from 'electron'
|
||||||
import { existsSync, readFileSync, writeFileSync } from 'fs'
|
import { existsSync, readFileSync, writeFileSync } from 'fs'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { getConfig, saveConfig, getDbPath, AppConfig } from '../config'
|
import { getConfig, saveConfig, getDbPath, AppConfig, ER_BACKEND_URL } from '../config'
|
||||||
import { getDocCount } from '../db/lancedb'
|
import { getDocCount } from '../db/lancedb'
|
||||||
import { embedOneLocal, resolveLocalModelPath } from '../rag/localEmbedder'
|
import { embedOneLocal, resolveLocalModelPath } from '../rag/localEmbedder'
|
||||||
import { syncNpuPolling } from './npuHandler'
|
import { syncNpuPolling } from './npuHandler'
|
||||||
@ -100,11 +100,12 @@ export function registerStatusHandlers(getWin: () => BrowserWindow | null): void
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
ipcMain.handle('chat-test', async (_event, llamacppConfig: AppConfig['llamacppServer']): Promise<TestResult> => {
|
// 生成服务连接测试(经 er_backend 代理转发到 llama-server)
|
||||||
|
ipcMain.handle('chat-test', async (_event, _llamacppConfig: AppConfig['llamacppServer']): Promise<TestResult> => {
|
||||||
const start = Date.now()
|
const start = Date.now()
|
||||||
const elapsed = (): string => `${Date.now() - start}ms`
|
const elapsed = (): string => `${Date.now() - start}ms`
|
||||||
try {
|
try {
|
||||||
const res = await fetch(`http://127.0.0.1:${llamacppConfig.port}/v1/chat/completions`, {
|
const res = await fetch(`${ER_BACKEND_URL}/v1/chat/completions`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
import type { SearchResult } from '../db/lancedb'
|
import type { SearchResult } from '../db/lancedb'
|
||||||
import type { AppConfig } from '../config'
|
import { ER_BACKEND_URL } from '../config'
|
||||||
|
|
||||||
export interface GeneratorStats {
|
export interface GeneratorStats {
|
||||||
tokensPerSecond: number
|
tokensPerSecond: number
|
||||||
@ -14,12 +14,11 @@ export interface GeneratorCallbacks {
|
|||||||
onError: (message: string) => void
|
onError: (message: string) => void
|
||||||
}
|
}
|
||||||
|
|
||||||
// 生成服务固定使用 llamacpp(ER160/ER25 均通过 er_backend 管理的 llama-server)
|
// 生成服务经 er_backend 代理访问其管理的 llama-server(OpenAI 兼容接口,SSE 流式)
|
||||||
export async function streamGenerate(
|
export async function streamGenerate(
|
||||||
question: string,
|
question: string,
|
||||||
chunks: SearchResult[],
|
chunks: SearchResult[],
|
||||||
systemPrompt: string,
|
systemPrompt: string,
|
||||||
llamacppConfig: AppConfig['llamacppServer'],
|
|
||||||
callbacks: GeneratorCallbacks
|
callbacks: GeneratorCallbacks
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const context = chunks
|
const context = chunks
|
||||||
@ -33,22 +32,21 @@ ${context}
|
|||||||
请回答以下问题:${question}`
|
请回答以下问题:${question}`
|
||||||
|
|
||||||
console.log('\n[generator] ---- 发送给 LLM 的完整消息 ----')
|
console.log('\n[generator] ---- 发送给 LLM 的完整消息 ----')
|
||||||
console.log(`[generator] 服务地址:http://127.0.0.1:${llamacppConfig.port}`)
|
console.log(`[generator] 服务地址:${ER_BACKEND_URL}(er_backend 代理)`)
|
||||||
console.log('[generator] system prompt:\n', systemPrompt)
|
console.log('[generator] system prompt:\n', systemPrompt)
|
||||||
console.log('[generator] user message:\n', userMessage)
|
console.log('[generator] user message:\n', userMessage)
|
||||||
console.log('[generator] ---- 消息结束 ----\n')
|
console.log('[generator] ---- 消息结束 ----\n')
|
||||||
|
|
||||||
await streamGenerateLlamaCpp(userMessage, systemPrompt, llamacppConfig, callbacks)
|
await streamGenerateLlamaCpp(userMessage, systemPrompt, callbacks)
|
||||||
}
|
}
|
||||||
|
|
||||||
// llama.cpp server OpenAI 兼容接口(/v1/chat/completions),SSE 格式
|
// llama.cpp server OpenAI 兼容接口(/v1/chat/completions),SSE 格式
|
||||||
async function streamGenerateLlamaCpp(
|
async function streamGenerateLlamaCpp(
|
||||||
userMessage: string,
|
userMessage: string,
|
||||||
systemPrompt: string,
|
systemPrompt: string,
|
||||||
llamacppConfig: AppConfig['llamacppServer'],
|
|
||||||
callbacks: GeneratorCallbacks
|
callbacks: GeneratorCallbacks
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const baseUrl = `http://127.0.0.1:${llamacppConfig.port}`
|
const baseUrl = ER_BACKEND_URL
|
||||||
let fullResponse = ''
|
let fullResponse = ''
|
||||||
let thinkOpen = false
|
let thinkOpen = false
|
||||||
const requestSentTime = Date.now()
|
const requestSentTime = Date.now()
|
||||||
@ -71,12 +69,12 @@ async function streamGenerateLlamaCpp(
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
callbacks.onError(`无法连接 llama.cpp server(${baseUrl}):${String(e)}`)
|
callbacks.onError(`无法连接生成服务(${baseUrl},er_backend 代理):${String(e)}`)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
callbacks.onError(`llama.cpp server 请求失败:${response.status}`)
|
callbacks.onError(`生成服务请求失败:${response.status}`)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user