Compare commits

..

2 Commits

Author SHA1 Message Date
51466d8ca7 Merge pull request 'feat/1.0.1' (#2) from feat/1.0.1 into main
Reviewed-on: #2
2025-06-30 09:52:04 +08:00
7e37d9ead0 Merge pull request 'feat/1.0.1' (#1) from feat/1.0.1 into main
Reviewed-on: #1
2025-06-28 20:08:10 +08:00
29 changed files with 136 additions and 916 deletions

2
.gitignore vendored
View File

@@ -114,5 +114,3 @@ node_modules/
*.njsproj
*.sln
*.sw?
*.mp3

View File

@@ -1,10 +0,0 @@
from fastapi import APIRouter
from app.constants.tts import SPEAKER_DATA
from app.schemas import SpeakerResponse
router = APIRouter()
@router.get("/list", response_model=SpeakerResponse)
async def get_model_vendors():
return SpeakerResponse(data=SPEAKER_DATA)

View File

@@ -1,17 +1,15 @@
# tts.py
import uuid
import websockets
import time
import fastrand
import json
import asyncio
import os
import aiofiles
from datetime import datetime
from typing import Dict, Any, Optional as OptionalType
from app.constants.tts import APP_ID, TOKEN, SPEAKER
# 协议常量
# 协议常量保持不变...
PROTOCOL_VERSION = 0b0001
DEFAULT_HEADER_SIZE = 0b0001
FULL_CLIENT_REQUEST = 0b0001
@@ -36,26 +34,8 @@ EVENT_TaskRequest = 200
EVENT_TTSSentenceEnd = 351
EVENT_TTSResponse = 352
# 音频文件保存目录
TEMP_AUDIO_DIR = "./temp_audio"
# 确保音频目录存在
async def ensure_audio_dir():
"""异步创建音频目录"""
if not os.path.exists(TEMP_AUDIO_DIR):
os.makedirs(TEMP_AUDIO_DIR, exist_ok=True)
# 生成时间戳文件名
def generate_audio_filename() -> str:
"""生成基于时间戳的音频文件名"""
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3] # 精确到毫秒
return f"{timestamp}.mp3"
# ... 保留所有原有的类定义和工具函数 ...
# 所有类定义保持不变...
class Header:
def __init__(self,
protocol_version=PROTOCOL_VERSION,
@@ -113,7 +93,7 @@ class Response:
self.payload_json = None
# 工具函数
# 工具函数保持不变...
def gen_log_id():
"""生成logID"""
ts = int(time.time() * 1000)
@@ -211,7 +191,7 @@ async def send_event(ws, header, optional=None, payload=None):
await ws.send(full_client_request)
# TTS状态管理类添加消息ID和任务追踪
# 修改:TTS状态管理类添加消息ID和任务追踪
class TTSState:
def __init__(self, message_id: str):
self.message_id = message_id
@@ -219,8 +199,6 @@ class TTSState:
self.session_id: OptionalType[str] = None
self.task: OptionalType[asyncio.Task] = None # 用于追踪异步任务
self.is_processing = False
self.audio_data = bytearray() # 用于收集音频数据
self.audio_filename = None # 保存的文件名
# 全局状态管理
@@ -327,64 +305,54 @@ async def create_tts_connection() -> websockets.WebSocketServerProtocol:
return volc_ws
# 异步保存音频文件
async def save_audio_file(audio_data: bytes, filename: str) -> str:
"""异步保存音频文件"""
await ensure_audio_dir()
file_path = os.path.join(TEMP_AUDIO_DIR, filename)
async with aiofiles.open(file_path, 'wb') as f:
await f.write(audio_data)
return file_path
# 处理单个TTS任务
async def process_tts_task(websocket, message_id: str, text: str, speaker: str = None):
async def process_tts_task(websocket, message_id: str, text: str):
"""处理单个TTS任务独立协程"""
tts_state = None
# 使用传入的speaker如果没有则使用默认的
selected_speaker = speaker if speaker else SPEAKER
try:
print(f"开始处理TTS任务 [{message_id}]: {text}, 使用说话人: {selected_speaker}")
print(f"开始处理TTS任务 [{message_id}]: {text}")
# 获取TTS状态
tts_state = tts_manager.get_tts_state(websocket, message_id)
if not tts_state:
raise Exception(f"找不到TTS状态: {message_id}")
tts_state.is_processing = True
# 生成音频文件名
tts_state.audio_filename = generate_audio_filename()
# 创建独立的TTS连接
tts_state.volc_ws = await create_tts_connection()
# 创建会话
tts_state.session_id = uuid.uuid4().__str__().replace('-', '')
tts_manager.register_session(tts_state.session_id, message_id)
print(f"创建TTS会话 [{message_id}]: {tts_state.session_id}")
header = Header(message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON).as_bytes()
optional = Optional(event=EVENT_StartSession, sessionId=tts_state.session_id).as_bytes()
# 使用选择的speaker
payload = get_payload_bytes(event=EVENT_StartSession, speaker=selected_speaker)
payload = get_payload_bytes(event=EVENT_StartSession, speaker=SPEAKER)
await send_event(tts_state.volc_ws, header, optional, payload)
raw_data = await tts_state.volc_ws.recv()
res = parser_response(raw_data)
if res.optional.event != EVENT_SessionStarted:
raise Exception("TTS会话启动失败")
print(f"TTS会话创建成功 [{message_id}]: {tts_state.session_id}")
# 发送文本到TTS服务
print(f"发送文本到TTS服务 [{message_id}]...")
header = Header(message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON).as_bytes()
optional = Optional(event=EVENT_TaskRequest, sessionId=tts_state.session_id).as_bytes()
# 使用选择的speaker
payload = get_payload_bytes(event=EVENT_TaskRequest, text=text, speaker=selected_speaker)
payload = get_payload_bytes(event=EVENT_TaskRequest, text=text, speaker=SPEAKER)
await send_event(tts_state.volc_ws, header, optional, payload)
# 接收TTS响应并发送到前端
print(f"开始接收TTS响应 [{message_id}]...")
audio_count = 0
try:
while True:
raw_data = await asyncio.wait_for(
@@ -392,48 +360,39 @@ async def process_tts_task(websocket, message_id: str, text: str, speaker: str =
timeout=30
)
res = parser_response(raw_data)
print(f"收到TTS事件 [{message_id}]: {res.optional.event}")
if res.optional.event == EVENT_TTSSentenceEnd:
print(f"句子结束事件 [{message_id}] - 直接完成")
break
elif res.optional.event == EVENT_SessionFinished:
print(f"收到会话结束事件 [{message_id}]")
break
elif res.optional.event == EVENT_TTSResponse:
audio_count += 1
print(f"收到音频数据 [{message_id}] #{audio_count},大小: {len(res.payload)}")
# 收集音频数据
tts_state.audio_data.extend(res.payload)
# 发送音频数据到前端
print(f"发送音频数据 [{message_id}] #{audio_count},大小: {len(res.payload)}")
# 发送音频数据包含消息ID
await websocket.send_json({
"id": audio_count,
"type": "tts_audio_data",
"messageId": message_id,
"audioData": res.payload.hex() # 转为hex字符串
})
else:
print(f"未知TTS事件 [{message_id}]: {res.optional.event}")
except asyncio.TimeoutError:
print(f"TTS响应超时 [{message_id}],强制结束")
# 异步保存音频文件
if tts_state.audio_data:
file_path = await save_audio_file(
bytes(tts_state.audio_data),
tts_state.audio_filename
)
print(f"音频文件已保存 [{message_id}]: {file_path}")
# 发送完成消息包含文件路径和使用的speaker
# 发送完成消息
await websocket.send_json({
"type": "tts_audio_complete",
"messageId": message_id,
"audioFile": tts_state.audio_filename,
"audioPath": os.path.join(TEMP_AUDIO_DIR, tts_state.audio_filename) if tts_state.audio_data else None,
"speaker": selected_speaker
"messageId": message_id
})
print(f"TTS处理完成 [{message_id}],共发送 {audio_count} 个音频包,使用说话人: {selected_speaker}")
print(f"TTS处理完成 [{message_id}],共发送 {audio_count} 个音频包")
except asyncio.CancelledError:
print(f"TTS任务被取消 [{message_id}]")
await websocket.send_json({
@@ -464,14 +423,14 @@ async def process_tts_task(websocket, message_id: str, text: str, speaker: str =
# 启动TTS文本转换
async def handle_tts_text(websocket, message_id: str, text: str, speaker: str = None):
async def handle_tts_text(websocket, message_id: str, text: str):
"""启动TTS文本转换"""
# 创建新的TTS状态
print(speaker)
tts_state = tts_manager.add_tts_state(websocket, message_id)
# 启动异步任务传入speaker参数
# 启动异步任务
tts_state.task = asyncio.create_task(
process_tts_task(websocket, message_id, text, speaker)
process_tts_task(websocket, message_id, text)
)

View File

@@ -1,105 +0,0 @@
import json
import aiohttp
import asyncio
from fastapi.encoders import jsonable_encoder
from starlette.websockets import WebSocket
from . import tts
from app.constants.model_data import tip_message, base_url, headers
async def process_voice_conversation(websocket: WebSocket, asr_text: str, message_id: str, speaker: str):
try:
print(f"开始处理语音对话 [{message_id}]: {asr_text}")
# 1. 发送ASR识别结果到前端
await websocket.send_json({
"type": "asr_result",
"messageId": message_id,
"result": asr_text
})
# 2. 构建LLM请求
messages = [
tip_message,
{"role": "user", "content": asr_text}
]
payload = {
"model": "gpt-4o",
"messages": messages,
"stream": True
}
print(f"发送LLM请求 [{message_id}]: {json.dumps(payload, ensure_ascii=False)}")
# 3. 流式处理LLM响应
full_response = ""
llm_completed = False
async with aiohttp.ClientSession() as session:
async with session.post(
base_url,
headers=headers,
json=jsonable_encoder(payload)
) as resp:
if resp.status != 200:
error_text = await resp.text()
raise Exception(f"LLM API请求失败: {resp.status} - {error_text}")
# 读取流式响应
async for line in resp.content:
if line:
line = line.decode('utf-8').strip()
if line.startswith('data: '):
data = line[6:].strip()
if data == '[DONE]':
llm_completed = True
print(f"LLM响应完成 [{message_id}]")
break
try:
result = json.loads(data)
# 提取内容
choices = result.get("choices", [])
if not choices:
# 跳过空choices数据包
continue
delta = choices[0].get("delta", {})
content = delta.get("content")
if content:
full_response += content
except json.JSONDecodeError as e:
print(f"JSON解析错误 [{message_id}]: {e}, 数据: {data}")
continue
except Exception as e:
print(f"处理数据包异常 [{message_id}]: {e}, 数据: {data}")
continue
# 4. LLM生成完成后启动完整的TTS处理
if llm_completed and full_response:
print(f"LLM生成完成 [{message_id}], 总内容长度: {len(full_response)}")
print(f"完整内容: {full_response}")
# 发送完成消息
await websocket.send_json({
"type": "llm_complete_response",
"messageId": message_id,
"content": full_response
})
# 启动TTS处理完整内容
print(f"启动完整TTS处理 [{message_id}]: {full_response}")
await tts.handle_tts_text(websocket, message_id, full_response, speaker)
except Exception as e:
print(f"语音对话处理异常 [{message_id}]: {e}")
import traceback
traceback.print_exc()
await websocket.send_json({
"type": "voice_conversation_error",
"messageId": message_id,
"message": f"处理失败: {str(e)}"
})

View File

@@ -1,14 +1,12 @@
# websocket_service.py
import uuid
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from typing import Set
from aip import AipSpeech
from app.constants.asr import APP_ID, API_KEY, SECRET_KEY
import json
# 导入修改后的TTS模块
from . import tts
from .voice_conversation import process_voice_conversation
router = APIRouter()
active_connections: Set[WebSocket] = set()
@@ -61,20 +59,13 @@ async def websocket_online_count(websocket: WebSocket):
elif msg_type == "asr_end":
asr_text = await asr_buffer(temp_buffer)
# 从data中获取messageId如果不存在则生成一个新的ID
message_id = data.get("messageId", "voice_" + str(uuid.uuid4()))
if data.get("voiceConversation"):
speaker = data.get("speaker")
await process_voice_conversation(websocket, asr_text, message_id, speaker)
else:
await websocket.send_json({"type": "asr_result", "result": asr_text})
temp_buffer = bytes()
# TTS处理
# 修改:TTS处理支持消息ID
elif msg_type == "tts_text":
message_id = data.get("messageId")
text = data.get("text", "")
speaker = data.get("speaker")
if not message_id:
await websocket.send_json({
@@ -85,7 +76,7 @@ async def websocket_online_count(websocket: WebSocket):
print(f"收到TTS文本请求 [{message_id}]: {text}")
try:
await tts.handle_tts_text(websocket, message_id, text, speaker)
await tts.handle_tts_text(websocket, message_id, text)
except Exception as e:
print(f"TTS文本处理异常 [{message_id}]: {e}")
await websocket.send_json({

View File

@@ -5,100 +5,3 @@
APP_ID = '2138450044'
TOKEN = 'V04_QumeQZhJrQ_In1Z0VBQm7n0ttMNO'
SPEAKER = 'zh_male_beijingxiaoye_moon_bigtts'
SPEAKER_DATA = [
{
"category": "趣味口音",
"speakers": [
{
"speaker_id": "zh_male_jingqiangkanye_moon_bigtts",
"speaker_name": "京腔侃爷/Harmony",
"language": "中文-北京口音、英文",
"platforms": ["豆包", "Cici", "web demo"]
},
{
"speaker_id": "zh_female_wanwanxiaohe_moon_bigtts",
"speaker_name": "湾湾小何",
"language": "中文-台湾口音",
"platforms": ["豆包", "Cici"]
},
{
"speaker_id": "zh_female_wanqudashu_moon_bigtts",
"speaker_name": "湾区大叔",
"language": "中文-广东口音",
"platforms": ["豆包", "Cici"]
},
{
"speaker_id": "zh_female_daimengchuanmei_moon_bigtts",
"speaker_name": "呆萌川妹",
"language": "中文-四川口音",
"platforms": ["豆包", "Cici"]
},
{
"speaker_id": "zh_male_guozhoudege_moon_bigtts",
"speaker_name": "广州德哥",
"language": "中文-广东口音",
"platforms": ["豆包", "Cici"]
},
{
"speaker_id": "zh_male_beijingxiaoye_moon_bigtts",
"speaker_name": "北京小爷",
"language": "中文-北京口音",
"platforms": ["豆包"]
},
{
"speaker_id": "zh_male_haoyuxiaoge_moon_bigtts",
"speaker_name": "浩宇小哥",
"language": "中文-青岛口音",
"platforms": ["豆包"]
},
{
"speaker_id": "zh_male_guangxiyuanzhou_moon_bigtts",
"speaker_name": "广西远舟",
"language": "中文-广西口音",
"platforms": ["豆包"]
},
{
"speaker_id": "zh_female_meituojieer_moon_bigtts",
"speaker_name": "妹坨洁儿",
"language": "中文-长沙口音",
"platforms": ["豆包", "剪映"]
},
{
"speaker_id": "zh_male_yuzhouzixuan_moon_bigtts",
"speaker_name": "豫州子轩",
"language": "中文-河南口音",
"platforms": ["豆包"]
}
]
},
{
"category": "角色扮演",
"speakers": [
{
"speaker_id": "zh_male_naiqimengwa_mars_bigtts",
"speaker_name": "奶气萌娃",
"language": "中文",
"platforms": ["剪映", "豆包"]
},
{
"speaker_id": "zh_female_popo_mars_bigtts",
"speaker_name": "婆婆",
"language": "中文",
"platforms": ["剪映C端", "抖音", "豆包"]
},
{
"speaker_id": "zh_female_gaolengyujie_moon_bigtts",
"speaker_name": "高冷御姐",
"language": "中文",
"platforms": ["豆包", "Cici"]
},
{
"speaker_id": "zh_male_aojiaobazong_moon_bigtts",
"speaker_name": "傲娇霸总",
"language": "中文",
"platforms": ["豆包"]
}
]
}
]

View File

@@ -1,5 +1,5 @@
from fastapi import FastAPI
from app.api.v1.endpoints import chat, model, websocket_service,speaker
from app.api.v1.endpoints import chat, model, websocket_service
app = FastAPI()
@@ -9,7 +9,6 @@ app.include_router(websocket_service.router, prefix="", tags=["websocket_service
app.include_router(chat.router, prefix="/v1/chat", tags=["chat"])
# 获取模型列表服务
app.include_router(model.router, prefix="/v1/model", tags=["model_list"])
app.include_router(speaker.router, prefix="/v1/speaker", tags=["speaker_list"])
if __name__ == "__main__":
import uvicorn

View File

@@ -5,5 +5,4 @@ from .chat import (
ModelInfo,
VendorModelList,
VendorModelResponse,
SpeakerResponse
)

View File

@@ -33,20 +33,3 @@ class VendorModelList(BaseModel):
class VendorModelResponse(BaseModel):
data: List[VendorModelList]
# Speaker相关模型
class Speaker(BaseModel):
speaker_id: str
speaker_name: str
language: str
platforms: List[str]
class CategorySpeakers(BaseModel):
category: str
speakers: List[Speaker]
class SpeakerResponse(BaseModel):
data: List[CategorySpeakers]

Binary file not shown.

3
web/components.d.ts vendored
View File

@@ -9,8 +9,6 @@ declare module 'vue' {
export interface GlobalComponents {
Avatar: typeof import('./src/components/avatar.vue')['default']
Markdown: typeof import('./src/components/markdown.vue')['default']
Message_tools: typeof import('./src/components/MessageTools.vue')['default']
MessageTools: typeof import('./src/components/MessageTools.vue')['default']
NButton: typeof import('naive-ui')['NButton']
NCollapse: typeof import('naive-ui')['NCollapse']
NCollapseItem: typeof import('naive-ui')['NCollapseItem']
@@ -23,7 +21,6 @@ declare module 'vue' {
NPopover: typeof import('naive-ui')['NPopover']
NScrollbar: typeof import('naive-ui')['NScrollbar']
NSelect: typeof import('naive-ui')['NSelect']
NSpin: typeof import('naive-ui')['NSpin']
NTag: typeof import('naive-ui')['NTag']
RouterLink: typeof import('vue-router')['RouterLink']
RouterView: typeof import('vue-router')['RouterView']

View File

@@ -32,7 +32,6 @@ export default antfu({
"ts/no-unsafe-function-type": "off",
"no-console": "off",
"unused-imports/no-unused-vars": "warn",
"ts/no-use-before-define": "off",
"vue/operator-linebreak": "off",
"ts/no-use-before-define": "off"
}
});

View File

@@ -1,5 +1,4 @@
export { default as ChevronLeftIcon } from "./svg/heroicons/ChevronLeftIcon.svg?component";
export { default as DocumentDuplicateIcon } from "./svg/heroicons/DocumentDuplicateIcon.svg?component";
export { default as ExclamationTriangleIcon } from "./svg/heroicons/ExclamationTriangleIcon.svg?component";
export { default as microphone } from "./svg/heroicons/MicrophoneIcon.svg?component";
export { default as PaperAirplaneIcon } from "./svg/heroicons/PaperAirplaneIcon.svg?component";

View File

@@ -1,3 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="size-6">
<path stroke-linecap="round" stroke-linejoin="round" d="M15.75 17.25v3.375c0 .621-.504 1.125-1.125 1.125h-9.75a1.125 1.125 0 0 1-1.125-1.125V7.875c0-.621.504-1.125 1.125-1.125H6.75a9.06 9.06 0 0 1 1.5.124m7.5 10.376h3.375c.621 0 1.125-.504 1.125-1.125V11.25c0-4.46-3.243-8.161-7.5-8.876a9.06 9.06 0 0 0-1.5-.124H9.375c-.621 0-1.125.504-1.125 1.125v3.5m7.5 10.375H9.375a1.125 1.125 0 0 1-1.125-1.125v-9.25m12 6.625v-1.875a3.375 3.375 0 0 0-3.375-3.375h-1.5a1.125 1.125 0 0 1-1.125-1.125v-1.5a3.375 3.375 0 0 0-3.375-3.375H9.75" />
</svg>

Before

Width:  |  Height:  |  Size: 668 B

View File

@@ -1,26 +0,0 @@
<script setup lang="ts">
import type { Message } from "@/interfaces";
import { DocumentDuplicateIcon } from "@/assets/Icons";
import { copy } from "@/utils";
const { msg } = defineProps<{
msg: Message;
}>();
</script>
<template>
<div class="flex items-center gap-2 justify-end mt-2">
<div v-if="msg.role !== 'user'">
<tts :text="msg.content || ''" :message-id="msg.id!" />
</div>
<NPopover trigger="hover">
<template #trigger>
<NButton quaternary circle @click="copy(msg.content || '')">
<DocumentDuplicateIcon class="!w-4 !h-4" />
</NButton>
</template>
<span>复制内容</span>
</NPopover>
</div>
</template>

View File

@@ -34,15 +34,13 @@ const handleClick = () => {
ttsStore.convertText(text, messageId);
}
};
// 文本改变清理之前的音频
// 当文本改变时清理之前的音频
watch(
() => text,
() => {
ttsStore.clearAudio(messageId);
}
);
onUnmounted(() => {
ttsStore.clearAudio(messageId);
});

View File

@@ -12,7 +12,6 @@ export interface Message {
role?: string;
usage?: UsageInfo;
id?: string;
type?: "chat" | "voice";
[property: string]: any;
}
@@ -32,97 +31,3 @@ export interface UsageInfo {
completion_tokens: number;
total_tokens: number;
}
/**
* Speaker 语音合成器基本信息
*/
export interface Speaker {
/** speaker唯一标识ID */
speaker_id: string;
/** speaker显示名称 */
speaker_name: string;
/** 支持的语言/口音 */
language: string;
/** 支持的平台列表 */
platforms: string[];
}
/**
* Speaker分类信息
*/
export interface CategorySpeakers {
/** 分类名称 */
category: string;
/** 该分类下的speaker列表 */
speakers: Speaker[];
}
/**
* Speaker分类枚举
*/
export enum SpeakerCategory {
/** 趣味口音 */
ACCENT = "趣味口音",
/** 角色扮演 */
ROLE_PLAY = "角色扮演"
}
/**
* 常用平台枚举
*/
export enum SpeakerPlatform {
DOUYIN = "抖音",
DOUBAO = "豆包",
CICI = "Cici",
JIANYING = "剪映",
JIANYING_C = "剪映C端",
WEB_DEMO = "web demo",
STORY_AI = "StoryAi",
MAOXIANG = "猫箱"
}
/**
* Speaker选择器组件Props
*/
export interface SpeakerSelectorProps {
/** 当前选中的speaker */
selectedSpeaker?: Speaker;
/** speaker选择回调 */
onSpeakerChange: (speaker: Speaker) => void;
/** 是否禁用 */
disabled?: boolean;
/** 过滤特定分类 */
filterCategories?: SpeakerCategory[];
/** 过滤特定平台 */
filterPlatforms?: SpeakerPlatform[];
}
/**
* 语音合成参数
*/
export interface VoiceSynthesisParams {
/** 使用的speaker */
speaker: Speaker;
/** 要合成的文本 */
text: string;
/** 语速 (0.5-2.0) */
speed?: number;
/** 音调 (0.5-2.0) */
pitch?: number;
/** 音量 (0.0-1.0) */
volume?: number;
}
/**
* 语音合成响应
*/
export interface VoiceSynthesisResponse {
/** 音频文件URL */
audio_url: string;
/** 音频时长(秒) */
duration: number;
/** 合成状态 */
status: "success" | "error";
/** 错误信息 */
error_message?: string;
}

View File

@@ -45,20 +45,7 @@ const { hiddenLeftSidebar, simpleMode } = storeToRefs(layoutStore);
"
to="/"
>
对话
</router-link>
<router-link
class="w-full h-[52px] px-8 flex items-center cursor-pointer"
:class="
$route.path === '/voice'
? [
'bg-[rgba(37,99,235,0.04)] text-[#0094c5] border-r-2 border-[#0094c5]'
]
: []
"
to="/voice"
>
语音聊天
聊天
</router-link>
<div class="w-full h-full flex flex-col items-center text-[#0094c5]">

View File

@@ -2,8 +2,7 @@ import { createRouter, createWebHistory } from "vue-router";
import BasicLayout from "@/layouts/BasicLayout.vue";
import { resetDescription, setTitle } from "@/utils";
import ChatLLM from "@/views/ChatLLMView.vue";
import VoiceView from "@/views/VoiceView.vue";
import community from "@/views/CommunityView.vue";
const router = createRouter({
history: createWebHistory(import.meta.env.BASE_URL),
@@ -14,19 +13,11 @@ const router = createRouter({
children: [
{
path: "",
name: "ChatLLM",
component: ChatLLM,
name: "community",
component: community,
meta: {
title: "对话"
}
},
{
path: "/voice",
name: "Voice",
component: VoiceView,
meta: {
title: "语音对话"
}
}
]
}

View File

@@ -137,9 +137,4 @@ export class ChatService {
public static GetModelList(config?: AxiosRequestConfig<any>) {
return BaseClientService.get(`${this.basePath}/model/list`, config);
}
// 获取音色列表
public static GetSpeakerList(config?: AxiosRequestConfig<any>) {
return BaseClientService.get(`${this.basePath}/speaker/list`, config);
}
}

View File

@@ -6,7 +6,6 @@ export const useWebSocketStore = defineStore("websocket", () => {
const connected = ref(false);
const chatStore = useChatStore();
const ttsStore = useTtsStore();
const router = useRouter();
const { onlineCount } = storeToRefs(chatStore);
@@ -15,11 +14,13 @@ export const useWebSocketStore = defineStore("websocket", () => {
if (e.data instanceof ArrayBuffer) {
// 处理二进制音频数据(兜底处理,新版本应该不会用到)
console.log("收到二进制音频数据,大小:", e.data.byteLength);
console.warn("收到旧格式的二进制数据无法确定messageId");
// 可以选择忽略或者作为兜底处理
} else if (e.data instanceof Blob) {
// 如果是Blob转换为ArrayBuffer兜底处理
e.data.arrayBuffer().then((buffer: ArrayBuffer) => {
console.log("收到Blob音频数据大小:", buffer.byteLength);
console.warn("收到旧格式的Blob数据无法确定messageId");
});
} else if (typeof e.data === "string") {
// 处理文本JSON消息
@@ -30,14 +31,7 @@ export const useWebSocketStore = defineStore("websocket", () => {
onlineCount.value = data.online_count;
break;
case "asr_result":
if (router.currentRoute.value.path === "/") {
chatStore.addMessageToHistory(data.result);
} else if (router.currentRoute.value.path === "/voice") {
// 在语音页面的处理
chatStore.addMessageToHistory(data.result, "user", "voice");
} else {
console.warn(data);
}
break;
// 新的TTS消息格式处理
@@ -82,6 +76,7 @@ export const useWebSocketStore = defineStore("websocket", () => {
ttsStore.finishConversion(data.messageId);
} else {
console.log("TTS音频传输完成无messageId");
// 兜底处理,可能是旧格式
ttsStore.finishConversion(data.messageId);
}
break;
@@ -90,6 +85,7 @@ export const useWebSocketStore = defineStore("websocket", () => {
// TTS会话结束
if (data.messageId) {
console.log(`TTS会话结束 [${data.messageId}]`);
// 可以添加额外的清理逻辑
} else {
console.log("TTS会话结束");
}
@@ -102,15 +98,17 @@ export const useWebSocketStore = defineStore("websocket", () => {
ttsStore.handleError(data.message, data.messageId);
} else {
console.error("TTS错误:", data.message);
// 兜底处理,可能是旧格式
ttsStore.handleError(data.message, data.messageId || "unknown");
}
break;
case "llm_complete_response":
// LLM部分响应
if (router.currentRoute.value.path === "/voice") {
chatStore.addMessageToHistory(data.content, "assistant", "voice");
}
// 保留旧的消息类型作为兜底处理
case "tts_audio_complete_legacy":
case "tts_complete_legacy":
case "tts_error_legacy":
console.log("收到旧格式TTS消息:", data.type);
// 可以选择处理或忽略
break;
default:

View File

@@ -1,6 +1,5 @@
import { useWebSocketStore } from "@/services";
import { convertToPCM16 } from "@/utils";
import { useChatStore } from "./chat_store";
export const useAsrStore = defineStore("asr", () => {
// 是否正在录音
@@ -12,8 +11,8 @@ export const useAsrStore = defineStore("asr", () => {
let mediaStreamSource: MediaStreamAudioSourceNode | null = null;
let workletNode: AudioWorkletNode | null = null;
// 获取 WebSocket store 实例
const webSocketStore = useWebSocketStore();
const router = useRouter();
/**
* 发送消息到 WebSocket
@@ -82,19 +81,16 @@ export const useAsrStore = defineStore("asr", () => {
const processorUrl = URL.createObjectURL(blob);
// 加载AudioWorklet模块
await audioContext.audioWorklet.addModule(processorUrl);
// 释放URL对象防止内存泄漏
// 释放URL对象防止内存泄漏
URL.revokeObjectURL(processorUrl);
// 创建音频源节点
mediaStreamSource = audioContext.createMediaStreamSource(stream);
// 创建AudioWorkletNode
workletNode = new AudioWorkletNode(audioContext, "audio-processor", {
numberOfInputs: 1,
numberOfOutputs: 1,
channelCount: 1
});
// 监听来自AudioWorklet的音频数据
workletNode.port.onmessage = (event) => {
if (event.data.type === "audiodata") {
@@ -120,15 +116,8 @@ export const useAsrStore = defineStore("asr", () => {
const stopRecording = () => {
if (!isRecording.value) return;
const messageId = `voice_${Date.now()}`;
// 通知后端录音结束
const msg: Record<string, any> = { type: "asr_end" };
if (router.currentRoute.value.path === "/voice") {
msg.messageId = messageId;
msg.voiceConversation = true;
msg.speaker = useChatStore().speakerInfo?.speaker_id;
}
sendMessage(JSON.stringify(msg));
sendMessage(JSON.stringify({ type: "asr_end" }));
// 停止所有音轨
if (mediaStreamSource?.mediaStream) {

View File

@@ -1,17 +1,13 @@
import type {
CategorySpeakers,
IChatWithLLMRequest,
ModelInfo,
ModelListInfo,
Speaker,
UsageInfo
} from "@/interfaces";
import { ChatService } from "@/services";
export const useChatStore = defineStore("chat", () => {
const router = useRouter();
const token = "sk-fkGVZBrAqvIxLjlF3b5f19EfBb63486c90Fa5a1fBd7076Ee";
// 默认模型
const modelInfo = ref<ModelInfo | null>(null);
// 历史消息
@@ -20,39 +16,32 @@ export const useChatStore = defineStore("chat", () => {
const completing = ref<boolean>(false);
// 是否正在思考
const thinking = ref<boolean>(false);
// 模型列表
const modelList = ref<ModelListInfo[]>([]);
// 音色列表
const speakerList = ref<CategorySpeakers[]>([]);
// 当前音色信息
const speakerInfo = ref<Speaker | null>(null);
// 在线人数
const onlineCount = ref<number>(0);
// 生成消息ID方法
const generateMessageId = () => new Date().getTime().toString();
// 获取最后一条消息
const getLastMessage = () =>
historyMessages.value[historyMessages.value.length - 1];
// 与 LLM 聊天
const chatWithLLM = async (
request: IChatWithLLMRequest,
onProgress: (content: string) => void,
getUsageInfo: (object: UsageInfo) => void = () => {},
getThinking: (thinkingContent: string) => void = () => {}
onProgress: (content: string) => void, // 接收内容进度回调
getUsageInfo: (object: UsageInfo) => void = () => {}, // 接收使用信息回调
getThinking: (thinkingContent: string) => void = () => {} // 接收思维链内容回调
) => {
if (completing.value) throw new Error("正在响应中");
completing.value = true;
completing.value = true; // 开始请求
try {
await ChatService.ChatWithLLM(
token,
request,
onProgress,
getUsageInfo,
getThinking
(content) => {
onProgress(content);
},
(object: UsageInfo) => {
getUsageInfo(object);
},
(thinkingContent: string) => {
getThinking(thinkingContent);
}
);
} catch (error) {
console.error("请求失败:", error);
@@ -62,33 +51,28 @@ export const useChatStore = defineStore("chat", () => {
};
// 添加消息到历史记录
const addMessageToHistory = (
message: string,
role: "user" | "assistant" = "user",
type: "chat" | "voice" = "chat"
) => {
const addMessageToHistory = (message: string) => {
const content = message.trim();
if (!content) return;
historyMessages.value.push({
role,
content,
type,
id: generateMessageId()
role: "user",
content
});
};
// 清除历史消息
const clearHistoryMessages = (type?: "chat" | "voice") => {
historyMessages.value = type
? historyMessages.value.filter((msg) => msg.type !== type)
: [];
const clearHistoryMessages = () => {
historyMessages.value = [];
};
// 确保最后一条消息是助手消息
// 确保最后一条消息是助手消息,如果最后一条消息不是,就加一条空的占位,不然后面的思维链会丢失
const ensureAssistantMessage = () => {
const lastMessage = getLastMessage();
if (!lastMessage || lastMessage.role !== "assistant") {
if (
historyMessages.value.length === 0 ||
historyMessages.value[historyMessages.value.length - 1].role !==
"assistant"
) {
historyMessages.value.push({
role: "assistant",
content: ""
@@ -96,57 +80,57 @@ export const useChatStore = defineStore("chat", () => {
}
};
// 处理聊天响应的逻辑
const handleChatResponse = async (
messages: IChatWithLLMRequest["messages"]
) => {
if (!modelInfo.value) return;
// 过滤出type为chat的聊天消息
const filteredMessages = computed(() =>
messages.filter((msg) => msg.type === "chat" || !msg.type)
);
await chatWithLLM(
{ messages: filteredMessages.value, model: modelInfo.value.model_id },
// 处理文本内容
(content) => {
ensureAssistantMessage();
thinking.value = false;
getLastMessage().content = content;
},
// 处理使用信息
(usageInfo: UsageInfo) => {
const lastMessage = getLastMessage();
if (lastMessage?.role === "assistant") {
lastMessage.usage = usageInfo;
}
},
// 处理思维链
(thinkingContent: string) => {
ensureAssistantMessage();
thinking.value = true;
getLastMessage().thinking = thinkingContent;
}
);
// 设置消息ID
getLastMessage().id = generateMessageId();
};
watch(
historyMessages,
(newVal) => {
if (newVal.length > 0 && router.currentRoute.value.path === "/") {
// 当历史消息变化时,发送请求
if (newVal.length > 0) {
const lastMessage = newVal[newVal.length - 1];
if (lastMessage.role === "user") {
handleChatResponse(newVal);
if (lastMessage.role === "user" && modelInfo.value) {
chatWithLLM(
{
messages: newVal,
model: modelInfo.value?.model_id
},
// 处理进度回调,文本
(content) => {
ensureAssistantMessage();
thinking.value = false;
historyMessages.value[historyMessages.value.length - 1].content =
content;
},
// 处理使用usage信息回调
(usageInfo: UsageInfo) => {
// 如果最后一条消息是助手的回复,则更新使用信息
if (
historyMessages.value.length > 0 &&
historyMessages.value[historyMessages.value.length - 1].role ===
"assistant"
) {
historyMessages.value[historyMessages.value.length - 1].usage =
usageInfo;
}
},
// 处理思维链内容回调
(thinkingContent: string) => {
ensureAssistantMessage();
thinking.value = true;
historyMessages.value[historyMessages.value.length - 1].thinking =
thinkingContent;
}
).then(() => {
historyMessages.value[historyMessages.value.length - 1].id =
new Date().getTime().toString();
});
}
}
},
{ deep: true }
);
// 模型列表
const modelList = ref<ModelListInfo[]>([]);
// 获取模型列表
const getModelList = async () => {
try {
@@ -157,30 +141,17 @@ export const useChatStore = defineStore("chat", () => {
}
};
// 获取音色列表
const getSpeakerList = async () => {
try {
const response = await ChatService.GetSpeakerList();
speakerList.value = response.data.data;
} catch (error) {
console.error("获取音色·列表失败:", error);
}
};
return {
token,
completing,
thinking,
modelInfo,
modelList,
historyMessages,
chatWithLLM,
historyMessages,
addMessageToHistory,
clearHistoryMessages,
getModelList,
modelList,
modelInfo,
onlineCount,
speakerList,
getSpeakerList,
speakerInfo
thinking
};
});

View File

@@ -1,6 +1,5 @@
import { useAudioWebSocket } from "@/services";
import { createAudioUrl, mergeAudioChunks } from "@/utils";
import { useChatStore } from "./chat_store";
interface AudioState {
isPlaying: boolean;
@@ -13,7 +12,6 @@ interface AudioState {
}
export const useTtsStore = defineStore("tts", () => {
const chatStore = useChatStore();
// 多音频状态管理 - 以消息ID为key
const audioStates = ref<Map<string, AudioState>>(new Map());
@@ -67,14 +65,7 @@ export const useTtsStore = defineStore("tts", () => {
hasActiveSession.value = true;
// 发送文本到TTS服务
sendMessage(
JSON.stringify({
type: "tts_text",
text,
messageId,
speaker: chatStore.speakerInfo?.speaker_id
})
);
sendMessage(JSON.stringify({ type: "tts_text", text, messageId }));
} catch (error) {
handleError(`连接失败: ${error}`, messageId);
}

View File

@@ -1,23 +0,0 @@
const leagacyCopy = (text: string) => {
const input = document.createElement("input");
input.value = text;
document.body.appendChild(input);
input.select();
try {
document.execCommand("copy");
} catch (err) {
console.error(err);
}
document.body.removeChild(input);
};
export const copy = (text: string) => {
if (navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(text).catch((err) => {
console.error(err);
leagacyCopy(text); // 如果现代API失败使用旧方法
});
} else {
leagacyCopy(text); // 如果没有现代API使用旧方法
}
};

View File

@@ -1,5 +1,4 @@
export * from "./audio";
export * from "./clipboard";
export * from "./context";
export * from "./format";
export * from "./media";

View File

@@ -29,11 +29,6 @@ const collapseActive = ref<string[]>(
historyMessages.value.map((msg, idx) => String(msg.id ?? idx))
);
// typechat
const filteredMessages = computed(() =>
historyMessages.value.filter((msg) => msg.type === "chat" || !msg.type)
);
const getName = (msg: Message, idx: number) => String(msg.id ?? idx);
// TODO: bugfix:
@@ -153,7 +148,7 @@ onMounted(() => {
</div>
<!-- 默认消息 历史消息 -->
<div
v-for="(msg, idx) in filteredMessages"
v-for="(msg, idx) in historyMessages"
:key="idx"
class="flex items-start mb-4"
>
@@ -204,7 +199,9 @@ onMounted(() => {
</NCollapse>
<!-- 内容 思维链 -->
<markdown :content="msg.content || ''" />
<MessageTools :msg="msg" />
<div v-if="msg.role !== 'user'" class="mt-2">
<tts :text="msg.content || ''" :message-id="msg.id!" />
</div>
<NDivider />
</div>
</div>
@@ -244,7 +241,7 @@ onMounted(() => {
:positive-button-props="{ type: 'error' }"
positive-text="清除"
negative-text="取消"
@positive-click="chatStore.clearHistoryMessages('chat')"
@positive-click="chatStore.clearHistoryMessages"
@negative-click="() => {}"
>
<template #icon>

View File

@@ -1,261 +0,0 @@
<script setup lang="ts">
import type { SelectGroupOption, SelectOption } from "naive-ui";
import type { Message } from "@/interfaces";
import { throttle } from "lodash-es";
import AIAvatar from "@/assets/ai_avatar.png";
import { ExclamationTriangleIcon, microphone, TrashIcon } from "@/assets/Icons";
import UserAvatar from "@/assets/user_avatar.jpg";
import markdown from "@/components/markdown.vue";
import { useAsrStore, useChatStore, useLayoutStore } from "@/stores";
const chatStore = useChatStore();
const { historyMessages, completing, speakerList, speakerInfo, thinking } =
storeToRefs(chatStore);
const asrStore = useAsrStore();
const { isRecording } = storeToRefs(asrStore);
const layoutStore = useLayoutStore();
const { hiddenLeftSidebar, simpleMode } = storeToRefs(layoutStore);
const scrollbarRef = ref<HTMLElement | null>(null);
const options = ref<Array<SelectGroupOption | SelectOption>>([]);
// NCollapse 组件的折叠状态
const collapseActive = ref<string[]>(
historyMessages.value.map((msg, idx) => String(msg.id ?? idx))
);
// 过滤出type为voice的聊天消息
const filteredMessages = computed(() =>
historyMessages.value.filter((msg) => msg.type === "voice")
);
const getName = (msg: Message, idx: number) => String(msg.id ?? idx);
// TODO: bugfix: 未能正确展开
watch(
historyMessages,
(newVal, oldVal) => {
// 取所有name
const newNames = newVal.map((msg, idx) => getName(msg, idx));
const oldNames = oldVal ? oldVal.map((msg, idx) => getName(msg, idx)) : [];
// 找出新增的name
const addedNames = newNames.filter((name) => !oldNames.includes(name));
// 保留原有已展开项
const currentActive = collapseActive.value.filter((name) =>
newNames.includes(name)
);
// 新增的默认展开
collapseActive.value = [...currentActive, ...addedNames];
},
{ immediate: true, deep: true }
);
// 处理折叠项的点击事件,切换折叠状态
const handleItemHeaderClick = (name: string) => {
if (collapseActive.value.includes(name)) {
collapseActive.value = collapseActive.value.filter((n) => n !== name);
} else {
collapseActive.value.push(name);
}
};
// 处理选中speaker的 ID
const selectedSpeakerId = computed({
get: () => speakerInfo.value?.speaker_id ?? null,
set: (id: string | null) => {
for (const category of speakerList.value) {
const found = category.speakers.find(
(speaker) => speaker.speaker_id === id
);
if (found) {
speakerInfo.value = found;
return;
}
}
speakerInfo.value = null;
}
});
// 监听speaker列表变化更新选项
watch(
() => speakerList.value,
(newVal) => {
if (newVal) {
options.value = newVal.map((category) => ({
type: "group",
label: category.category,
key: category.category,
children: category.speakers.map((speaker) => ({
label: speaker.speaker_name,
value: speaker.speaker_id,
language: speaker.language,
platforms: speaker.platforms
}))
}));
// 默认选择第一个speaker
if (newVal.length > 0 && newVal[0].speakers.length > 0) {
speakerInfo.value = newVal[0].speakers[0];
}
}
},
{ immediate: true, deep: true }
);
// 开关语音输入
const toggleRecording = throttle(() => {
if (isRecording.value) {
asrStore.stopRecording();
} else {
asrStore.startRecording();
}
}, 500);
watch(completing, (newVal) => {
if (newVal) {
nextTick(() => {
scrollbarRef.value?.scrollTo({ top: 99999, behavior: "smooth" });
});
}
});
onMounted(() => {
chatStore.getSpeakerList();
});
</script>
<template>
<div
class="p-8 !pr-4 h-full w-full flex flex-col gap-4 border-l-[24px] border-l-[#FAFAFA] transition-all ease-in-out text-base"
:class="{ '!border-l-0': hiddenLeftSidebar || simpleMode }"
>
<!-- 历史消息区 -->
<NScrollbar ref="scrollbarRef" class="flex-1 pr-4 relative">
<div class="flex items-start mb-4">
<span class="rounded-lg overflow-hidden !w-16 !min-w-16 !h-16">
<avatar :avatar="AIAvatar" />
</span>
<div class="text-base w-full max-w-full ml-2 flex flex-col items-start">
<span class="text-base font-bold mb-4">助手</span>
<span class="text-base"
>你好我是你的智能助手请问有什么可以帮助你的吗</span
>
<NDivider />
</div>
</div>
<!-- 默认消息 历史消息 -->
<div
v-for="(msg, idx) in filteredMessages"
:key="idx"
class="flex items-start mb-4"
>
<!-- 头像 -->
<span
v-if="msg.role === 'user'"
class="rounded-lg overflow-hidden !w-16 !min-w-16 !h-16"
>
<avatar :avatar="UserAvatar" />
</span>
<span v-else class="rounded-lg overflow-hidden">
<avatar :avatar="AIAvatar" />
</span>
<!-- 头像 名称 -->
<div class="text-base w-full max-w-full ml-2 flex flex-col items-start">
<span class="text-base font-bold">{{
msg.role === "user" ? "你:" : "助手:"
}}</span>
<!-- 使用信息 -->
<div
v-if="msg.role !== 'user'"
class="text-[12px] text-[#7A7A7A] mb-[2px]"
>
Tokens: <span class="mr-1">{{ msg.usage?.total_tokens }}</span>
</div>
<div class="w-full max-w-full">
<NCollapse
v-if="msg.thinking?.trim()"
:expanded-names="collapseActive[idx]"
>
<NCollapseItem
:title="
thinking && idx === historyMessages.length - 1
? '思考中...'
: '已深度思考'
"
:name="getName(msg, idx)"
@item-header-click="
() => handleItemHeaderClick(getName(msg, idx))
"
>
<div
class="text-[#7A7A7A] mb-4 border-l-2 border-[#E5E5E5] ml-2 pl-2"
>
<markdown :content="msg.thinking || ''" />
</div>
</NCollapseItem>
</NCollapse>
<!-- 内容↓ 思维链↑ -->
<markdown :content="msg.content || ''" />
<MessageTools :msg="msg" />
<NDivider />
</div>
</div>
</div>
<div
v-if="isRecording"
class="absolute inset-0 pointer-events-none flex items-center justify-center text-[#7A7A7A] text-2xl bg-white/80"
>
正在语音输入...
</div>
</NScrollbar>
<!-- 操作区 -->
<div class="flex justify-between items-center gap-2">
<div class="flex items-center gap-2">
<NSelect
v-model:value="selectedSpeakerId"
label-field="label"
value-field="value"
children-field="children"
filterable
:options="options"
/>
</div>
<div class="flex items-center gap-2">
<NPopconfirm
:positive-button-props="{ type: 'error' }"
positive-text="清除"
negative-text="取消"
@positive-click="chatStore.clearHistoryMessages('voice')"
@negative-click="() => {}"
>
<template #icon>
<ExclamationTriangleIcon class="!w-6 !h-6 text-[#d03050]" />
</template>
<template #trigger>
<NButton :disabled="isRecording || completing" type="warning">
<template v-if="!simpleMode"> 清除历史 </template>
<TrashIcon
class="!w-4 !h-4"
:class="{
'ml-1': !simpleMode
}"
/>
</NButton>
</template>
<span>确定要清除历史消息吗?</span>
</NPopconfirm>
<NButton :disabled="completing" @click="toggleRecording">
<template v-if="!simpleMode">
{{ isRecording ? "停止输入" : "语音输入" }}
</template>
<microphone
class="!w-4 !h-4"
:class="{
'ml-1': !simpleMode
}"
/>
</NButton>
</div>
</div>
</div>
</template>