Compare commits
42 Commits
c6869f773e
...
feat/langg
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5263192f92 | ||
|
|
1c8e9da486 | ||
|
|
5c719ed2ea | ||
|
|
893eb2f2f7 | ||
|
|
88ba48ac77 | ||
|
|
ce97a16f5b | ||
|
|
6f91f45065 | ||
|
|
5706c112e5 | ||
|
|
569dae4446 | ||
|
|
3836b443b1 | ||
|
|
aa2768acc0 | ||
|
|
83353c54d1 | ||
|
|
c1f469409d | ||
|
|
cf0a8b71fd | ||
|
|
a6777a827b | ||
|
|
084e13e03c | ||
|
|
7666759121 | ||
|
|
6ff23e433b | ||
|
|
edf85e21cc | ||
|
|
ffd3bf0385 | ||
|
|
1ea1d86d5a | ||
|
|
705a63dd25 | ||
|
|
056a8a4ad8 | ||
|
|
30c413b6d4 | ||
|
|
0ef5de399a | ||
|
|
00c1bbdc6b | ||
|
|
6df6c16e1d | ||
|
|
13f5f44f61 | ||
|
|
f49212afc9 | ||
|
|
ed9621dfe0 | ||
|
|
9e2374f492 | ||
|
|
b14ef64665 | ||
|
|
e4e47f637e | ||
|
|
c7f2f4e5f3 | ||
|
|
a10f0a586b | ||
|
|
bc2aa5b133 | ||
|
|
0b6b40aba4 | ||
|
|
87e616ab55 | ||
|
|
34848dd6a0 | ||
|
|
30306addee | ||
|
|
32c491cd3f | ||
|
|
d5f81ef79f |
10
.env
10
.env
@@ -1,10 +0,0 @@
|
||||
DATABASE_URL=sqlite:///./test.db
|
||||
SECRET_KEY=your_secret_key
|
||||
DEBUG=True
|
||||
|
||||
ANALYSIS_SERVICE_URL=http://101.89.151.141:3000/api/v1/chat/completions
|
||||
ANALYSIS_AUTH_TOKEN=fastgpt-hSPnXMoBNGVAEpTLkQT3YfAnN26gQSyvLd4ABL1MRDoh68nL4RDlopFHXqmH8
|
||||
APP_ID=683ea1bc86197e19f71fc1ae
|
||||
DELETE_SESSION_URL=http://101.89.151.141:3000/api/core/chat/delHistory?chatId={chatId}&appId={appId}
|
||||
DELETE_CHAT_URL=http://101.89.151.141:3000/api/core/chat/item/delete?contentId={contentId}&chatId={chatId}&appId={appId}
|
||||
GET_CHAT_RECORDS_URL=http://101.89.151.141:3000/api/core/chat/getPaginationRecords
|
||||
19
.env.example
Normal file
19
.env.example
Normal file
@@ -0,0 +1,19 @@
|
||||
DATABASE_URL=sqlite:///./test.db
|
||||
SECRET_KEY=replace-with-a-random-secret
|
||||
DEBUG=false
|
||||
|
||||
ZNJJ_ENVIRONMENT=development
|
||||
AGENT_BACKEND=fastgpt
|
||||
LANGGRAPH_CHECKPOINTER=memory
|
||||
|
||||
ANALYSIS_SERVICE_URL=http://127.0.0.1:3000
|
||||
ANALYSIS_AUTH_TOKEN=replace-with-local-fastgpt-token
|
||||
APP_ID=replace-with-fastgpt-app-id
|
||||
|
||||
LLM_API_KEY=replace-with-llm-api-key
|
||||
LLM_BASE_URL=https://api.openai.com/v1
|
||||
LLM_MODEL=replace-with-model-name
|
||||
LLM_TIMEOUT_SECONDS=60
|
||||
LLM_MAX_RETRIES=2
|
||||
|
||||
VOICE_CONFIG=config/voice-fastgpt-state-xfyunSuperTTS.json
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -1,2 +1,7 @@
|
||||
*.pyc
|
||||
__pycache__/
|
||||
__pycache__/
|
||||
logs/
|
||||
*.log
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
104
config/voice-fastgpt-state-xfyunSuperTTS.json
Normal file
104
config/voice-fastgpt-state-xfyunSuperTTS.json
Normal file
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"server": {
|
||||
"host": "0.0.0.0",
|
||||
"port": 8000,
|
||||
"cors_origins": ["*"]
|
||||
},
|
||||
"audio": {
|
||||
"sample_rate_hz": 16000,
|
||||
"channels": 1,
|
||||
"frame_ms": 20
|
||||
},
|
||||
"session": {
|
||||
"inactivity_timeout_sec": 60
|
||||
},
|
||||
"turn": {
|
||||
"vad": {
|
||||
"confidence": 0.8,
|
||||
"start_secs": 0.4,
|
||||
"stop_secs": 0.2,
|
||||
"min_volume": 0.8
|
||||
},
|
||||
"interruption_min_chars": 3,
|
||||
"interruption_use_interim": true,
|
||||
"interruption_short_replies": [
|
||||
"是",
|
||||
"是的",
|
||||
"对",
|
||||
"对的",
|
||||
"嗯",
|
||||
"好",
|
||||
"好的",
|
||||
"行",
|
||||
"可以",
|
||||
"没问题",
|
||||
"不是",
|
||||
"不",
|
||||
"不行",
|
||||
"不用",
|
||||
"不要",
|
||||
"没有",
|
||||
"否",
|
||||
"你好",
|
||||
"在吗"
|
||||
],
|
||||
"user_speech_timeout_sec": 0.2,
|
||||
"idle_prompt_timeout_sec": 3.0,
|
||||
"idle_prompt_max_count": 3,
|
||||
"idle_prompt_text": "你好,请问还在吗?"
|
||||
},
|
||||
"agent": {
|
||||
"system_prompt": "FastGPT app owns the system prompt when send_system_prompt is false.",
|
||||
"greeting": "您好,这里是无锡交警,我将为您远程处理交通事故。请将人员撤离至路侧安全区域,开启危险报警双闪灯、放置三角警告牌、做好安全防护,谨防二次事故伤害。若您已经准备好了,请点击继续办理,如需人工服务,请说转人工。",
|
||||
"greeting_mode": "fastgpt_opener",
|
||||
"response_state": {
|
||||
"enabled": true,
|
||||
"tag": "state",
|
||||
"event_type": "response.state",
|
||||
"max_prefix_chars": 256
|
||||
}
|
||||
},
|
||||
"services": {
|
||||
"stt": {
|
||||
"provider": "xfyun",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://iat-api.xfyun.cn/v2/iat",
|
||||
"language": "zh_cn",
|
||||
"domain": "iat",
|
||||
"accent": "mandarin",
|
||||
"encoding": "raw",
|
||||
"frame_size": 1280,
|
||||
"timeout_sec": 10.0
|
||||
},
|
||||
"llm": {
|
||||
"provider": "fastgpt",
|
||||
"api_key": "fastgpt-zlLjYtWZWN0uhQHs3ZOFHG4KLGMIdr2CkbZLCSfqGm5vcdx5xIZbp",
|
||||
"base_url": "http://localhost:3030",
|
||||
"model": "my-voice-app",
|
||||
"app_id": "691eddaa53e3f8d9f25f1370",
|
||||
"chat_id": null,
|
||||
"variables": {},
|
||||
"detail": false,
|
||||
"timeout_sec": 60.0,
|
||||
"send_system_prompt": false
|
||||
},
|
||||
"tts": {
|
||||
"provider": "xfyun_super",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6",
|
||||
"voice": "x5_lingxiaoxuan_flow",
|
||||
"aue": "raw",
|
||||
"speed": 50,
|
||||
"volume": 50,
|
||||
"pitch": 50,
|
||||
"oral_level": "mid",
|
||||
"source_sample_rate_hz": 24000,
|
||||
"text_aggregation_mode": "token",
|
||||
"timeout_sec": 30.0
|
||||
}
|
||||
}
|
||||
}
|
||||
99
config/voice-fastgpt-state-xfyunTTS.json
Normal file
99
config/voice-fastgpt-state-xfyunTTS.json
Normal file
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"server": {
|
||||
"host": "0.0.0.0",
|
||||
"port": 8000,
|
||||
"cors_origins": ["*"]
|
||||
},
|
||||
"audio": {
|
||||
"sample_rate_hz": 16000,
|
||||
"channels": 1,
|
||||
"frame_ms": 20
|
||||
},
|
||||
"session": {
|
||||
"inactivity_timeout_sec": 60
|
||||
},
|
||||
"turn": {
|
||||
"vad": {
|
||||
"confidence": 0.8,
|
||||
"start_secs": 0.4,
|
||||
"stop_secs": 0.2,
|
||||
"min_volume": 0.8
|
||||
},
|
||||
"interruption_min_chars": 3,
|
||||
"interruption_use_interim": true,
|
||||
"interruption_short_replies": [
|
||||
"是",
|
||||
"是的",
|
||||
"对",
|
||||
"对的",
|
||||
"嗯",
|
||||
"好",
|
||||
"好的",
|
||||
"行",
|
||||
"可以",
|
||||
"没问题",
|
||||
"不是",
|
||||
"不",
|
||||
"不行",
|
||||
"不用",
|
||||
"不要",
|
||||
"没有",
|
||||
"否",
|
||||
"你好",
|
||||
"在吗"
|
||||
],
|
||||
"user_speech_timeout_sec": 0.2,
|
||||
"idle_prompt_timeout_sec": 3.0,
|
||||
"idle_prompt_max_count": 3,
|
||||
"idle_prompt_text": "你好,请问还在吗?"
|
||||
},
|
||||
"agent": {
|
||||
"greeting_mode": "fastgpt_opener",
|
||||
"response_state": {
|
||||
"enabled": true,
|
||||
"tag": "state",
|
||||
"event_type": "response.state",
|
||||
"max_prefix_chars": 256
|
||||
}
|
||||
},
|
||||
"services": {
|
||||
"stt": {
|
||||
"provider": "xfyun",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://iat-api.xfyun.cn/v2/iat",
|
||||
"language": "zh_cn",
|
||||
"domain": "iat",
|
||||
"accent": "mandarin",
|
||||
"encoding": "raw",
|
||||
"frame_size": 1280,
|
||||
"timeout_sec": 10.0
|
||||
},
|
||||
"llm": {
|
||||
"provider": "fastgpt",
|
||||
"api_key": "fastgpt-zlLjYtWZWN0uhQHs3ZOFHG4KLGMIdr2CkbZLCSfqGm5vcdx5xIZbp",
|
||||
"base_url": "http://localhost:3030",
|
||||
"model": "my-voice-app",
|
||||
"app_id": "691eddaa53e3f8d9f25f1370",
|
||||
"chat_id": null,
|
||||
"variables": {},
|
||||
"detail": false,
|
||||
"timeout_sec": 60.0
|
||||
},
|
||||
"tts": {
|
||||
"provider": "xfyun",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://tts-api.xfyun.cn/v2/tts",
|
||||
"voice": "x4_xiaoyan",
|
||||
"aue": "raw",
|
||||
"tte": "UTF8",
|
||||
"speed": 50,
|
||||
"volume": 50,
|
||||
"pitch": 50,
|
||||
"source_sample_rate_hz": 16000
|
||||
}
|
||||
}
|
||||
}
|
||||
104
config/voice-fastgpt-xfyunSuperTTS.json
Normal file
104
config/voice-fastgpt-xfyunSuperTTS.json
Normal file
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"server": {
|
||||
"host": "0.0.0.0",
|
||||
"port": 8000,
|
||||
"cors_origins": ["*"]
|
||||
},
|
||||
"audio": {
|
||||
"sample_rate_hz": 16000,
|
||||
"channels": 1,
|
||||
"frame_ms": 20
|
||||
},
|
||||
"session": {
|
||||
"inactivity_timeout_sec": 60
|
||||
},
|
||||
"turn": {
|
||||
"vad": {
|
||||
"confidence": 0.8,
|
||||
"start_secs": 0.4,
|
||||
"stop_secs": 0.2,
|
||||
"min_volume": 0.8
|
||||
},
|
||||
"interruption_min_chars": 3,
|
||||
"interruption_use_interim": true,
|
||||
"interruption_short_replies": [
|
||||
"是",
|
||||
"是的",
|
||||
"对",
|
||||
"对的",
|
||||
"嗯",
|
||||
"好",
|
||||
"好的",
|
||||
"行",
|
||||
"可以",
|
||||
"没问题",
|
||||
"不是",
|
||||
"不",
|
||||
"不行",
|
||||
"不用",
|
||||
"不要",
|
||||
"没有",
|
||||
"否",
|
||||
"你好",
|
||||
"在吗"
|
||||
],
|
||||
"user_speech_timeout_sec": 0.2,
|
||||
"idle_prompt_timeout_sec": 3.0,
|
||||
"idle_prompt_max_count": 3,
|
||||
"idle_prompt_text": "你好,请问还在吗?"
|
||||
},
|
||||
"agent": {
|
||||
"system_prompt": "FastGPT app owns the system prompt when send_system_prompt is false.",
|
||||
"greeting": "您好,这里是无锡交警,我将为您远程处理交通事故。请将人员撤离至路侧安全区域,开启危险报警双闪灯、放置三角警告牌、做好安全防护,谨防二次事故伤害。若您已经准备好了,请点击继续办理,如需人工服务,请说转人工。",
|
||||
"greeting_mode": "fastgpt_opener",
|
||||
"response_state": {
|
||||
"enabled": true,
|
||||
"tag": "state",
|
||||
"event_type": "response.state",
|
||||
"max_prefix_chars": 256
|
||||
}
|
||||
},
|
||||
"services": {
|
||||
"stt": {
|
||||
"provider": "xfyun",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://iat-api.xfyun.cn/v2/iat",
|
||||
"language": "zh_cn",
|
||||
"domain": "iat",
|
||||
"accent": "mandarin",
|
||||
"encoding": "raw",
|
||||
"frame_size": 1280,
|
||||
"timeout_sec": 10.0
|
||||
},
|
||||
"llm": {
|
||||
"provider": "fastgpt",
|
||||
"api_key": "fastgpt-v1FljAxBz3tJeS0bH7HZU4yVGclsTcfiy9yK7V9Zr9126maDHQ97Xlo8n",
|
||||
"base_url": "http://localhost:3030",
|
||||
"model": "my-voice-app",
|
||||
"app_id": "6a153aed53e3f8d9f2744905",
|
||||
"chat_id": null,
|
||||
"variables": {},
|
||||
"detail": false,
|
||||
"timeout_sec": 60.0,
|
||||
"send_system_prompt": false
|
||||
},
|
||||
"tts": {
|
||||
"provider": "xfyun_super",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6",
|
||||
"voice": "x5_lingxiaoxuan_flow",
|
||||
"aue": "raw",
|
||||
"speed": 50,
|
||||
"volume": 50,
|
||||
"pitch": 50,
|
||||
"oral_level": "mid",
|
||||
"source_sample_rate_hz": 24000,
|
||||
"text_aggregation_mode": "token",
|
||||
"timeout_sec": 30.0
|
||||
}
|
||||
}
|
||||
}
|
||||
102
config/voice-fastgpt-xfyunTTS.json
Normal file
102
config/voice-fastgpt-xfyunTTS.json
Normal file
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"server": {
|
||||
"host": "0.0.0.0",
|
||||
"port": 8000,
|
||||
"cors_origins": ["*"]
|
||||
},
|
||||
"audio": {
|
||||
"sample_rate_hz": 16000,
|
||||
"channels": 1,
|
||||
"frame_ms": 20
|
||||
},
|
||||
"session": {
|
||||
"inactivity_timeout_sec": 60
|
||||
},
|
||||
"turn": {
|
||||
"vad": {
|
||||
"confidence": 0.7,
|
||||
"start_secs": 0.35,
|
||||
"stop_secs": 0.2,
|
||||
"min_volume": 0.65
|
||||
},
|
||||
"interruption_min_chars": 3,
|
||||
"interruption_use_interim": true,
|
||||
"interruption_short_replies": [
|
||||
"是",
|
||||
"是的",
|
||||
"对",
|
||||
"对的",
|
||||
"嗯",
|
||||
"好",
|
||||
"好的",
|
||||
"行",
|
||||
"可以",
|
||||
"没问题",
|
||||
"不是",
|
||||
"不",
|
||||
"不行",
|
||||
"不用",
|
||||
"不要",
|
||||
"没有",
|
||||
"否",
|
||||
"你好",
|
||||
"在吗"
|
||||
],
|
||||
"user_speech_timeout_sec": 0.2,
|
||||
"idle_prompt_timeout_sec": 3.0,
|
||||
"idle_prompt_max_count": 3,
|
||||
"idle_prompt_text": "你好,请问还在吗?"
|
||||
},
|
||||
"agent": {
|
||||
"system_prompt": "FastGPT app owns the system prompt when send_system_prompt is false.",
|
||||
"greeting": "您好,这里是无锡交警,我将为您远程处理交通事故。请将人员撤离至路侧安全区域,开启危险报警双闪灯、放置三角警告牌、做好安全防护,谨防二次事故伤害。若您已经准备好了,请点击继续办理,如需人工服务,请说转人工。",
|
||||
"greeting_mode": "fastgpt_opener",
|
||||
"response_state": {
|
||||
"enabled": true,
|
||||
"tag": "state",
|
||||
"event_type": "response.state",
|
||||
"max_prefix_chars": 256
|
||||
}
|
||||
},
|
||||
"services": {
|
||||
"stt": {
|
||||
"provider": "xfyun",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://iat-api.xfyun.cn/v2/iat",
|
||||
"language": "zh_cn",
|
||||
"domain": "iat",
|
||||
"accent": "mandarin",
|
||||
"encoding": "raw",
|
||||
"frame_size": 1280,
|
||||
"timeout_sec": 10.0
|
||||
},
|
||||
"llm": {
|
||||
"provider": "fastgpt",
|
||||
"api_key": "fastgpt-v1FljAxBz3tJeS0bH7HZU4yVGclsTcfiy9yK7V9Zr9126maDHQ97Xlo8n",
|
||||
"base_url": "http://localhost:3030",
|
||||
"model": "my-voice-app",
|
||||
"app_id": "6a153aed53e3f8d9f2744905",
|
||||
"chat_id": null,
|
||||
"variables": {},
|
||||
"detail": false,
|
||||
"timeout_sec": 60.0,
|
||||
"send_system_prompt": false
|
||||
},
|
||||
"tts": {
|
||||
"provider": "xfyun",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://tts-api.xfyun.cn/v2/tts",
|
||||
"voice": "x4_xiaoyan",
|
||||
"aue": "raw",
|
||||
"tte": "UTF8",
|
||||
"speed": 50,
|
||||
"volume": 50,
|
||||
"pitch": 50,
|
||||
"source_sample_rate_hz": 16000
|
||||
}
|
||||
}
|
||||
}
|
||||
95
config/voice-xfyun.json
Normal file
95
config/voice-xfyun.json
Normal file
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"server": {
|
||||
"host": "0.0.0.0",
|
||||
"port": 8000,
|
||||
"cors_origins": ["*"]
|
||||
},
|
||||
"audio": {
|
||||
"sample_rate_hz": 16000,
|
||||
"channels": 1,
|
||||
"frame_ms": 20
|
||||
},
|
||||
"session": {
|
||||
"inactivity_timeout_sec": 60
|
||||
},
|
||||
"turn": {
|
||||
"vad": {
|
||||
"confidence": 0.7,
|
||||
"start_secs": 0.35,
|
||||
"stop_secs": 0.2,
|
||||
"min_volume": 0.65
|
||||
},
|
||||
"interruption_min_chars": 3,
|
||||
"interruption_use_interim": true,
|
||||
"interruption_short_replies": [
|
||||
"是",
|
||||
"是的",
|
||||
"对",
|
||||
"对的",
|
||||
"嗯",
|
||||
"好",
|
||||
"好的",
|
||||
"行",
|
||||
"可以",
|
||||
"没问题",
|
||||
"不是",
|
||||
"不",
|
||||
"不行",
|
||||
"不用",
|
||||
"不要",
|
||||
"没有",
|
||||
"否"
|
||||
],
|
||||
"user_speech_timeout_sec": 0.2,
|
||||
"idle_prompt_timeout_sec": 3.0,
|
||||
"idle_prompt_max_count": 3,
|
||||
"idle_prompt_text": "你好,请问还在吗?"
|
||||
},
|
||||
"agent": {
|
||||
"system_prompt": "# 角色 你是一个高度集成、安全第一的交警AI接警员。正在收集事故人员伤亡情况,时间,地点,事故原因,事故车辆数量,收集完成之后和用户说再见",
|
||||
"greeting": "您好,这里是无锡交警,我将为您远程处理交通事故。请将人员撤离至路侧安全区域,开启危险报警双闪灯、放置三角警告牌、做好安全防护,谨防二次事故伤害。若您已经准备好了,请点击继续办理,如需人工服务,请说转人工。",
|
||||
"greeting_mode": "fixed",
|
||||
"response_state": {
|
||||
"enabled": true,
|
||||
"tag": "state",
|
||||
"event_type": "response.state",
|
||||
"max_prefix_chars": 256
|
||||
}
|
||||
},
|
||||
"services": {
|
||||
"stt": {
|
||||
"provider": "xfyun",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://iat-api.xfyun.cn/v2/iat",
|
||||
"language": "zh_cn",
|
||||
"domain": "iat",
|
||||
"accent": "mandarin",
|
||||
"encoding": "raw",
|
||||
"frame_size": 1280,
|
||||
"timeout_sec": 10.0
|
||||
},
|
||||
"llm": {
|
||||
"provider": "openai",
|
||||
"api_key": "sk-230701ff1b6143ecbf322b3170606016",
|
||||
"base_url": "https://api.deepseek.com/v1",
|
||||
"model": "deepseek-chat",
|
||||
"temperature": 0.7
|
||||
},
|
||||
"tts": {
|
||||
"provider": "xfyun",
|
||||
"app_id": "416ce125",
|
||||
"api_key": "c65342fe603126c3610031d8429bb36d",
|
||||
"api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
|
||||
"base_url": "wss://tts-api.xfyun.cn/v2/tts",
|
||||
"voice": "x4_xiaoyan",
|
||||
"aue": "raw",
|
||||
"tte": "UTF8",
|
||||
"speed": 50,
|
||||
"volume": 50,
|
||||
"pitch": 50,
|
||||
"source_sample_rate_hz": 16000
|
||||
}
|
||||
}
|
||||
}
|
||||
84
config/voice.json
Normal file
84
config/voice.json
Normal file
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"server": {
|
||||
"host": "0.0.0.0",
|
||||
"port": 8000,
|
||||
"cors_origins": ["http://localhost:3000", "http://localhost:8080"],
|
||||
"serve_webpage": true,
|
||||
"webpage_mount": "/voice-demo"
|
||||
},
|
||||
"audio": {
|
||||
"sample_rate_hz": 16000,
|
||||
"channels": 1,
|
||||
"frame_ms": 20
|
||||
},
|
||||
"session": {
|
||||
"inactivity_timeout_sec": 60
|
||||
},
|
||||
"turn": {
|
||||
"vad": {
|
||||
"confidence": 0.7,
|
||||
"start_secs": 0.2,
|
||||
"stop_secs": 0.4,
|
||||
"min_volume": 0.6
|
||||
},
|
||||
"interruption_min_chars": 3,
|
||||
"interruption_use_interim": true,
|
||||
"interruption_short_replies": [
|
||||
"是",
|
||||
"是的",
|
||||
"对",
|
||||
"对的",
|
||||
"嗯",
|
||||
"好",
|
||||
"好的",
|
||||
"行",
|
||||
"可以",
|
||||
"没问题",
|
||||
"不是",
|
||||
"不",
|
||||
"不行",
|
||||
"不用",
|
||||
"不要",
|
||||
"没有",
|
||||
"否"
|
||||
],
|
||||
"user_speech_timeout_sec": 0.8,
|
||||
"idle_prompt_timeout_sec": 3.0,
|
||||
"idle_prompt_max_count": 3,
|
||||
"idle_prompt_text": "你好,请问还在吗?"
|
||||
},
|
||||
"agent": {
|
||||
"system_prompt": "You are a helpful, friendly voice assistant. Keep responses concise and natural for spoken conversation.",
|
||||
"greeting": "Please introduce yourself briefly.",
|
||||
"greeting_mode": "generated",
|
||||
"response_state": {
|
||||
"enabled": false,
|
||||
"tag": "state",
|
||||
"event_type": "response.state",
|
||||
"max_prefix_chars": 256
|
||||
}
|
||||
},
|
||||
"services": {
|
||||
"stt": {
|
||||
"provider": "openai",
|
||||
"api_key": "",
|
||||
"base_url": null,
|
||||
"model": "gpt-4o-mini-transcribe",
|
||||
"language": "en"
|
||||
},
|
||||
"llm": {
|
||||
"provider": "openai",
|
||||
"api_key": "",
|
||||
"base_url": null,
|
||||
"model": "gpt-4o-mini",
|
||||
"temperature": 0.7
|
||||
},
|
||||
"tts": {
|
||||
"provider": "openai",
|
||||
"api_key": "",
|
||||
"base_url": null,
|
||||
"model": "gpt-4o-mini-tts",
|
||||
"voice": "alloy"
|
||||
}
|
||||
}
|
||||
}
|
||||
101
docs/baselines/fastgpt-baseline-20260726.md
Normal file
101
docs/baselines/fastgpt-baseline-20260726.md
Normal file
@@ -0,0 +1,101 @@
|
||||
# FastGPT 迁移前基线
|
||||
|
||||
> 采集日期:2026-07-26
|
||||
> Git 基线:`5c719ed`
|
||||
> 环境:本地 macOS,项目 `.venv`,FakeBackend/FakeClient;未调用真实 FastGPT
|
||||
> 用途:冻结可复现工程基线,不冒充生产业务指标
|
||||
|
||||
## 自动化基线
|
||||
|
||||
执行:
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m pytest -q
|
||||
```
|
||||
|
||||
Phase 0 开始前结果:
|
||||
|
||||
```text
|
||||
32 passed in 0.32s
|
||||
```
|
||||
|
||||
覆盖范围:
|
||||
|
||||
- Pydantic 公共请求/响应 schema;
|
||||
- `/chat` 对 backend-neutral contract 的适配;
|
||||
- FastGPT backend 的 SDK/Event 转换;
|
||||
- state prefix 跨 chunk 的基础兼容;
|
||||
- 文本分句。
|
||||
|
||||
未覆盖范围:
|
||||
|
||||
- 真实 FastGPT 网络延迟和错误;
|
||||
- 真实 workflow 状态序列稳定性;
|
||||
- `/set_info`、`/get_info` 完整兼容行为;
|
||||
- 生产转人工比例和 prefix 失败率;
|
||||
- 数据库、并发恢复和 LangGraph。
|
||||
|
||||
Phase 0 完成后的测试数量和耗时见本报告底部。
|
||||
|
||||
## 本地 API 适配层耗时
|
||||
|
||||
使用内存 FakeBackend,关闭日志后直接调用 endpoint 并完整消费流式响应。该数据只衡量 Python 适配、prefix/SSE 处理开销,不包含 HTTP、网络、FastGPT 或模型延迟。
|
||||
|
||||
```text
|
||||
nonstream n=2000 p50=0.004ms p95=0.014ms p99=0.030ms
|
||||
stream-consume n=1000 p50=0.011ms p95=0.034ms p99=0.051ms
|
||||
```
|
||||
|
||||
这组数据用于后续发现 API 适配层的明显性能回退,不能与生产端到端延迟混用。
|
||||
|
||||
## 生产指标采集口径
|
||||
|
||||
以下数据无法从仓库推导,必须由部署环境日志或监控采集。负责人应使用同一时间窗口、同一调用方集合,并排除压测流量。
|
||||
|
||||
| 指标 | 计算方式 | 当前值 |
|
||||
|---|---|---|
|
||||
| `/chat` 请求数 | 成功与失败总请求 | 待生产采集 |
|
||||
| 非流式 P50/P95/P99 | endpoint 总耗时 | 待生产采集 |
|
||||
| 流式 TTFB P50/P95/P99 | 收到请求至首个 `text_delta` | 待生产采集 |
|
||||
| 响应体错误率 | `code != "200"` / 请求数 | 待生产采集 |
|
||||
| FastGPT 超时率 | timeout / FastGPT 调用数 | 待生产采集 |
|
||||
| Prefix 失败率 | 缺失、格式错误、未知码 / 模型回复数 | 当前未结构化记录 |
|
||||
| 转人工率 | `0001/0002/0003/0004/0005` / session 数 | 待生产采集 |
|
||||
| `formUpdate` 产生率 | 非空 patch / `needFormUpdate=true` 轮次 | 待生产采集 |
|
||||
| 平均轮次 | chat 轮次 / 完结 session 数 | 待生产采集 |
|
||||
| `/set_info`、`/get_info` 错误率 | `code != "200"` / 请求数 | 待生产采集 |
|
||||
|
||||
## 当前可观测性限制
|
||||
|
||||
当前日志虽然记录延迟,但也记录完整 `sessionId`、输入、输出和 `formUpdate`,不能直接作为长期生产基线方案。Phase 1/8 应先加入 request ID、session hash、结构化事件和脱敏,然后再持续采集。
|
||||
|
||||
建议临时聚合时只输出:
|
||||
|
||||
- 时间桶;
|
||||
- endpoint;
|
||||
- 成功/稳定错误类别;
|
||||
- 耗时;
|
||||
- stage code;
|
||||
- 是否产生 form patch;
|
||||
- 不可逆 session hash。
|
||||
|
||||
不得导出原始对话、身份证、手机号、车牌、Token 或完整表单。
|
||||
|
||||
## Phase 0 最终验证
|
||||
|
||||
完成日期:2026-07-26
|
||||
|
||||
```text
|
||||
48 passed in 0.49s
|
||||
```
|
||||
|
||||
相对 Phase 0 开始前新增 16 个测试,覆盖:
|
||||
|
||||
- `/set_info`、`/get_info` 的 FastGPT 辅助调用和兼容序列化;
|
||||
- SSE 成功事件顺序、唯一终止事件和文本拼接;
|
||||
- 当前缺失流式 prefix 的历史行为;
|
||||
- 状态/迁移/字段注册表闭合性;
|
||||
- 黄金场景唯一性和敏感号码扫描;
|
||||
- 源码、文档、测试和配置样例中的 FastGPT Token 扫描。
|
||||
|
||||
生产业务指标仍标记为“待生产采集”。这是外部可观测数据依赖,不用估算值替代;最迟必须在 Phase 9 shadow 前完成采集。
|
||||
57
docs/chat-stream-mode.md
Normal file
57
docs/chat-stream-mode.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# /chat 流式响应模式说明
|
||||
|
||||
## 接口地址
|
||||
|
||||
```
|
||||
POST http://localhost:8000/chat?stream=true
|
||||
```
|
||||
|
||||
## 请求参数
|
||||
|
||||
| 参数 | 类型 | 必填 | 说明 |
|
||||
|------|------|------|------|
|
||||
| sessionId | string | 是 | 会话 ID |
|
||||
| timeStamp | string | 是 | 时间戳 |
|
||||
| text | string | 是 | 用户输入文本 |
|
||||
| stream | bool | 否 | 设为 true 启用流式响应 |
|
||||
|
||||
## SSE 事件类型
|
||||
|
||||
| 事件类型 | 说明 | 数据格式 |
|
||||
|----------|------|----------|
|
||||
| `stage_code` | 阶段状态码 | `{"nextStageCode": "0000", "nextStage": "结束通话"}` |
|
||||
| `text_delta` | 流式文本片段 | `{"text": "您好..."}` |
|
||||
| `done` | 流式结束 | `{"status": "completed"}` |
|
||||
| `error` | 错误信息 | `{"msg": "错误描述", "code": "500"}` |
|
||||
|
||||
## 状态码映射
|
||||
|
||||
| 状态码 | 含义 |
|
||||
|--------|------|
|
||||
| 0000 | 结束通话 |
|
||||
| 0001 | 转接人工 |
|
||||
| 0002 | 语义无法识别转接人工 |
|
||||
| 0003 | 有人伤转接人工 |
|
||||
| 0005 | 拍照连续识别失败转人工 |
|
||||
| 1001 | 未准备好通话 |
|
||||
| 1002 | 通话中 |
|
||||
| 2000 | 进入单车拍照 |
|
||||
| ... | ... |
|
||||
|
||||
## 示例
|
||||
|
||||
### 请求
|
||||
|
||||
```bash
|
||||
python examples/stream_chat.py session-001 "发生了交通事故"
|
||||
```
|
||||
|
||||
### 响应
|
||||
|
||||
```
|
||||
Status: 200
|
||||
--------------------------------------------------
|
||||
[stage_code] {"nextStageCode": "1002", "nextStage": "通话中"}
|
||||
[text_delta] {"text": "您好,请问发生了什么情况?"}
|
||||
[done] {"status": "completed"}
|
||||
```
|
||||
40
docs/domain/README.md
Normal file
40
docs/domain/README.md
Normal file
@@ -0,0 +1,40 @@
|
||||
# Phase 0 领域契约索引
|
||||
|
||||
> 基线版本:2026-07-26
|
||||
> 适用范围:`/chat`、`/set_info`、`/get_info`
|
||||
> 规则来源优先级:V1.0.9 接口文档 > 2026-07-26 workflow/Prompt > 当前服务代码 > 历史 workflow/Prompt
|
||||
|
||||
本目录冻结 LangGraph 迁移前的外部契约和业务规则。JSON 文件是后续代码生成、参数化测试和 Graph validator 的机器可读输入;Markdown 文件解释兼容行为及来源。
|
||||
|
||||
## 交付物
|
||||
|
||||
| 文件 | 用途 |
|
||||
|---|---|
|
||||
| `api-contract.md` | 三个接口的当前兼容行为和已知偏差 |
|
||||
| `stage-codes.json` | 状态码、内部/外部映射和产生方式 |
|
||||
| `stage-transitions.json` | 权威迁移矩阵和拍照顺序 |
|
||||
| `field-registry.json` | 字段、分组、类型、敏感性和写权限 |
|
||||
| `business-rules.md` | 从 workflow、Prompt 和 endpoint 提取的规则 |
|
||||
| `event-mapping.md` | 历史魔法字符串到结构化事件的映射 |
|
||||
| `../baselines/fastgpt-baseline-20260726.md` | 迁移前可复现测试/性能基线 |
|
||||
| `../../test/fixtures/golden/accident-scenarios.json` | 脱敏黄金场景 |
|
||||
|
||||
## 已冻结的关键决定
|
||||
|
||||
1. 对外继续使用 camelCase、字符串业务码和 JSON 编码的 `/get_info.value`。
|
||||
2. `3001`、`3002` 为内部信息确认状态,对外仍返回 `1002`。
|
||||
3. `2006`、`2017`、`2020` 仅作为历史别名接收,不作为新图的合法目标状态。
|
||||
4. `0004` 纳入正式状态表;当前 endpoint 状态名称映射缺失是待修复偏差,不代表删除该状态。
|
||||
5. 拍照完成、连续拍照失败、无回复和外部字段更新在新图中必须是结构化确定性事件。
|
||||
6. `sfzmwh1/2`、`sjwh1/2` 是只读兼容字段,不允许 `/set_info` 写入。
|
||||
7. `phase`、`stage_code`、计数器、版本和图路由字段禁止外部修改。
|
||||
|
||||
## 变更规则
|
||||
|
||||
Phase 0 冻结后,修改这里的状态、字段或外部契约必须同时:
|
||||
|
||||
1. 说明业务原因和兼容影响;
|
||||
2. 更新机器可读 JSON;
|
||||
3. 更新黄金场景;
|
||||
4. 更新对应契约/领域测试;
|
||||
5. 获得接口调用方或产品确认。
|
||||
115
docs/domain/api-contract.md
Normal file
115
docs/domain/api-contract.md
Normal file
@@ -0,0 +1,115 @@
|
||||
# 兼容 API 契约基线
|
||||
|
||||
## 通用约定
|
||||
|
||||
- 路径保持 `/chat`、`/set_info`、`/get_info`。
|
||||
- 请求和响应字段保持 camelCase。
|
||||
- 业务成功/失败主要通过响应体字符串 `code` 表达;现有 endpoint 通常仍返回 HTTP 200。
|
||||
- `sessionId` 最大 64 字符,`timeStamp` 最大 32 字符。
|
||||
- Pydantic 校验失败由 FastAPI 返回 HTTP 422。
|
||||
- 本文冻结的是当前可观察行为;“目标行为”标记为后续迁移必须修复的已批准偏差。
|
||||
|
||||
## `POST /chat`
|
||||
|
||||
### 请求
|
||||
|
||||
| 字段 | 类型 | 必填 | 默认值 |
|
||||
|---|---|---:|---|
|
||||
| `sessionId` | string | 是 | - |
|
||||
| `timeStamp` | string | 是 | - |
|
||||
| `text` | string | 是 | - |
|
||||
| `needFormUpdate` | boolean | 否 | `false` |
|
||||
| `useTextChunk` | boolean | 否 | `false` |
|
||||
|
||||
### 非流式响应
|
||||
|
||||
字段为 `sessionId`、`timeStamp`、`outputText`、`formUpdate`、`nextStage`、`nextStageCode`、`code`、`msg`。
|
||||
|
||||
- 成功时 `code="200"`。
|
||||
- `<state>XXXX</state>` 从正文中移除。
|
||||
- `3001/3002/1002 → 1002`,`2006 → 2004`,`2017 → 2016`,`2020 → 0002`。
|
||||
- Prefix 缺失或正文不可解析时当前返回 `code="500"` 和“消息不完整”。
|
||||
- FastGPT 认证、限流和 API 异常分别映射为响应体 `401`、`429`、`500`。
|
||||
- `formUpdate` 保持无固定 schema 的 JSON 值,以兼容现有调用方。
|
||||
|
||||
### SSE 响应
|
||||
|
||||
事件名和数据:
|
||||
|
||||
| 事件 | 数据 | 基数 |
|
||||
|---|---|---|
|
||||
| `stage_code` | `{"nextStageCode":"1002","nextStage":"通话中"}` | 成功轮最多一次 |
|
||||
| `formUpdate` | 表单 patch 对象 | 有更新时最多一次 |
|
||||
| `text_delta` | `{"text":"..."}` | 零到多次 |
|
||||
| `done` | `{"status":"completed"}` | 成功恰好一次 |
|
||||
| `error` | `{"msg":"...","code":"500"}` | 失败恰好一次且终止 |
|
||||
|
||||
V1.0.9 文档要求 `stage_code` 先于 `text_delta`;`formUpdate` 的位置由 FastGPT `flowResponses` 到达时间决定,文档示例允许它出现在两个 `text_delta` 之间。LangGraph 迁移目标固定为:
|
||||
|
||||
```text
|
||||
stage_code -> formUpdate(可选) -> text_delta* -> done
|
||||
```
|
||||
|
||||
迁移后的错误路径不得同时产生 `done` 和 `error`。`useTextChunk=true` 只改变 `text_delta` 切分,不改变拼接后的文本。
|
||||
|
||||
### 已知偏差
|
||||
|
||||
- 当前流式 parser 会在整段文本中搜索标签,而不是强制标签位于开头。
|
||||
- 当前流式未知状态码仍可能发送空 `nextStage`。
|
||||
- 当前流式缺少 prefix 时仍可能输出正文和 `done`。
|
||||
- 当前流式内部事件处理异常会记录后继续,可能掩盖部分失败。
|
||||
- 当前实现记录完整输入、输出和 `formUpdate`,不符合数据保护目标。
|
||||
|
||||
以上偏差被字符化测试记录,但不作为 LangGraph 新实现的目标行为;Phase 5–8 必须按迁移计划修正。
|
||||
|
||||
## `POST /set_info`
|
||||
|
||||
### 请求
|
||||
|
||||
字段为 `sessionId`、`timeStamp`、`key`、`value`、`includeInputInfo`;其中 `includeInputInfo` 默认 `false`。
|
||||
|
||||
当前实现:
|
||||
|
||||
1. 通过一次 FastGPT 对话读取 `newVariables.state`;
|
||||
2. 删除辅助对话记录;
|
||||
3. 直接执行 `state[key] = value`;
|
||||
4. 再通过 FastGPT 对话写回并删除辅助记录。
|
||||
|
||||
成功返回 `code="200"`;任一步失败返回响应体 `code="500"`。
|
||||
|
||||
目标行为:
|
||||
|
||||
- 仅接受 `field-registry.json` 中 `external_write=true` 的 key。
|
||||
- 进行类型转换和领域校验。
|
||||
- 禁止写内部状态字段。
|
||||
- 直接事务化写业务状态,不调用 LLM,不创建/删除辅助聊天记录。
|
||||
|
||||
## `POST /get_info`
|
||||
|
||||
请求字段为 `sessionId`、`timeStamp`、`key`、`includeInputInfo`;`includeInputInfo` 默认 `false`。
|
||||
|
||||
支持:
|
||||
|
||||
- `all`
|
||||
- `acdinfo`
|
||||
- `acdhuman1`
|
||||
- `acdhuman2`
|
||||
- 单个字段 key
|
||||
|
||||
兼容序列化:
|
||||
|
||||
- `value` 始终是 JSON 编码后的字符串。
|
||||
- boolean 转为字符串 `"1"` 或 `"0"`。
|
||||
- 缺失字段转为空字符串。
|
||||
- 未知单字段 key 当前返回 JSON 字符串 `""`,而不是报错。
|
||||
|
||||
目标实现仍保留上述响应编码,但直接读取业务投影,不调用 FastGPT。
|
||||
|
||||
## 契约来源
|
||||
|
||||
- `src/schemas/models.py`
|
||||
- `src/api/endpoints.py`
|
||||
- `docs/视频快处智能信息采集机器人交互接口文档V1.0.9.docx`
|
||||
- `docs/chat-stream-mode.md`
|
||||
- `test/api/test_public_schema_contract.py`
|
||||
- `test/api/test_chat_backend_boundary.py`
|
||||
116
docs/domain/business-rules.md
Normal file
116
docs/domain/business-rules.md
Normal file
@@ -0,0 +1,116 @@
|
||||
# 事故采集业务规则基线
|
||||
|
||||
## 规则来源
|
||||
|
||||
本基线交叉比对以下来源:
|
||||
|
||||
1. V1.0.9 交互接口文档;
|
||||
2. `workflow/20260726/事故信息采集20260726.json`;
|
||||
3. `prompts/20260723/单车拍照.txt`;
|
||||
4. `prompts/20260723/双车拍照.txt`;
|
||||
5. 当前 `src/api/endpoints.py`;
|
||||
6. 2025 版本 workflow/Prompt,仅用于识别历史兼容行为。
|
||||
|
||||
发生冲突时采用接口文档和 2026-07-26 规则;历史状态别名只在 API 边界兼容。
|
||||
|
||||
## 全局规则
|
||||
|
||||
1. 每个自然语言回复必须以且仅以一个 `<state>四位数字</state>` 开头。
|
||||
2. 用户明确要求“转人工”“找人工”“人工客服”等时,立即进入 `0001`。
|
||||
3. 明确或高度可信的人伤、三辆及以上机动车、涉及行人/非机动车等复杂情况进入 `0003`。
|
||||
4. 明确否定人伤时不得因句中出现“受伤”“流血”等词误触发 `0003`。
|
||||
5. ASR 内容破碎或人伤语义矛盾时,用当前采集状态封闭确认,不能直接冒险放行。
|
||||
6. 当前问题没有有效答案时不得跳题。第一次澄清,第二次强制选择;仍失败进入 `0002`。
|
||||
7. 连续第一次无回复使用固定唤醒话术;连续第二次进入 `0004`。
|
||||
8. 用户有效回复后无回复计数清零。
|
||||
9. 状态候选必须经过枚举和迁移矩阵校验才能持久化。
|
||||
|
||||
## 准备与事故信息采集
|
||||
|
||||
1. 新 session 初始为 `1001`,提示撤离到安全区域、开启双闪、放置警告牌。
|
||||
2. `【开始】` 或 `【继续办理】` 后进入 `1002`。
|
||||
3. 采集顺序:
|
||||
- 事故经过;
|
||||
- 是否有人伤;
|
||||
- 是否涉及非机动车/摩托车/自行车;
|
||||
- 事故时间并校验不能晚于当前时间;
|
||||
- 是否仍在现场;
|
||||
- 机动车数量。
|
||||
4. 用户提前提供的字段用于填槽,但进入下一项前应做封闭式确认。
|
||||
5. 一辆机动车、无人伤且不涉及非机动车/行人:进入 `2000`。
|
||||
6. 两辆机动车、无人伤且不涉及非机动车/行人:进入 `2010`。
|
||||
7. 三辆及以上,或涉及非机动车/行人,或有人伤:进入 `0003`。
|
||||
|
||||
## 单车拍照
|
||||
|
||||
严格顺序:
|
||||
|
||||
```text
|
||||
2000 车前/车牌
|
||||
-> 2001 车辆碰撞部位
|
||||
-> 2002 被撞物品
|
||||
-> 2003 本人正面
|
||||
-> 2004 确认或纠正车牌
|
||||
-> 2005 确认车损位置
|
||||
-> 3001 单车信息确认
|
||||
```
|
||||
|
||||
- `2000`–`2003` 只有 `PhotoCompletedEvent` 可以正常推进;其他普通输入重复当前固定指令。
|
||||
- `2004` 肯定车牌或提供完整新车牌后进入 `2005`;仅否定但不提供号码时停留并追问。
|
||||
- `2005` 获得有效车损位置后进入 `3001`;连续两次无效回答进入 `0002`。
|
||||
- 任意单车照片状态收到拍照失败事件立即进入 `0005`。
|
||||
|
||||
## 双车拍照
|
||||
|
||||
严格顺序:
|
||||
|
||||
```text
|
||||
2010 第一辆车侧前方/车牌
|
||||
-> 2011 第一辆车碰撞部位
|
||||
-> 2012 第二辆车碰撞部位
|
||||
-> 2013 第二辆车侧后方/车牌
|
||||
-> 2014 另一方驾驶人正面
|
||||
-> 2015 本人正面
|
||||
-> 2016 确认或纠正车牌
|
||||
-> 3002 双车信息确认
|
||||
```
|
||||
|
||||
- `2010`–`2015` 只有 `PhotoCompletedEvent` 可以正常推进。
|
||||
- `2016` 肯定或提供完整新车牌后进入 `3002`;无关或不完整回答停留,连续两次失败进入 `0002`。
|
||||
- 任意双车照片状态收到拍照失败事件立即进入 `0005`。
|
||||
|
||||
## 当事人信息确认
|
||||
|
||||
### 单车 `3001`
|
||||
|
||||
依次确认:
|
||||
|
||||
1. 是否为对应车辆车主/驾驶人;
|
||||
2. 姓名;
|
||||
3. 身份证后四位;不一致时采集完整号码并二次确认;
|
||||
4. 手机号后四位;不一致时采集完整号码并二次确认;
|
||||
5. 完成后进入 `0000`。
|
||||
|
||||
### 双车 `3002`
|
||||
|
||||
先完成第一位驾驶人上述信息,再要求将电话交给第二位驾驶人,重复相同步骤。第二位完成后进入 `0000`。
|
||||
|
||||
身份证和手机号允许分段输入;中间态只保存已接收片段,不应把未完成号码写入已确认业务字段。日志、trace 和黄金数据不得包含真实号码。
|
||||
|
||||
## 表单更新
|
||||
|
||||
- `needFormUpdate=false` 时无需返回 `formUpdate`。
|
||||
- `needFormUpdate=true` 时只返回本轮相对当前表单发生变化的字段。
|
||||
- LLM 提取结果必须经过 `field-registry.json` 白名单和类型校验。
|
||||
- patch 之外的原字段保持不变。
|
||||
- 不允许 LLM 更新 phase、状态码、计数器或版本号。
|
||||
|
||||
## 当前实现与目标规则的差异
|
||||
|
||||
- FastGPT Prompt 承担了多数计数和迁移逻辑,服务端未校验迁移合法性。
|
||||
- 当前 prefix 正则不是开头锚定且接受任意位数字。
|
||||
- 当前 `/set_info` 可写任意 key。
|
||||
- 当前 `/get_info` 和 `/set_info` 通过辅助 LLM 对话访问状态。
|
||||
- 当前日志会记录完整用户输入、回复和表单。
|
||||
|
||||
这些差异是 Phase 1–8 的明确改造项,不能被解释为本基线认可的目标行为。
|
||||
37
docs/domain/event-mapping.md
Normal file
37
docs/domain/event-mapping.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# 历史输入到结构化事件的映射
|
||||
|
||||
API 兼容层可以继续接收历史字符串,但进入 LangGraph 前必须转换为以下事件。图节点不得再通过自然语言字符串判断系统事件。
|
||||
|
||||
| 历史输入 | 结构化事件 | 必要字段 | 确定性效果 |
|
||||
|---|---|---|---|
|
||||
| 普通用户文本 | `UserMessageEvent` | `event_id`, `text`, `need_form_update`, `use_text_chunk` | 进入当前阶段处理;自然语言轮次通常调用一次 LLM |
|
||||
| `【开始】` | `SessionStartedEvent` | `event_id` | `1001 → 1002`,开始事故描述采集 |
|
||||
| `【继续办理】` | `SessionStartedEvent` | `event_id` | 与 `【开始】` 相同 |
|
||||
| `【拍摄完成】` | `PhotoCompletedEvent` | `event_id`, `photo_step` | 仅按单车/双车严格顺序推进一步,0 次 LLM |
|
||||
| `【客户端连续3次拍摄识别失败:原因】` | `PhotoRecognitionFailedEvent` | `event_id`, `reason` | 任意照片阶段立即进入 `0005`,0 次 LLM |
|
||||
| `【用户无回复】` | `NoResponseEvent` | `event_id` | 第一次重复唤醒;连续第二次进入 `0004` |
|
||||
| `【用户未回复】` | `NoResponseEvent` | `event_id` | 历史别名,效果同上 |
|
||||
| `/set_info` 请求 | `SetInfoEvent` | `event_id`, `key`, `value` | 校验白名单/类型后幂等更新,不调用 LLM |
|
||||
|
||||
## 优先级
|
||||
|
||||
同一轮只允许一个输入事件。事件处理优先级为:
|
||||
|
||||
```text
|
||||
显式人工请求
|
||||
> 连续拍照失败
|
||||
> 明确人伤/复杂情况
|
||||
> 无回复
|
||||
> 拍照完成
|
||||
> 外部字段更新
|
||||
> 普通用户文本
|
||||
```
|
||||
|
||||
显式人工请求来自普通文本时允许确定性关键词 gate 先处理;语义模糊的人伤内容交由同一轮阶段 LLM 判断,但候选状态仍需迁移校验。
|
||||
|
||||
## 计数重置
|
||||
|
||||
- 收到有效用户回答后,`no_response_count` 清零。
|
||||
- 当前问题得到有效答案后,对应 `clarification_counts[question_id]` 清零。
|
||||
- 非连续的无回复不能累计到 `0004`。
|
||||
- 客户端已经负责累计三次拍照识别失败;服务端收到一次结构化失败事件就进入 `0005`,不得再次累计三次。
|
||||
51
docs/domain/field-registry.json
Normal file
51
docs/domain/field-registry.json
Normal file
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"version": "2026-07-26",
|
||||
"groups": {
|
||||
"acdinfo": ["ywrysw", "ywfjdc", "ywmtc", "bjrjs", "sgfssj", "sfsgxc", "jdcsl", "sgyy"],
|
||||
"acdhuman1": ["xm1", "hpzl1", "hphm1", "sfzmhm1", "sfzmwh1", "sjhm1", "sjwh1", "csbw1"],
|
||||
"acdhuman2": ["xm2", "hpzl2", "hphm2", "sfzmhm2", "sfzmwh2", "sjhm2", "sjwh2", "csbw2"]
|
||||
},
|
||||
"fields": [
|
||||
{"key": "ywrysw", "group": "acdinfo", "type": "boolean", "description": "是否有人员伤亡", "sensitive": false, "external_write": true},
|
||||
{"key": "ywfjdc", "group": "acdinfo", "type": "boolean", "description": "是否涉及非机动车", "sensitive": false, "external_write": true},
|
||||
{"key": "ywmtc", "group": "acdinfo", "type": "boolean", "description": "是否涉及摩托车", "sensitive": false, "external_write": true},
|
||||
{"key": "bjrjs", "group": "acdinfo", "type": "string", "description": "报警人角色/描述", "sensitive": false, "external_write": true},
|
||||
{"key": "sgfssj", "group": "acdinfo", "type": "datetime", "description": "事故发生时间", "sensitive": false, "external_write": true},
|
||||
{"key": "sfsgxc", "group": "acdinfo", "type": "boolean", "description": "是否在事故现场", "sensitive": false, "external_write": true},
|
||||
{"key": "jdcsl", "group": "acdinfo", "type": "integer", "description": "事故机动车数量", "sensitive": false, "external_write": true},
|
||||
{"key": "sgyy", "group": "acdinfo", "type": "string", "description": "事故原因/经过", "sensitive": false, "external_write": true},
|
||||
{"key": "xm1", "group": "acdhuman1", "type": "string", "description": "驾驶员1姓名", "sensitive": true, "external_write": true},
|
||||
{"key": "hpzl1", "group": "acdhuman1", "type": "string", "description": "驾驶员1号牌种类", "sensitive": false, "external_write": true},
|
||||
{"key": "hphm1", "group": "acdhuman1", "type": "license_plate", "description": "驾驶员1车牌号", "sensitive": true, "external_write": true},
|
||||
{"key": "sfzmhm1", "group": "acdhuman1", "type": "national_id", "description": "驾驶员1身份证号码", "sensitive": true, "external_write": true},
|
||||
{"key": "sfzmwh1", "group": "acdhuman1", "type": "string", "description": "驾驶员1身份证尾号兼容字段", "sensitive": true, "external_write": false},
|
||||
{"key": "sjhm1", "group": "acdhuman1", "type": "phone", "description": "驾驶员1手机号码", "sensitive": true, "external_write": true},
|
||||
{"key": "sjwh1", "group": "acdhuman1", "type": "string", "description": "驾驶员1手机号尾号兼容字段", "sensitive": true, "external_write": false},
|
||||
{"key": "csbw1", "group": "acdhuman1", "type": "string", "description": "驾驶员1车辆车损部位", "sensitive": false, "external_write": true},
|
||||
{"key": "xm2", "group": "acdhuman2", "type": "string", "description": "驾驶员2姓名", "sensitive": true, "external_write": true},
|
||||
{"key": "hpzl2", "group": "acdhuman2", "type": "string", "description": "驾驶员2号牌种类", "sensitive": false, "external_write": true},
|
||||
{"key": "hphm2", "group": "acdhuman2", "type": "license_plate", "description": "驾驶员2车牌号", "sensitive": true, "external_write": true},
|
||||
{"key": "sfzmhm2", "group": "acdhuman2", "type": "national_id", "description": "驾驶员2身份证号码", "sensitive": true, "external_write": true},
|
||||
{"key": "sfzmwh2", "group": "acdhuman2", "type": "string", "description": "驾驶员2身份证尾号兼容字段", "sensitive": true, "external_write": false},
|
||||
{"key": "sjhm2", "group": "acdhuman2", "type": "phone", "description": "驾驶员2手机号码", "sensitive": true, "external_write": true},
|
||||
{"key": "sjwh2", "group": "acdhuman2", "type": "string", "description": "驾驶员2手机号尾号兼容字段", "sensitive": true, "external_write": false},
|
||||
{"key": "csbw2", "group": "acdhuman2", "type": "string", "description": "驾驶员2车辆车损部位", "sensitive": false, "external_write": true}
|
||||
],
|
||||
"read_keys": ["all", "acdinfo", "acdhuman1", "acdhuman2"],
|
||||
"internal_write_denylist": [
|
||||
"phase",
|
||||
"stage_code",
|
||||
"clarification_counts",
|
||||
"no_response_count",
|
||||
"handoff_reason",
|
||||
"state_version",
|
||||
"messages",
|
||||
"input_event",
|
||||
"photo_step"
|
||||
],
|
||||
"legacy_serialization": {
|
||||
"get_info_value": "json_encoded_string",
|
||||
"boolean": {"true": "1", "false": "0"},
|
||||
"missing": ""
|
||||
}
|
||||
}
|
||||
40
docs/domain/stage-codes.json
Normal file
40
docs/domain/stage-codes.json
Normal file
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"version": "2026-07-26",
|
||||
"sources": [
|
||||
"src/api/endpoints.py",
|
||||
"docs/视频快处智能信息采集机器人交互接口文档V1.0.9.docx",
|
||||
"workflow/20260726/事故信息采集20260726.json",
|
||||
"prompts/20260723/单车拍照.txt",
|
||||
"prompts/20260723/双车拍照.txt"
|
||||
],
|
||||
"codes": [
|
||||
{"code": "0000", "name": "通话结束", "phase": "complete", "terminal": true, "external_code": "0000", "producer": "workflow"},
|
||||
{"code": "0001", "name": "主动转人工", "phase": "handoff", "terminal": true, "external_code": "0001", "producer": "deterministic_or_llm"},
|
||||
{"code": "0002", "name": "语义连续无法识别", "phase": "handoff", "terminal": true, "external_code": "0002", "producer": "deterministic"},
|
||||
{"code": "0003", "name": "人伤或复杂情况转人工", "phase": "handoff", "terminal": true, "external_code": "0003", "producer": "deterministic_or_llm"},
|
||||
{"code": "0004", "name": "长时间无回复", "phase": "handoff", "terminal": true, "external_code": "0004", "producer": "deterministic"},
|
||||
{"code": "0005", "name": "连续拍照识别失败", "phase": "handoff", "terminal": true, "external_code": "0005", "producer": "deterministic"},
|
||||
{"code": "1001", "name": "未准备好通话", "phase": "ready_gate", "terminal": false, "external_code": "1001", "producer": "deterministic"},
|
||||
{"code": "1002", "name": "事故信息采集中", "phase": "collection", "terminal": false, "external_code": "1002", "producer": "llm"},
|
||||
{"code": "2000", "name": "单车车前照片", "phase": "single_photo", "terminal": false, "external_code": "2000", "producer": "deterministic_or_llm"},
|
||||
{"code": "2001", "name": "单车碰撞部位照片", "phase": "single_photo", "terminal": false, "external_code": "2001", "producer": "deterministic"},
|
||||
{"code": "2002", "name": "被撞物品照片", "phase": "single_photo", "terminal": false, "external_code": "2002", "producer": "deterministic"},
|
||||
{"code": "2003", "name": "本人正面照片", "phase": "single_photo", "terminal": false, "external_code": "2003", "producer": "deterministic"},
|
||||
{"code": "2004", "name": "确认单车车牌", "phase": "single_photo", "terminal": false, "external_code": "2004", "producer": "llm"},
|
||||
{"code": "2005", "name": "确认单车车损位置", "phase": "single_photo", "terminal": false, "external_code": "2005", "producer": "llm"},
|
||||
{"code": "2010", "name": "第一辆车侧前方照片", "phase": "double_photo", "terminal": false, "external_code": "2010", "producer": "deterministic_or_llm"},
|
||||
{"code": "2011", "name": "第一辆车碰撞部位照片", "phase": "double_photo", "terminal": false, "external_code": "2011", "producer": "deterministic"},
|
||||
{"code": "2012", "name": "第二辆车碰撞部位照片", "phase": "double_photo", "terminal": false, "external_code": "2012", "producer": "deterministic"},
|
||||
{"code": "2013", "name": "第二辆车侧后方车牌照片", "phase": "double_photo", "terminal": false, "external_code": "2013", "producer": "deterministic"},
|
||||
{"code": "2014", "name": "另一方驾驶人正面照片", "phase": "double_photo", "terminal": false, "external_code": "2014", "producer": "deterministic"},
|
||||
{"code": "2015", "name": "本人正面照片", "phase": "double_photo", "terminal": false, "external_code": "2015", "producer": "deterministic"},
|
||||
{"code": "2016", "name": "确认双车车牌", "phase": "double_photo", "terminal": false, "external_code": "2016", "producer": "llm"},
|
||||
{"code": "3001", "name": "单车当事人信息确认", "phase": "single_verification", "terminal": false, "external_code": "1002", "producer": "llm"},
|
||||
{"code": "3002", "name": "双车当事人信息确认", "phase": "double_verification", "terminal": false, "external_code": "1002", "producer": "llm"}
|
||||
],
|
||||
"legacy_aliases": {
|
||||
"2006": "2004",
|
||||
"2017": "2016",
|
||||
"2020": "0002"
|
||||
}
|
||||
}
|
||||
46
docs/domain/stage-transitions.json
Normal file
46
docs/domain/stage-transitions.json
Normal file
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"version": "2026-07-26",
|
||||
"global_transitions": {
|
||||
"explicit_handoff": "0001",
|
||||
"injury_or_complex": "0003",
|
||||
"two_invalid_clarifications": "0002",
|
||||
"two_no_response_events": "0004",
|
||||
"photo_recognition_failed_event": "0005"
|
||||
},
|
||||
"allowed": {
|
||||
"1001": ["1001", "1002", "0001", "0003", "0004"],
|
||||
"1002": ["1002", "2000", "2010", "0000", "0001", "0002", "0003", "0004"],
|
||||
"2000": ["2000", "2001", "0001", "0003", "0004", "0005"],
|
||||
"2001": ["2001", "2002", "0001", "0003", "0004", "0005"],
|
||||
"2002": ["2002", "2003", "0001", "0003", "0004", "0005"],
|
||||
"2003": ["2003", "2004", "0001", "0003", "0004", "0005"],
|
||||
"2004": ["2004", "2005", "0001", "0002", "0003", "0004", "0005"],
|
||||
"2005": ["2005", "3001", "0001", "0002", "0003", "0004", "0005"],
|
||||
"2010": ["2010", "2011", "0001", "0003", "0004", "0005"],
|
||||
"2011": ["2011", "2012", "0001", "0003", "0004", "0005"],
|
||||
"2012": ["2012", "2013", "0001", "0003", "0004", "0005"],
|
||||
"2013": ["2013", "2014", "0001", "0003", "0004", "0005"],
|
||||
"2014": ["2014", "2015", "0001", "0003", "0004", "0005"],
|
||||
"2015": ["2015", "2016", "0001", "0003", "0004", "0005"],
|
||||
"2016": ["2016", "3002", "0001", "0002", "0003", "0004", "0005"],
|
||||
"3001": ["3001", "0000", "0001", "0002", "0003", "0004"],
|
||||
"3002": ["3002", "0000", "0001", "0002", "0003", "0004"],
|
||||
"0000": [],
|
||||
"0001": [],
|
||||
"0002": [],
|
||||
"0003": [],
|
||||
"0004": [],
|
||||
"0005": []
|
||||
},
|
||||
"photo_sequences": {
|
||||
"single": ["2000", "2001", "2002", "2003", "2004", "2005", "3001"],
|
||||
"double": ["2010", "2011", "2012", "2013", "2014", "2015", "2016", "3002"]
|
||||
},
|
||||
"notes": [
|
||||
"同状态迁移表示无效输入后重复当前问题或固定指令。",
|
||||
"3001 和 3002 是内部信息确认状态,对外兼容码统一为 1002。",
|
||||
"终止状态不允许继续迁移;新请求必须创建或显式重置 session。",
|
||||
"0003 在照片和信息确认阶段仍保留全局安全优先级。",
|
||||
"迁移矩阵是 LangGraph 实现的权威基线;FastGPT 当前不会在服务端执行该校验。"
|
||||
]
|
||||
}
|
||||
1421
docs/langgraph-backend-migration-plan.md
Normal file
1421
docs/langgraph-backend-migration-plan.md
Normal file
File diff suppressed because it is too large
Load Diff
82
docs/langgraph-minimal-slice.md
Normal file
82
docs/langgraph-minimal-slice.md
Normal file
@@ -0,0 +1,82 @@
|
||||
# LangGraph 最小开发纵切
|
||||
|
||||
> 状态:仅用于本地开发和测试
|
||||
> 日期:2026-07-27
|
||||
|
||||
## 启用
|
||||
|
||||
在被 Git 忽略的 `.env.local` 中配置:
|
||||
|
||||
```text
|
||||
ZNJJ_ENVIRONMENT=development
|
||||
AGENT_BACKEND=langgraph
|
||||
LANGGRAPH_CHECKPOINTER=memory
|
||||
|
||||
LLM_API_KEY=...
|
||||
LLM_BASE_URL=https://api.openai.com/v1
|
||||
LLM_MODEL=...
|
||||
LLM_TIMEOUT_SECONDS=60
|
||||
LLM_MAX_RETRIES=2
|
||||
```
|
||||
|
||||
不设置 `AGENT_BACKEND` 时默认使用 FastGPT。
|
||||
|
||||
## 当前调用链
|
||||
|
||||
```text
|
||||
/chat
|
||||
-> ChatBackend
|
||||
-> LangGraphBackend
|
||||
-> StateGraph
|
||||
-> generate_response
|
||||
-> OpenAI-compatible LLM
|
||||
-> ChatResult
|
||||
-> 现有非流式/SSE API Adapter
|
||||
```
|
||||
|
||||
`InMemorySaver` 使用 `sessionId` 作为 `thread_id`,同一进程内同一 session
|
||||
可以恢复 `turn_count` 等图状态;不同 session 相互隔离。
|
||||
|
||||
## 当前已有能力
|
||||
|
||||
- FastGPT/LangGraph 后端配置切换。
|
||||
- 最小 Pydantic Settings 和条件化启动校验。
|
||||
- 实际使用 LangGraph `StateGraph`。
|
||||
- 开发/测试使用 `InMemorySaver`。
|
||||
- OpenAI-compatible LLM 节点。
|
||||
- 非流式 `/chat` 适配。
|
||||
- 通过完整结果桥接现有 SSE;`formUpdate` 先于文本发送。
|
||||
- Fake LLM 下的多轮 thread 隔离测试。
|
||||
|
||||
## 明确限制
|
||||
|
||||
当前图只有一个模型节点,目的是尽早建立可执行骨架。以下尚未实现:
|
||||
|
||||
- 领域状态、输入事件和确定性路由节点;
|
||||
- 状态码枚举及迁移合法性校验;
|
||||
- 单车/双车拍照状态机;
|
||||
- 字段提取、验证和真实 `formUpdate`;
|
||||
- 模型 token 级流式输出;
|
||||
- PostgreSQL checkpointer;
|
||||
- session 并发、版本和幂等;
|
||||
- `/set_info`、`/get_info` 的业务状态迁移。
|
||||
|
||||
因此:
|
||||
|
||||
- `LANGGRAPH_CHECKPOINTER=postgres` 当前会启动失败;
|
||||
- staging/production 禁止使用 memory checkpointer;
|
||||
- `/set_info`、`/get_info`、`/delete_session` 暂时仍需要 FastGPT 配置;
|
||||
- 不得把当前 LangGraph backend 接入生产流量。
|
||||
|
||||
## 下一步
|
||||
|
||||
直接在现有图中加入 Phase 5 的纯确定性骨架:
|
||||
|
||||
1. `AccidentState`、输入事件和状态枚举;
|
||||
2. state prefix parser 与迁移 validator;
|
||||
3. `normalize_input`、`route_phase`;
|
||||
4. 单车/双车拍照事件推进;
|
||||
5. 无回复和澄清计数;
|
||||
6. 对应参数化 Graph 测试。
|
||||
|
||||
这些节点完成后再接 PostgreSQL 和业务状态 repository。
|
||||
376
docs/voice-websocket.md
Normal file
376
docs/voice-websocket.md
Normal file
@@ -0,0 +1,376 @@
|
||||
# Voice WebSocket 使用说明
|
||||
|
||||
基于 `src/voice` 产品语音管线与 `static/voice-demo` 浏览器示例整理。
|
||||
|
||||
## 概览
|
||||
|
||||
| 项目 | 说明 |
|
||||
|------|------|
|
||||
| WebSocket 路径 | `/ws-product` |
|
||||
| 协议标识 | `va.ws.v1`(JSON + base64;音频上行也支持二进制 PCM) |
|
||||
| 默认音频 | PCM16 小端(`pcm_s16le`)、16 kHz、单声道 |
|
||||
| 会话 ID | 连接 URL 查询参数 `chatId` 或 `chat_id`;未传时服务端自动生成 |
|
||||
| 健康检查 | `GET /voice/health` |
|
||||
| 浏览器 Demo | 默认挂载于 `/voice-demo`(由 voice 配置 `server.serve_webpage` 控制) |
|
||||
|
||||
完整 URL 示例:
|
||||
|
||||
```
|
||||
ws://127.0.0.1:8000/ws-product?chatId=voice_abc123
|
||||
wss://your-host/ws-product?chatId=voice_abc123
|
||||
```
|
||||
|
||||
## 连接流程
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client
|
||||
participant Server
|
||||
|
||||
Client->>Server: WebSocket connect (?chatId=...)
|
||||
Server-->>Client: 101 Switching Protocols
|
||||
Client->>Server: session.start (JSON)
|
||||
Note over Client,Server: 可选:固定开场白 / FastGPT opener / LLM 生成问候
|
||||
loop 会话中
|
||||
Client->>Server: input.audio (binary 或 JSON)
|
||||
Client->>Server: input.text
|
||||
Server-->>Client: input.transcript.* / response.text.* / response.audio.*
|
||||
Server-->>Client: response.state(若启用状态标签)
|
||||
end
|
||||
Client->>Server: session.stop
|
||||
Server-->>Client: WebSocket close
|
||||
```
|
||||
|
||||
推荐顺序(与 `voice-demo/app.js` 一致):
|
||||
|
||||
1. 建立 WebSocket 连接(建议 `binaryType = "arraybuffer"`)。
|
||||
2. 连接成功后立即发送 `session.start`。
|
||||
3. 开始推送麦克风音频(二进制帧或 `input.audio` JSON)。
|
||||
4. 处理服务端 JSON 事件(文本、转写、TTS 音频等)。
|
||||
5. 断开前发送 `session.stop`,再关闭连接。
|
||||
|
||||
## 消息信封
|
||||
|
||||
除二进制音频外,所有消息均为 UTF-8 JSON 对象。服务端下发事件统一包含:
|
||||
|
||||
| 字段 | 类型 | 说明 |
|
||||
|------|------|------|
|
||||
| `type` | string | 事件类型 |
|
||||
| `protocol` | string | 固定为 `va.ws.v1` |
|
||||
| `seq` | number | 单调递增序号(仅服务端事件) |
|
||||
|
||||
## 客户端 → 服务端
|
||||
|
||||
### `session.start`
|
||||
|
||||
开始会话,必须在发送音频或文本输入之前调用。
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "session.start",
|
||||
"protocol": "va.ws.v1",
|
||||
"chatId": "voice_abc123",
|
||||
"audio": {
|
||||
"encoding": "pcm_s16le",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`chatId` 也可写作 `chat_id`。若省略,服务端使用 URL 查询参数或自动生成 ID。
|
||||
|
||||
### `session.stop`
|
||||
|
||||
正常结束会话。
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "session.stop",
|
||||
"reason": "client_disconnect"
|
||||
}
|
||||
```
|
||||
|
||||
### `input.audio`(JSON 形式)
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "input.audio",
|
||||
"audio": "<base64 PCM16>",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1
|
||||
}
|
||||
```
|
||||
|
||||
`audio` 字段也可命名为 `data`。`sample_rate` / `channels` 可省略,默认与服务端配置一致。
|
||||
|
||||
### 二进制音频(推荐)
|
||||
|
||||
直接发送 **原始 PCM16 小端** 字节流,无需 JSON 包装。`voice-demo` 通过 AudioWorklet 每 20 ms 发送一帧(16 kHz 单声道下约 640 字节/帧)。
|
||||
|
||||
服务端同时接受 JSON 与二进制两种上行格式。
|
||||
|
||||
### `input.text`
|
||||
|
||||
发送文本回合;默认会打断当前 bot 回复(`interrupt: true`)。
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "input.text",
|
||||
"text": "你好,我想报案",
|
||||
"interrupt": true
|
||||
}
|
||||
```
|
||||
|
||||
注意:文本输入**不会**以 `input.transcript.final` 回显,客户端需自行在 UI 中展示用户消息(Demo 即如此处理)。Demo 的相机步骤通过发送 `input.text`(如 `【拍摄完成】`)完成,不上传图片帧。
|
||||
|
||||
## 服务端 → 客户端
|
||||
|
||||
### 用户语音转写
|
||||
|
||||
| 事件 | 说明 |
|
||||
|------|------|
|
||||
| `input.transcript.interim` | ASR 中间结果(流式识别过程中) |
|
||||
| `input.transcript.final` | 用户一句话结束后的最终转写 |
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "input.transcript.final",
|
||||
"protocol": "va.ws.v1",
|
||||
"seq": 12,
|
||||
"text": "发生了交通事故",
|
||||
"user_id": "product-user",
|
||||
"timestamp": "2026-06-01T10:00:00.000Z"
|
||||
}
|
||||
```
|
||||
|
||||
### 助手文本流
|
||||
|
||||
文本通常**早于**对应 TTS 音频到达,便于客户端先渲染字幕。
|
||||
|
||||
| 事件 | 说明 |
|
||||
|------|------|
|
||||
| `response.text.started` | 新一轮助手回复开始 |
|
||||
| `response.text.delta` | 流式文本片段 |
|
||||
| `response.text.final` | 本轮文本结束;`interrupted: true` 表示被打断 |
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "response.text.delta",
|
||||
"protocol": "va.ws.v1",
|
||||
"seq": 20,
|
||||
"text": "您好,"
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "response.text.final",
|
||||
"protocol": "va.ws.v1",
|
||||
"seq": 45,
|
||||
"text": "您好,请问发生了什么情况?",
|
||||
"interrupted": false
|
||||
}
|
||||
```
|
||||
|
||||
### 助手语音(TTS)
|
||||
|
||||
| 事件 | 说明 |
|
||||
|------|------|
|
||||
| `response.audio.started` | Bot 开始说话 |
|
||||
| `response.audio.delta` | PCM16 音频块(base64) |
|
||||
| `response.audio.stopped` | Bot 说完 |
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "response.audio.delta",
|
||||
"protocol": "va.ws.v1",
|
||||
"seq": 30,
|
||||
"audio": "<base64 PCM16>",
|
||||
"bytes": 640,
|
||||
"sample_rate": 16000,
|
||||
"channels": 1
|
||||
}
|
||||
```
|
||||
|
||||
客户端应将各 `delta` 块按序解码并无缝拼接播放(Demo 使用 Web Audio `AudioContext` 调度)。
|
||||
|
||||
### 助手状态(可选)
|
||||
|
||||
当 voice 配置启用 `agent.response_state` 时,LLM 输出开头的 `<state>...</state>` 标签会被剥离,并单独下发:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "response.state",
|
||||
"protocol": "va.ws.v1",
|
||||
"seq": 18,
|
||||
"state": "2000"
|
||||
}
|
||||
```
|
||||
|
||||
Demo 根据状态码展示拍照引导(如 `2000`–`2015` 等车险场景状态)。
|
||||
|
||||
## 音频参数
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| 采样率 | 16000 Hz | 配置项 `audio.sample_rate_hz` |
|
||||
| 声道 | 1(mono) | 配置项 `audio.channels` |
|
||||
| 帧长 | 20 ms | 配置项 `audio.frame_ms`;每帧 640 字节 |
|
||||
| 编码 | PCM signed 16-bit LE | 小端有符号 16 位整数 |
|
||||
|
||||
## 会话与打断行为
|
||||
|
||||
- **chatId**:同一 ID 用于 LLM(如 FastGPT)多轮上下文;连接时可写在 URL 或 `session.start` 中。
|
||||
- **语音回合**:VAD + 静音超时判定用户说完;说完后触发 STT 最终转写与 LLM。
|
||||
- **打断**:用户说话或 `input.text`(`interrupt: true`)可打断 bot;被打断的助手文本在 `response.text.final` 中带 `interrupted: true`。
|
||||
- **空闲超时**:长时间无活动会断开(`session.inactivity_timeout_sec`,默认 60 秒);可配置空闲提示语。
|
||||
- **开场白**:由 `agent.greeting_mode` 控制(`fixed` / `fastgpt_opener` / `generated` 等)。
|
||||
|
||||
## 浏览器 Demo 参考实现
|
||||
|
||||
Demo 位于 `static/voice-demo/`,无构建步骤,核心文件:
|
||||
|
||||
| 文件 | 职责 |
|
||||
|------|------|
|
||||
| `app.js` | WebSocket 连接、事件处理、聊天 UI、TTS 播放 |
|
||||
| `pcm-recorder.worklet.js` | 麦克风采集、重采样至 16 kHz、20 ms 二进制帧 |
|
||||
| `index.html` / `styles.css` | 页面与样式 |
|
||||
|
||||
### 启动 Demo
|
||||
|
||||
1. 启动 API 服务并加载 voice 配置(环境变量 `VOICE_CONFIG` 指向 JSON,默认 `config/voice.json`)。
|
||||
2. 浏览器打开 `http://127.0.0.1:8000/voice-demo/`(挂载路径见配置 `server.webpage_mount`)。
|
||||
3. 点击 **Connect** → **Enable mic** 开始对话。
|
||||
|
||||
### Demo 关键实现要点
|
||||
|
||||
**连接与握手**
|
||||
|
||||
```javascript
|
||||
const ws = new WebSocket("ws://127.0.0.1:8000/ws-product?chatId=voice_xxx");
|
||||
ws.binaryType = "arraybuffer";
|
||||
|
||||
ws.onopen = () => {
|
||||
ws.send(JSON.stringify({
|
||||
type: "session.start",
|
||||
protocol: "va.ws.v1",
|
||||
chatId: "voice_xxx",
|
||||
audio: { encoding: "pcm_s16le", sample_rate: 16000, channels: 1 },
|
||||
}));
|
||||
};
|
||||
```
|
||||
|
||||
**发送麦克风(二进制,与 Demo 一致)**
|
||||
|
||||
```javascript
|
||||
// AudioWorklet 每 20ms postMessage { type: "frame", buffer: ArrayBuffer }
|
||||
recorderNode.port.onmessage = (event) => {
|
||||
if (event.data?.type === "frame") {
|
||||
ws.send(event.data.buffer);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
**播放 TTS**
|
||||
|
||||
```javascript
|
||||
function decodeBase64ToInt16(b64) {
|
||||
const binary = atob(b64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
||||
return new Int16Array(bytes.buffer);
|
||||
}
|
||||
|
||||
// 收到 response.audio.delta 后,将 Int16 转为 Float32 并调度到 AudioContext
|
||||
```
|
||||
|
||||
**发送文本并打断**
|
||||
|
||||
```javascript
|
||||
ws.send(JSON.stringify({
|
||||
type: "input.text",
|
||||
text: "【拍摄完成】",
|
||||
interrupt: true,
|
||||
}));
|
||||
// 客户端应停止本地 TTS 播放队列;服务端会发 response.text.final(interrupted=true)
|
||||
```
|
||||
|
||||
### 跨域静态页
|
||||
|
||||
若 Demo 托管在其他端口,需在 voice 配置中设置 `server.cors_origins`,并将 WebSocket URL 指向 API 主机。
|
||||
|
||||
> 浏览器 `getUserMedia` 需要安全上下文:`https://` 或 `http://localhost` 可用;其他 HTTP 源需改用 HTTPS + `wss://`。
|
||||
|
||||
## 最小客户端示例(伪代码)
|
||||
|
||||
```javascript
|
||||
const ws = new WebSocket(`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/ws-product?chatId=voice_demo_1`);
|
||||
ws.binaryType = "arraybuffer";
|
||||
|
||||
ws.onopen = () => {
|
||||
ws.send(JSON.stringify({
|
||||
type: "session.start",
|
||||
protocol: "va.ws.v1",
|
||||
audio: { encoding: "pcm_s16le", sample_rate: 16000, channels: 1 },
|
||||
}));
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
if (typeof event.data !== "string") return;
|
||||
const msg = JSON.parse(event.data);
|
||||
switch (msg.type) {
|
||||
case "input.transcript.final":
|
||||
console.log("User:", msg.text);
|
||||
break;
|
||||
case "response.text.delta":
|
||||
process.stdout?.write?.(msg.text); // 流式打印助手文本
|
||||
break;
|
||||
case "response.audio.delta":
|
||||
playPcm16(decodeBase64(msg.audio));
|
||||
break;
|
||||
case "response.state":
|
||||
console.log("State:", msg.state);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
function disconnect() {
|
||||
ws.send(JSON.stringify({ type: "session.stop", reason: "done" }));
|
||||
ws.close(1000, "done");
|
||||
}
|
||||
```
|
||||
|
||||
## 健康检查响应示例
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/voice/health
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "healthy",
|
||||
"config": "/path/to/config/voice.json",
|
||||
"protocols": {
|
||||
"/ws-product": "va.ws.v1.json_base64"
|
||||
},
|
||||
"features": {
|
||||
"product_text_input": true,
|
||||
"product_text_interrupt": true
|
||||
},
|
||||
"demo": "/voice-demo",
|
||||
"llm_provider": "fastgpt",
|
||||
"stt_provider": "xfyun",
|
||||
"tts_provider": "xfyun"
|
||||
}
|
||||
```
|
||||
|
||||
## 常见问题
|
||||
|
||||
| 现象 | 可能原因 |
|
||||
|------|----------|
|
||||
| 连接后立即断开 | 未发送 `session.start`;或超过 inactivity 超时 |
|
||||
| 无 bot 语音 | 未处理 `response.audio.delta`;AudioContext 未在用户手势后 resume |
|
||||
| 回声/啸叫 | 建议使用耳机;Demo 已开启浏览器 AEC,但扬声器外放仍可能串音 |
|
||||
| 文本发送无用户气泡 | 设计如此,需客户端本地展示 `input.text` 内容 |
|
||||
| 跨域 WebSocket 失败 | 检查 `cors_origins` 与 `wss` 证书 |
|
||||
BIN
docs/视频快处智能信息采集机器人交互接口文档V1.0.9.docx
Normal file
BIN
docs/视频快处智能信息采集机器人交互接口文档V1.0.9.docx
Normal file
Binary file not shown.
55
examples/nostream_chat.py
Normal file
55
examples/nostream_chat.py
Normal file
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple CLI script to interact with /chat endpoint in non-stream mode.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
API_BASE_URL = "http://localhost:8000"
|
||||
|
||||
|
||||
async def chat(session_id: str, text: str):
|
||||
"""Send a non-streaming chat request."""
|
||||
timestamp = datetime.now().isoformat()
|
||||
|
||||
payload = {
|
||||
"sessionId": session_id,
|
||||
"timeStamp": timestamp,
|
||||
"text": text
|
||||
}
|
||||
|
||||
async with aiohttp.ClientSession() as http_session:
|
||||
async with http_session.post(
|
||||
f"{API_BASE_URL}/chat",
|
||||
json=payload,
|
||||
) as response:
|
||||
data = await response.json()
|
||||
|
||||
print(f"Status: {response.status}")
|
||||
print("-" * 50)
|
||||
print(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
async def main():
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: python nostream_chat.py <session_id> <message>")
|
||||
print("Example: python nostream_chat.py test-session-123 '发生了交通事故'")
|
||||
sys.exit(1)
|
||||
|
||||
session_id = sys.argv[1]
|
||||
text = " ".join(sys.argv[2:])
|
||||
|
||||
print(f"Session ID: {session_id}")
|
||||
print(f"Message: {text}")
|
||||
print("-" * 50)
|
||||
|
||||
await chat(session_id, text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
96
examples/stream_chat.py
Normal file
96
examples/stream_chat.py
Normal file
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Stream Chat CLI - 与 /chat 端点进行流式交互的脚本。
|
||||
|
||||
用法:
|
||||
python stream_chat.py <session_id> <消息>
|
||||
|
||||
示例:
|
||||
python stream_chat.py test-001 "发生了交通事故"
|
||||
|
||||
输出说明:
|
||||
- [stage_code]: 阶段状态码,如 {"nextStageCode": "0000", "nextStage": "结束通话"}
|
||||
- [text_delta]: 流式文本片段
|
||||
- [done]: 流式结束
|
||||
- [error]: 错误信息
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import ssl
|
||||
import aiohttp
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
#API_BASE_URL = "http://localhost:8000"
|
||||
API_BASE_URL = "https://101.89.108.122:8000"
|
||||
|
||||
|
||||
async def stream_chat(session_id: str, text: str):
|
||||
"""Send a streaming chat request."""
|
||||
timestamp = datetime.now().isoformat()
|
||||
|
||||
payload = {
|
||||
"sessionId": session_id,
|
||||
"timeStamp": timestamp,
|
||||
"text": text
|
||||
}
|
||||
|
||||
ssl_ctx = ssl.create_default_context()
|
||||
ssl_ctx.check_hostname = False
|
||||
ssl_ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_ctx)) as http_session:
|
||||
async with http_session.post(
|
||||
f"{API_BASE_URL}/chat",
|
||||
json=payload,
|
||||
params={"stream": "true"},
|
||||
) as response:
|
||||
print(f"Status: {response.status}")
|
||||
print("-" * 50)
|
||||
|
||||
buffer = ""
|
||||
# Use async iterator on response.content
|
||||
async for chunk in response.content.iter_chunked(1024):
|
||||
chunk_str = chunk.decode("utf-8")
|
||||
buffer += chunk_str
|
||||
|
||||
# Process complete SSE messages
|
||||
while "\n\n" in buffer:
|
||||
message, buffer = buffer.split("\n\n", 1)
|
||||
if message.startswith("event: "):
|
||||
event_type = message[7:].split("\n")[0]
|
||||
data_line = message.split("data: ", 1)[-1]
|
||||
try:
|
||||
data = json.loads(data_line)
|
||||
print(f"[{event_type}] {json.dumps(data, ensure_ascii=False)}")
|
||||
except json.JSONDecodeError:
|
||||
print(f"[{event_type}] {data_line}")
|
||||
elif message.startswith("data: "):
|
||||
data_str = message[6:]
|
||||
try:
|
||||
data = json.loads(data_str)
|
||||
print(f"[data] {json.dumps(data, ensure_ascii=False)}")
|
||||
except json.JSONDecodeError:
|
||||
print(f"[data] {data_str}")
|
||||
|
||||
|
||||
async def main():
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: python stream_chat.py <session_id> <message>")
|
||||
print("Example: python stream_chat.py test-session-123 '发生了交通事故'")
|
||||
sys.exit(1)
|
||||
|
||||
session_id = sys.argv[1]
|
||||
text = " ".join(sys.argv[2:])
|
||||
|
||||
print(f"Session ID: {session_id}")
|
||||
print(f"Message: {text}")
|
||||
print("-" * 50)
|
||||
|
||||
await stream_chat(session_id, text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
92
prompts/20251108/单车拍照.txt
Normal file
92
prompts/20251108/单车拍照.txt
Normal file
@@ -0,0 +1,92 @@
|
||||
# 角色
|
||||
你现在进入了**事故现场拍照引导**阶段。你的角色是一名严谨、清晰的AI现场勘查引导员。你的任务是严格按照预设流程引导当事人。在此期间,你的行为模式会根据具体步骤发生改变。
|
||||
|
||||
# 最高原则 (不可违背的绝对指令)
|
||||
|
||||
**全局中断指令 (最高优先级)**: 在对话的**任何阶段**,如果用户的输入包含明确的转人工意图(例如:“转人工”、“找人工”、“人工客服”),你必须**立即停止**当前所有流程,并**只输出** `<state>0001</state>好的,正在为您转接人工客服,请稍候。`
|
||||
|
||||
**严格的输出格式**: 你的所有回复都**必须**以 `<state>状态编码</state>` 开头。
|
||||
|
||||
**混合流程锁定协议 (Hybrid Process Lock Protocol)**:
|
||||
* **严格顺序**: 你必须严格按照 `2000 -> 2001 -> 2002 -> 2003 -> 2004 -> 2005` 的顺序执行任务,绝不可以跳过或打乱。
|
||||
* **行为模式A (针对步骤 2000, 2001, 2002, 2003)**:
|
||||
* **唯一触发**: **只有在接收到系统输入 `【拍摄完成】` 时**,你才能推进到流程的下一步。
|
||||
* **无效输入处理**: 对于**任何非 `【拍摄完成】`** 的输入(包括用户的任何口头回答),你都必须**重复当前状态的完整引导指令**。
|
||||
* **行为模式B (针对步骤 2004, 2005)**:
|
||||
* **触发**: 你**不再等待`【拍摄完成】`指令**。你需要**理解用户的自然语言回答**。
|
||||
* **无效输入处理**: 如果用户的回答与当前问题无关,你必须**重复当前状态的完整指令**。
|
||||
|
||||
#### # 状态编码表 (拍照引导阶段)
|
||||
| 状态编码 | 引导指令 / 任务 |
|
||||
|:---|:---|
|
||||
| **2000** | 请对准车辆前方看清车牌,拍摄车前方照片。 |
|
||||
| **2001** | 请对准车辆碰撞部位拍摄照片。 |
|
||||
| **2002** | 请对准被撞物品拍摄照片。 |
|
||||
| **2003** | 请切换摄像头对准本人拍摄一张正面照片。 |
|
||||
| **2004** | **(确认与纠错合并)** 请您核对车牌号码是否为{{$i2xYvUNWE7Zv.rWawP4BJxuH1$}}。如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。 |
|
||||
| **2005** | 请确认车损位置是在车辆前方、后方还是侧面? |
|
||||
| **3001** | 好的,已记录车损位置。拍摄记录过程已完毕。下面将进入信息确认阶段。 |
|
||||
| **0001** | **(全局转人工)** 好的,正在为您转接人工客服,请稍候。 |
|
||||
| **0002** | **(流程异常转人工)** 抱歉,操作遇到问题,为确保信息准确,将为您转接人工处理。 |
|
||||
---
|
||||
|
||||
### # 任务流程 (体现混合模式)
|
||||
|
||||
**流程起点:步骤 2000**
|
||||
* **你的初始输出**: `<state>2000</state>请对准车辆前方看清车牌,拍摄车前方照片。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2000 -> 步骤 2001 (行为模式A)**
|
||||
* **当系统输入为**: `【拍摄完成】`
|
||||
* **你的输出**: `<state>2001</state>请对准车辆碰撞部位拍摄照片。`
|
||||
* **当用户输入为**: **任何其他内容** (例如: "好的", "拍好了")
|
||||
* **你的输出 (重复指令)**: `<state>2000</state>请对准车辆前方看清车牌,拍摄车前方照片。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2001 -> 步骤 2002 (行为模式A)**
|
||||
* **当系统输入为**: `【拍摄完成】`
|
||||
* **你的输出**: `<state>2002</state>请对准被撞物品拍摄照片。`
|
||||
* **当用户输入为**: **任何其他内容** (例如: "OK", "已经拍了")
|
||||
* **你的输出 (重复指令)**: `<state>2001</state>请对准车辆碰撞部位拍摄照片。`
|
||||
---
|
||||
|
||||
**步骤 2002 -> 步骤 2003 (行为模式A)**
|
||||
* **当系统输入为**: `【拍摄完成】`
|
||||
* **你的输出**: `<state>2003</state>请切换摄像头对准本人拍摄一张正面照片。`
|
||||
* **当用户输入为**: **任何其他内容**
|
||||
* **你的输出 (重复指令)**: `<state>2002</state>请对准被撞物品拍摄照片。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2003 -> 2004 (行为模式A -> B的切换点)**
|
||||
* **当系统输入为**: `【拍摄完成】`
|
||||
* **你的输出**: `<state>2004</state>请您核对车牌号码是否为{{$i2xYvUNWE7Zv.rWawP4BJxuH1$}}。如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。`
|
||||
* **当用户输入为**: **任何其他内容**
|
||||
* **你的输出 (重复指令)**: `<state>2003</state>请切换摄像头对准本人拍摄一张正面照片。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2004 -> 2005 (合并后的确认与纠错流程)**
|
||||
* **(AI当前状态为2004)**
|
||||
* **场景A: 用户确认车牌正确**
|
||||
* **当用户输入为**: "是的", "没错", "对的" (肯定回答)
|
||||
* **你的输出 (继续主流程)**: `<state>2005</state>好的,车牌号已确认。请确认车损位置是在车辆前方、后方还是侧面?`
|
||||
* **场景B: 用户提供新的车牌号**
|
||||
* **当用户输入为**: "不对,车牌号是苏BXXXXX", "我的车牌是沪A12345" (直接提供新信息)
|
||||
* **你的输出 (记录新信息并继续主流程)**: `<state>2005</state>收到,已将正确的车牌号记录为苏BXXXXX。现在,请确认车损位置是在车辆前方、后方还是侧面?`
|
||||
* **场景C: 用户回答不清晰或无关**
|
||||
* **当用户输入为**: "不是这辆车", "不对", "什么意思?" (否定但未提供新信息,或无关回答)
|
||||
* **你的输出 (重复当前问题,引导用户)**: `<state>2004</state>请您核对车牌号码是否为{{$i2xYvUNWE7Zv.rWawP4BJxuH1$}}。如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。`
|
||||
|
||||
---
|
||||
|
||||
**流程终点:步骤 2005 (行为模式B)**
|
||||
* **(AI当前状态为2005)**
|
||||
* **当用户输入为**: "是车子的左前方。" (有效的自然语言)
|
||||
* **你的输出**: `<state>3001</state>好的,已记录车损位置在车辆前方和侧面。拍摄记录过程已完毕。下面将进入信息确认阶段。请问您是否是车牌号为{{$i2xYvUNWE7Zv.rWawP4BJxuH1$}}的车辆的车主`
|
||||
* **当用户输入为**: "什么意思?" (无效的自然语言 - 第一次)
|
||||
* **你的输出 (重复完整指令)**: `<state>2005</state>请确认车损位置是在车辆前方、后方还是侧面?`
|
||||
* **当用户输入仍无效 (第二次)**:
|
||||
* **你的输出 (转接人工)**: `<state>0002</state>抱歉,未能成功记录车损位置,为确保信息准确,将为您转接人工处理。`转接人工处理。`
|
||||
83
prompts/20251108/双车拍照.txt
Normal file
83
prompts/20251108/双车拍照.txt
Normal file
@@ -0,0 +1,83 @@
|
||||
# 角色
|
||||
你现在进入了**双车事故现场拍照与核验**阶段。你的角色是一名严谨、精确的AI现场勘查引导员。你的唯一任务是严格按照预设流程,引导当事人完成一系列拍照和信息确认。你的行为模式会根据具体步骤发生改变。
|
||||
|
||||
# 最高原则 (不可违背的绝对指令)
|
||||
|
||||
**全局中断指令 (最高优先级)**: 在对话的**任何阶段**,如果用户的输入包含明确的转人工意图(例如:“转人工”、“找人工”、“人工客服”),你必须**立即停止**当前所有流程,并**只输出** `<state>0001</state>好的,正在为您转接人工客服,请稍候。`
|
||||
|
||||
**严格的输出格式**: 你的所有回复都**必须**以 `<state>状态编码</state>` 开头。
|
||||
|
||||
**混合流程锁定协议 (Hybrid Process Lock Protocol)**:
|
||||
* **严格顺序**: 你必须严格按照 `2010 -> 2011 -> 2012 -> 2013 -> 2014 -> 2015 -> 2016` 的顺序执行任务。
|
||||
* **行为模式A (拍照监控,适用于步骤 2010 至 2015)**:
|
||||
* **唯一触发**: **只有在接收到 `【拍摄完成】` 指令时**,才能推进到下一步。
|
||||
* **无效输入处理**: 对于**任何非 `【拍摄完成】`** 的输入,你都必须**重复当前状态的完整引导指令**。
|
||||
* **行为模式B (确认与纠错,专门适用于步骤 2016)**:
|
||||
* **任务**: 你的任务是核对一个预设的车牌号,并能直接接收并记录正确的号码。你需要**理解用户的肯定、否定、直接提供新信息或无关回答**。
|
||||
* **触发**: 你不再等待`【拍摄完成】`指令。
|
||||
* **如果回答无关**: **重复`2016`状态的完整指令**进行澄清。
|
||||
|
||||
#### # 状态编码表 (双车拍照引导阶段)
|
||||
| 状态编码 | 引导指令 / 任务 |
|
||||
|:---|:---|
|
||||
| **2010** | 请对准第一辆车的侧前方,看清车牌拍摄。 |
|
||||
| **2011** | 请对准第一辆车碰撞部位拍摄。 |
|
||||
| **2012** | 请对准第二辆车碰撞部位拍摄。 |
|
||||
| **2013** | 请对准第二方车辆侧后方,看清车牌拍摄。 |
|
||||
| **2014** | 请拍摄另一方驾驶人的正面照片。 |
|
||||
| **2015** | 请切换前置摄像头对准本人拍摄一张正面照片。 |
|
||||
| **2016** | **(确认与纠错合并)** 请您核对,事故一方车辆的车牌号是否为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}?如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。 |
|
||||
| **3002** | 好的,车牌号已记录。感谢您的配合,拍摄记录流程已结束,下面将进入信息确认阶段。 请问车牌号为 {{$mbP4DRmqf3qT.rWawP4BJxuH1$}} 的车辆是由您驾驶的吗?|
|
||||
| **0001** | **(全局转人工)** 好的,正在为您转接人工客服,请稍候。 |
|
||||
| **0002** | **(流程异常转人工)** 抱歉,未能成功记录车牌号,为确保信息准确,将为您转接人工处理。 |
|
||||
---
|
||||
|
||||
# 任务流程
|
||||
|
||||
**流程起点:(由主流程转入)**
|
||||
* **你的初始输出**: `<state>2010</state>请对准第一辆车的侧前方,看清车牌拍摄。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2010 -> 2011 (行为模式A)**
|
||||
* **当系统输入为**: `【拍摄完成】`
|
||||
* **你的输出**: `<state>2011</state>请对准第一辆车碰撞部位拍摄。`
|
||||
* **当用户输入为**: **任何其他内容** (例如: "拍了")
|
||||
* **你的输出 (重复指令)**: `<state>2010</state>请对准第一辆车的侧前方,看清车牌拍摄。`
|
||||
|
||||
---
|
||||
|
||||
**... (步骤 2011 至 2014 的逻辑与此完全相同,依次推进) ...**
|
||||
|
||||
---
|
||||
|
||||
**步骤 2014 -> 2015 (行为模式A)**
|
||||
* **当系统输入为**: `【拍摄完成】`
|
||||
* **你的输出**: `<state>2015</state>请切换前置摄像头对准本人拍摄一张正面照片。`
|
||||
* **当系统输入为**: **任何其他内容**
|
||||
* **你的输出 (重复指令)**: `<state>2014</state>请拍摄另一方驾驶人的正面照片。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2015 -> 2016 (行为模式A -> B的切换点)**
|
||||
* **当系统输入为**: `【拍摄完成】`
|
||||
* **你的输出**: `<state>2016</state>请您核对,事故一方车辆的车牌号是否为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}?如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。`
|
||||
* **当用户输入为**: **任何其他内容**
|
||||
* **你的输出 (重复指令)**: `<state>2015</state>请切换前置摄像头对准本人拍摄一张正面照片。`
|
||||
|
||||
---
|
||||
|
||||
**流程终点:步骤 2016 (合并后的确认与纠错流程)**
|
||||
* **(AI当前状态为2016)**
|
||||
* **场景A: 用户确认车牌正确**
|
||||
* **当用户输入为**: "是的", "没错", "对的" (肯定回答)
|
||||
* **你的输出 (结束流程)**: `<state>3002</state>好的,车牌号已确认。感谢您的配合,拍摄记录流程已结束,下面将进入信息确认阶段。`
|
||||
* **场景B: 用户提供新的车牌号**
|
||||
* **当用户输入为**: "不对,车牌号是苏BXXXXX", "我的车牌是沪A12345" (直接提供新信息)
|
||||
* **你的输出 (记录新信息并结束流程)**: `<state>3002</state>收到,已将正确的车牌号记录为【苏BXXXXX】。感谢您的配合,拍摄记录流程已结束,下面将进入信息确认阶段。请问您是否是车牌号为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}的车辆的车主`
|
||||
* **场景C: 用户回答不清晰或无关 (第一次)**
|
||||
* **当用户输入为**: "不是这辆车", "不对", "什么意思?" (否定但未提供新信息,或无关回答)
|
||||
* **你的输出 (重复当前问题,引导用户)**: `<state>2016</state>请您核对,事故一方车辆的车牌号是否为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}?如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。`
|
||||
* **场景D: 用户回答仍不清晰 (第二次)**
|
||||
* **当用户输入仍为**: 无关内容
|
||||
* **你的输出 (转接人工)**: `<state>0002</state>抱歉,未能成功记录车牌号,为确保信息准确,将为您转接人工处理。`
|
||||
284
prompts/20260723/单车拍照.txt
Normal file
284
prompts/20260723/单车拍照.txt
Normal file
@@ -0,0 +1,284 @@
|
||||
# 角色
|
||||
|
||||
你现在进入了**事故现场拍照引导**阶段。你的角色是一名严谨、清晰的AI现场勘查引导员。你的任务是严格按照预设流程引导当事人。在此期间,你的行为模式会根据具体步骤发生改变。
|
||||
|
||||
# 最高原则(不可违背的绝对指令)
|
||||
|
||||
**全局中断指令(最高优先级)**:
|
||||
|
||||
在对话的**任何阶段**,如果用户的输入包含明确的转人工意图(例如:“转人工”、“找人工”、“人工客服”),你必须**立即停止**当前所有流程,并**只输出**:
|
||||
|
||||
`<state>0001</state>好的,正在为您转接人工客服,请稍候。`
|
||||
|
||||
**客户端连续拍摄识别失败指令(高优先级)**:
|
||||
|
||||
在单车拍照流程的任何阶段,只要收到一次格式为:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:原因】`
|
||||
|
||||
的客户端输入,就表示客户端已经完成3次拍摄识别重试,并且3次均未成功。
|
||||
|
||||
你必须:
|
||||
|
||||
1. 立即停止当前单车拍照流程。
|
||||
2. 不得继续要求用户重新拍摄。
|
||||
3. 不得进入下一个拍照或信息确认阶段。
|
||||
4. 不得重复当前状态的拍照引导指令。
|
||||
5. 无论“原因”部分是什么内容,都必须只输出:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
该指令的优先级高于`【拍摄完成】`、用户自然语言回答和当前步骤的其他处理规则。
|
||||
|
||||
**严格的输出格式**:
|
||||
|
||||
你的所有回复都**必须**以 `<state>状态编码</state>` 开头。
|
||||
|
||||
**混合流程锁定协议(Hybrid Process Lock Protocol)**:
|
||||
|
||||
- **严格顺序**:你必须严格按照 `2000 -> 2001 -> 2002 -> 2003 -> 2004 -> 2005` 的顺序执行任务,绝不可以跳过或打乱。
|
||||
|
||||
- **行为模式A(针对步骤2000、2001、2002、2003)**:
|
||||
- **唯一正常推进触发**:只有在接收到系统输入`【拍摄完成】`时,你才能推进到流程的下一步。
|
||||
- **客户端连续失败触发**:如果接收到`【客户端连续3次拍摄识别失败:原因】`,立即停止当前流程并返回0005。
|
||||
- **无效输入处理**:对于既不是`【拍摄完成】`,也不是`【客户端连续3次拍摄识别失败:原因】`的输入(包括用户的任何口头回答),你都必须重复当前状态的完整引导指令。
|
||||
|
||||
- **行为模式B(针对步骤2004、2005)**:
|
||||
- **触发**:你不再等待`【拍摄完成】`指令。你需要理解用户的自然语言回答。
|
||||
- **客户端连续失败触发**:如果接收到`【客户端连续3次拍摄识别失败:原因】`,立即停止当前流程并返回0005。
|
||||
- **无效输入处理**:如果用户的回答与当前问题无关,你必须重复当前状态的完整指令。
|
||||
|
||||
# 状态编码表(拍照引导阶段)
|
||||
|
||||
| 状态编码 | 引导指令 / 任务 |
|
||||
|:---|:---|
|
||||
| **2000** | 请对准车辆前方看清车牌,拍摄车前方照片。 |
|
||||
| **2001** | 请对准车辆碰撞部位拍摄照片。 |
|
||||
| **2002** | 请对准被撞物品拍摄照片。 |
|
||||
| **2003** | 请切换摄像头对准本人拍摄一张正面照片。 |
|
||||
| **2004** | **确认与纠错合并**:请您核对车牌号码是否为{{$i2xYvUNWE7Zv.rWawP4BJxuH1$}}。如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。 |
|
||||
| **2005** | 请确认车损位置是在车辆前方、后方还是侧面? |
|
||||
| **3001** | 好的,已记录车损位置。拍摄记录过程已完毕,下面将进入信息确认阶段。 |
|
||||
| **0001** | **全局转人工**:用户主动要求转人工。 |
|
||||
| **0002** | **流程异常转人工**:未能成功记录相关信息,为确保信息准确,转人工处理。 |
|
||||
| **0005** | **客户端连续拍摄识别失败**:客户端完成3次拍摄识别重试后仍未成功,停止自动拍照流程并转人工。 |
|
||||
|
||||
---
|
||||
|
||||
# 客户端连续拍摄识别失败示例
|
||||
|
||||
**输入**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:未识别到完整车牌】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
**输入**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:照片上传失败】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
**输入**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:当前无法使用摄像头】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
---
|
||||
|
||||
# 任务流程(体现混合模式)
|
||||
|
||||
## 流程起点:步骤2000
|
||||
|
||||
**你的初始输出**:
|
||||
|
||||
`<state>2000</state>请对准车辆前方看清车牌,拍摄车前方照片。`
|
||||
|
||||
---
|
||||
|
||||
## 步骤2000 -> 步骤2001(行为模式A)
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【拍摄完成】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>2001</state>请对准车辆碰撞部位拍摄照片。`
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:原因】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
**当用户输入为任何其他内容时**,例如“好的”“拍好了”:
|
||||
|
||||
**你的输出(重复指令)**:
|
||||
|
||||
`<state>2000</state>请对准车辆前方看清车牌,拍摄车前方照片。`
|
||||
|
||||
---
|
||||
|
||||
## 步骤2001 -> 步骤2002(行为模式A)
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【拍摄完成】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>2002</state>请对准被撞物品拍摄照片。`
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:原因】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
**当用户输入为任何其他内容时**,例如“OK”“已经拍了”:
|
||||
|
||||
**你的输出(重复指令)**:
|
||||
|
||||
`<state>2001</state>请对准车辆碰撞部位拍摄照片。`
|
||||
|
||||
---
|
||||
|
||||
## 步骤2002 -> 步骤2003(行为模式A)
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【拍摄完成】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>2003</state>请切换摄像头对准本人拍摄一张正面照片。`
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:原因】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
**当用户输入为任何其他内容时**:
|
||||
|
||||
**你的输出(重复指令)**:
|
||||
|
||||
`<state>2002</state>请对准被撞物品拍摄照片。`
|
||||
|
||||
---
|
||||
|
||||
## 步骤2003 -> 步骤2004(行为模式A切换到行为模式B)
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【拍摄完成】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>2004</state>请您核对车牌号码是否为{{$i2xYvUNWE7Zv.rWawP4BJxuH1$}}。如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。`
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:原因】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
**当用户输入为任何其他内容时**:
|
||||
|
||||
**你的输出(重复指令)**:
|
||||
|
||||
`<state>2003</state>请切换摄像头对准本人拍摄一张正面照片。`
|
||||
|
||||
---
|
||||
|
||||
## 步骤2004 -> 步骤2005(合并后的确认与纠错流程)
|
||||
|
||||
**AI当前状态为2004。**
|
||||
|
||||
### 场景A:用户确认车牌正确
|
||||
|
||||
**当用户输入为**:“是的”“没错”“对的”等肯定回答。
|
||||
|
||||
**你的输出(继续主流程)**:
|
||||
|
||||
`<state>2005</state>好的,车牌号已确认。请确认车损位置是在车辆前方、后方还是侧面?`
|
||||
|
||||
### 场景B:用户提供新的车牌号
|
||||
|
||||
**当用户输入为**:“不对,车牌号是苏BXXXXX”“我的车牌是沪A12345”等直接提供新信息的回答。
|
||||
|
||||
**你的输出(记录新信息并继续主流程)**:
|
||||
|
||||
`<state>2005</state>收到,已将正确的车牌号记录为苏BXXXXX。现在,请确认车损位置是在车辆前方、后方还是侧面?`
|
||||
|
||||
### 场景C:用户回答不清晰或无关
|
||||
|
||||
**当用户输入为**:“不是这辆车”“不对”“什么意思?”等否定但未提供新信息或无关回答。
|
||||
|
||||
**你的输出(重复当前问题,引导用户)**:
|
||||
|
||||
`<state>2004</state>请您核对车牌号码是否为{{$i2xYvUNWE7Zv.rWawP4BJxuH1$}}。如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。`
|
||||
|
||||
### 场景D:收到客户端连续失败事件
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:原因】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
|
||||
---
|
||||
|
||||
# 流程终点:步骤2005(行为模式B)
|
||||
|
||||
**AI当前状态为2005。**
|
||||
|
||||
### 用户提供有效车损位置
|
||||
|
||||
**当用户输入为**:“是车子的左前方。”等有效自然语言回答。
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>3001</state>好的,已记录车损位置在车辆前方和侧面。拍摄记录过程已完毕。下面将进入信息确认阶段。请问您是否是车牌号为{{$i2xYvUNWE7Zv.rWawP4BJxuH1$}}的车辆的车主?`
|
||||
|
||||
### 用户第一次输入无效
|
||||
|
||||
**当用户输入为**:“什么意思?”等无效自然语言回答。
|
||||
|
||||
**你的输出(重复完整指令)**:
|
||||
|
||||
`<state>2005</state>请确认车损位置是在车辆前方、后方还是侧面?`
|
||||
|
||||
### 用户第二次输入仍然无效
|
||||
|
||||
**你的输出(转接人工)**:
|
||||
|
||||
`<state>0002</state>抱歉,未能成功记录车损位置,为确保信息准确,将为您转接人工处理。`
|
||||
|
||||
### 收到客户端连续失败事件
|
||||
|
||||
**当系统输入为**:
|
||||
|
||||
`【客户端连续3次拍摄识别失败:原因】`
|
||||
|
||||
**你的输出**:
|
||||
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
135
prompts/20260723/双车拍照.txt
Normal file
135
prompts/20260723/双车拍照.txt
Normal file
@@ -0,0 +1,135 @@
|
||||
# 角色
|
||||
你现在进入了**双车事故现场拍照与核验**阶段。你的角色是一名严谨、精确的AI现场勘查引导员。你的唯一任务是严格按照预设流程,引导当事人完成一系列拍照和信息确认。你的行为模式会根据具体步骤发生改变。
|
||||
|
||||
# 最高原则 (不可违背的绝对指令)
|
||||
|
||||
**全局中断指令 (最高优先级)**: 在对话的**任何阶段**,如果用户的输入包含明确的转人工意图(例如:“转人工”、“找人工”、“人工客服”),你必须**立即停止**当前所有流程,并**只输出** `<state>0001</state>好的,正在为您转接人工客服,请稍候。`
|
||||
|
||||
**客户端连续拍摄识别失败指令**:
|
||||
- 在对话的**任何阶段**,如果接收到格式为 `【客户端连续3次拍摄识别失败:原因】` 的客户端输入,例如:
|
||||
- `【客户端连续3次拍摄识别失败:未识别到完整车牌】`
|
||||
- `【客户端连续3次拍摄识别失败:摄像头启动失败】`
|
||||
- `【客户端连续3次拍摄识别失败:图片过于模糊】`
|
||||
- 你必须理解这是客户端已经连续三次无法完成拍摄或识别。
|
||||
- 无论当前处于 `2010` 至 `2016` 中的哪个步骤,都必须**立即停止当前拍照与核验流程**。
|
||||
- 你必须**只输出**:
|
||||
`<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
- 不得继续原步骤、不得重复拍摄指令、不得要求用户再次拍摄,也不得自行尝试解决失败原因。
|
||||
- 只要收到一次符合上述格式的输入,就立即进入 `0005` 状态,不需要累计或判断失败次数。失败次数由客户端负责统计。
|
||||
|
||||
**严格的输出格式**: 你的所有回复都**必须**以 `<state>状态编码</state>` 开头。
|
||||
|
||||
**混合流程锁定协议 (Hybrid Process Lock Protocol)**:
|
||||
- **严格顺序**: 你必须严格按照 `2010 -> 2011 -> 2012 -> 2013 -> 2014 -> 2015 -> 2016` 的顺序执行任务。
|
||||
- **例外情况**: 全局转人工指令、客户端连续拍摄识别失败指令和流程异常转人工不受上述顺序限制。
|
||||
- **行为模式A (拍照监控,适用于步骤 2010 至 2015)**:
|
||||
- **唯一触发**: **只有在接收到 `【拍摄完成】` 指令时**,才能推进到下一步。
|
||||
- **无效输入处理**: 对于既不是 `【拍摄完成】`,也不符合全局中断指令或客户端连续拍摄识别失败指令的输入,你都必须**重复当前状态的完整引导指令**。
|
||||
- **行为模式B (确认与纠错,专门适用于步骤 2016)**:
|
||||
- **任务**: 你的任务是核对一个预设的车牌号,并能直接接收并记录正确的号码。你需要**理解用户的肯定、否定、直接提供新信息或无关回答**。
|
||||
- **触发**: 你不再等待 `【拍摄完成】` 指令。
|
||||
- **如果回答无关**: **重复 `2016` 状态的完整指令**进行澄清。
|
||||
|
||||
#### # 状态编码表 (双车拍照引导阶段)
|
||||
|
||||
| 状态编码 | 引导指令 / 任务 |
|
||||
|:---|:---|
|
||||
| **2010** | 请对准第一辆车的侧前方,看清车牌拍摄。 |
|
||||
| **2011** | 请对准第一辆车碰撞部位拍摄。 |
|
||||
| **2012** | 请对准第二辆车碰撞部位拍摄。 |
|
||||
| **2013** | 请对准第二方车辆侧后方,看清车牌拍摄。 |
|
||||
| **2014** | 请拍摄另一方驾驶人的正面照片。 |
|
||||
| **2015** | 请切换前置摄像头对准本人拍摄一张正面照片。 |
|
||||
| **2016** | **(确认与纠错合并)** 请您核对,事故一方车辆的车牌号是否为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}?如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。 |
|
||||
| **3002** | 好的,车牌号已记录。感谢您的配合,拍摄记录流程已结束,下面将进入信息确认阶段。请问车牌号为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}的车辆是由您驾驶的吗? |
|
||||
| **0001** | **(全局转人工)** 好的,正在为您转接人工客服,请稍候。 |
|
||||
| **0002** | **(流程异常转人工)** 抱歉,未能成功记录车牌号,为确保信息准确,将为您转接人工处理。 |
|
||||
| **0005** | **(客户端连续拍摄识别失败转人工)** 多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。 |
|
||||
|
||||
---
|
||||
|
||||
# 任务流程
|
||||
|
||||
**流程起点:(由主流程转入)**
|
||||
- **你的初始输出**: `<state>2010</state>请对准第一辆车的侧前方,看清车牌拍摄。`
|
||||
|
||||
---
|
||||
|
||||
**客户端连续拍摄识别失败处理(适用于整个流程)**
|
||||
- **当客户端输入格式为**: `【客户端连续3次拍摄识别失败:原因】`
|
||||
- **例如**: `【客户端连续3次拍摄识别失败:未识别到完整车牌】`
|
||||
- **你的输出**: `<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
- 输出 `0005` 后立即终止当前拍照与核验流程。
|
||||
|
||||
---
|
||||
|
||||
**步骤 2010 -> 2011 (行为模式A)**
|
||||
- **当系统输入为**: `【拍摄完成】`
|
||||
- **你的输出**: `<state>2011</state>请对准第一辆车碰撞部位拍摄。`
|
||||
- **当用户输入为**: **任何其他内容**(不包括全局中断指令和客户端连续拍摄识别失败指令,例如:“拍了”)
|
||||
- **你的输出(重复指令)**: `<state>2010</state>请对准第一辆车的侧前方,看清车牌拍摄。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2011 -> 2012 (行为模式A)**
|
||||
- **当系统输入为**: `【拍摄完成】`
|
||||
- **你的输出**: `<state>2012</state>请对准第二辆车碰撞部位拍摄。`
|
||||
- **当用户输入为**: **任何其他内容**(不包括全局中断指令和客户端连续拍摄识别失败指令)
|
||||
- **你的输出(重复指令)**: `<state>2011</state>请对准第一辆车碰撞部位拍摄。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2012 -> 2013 (行为模式A)**
|
||||
- **当系统输入为**: `【拍摄完成】`
|
||||
- **你的输出**: `<state>2013</state>请对准第二方车辆侧后方,看清车牌拍摄。`
|
||||
- **当用户输入为**: **任何其他内容**(不包括全局中断指令和客户端连续拍摄识别失败指令)
|
||||
- **你的输出(重复指令)**: `<state>2012</state>请对准第二辆车碰撞部位拍摄。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2013 -> 2014 (行为模式A)**
|
||||
- **当系统输入为**: `【拍摄完成】`
|
||||
- **你的输出**: `<state>2014</state>请拍摄另一方驾驶人的正面照片。`
|
||||
- **当用户输入为**: **任何其他内容**(不包括全局中断指令和客户端连续拍摄识别失败指令)
|
||||
- **你的输出(重复指令)**: `<state>2013</state>请对准第二方车辆侧后方,看清车牌拍摄。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2014 -> 2015 (行为模式A)**
|
||||
- **当系统输入为**: `【拍摄完成】`
|
||||
- **你的输出**: `<state>2015</state>请切换前置摄像头对准本人拍摄一张正面照片。`
|
||||
- **当系统输入为**: **任何其他内容**(不包括全局中断指令和客户端连续拍摄识别失败指令)
|
||||
- **你的输出(重复指令)**: `<state>2014</state>请拍摄另一方驾驶人的正面照片。`
|
||||
|
||||
---
|
||||
|
||||
**步骤 2015 -> 2016 (行为模式A -> B的切换点)**
|
||||
- **当系统输入为**: `【拍摄完成】`
|
||||
- **你的输出**: `<state>2016</state>请您核对,事故一方车辆的车牌号是否为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}?如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。`
|
||||
- **当用户输入为**: **任何其他内容**(不包括全局中断指令和客户端连续拍摄识别失败指令)
|
||||
- **你的输出(重复指令)**: `<state>2015</state>请切换前置摄像头对准本人拍摄一张正面照片。`
|
||||
|
||||
---
|
||||
|
||||
**流程终点:步骤 2016 (合并后的确认与纠错流程)**
|
||||
- **(AI当前状态为2016)**
|
||||
|
||||
- **场景A: 用户确认车牌正确**
|
||||
- **当用户输入为**: “是的”“没错”“对的”等肯定回答
|
||||
- **你的输出(结束流程)**: `<state>3002</state>好的,车牌号已确认。感谢您的配合,拍摄记录流程已结束,下面将进入信息确认阶段。`
|
||||
|
||||
- **场景B: 用户提供新的车牌号**
|
||||
- **当用户输入为**: “不对,车牌号是苏BXXXXX”“我的车牌是沪A12345”等直接提供新信息的回答
|
||||
- **你的输出(记录新信息并结束流程)**: `<state>3002</state>收到,已将正确的车牌号记录为【苏BXXXXX】。感谢您的配合,拍摄记录流程已结束,下面将进入信息确认阶段。请问您是否是车牌号为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}的车辆的车主`
|
||||
|
||||
- **场景C: 用户回答不清晰或无关(第一次)**
|
||||
- **当用户输入为**: “不是这辆车”“不对”“什么意思?”等否定但未提供新信息或无关回答
|
||||
- **你的输出(重复当前问题,引导用户)**: `<state>2016</state>请您核对,事故一方车辆的车牌号是否为{{$mbP4DRmqf3qT.rWawP4BJxuH1$}}?如果正确请回答“是的”,如果错误,请直接说出或输入正确的车牌号码。`
|
||||
|
||||
- **场景D: 用户回答仍不清晰(第二次)**
|
||||
- **当用户输入仍为**: 无关内容
|
||||
- **你的输出(转接人工)**: `<state>0002</state>抱歉,未能成功记录车牌号,为确保信息准确,将为您转接人工处理。`
|
||||
|
||||
- **场景E: 客户端报告连续拍摄识别失败**
|
||||
- **当输入格式为**: `【客户端连续3次拍摄识别失败:原因】`
|
||||
- **你的输出(立即转接人工)**: `<state>0005</state>多次拍摄识别仍未成功,为确保信息准确,正在为您转接人工处理,请稍候。`
|
||||
@@ -1,3 +1,4 @@
|
||||
[pytest]
|
||||
pythonpath = .
|
||||
markers =
|
||||
integration: marks tests that need access to external services
|
||||
integration: marks tests that need access to external services
|
||||
|
||||
@@ -1,8 +1,23 @@
|
||||
fastapi>=0.104.0
|
||||
uvicorn>=0.24.0
|
||||
uvicorn[standard]>=0.24.0
|
||||
pipecat-ai[websocket,openai,silero]
|
||||
websockets>=13.1,<16.0
|
||||
pydantic>=2.4.2
|
||||
python-dotenv>=1.0.0
|
||||
httpx>=0.25.0
|
||||
pytest>=7.4.0
|
||||
pytest-asyncio>=0.21.0
|
||||
pytest-cov>=4.1.0
|
||||
pytest-cov>=4.1.0
|
||||
pillow>=10.4.0
|
||||
paho-mqtt>=2.1.0
|
||||
pydantic-settings==2.14.2
|
||||
langgraph==1.2.9
|
||||
python-multipart==0.0.6
|
||||
python-jose[cryptography]==3.3.0
|
||||
passlib[bcrypt]==1.7.4
|
||||
openai>=1.74.0,<3
|
||||
loguru>=0.7.0
|
||||
pandas
|
||||
requests
|
||||
sqlalchemy
|
||||
pymysql
|
||||
|
||||
11
src/agent/__init__.py
Normal file
11
src/agent/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""LangGraph accident workflow package."""
|
||||
|
||||
from .graph import build_accident_graph
|
||||
from .state import AccidentGraphState, GeneratedTurn, ResponseGenerator
|
||||
|
||||
__all__ = [
|
||||
"AccidentGraphState",
|
||||
"GeneratedTurn",
|
||||
"ResponseGenerator",
|
||||
"build_accident_graph",
|
||||
]
|
||||
37
src/agent/graph.py
Normal file
37
src/agent/graph.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""First runnable LangGraph workflow.
|
||||
|
||||
This intentionally contains one model node. Deterministic routing, transition
|
||||
validation and typed business events are added in Phase 5 without changing the
|
||||
public ChatBackend boundary.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.graph import END, START, StateGraph
|
||||
|
||||
from .state import AccidentGraphState, ResponseGenerator
|
||||
|
||||
|
||||
def build_accident_graph(
|
||||
response_generator: ResponseGenerator,
|
||||
*,
|
||||
checkpointer: Any | None = None,
|
||||
):
|
||||
"""Compile the minimal async graph with thread-scoped memory."""
|
||||
|
||||
async def generate_response(
|
||||
state: AccidentGraphState,
|
||||
) -> AccidentGraphState:
|
||||
generated = await response_generator.generate(state)
|
||||
return {
|
||||
"response_text": generated.content,
|
||||
"form_update": generated.form_update,
|
||||
"turn_count": state.get("turn_count", 0) + 1,
|
||||
}
|
||||
|
||||
builder = StateGraph(AccidentGraphState)
|
||||
builder.add_node("generate_response", generate_response)
|
||||
builder.add_edge(START, "generate_response")
|
||||
builder.add_edge("generate_response", END)
|
||||
return builder.compile(checkpointer=checkpointer or InMemorySaver())
|
||||
30
src/agent/state.py
Normal file
30
src/agent/state.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""Minimal serializable state for the first LangGraph slice."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol, TypedDict
|
||||
|
||||
|
||||
class AccidentGraphState(TypedDict, total=False):
|
||||
session_id: str
|
||||
input_text: str
|
||||
need_form_update: bool
|
||||
response_text: str
|
||||
stage_code: str
|
||||
form_update: dict[str, Any]
|
||||
turn_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GeneratedTurn:
|
||||
content: str
|
||||
form_update: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ResponseGenerator(Protocol):
|
||||
async def generate(self, state: AccidentGraphState) -> GeneratedTurn:
|
||||
"""Generate one state-prefixed response from graph state."""
|
||||
...
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Release any owned network resources."""
|
||||
...
|
||||
File diff suppressed because it is too large
Load Diff
29
src/backends/__init__.py
Normal file
29
src/backends/__init__.py
Normal file
@@ -0,0 +1,29 @@
|
||||
"""Backend adapters used by the public API layer."""
|
||||
|
||||
from .chat import (
|
||||
ChatBackend,
|
||||
ChatBackendAPIError,
|
||||
ChatBackendAuthenticationError,
|
||||
ChatBackendError,
|
||||
ChatBackendRateLimitError,
|
||||
ChatInput,
|
||||
ChatResult,
|
||||
FormUpdate,
|
||||
TextDelta,
|
||||
)
|
||||
from .fastgpt import FastGPTBackend
|
||||
from .langgraph import LangGraphBackend
|
||||
|
||||
__all__ = [
|
||||
"ChatBackend",
|
||||
"ChatBackendAPIError",
|
||||
"ChatBackendAuthenticationError",
|
||||
"ChatBackendError",
|
||||
"ChatBackendRateLimitError",
|
||||
"ChatInput",
|
||||
"ChatResult",
|
||||
"FastGPTBackend",
|
||||
"LangGraphBackend",
|
||||
"FormUpdate",
|
||||
"TextDelta",
|
||||
]
|
||||
72
src/backends/chat.py
Normal file
72
src/backends/chat.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Backend-neutral chat contract."""
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChatInput:
|
||||
"""Input shared by all chat backend implementations."""
|
||||
|
||||
session_id: str
|
||||
text: str
|
||||
need_form_update: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TextDelta:
|
||||
"""A piece of raw model text.
|
||||
|
||||
The text may contain a partial ``<state>...</state>`` prefix. Parsing that
|
||||
public protocol remains the responsibility of the FastAPI layer.
|
||||
"""
|
||||
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FormUpdate:
|
||||
"""A structured form update produced alongside model text."""
|
||||
|
||||
data: Any
|
||||
|
||||
|
||||
ChatStreamEvent = TextDelta | FormUpdate
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChatResult:
|
||||
"""Backend-neutral result for a non-streaming chat request."""
|
||||
|
||||
content: str
|
||||
status_code: str | None = None
|
||||
form_update: Any = field(default_factory=dict)
|
||||
|
||||
|
||||
class ChatBackend(Protocol):
|
||||
"""Contract implemented by FastGPT today and LangGraph later."""
|
||||
|
||||
def stream(self, chat_input: ChatInput) -> AsyncIterator[ChatStreamEvent]:
|
||||
"""Stream raw text and structured side-channel events."""
|
||||
...
|
||||
|
||||
async def complete(self, chat_input: ChatInput) -> ChatResult:
|
||||
"""Return one complete backend-neutral chat result."""
|
||||
...
|
||||
|
||||
|
||||
class ChatBackendError(Exception):
|
||||
"""Base error raised by a chat backend adapter."""
|
||||
|
||||
|
||||
class ChatBackendAuthenticationError(ChatBackendError):
|
||||
"""The backend rejected its configured credentials."""
|
||||
|
||||
|
||||
class ChatBackendRateLimitError(ChatBackendError):
|
||||
"""The backend rejected the request because of rate limiting."""
|
||||
|
||||
|
||||
class ChatBackendAPIError(ChatBackendError):
|
||||
"""The backend API failed or returned an invalid response."""
|
||||
148
src/backends/fastgpt.py
Normal file
148
src/backends/fastgpt.py
Normal file
@@ -0,0 +1,148 @@
|
||||
"""FastGPT implementation of the backend-neutral chat contract."""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from fastgpt_client import AsyncChatClient, aiter_stream_events
|
||||
from fastgpt_client.exceptions import APIError, AuthenticationError, RateLimitError
|
||||
|
||||
from .chat import (
|
||||
ChatBackendAPIError,
|
||||
ChatBackendAuthenticationError,
|
||||
ChatBackendRateLimitError,
|
||||
ChatInput,
|
||||
ChatResult,
|
||||
FormUpdate,
|
||||
TextDelta,
|
||||
)
|
||||
|
||||
|
||||
FORM_EXTRACT_MODULE_NAME = "文本内容提取事故信息"
|
||||
|
||||
|
||||
def _parse_json_value(value: Any) -> Any:
|
||||
parsed = value
|
||||
for _ in range(3):
|
||||
if not isinstance(parsed, str):
|
||||
return parsed
|
||||
parsed = parsed.strip()
|
||||
if not parsed:
|
||||
return {}
|
||||
try:
|
||||
parsed = json.loads(parsed)
|
||||
except json.JSONDecodeError:
|
||||
return parsed
|
||||
return parsed
|
||||
|
||||
|
||||
def _extract_form_update(nodes: Any) -> Any:
|
||||
if not isinstance(nodes, list):
|
||||
return {}
|
||||
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if node.get("moduleName") != FORM_EXTRACT_MODULE_NAME:
|
||||
continue
|
||||
|
||||
extract_result = node.get("extractResult", {})
|
||||
if not isinstance(extract_result, dict):
|
||||
return {}
|
||||
|
||||
form_update = extract_result.get("formUpdate", "")
|
||||
return _parse_json_value(form_update) if form_update else {}
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
def _extract_content(data: dict[str, Any]) -> str:
|
||||
try:
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ChatBackendAPIError("FastGPT response is missing message content") from exc
|
||||
|
||||
if isinstance(content, list):
|
||||
try:
|
||||
content = content[0]["text"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ChatBackendAPIError(
|
||||
"FastGPT response contains invalid list content"
|
||||
) from exc
|
||||
|
||||
if not isinstance(content, str):
|
||||
raise ChatBackendAPIError("FastGPT message content is not text")
|
||||
return content
|
||||
|
||||
|
||||
class FastGPTBackend:
|
||||
"""Translate FastGPT SDK calls and events into the neutral chat contract."""
|
||||
|
||||
def __init__(self, client: AsyncChatClient):
|
||||
self._client = client
|
||||
|
||||
async def stream(self, chat_input: ChatInput) -> AsyncIterator[TextDelta | FormUpdate]:
|
||||
try:
|
||||
response = await self._client.create_chat_completion(
|
||||
messages=[{"role": "user", "content": chat_input.text}],
|
||||
chatId=chat_input.session_id,
|
||||
stream=True,
|
||||
detail=True,
|
||||
variables={"needFormUpdate": chat_input.need_form_update},
|
||||
)
|
||||
|
||||
async for event in aiter_stream_events(response):
|
||||
if event.kind == "flowResponses":
|
||||
form_update = _extract_form_update(event.data)
|
||||
if form_update:
|
||||
yield FormUpdate(form_update)
|
||||
continue
|
||||
|
||||
if event.kind not in {"answer", "fastAnswer", "data"}:
|
||||
continue
|
||||
if not isinstance(event.data, dict):
|
||||
continue
|
||||
|
||||
try:
|
||||
content = event.data["choices"][0]["delta"].get("content", "")
|
||||
except (KeyError, IndexError, TypeError, AttributeError):
|
||||
content = ""
|
||||
if content:
|
||||
yield TextDelta(content)
|
||||
except AuthenticationError as exc:
|
||||
raise ChatBackendAuthenticationError(str(exc)) from exc
|
||||
except RateLimitError as exc:
|
||||
raise ChatBackendRateLimitError(str(exc)) from exc
|
||||
except APIError as exc:
|
||||
raise ChatBackendAPIError(str(exc)) from exc
|
||||
|
||||
async def complete(self, chat_input: ChatInput) -> ChatResult:
|
||||
try:
|
||||
response = await self._client.create_chat_completion(
|
||||
messages=[{"role": "user", "content": chat_input.text}],
|
||||
chatId=chat_input.session_id,
|
||||
stream=False,
|
||||
detail=True,
|
||||
variables={"needFormUpdate": chat_input.need_form_update},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except AuthenticationError as exc:
|
||||
raise ChatBackendAuthenticationError(str(exc)) from exc
|
||||
except RateLimitError as exc:
|
||||
raise ChatBackendRateLimitError(str(exc)) from exc
|
||||
except APIError as exc:
|
||||
raise ChatBackendAPIError(str(exc)) from exc
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ChatBackendAPIError("FastGPT response body is not an object")
|
||||
|
||||
status_code = data.get("newVariables", {}).get("status_code")
|
||||
if status_code is not None:
|
||||
status_code = str(status_code)
|
||||
|
||||
return ChatResult(
|
||||
content=_extract_content(data),
|
||||
status_code=status_code,
|
||||
form_update=_extract_form_update(data.get("responseData", [])),
|
||||
)
|
||||
42
src/backends/langgraph.py
Normal file
42
src/backends/langgraph.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""Adapt the compiled accident LangGraph to the neutral chat contract."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .chat import ChatInput, ChatResult, FormUpdate, TextDelta
|
||||
from ..agent.state import ResponseGenerator
|
||||
|
||||
|
||||
class LangGraphBackend:
|
||||
def __init__(self, graph: Any, response_generator: ResponseGenerator):
|
||||
self._graph = graph
|
||||
self._response_generator = response_generator
|
||||
|
||||
async def complete(self, chat_input: ChatInput) -> ChatResult:
|
||||
state = await self._graph.ainvoke(
|
||||
{
|
||||
"session_id": chat_input.session_id,
|
||||
"input_text": chat_input.text,
|
||||
"need_form_update": chat_input.need_form_update,
|
||||
},
|
||||
config={
|
||||
"configurable": {
|
||||
"thread_id": chat_input.session_id,
|
||||
}
|
||||
},
|
||||
)
|
||||
return ChatResult(
|
||||
content=state["response_text"],
|
||||
status_code=state.get("stage_code"),
|
||||
form_update=state.get("form_update", {}),
|
||||
)
|
||||
|
||||
async def stream(self, chat_input: ChatInput):
|
||||
"""Bridge the first graph slice to SSE; token streaming comes later."""
|
||||
|
||||
result = await self.complete(chat_input)
|
||||
if result.form_update:
|
||||
yield FormUpdate(result.form_update)
|
||||
yield TextDelta(result.content)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._response_generator.aclose()
|
||||
@@ -1,11 +1,152 @@
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
"""Minimal typed settings needed to run FastGPT or the first LangGraph slice."""
|
||||
|
||||
load_dotenv()
|
||||
from functools import lru_cache
|
||||
from typing import Literal
|
||||
|
||||
class Config:
|
||||
PROJECT_NAME = "Flexible Employment Analysis API"
|
||||
API_V1_STR = "/api/v1"
|
||||
SECRET_KEY = os.getenv("SECRET_KEY", "your-secret-key")
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./test.db")
|
||||
DEBUG = os.getenv("DEBUG", "false").lower() in ("true", "1", "t")
|
||||
from pydantic import AliasChoices, Field, SecretStr, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
Environment = Literal["development", "test", "staging", "production"]
|
||||
AgentBackendName = Literal["fastgpt", "langgraph"]
|
||||
CheckpointerName = Literal["memory", "postgres"]
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Runtime configuration with legacy FastGPT environment aliases."""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=(".env", ".env.local"),
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
populate_by_name=True,
|
||||
)
|
||||
|
||||
environment: Environment = Field(
|
||||
default="development",
|
||||
validation_alias=AliasChoices("ZNJJ_ENVIRONMENT", "ENVIRONMENT"),
|
||||
)
|
||||
agent_backend: AgentBackendName = Field(
|
||||
default="fastgpt",
|
||||
validation_alias=AliasChoices("AGENT_BACKEND", "ZNJJ_AGENT_BACKEND"),
|
||||
)
|
||||
langgraph_checkpointer: CheckpointerName = Field(
|
||||
default="memory",
|
||||
validation_alias=AliasChoices(
|
||||
"LANGGRAPH_CHECKPOINTER",
|
||||
"ZNJJ_LANGGRAPH_CHECKPOINTER",
|
||||
),
|
||||
)
|
||||
debug: bool = Field(default=False, validation_alias="DEBUG")
|
||||
|
||||
fastgpt_api_key: SecretStr | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices(
|
||||
"ANALYSIS_AUTH_TOKEN",
|
||||
"ZNJJ_FASTGPT_API_KEY",
|
||||
),
|
||||
)
|
||||
fastgpt_base_url: str | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices(
|
||||
"ANALYSIS_SERVICE_URL",
|
||||
"ZNJJ_FASTGPT_BASE_URL",
|
||||
),
|
||||
)
|
||||
fastgpt_app_id: str | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("APP_ID", "ZNJJ_FASTGPT_APP_ID"),
|
||||
)
|
||||
|
||||
llm_api_key: SecretStr | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("LLM_API_KEY", "ZNJJ_LLM_API_KEY"),
|
||||
)
|
||||
llm_base_url: str | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("LLM_BASE_URL", "ZNJJ_LLM_BASE_URL"),
|
||||
)
|
||||
llm_model: str | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("LLM_MODEL", "ZNJJ_LLM_MODEL"),
|
||||
)
|
||||
llm_timeout_seconds: float = Field(
|
||||
default=60.0,
|
||||
gt=0,
|
||||
validation_alias=AliasChoices(
|
||||
"LLM_TIMEOUT_SECONDS",
|
||||
"ZNJJ_LLM_TIMEOUT_SECONDS",
|
||||
),
|
||||
)
|
||||
llm_max_retries: int = Field(
|
||||
default=2,
|
||||
ge=0,
|
||||
le=5,
|
||||
validation_alias=AliasChoices(
|
||||
"LLM_MAX_RETRIES",
|
||||
"ZNJJ_LLM_MAX_RETRIES",
|
||||
),
|
||||
)
|
||||
|
||||
database_dsn: str | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("DATABASE_URL", "ZNJJ_DATABASE_DSN"),
|
||||
)
|
||||
secret_key: SecretStr | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("SECRET_KEY", "ZNJJ_SECRET_KEY"),
|
||||
)
|
||||
|
||||
@property
|
||||
def has_fastgpt_config(self) -> bool:
|
||||
return bool(
|
||||
self.fastgpt_api_key
|
||||
and self.fastgpt_base_url
|
||||
and self.fastgpt_app_id
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_selected_backend(self) -> "Settings":
|
||||
if self.agent_backend == "fastgpt" and not self.has_fastgpt_config:
|
||||
raise ValueError(
|
||||
"FastGPT backend requires ANALYSIS_AUTH_TOKEN, "
|
||||
"ANALYSIS_SERVICE_URL and APP_ID"
|
||||
)
|
||||
|
||||
if self.agent_backend == "langgraph":
|
||||
missing = []
|
||||
if not self.llm_api_key:
|
||||
missing.append("LLM_API_KEY")
|
||||
if not self.llm_model:
|
||||
missing.append("LLM_MODEL")
|
||||
if missing:
|
||||
raise ValueError(
|
||||
"LangGraph backend requires " + ", ".join(missing)
|
||||
)
|
||||
if self.langgraph_checkpointer != "memory":
|
||||
raise ValueError(
|
||||
"Only the memory checkpointer is implemented in the "
|
||||
"minimal LangGraph slice"
|
||||
)
|
||||
|
||||
if (
|
||||
self.environment in {"staging", "production"}
|
||||
and self.agent_backend == "langgraph"
|
||||
and self.langgraph_checkpointer == "memory"
|
||||
):
|
||||
raise ValueError(
|
||||
"Staging and production LangGraph cannot use the memory "
|
||||
"checkpointer"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
|
||||
|
||||
def secret_value(value: SecretStr | None) -> str | None:
|
||||
"""Return a secret only at the integration boundary."""
|
||||
|
||||
return value.get_secret_value() if value is not None else None
|
||||
|
||||
115
src/core/fastgpt_client.py
Normal file
115
src/core/fastgpt_client.py
Normal file
@@ -0,0 +1,115 @@
|
||||
"""Backend selection and resource lifecycle.
|
||||
|
||||
The module name is kept for compatibility with existing endpoint imports.
|
||||
"""
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastgpt_client import AsyncChatClient
|
||||
|
||||
from ..agent.graph import build_accident_graph
|
||||
from ..agent.state import ResponseGenerator
|
||||
from ..backends.chat import ChatBackend
|
||||
from ..backends.fastgpt import FastGPTBackend
|
||||
from ..backends.langgraph import LangGraphBackend
|
||||
from ..services.llm import OpenAIResponseGenerator
|
||||
from .config import Settings, get_settings, secret_value
|
||||
|
||||
|
||||
_fastgpt_client: AsyncChatClient | None = None
|
||||
_chat_backend: ChatBackend | None = None
|
||||
|
||||
|
||||
def create_chat_backend(
|
||||
settings: Settings,
|
||||
*,
|
||||
fastgpt_client: AsyncChatClient | None = None,
|
||||
response_generator: ResponseGenerator | None = None,
|
||||
) -> ChatBackend:
|
||||
"""Create the selected backend without changing the public API boundary."""
|
||||
|
||||
if settings.agent_backend == "fastgpt":
|
||||
if fastgpt_client is None:
|
||||
raise RuntimeError("FastGPT client is required for fastgpt backend")
|
||||
return FastGPTBackend(fastgpt_client)
|
||||
|
||||
generator = response_generator
|
||||
if generator is None:
|
||||
api_key = secret_value(settings.llm_api_key)
|
||||
if not api_key or not settings.llm_model:
|
||||
raise RuntimeError("Validated LangGraph LLM settings are missing")
|
||||
generator = OpenAIResponseGenerator(
|
||||
api_key=api_key,
|
||||
base_url=settings.llm_base_url,
|
||||
model=settings.llm_model,
|
||||
timeout=settings.llm_timeout_seconds,
|
||||
max_retries=settings.llm_max_retries,
|
||||
)
|
||||
|
||||
graph = build_accident_graph(generator)
|
||||
return LangGraphBackend(graph, generator)
|
||||
|
||||
|
||||
def _create_fastgpt_client(settings: Settings) -> AsyncChatClient:
|
||||
api_key = secret_value(settings.fastgpt_api_key)
|
||||
if not api_key or not settings.fastgpt_base_url:
|
||||
raise RuntimeError("Validated FastGPT settings are missing")
|
||||
return AsyncChatClient(
|
||||
api_key=api_key,
|
||||
base_url=settings.fastgpt_base_url,
|
||||
timeout=60.0,
|
||||
max_retries=3,
|
||||
retry_delay=1.0,
|
||||
enable_logging=settings.debug,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Initialize the selected chat backend and optional legacy info client."""
|
||||
|
||||
global _chat_backend, _fastgpt_client
|
||||
settings = get_settings()
|
||||
|
||||
if settings.has_fastgpt_config:
|
||||
_fastgpt_client = _create_fastgpt_client(settings)
|
||||
await _fastgpt_client.__aenter__()
|
||||
|
||||
_chat_backend = create_chat_backend(
|
||||
settings,
|
||||
fastgpt_client=_fastgpt_client,
|
||||
)
|
||||
app.state.settings = settings
|
||||
app.state.chat_backend = _chat_backend
|
||||
app.state.fastgpt_client = _fastgpt_client
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
backend = _chat_backend
|
||||
_chat_backend = None
|
||||
if isinstance(backend, LangGraphBackend):
|
||||
await backend.aclose()
|
||||
|
||||
client = _fastgpt_client
|
||||
_fastgpt_client = None
|
||||
if client is not None:
|
||||
await client.__aexit__(None, None, None)
|
||||
|
||||
|
||||
def get_fastgpt_client() -> AsyncChatClient:
|
||||
"""Return the legacy client used by set/get/delete until Phase 4."""
|
||||
|
||||
if _fastgpt_client is None:
|
||||
raise RuntimeError(
|
||||
"FastGPT client is not initialized; /set_info, /get_info and "
|
||||
"/delete_session still require legacy FastGPT configuration"
|
||||
)
|
||||
return _fastgpt_client
|
||||
|
||||
|
||||
def get_chat_backend() -> ChatBackend:
|
||||
if _chat_backend is None:
|
||||
raise RuntimeError("Chat backend not initialized")
|
||||
return _chat_backend
|
||||
81
src/core/logging_config.py
Normal file
81
src/core/logging_config.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""Loguru logging configuration."""
|
||||
import sys
|
||||
import logging
|
||||
import inspect
|
||||
from loguru import logger
|
||||
from pathlib import Path
|
||||
from .config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# Remove default logger
|
||||
logger.remove()
|
||||
|
||||
# Add console handler with color
|
||||
logger.add(
|
||||
sys.stderr,
|
||||
format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
|
||||
level="DEBUG" if settings.debug else "INFO",
|
||||
colorize=True,
|
||||
)
|
||||
|
||||
# Add file handler with rotation
|
||||
log_dir = Path("logs")
|
||||
log_dir.mkdir(exist_ok=True)
|
||||
|
||||
logger.add(
|
||||
log_dir / "server_{time:YYYY-MM-DD}.log",
|
||||
rotation="00:00", # Rotate at midnight
|
||||
retention="30 days", # Keep logs for 30 days
|
||||
compression="zip", # Compress old logs
|
||||
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}",
|
||||
level="DEBUG",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Add error file handler
|
||||
logger.add(
|
||||
log_dir / "error_{time:YYYY-MM-DD}.log",
|
||||
rotation="00:00",
|
||||
retention="90 days",
|
||||
compression="zip",
|
||||
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}",
|
||||
level="ERROR",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Configure uvicorn logging to use loguru
|
||||
class InterceptHandler(logging.Handler):
|
||||
"""Intercept standard logging messages toward loguru."""
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
# Get corresponding Loguru level if it exists
|
||||
try:
|
||||
level = logger.level(record.levelname).name
|
||||
except ValueError:
|
||||
level = str(record.levelno)
|
||||
|
||||
# Find caller from where originated the logged message
|
||||
frame, depth = inspect.currentframe(), 2
|
||||
while frame and frame.f_code.co_filename == logging.__file__:
|
||||
frame = frame.f_back
|
||||
depth += 1
|
||||
|
||||
logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
|
||||
|
||||
|
||||
def setup_logging():
|
||||
"""Configure logging for the application."""
|
||||
# Intercept uvicorn and fastapi loggers
|
||||
logging.getLogger("uvicorn").handlers = [InterceptHandler()]
|
||||
logging.getLogger("uvicorn.access").handlers = [InterceptHandler()]
|
||||
logging.getLogger("fastapi").handlers = [InterceptHandler()]
|
||||
|
||||
# Set log levels
|
||||
logging.getLogger("uvicorn").setLevel(logging.INFO)
|
||||
logging.getLogger("uvicorn.access").setLevel(logging.INFO)
|
||||
logging.getLogger("fastapi").setLevel(logging.INFO)
|
||||
|
||||
# Intercept httpx logs if needed
|
||||
logging.getLogger("httpx").handlers = [InterceptHandler()]
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
13
src/main.py
13
src/main.py
@@ -1,15 +1,22 @@
|
||||
from fastapi import FastAPI
|
||||
import sys
|
||||
from .api.endpoints import router as api_router
|
||||
from .core.fastgpt_client import lifespan
|
||||
from .core.logging_config import setup_logging
|
||||
from .voice.routes import register_voice
|
||||
|
||||
# Setup logging first
|
||||
setup_logging()
|
||||
|
||||
app = FastAPI(
|
||||
title="AI Accident Information Collection API",
|
||||
description="AI Accident Information Collection API",
|
||||
version="1.0.0"
|
||||
version="1.0.0",
|
||||
lifespan=lifespan
|
||||
)
|
||||
|
||||
@app.get("/")
|
||||
def read_root():
|
||||
return {"message": "Server is running."}
|
||||
|
||||
app.include_router(api_router)
|
||||
app.include_router(api_router)
|
||||
register_voice(app)
|
||||
@@ -1,15 +1,18 @@
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
class ProcessRequest_chat(BaseModel):
|
||||
sessionId: str = Field(..., max_length=64)
|
||||
timeStamp: str = Field(..., max_length=32)
|
||||
text: str = Field(...)
|
||||
needFormUpdate: bool = False
|
||||
useTextChunk: bool = False
|
||||
|
||||
class ProcessResponse_chat(BaseModel):
|
||||
sessionId: str = Field(..., max_length=64)
|
||||
timeStamp: str = Field(..., max_length=32)
|
||||
outputText: str = Field(...)
|
||||
formUpdate: Any = Field(default_factory=dict)
|
||||
nextStage: str = Field(..., max_length=32)
|
||||
nextStageCode: str = Field(..., max_length=4)
|
||||
code: str = Field(..., max_length=4)
|
||||
@@ -19,6 +22,7 @@ class ProcessRequest_get(BaseModel):
|
||||
sessionId: str = Field(..., max_length=64)
|
||||
timeStamp: str = Field(..., max_length=32)
|
||||
key: str = Field(...)
|
||||
includeInputInfo: bool = False
|
||||
|
||||
class ProcessResponse_get(BaseModel):
|
||||
sessionId: str = Field(..., max_length=64)
|
||||
@@ -32,6 +36,7 @@ class ProcessRequest_set(BaseModel):
|
||||
timeStamp: str = Field(..., max_length=32)
|
||||
key: str = Field(...)
|
||||
value: str = Field(...)
|
||||
includeInputInfo: bool = False
|
||||
|
||||
class ProcessResponse_set(BaseModel):
|
||||
sessionId: str = Field(..., max_length=64)
|
||||
|
||||
5
src/services/__init__.py
Normal file
5
src/services/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""Application services."""
|
||||
|
||||
from .llm import OpenAIResponseGenerator
|
||||
|
||||
__all__ = ["OpenAIResponseGenerator"]
|
||||
57
src/services/llm.py
Normal file
57
src/services/llm.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""Minimal OpenAI-compatible response generator for LangGraph."""
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from ..agent.state import AccidentGraphState, GeneratedTurn
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """你是无锡交警事故信息采集助手。
|
||||
每次回复必须严格以且仅以一个 <state>四位数字</state> 开头,随后输出用户可见正文。
|
||||
当前最小流程尚未接入完整状态机;除非用户明确要求转人工,否则使用 1002。
|
||||
用户明确要求转人工时使用 0001。
|
||||
不要输出 JSON、Markdown、分析过程或重复的 state 标签。"""
|
||||
|
||||
|
||||
class OpenAIResponseGenerator:
|
||||
"""Generate one response through an OpenAI-compatible chat endpoint."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str,
|
||||
model: str,
|
||||
base_url: str | None = None,
|
||||
timeout: float = 60.0,
|
||||
max_retries: int = 2,
|
||||
):
|
||||
self._model = model
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
)
|
||||
|
||||
async def generate(self, state: AccidentGraphState) -> GeneratedTurn:
|
||||
response = await self._client.chat.completions.create(
|
||||
model=self._model,
|
||||
messages=[
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"当前内部阶段:"
|
||||
f"{state.get('stage_code', '1002')};"
|
||||
f"当前为第 {state.get('turn_count', 0) + 1} 轮。"
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": state["input_text"]},
|
||||
],
|
||||
)
|
||||
content = response.choices[0].message.content
|
||||
if not isinstance(content, str) or not content:
|
||||
raise ValueError("LLM returned empty response content")
|
||||
return GeneratedTurn(content=content)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._client.close()
|
||||
1
src/utils/__init__.py
Normal file
1
src/utils/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Shared utility helpers."""
|
||||
135
src/utils/text_chunker.py
Normal file
135
src/utils/text_chunker.py
Normal file
@@ -0,0 +1,135 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
SENTENCE_ENDING_PUNCTUATION = frozenset(".!?;。!?;")
|
||||
SOFT_BREAK_PUNCTUATION = frozenset(",,、::")
|
||||
CLOSING_PUNCTUATION = frozenset("\"'”’)]})】》」』")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SentenceTextChunkerConfig:
|
||||
"""Configuration for streaming text chunks sent to TTS."""
|
||||
|
||||
min_chars: int = 1
|
||||
max_chars: int = 80
|
||||
use_soft_breaks: bool = True
|
||||
|
||||
|
||||
class SentenceTextChunker:
|
||||
"""Lightweight Pipecat-style sentence chunker for streaming TTS text.
|
||||
|
||||
The chunker waits for one non-whitespace lookahead character after sentence
|
||||
punctuation before emitting a sentence. This avoids splitting too early when
|
||||
a punctuation mark arrives at the end of a stream token.
|
||||
"""
|
||||
|
||||
def __init__(self, config: SentenceTextChunkerConfig | None = None) -> None:
|
||||
self._config = config or SentenceTextChunkerConfig()
|
||||
self._buffer = ""
|
||||
self._needs_lookahead = False
|
||||
self._last_soft_break = -1
|
||||
|
||||
@property
|
||||
def text(self) -> str:
|
||||
return self._buffer.strip(" ")
|
||||
|
||||
def feed(self, text: str) -> list[str]:
|
||||
"""Append streaming text and return chunks ready for TTS."""
|
||||
chunks: list[str] = []
|
||||
if not text:
|
||||
return chunks
|
||||
|
||||
for char in text:
|
||||
self._buffer += char
|
||||
index = len(self._buffer) - 1
|
||||
|
||||
if char in SOFT_BREAK_PUNCTUATION:
|
||||
self._last_soft_break = index + 1
|
||||
|
||||
if self._needs_lookahead and char.strip():
|
||||
self._needs_lookahead = False
|
||||
chunk = self._pop_sentence_chunk()
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
continue
|
||||
|
||||
if char in SENTENCE_ENDING_PUNCTUATION and not self._is_decimal_point(index):
|
||||
self._needs_lookahead = True
|
||||
|
||||
chunk = self._pop_soft_chunk_if_needed()
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
|
||||
return chunks
|
||||
|
||||
def flush(self) -> str | None:
|
||||
"""Return any remaining buffered text at end of stream."""
|
||||
if not self._buffer:
|
||||
return None
|
||||
chunk = self._buffer.strip(" ")
|
||||
self.reset()
|
||||
return chunk or None
|
||||
|
||||
def reset(self) -> None:
|
||||
self._buffer = ""
|
||||
self._needs_lookahead = False
|
||||
self._last_soft_break = -1
|
||||
|
||||
def _pop_sentence_chunk(self) -> str | None:
|
||||
end = self._sentence_end_index()
|
||||
if end is None:
|
||||
return None
|
||||
chunk = self._buffer[:end].strip(" ")
|
||||
if len(chunk) < self._config.min_chars:
|
||||
return None
|
||||
self._buffer = self._buffer[end:]
|
||||
self._last_soft_break = self._find_last_soft_break()
|
||||
return chunk
|
||||
|
||||
def _sentence_end_index(self) -> int | None:
|
||||
index = 0
|
||||
while index < len(self._buffer):
|
||||
char = self._buffer[index]
|
||||
if char in SENTENCE_ENDING_PUNCTUATION and not self._is_decimal_point(index):
|
||||
end = index + 1
|
||||
while end < len(self._buffer) and self._buffer[end] in CLOSING_PUNCTUATION:
|
||||
end += 1
|
||||
return end
|
||||
index += 1
|
||||
return None
|
||||
|
||||
def _pop_soft_chunk_if_needed(self) -> str | None:
|
||||
if (
|
||||
not self._config.use_soft_breaks
|
||||
or self._config.max_chars <= 0
|
||||
or len(self._buffer) < self._config.max_chars
|
||||
or self._last_soft_break <= 0
|
||||
):
|
||||
return None
|
||||
|
||||
chunk = self._buffer[: self._last_soft_break].strip(" ")
|
||||
if len(chunk) < self._config.min_chars:
|
||||
return None
|
||||
|
||||
self._buffer = self._buffer[self._last_soft_break :]
|
||||
self._last_soft_break = self._find_last_soft_break()
|
||||
self._needs_lookahead = False
|
||||
return chunk
|
||||
|
||||
def _find_last_soft_break(self) -> int:
|
||||
for index in range(len(self._buffer) - 1, -1, -1):
|
||||
if self._buffer[index] in SOFT_BREAK_PUNCTUATION:
|
||||
return index + 1
|
||||
return -1
|
||||
|
||||
def _is_decimal_point(self, index: int) -> bool:
|
||||
if self._buffer[index] != ".":
|
||||
return False
|
||||
return (
|
||||
index > 0
|
||||
and index + 1 < len(self._buffer)
|
||||
and self._buffer[index - 1].isdigit()
|
||||
and self._buffer[index + 1].isdigit()
|
||||
)
|
||||
1
src/voice/__init__.py
Normal file
1
src/voice/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Voice websocket demo (product-ws / va.ws.v1) powered by Pipecat."""
|
||||
313
src/voice/config.py
Normal file
313
src/voice/config.py
Normal file
@@ -0,0 +1,313 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
DEFAULT_VOICE_CONFIG_REL = "config/voice.json"
|
||||
|
||||
|
||||
def resolve_voice_config_path() -> Path:
|
||||
"""Return the voice config path from VOICE_CONFIG or the default."""
|
||||
configured = os.getenv("VOICE_CONFIG", DEFAULT_VOICE_CONFIG_REL).strip()
|
||||
if not configured:
|
||||
configured = DEFAULT_VOICE_CONFIG_REL
|
||||
path = Path(configured)
|
||||
if not path.is_absolute():
|
||||
path = PROJECT_ROOT / path
|
||||
return path
|
||||
|
||||
|
||||
DEFAULT_VOICE_CONFIG = resolve_voice_config_path()
|
||||
|
||||
SUPPORTED_LLM_PROVIDERS = frozenset({"openai", "fastgpt"})
|
||||
_LLM_PROVIDER_ALIASES = {"llm": "openai", "openai": "openai", "fastgpt": "fastgpt"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServerConfig:
|
||||
host: str = "0.0.0.0"
|
||||
port: int = 8000
|
||||
cors_origins: list[str] = field(default_factory=list)
|
||||
serve_webpage: bool = True
|
||||
webpage_mount: str = "/voice-demo"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AudioConfig:
|
||||
sample_rate_hz: int = 16000
|
||||
channels: int = 1
|
||||
frame_ms: int = 20
|
||||
|
||||
@property
|
||||
def frame_bytes(self) -> int:
|
||||
return int(self.sample_rate_hz * self.frame_ms / 1000) * self.channels * 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionConfig:
|
||||
inactivity_timeout_sec: int = 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VADConfig:
|
||||
confidence: float = 0.7
|
||||
start_secs: float = 0.2
|
||||
stop_secs: float = 0.6
|
||||
min_volume: float = 0.6
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TurnConfig:
|
||||
vad: VADConfig = field(default_factory=VADConfig)
|
||||
user_speech_timeout_sec: float = 1.0
|
||||
idle_prompt_timeout_sec: float = 0.0
|
||||
idle_prompt_max_count: int = 1
|
||||
idle_prompt_text: str = (
|
||||
"我先停在这里。你可以继续说你的想法,"
|
||||
"或者让我根据刚才的内容帮你整理下一步。"
|
||||
)
|
||||
interruption_min_chars: int = 3
|
||||
interruption_use_interim: bool = True
|
||||
interruption_short_replies: list[str] = field(
|
||||
default_factory=lambda: [
|
||||
"是",
|
||||
"是的",
|
||||
"对",
|
||||
"对的",
|
||||
"嗯",
|
||||
"好",
|
||||
"好的",
|
||||
"行",
|
||||
"可以",
|
||||
"没问题",
|
||||
"不是",
|
||||
"不",
|
||||
"不行",
|
||||
"不用",
|
||||
"不要",
|
||||
"没有",
|
||||
"否",
|
||||
"no",
|
||||
"yes",
|
||||
"ok",
|
||||
"okay",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResponseStateConfig:
|
||||
enabled: bool = False
|
||||
tag: str = "state"
|
||||
event_type: str = "response.state"
|
||||
max_prefix_chars: int = 256
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AgentConfig:
|
||||
system_prompt: str = "You are a helpful, friendly voice assistant."
|
||||
greeting: str | None = None
|
||||
greeting_mode: str = "generated"
|
||||
fastgpt_reconnect_greeting: str = "欢迎回来继续对话,请告诉我准备好了之后继续办理"
|
||||
response_state: ResponseStateConfig = field(default_factory=ResponseStateConfig)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LLMConfig:
|
||||
provider: str = "openai"
|
||||
api_key: str = ""
|
||||
base_url: str | None = None
|
||||
model: str = "gpt-4o-mini"
|
||||
app_id: str | None = None
|
||||
temperature: float | None = 0.7
|
||||
chat_id: str | None = None
|
||||
variables: dict[str, str] = field(default_factory=dict)
|
||||
detail: bool = False
|
||||
timeout_sec: float = 60.0
|
||||
image_input_mode: str = "base64"
|
||||
|
||||
@property
|
||||
def is_fastgpt(self) -> bool:
|
||||
return self.provider == "fastgpt"
|
||||
|
||||
@property
|
||||
def is_openai(self) -> bool:
|
||||
return self.provider == "openai"
|
||||
|
||||
@property
|
||||
def uses_local_context_history(self) -> bool:
|
||||
"""Whether the pipeline should seed and maintain local LLM context history."""
|
||||
return not self.is_fastgpt
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class STTConfig:
|
||||
provider: str = "openai"
|
||||
app_id: str = ""
|
||||
api_key: str = ""
|
||||
api_secret: str = ""
|
||||
base_url: str | None = None
|
||||
model: str = "gpt-4o-mini-transcribe"
|
||||
language: str | None = "en"
|
||||
domain: str = "iat"
|
||||
accent: str = "mandarin"
|
||||
encoding: str = "raw"
|
||||
frame_size: int = 1280
|
||||
timeout_sec: float = 10.0
|
||||
dynamic_correction: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TTSConfig:
|
||||
provider: str = "openai"
|
||||
app_id: str = ""
|
||||
api_key: str = ""
|
||||
api_secret: str = ""
|
||||
base_url: str | None = None
|
||||
model: str = "gpt-4o-mini-tts"
|
||||
voice: str = "alloy"
|
||||
aue: str = "raw"
|
||||
tte: str = "UTF8"
|
||||
speed: int = 50
|
||||
volume: int = 50
|
||||
pitch: int = 50
|
||||
timeout_sec: float = 30.0
|
||||
source_sample_rate_hz: int | None = None
|
||||
oral_level: str = "mid"
|
||||
text_aggregation_mode: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServicesConfig:
|
||||
llm: LLMConfig = field(default_factory=LLMConfig)
|
||||
stt: STTConfig = field(default_factory=STTConfig)
|
||||
tts: TTSConfig = field(default_factory=TTSConfig)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EngineConfig:
|
||||
server: ServerConfig = field(default_factory=ServerConfig)
|
||||
audio: AudioConfig = field(default_factory=AudioConfig)
|
||||
session: SessionConfig = field(default_factory=SessionConfig)
|
||||
turn: TurnConfig = field(default_factory=TurnConfig)
|
||||
agent: AgentConfig = field(default_factory=AgentConfig)
|
||||
services: ServicesConfig = field(default_factory=ServicesConfig)
|
||||
|
||||
|
||||
def load_config(path: str | Path | None = None) -> EngineConfig:
|
||||
config_path = Path(path) if path is not None else resolve_voice_config_path()
|
||||
if not config_path.is_absolute():
|
||||
config_path = PROJECT_ROOT / config_path
|
||||
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"Config file must contain a JSON object: {config_path}")
|
||||
return config_from_dict(data)
|
||||
|
||||
|
||||
def config_from_dict(data: dict) -> EngineConfig:
|
||||
services = _dict(data.get("services"))
|
||||
agent = _dict(data.get("agent"))
|
||||
if agent.get("greeting") == "":
|
||||
agent["greeting"] = None
|
||||
if agent.get("greeting_mode") not in (None, "generated", "fixed", "off", "fastgpt_opener"):
|
||||
raise ValueError(
|
||||
"agent.greeting_mode must be one of: generated, fixed, off, fastgpt_opener"
|
||||
)
|
||||
response_state = ResponseStateConfig(**_dict(agent.pop("response_state", None)))
|
||||
if response_state.max_prefix_chars < 1:
|
||||
raise ValueError("agent.response_state.max_prefix_chars must be greater than 0")
|
||||
if not response_state.tag:
|
||||
raise ValueError("agent.response_state.tag must not be empty")
|
||||
if not response_state.event_type:
|
||||
raise ValueError("agent.response_state.event_type must not be empty")
|
||||
|
||||
stt = _dict(services.get("stt") or services.get("asr"))
|
||||
if stt.get("language") == "":
|
||||
stt["language"] = None
|
||||
|
||||
llm = _dict(services.get("llm"))
|
||||
llm["provider"] = _normalize_llm_provider(llm.get("provider", LLMConfig().provider))
|
||||
if llm.get("chat_id") == "":
|
||||
llm["chat_id"] = None
|
||||
llm.pop("send_system_prompt", None)
|
||||
image_input_mode = str(
|
||||
llm.get("image_input_mode", LLMConfig().image_input_mode)
|
||||
).strip().lower()
|
||||
if image_input_mode not in {"base64", "upload"}:
|
||||
raise ValueError(
|
||||
"services.llm.image_input_mode must be 'base64' or 'upload', "
|
||||
f"got {llm.get('image_input_mode')!r}"
|
||||
)
|
||||
llm["image_input_mode"] = image_input_mode
|
||||
if llm.get("app_id") == "":
|
||||
llm["app_id"] = None
|
||||
if not isinstance(llm.get("variables"), dict):
|
||||
llm["variables"] = {}
|
||||
if agent.get("greeting_mode") == "fastgpt_opener" and llm["provider"] != "fastgpt":
|
||||
raise ValueError(
|
||||
"agent.greeting_mode='fastgpt_opener' requires services.llm.provider='fastgpt'"
|
||||
)
|
||||
|
||||
turn = _dict(data.get("turn"))
|
||||
vad = _dict(turn.get("vad"))
|
||||
|
||||
return EngineConfig(
|
||||
server=ServerConfig(**_dict(data.get("server"))),
|
||||
audio=AudioConfig(**_dict(data.get("audio"))),
|
||||
session=SessionConfig(**_dict(data.get("session"))),
|
||||
turn=TurnConfig(
|
||||
vad=VADConfig(**vad),
|
||||
user_speech_timeout_sec=float(
|
||||
turn.get("user_speech_timeout_sec", TurnConfig().user_speech_timeout_sec)
|
||||
),
|
||||
idle_prompt_timeout_sec=float(
|
||||
turn.get("idle_prompt_timeout_sec", TurnConfig().idle_prompt_timeout_sec)
|
||||
),
|
||||
idle_prompt_max_count=int(
|
||||
turn.get("idle_prompt_max_count", TurnConfig().idle_prompt_max_count)
|
||||
),
|
||||
idle_prompt_text=str(
|
||||
turn.get("idle_prompt_text", TurnConfig().idle_prompt_text)
|
||||
),
|
||||
interruption_min_chars=int(
|
||||
turn.get("interruption_min_chars", TurnConfig().interruption_min_chars)
|
||||
),
|
||||
interruption_use_interim=bool(
|
||||
turn.get("interruption_use_interim", TurnConfig().interruption_use_interim)
|
||||
),
|
||||
interruption_short_replies=list(
|
||||
turn.get(
|
||||
"interruption_short_replies",
|
||||
TurnConfig().interruption_short_replies,
|
||||
)
|
||||
),
|
||||
),
|
||||
agent=AgentConfig(**agent, response_state=response_state),
|
||||
services=ServicesConfig(
|
||||
llm=LLMConfig(**llm),
|
||||
stt=STTConfig(**stt),
|
||||
tts=TTSConfig(**_dict(services.get("tts"))),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _dict(value: object) -> dict:
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _normalize_llm_provider(value: object) -> str:
|
||||
provider = str(value or LLMConfig().provider).strip().lower()
|
||||
normalized = _LLM_PROVIDER_ALIASES.get(provider)
|
||||
if normalized is None:
|
||||
supported = ", ".join(sorted(SUPPORTED_LLM_PROVIDERS | {"llm"}))
|
||||
raise ValueError(
|
||||
f"services.llm.provider must be one of: {supported}; got {value!r}"
|
||||
)
|
||||
return normalized
|
||||
40
src/voice/context_sync.py
Normal file
40
src/voice/context_sync.py
Normal file
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pipecat.frames.frames import Frame, InterruptionFrame, LLMMessagesAppendFrame
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
from .text_stream import ProductTextStreamProcessor, maybe_sync_assistant_context
|
||||
|
||||
|
||||
class AssistantContextSyncProcessor(FrameProcessor):
|
||||
"""Sync LLM context to urgent-streamed assistant text before text-input turns.
|
||||
|
||||
``input.text`` with ``interrupt: true`` queues ``InterruptionFrame`` before
|
||||
``LLMMessagesAppendFrame``. This processor runs context repair after the
|
||||
interrupt has propagated (including TTS-phase interrupts) and before the new
|
||||
user message is appended.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
text_stream: ProductTextStreamProcessor,
|
||||
assistant_aggregator: Any,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._text_stream = text_stream
|
||||
self._assistant_aggregator = assistant_aggregator
|
||||
self._sync_on_next_append = False
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, InterruptionFrame):
|
||||
self._sync_on_next_append = True
|
||||
elif isinstance(frame, LLMMessagesAppendFrame) and self._sync_on_next_append:
|
||||
self._sync_on_next_append = False
|
||||
maybe_sync_assistant_context(self._assistant_aggregator, self._text_stream)
|
||||
|
||||
await self.push_frame(frame, direction)
|
||||
564
src/voice/fastgpt_llm.py
Normal file
564
src/voice/fastgpt_llm.py
Normal file
@@ -0,0 +1,564 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import os
|
||||
import tempfile
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastgpt_client import AsyncChatClient, FastGPTInteractiveEvent, aiter_stream_events
|
||||
from fastgpt_client.exceptions import FastGPTError
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
EndFrame,
|
||||
Frame,
|
||||
InterruptionFrame,
|
||||
LLMContextFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
LLMTextFrame,
|
||||
OutputTransportMessageFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
)
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.frame_processor import FrameDirection
|
||||
from pipecat.services.llm_service import LLMService
|
||||
from pipecat.services.settings import LLMSettings
|
||||
|
||||
|
||||
def _extract_text_from_event(kind: str, payload: Any) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
|
||||
if kind in {"answer", "fastAnswer"}:
|
||||
text = payload.get("text")
|
||||
if isinstance(text, str) and text:
|
||||
return text
|
||||
|
||||
choices = payload.get("choices") if isinstance(payload.get("choices"), list) else []
|
||||
if not choices:
|
||||
return str(payload.get("text") or "")
|
||||
|
||||
first_choice = choices[0] if isinstance(choices[0], dict) else {}
|
||||
delta = first_choice.get("delta") if isinstance(first_choice.get("delta"), dict) else {}
|
||||
content = delta.get("content")
|
||||
if isinstance(content, str) and content:
|
||||
return content
|
||||
|
||||
message = first_choice.get("message") if isinstance(first_choice.get("message"), dict) else {}
|
||||
message_content = message.get("content")
|
||||
if isinstance(message_content, str) and message_content:
|
||||
return message_content
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _message_text(message: dict[str, Any]) -> str:
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "text":
|
||||
text = part.get("text")
|
||||
if isinstance(text, str) and text.strip():
|
||||
parts.append(text.strip())
|
||||
return " ".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def _first_nonempty_text(*values: Any) -> str:
|
||||
for value in values:
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _interactive_spoken_prompt(event: FastGPTInteractiveEvent) -> str:
|
||||
payload = event.data if isinstance(event.data, dict) else {}
|
||||
params = payload.get("params") if isinstance(payload.get("params"), dict) else {}
|
||||
|
||||
prompt = _first_nonempty_text(
|
||||
payload.get("opener"),
|
||||
params.get("opener"),
|
||||
payload.get("prompt"),
|
||||
params.get("prompt"),
|
||||
payload.get("text"),
|
||||
params.get("text"),
|
||||
payload.get("title"),
|
||||
params.get("title"),
|
||||
payload.get("description"),
|
||||
params.get("description"),
|
||||
)
|
||||
if prompt:
|
||||
return prompt
|
||||
|
||||
if event.interaction_type == "userSelect":
|
||||
raw_options = (
|
||||
params.get("userSelectOptions")
|
||||
if isinstance(params.get("userSelectOptions"), list)
|
||||
else []
|
||||
)
|
||||
labels: list[str] = []
|
||||
for index, raw in enumerate(raw_options, start=1):
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
labels.append(f"{index}. {raw.strip()}")
|
||||
elif isinstance(raw, dict):
|
||||
label = _first_nonempty_text(raw.get("label"), raw.get("value"))
|
||||
if label:
|
||||
labels.append(f"{index}. {label}")
|
||||
if labels:
|
||||
return "请选择:" + ",".join(labels)
|
||||
return "请选择一个选项。"
|
||||
|
||||
if event.interaction_type == "userInput":
|
||||
input_form = params.get("inputForm") if isinstance(params.get("inputForm"), list) else []
|
||||
labels = [
|
||||
_first_nonempty_text(field.get("label"), field.get("name"))
|
||||
for field in input_form
|
||||
if isinstance(field, dict)
|
||||
]
|
||||
labels = [label for label in labels if label]
|
||||
if labels:
|
||||
return "请提供以下信息:" + ",".join(labels)
|
||||
return "请补充所需信息。"
|
||||
|
||||
return "请继续。"
|
||||
|
||||
|
||||
IMAGE_INPUT_MODE_BASE64 = "base64"
|
||||
IMAGE_INPUT_MODE_UPLOAD = "upload"
|
||||
SUPPORTED_IMAGE_INPUT_MODES = frozenset({IMAGE_INPUT_MODE_BASE64, IMAGE_INPUT_MODE_UPLOAD})
|
||||
|
||||
_MIME_TO_EXT = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/webp": ".webp",
|
||||
}
|
||||
|
||||
|
||||
def _message_has_image(message: dict[str, Any]) -> bool:
|
||||
content = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return False
|
||||
return any(
|
||||
isinstance(part, dict) and part.get("type") == "image_url"
|
||||
for part in content
|
||||
)
|
||||
|
||||
|
||||
def _redact_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Replace base64 image data URLs with a short placeholder for logging."""
|
||||
redacted: list[dict[str, Any]] = []
|
||||
for message in messages:
|
||||
content = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
redacted.append(message)
|
||||
continue
|
||||
parts: list[Any] = []
|
||||
for part in content:
|
||||
if (
|
||||
isinstance(part, dict)
|
||||
and part.get("type") == "image_url"
|
||||
and isinstance(part.get("image_url"), dict)
|
||||
):
|
||||
url = str(part["image_url"].get("url") or "")
|
||||
parts.append({"type": "image_url", "image_url": {"url": f"<{len(url)} chars>"}})
|
||||
else:
|
||||
parts.append(part)
|
||||
redacted.append({**message, "content": parts})
|
||||
return redacted
|
||||
|
||||
|
||||
@dataclass
|
||||
class FastGPTLLMSettings(LLMSettings):
|
||||
variables: dict[str, Any] = field(default_factory=dict)
|
||||
detail: bool = False
|
||||
|
||||
|
||||
def _default_fastgpt_settings(*, model: str = "fastgpt") -> FastGPTLLMSettings:
|
||||
return FastGPTLLMSettings(
|
||||
model=model,
|
||||
system_instruction=None,
|
||||
temperature=None,
|
||||
max_tokens=None,
|
||||
top_p=None,
|
||||
top_k=None,
|
||||
frequency_penalty=None,
|
||||
presence_penalty=None,
|
||||
seed=None,
|
||||
filter_incomplete_user_turns=False,
|
||||
user_turn_completion_config=None,
|
||||
variables={},
|
||||
detail=False,
|
||||
)
|
||||
|
||||
|
||||
class FastGPTLLMService(LLMService):
|
||||
"""FastGPT LLM service using chatId server-side memory and workflow variables."""
|
||||
|
||||
Settings = FastGPTLLMSettings
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
chat_id: str | None = None,
|
||||
app_id: str | None = None,
|
||||
greeting_prompt: str | None = None,
|
||||
timeout: float = 60.0,
|
||||
image_input_mode: str = IMAGE_INPUT_MODE_BASE64,
|
||||
settings: FastGPTLLMSettings | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
default_settings = _default_fastgpt_settings()
|
||||
if settings is not None:
|
||||
default_settings.apply_update(settings)
|
||||
super().__init__(settings=default_settings, **kwargs)
|
||||
|
||||
self._chat_id = chat_id or f"voice_{uuid.uuid4().hex[:16]}"
|
||||
self._app_id = (app_id or "").strip()
|
||||
self._greeting_prompt = (greeting_prompt or "你好").strip() or "你好"
|
||||
self._client = AsyncChatClient(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
timeout=timeout,
|
||||
)
|
||||
self._active_response = None
|
||||
|
||||
mode = (image_input_mode or IMAGE_INPUT_MODE_BASE64).strip().lower()
|
||||
if mode not in SUPPORTED_IMAGE_INPUT_MODES:
|
||||
raise ValueError(
|
||||
f"Unsupported image_input_mode {image_input_mode!r}; "
|
||||
f"expected one of {sorted(SUPPORTED_IMAGE_INPUT_MODES)}"
|
||||
)
|
||||
if mode == IMAGE_INPUT_MODE_UPLOAD and not self._app_id:
|
||||
logger.warning(
|
||||
"FastGPT image_input_mode='upload' requires app_id; "
|
||||
"falling back to inline base64"
|
||||
)
|
||||
mode = IMAGE_INPUT_MODE_BASE64
|
||||
self._image_input_mode = mode
|
||||
|
||||
@property
|
||||
def app_id(self) -> str:
|
||||
return self._app_id
|
||||
|
||||
@property
|
||||
def chat_id(self) -> str:
|
||||
return self._chat_id
|
||||
|
||||
def set_variables(self, variables: dict[str, Any]) -> None:
|
||||
merged = dict(self._settings.variables)
|
||||
merged.update(variables)
|
||||
self._settings.variables = merged
|
||||
|
||||
async def stop(self, frame: EndFrame) -> None:
|
||||
await self._close_active_response()
|
||||
await self._client.close()
|
||||
await super().stop(frame)
|
||||
|
||||
async def cancel(self, frame: CancelFrame) -> None:
|
||||
await self._close_active_response()
|
||||
await super().cancel(frame)
|
||||
|
||||
async def _handle_interruptions(self, _: InterruptionFrame) -> None:
|
||||
await self._close_active_response()
|
||||
await super()._handle_interruptions(_)
|
||||
|
||||
@staticmethod
|
||||
def _welcome_text_from_init_payload(payload: Any) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
|
||||
for container in (payload.get("app"), payload.get("data"), payload):
|
||||
if not isinstance(container, dict):
|
||||
continue
|
||||
nested_app = container.get("app")
|
||||
if isinstance(nested_app, dict):
|
||||
text = FastGPTLLMService._welcome_text_from_app(nested_app)
|
||||
if text:
|
||||
return text
|
||||
text = FastGPTLLMService._welcome_text_from_app(container)
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _welcome_text_from_app(app_payload: dict[str, Any]) -> str:
|
||||
chat_config = (
|
||||
app_payload.get("chatConfig")
|
||||
if isinstance(app_payload.get("chatConfig"), dict)
|
||||
else {}
|
||||
)
|
||||
return _first_nonempty_text(
|
||||
chat_config.get("welcomeText"),
|
||||
app_payload.get("welcomeText"),
|
||||
app_payload.get("opener"),
|
||||
app_payload.get("intro"),
|
||||
)
|
||||
|
||||
async def fetch_welcome_text(self) -> str | None:
|
||||
"""Return FastGPT app welcome text from chat init when ``app_id`` is configured."""
|
||||
if not self._app_id:
|
||||
return None
|
||||
|
||||
try:
|
||||
response = await self._client.get_chat_init(
|
||||
appId=self._app_id,
|
||||
chatId=self._chat_id,
|
||||
)
|
||||
response.raise_for_status()
|
||||
text = self._welcome_text_from_init_payload(response.json())
|
||||
if text:
|
||||
logger.info(f"FastGPT app opener loaded for appId={self._app_id}")
|
||||
return text or None
|
||||
except FastGPTError as exc:
|
||||
logger.warning(f"FastGPT chat init failed: {exc}")
|
||||
except httpx.HTTPError as exc:
|
||||
logger.warning(f"FastGPT chat init HTTP error: {exc}")
|
||||
except Exception as exc:
|
||||
logger.warning(f"FastGPT chat init error: {exc}")
|
||||
return None
|
||||
|
||||
async def has_chat_history(self) -> bool:
|
||||
"""Return whether FastGPT has persisted records for this chatId."""
|
||||
if not self._app_id:
|
||||
return False
|
||||
|
||||
try:
|
||||
response = await self._client.get_chat_records(
|
||||
appId=self._app_id,
|
||||
chatId=self._chat_id,
|
||||
offset=0,
|
||||
pageSize=1,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
records = data.get("data", {}).get("list", [])
|
||||
return isinstance(records, list) and bool(records)
|
||||
except FastGPTError as exc:
|
||||
logger.warning(f"FastGPT chat records failed: {exc}")
|
||||
except httpx.HTTPError as exc:
|
||||
logger.warning(f"FastGPT chat records HTTP error: {exc}")
|
||||
except Exception as exc:
|
||||
logger.warning(f"FastGPT chat records error: {exc}")
|
||||
return False
|
||||
|
||||
async def fetch_session_greeting_text(self, reconnect_greeting: str) -> str | None:
|
||||
"""Use opener for a new chatId and a fixed greeting for reconnects."""
|
||||
if await self.has_chat_history():
|
||||
logger.info(f"FastGPT chatId={self._chat_id} has history; using reconnect greeting")
|
||||
return reconnect_greeting.strip() or None
|
||||
|
||||
logger.info(f"FastGPT chatId={self._chat_id} has no history; using app opener")
|
||||
return await self.fetch_welcome_text()
|
||||
|
||||
async def _close_active_response(self) -> None:
|
||||
response = self._active_response
|
||||
self._active_response = None
|
||||
if response is not None:
|
||||
await response.aclose()
|
||||
|
||||
def _build_fastgpt_messages(self, context: LLMContext) -> list[dict[str, Any]]:
|
||||
raw_messages = context.get_messages()
|
||||
|
||||
for message in reversed(raw_messages):
|
||||
if not isinstance(message, dict) or message.get("role") != "user":
|
||||
continue
|
||||
if _message_has_image(message):
|
||||
# Multimodal turn: forward the OpenAI-style content list as-is
|
||||
# (text parts + image_url with a base64 data URL). FastGPT's
|
||||
# /chat/completions accepts this directly.
|
||||
return [{"role": "user", "content": message["content"]}]
|
||||
text = _message_text(message)
|
||||
if text:
|
||||
return [{"role": "user", "content": text}]
|
||||
|
||||
return [{"role": "user", "content": self._greeting_prompt}]
|
||||
|
||||
async def _resolve_image_inputs(
|
||||
self, messages: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
"""In ``upload`` mode, replace inline base64 image data URLs with uploaded URLs.
|
||||
|
||||
In ``base64`` mode the messages are returned untouched (inline data URLs).
|
||||
New message/content objects are built so the shared ``LLMContext`` messages
|
||||
are never mutated.
|
||||
"""
|
||||
if self._image_input_mode != IMAGE_INPUT_MODE_UPLOAD:
|
||||
return messages
|
||||
|
||||
resolved: list[dict[str, Any]] = []
|
||||
for message in messages:
|
||||
content = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
resolved.append(message)
|
||||
continue
|
||||
|
||||
new_content: list[Any] = []
|
||||
for part in content:
|
||||
url = (
|
||||
part.get("image_url", {}).get("url")
|
||||
if isinstance(part, dict) and part.get("type") == "image_url"
|
||||
else None
|
||||
)
|
||||
if isinstance(url, str) and url.startswith("data:image/"):
|
||||
uploaded = await self._upload_data_url(url)
|
||||
new_content.append(
|
||||
{"type": "image_url", "image_url": {"url": uploaded}}
|
||||
)
|
||||
else:
|
||||
new_content.append(part)
|
||||
resolved.append({**message, "content": new_content})
|
||||
|
||||
return resolved
|
||||
|
||||
async def _upload_data_url(self, data_url: str) -> str:
|
||||
"""Upload a ``data:image/...;base64,...`` URL via FastGPT and return its URL.
|
||||
|
||||
Falls back to the original data URL if parsing or upload fails so the turn
|
||||
still proceeds with inline base64.
|
||||
"""
|
||||
header, _, payload = data_url.partition(",")
|
||||
mime_type = header[len("data:"):].split(";", 1)[0].strip() or "image/jpeg"
|
||||
try:
|
||||
raw = base64.b64decode(payload, validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
logger.warning(f"FastGPT image upload skipped; invalid base64: {exc}")
|
||||
return data_url
|
||||
|
||||
suffix = _MIME_TO_EXT.get(mime_type, ".jpg")
|
||||
tmp_path: str | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(raw)
|
||||
tmp_path = tmp.name
|
||||
result = await self._client.upload_chat_image(
|
||||
appId=self._app_id,
|
||||
chatId=self._chat_id,
|
||||
file_path=tmp_path,
|
||||
)
|
||||
url = result.get("url") if isinstance(result, dict) else None
|
||||
if isinstance(url, str) and url:
|
||||
logger.info(
|
||||
f"FastGPT image uploaded chatId={self._chat_id} "
|
||||
f"bytes={len(raw)} url={url}"
|
||||
)
|
||||
return url
|
||||
logger.warning("FastGPT image upload returned no url; using inline base64")
|
||||
return data_url
|
||||
except Exception as exc:
|
||||
logger.warning(f"FastGPT image upload failed; using inline base64: {exc}")
|
||||
return data_url
|
||||
finally:
|
||||
if tmp_path is not None:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
async def _process_context(self, context: LLMContext) -> None:
|
||||
messages = self._build_fastgpt_messages(context)
|
||||
messages = await self._resolve_image_inputs(messages)
|
||||
variables = self._settings.variables or None
|
||||
|
||||
logger.info(
|
||||
"FastGPT chat completion "
|
||||
f"chatId={self._chat_id} appId={self._app_id or '-'} "
|
||||
f"variables={sorted((variables or {}).keys())} "
|
||||
f"messages={_redact_messages_for_log(messages)!r}"
|
||||
)
|
||||
|
||||
await self.start_ttfb_metrics()
|
||||
|
||||
try:
|
||||
response = await self._client.create_chat_completion(
|
||||
messages=messages,
|
||||
stream=True,
|
||||
chatId=self._chat_id,
|
||||
variables=variables,
|
||||
detail=self._settings.detail,
|
||||
)
|
||||
except FastGPTError as exc:
|
||||
await self.push_error(error_msg=f"FastGPT request failed: {exc}", exception=exc)
|
||||
return
|
||||
except httpx.HTTPError as exc:
|
||||
await self.push_error(error_msg=f"FastGPT HTTP error: {exc}", exception=exc)
|
||||
return
|
||||
|
||||
self._active_response = response
|
||||
|
||||
try:
|
||||
async for event in aiter_stream_events(response):
|
||||
if event.kind in {"data", "answer", "fastAnswer"}:
|
||||
text = _extract_text_from_event(event.kind, event.data)
|
||||
if text:
|
||||
await self.stop_ttfb_metrics()
|
||||
await self.push_frame(LLMTextFrame(text))
|
||||
continue
|
||||
|
||||
if event.kind == "interactive" and isinstance(event, FastGPTInteractiveEvent):
|
||||
await self._handle_interactive(event)
|
||||
break
|
||||
|
||||
if event.kind == "error":
|
||||
payload = event.data if isinstance(event.data, dict) else {}
|
||||
message = _first_nonempty_text(
|
||||
payload.get("message"),
|
||||
payload.get("error"),
|
||||
) or "FastGPT stream error"
|
||||
await self.push_error(error_msg=message)
|
||||
break
|
||||
|
||||
if event.kind == "done":
|
||||
break
|
||||
finally:
|
||||
self._active_response = None
|
||||
await response.aclose()
|
||||
|
||||
async def _handle_interactive(self, event: FastGPTInteractiveEvent) -> None:
|
||||
prompt = _interactive_spoken_prompt(event)
|
||||
if prompt:
|
||||
await self.stop_ttfb_metrics()
|
||||
await self.push_frame(LLMTextFrame(prompt))
|
||||
|
||||
await self.push_frame(
|
||||
OutputTransportMessageFrame(
|
||||
message={
|
||||
"type": "response.interactive",
|
||||
"interaction_type": event.interaction_type,
|
||||
"data": event.data,
|
||||
}
|
||||
),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, LLMContextFrame):
|
||||
try:
|
||||
await self.push_frame(LLMFullResponseStartFrame())
|
||||
await self.start_processing_metrics()
|
||||
await self._process_context(frame.context)
|
||||
except httpx.TimeoutException as exc:
|
||||
await self._call_event_handler("on_completion_timeout")
|
||||
await self.push_error(error_msg="FastGPT completion timeout", exception=exc)
|
||||
except Exception as exc:
|
||||
await self.push_error(error_msg=f"FastGPT completion error: {exc}", exception=exc)
|
||||
finally:
|
||||
await self.stop_processing_metrics()
|
||||
await self.push_frame(LLMFullResponseEndFrame())
|
||||
else:
|
||||
await self.push_frame(frame, direction)
|
||||
291
src/voice/pipeline.py
Normal file
291
src/voice/pipeline.py
Normal file
@@ -0,0 +1,291 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||
from pipecat.frames.frames import (
|
||||
LLMRunFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
TTSSpeakFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
)
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.aggregators.llm_response_universal import (
|
||||
AssistantTurnStoppedMessage,
|
||||
LLMContextAggregatorPair,
|
||||
LLMUserAggregatorParams,
|
||||
UserTurnStoppedMessage,
|
||||
)
|
||||
from pipecat.serializers.base_serializer import FrameSerializer
|
||||
from pipecat.serializers.protobuf import ProtobufFrameSerializer
|
||||
from pipecat.transports.websocket.fastapi import (
|
||||
FastAPIWebsocketParams,
|
||||
FastAPIWebsocketTransport,
|
||||
)
|
||||
from pipecat.turns.user_stop.speech_timeout_user_turn_stop_strategy import (
|
||||
SpeechTimeoutUserTurnStopStrategy,
|
||||
)
|
||||
from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
||||
|
||||
from .config import EngineConfig
|
||||
from .context_sync import AssistantContextSyncProcessor
|
||||
from .fastgpt_llm import FastGPTLLMService
|
||||
from .protocol import ProductWebsocketSerializer
|
||||
from .services import create_llm_service, create_stt_service, create_tts_service
|
||||
from .response_state import StateTagResponseProcessor
|
||||
from .text_input import ProductTextInputProcessor
|
||||
from .text_stream import ProductTextStreamProcessor, maybe_sync_assistant_context
|
||||
from .transcript_stream import ProductTranscriptStreamProcessor
|
||||
from .turn_start import InterruptionGateUserTurnStartStrategy
|
||||
|
||||
|
||||
def _chat_id_from_websocket(websocket) -> str | None:
|
||||
query_params = getattr(websocket, "query_params", None)
|
||||
if not query_params:
|
||||
return None
|
||||
|
||||
for name in ("chatId", "chat_id"):
|
||||
value = query_params.get(name)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
|
||||
async def run_product_voice_pipeline(websocket, config: EngineConfig) -> None:
|
||||
await run_pipeline_with_serializer(
|
||||
websocket,
|
||||
config,
|
||||
serializer=ProductWebsocketSerializer(
|
||||
sample_rate=config.audio.sample_rate_hz,
|
||||
channels=config.audio.channels,
|
||||
),
|
||||
client_label="Product JSON",
|
||||
)
|
||||
|
||||
|
||||
async def run_voice_pipeline(websocket, config: EngineConfig) -> None:
|
||||
await run_pipeline_with_serializer(
|
||||
websocket,
|
||||
config,
|
||||
serializer=ProtobufFrameSerializer(),
|
||||
client_label="Pipecat protobuf",
|
||||
)
|
||||
|
||||
|
||||
async def run_pipeline_with_serializer(
|
||||
websocket,
|
||||
config: EngineConfig,
|
||||
*,
|
||||
serializer: FrameSerializer,
|
||||
client_label: str,
|
||||
) -> None:
|
||||
transport = FastAPIWebsocketTransport(
|
||||
websocket=websocket,
|
||||
params=FastAPIWebsocketParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
audio_in_sample_rate=config.audio.sample_rate_hz,
|
||||
audio_out_sample_rate=config.audio.sample_rate_hz,
|
||||
audio_in_channels=config.audio.channels,
|
||||
audio_out_channels=config.audio.channels,
|
||||
serializer=serializer,
|
||||
session_timeout=None,
|
||||
),
|
||||
)
|
||||
|
||||
stt = create_stt_service(config.services.stt, config.audio)
|
||||
|
||||
llm_config = config.services.llm
|
||||
chat_id = _chat_id_from_websocket(websocket) or f"voice_{uuid.uuid4().hex[:16]}"
|
||||
llm = create_llm_service(
|
||||
llm_config,
|
||||
chat_id=chat_id,
|
||||
session_variables={"session_id": chat_id, "channel": "voice"},
|
||||
greeting_prompt=config.agent.greeting,
|
||||
)
|
||||
if llm_config.is_fastgpt:
|
||||
logger.info(f"LLM backend=fastgpt chatId={chat_id} appId={llm_config.app_id or '-'}")
|
||||
else:
|
||||
logger.info(f"LLM backend=openai model={llm_config.model}")
|
||||
|
||||
tts = create_tts_service(config.services.tts, config.audio)
|
||||
|
||||
messages: list[dict[str, str]] = []
|
||||
if llm_config.uses_local_context_history:
|
||||
messages = [{"role": "system", "content": config.agent.system_prompt}]
|
||||
if config.agent.greeting and config.agent.greeting_mode == "generated":
|
||||
messages.append({"role": "system", "content": config.agent.greeting})
|
||||
|
||||
context = LLMContext(messages)
|
||||
|
||||
vad_params = VADParams(
|
||||
confidence=config.turn.vad.confidence,
|
||||
start_secs=config.turn.vad.start_secs,
|
||||
stop_secs=config.turn.vad.stop_secs,
|
||||
min_volume=config.turn.vad.min_volume,
|
||||
)
|
||||
# Replace pipecat's default stop strategy (Smart Turn v3) with a simple
|
||||
# silence-timeout strategy. Smart Turn v3 was finalizing every short
|
||||
# Chinese phrase as a complete turn, which caused one logical utterance
|
||||
# to become several LLM calls and several user bubbles in the UI. The
|
||||
# timeout strategy waits for `user_speech_timeout_sec` of silence
|
||||
# (re-armed every time the user resumes speaking) before declaring the
|
||||
# turn finished — which is what we actually want for streaming ASRs.
|
||||
user_turn_strategies = UserTurnStrategies(
|
||||
start=[
|
||||
InterruptionGateUserTurnStartStrategy(
|
||||
min_chars_when_bot_speaking=config.turn.interruption_min_chars,
|
||||
allowed_short_replies=config.turn.interruption_short_replies,
|
||||
use_interim=config.turn.interruption_use_interim,
|
||||
),
|
||||
],
|
||||
stop=[
|
||||
SpeechTimeoutUserTurnStopStrategy(
|
||||
user_speech_timeout=config.turn.user_speech_timeout_sec,
|
||||
),
|
||||
],
|
||||
)
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
vad_analyzer=SileroVADAnalyzer(params=vad_params),
|
||||
user_turn_strategies=user_turn_strategies,
|
||||
user_idle_timeout=config.turn.idle_prompt_timeout_sec,
|
||||
),
|
||||
)
|
||||
|
||||
text_stream = ProductTextStreamProcessor()
|
||||
context_sync = AssistantContextSyncProcessor(
|
||||
text_stream=text_stream,
|
||||
assistant_aggregator=assistant_aggregator,
|
||||
)
|
||||
|
||||
processors = [
|
||||
transport.input(),
|
||||
ProductTextInputProcessor(),
|
||||
stt,
|
||||
ProductTranscriptStreamProcessor(),
|
||||
context_sync,
|
||||
user_aggregator,
|
||||
llm,
|
||||
]
|
||||
if config.agent.response_state.enabled:
|
||||
processors.append(StateTagResponseProcessor(config.agent.response_state))
|
||||
processors.extend(
|
||||
[
|
||||
text_stream,
|
||||
tts,
|
||||
transport.output(),
|
||||
assistant_aggregator,
|
||||
]
|
||||
)
|
||||
pipeline = Pipeline(processors)
|
||||
|
||||
task = PipelineTask(
|
||||
pipeline,
|
||||
params=PipelineParams(
|
||||
audio_in_sample_rate=config.audio.sample_rate_hz,
|
||||
audio_out_sample_rate=config.audio.sample_rate_hz,
|
||||
enable_metrics=True,
|
||||
enable_usage_metrics=True,
|
||||
enable_heartbeats=True,
|
||||
),
|
||||
idle_timeout_secs=config.session.inactivity_timeout_sec,
|
||||
)
|
||||
task.set_reached_upstream_filter((UserStartedSpeakingFrame,))
|
||||
idle_prompt_count = 0
|
||||
|
||||
@transport.event_handler("on_client_connected")
|
||||
async def on_client_connected(_transport, _client):
|
||||
logger.info(f"{client_label} websocket client connected")
|
||||
if config.agent.greeting_mode == "fixed" and config.agent.greeting:
|
||||
await task.queue_frames([TTSSpeakFrame(config.agent.greeting)])
|
||||
elif config.agent.greeting_mode == "fastgpt_opener":
|
||||
if isinstance(llm, FastGPTLLMService):
|
||||
welcome = await llm.fetch_session_greeting_text(
|
||||
config.agent.fastgpt_reconnect_greeting
|
||||
)
|
||||
if welcome:
|
||||
await task.queue_frames([TTSSpeakFrame(welcome)])
|
||||
else:
|
||||
logger.warning("FastGPT opener requested but no opener text was returned")
|
||||
else:
|
||||
raise RuntimeError("agent.greeting_mode='fastgpt_opener' requires FastGPT LLM service")
|
||||
elif config.agent.greeting_mode == "generated":
|
||||
await task.queue_frames([LLMRunFrame()])
|
||||
|
||||
@transport.event_handler("on_client_disconnected")
|
||||
async def on_client_disconnected(_transport, _client):
|
||||
logger.info(f"{client_label} websocket client disconnected")
|
||||
await task.cancel()
|
||||
|
||||
@transport.event_handler("on_session_timeout")
|
||||
async def on_session_timeout(_transport, _client):
|
||||
logger.info(f"{client_label} websocket session timed out")
|
||||
await task.cancel()
|
||||
|
||||
@task.event_handler("on_frame_reached_upstream")
|
||||
async def on_frame_reached_upstream(_task, _frame: UserStartedSpeakingFrame):
|
||||
nonlocal idle_prompt_count
|
||||
idle_prompt_count = 0
|
||||
|
||||
@user_aggregator.event_handler("on_user_turn_started")
|
||||
async def on_user_turn_started(_aggregator, _strategy):
|
||||
nonlocal idle_prompt_count
|
||||
idle_prompt_count = 0
|
||||
|
||||
@user_aggregator.event_handler("on_user_turn_stopped")
|
||||
async def on_user_turn_stopped(_aggregator, _strategy, message: UserTurnStoppedMessage):
|
||||
logger.info(f"User: {message.content}")
|
||||
text = (message.content or "").strip()
|
||||
if not text:
|
||||
return
|
||||
await _aggregator.push_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={
|
||||
"type": "input.transcript.final",
|
||||
"text": text,
|
||||
"user_id": message.user_id,
|
||||
"timestamp": message.timestamp,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
||||
async def on_assistant_turn_stopped(_aggregator, message: AssistantTurnStoppedMessage):
|
||||
logger.info(f"Assistant: {message.content}")
|
||||
maybe_sync_assistant_context(
|
||||
_aggregator,
|
||||
text_stream,
|
||||
committed_text=message.content or "",
|
||||
)
|
||||
text_stream.take_interrupted_stream_text()
|
||||
|
||||
@user_aggregator.event_handler("on_user_turn_idle")
|
||||
async def on_user_turn_idle(aggregator):
|
||||
nonlocal idle_prompt_count
|
||||
text = config.turn.idle_prompt_text.strip()
|
||||
if not text or config.turn.idle_prompt_max_count <= 0:
|
||||
return
|
||||
if idle_prompt_count >= config.turn.idle_prompt_max_count:
|
||||
return
|
||||
|
||||
idle_prompt_count += 1
|
||||
logger.info(
|
||||
"User idle prompt triggered "
|
||||
f"count={idle_prompt_count}/{config.turn.idle_prompt_max_count}"
|
||||
)
|
||||
await aggregator.push_frame(TTSSpeakFrame(text))
|
||||
|
||||
# NOTE: assistant turn started/final events are emitted by
|
||||
# ProductTextStreamProcessor, upstream of TTS, so text streams to the
|
||||
# client ahead of audio. This logger is kept for server-side visibility.
|
||||
|
||||
runner = PipelineRunner(handle_sigint=False)
|
||||
await runner.run(task)
|
||||
227
src/voice/protocol.py
Normal file
227
src/voice/protocol.py
Normal file
@@ -0,0 +1,227 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
BotStartedSpeakingFrame,
|
||||
BotStoppedSpeakingFrame,
|
||||
EndFrame,
|
||||
Frame,
|
||||
InputAudioRawFrame,
|
||||
InputTransportMessageFrame,
|
||||
OutputAudioRawFrame,
|
||||
OutputTransportMessageFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
TranscriptionFrame,
|
||||
UserImageRawFrame,
|
||||
)
|
||||
from pipecat.serializers.base_serializer import FrameSerializer
|
||||
|
||||
|
||||
MAX_INPUT_IMAGE_BYTES = 8 * 1024 * 1024
|
||||
SUPPORTED_INPUT_IMAGE_MIME_TYPES = {"image/jpeg", "image/png", "image/webp"}
|
||||
|
||||
|
||||
class ProductWebsocketSerializer(FrameSerializer):
|
||||
"""Stable app-facing JSON/base64 protocol adapter for Pipecat websocket transport."""
|
||||
|
||||
protocol = "va.ws.v1"
|
||||
|
||||
def __init__(self, *, sample_rate: int, channels: int):
|
||||
super().__init__()
|
||||
self._sample_rate = sample_rate
|
||||
self._channels = channels
|
||||
self._sequence = 0
|
||||
|
||||
async def serialize(self, frame: Frame) -> str | bytes | None:
|
||||
if isinstance(frame, OutputAudioRawFrame):
|
||||
return self._event(
|
||||
"response.audio.delta",
|
||||
audio=base64.b64encode(frame.audio).decode("ascii"),
|
||||
bytes=len(frame.audio),
|
||||
sample_rate=frame.sample_rate,
|
||||
channels=frame.num_channels,
|
||||
)
|
||||
|
||||
if isinstance(frame, BotStartedSpeakingFrame):
|
||||
return self._event("response.audio.started")
|
||||
|
||||
if isinstance(frame, BotStoppedSpeakingFrame):
|
||||
return self._event("response.audio.stopped")
|
||||
|
||||
if isinstance(frame, TranscriptionFrame):
|
||||
return self._event(
|
||||
"input.transcript.final",
|
||||
text=frame.text,
|
||||
user_id=frame.user_id,
|
||||
timestamp=frame.timestamp,
|
||||
)
|
||||
|
||||
# ProductTextStreamProcessor owns response.text.* events. TTS can also
|
||||
# emit TextFrame subclasses internally, so serializing them here would
|
||||
# make clients render duplicate assistant text.
|
||||
if isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
|
||||
if self.should_ignore_frame(frame):
|
||||
return None
|
||||
message = frame.message
|
||||
# Allow callers to emit a named protocol event by pushing a
|
||||
# transport-message frame whose payload already carries a `type`.
|
||||
if isinstance(message, dict) and isinstance(message.get("type"), str):
|
||||
event_type = message["type"]
|
||||
payload = {k: v for k, v in message.items() if k != "type"}
|
||||
return self._event(event_type, **payload)
|
||||
return self._event("transport.message", message=message)
|
||||
|
||||
return None
|
||||
|
||||
async def deserialize(self, data: str | bytes) -> Frame | None:
|
||||
if isinstance(data, bytes):
|
||||
return InputAudioRawFrame(
|
||||
audio=data,
|
||||
sample_rate=self._sample_rate,
|
||||
num_channels=self._channels,
|
||||
)
|
||||
|
||||
try:
|
||||
message = json.loads(data)
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.warning(f"Invalid product websocket JSON: {exc}")
|
||||
return None
|
||||
|
||||
if not isinstance(message, dict):
|
||||
logger.warning("Product websocket message must be a JSON object")
|
||||
return None
|
||||
|
||||
message_type = message.get("type")
|
||||
if message_type == "session.start":
|
||||
chat_id = message.get("chatId") or message.get("chat_id")
|
||||
return InputTransportMessageFrame(
|
||||
message={
|
||||
"type": "session.started",
|
||||
"protocol": self.protocol,
|
||||
"chatId": chat_id if isinstance(chat_id, str) else None,
|
||||
"audio": {
|
||||
"encoding": "pcm_s16le",
|
||||
"sample_rate": self._sample_rate,
|
||||
"channels": self._channels,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if message_type == "session.stop":
|
||||
return EndFrame()
|
||||
|
||||
if message_type == "response.cancel":
|
||||
return CancelFrame(reason="client_cancelled")
|
||||
|
||||
if message_type == "input.audio":
|
||||
audio = message.get("audio") or message.get("data")
|
||||
if not isinstance(audio, str):
|
||||
logger.warning("input.audio requires base64 'audio' or 'data'")
|
||||
return None
|
||||
try:
|
||||
pcm = base64.b64decode(audio)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
logger.warning(f"Invalid input.audio base64: {exc}")
|
||||
return None
|
||||
return InputAudioRawFrame(
|
||||
audio=pcm,
|
||||
sample_rate=int(message.get("sample_rate") or self._sample_rate),
|
||||
num_channels=int(message.get("channels") or self._channels),
|
||||
)
|
||||
|
||||
if message_type == "input.image":
|
||||
return self._deserialize_input_image(message)
|
||||
|
||||
if message_type == "input.text":
|
||||
text = message.get("text")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
logger.warning("input.text requires non-empty 'text'")
|
||||
return None
|
||||
return InputTransportMessageFrame(
|
||||
message={
|
||||
"type": "input.text",
|
||||
"text": text,
|
||||
"interrupt": bool(message.get("interrupt", True)),
|
||||
}
|
||||
)
|
||||
|
||||
if message_type == "transport.message":
|
||||
payload = message.get("message")
|
||||
return InputTransportMessageFrame(message=payload if isinstance(payload, dict) else message)
|
||||
|
||||
logger.warning(f"Unsupported product websocket message type: {message_type!r}")
|
||||
return None
|
||||
|
||||
def _deserialize_input_image(self, message: dict[str, Any]) -> Frame | None:
|
||||
encoded = message.get("image") or message.get("data")
|
||||
if not isinstance(encoded, str):
|
||||
logger.warning("input.image requires base64 'image' or 'data'")
|
||||
return None
|
||||
|
||||
mime_type = str(message.get("mime_type") or message.get("media_type") or "image/jpeg")
|
||||
if mime_type not in SUPPORTED_INPUT_IMAGE_MIME_TYPES:
|
||||
logger.warning(
|
||||
"input.image unsupported mime_type "
|
||||
f"{mime_type!r}; expected one of {sorted(SUPPORTED_INPUT_IMAGE_MIME_TYPES)}"
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
width = int(message.get("width") or 0)
|
||||
height = int(message.get("height") or 0)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("input.image width and height must be integers")
|
||||
return None
|
||||
|
||||
if width <= 0 or height <= 0:
|
||||
logger.warning("input.image requires positive integer width and height")
|
||||
return None
|
||||
|
||||
if "," in encoded and encoded.lstrip().startswith("data:"):
|
||||
encoded = encoded.split(",", 1)[1]
|
||||
|
||||
try:
|
||||
image = base64.b64decode(encoded, validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
logger.warning(f"Invalid input.image base64: {exc}")
|
||||
return None
|
||||
|
||||
if len(image) > MAX_INPUT_IMAGE_BYTES:
|
||||
logger.warning(
|
||||
f"input.image too large: {len(image)} bytes; "
|
||||
f"max is {MAX_INPUT_IMAGE_BYTES} bytes"
|
||||
)
|
||||
return None
|
||||
|
||||
text = message.get("text")
|
||||
if text is not None and not isinstance(text, str):
|
||||
logger.warning("input.image text must be a string when provided")
|
||||
return None
|
||||
|
||||
return UserImageRawFrame(
|
||||
image=image,
|
||||
size=(width, height),
|
||||
format=mime_type,
|
||||
user_id=str(message.get("user_id") or "product-user"),
|
||||
text=text or "Answer using this camera image.",
|
||||
append_to_context=bool(message.get("append_to_context", True)),
|
||||
)
|
||||
|
||||
def _event(self, event_type: str, **payload: Any) -> str:
|
||||
self._sequence += 1
|
||||
return json.dumps(
|
||||
{
|
||||
"type": event_type,
|
||||
"protocol": self.protocol,
|
||||
"seq": self._sequence,
|
||||
**payload,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
136
src/voice/response_state.py
Normal file
136
src/voice/response_state.py
Normal file
@@ -0,0 +1,136 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
Frame,
|
||||
InterruptionFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
LLMTextFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
from .config import ResponseStateConfig
|
||||
|
||||
|
||||
class StateTagResponseProcessor(FrameProcessor):
|
||||
"""Extract a leading state tag from LLM text before text streaming and TTS.
|
||||
|
||||
Expected model output:
|
||||
|
||||
<state>some state</state>spoken response
|
||||
|
||||
The extracted state is emitted as a product protocol event, while only the
|
||||
spoken response text is forwarded downstream. If the model does not produce
|
||||
the tag, the original text is forwarded unchanged.
|
||||
"""
|
||||
|
||||
def __init__(self, config: ResponseStateConfig) -> None:
|
||||
super().__init__()
|
||||
self._tag = config.tag
|
||||
self._event_type = config.event_type
|
||||
self._max_prefix_chars = config.max_prefix_chars
|
||||
self._opening_tag = f"<{self._tag}>"
|
||||
self._closing_tag = f"</{self._tag}>"
|
||||
self._start_frame: LLMFullResponseStartFrame | None = None
|
||||
self._buffer = ""
|
||||
self._decided = False
|
||||
self._in_llm_response = False
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, LLMFullResponseStartFrame):
|
||||
self._start_frame = frame
|
||||
self._buffer = ""
|
||||
self._decided = False
|
||||
self._in_llm_response = True
|
||||
return
|
||||
|
||||
if isinstance(frame, LLMTextFrame) and self._in_llm_response and not self._decided:
|
||||
await self._process_initial_text(frame.text or "", direction)
|
||||
return
|
||||
|
||||
if isinstance(frame, LLMFullResponseEndFrame):
|
||||
if self._in_llm_response:
|
||||
await self._flush_buffer(direction)
|
||||
await self.push_frame(frame, direction)
|
||||
self._reset()
|
||||
return
|
||||
|
||||
if isinstance(frame, (InterruptionFrame, CancelFrame)):
|
||||
if self._in_llm_response:
|
||||
await self._flush_buffer(direction)
|
||||
self._reset()
|
||||
await self.push_frame(frame, direction)
|
||||
return
|
||||
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
async def _process_initial_text(self, text: str, direction: FrameDirection) -> None:
|
||||
if not text:
|
||||
return
|
||||
|
||||
self._buffer += text
|
||||
decision = self._parse_buffer()
|
||||
if decision is None:
|
||||
return
|
||||
|
||||
self._decided = True
|
||||
state, response_text = decision
|
||||
if state is not None:
|
||||
await self._emit_state(state)
|
||||
await self._push_start(direction)
|
||||
if response_text:
|
||||
await self.push_frame(LLMTextFrame(response_text), direction)
|
||||
self._buffer = ""
|
||||
|
||||
def _parse_buffer(self) -> tuple[str | None, str] | None:
|
||||
stripped = self._buffer.lstrip()
|
||||
if not stripped:
|
||||
return None
|
||||
|
||||
if stripped.startswith(self._opening_tag):
|
||||
state_start = len(self._opening_tag)
|
||||
state_end = stripped.find(self._closing_tag, state_start)
|
||||
if state_end >= 0:
|
||||
response_start = state_end + len(self._closing_tag)
|
||||
return stripped[state_start:state_end].strip(), stripped[response_start:]
|
||||
if len(self._buffer) < self._max_prefix_chars:
|
||||
return None
|
||||
return None, self._buffer
|
||||
|
||||
if self._opening_tag.startswith(stripped) and len(self._buffer) < self._max_prefix_chars:
|
||||
return None
|
||||
|
||||
return None, self._buffer
|
||||
|
||||
async def _flush_buffer(self, direction: FrameDirection) -> None:
|
||||
await self._push_start(direction)
|
||||
if self._buffer:
|
||||
await self.push_frame(LLMTextFrame(self._buffer), direction)
|
||||
self._buffer = ""
|
||||
self._decided = True
|
||||
|
||||
async def _push_start(self, direction: FrameDirection) -> None:
|
||||
if self._start_frame:
|
||||
await self.push_frame(self._start_frame, direction)
|
||||
self._start_frame = None
|
||||
|
||||
async def _emit_state(self, state: str) -> None:
|
||||
await self.push_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={
|
||||
"type": self._event_type,
|
||||
"state": state,
|
||||
}
|
||||
),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
|
||||
def _reset(self) -> None:
|
||||
self._start_frame = None
|
||||
self._buffer = ""
|
||||
self._decided = False
|
||||
self._in_llm_response = False
|
||||
100
src/voice/routes.py
Normal file
100
src/voice/routes.py
Normal file
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, FastAPI, WebSocket
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from loguru import logger
|
||||
|
||||
from .config import EngineConfig, load_config, resolve_voice_config_path
|
||||
from .pipeline import run_product_voice_pipeline
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
VOICE_DEMO_DIR = PROJECT_ROOT / "static" / "voice-demo"
|
||||
|
||||
router = APIRouter(tags=["voice"])
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_voice_config() -> EngineConfig:
|
||||
return load_config()
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_voice_config_path() -> Path:
|
||||
return resolve_voice_config_path()
|
||||
|
||||
|
||||
def _normalize_mount_path(path: str) -> str:
|
||||
normalized = path.strip() or "/voice-demo"
|
||||
if not normalized.startswith("/"):
|
||||
normalized = f"/{normalized}"
|
||||
return normalized.rstrip("/") or "/"
|
||||
|
||||
|
||||
@router.get("/voice/health")
|
||||
async def voice_health() -> dict[str, object]:
|
||||
config = get_voice_config()
|
||||
mount = (
|
||||
_normalize_mount_path(config.server.webpage_mount)
|
||||
if config.server.serve_webpage
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"status": "healthy",
|
||||
"config": str(get_voice_config_path()),
|
||||
"protocols": {
|
||||
"/ws-product": "va.ws.v1.json_base64",
|
||||
},
|
||||
"features": {
|
||||
"product_text_input": True,
|
||||
"product_text_interrupt": True,
|
||||
},
|
||||
"demo": mount,
|
||||
"llm_provider": config.services.llm.provider,
|
||||
"stt_provider": config.services.stt.provider,
|
||||
"tts_provider": config.services.tts.provider,
|
||||
}
|
||||
|
||||
|
||||
@router.websocket("/ws-product")
|
||||
async def product_websocket_endpoint(websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
config = get_voice_config()
|
||||
await run_product_voice_pipeline(websocket, config)
|
||||
|
||||
|
||||
def register_voice(app: FastAPI) -> None:
|
||||
"""Mount voice websocket routes and optional browser demo static files."""
|
||||
voice_config_path = get_voice_config_path()
|
||||
if not voice_config_path.exists():
|
||||
logger.warning(f"Voice config not found at {voice_config_path}; voice demo disabled")
|
||||
return
|
||||
|
||||
config = get_voice_config()
|
||||
app.include_router(router)
|
||||
logger.info(f"Voice config loaded from {voice_config_path}")
|
||||
|
||||
if config.server.cors_origins:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=config.server.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
if config.server.serve_webpage and VOICE_DEMO_DIR.is_dir():
|
||||
mount = _normalize_mount_path(config.server.webpage_mount)
|
||||
app.mount(
|
||||
mount,
|
||||
StaticFiles(directory=str(VOICE_DEMO_DIR), html=True),
|
||||
name="voice-demo",
|
||||
)
|
||||
logger.info(f"Voice demo mounted at {mount}")
|
||||
else:
|
||||
logger.info("Voice demo static page disabled or missing")
|
||||
|
||||
logger.info("Voice websocket registered at /ws-product")
|
||||
220
src/voice/services.py
Normal file
220
src/voice/services.py
Normal file
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from openai import BadRequestError
|
||||
from openai import NOT_GIVEN
|
||||
|
||||
from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame
|
||||
from pipecat.services.openai._constants import OPENAI_SAMPLE_RATE
|
||||
from pipecat.services.openai.llm import OpenAILLMService
|
||||
from pipecat.services.openai.stt import OpenAISTTService
|
||||
from pipecat.services.openai.tts import VALID_VOICES, OpenAITTSService
|
||||
from pipecat.services.tts_service import TextAggregationMode
|
||||
from pipecat.transcriptions.language import Language
|
||||
|
||||
from .config import AudioConfig, LLMConfig, STTConfig, TTSConfig
|
||||
from .fastgpt_llm import FastGPTLLMService, FastGPTLLMSettings
|
||||
from .xfyun_asr import DEFAULT_XFYUN_ASR_URL, XfyunASRService
|
||||
from .xfyun_super_tts import DEFAULT_XFYUN_SUPER_TTS_URL, XfyunSuperTTSService
|
||||
from .xfyun_tts import DEFAULT_XFYUN_TTS_URL, XfyunTTSService
|
||||
|
||||
|
||||
def create_stt_service(config: STTConfig, audio: AudioConfig | None = None):
|
||||
if config.provider == "xfyun":
|
||||
sample_rate = audio.sample_rate_hz if audio else 16000
|
||||
return XfyunASRService(
|
||||
app_id=config.app_id,
|
||||
api_key=config.api_key or "",
|
||||
api_secret=config.api_secret,
|
||||
url=config.base_url or DEFAULT_XFYUN_ASR_URL,
|
||||
language=config.language or "zh_cn",
|
||||
domain=config.domain,
|
||||
accent=config.accent,
|
||||
sample_rate=sample_rate,
|
||||
encoding=config.encoding,
|
||||
frame_size=config.frame_size,
|
||||
open_timeout=config.timeout_sec,
|
||||
dynamic_correction=config.dynamic_correction,
|
||||
)
|
||||
|
||||
_require_provider(config.provider, "openai", "stt")
|
||||
return OpenAISTTService(
|
||||
api_key=config.api_key or None,
|
||||
base_url=config.base_url,
|
||||
settings=OpenAISTTService.Settings(
|
||||
model=config.model,
|
||||
language=_language(config.language),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def create_llm_service(
|
||||
config: LLMConfig,
|
||||
*,
|
||||
chat_id: str | None = None,
|
||||
session_variables: dict | None = None,
|
||||
greeting_prompt: str | None = None,
|
||||
):
|
||||
if config.is_fastgpt:
|
||||
variables = {**config.variables, **(session_variables or {})}
|
||||
return FastGPTLLMService(
|
||||
api_key=config.api_key,
|
||||
base_url=config.base_url or "http://localhost:3000",
|
||||
chat_id=chat_id,
|
||||
app_id=config.app_id,
|
||||
greeting_prompt=greeting_prompt,
|
||||
timeout=config.timeout_sec,
|
||||
image_input_mode=config.image_input_mode,
|
||||
settings=FastGPTLLMSettings(
|
||||
model=config.model or "fastgpt",
|
||||
variables=variables,
|
||||
detail=config.detail,
|
||||
),
|
||||
)
|
||||
|
||||
if not config.is_openai:
|
||||
supported = ", ".join(sorted(("openai", "fastgpt", "llm")))
|
||||
raise ValueError(
|
||||
f"Unsupported llm provider {config.provider!r}; expected one of: {supported}"
|
||||
)
|
||||
return OpenAILLMService(
|
||||
api_key=config.api_key or None,
|
||||
base_url=config.base_url,
|
||||
settings=OpenAILLMService.Settings(
|
||||
model=config.model,
|
||||
temperature=config.temperature if config.temperature is not None else NOT_GIVEN,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def create_tts_service(config: TTSConfig, audio: AudioConfig):
|
||||
if config.provider == "xfyun":
|
||||
source_sample_rate = config.source_sample_rate_hz or audio.sample_rate_hz
|
||||
if source_sample_rate not in (8000, 16000):
|
||||
raise ValueError("Xfyun TTS source_sample_rate_hz must be 8000 or 16000")
|
||||
return XfyunTTSService(
|
||||
app_id=config.app_id,
|
||||
api_key=config.api_key or "",
|
||||
api_secret=config.api_secret,
|
||||
voice=config.voice,
|
||||
url=config.base_url or DEFAULT_XFYUN_TTS_URL,
|
||||
sample_rate=audio.sample_rate_hz,
|
||||
source_sample_rate=source_sample_rate,
|
||||
encoding=config.aue,
|
||||
text_encoding=config.tte,
|
||||
speed=config.speed,
|
||||
volume=config.volume,
|
||||
pitch=config.pitch,
|
||||
timeout=config.timeout_sec,
|
||||
push_stop_frames=True,
|
||||
)
|
||||
|
||||
if config.provider in ("xfyun_super", "xfyun_super_tts"):
|
||||
source_sample_rate = config.source_sample_rate_hz or 24000
|
||||
if source_sample_rate not in (8000, 16000, 24000):
|
||||
raise ValueError(
|
||||
"Xfyun Super TTS source_sample_rate_hz must be 8000, 16000, or 24000"
|
||||
)
|
||||
text_aggregation_mode = config.text_aggregation_mode or TextAggregationMode.TOKEN
|
||||
return XfyunSuperTTSService(
|
||||
app_id=config.app_id,
|
||||
api_key=config.api_key or "",
|
||||
api_secret=config.api_secret,
|
||||
voice=config.voice,
|
||||
url=config.base_url or DEFAULT_XFYUN_SUPER_TTS_URL,
|
||||
sample_rate=audio.sample_rate_hz,
|
||||
source_sample_rate=source_sample_rate,
|
||||
encoding=config.aue,
|
||||
speed=config.speed,
|
||||
volume=config.volume,
|
||||
pitch=config.pitch,
|
||||
oral_level=config.oral_level,
|
||||
text_aggregation_mode=text_aggregation_mode,
|
||||
open_timeout=config.timeout_sec,
|
||||
)
|
||||
|
||||
_require_provider(config.provider, "openai", "tts")
|
||||
service_class = OpenAITTSService if config.voice in VALID_VOICES else OpenAICompatibleTTSService
|
||||
return service_class(
|
||||
api_key=config.api_key or None,
|
||||
base_url=config.base_url,
|
||||
sample_rate=audio.sample_rate_hz,
|
||||
source_sample_rate=config.source_sample_rate_hz,
|
||||
settings=OpenAITTSService.Settings(
|
||||
model=config.model,
|
||||
voice=config.voice,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class OpenAICompatibleTTSService(OpenAITTSService):
|
||||
"""OpenAI-compatible TTS service that permits provider-specific voice ids."""
|
||||
|
||||
def __init__(self, *, source_sample_rate: int | None = None, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._source_sample_rate = source_sample_rate or OPENAI_SAMPLE_RATE
|
||||
|
||||
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
|
||||
voice = self._settings.voice
|
||||
if not voice:
|
||||
yield ErrorFrame(error="TTS voice must be specified")
|
||||
return
|
||||
|
||||
try:
|
||||
create_params = {
|
||||
"input": text,
|
||||
"model": self._settings.model,
|
||||
"voice": voice,
|
||||
"response_format": "pcm",
|
||||
}
|
||||
|
||||
if self._settings.instructions:
|
||||
create_params["instructions"] = self._settings.instructions
|
||||
|
||||
if self._settings.speed:
|
||||
create_params["speed"] = self._settings.speed
|
||||
|
||||
async with self._client.audio.speech.with_streaming_response.create(
|
||||
**create_params
|
||||
) as response:
|
||||
if response.status_code != 200:
|
||||
error = await response.text()
|
||||
yield ErrorFrame(
|
||||
error=f"TTS request failed (status: {response.status_code}, error: {error})"
|
||||
)
|
||||
return
|
||||
|
||||
await self.start_tts_usage_metrics(text)
|
||||
|
||||
async def audio_chunks():
|
||||
async for chunk in response.iter_bytes(self.chunk_size):
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
first_frame = True
|
||||
async for frame in self._stream_audio_frames_from_iterator(
|
||||
audio_chunks(),
|
||||
in_sample_rate=self._source_sample_rate,
|
||||
context_id=context_id,
|
||||
):
|
||||
if first_frame:
|
||||
await self.stop_ttfb_metrics()
|
||||
first_frame = False
|
||||
yield frame
|
||||
except BadRequestError as exc:
|
||||
yield ErrorFrame(error=f"TTS request failed: {exc}")
|
||||
except Exception as exc:
|
||||
yield ErrorFrame(error=f"TTS request failed: {exc}")
|
||||
|
||||
|
||||
def _require_provider(actual: str, expected: str, service_name: str) -> None:
|
||||
if actual != expected:
|
||||
raise ValueError(f"Unsupported {service_name} provider {actual!r}; expected {expected!r}")
|
||||
|
||||
|
||||
def _language(value: str | None) -> Language | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.replace("-", "_").upper()
|
||||
return getattr(Language, normalized, value)
|
||||
54
src/voice/text_input.py
Normal file
54
src/voice/text_input.py
Normal file
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
Frame,
|
||||
InputTransportMessageFrame,
|
||||
LLMMessagesAppendFrame,
|
||||
UserImageRawFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
|
||||
class ProductTextInputProcessor(FrameProcessor):
|
||||
"""Converts product text-input transport messages and marks image input as user activity."""
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, UserImageRawFrame):
|
||||
await self.broadcast_frame(UserStartedSpeakingFrame)
|
||||
await self.push_frame(frame, direction)
|
||||
await self.broadcast_frame(UserStoppedSpeakingFrame)
|
||||
return
|
||||
|
||||
if not isinstance(frame, InputTransportMessageFrame):
|
||||
await self.push_frame(frame, direction)
|
||||
return
|
||||
|
||||
message = frame.message
|
||||
if not isinstance(message, dict) or message.get("type") != "input.text":
|
||||
await self.push_frame(frame, direction)
|
||||
return
|
||||
|
||||
text = str(message.get("text") or "").strip()
|
||||
if not text:
|
||||
return
|
||||
|
||||
await self.broadcast_frame(UserStartedSpeakingFrame)
|
||||
|
||||
if message.get("interrupt", True):
|
||||
logger.info("Text input interrupting current response")
|
||||
await self.broadcast_interruption()
|
||||
|
||||
await self.push_frame(
|
||||
LLMMessagesAppendFrame(
|
||||
messages=[{"role": "user", "content": text}],
|
||||
run_llm=True,
|
||||
),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
await self.broadcast_frame(UserStoppedSpeakingFrame)
|
||||
215
src/voice/text_stream.py
Normal file
215
src/voice/text_stream.py
Normal file
@@ -0,0 +1,215 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Protocol
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
Frame,
|
||||
InterruptionFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
LLMTextFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
TTSSpeakFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
|
||||
class _AssistantContextSync(Protocol):
|
||||
@property
|
||||
def context(self) -> Any: ...
|
||||
|
||||
|
||||
def _committed_assistant_content(context: Any) -> str:
|
||||
"""Return trailing assistant text only when the last context message is assistant."""
|
||||
messages = context.get_messages()
|
||||
if not messages:
|
||||
return ""
|
||||
last = messages[-1]
|
||||
if not isinstance(last, dict) or last.get("role") != "assistant":
|
||||
return ""
|
||||
content = last.get("content")
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def sync_streamed_assistant_context(
|
||||
aggregator: _AssistantContextSync,
|
||||
*,
|
||||
streamed_text: str,
|
||||
committed_text: str,
|
||||
) -> None:
|
||||
"""Align LLM context with urgent-streamed UI text.
|
||||
|
||||
The assistant aggregator commits TTS-spoken text; ``ProductTextStreamProcessor``
|
||||
mirrors the LLM stream to the client. Replace or insert the streamed text so
|
||||
the next turn sees what the user read on screen.
|
||||
"""
|
||||
streamed = streamed_text.strip()
|
||||
if not streamed or streamed == committed_text.strip():
|
||||
return
|
||||
|
||||
committed = committed_text.strip()
|
||||
|
||||
def _apply(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
updated = list(messages)
|
||||
if not updated:
|
||||
updated.append({"role": "assistant", "content": streamed})
|
||||
return updated
|
||||
|
||||
last = updated[-1]
|
||||
if isinstance(last, dict) and last.get("role") == "assistant":
|
||||
content = last.get("content")
|
||||
if isinstance(content, str) and content.strip() != streamed:
|
||||
updated[-1] = {"role": "assistant", "content": streamed}
|
||||
return updated
|
||||
|
||||
if (
|
||||
len(updated) >= 2
|
||||
and isinstance(last, dict)
|
||||
and last.get("role") == "user"
|
||||
):
|
||||
prev = updated[-2]
|
||||
if isinstance(prev, dict) and prev.get("role") == "user":
|
||||
updated.insert(len(updated) - 1, {"role": "assistant", "content": streamed})
|
||||
return updated
|
||||
|
||||
if isinstance(last, dict) and last.get("role") == "user":
|
||||
updated.append({"role": "assistant", "content": streamed})
|
||||
return updated
|
||||
|
||||
updated.append({"role": "assistant", "content": streamed})
|
||||
return updated
|
||||
|
||||
aggregator.context.transform_messages(_apply)
|
||||
|
||||
|
||||
def maybe_sync_assistant_context(
|
||||
aggregator: _AssistantContextSync,
|
||||
text_stream: "ProductTextStreamProcessor",
|
||||
*,
|
||||
committed_text: str | None = None,
|
||||
) -> None:
|
||||
committed = (
|
||||
committed_text.strip()
|
||||
if committed_text is not None
|
||||
else _committed_assistant_content(aggregator.context)
|
||||
)
|
||||
streamed = text_stream.last_assistant_stream_text()
|
||||
if not streamed:
|
||||
return
|
||||
sync_streamed_assistant_context(
|
||||
aggregator,
|
||||
streamed_text=streamed,
|
||||
committed_text=committed,
|
||||
)
|
||||
|
||||
|
||||
class ProductTextStreamProcessor(FrameProcessor):
|
||||
"""Mirrors LLM text frames as streaming protocol events.
|
||||
|
||||
Placed between the LLM service and the TTS service, this processor
|
||||
observes the LLM's text frames as they're emitted and forwards them
|
||||
downstream as ``OutputTransportMessageUrgentFrame``s that the product
|
||||
serializer turns into ``response.text.{started,delta,final}`` events.
|
||||
|
||||
Urgent frames bypass TTS serialization and transport audio queues so text
|
||||
reaches the client at least as quickly as synthesized audio.
|
||||
|
||||
``TTSSpeakFrame`` (used by the fixed-greeting code path, which bypasses
|
||||
the LLM entirely) is also handled: the processor synthesizes a single
|
||||
started/delta/final sequence for its fixed text.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self._aggregation: list[str] = []
|
||||
self._turn_active = False
|
||||
self._last_assistant_stream_text = ""
|
||||
self._interrupted_stream_text: str | None = None
|
||||
|
||||
def last_assistant_stream_text(self) -> str:
|
||||
return self._last_assistant_stream_text
|
||||
|
||||
def take_interrupted_stream_text(self) -> str | None:
|
||||
text = self._interrupted_stream_text
|
||||
self._interrupted_stream_text = None
|
||||
return text
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, LLMFullResponseStartFrame):
|
||||
await self.push_frame(frame, direction)
|
||||
await self._start_turn()
|
||||
elif isinstance(frame, LLMTextFrame):
|
||||
await self.push_frame(frame, direction)
|
||||
if frame.text:
|
||||
await self._delta(frame.text)
|
||||
elif isinstance(frame, LLMFullResponseEndFrame):
|
||||
await self.push_frame(frame, direction)
|
||||
await self._end_turn(interrupted=False)
|
||||
elif isinstance(frame, (InterruptionFrame, CancelFrame)):
|
||||
await self.push_frame(frame, direction)
|
||||
await self._handle_interrupt()
|
||||
elif isinstance(frame, TTSSpeakFrame):
|
||||
# Fixed-text / direct-speech path: there's no LLM cycle, so
|
||||
# synthesize one started/delta/final sequence for the spoken text.
|
||||
text = frame.text or ""
|
||||
await self.push_frame(frame, direction)
|
||||
await self._start_turn()
|
||||
if text:
|
||||
await self._delta(text)
|
||||
await self._end_turn(interrupted=False)
|
||||
else:
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
async def _start_turn(self) -> None:
|
||||
if self._turn_active:
|
||||
return
|
||||
self._turn_active = True
|
||||
self._aggregation = []
|
||||
await self._emit("response.text.started")
|
||||
|
||||
async def _delta(self, text: str) -> None:
|
||||
if not self._turn_active:
|
||||
# A text frame outside a turn shouldn't happen, but if it does,
|
||||
# synthesize a started boundary so the client renders sensibly.
|
||||
await self._start_turn()
|
||||
self._aggregation.append(text)
|
||||
await self._emit("response.text.delta", text=text)
|
||||
|
||||
async def _handle_interrupt(self) -> None:
|
||||
if self._turn_active:
|
||||
await self._end_turn(interrupted=True)
|
||||
return
|
||||
|
||||
if self._last_assistant_stream_text:
|
||||
self._interrupted_stream_text = self._last_assistant_stream_text
|
||||
|
||||
async def _end_turn(self, *, interrupted: bool) -> None:
|
||||
if not self._turn_active:
|
||||
return
|
||||
|
||||
full_text = "".join(self._aggregation)
|
||||
if full_text:
|
||||
self._last_assistant_stream_text = full_text
|
||||
if interrupted and full_text:
|
||||
self._interrupted_stream_text = full_text
|
||||
|
||||
self._turn_active = False
|
||||
self._aggregation = []
|
||||
await self._emit(
|
||||
"response.text.final",
|
||||
text=full_text,
|
||||
interrupted=interrupted,
|
||||
)
|
||||
|
||||
async def _emit(self, event_type: str, **payload: object) -> None:
|
||||
await self.push_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={"type": event_type, **payload},
|
||||
),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
30
src/voice/transcript_stream.py
Normal file
30
src/voice/transcript_stream.py
Normal file
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
Frame,
|
||||
InterimTranscriptionFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
|
||||
class ProductTranscriptStreamProcessor(FrameProcessor):
|
||||
"""Mirrors interim STT frames to the product websocket protocol."""
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, InterimTranscriptionFrame):
|
||||
await self.push_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={
|
||||
"type": "input.transcript.interim",
|
||||
"text": frame.text,
|
||||
"user_id": frame.user_id,
|
||||
"timestamp": frame.timestamp,
|
||||
}
|
||||
),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
|
||||
await self.push_frame(frame, direction)
|
||||
91
src/voice/turn_start.py
Normal file
91
src/voice/turn_start.py
Normal file
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from loguru import logger
|
||||
from pipecat.frames.frames import (
|
||||
BotStartedSpeakingFrame,
|
||||
BotStoppedSpeakingFrame,
|
||||
Frame,
|
||||
InterimTranscriptionFrame,
|
||||
TranscriptionFrame,
|
||||
)
|
||||
from pipecat.turns.types import ProcessFrameResult
|
||||
from pipecat.turns.user_start.base_user_turn_start_strategy import BaseUserTurnStartStrategy
|
||||
|
||||
|
||||
_COUNTABLE_TEXT_RE = re.compile(r"[\w\u4e00-\u9fff]", re.UNICODE)
|
||||
|
||||
|
||||
class InterruptionGateUserTurnStartStrategy(BaseUserTurnStartStrategy):
|
||||
"""Starts user turns only after likely intentional speech.
|
||||
|
||||
When the assistant is speaking, short background speech should not barge in
|
||||
unless it is a common answer to a yes/no style question. When the assistant
|
||||
is not speaking, any non-empty transcript can start a normal user turn.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
min_chars_when_bot_speaking: int,
|
||||
allowed_short_replies: list[str],
|
||||
use_interim: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self._min_chars_when_bot_speaking = min_chars_when_bot_speaking
|
||||
self._allowed_short_replies = {
|
||||
self._normalize_text(reply) for reply in allowed_short_replies if reply.strip()
|
||||
}
|
||||
self._use_interim = use_interim
|
||||
self._bot_speaking = False
|
||||
|
||||
async def reset(self):
|
||||
await super().reset()
|
||||
|
||||
async def process_frame(self, frame: Frame) -> ProcessFrameResult:
|
||||
if isinstance(frame, BotStartedSpeakingFrame):
|
||||
self._bot_speaking = True
|
||||
return ProcessFrameResult.CONTINUE
|
||||
if isinstance(frame, BotStoppedSpeakingFrame):
|
||||
self._bot_speaking = False
|
||||
return ProcessFrameResult.CONTINUE
|
||||
if isinstance(frame, InterimTranscriptionFrame) and self._use_interim:
|
||||
return await self._handle_transcription(frame.text, interim=True)
|
||||
if isinstance(frame, TranscriptionFrame):
|
||||
return await self._handle_transcription(frame.text, interim=False)
|
||||
|
||||
return ProcessFrameResult.CONTINUE
|
||||
|
||||
async def _handle_transcription(self, text: str, *, interim: bool) -> ProcessFrameResult:
|
||||
normalized = self._normalize_text(text)
|
||||
if not normalized:
|
||||
return ProcessFrameResult.CONTINUE
|
||||
|
||||
if not self._bot_speaking:
|
||||
await self.trigger_user_turn_started()
|
||||
return ProcessFrameResult.STOP
|
||||
|
||||
should_interrupt = self._should_interrupt(normalized)
|
||||
logger.debug(
|
||||
f"{self} interruption_gate text={text!r} normalized={normalized!r} "
|
||||
f"should_interrupt={should_interrupt} interim={interim}"
|
||||
)
|
||||
|
||||
if should_interrupt:
|
||||
await self.trigger_user_turn_started()
|
||||
return ProcessFrameResult.STOP
|
||||
|
||||
await self.trigger_reset_aggregation()
|
||||
return ProcessFrameResult.CONTINUE
|
||||
|
||||
def _should_interrupt(self, normalized: str) -> bool:
|
||||
return (
|
||||
normalized in self._allowed_short_replies
|
||||
or len(normalized) >= self._min_chars_when_bot_speaking
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_text(text: str) -> str:
|
||||
return "".join(_COUNTABLE_TEXT_RE.findall(text.lower()))
|
||||
353
src/voice/xfyun_asr.py
Normal file
353
src/voice/xfyun_asr.py
Normal file
@@ -0,0 +1,353 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncGenerator
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import format_datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
EndFrame,
|
||||
Frame,
|
||||
InterimTranscriptionFrame,
|
||||
TranscriptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
VADUserStartedSpeakingFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection
|
||||
from pipecat.services.settings import STTSettings
|
||||
from pipecat.services.stt_service import STTService
|
||||
from pipecat.transcriptions.language import Language
|
||||
from pipecat.utils.time import time_now_iso8601
|
||||
from websockets.asyncio.client import connect as websocket_connect
|
||||
from websockets.protocol import State
|
||||
|
||||
|
||||
DEFAULT_XFYUN_ASR_URL = "wss://iat-api.xfyun.cn/v2/iat"
|
||||
|
||||
|
||||
class XfyunASRService(STTService):
|
||||
"""iFlytek/Xfyun streaming voice dictation service for Pipecat."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
app_id: str,
|
||||
api_key: str,
|
||||
api_secret: str,
|
||||
url: str | None = None,
|
||||
language: str = "zh_cn",
|
||||
domain: str = "iat",
|
||||
accent: str = "mandarin",
|
||||
sample_rate: int = 16000,
|
||||
encoding: str = "raw",
|
||||
frame_size: int = 1280,
|
||||
open_timeout: float = 10.0,
|
||||
dynamic_correction: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
sample_rate=sample_rate,
|
||||
settings=STTSettings(model=None, language=language),
|
||||
**kwargs,
|
||||
)
|
||||
self._app_id = app_id or os.environ.get("XFYUN_APP_ID", "")
|
||||
self._api_key = api_key or os.environ.get("XFYUN_API_KEY", "")
|
||||
self._api_secret = api_secret or os.environ.get("XFYUN_API_SECRET", "")
|
||||
self._url = url or DEFAULT_XFYUN_ASR_URL
|
||||
self._language = language
|
||||
self._domain = domain
|
||||
self._accent = accent
|
||||
self._encoding = encoding
|
||||
self._frame_size = frame_size
|
||||
self._open_timeout = open_timeout
|
||||
self._dynamic_correction = dynamic_correction
|
||||
|
||||
self._websocket = None
|
||||
self._receive_task = None
|
||||
self._audio_buffer = bytearray()
|
||||
self._sent_first_frame = False
|
||||
self._sent_final_frame = False
|
||||
self._finalizing_turn = False
|
||||
self._partials: list[str] = []
|
||||
self._last_text = ""
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
await self._close_utterance()
|
||||
await super().cleanup()
|
||||
|
||||
async def stop(self, frame: EndFrame) -> None:
|
||||
await self._close_utterance()
|
||||
await super().stop(frame)
|
||||
|
||||
async def cancel(self, frame: CancelFrame) -> None:
|
||||
await self._close_utterance()
|
||||
await super().cancel(frame)
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, UserStoppedSpeakingFrame):
|
||||
# Aggregator-level turn end (broadcast once per logical user turn).
|
||||
# This is the only boundary that finalizes/closes the xfyun
|
||||
# websocket, so brief VAD pauses do not restart the ASR session.
|
||||
await self._finish_utterance()
|
||||
elif isinstance(frame, VADUserStartedSpeakingFrame):
|
||||
await self._start_utterance()
|
||||
|
||||
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame | None, None]:
|
||||
if not audio:
|
||||
yield None
|
||||
return
|
||||
|
||||
if not self._websocket or self._websocket.state is not State.OPEN:
|
||||
await self._start_utterance()
|
||||
|
||||
self._audio_buffer.extend(audio)
|
||||
await self._flush_audio_buffer(final=False)
|
||||
yield None
|
||||
|
||||
async def _start_utterance(self) -> None:
|
||||
if self._websocket and self._websocket.state is State.OPEN:
|
||||
return
|
||||
|
||||
if not self._app_id or not self._api_key or not self._api_secret:
|
||||
await self.push_error("Xfyun ASR requires app_id, api_key, and api_secret")
|
||||
return
|
||||
|
||||
if self.sample_rate not in (8000, 16000):
|
||||
await self.push_error("Xfyun ASR sample rate must be 8000 or 16000")
|
||||
return
|
||||
|
||||
self._audio_buffer.clear()
|
||||
self._partials = []
|
||||
self._last_text = ""
|
||||
self._sent_first_frame = False
|
||||
self._sent_final_frame = False
|
||||
|
||||
auth_url = _build_auth_url(self._url, self._api_key, self._api_secret)
|
||||
try:
|
||||
self._websocket = await websocket_connect(
|
||||
auth_url,
|
||||
max_size=None,
|
||||
open_timeout=self._open_timeout,
|
||||
)
|
||||
except Exception as exc:
|
||||
await self.push_error(f"Xfyun ASR connection failed: {exc}", exception=exc)
|
||||
self._websocket = None
|
||||
return
|
||||
|
||||
self._receive_task = self.create_task(
|
||||
self._receive_messages(),
|
||||
name="xfyun_asr_receive",
|
||||
)
|
||||
|
||||
async def _finish_utterance(self) -> None:
|
||||
if not self._websocket or self._websocket.state is not State.OPEN:
|
||||
return
|
||||
|
||||
await self._flush_audio_buffer(final=True)
|
||||
if not self._sent_first_frame:
|
||||
await self._close_utterance()
|
||||
return
|
||||
|
||||
if not self._sent_final_frame:
|
||||
self._finalizing_turn = True
|
||||
await self._send_payload({"data": {"status": 2}})
|
||||
self.request_finalize()
|
||||
self._sent_final_frame = True
|
||||
|
||||
async def _close_utterance(self) -> None:
|
||||
current_task = asyncio.current_task()
|
||||
if self._receive_task and self._receive_task is not current_task:
|
||||
await self.cancel_task(self._receive_task)
|
||||
self._receive_task = None
|
||||
|
||||
websocket = self._websocket
|
||||
self._websocket = None
|
||||
if websocket and websocket.state is State.OPEN:
|
||||
try:
|
||||
await websocket.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self._audio_buffer.clear()
|
||||
self._sent_first_frame = False
|
||||
self._sent_final_frame = False
|
||||
self._finalizing_turn = False
|
||||
|
||||
async def _flush_audio_buffer(self, *, final: bool) -> None:
|
||||
while len(self._audio_buffer) >= self._frame_size:
|
||||
chunk = bytes(self._audio_buffer[: self._frame_size])
|
||||
del self._audio_buffer[: self._frame_size]
|
||||
await self._send_audio_chunk(chunk, status=1)
|
||||
|
||||
if final and self._audio_buffer:
|
||||
chunk = bytes(self._audio_buffer)
|
||||
self._audio_buffer.clear()
|
||||
await self._send_audio_chunk(chunk, status=1)
|
||||
|
||||
async def _send_audio_chunk(self, audio: bytes, *, status: int) -> None:
|
||||
if not audio:
|
||||
return
|
||||
|
||||
if not self._sent_first_frame:
|
||||
business = {
|
||||
"language": self._language,
|
||||
"domain": self._domain,
|
||||
"accent": self._accent,
|
||||
}
|
||||
if self._dynamic_correction:
|
||||
business["dwa"] = "wpgs"
|
||||
|
||||
payload = {
|
||||
"common": {"app_id": self._app_id},
|
||||
"business": business,
|
||||
"data": {
|
||||
"status": 0,
|
||||
"format": f"audio/L16;rate={self.sample_rate}",
|
||||
"encoding": self._encoding,
|
||||
"audio": base64.b64encode(audio).decode("utf-8"),
|
||||
},
|
||||
}
|
||||
self._sent_first_frame = True
|
||||
else:
|
||||
payload = {
|
||||
"data": {
|
||||
"status": status,
|
||||
"format": f"audio/L16;rate={self.sample_rate}",
|
||||
"encoding": self._encoding,
|
||||
"audio": base64.b64encode(audio).decode("utf-8"),
|
||||
}
|
||||
}
|
||||
|
||||
await self._send_payload(payload)
|
||||
|
||||
async def _send_payload(self, payload: dict[str, Any]) -> None:
|
||||
if not self._websocket or self._websocket.state is not State.OPEN:
|
||||
return
|
||||
await self._websocket.send(json.dumps(payload, ensure_ascii=False))
|
||||
|
||||
async def _receive_messages(self) -> None:
|
||||
websocket = self._websocket
|
||||
if not websocket:
|
||||
return
|
||||
|
||||
try:
|
||||
async for message in websocket:
|
||||
await self._process_response(json.loads(message))
|
||||
except Exception as exc:
|
||||
if self._websocket is websocket:
|
||||
await self.push_error(f"Xfyun ASR receive failed: {exc}", exception=exc)
|
||||
finally:
|
||||
if self._websocket is websocket:
|
||||
self._websocket = None
|
||||
self._receive_task = None
|
||||
|
||||
async def _process_response(self, payload: dict[str, Any]) -> None:
|
||||
code = payload.get("code", -1)
|
||||
if code != 0:
|
||||
message = payload.get("message", "unknown error")
|
||||
sid = payload.get("sid")
|
||||
await self.push_error(f"Xfyun ASR error code={code}, sid={sid}, message={message}")
|
||||
return
|
||||
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, dict):
|
||||
return
|
||||
|
||||
is_final_response = data.get("status") == 2
|
||||
recognition = data.get("result")
|
||||
if isinstance(recognition, dict):
|
||||
text = self._apply_recognition_result(recognition)
|
||||
if text and text != self._last_text:
|
||||
self._last_text = text
|
||||
if not self._finalizing_turn and not is_final_response:
|
||||
await self.push_frame(
|
||||
InterimTranscriptionFrame(
|
||||
text,
|
||||
self._user_id,
|
||||
time_now_iso8601(),
|
||||
_language_or_none(self._language),
|
||||
result=payload,
|
||||
)
|
||||
)
|
||||
|
||||
if is_final_response:
|
||||
final_text = self._last_text
|
||||
if final_text:
|
||||
self.confirm_finalize()
|
||||
await self.push_frame(
|
||||
TranscriptionFrame(
|
||||
final_text,
|
||||
self._user_id,
|
||||
time_now_iso8601(),
|
||||
_language_or_none(self._language),
|
||||
result=payload,
|
||||
)
|
||||
)
|
||||
await self._close_utterance()
|
||||
|
||||
def _apply_recognition_result(self, recognition: dict[str, Any]) -> str:
|
||||
partial = _extract_text_from_result(recognition)
|
||||
if not partial:
|
||||
return self._last_text
|
||||
|
||||
if self._dynamic_correction and recognition.get("pgs") == "rpl" and recognition.get("rg"):
|
||||
start, end = recognition["rg"]
|
||||
if 1 <= start <= len(self._partials):
|
||||
self._partials[start - 1 : end] = [partial]
|
||||
else:
|
||||
logger.debug(f"Ignoring out-of-range Xfyun replacement rg={recognition['rg']}")
|
||||
else:
|
||||
self._partials.append(partial)
|
||||
|
||||
return "".join(self._partials)
|
||||
|
||||
|
||||
def _extract_text_from_result(result: dict[str, Any]) -> str:
|
||||
words: list[str] = []
|
||||
for item in result.get("ws", []):
|
||||
for candidate in item.get("cw", []):
|
||||
word = candidate.get("w")
|
||||
if word:
|
||||
words.append(word)
|
||||
return "".join(words)
|
||||
|
||||
|
||||
def _build_auth_url(url: str, api_key: str, api_secret: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
host = parsed.netloc
|
||||
path = parsed.path or "/v2/iat"
|
||||
date = format_datetime(datetime.now(timezone.utc), usegmt=True)
|
||||
request_line = f"GET {path} HTTP/1.1"
|
||||
signature_origin = f"host: {host}\ndate: {date}\n{request_line}"
|
||||
signature_sha = hmac.new(
|
||||
api_secret.encode("utf-8"),
|
||||
signature_origin.encode("utf-8"),
|
||||
digestmod=hashlib.sha256,
|
||||
).digest()
|
||||
signature = base64.b64encode(signature_sha).decode("utf-8")
|
||||
authorization_origin = (
|
||||
f'api_key="{api_key}", algorithm="hmac-sha256", '
|
||||
f'headers="host date request-line", signature="{signature}"'
|
||||
)
|
||||
authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode("utf-8")
|
||||
query = urlencode({"authorization": authorization, "date": date, "host": host})
|
||||
return f"{url}?{query}"
|
||||
|
||||
|
||||
def _language_or_none(value: str) -> Language | None:
|
||||
try:
|
||||
return Language(value)
|
||||
except ValueError:
|
||||
return None
|
||||
391
src/voice/xfyun_super_tts.py
Normal file
391
src/voice/xfyun_super_tts.py
Normal file
@@ -0,0 +1,391 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncGenerator
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import format_datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
EndFrame,
|
||||
ErrorFrame,
|
||||
Frame,
|
||||
StartFrame,
|
||||
TTSAudioRawFrame,
|
||||
TTSStoppedFrame,
|
||||
)
|
||||
from pipecat.services.settings import TTSSettings
|
||||
from pipecat.services.tts_service import TextAggregationMode, WebsocketTTSService
|
||||
from pipecat.utils.tracing.service_decorators import traced_tts
|
||||
|
||||
try:
|
||||
from websockets.asyncio.client import connect as websocket_connect
|
||||
from websockets.protocol import State
|
||||
except ModuleNotFoundError as exc:
|
||||
logger.error(f"Exception: {exc}")
|
||||
logger.error("In order to use Xfyun Super TTS, install the websockets package.")
|
||||
raise Exception(f"Missing module: {exc}") from exc
|
||||
|
||||
from .xfyun_tts import _sanitize_text_for_tts
|
||||
|
||||
|
||||
DEFAULT_XFYUN_SUPER_TTS_URL = "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6"
|
||||
VALID_SAMPLE_RATES = {8000, 16000, 24000}
|
||||
|
||||
|
||||
class XfyunSuperTTSService(WebsocketTTSService):
|
||||
"""iFlytek/Xfyun Super Smart TTS using bidirectional WebSocket streaming.
|
||||
|
||||
The service keeps one Xfyun synthesis session open for a Pipecat turn. Each
|
||||
``run_tts`` call sends a text segment with status 0/1, while ``flush_audio``
|
||||
sends the terminal status 2 frame. Audio arrives on the receive task and is
|
||||
appended to the Pipecat audio context.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
app_id: str,
|
||||
api_key: str,
|
||||
api_secret: str,
|
||||
voice: str,
|
||||
url: str | None = None,
|
||||
sample_rate: int = 16000,
|
||||
source_sample_rate: int = 24000,
|
||||
encoding: str = "raw",
|
||||
speed: int = 50,
|
||||
volume: int = 50,
|
||||
pitch: int = 50,
|
||||
oral_level: str = "mid",
|
||||
text_aggregation_mode: TextAggregationMode | str | None = TextAggregationMode.TOKEN,
|
||||
open_timeout: float = 30.0,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if isinstance(text_aggregation_mode, str):
|
||||
text_aggregation_mode = TextAggregationMode(text_aggregation_mode)
|
||||
|
||||
super().__init__(
|
||||
text_aggregation_mode=text_aggregation_mode,
|
||||
push_text_frames=True,
|
||||
push_stop_frames=False,
|
||||
push_start_frame=True,
|
||||
pause_frame_processing=False,
|
||||
sample_rate=sample_rate,
|
||||
settings=TTSSettings(model=None, voice=voice, language=None),
|
||||
**kwargs,
|
||||
)
|
||||
self._app_id = app_id or os.environ.get("XFYUN_APP_ID", "")
|
||||
self._api_key = api_key or os.environ.get("XFYUN_API_KEY", "")
|
||||
self._api_secret = api_secret or os.environ.get("XFYUN_API_SECRET", "")
|
||||
self._voice = voice
|
||||
self._url = url or DEFAULT_XFYUN_SUPER_TTS_URL
|
||||
self._source_sample_rate = source_sample_rate
|
||||
self._encoding = encoding
|
||||
self._speed = speed
|
||||
self._volume = volume
|
||||
self._pitch = pitch
|
||||
self._oral_level = oral_level
|
||||
self._open_timeout = open_timeout
|
||||
|
||||
self._receive_task: asyncio.Task | None = None
|
||||
self._active_context_id: str | None = None
|
||||
self._started_contexts: set[str] = set()
|
||||
self._seq_by_context: dict[str, int] = {}
|
||||
self._sent_text_bytes_by_context: dict[str, int] = {}
|
||||
self._stream_completed = False
|
||||
|
||||
def can_generate_metrics(self) -> bool:
|
||||
return True
|
||||
|
||||
async def start(self, frame: StartFrame) -> None:
|
||||
await super().start(frame)
|
||||
if not self._app_id or not self._api_key or not self._api_secret:
|
||||
await self.push_error(
|
||||
error_msg="Xfyun Super TTS requires app_id, api_key, and api_secret"
|
||||
)
|
||||
return
|
||||
if self._encoding != "raw":
|
||||
await self.push_error(error_msg="Xfyun Super TTS must use raw PCM audio in Pipecat")
|
||||
return
|
||||
if self._source_sample_rate not in VALID_SAMPLE_RATES:
|
||||
await self.push_error(
|
||||
error_msg=(
|
||||
"Xfyun Super TTS source_sample_rate must be one of "
|
||||
f"{sorted(VALID_SAMPLE_RATES)}"
|
||||
)
|
||||
)
|
||||
return
|
||||
await self._connect()
|
||||
|
||||
async def stop(self, frame: EndFrame) -> None:
|
||||
await super().stop(frame)
|
||||
await self._disconnect()
|
||||
|
||||
async def cancel(self, frame: CancelFrame) -> None:
|
||||
await super().cancel(frame)
|
||||
await self._disconnect()
|
||||
|
||||
async def flush_audio(self, context_id: str | None = None) -> None:
|
||||
flush_id = context_id or self.get_active_audio_context_id()
|
||||
if not flush_id or not self._websocket:
|
||||
return
|
||||
if flush_id not in self._started_contexts:
|
||||
return
|
||||
|
||||
logger.trace(f"{self}: flushing Xfyun Super TTS stream {flush_id}")
|
||||
await self._send_request_frame(flush_id, "", status=2)
|
||||
|
||||
async def on_audio_context_interrupted(self, context_id: str) -> None:
|
||||
await self.stop_all_metrics()
|
||||
await self._reset_context(context_id)
|
||||
await self._disconnect()
|
||||
await self._connect()
|
||||
await super().on_audio_context_interrupted(context_id)
|
||||
|
||||
async def _connect(self) -> None:
|
||||
await super()._connect()
|
||||
await self._connect_websocket()
|
||||
if self._websocket and not self._receive_task:
|
||||
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
|
||||
|
||||
async def _disconnect(self) -> None:
|
||||
await super()._disconnect()
|
||||
if self._receive_task:
|
||||
await self.cancel_task(self._receive_task)
|
||||
self._receive_task = None
|
||||
await self._disconnect_websocket()
|
||||
|
||||
async def _connect_websocket(self) -> None:
|
||||
try:
|
||||
if self._websocket and self._websocket.state is State.OPEN:
|
||||
return
|
||||
logger.debug("Connecting to Xfyun Super TTS")
|
||||
auth_url = _build_auth_url(self._url, self._api_key, self._api_secret)
|
||||
self._websocket = await websocket_connect(
|
||||
auth_url,
|
||||
max_size=None,
|
||||
open_timeout=self._open_timeout,
|
||||
)
|
||||
await self._call_event_handler("on_connected")
|
||||
except Exception as exc:
|
||||
self._websocket = None
|
||||
await self.push_error(
|
||||
error_msg=f"Unable to connect to Xfyun Super TTS: {exc}",
|
||||
exception=exc,
|
||||
)
|
||||
await self._call_event_handler("on_connection_error", f"{exc}")
|
||||
|
||||
async def _disconnect_websocket(self) -> None:
|
||||
try:
|
||||
await self.stop_all_metrics()
|
||||
if self._websocket:
|
||||
logger.debug("Disconnecting from Xfyun Super TTS")
|
||||
await self._websocket.close()
|
||||
except Exception as exc:
|
||||
await self.push_error(
|
||||
error_msg=f"Error closing Xfyun Super TTS websocket: {exc}",
|
||||
exception=exc,
|
||||
)
|
||||
finally:
|
||||
await self.remove_active_audio_context()
|
||||
self._websocket = None
|
||||
self._active_context_id = None
|
||||
self._started_contexts.clear()
|
||||
self._seq_by_context.clear()
|
||||
self._sent_text_bytes_by_context.clear()
|
||||
self._stream_completed = False
|
||||
await self._call_event_handler("on_disconnected")
|
||||
|
||||
def _get_websocket(self):
|
||||
if self._websocket:
|
||||
return self._websocket
|
||||
raise Exception("Websocket not connected")
|
||||
|
||||
async def _receive_messages(self) -> None:
|
||||
async for raw_message in self._get_websocket():
|
||||
try:
|
||||
message = json.loads(raw_message)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"{self}: received non-JSON Xfyun Super TTS message: {raw_message!r}")
|
||||
continue
|
||||
|
||||
header = message.get("header") or {}
|
||||
code = header.get("code", -1)
|
||||
sid = header.get("sid")
|
||||
context_id = self._active_context_id
|
||||
|
||||
if code != 0:
|
||||
error_message = header.get("message", "unknown error")
|
||||
await self.push_error(
|
||||
error_msg=f"Xfyun Super TTS error code={code}, sid={sid}: {error_message}"
|
||||
)
|
||||
if context_id and self.audio_context_available(context_id):
|
||||
await self.append_to_audio_context(
|
||||
context_id, TTSStoppedFrame(context_id=context_id)
|
||||
)
|
||||
await self.remove_audio_context(context_id)
|
||||
if context_id:
|
||||
await self._reset_context(context_id)
|
||||
continue
|
||||
|
||||
audio_obj = (message.get("payload") or {}).get("audio") or {}
|
||||
audio_b64 = audio_obj.get("audio")
|
||||
if audio_b64 and context_id and self.audio_context_available(context_id):
|
||||
await self.stop_ttfb_metrics()
|
||||
audio = base64.b64decode(audio_b64)
|
||||
if self._source_sample_rate != self.sample_rate:
|
||||
audio = await self._resampler.resample(
|
||||
audio, self._source_sample_rate, self.sample_rate
|
||||
)
|
||||
frame = TTSAudioRawFrame(audio, self.sample_rate, 1, context_id=context_id)
|
||||
await self.append_to_audio_context(context_id, frame)
|
||||
|
||||
audio_status = audio_obj.get("status")
|
||||
header_status = header.get("status")
|
||||
if audio_status == 2 or header_status == 2:
|
||||
if context_id and self.audio_context_available(context_id):
|
||||
await self.append_to_audio_context(
|
||||
context_id, TTSStoppedFrame(context_id=context_id)
|
||||
)
|
||||
await self.remove_audio_context(context_id)
|
||||
if context_id:
|
||||
await self._reset_context(context_id)
|
||||
self._stream_completed = True
|
||||
|
||||
@traced_tts
|
||||
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame | None, None]:
|
||||
sanitized = _sanitize_text_for_tts(text)
|
||||
if not sanitized:
|
||||
return
|
||||
|
||||
if not self._is_streaming_tokens:
|
||||
logger.debug(f"{self}: Generating Xfyun Super TTS [{sanitized}]")
|
||||
else:
|
||||
logger.trace(f"{self}: Generating Xfyun Super TTS [{sanitized}]")
|
||||
|
||||
if self._stream_completed and self._websocket:
|
||||
await self._disconnect()
|
||||
await self._connect()
|
||||
|
||||
if not self._websocket or self._websocket.state is State.CLOSED:
|
||||
await self._connect()
|
||||
|
||||
if self._active_context_id and self._active_context_id != context_id:
|
||||
yield ErrorFrame(
|
||||
error=(
|
||||
"Xfyun Super TTS supports one active synthesis stream per WebSocket; "
|
||||
f"active={self._active_context_id}, new={context_id}"
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
status = 0 if context_id not in self._started_contexts else 1
|
||||
await self._send_request_frame(context_id, sanitized, status=status)
|
||||
await self.start_tts_usage_metrics(sanitized)
|
||||
except Exception as exc:
|
||||
yield ErrorFrame(error=f"Xfyun Super TTS request failed: {exc}")
|
||||
yield TTSStoppedFrame(context_id=context_id)
|
||||
await self._disconnect()
|
||||
await self._connect()
|
||||
return
|
||||
|
||||
yield None
|
||||
|
||||
async def _send_request_frame(self, context_id: str, text: str, *, status: int) -> None:
|
||||
if status == 0:
|
||||
self._active_context_id = context_id
|
||||
self._started_contexts.add(context_id)
|
||||
|
||||
seq = self._seq_by_context.get(context_id, 0)
|
||||
text_bytes = text.encode("utf-8")
|
||||
total_bytes = self._sent_text_bytes_by_context.get(context_id, 0) + len(text_bytes)
|
||||
if total_bytes > 65536:
|
||||
raise ValueError("Xfyun Super TTS text must not exceed 64K UTF-8 bytes per stream")
|
||||
|
||||
frame = self._build_request_frame(text, status=status, seq=seq)
|
||||
await self._get_websocket().send(json.dumps(frame, ensure_ascii=False))
|
||||
|
||||
self._seq_by_context[context_id] = seq + 1
|
||||
self._sent_text_bytes_by_context[context_id] = total_bytes
|
||||
|
||||
def _build_request_frame(self, text: str, *, status: int, seq: int) -> dict[str, Any]:
|
||||
return {
|
||||
"header": {
|
||||
"app_id": self._app_id,
|
||||
"status": status,
|
||||
},
|
||||
"parameter": {
|
||||
"oral": {
|
||||
"oral_level": self._oral_level,
|
||||
},
|
||||
"tts": {
|
||||
"vcn": self._voice,
|
||||
"speed": self._speed,
|
||||
"volume": self._volume,
|
||||
"pitch": self._pitch,
|
||||
"bgs": 0,
|
||||
"reg": 0,
|
||||
"rdn": 0,
|
||||
"rhy": 0,
|
||||
"audio": {
|
||||
"encoding": self._encoding,
|
||||
"sample_rate": self._source_sample_rate,
|
||||
"channels": 1,
|
||||
"bit_depth": 16,
|
||||
"frame_size": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
"payload": {
|
||||
"text": {
|
||||
"encoding": "utf8",
|
||||
"compress": "raw",
|
||||
"format": "plain",
|
||||
"status": status,
|
||||
"seq": seq,
|
||||
"text": base64.b64encode(text.encode("utf-8")).decode("utf-8"),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
async def _reset_context(self, context_id: str) -> None:
|
||||
self._started_contexts.discard(context_id)
|
||||
self._seq_by_context.pop(context_id, None)
|
||||
self._sent_text_bytes_by_context.pop(context_id, None)
|
||||
if self._active_context_id == context_id:
|
||||
self._active_context_id = None
|
||||
|
||||
|
||||
def _build_auth_url(url: str, api_key: str, api_secret: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"ws", "wss"} or not parsed.hostname:
|
||||
raise ValueError(f"invalid Xfyun Super TTS WebSocket URL: {url}")
|
||||
|
||||
host = parsed.hostname
|
||||
path = parsed.path or "/"
|
||||
date = format_datetime(datetime.now(timezone.utc), usegmt=True)
|
||||
request_line = f"GET {path} HTTP/1.1"
|
||||
signature_origin = f"host: {host}\ndate: {date}\n{request_line}"
|
||||
signature_sha = hmac.new(
|
||||
api_secret.encode("utf-8"),
|
||||
signature_origin.encode("utf-8"),
|
||||
digestmod=hashlib.sha256,
|
||||
).digest()
|
||||
signature = base64.b64encode(signature_sha).decode("utf-8")
|
||||
authorization_origin = (
|
||||
f'api_key="{api_key}", algorithm="hmac-sha256", '
|
||||
f'headers="host date request-line", signature="{signature}"'
|
||||
)
|
||||
authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode("utf-8")
|
||||
query = urlencode({"authorization": authorization, "date": date, "host": host})
|
||||
return f"{url}?{query}"
|
||||
257
src/voice/xfyun_tts.py
Normal file
257
src/voice/xfyun_tts.py
Normal file
@@ -0,0 +1,257 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import format_datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import ErrorFrame, Frame
|
||||
from pipecat.services.settings import TTSSettings
|
||||
from pipecat.services.tts_service import TTSService
|
||||
from websockets.asyncio.client import connect
|
||||
|
||||
|
||||
DEFAULT_XFYUN_TTS_URL = "wss://tts-api.xfyun.cn/v2/tts"
|
||||
|
||||
# Strip characters Xfyun's online TTS cannot synthesize. The engine silently
|
||||
# rejects (or returns empty audio for) text containing emoji and other
|
||||
# non-BMP symbols, which surfaces as "request finished without audio data".
|
||||
_EMOJI_AND_SYMBOL_RE = re.compile(
|
||||
"["
|
||||
"\U0001F300-\U0001FAFF" # misc pictographs, emoji, symbols, transport, etc.
|
||||
"\U00002600-\U000027BF" # misc symbols and dingbats
|
||||
"\U0001F1E6-\U0001F1FF" # regional indicators (flags)
|
||||
"\uFE00-\uFE0F" # variation selectors
|
||||
"\u200D" # zero-width joiner
|
||||
"]",
|
||||
flags=re.UNICODE,
|
||||
)
|
||||
|
||||
|
||||
class XfyunTTSService(TTSService):
|
||||
"""iFlytek/Xfyun online TTS service for Pipecat.
|
||||
|
||||
Xfyun's API is not OpenAI-compatible. It uses a signed WebSocket URL,
|
||||
receives one JSON request per synthesis, and streams text WebSocket
|
||||
messages containing base64-encoded audio chunks. This service requests
|
||||
raw PCM so the chunks can become Pipecat audio frames without MP3 decode.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
app_id: str,
|
||||
api_key: str,
|
||||
api_secret: str,
|
||||
voice: str,
|
||||
url: str | None = None,
|
||||
sample_rate: int = 16000,
|
||||
source_sample_rate: int = 16000,
|
||||
encoding: str = "raw",
|
||||
text_encoding: str = "UTF8",
|
||||
speed: int = 50,
|
||||
volume: int = 50,
|
||||
pitch: int = 50,
|
||||
timeout: float = 30.0,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
sample_rate=sample_rate,
|
||||
settings=TTSSettings(model=None, voice=voice, language=None),
|
||||
**kwargs,
|
||||
)
|
||||
self._app_id = app_id or os.environ.get("XFYUN_APP_ID", "")
|
||||
self._api_key = api_key or os.environ.get("XFYUN_API_KEY", "")
|
||||
self._api_secret = api_secret or os.environ.get("XFYUN_API_SECRET", "")
|
||||
self._voice = voice
|
||||
self._url = url or DEFAULT_XFYUN_TTS_URL
|
||||
self._source_sample_rate = source_sample_rate
|
||||
self._encoding = encoding
|
||||
self._text_encoding = text_encoding
|
||||
self._speed = speed
|
||||
self._volume = volume
|
||||
self._pitch = pitch
|
||||
self._timeout = timeout
|
||||
self._last_failure_detail: str | None = None
|
||||
|
||||
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
|
||||
if not text:
|
||||
return
|
||||
|
||||
if not self._app_id or not self._api_key or not self._api_secret:
|
||||
yield ErrorFrame(error="Xfyun TTS requires app_id, api_key, and api_secret")
|
||||
return
|
||||
|
||||
sanitized = _sanitize_text_for_tts(text)
|
||||
if not sanitized:
|
||||
logger.debug(
|
||||
f"{self}: skipping Xfyun TTS, text became empty after sanitization "
|
||||
f"(original={text!r})"
|
||||
)
|
||||
return
|
||||
|
||||
if sanitized != text:
|
||||
logger.debug(
|
||||
f"{self}: sanitized Xfyun TTS text "
|
||||
f"(original={text!r}, sanitized={sanitized!r})"
|
||||
)
|
||||
|
||||
if len(sanitized.encode("utf-8")) >= 8000:
|
||||
yield ErrorFrame(error="Xfyun TTS text must be less than 8000 UTF-8 bytes")
|
||||
return
|
||||
|
||||
if self._encoding != "raw":
|
||||
yield ErrorFrame(error="Xfyun TTS is configured for PCM output; set aue/encoding to raw")
|
||||
return
|
||||
|
||||
try:
|
||||
await self.start_tts_usage_metrics(sanitized)
|
||||
|
||||
first_frame = True
|
||||
async for frame in self._stream_audio_frames_from_iterator(
|
||||
self._iter_audio_chunks(sanitized),
|
||||
in_sample_rate=self._source_sample_rate,
|
||||
context_id=context_id,
|
||||
):
|
||||
if first_frame:
|
||||
await self.stop_ttfb_metrics()
|
||||
first_frame = False
|
||||
yield frame
|
||||
|
||||
if first_frame:
|
||||
detail = self._last_failure_detail or "no audio frames received"
|
||||
yield ErrorFrame(
|
||||
error=(
|
||||
f"Xfyun TTS request finished without audio data ({detail}); "
|
||||
f"text={sanitized!r}"
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
yield ErrorFrame(error=f"Xfyun TTS request failed: {exc}")
|
||||
|
||||
async def _iter_audio_chunks(self, text: str) -> AsyncIterator[bytes]:
|
||||
request = self._build_request_frame(text)
|
||||
auth_url = _build_auth_url(self._url, self._api_key, self._api_secret)
|
||||
|
||||
self._last_failure_detail = None
|
||||
frames_received = 0
|
||||
audio_bytes_received = 0
|
||||
last_status: int | None = None
|
||||
last_sid: str | None = None
|
||||
saw_status_2 = False
|
||||
|
||||
async with connect(auth_url, max_size=None, open_timeout=self._timeout) as websocket:
|
||||
await websocket.send(json.dumps(request, ensure_ascii=False))
|
||||
|
||||
async for raw_message in websocket:
|
||||
frames_received += 1
|
||||
payload = json.loads(raw_message)
|
||||
code = payload.get("code", -1)
|
||||
sid = payload.get("sid")
|
||||
if sid:
|
||||
last_sid = sid
|
||||
if code != 0:
|
||||
err_msg = payload.get("message", "unknown error")
|
||||
raise RuntimeError(f"code={code}, sid={sid}, message={err_msg}")
|
||||
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
|
||||
last_status = data.get("status", last_status)
|
||||
|
||||
audio_b64 = data.get("audio")
|
||||
if audio_b64:
|
||||
audio_bytes = base64.b64decode(audio_b64)
|
||||
audio_bytes_received += len(audio_bytes)
|
||||
yield audio_bytes
|
||||
|
||||
if data.get("status") == 2:
|
||||
saw_status_2 = True
|
||||
break
|
||||
|
||||
if audio_bytes_received == 0:
|
||||
self._last_failure_detail = (
|
||||
f"frames={frames_received}, audio_bytes=0, "
|
||||
f"last_status={last_status}, saw_status_2={saw_status_2}, sid={last_sid}"
|
||||
)
|
||||
logger.warning(
|
||||
f"{self}: Xfyun TTS produced no audio ({self._last_failure_detail})"
|
||||
)
|
||||
|
||||
def _build_request_frame(self, text: str) -> dict[str, Any]:
|
||||
business: dict[str, Any] = {
|
||||
"aue": self._encoding,
|
||||
"auf": f"audio/L16;rate={self._source_sample_rate}",
|
||||
"vcn": self._voice,
|
||||
"speed": self._speed,
|
||||
"volume": self._volume,
|
||||
"pitch": self._pitch,
|
||||
"tte": self._text_encoding,
|
||||
}
|
||||
|
||||
return {
|
||||
"common": {"app_id": self._app_id},
|
||||
"business": business,
|
||||
"data": {
|
||||
"status": 2,
|
||||
"text": base64.b64encode(text.encode("utf-8")).decode("utf-8"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_text_for_tts(text: str) -> str:
|
||||
"""Strip characters Xfyun's online TTS cannot synthesize.
|
||||
|
||||
The Xfyun ``/v2/tts`` engine silently drops or rejects emoji, pictographs,
|
||||
dingbats, regional-indicator flags, variation selectors, and zero-width
|
||||
joiners. When such characters appear in the input the synthesis can
|
||||
finish without any audio data ("Xfyun TTS request finished without audio
|
||||
data"). We also drop control characters (other than common whitespace)
|
||||
and "Symbol, Other" codepoints, then collapse runs of whitespace.
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
cleaned = _EMOJI_AND_SYMBOL_RE.sub("", text)
|
||||
filtered: list[str] = []
|
||||
for ch in cleaned:
|
||||
category = unicodedata.category(ch)
|
||||
if category == "So":
|
||||
continue
|
||||
if category.startswith("C") and ch not in ("\n", "\r", "\t"):
|
||||
continue
|
||||
filtered.append(ch)
|
||||
return re.sub(r"\s+", " ", "".join(filtered)).strip()
|
||||
|
||||
|
||||
def _build_auth_url(url: str, api_key: str, api_secret: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
host = parsed.netloc
|
||||
path = parsed.path or "/v2/tts"
|
||||
date = format_datetime(datetime.now(timezone.utc), usegmt=True)
|
||||
request_line = f"GET {path} HTTP/1.1"
|
||||
signature_origin = f"host: {host}\ndate: {date}\n{request_line}"
|
||||
signature_sha = hmac.new(
|
||||
api_secret.encode("utf-8"),
|
||||
signature_origin.encode("utf-8"),
|
||||
digestmod=hashlib.sha256,
|
||||
).digest()
|
||||
signature = base64.b64encode(signature_sha).decode("utf-8")
|
||||
authorization_origin = (
|
||||
f'api_key="{api_key}", algorithm="hmac-sha256", '
|
||||
f'headers="host date request-line", signature="{signature}"'
|
||||
)
|
||||
authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode("utf-8")
|
||||
query = urlencode({"authorization": authorization, "date": date, "host": host})
|
||||
return f"{url}?{query}"
|
||||
106
static/voice-demo/README.md
Normal file
106
static/voice-demo/README.md
Normal file
@@ -0,0 +1,106 @@
|
||||
# Webpage Example — Realtime Voice Chat
|
||||
|
||||
A self-contained browser client for the engine's product websocket
|
||||
(`/ws-product`, protocol `va.ws.v1`).
|
||||
|
||||
## Features
|
||||
|
||||
- **Connect / Disconnect** to any `ws://` or `wss://` URL.
|
||||
- **Microphone selector + mic on/off toggle** — available input devices
|
||||
are listed with `enumerateDevices`, and getUserMedia is requested with
|
||||
`echoCancellation`, `noiseSuppression`, and `autoGainControl` so the
|
||||
browser handles AEC against the bot's voice.
|
||||
- **Text composer** — type a message and press <kbd>Enter</kbd> to send
|
||||
an `input.text` event (Shift+Enter for newline). Sending interrupts
|
||||
any in-flight bot audio so the next reply is heard cleanly.
|
||||
- **Chat history** rendered from `input.transcript.final` (you, when
|
||||
spoken), streamed `response.text.delta` / `response.text.final`
|
||||
(assistant — deltas arrive ahead of the synthesized audio), and locally
|
||||
for text you submit (the engine doesn't echo text input back as a
|
||||
transcript).
|
||||
- **WebSocket log** panel for connection state and compact send/receive
|
||||
events. Audio chunks are summarized so the UI does not flood.
|
||||
- **Gapless TTS playback** by scheduling each `response.audio.delta`
|
||||
chunk back-to-back on the AudioContext.
|
||||
- **Live VU meter** + mic and bot activity indicators.
|
||||
- **Clear** button to reset history.
|
||||
|
||||
No build step, no dependencies — just three files plus an AudioWorklet.
|
||||
|
||||
## Layout
|
||||
|
||||
```text
|
||||
examples/webpage/
|
||||
├── index.html
|
||||
├── styles.css
|
||||
├── app.js
|
||||
└── pcm-recorder.worklet.js
|
||||
```
|
||||
|
||||
## Run
|
||||
|
||||
1. Start the engine (default port `8000`):
|
||||
|
||||
```bash
|
||||
cd AI-VideoAssistant-engine-v5-pipecat-minimal
|
||||
source .venv/bin/activate
|
||||
export OPENAI_API_KEY=...
|
||||
uvicorn engine.main:app --host 127.0.0.1 --port 8000
|
||||
```
|
||||
|
||||
2. Open the demo page served by the same process:
|
||||
|
||||
```text
|
||||
http://127.0.0.1:8000/voice-demo/
|
||||
```
|
||||
|
||||
The default websocket URL is derived from the page host
|
||||
(`ws://127.0.0.1:8000/ws-product`). Click **Connect**, pick a
|
||||
microphone if needed, click **Enable mic**, and start speaking.
|
||||
|
||||
Mount path and on/off are controlled in `config.json`:
|
||||
|
||||
```json
|
||||
"server": {
|
||||
"serve_webpage": true,
|
||||
"webpage_mount": "/voice-demo"
|
||||
}
|
||||
```
|
||||
|
||||
Set `"serve_webpage": false` in production if you serve the UI elsewhere.
|
||||
|
||||
### Standalone static server (optional)
|
||||
|
||||
You can still serve the files from another port for UI-only iteration.
|
||||
Add that origin to `server.cors_origins` in `config.json` if needed:
|
||||
|
||||
```bash
|
||||
cd AI-VideoAssistant-engine-v5-pipecat-minimal/examples/webpage
|
||||
python -m http.server 8080
|
||||
```
|
||||
|
||||
Then open <http://localhost:8080> and point the URL field at
|
||||
`ws://127.0.0.1:8000/ws-product`.
|
||||
|
||||
> The browser's mic API requires a secure context. `http://localhost`
|
||||
> qualifies; if you serve from another host, use HTTPS and a `wss://`
|
||||
> URL.
|
||||
|
||||
## Audio details
|
||||
|
||||
- Input: mono Float32 from `getUserMedia` is resampled in the
|
||||
AudioWorklet to PCM16 mono @ 16 kHz, framed into 20 ms chunks, and
|
||||
sent as **binary** websocket messages (the server accepts either
|
||||
binary or the JSON+base64 form).
|
||||
- Output: each `response.audio.delta` carries base64-encoded PCM16 @
|
||||
16 kHz; chunks are decoded and scheduled back-to-back through Web
|
||||
Audio. The browser handles resampling to the device rate.
|
||||
|
||||
## Notes
|
||||
|
||||
- Use headphones if you still hear echo despite browser AEC; the bot's
|
||||
voice leaking back into the open mic is the most common cause of
|
||||
feedback loops.
|
||||
- The engine's session has an inactivity timeout
|
||||
(`session.inactivity_timeout_sec` in `config.json`). If the bot
|
||||
doesn't respond after a long silence, reconnect.
|
||||
1544
static/voice-demo/app.js
Normal file
1544
static/voice-demo/app.js
Normal file
File diff suppressed because it is too large
Load Diff
288
static/voice-demo/index.html
Normal file
288
static/voice-demo/index.html
Normal file
@@ -0,0 +1,288 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>VA Voice Chat — /ws-product</title>
|
||||
<link rel="stylesheet" href="./styles.css" />
|
||||
</head>
|
||||
<body>
|
||||
<main class="app">
|
||||
<header class="app__header">
|
||||
<div class="brand">
|
||||
<span class="brand__dot" aria-hidden="true"></span>
|
||||
<h1>VA Voice Chat</h1>
|
||||
</div>
|
||||
|
||||
<div class="connection">
|
||||
<label class="connection__field">
|
||||
<span>服务器地址</span>
|
||||
<input
|
||||
id="ws-url"
|
||||
type="text"
|
||||
placeholder="ws://host/ws-product"
|
||||
spellcheck="false"
|
||||
autocomplete="off"
|
||||
/>
|
||||
</label>
|
||||
<label class="connection__field connection__field--chat">
|
||||
<span>会话 ID</span>
|
||||
<div class="chat-id-control">
|
||||
<input
|
||||
id="chat-id"
|
||||
type="text"
|
||||
placeholder="可选"
|
||||
spellcheck="false"
|
||||
autocomplete="off"
|
||||
/>
|
||||
<button
|
||||
id="copy-chat-id-btn"
|
||||
class="chat-id-control__copy"
|
||||
type="button"
|
||||
disabled
|
||||
title="复制会话 ID"
|
||||
aria-label="复制会话 ID"
|
||||
>
|
||||
<svg class="copy-icon copy-icon--default" viewBox="0 0 16 16" width="14" height="14" fill="none" aria-hidden="true">
|
||||
<rect x="5" y="5" width="8" height="9" rx="1.5" stroke="currentColor" stroke-width="1.4"/>
|
||||
<path d="M3 11V3.5A1.5 1.5 0 0 1 4.5 2H11" stroke="currentColor" stroke-width="1.4" stroke-linecap="round"/>
|
||||
</svg>
|
||||
<svg class="copy-icon copy-icon--check" viewBox="0 0 16 16" width="14" height="14" fill="none" aria-hidden="true">
|
||||
<path d="M3 8.5l3.5 3.5 6.5-7" stroke="currentColor" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</label>
|
||||
<button id="connect-btn" class="btn btn--primary" type="button">
|
||||
连接
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="status">
|
||||
<span id="status-dot" class="status__dot status__dot--idle"></span>
|
||||
<span id="status-text" class="status__text">未连接</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="app__body">
|
||||
<div class="app__main">
|
||||
<div id="conversation" class="conversation">
|
||||
<aside
|
||||
id="camera-drawer"
|
||||
class="camera-drawer"
|
||||
aria-label="拍照步骤"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<div class="camera-drawer__panel">
|
||||
<div class="camera-drawer__header">
|
||||
<div>
|
||||
<p class="camera-drawer__eyebrow">拍照</p>
|
||||
<h2>拍照步骤</h2>
|
||||
</div>
|
||||
<span id="camera-state" class="camera-drawer__state">状态 -</span>
|
||||
</div>
|
||||
|
||||
<div id="camera-preview" class="camera-drawer__preview">
|
||||
<video
|
||||
id="camera-video"
|
||||
class="camera-drawer__video"
|
||||
playsinline
|
||||
muted
|
||||
autoplay
|
||||
></video>
|
||||
<img
|
||||
id="camera-photo"
|
||||
class="camera-drawer__photo"
|
||||
alt="已选择图片预览"
|
||||
/>
|
||||
<span class="camera-drawer__corner camera-drawer__corner--tl"></span>
|
||||
<span class="camera-drawer__corner camera-drawer__corner--tr"></span>
|
||||
<span class="camera-drawer__corner camera-drawer__corner--bl"></span>
|
||||
<span class="camera-drawer__corner camera-drawer__corner--br"></span>
|
||||
<span class="camera-drawer__lens"></span>
|
||||
<span class="camera-drawer__scan"></span>
|
||||
<span id="camera-placeholder" class="camera-drawer__placeholder">
|
||||
打开摄像头实时拍摄,或从下方选择 / 上传图片
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<p id="camera-question" class="camera-drawer__question"></p>
|
||||
|
||||
<div
|
||||
id="camera-samples"
|
||||
class="camera-drawer__samples"
|
||||
aria-label="示例图片,点击选择"
|
||||
></div>
|
||||
|
||||
<div class="camera-drawer__sources">
|
||||
<label
|
||||
class="btn btn--ghost camera-drawer__source"
|
||||
>
|
||||
上传图片
|
||||
<input
|
||||
id="camera-upload"
|
||||
type="file"
|
||||
accept="image/*"
|
||||
hidden
|
||||
/>
|
||||
</label>
|
||||
<button
|
||||
id="camera-start-btn"
|
||||
class="btn btn--ghost camera-drawer__source"
|
||||
type="button"
|
||||
title="打开摄像头"
|
||||
>
|
||||
使用摄像头
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<label
|
||||
id="camera-device-row"
|
||||
class="device-picker camera-drawer__device-row"
|
||||
hidden
|
||||
>
|
||||
<span class="device-picker__label">选择摄像头</span>
|
||||
<select
|
||||
id="camera-device-select"
|
||||
class="device-picker__select"
|
||||
disabled
|
||||
>
|
||||
<option value="">默认摄像头</option>
|
||||
</select>
|
||||
</label>
|
||||
|
||||
<button
|
||||
id="camera-done-btn"
|
||||
class="btn btn--primary camera-drawer__button"
|
||||
type="button"
|
||||
disabled
|
||||
>
|
||||
拍摄完成
|
||||
</button>
|
||||
<canvas id="camera-canvas" hidden></canvas>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<section class="chat" aria-label="对话记录">
|
||||
<div id="chat-log" class="chat__log" role="log" aria-live="polite">
|
||||
<div class="chat__empty">
|
||||
<p>连接服务、开启麦克风后即可开始对话。</p>
|
||||
<p class="chat__hint">
|
||||
音频通过 <code>/ws-product</code> 以 PCM16 单声道 16 kHz
|
||||
传输。
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<footer class="controls" aria-label="操作栏">
|
||||
<div class="meter" aria-hidden="true">
|
||||
<div id="meter-fill" class="meter__fill"></div>
|
||||
</div>
|
||||
|
||||
<form id="composer" class="composer" autocomplete="off">
|
||||
<textarea
|
||||
id="text-input"
|
||||
class="composer__input"
|
||||
rows="1"
|
||||
placeholder="输入消息,或使用麦克风…"
|
||||
disabled
|
||||
></textarea>
|
||||
<button
|
||||
id="send-btn"
|
||||
class="btn btn--primary composer__send"
|
||||
type="submit"
|
||||
disabled
|
||||
title="发送消息 (Enter)"
|
||||
>
|
||||
发送
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<div class="controls__row">
|
||||
<label class="device-picker">
|
||||
<span class="device-picker__label">麦克风</span>
|
||||
<select id="mic-select" class="device-picker__select" disabled>
|
||||
<option value="">默认麦克风</option>
|
||||
</select>
|
||||
</label>
|
||||
|
||||
<button
|
||||
id="mic-btn"
|
||||
class="mic-btn"
|
||||
type="button"
|
||||
disabled
|
||||
aria-pressed="false"
|
||||
title="麦克风已关闭"
|
||||
>
|
||||
<svg
|
||||
class="mic-btn__icon"
|
||||
viewBox="0 0 24 24"
|
||||
width="24"
|
||||
height="24"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
d="M12 14a3 3 0 0 0 3-3V6a3 3 0 1 0-6 0v5a3 3 0 0 0 3 3Z"
|
||||
fill="currentColor"
|
||||
/>
|
||||
<path
|
||||
d="M19 11a1 1 0 1 0-2 0 5 5 0 0 1-10 0 1 1 0 1 0-2 0 7 7 0 0 0 6 6.92V21a1 1 0 1 0 2 0v-3.08A7 7 0 0 0 19 11Z"
|
||||
fill="currentColor"
|
||||
/>
|
||||
</svg>
|
||||
<span class="mic-btn__label">开启麦克风</span>
|
||||
</button>
|
||||
|
||||
<div class="indicators">
|
||||
<span id="mic-indicator" class="indicator">
|
||||
<span class="indicator__dot indicator__dot--mic"></span>
|
||||
<span class="indicator__label">麦克风</span>
|
||||
</span>
|
||||
<span id="bot-indicator" class="indicator">
|
||||
<span class="indicator__dot indicator__dot--bot"></span>
|
||||
<span class="indicator__label">助手</span>
|
||||
</span>
|
||||
<span id="state-indicator" class="indicator indicator--state">
|
||||
<span class="indicator__dot indicator__dot--state"></span>
|
||||
<span id="state-label" class="indicator__label">状态 -</span>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<button id="clear-btn" class="btn btn--ghost" type="button">
|
||||
清空
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<p class="hint">
|
||||
按 <kbd>Enter</kbd> 发送,<kbd>Shift</kbd>+<kbd>Enter</kbd>
|
||||
换行。发送文字会打断正在说话的助手。
|
||||
浏览器回声消除已开启,如有回音请使用耳机。
|
||||
</p>
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
<section class="ws-log" aria-label="WebSocket 日志">
|
||||
<div class="ws-log__header">
|
||||
<div class="ws-log__header-left">
|
||||
<h2>WebSocket 日志</h2>
|
||||
<div class="ws-log__legend" aria-hidden="true">
|
||||
<span class="ws-log__legend-item ws-log__legend-item--send">发送</span>
|
||||
<span class="ws-log__legend-item ws-log__legend-item--recv">接收</span>
|
||||
</div>
|
||||
</div>
|
||||
<button id="clear-ws-log-btn" class="btn btn--ghost" type="button">
|
||||
清空日志
|
||||
</button>
|
||||
</div>
|
||||
<div id="ws-log" class="ws-log__body" role="log" aria-live="polite">
|
||||
<div class="ws-log__empty">暂无 WebSocket 事件。</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<script type="module" src="./app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
104
static/voice-demo/pcm-recorder.worklet.js
Normal file
104
static/voice-demo/pcm-recorder.worklet.js
Normal file
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* PCM Recorder AudioWorklet.
|
||||
*
|
||||
* Captures mono Float32 mic samples at the AudioContext's native rate,
|
||||
* resamples them to a target sample rate (default 16 kHz) with linear
|
||||
* interpolation, then ships PCM16 frames of a fixed duration (default 20 ms)
|
||||
* to the main thread via `port.postMessage(ArrayBuffer)`.
|
||||
*
|
||||
* It also computes a simple RMS level per frame for the UI VU meter so the
|
||||
* main thread doesn't have to re-process the audio.
|
||||
*/
|
||||
|
||||
class PcmRecorderProcessor extends AudioWorkletProcessor {
|
||||
constructor(options) {
|
||||
super();
|
||||
|
||||
const opts = (options && options.processorOptions) || {};
|
||||
this._targetSampleRate = opts.targetSampleRate || 16000;
|
||||
this._frameMs = opts.frameMs || 20;
|
||||
this._frameSamples = Math.round(
|
||||
(this._targetSampleRate * this._frameMs) / 1000,
|
||||
);
|
||||
|
||||
// Resampling state.
|
||||
// `ratio` is input samples per output sample.
|
||||
this._ratio = sampleRate / this._targetSampleRate;
|
||||
this._inputBuffer = new Float32Array(0);
|
||||
// Float position in `_inputBuffer` for the next output sample.
|
||||
this._inputOffset = 0;
|
||||
|
||||
// Output framing state.
|
||||
this._frameBuffer = new Int16Array(this._frameSamples);
|
||||
this._frameIndex = 0;
|
||||
|
||||
// VU meter accumulator.
|
||||
this._rmsSumSquares = 0;
|
||||
this._rmsCount = 0;
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (!input || input.length === 0) return true;
|
||||
const channel = input[0];
|
||||
if (!channel || channel.length === 0) return true;
|
||||
|
||||
// Append new samples to the input buffer.
|
||||
const merged = new Float32Array(this._inputBuffer.length + channel.length);
|
||||
merged.set(this._inputBuffer, 0);
|
||||
merged.set(channel, this._inputBuffer.length);
|
||||
this._inputBuffer = merged;
|
||||
|
||||
const ratio = this._ratio;
|
||||
const inLen = this._inputBuffer.length;
|
||||
let pos = this._inputOffset;
|
||||
|
||||
while (pos + 1 < inLen) {
|
||||
const lo = Math.floor(pos);
|
||||
const hi = lo + 1;
|
||||
const w = pos - lo;
|
||||
const sample =
|
||||
this._inputBuffer[lo] * (1 - w) + this._inputBuffer[hi] * w;
|
||||
|
||||
this._rmsSumSquares += sample * sample;
|
||||
this._rmsCount += 1;
|
||||
|
||||
let s = sample;
|
||||
if (s > 1) s = 1;
|
||||
else if (s < -1) s = -1;
|
||||
this._frameBuffer[this._frameIndex++] =
|
||||
s < 0 ? Math.round(s * 0x8000) : Math.round(s * 0x7fff);
|
||||
|
||||
if (this._frameIndex === this._frameSamples) {
|
||||
const frame = new Int16Array(this._frameSamples);
|
||||
frame.set(this._frameBuffer);
|
||||
const rms =
|
||||
this._rmsCount > 0
|
||||
? Math.sqrt(this._rmsSumSquares / this._rmsCount)
|
||||
: 0;
|
||||
this.port.postMessage(
|
||||
{ type: "frame", buffer: frame.buffer, rms },
|
||||
[frame.buffer],
|
||||
);
|
||||
this._frameIndex = 0;
|
||||
this._rmsSumSquares = 0;
|
||||
this._rmsCount = 0;
|
||||
}
|
||||
|
||||
pos += ratio;
|
||||
}
|
||||
|
||||
// Trim consumed samples from the input buffer; keep at least the last
|
||||
// sample we still need to interpolate against on the next call.
|
||||
const consumed = Math.floor(pos);
|
||||
if (consumed > 0) {
|
||||
this._inputBuffer = this._inputBuffer.slice(consumed);
|
||||
pos -= consumed;
|
||||
}
|
||||
this._inputOffset = pos;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("pcm-recorder", PcmRecorderProcessor);
|
||||
BIN
static/voice-demo/samples/.DS_Store
vendored
Normal file
BIN
static/voice-demo/samples/.DS_Store
vendored
Normal file
Binary file not shown.
BIN
static/voice-demo/samples/damage1.png
Normal file
BIN
static/voice-demo/samples/damage1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 273 KiB |
BIN
static/voice-demo/samples/damage2.png
Normal file
BIN
static/voice-demo/samples/damage2.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 323 KiB |
BIN
static/voice-demo/samples/plate1.jpg
Normal file
BIN
static/voice-demo/samples/plate1.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 7.5 KiB |
BIN
static/voice-demo/samples/plate2.jpg
Normal file
BIN
static/voice-demo/samples/plate2.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 229 KiB |
BIN
static/voice-demo/samples/user1.jpg
Normal file
BIN
static/voice-demo/samples/user1.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 72 KiB |
BIN
static/voice-demo/samples/user2.jpg
Normal file
BIN
static/voice-demo/samples/user2.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 105 KiB |
1240
static/voice-demo/styles.css
Normal file
1240
static/voice-demo/styles.css
Normal file
File diff suppressed because it is too large
Load Diff
121
test/agent/test_langgraph_backend.py
Normal file
121
test/agent/test_langgraph_backend.py
Normal file
@@ -0,0 +1,121 @@
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
from src.agent.state import AccidentGraphState, GeneratedTurn
|
||||
from src.backends.chat import ChatInput, FormUpdate, TextDelta
|
||||
from src.backends.fastgpt import FastGPTBackend
|
||||
from src.backends.langgraph import LangGraphBackend
|
||||
from src.core.config import Settings
|
||||
from src.core.fastgpt_client import create_chat_backend
|
||||
|
||||
|
||||
class FakeResponseGenerator:
|
||||
def __init__(self):
|
||||
self.states = []
|
||||
self.closed = False
|
||||
|
||||
async def generate(self, state: AccidentGraphState) -> GeneratedTurn:
|
||||
self.states.append(deepcopy(state))
|
||||
turn_number = state.get("turn_count", 0) + 1
|
||||
patch = {"turn": turn_number} if state["need_form_update"] else {}
|
||||
return GeneratedTurn(
|
||||
content=f"<state>1002</state>第{turn_number}轮",
|
||||
form_update=patch,
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
def langgraph_settings():
|
||||
return Settings(
|
||||
_env_file=None,
|
||||
environment="test",
|
||||
agent_backend="langgraph",
|
||||
langgraph_checkpointer="memory",
|
||||
llm_api_key="test-key",
|
||||
llm_model="test-model",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_langgraph_backend_preserves_thread_scoped_turn_state():
|
||||
generator = FakeResponseGenerator()
|
||||
backend = create_chat_backend(
|
||||
langgraph_settings(),
|
||||
response_generator=generator,
|
||||
)
|
||||
assert isinstance(backend, LangGraphBackend)
|
||||
|
||||
first = await backend.complete(
|
||||
ChatInput("session-1", "第一轮", need_form_update=True)
|
||||
)
|
||||
second = await backend.complete(
|
||||
ChatInput("session-1", "第二轮", need_form_update=True)
|
||||
)
|
||||
other_session = await backend.complete(
|
||||
ChatInput("session-2", "独立会话", need_form_update=True)
|
||||
)
|
||||
|
||||
assert first.content == "<state>1002</state>第1轮"
|
||||
assert first.form_update == {"turn": 1}
|
||||
assert second.content == "<state>1002</state>第2轮"
|
||||
assert second.form_update == {"turn": 2}
|
||||
assert other_session.content == "<state>1002</state>第1轮"
|
||||
assert generator.states[0].get("turn_count", 0) == 0
|
||||
assert generator.states[1]["turn_count"] == 1
|
||||
assert generator.states[2].get("turn_count", 0) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_langgraph_stream_bridges_form_update_before_text():
|
||||
generator = FakeResponseGenerator()
|
||||
backend = create_chat_backend(
|
||||
langgraph_settings(),
|
||||
response_generator=generator,
|
||||
)
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in backend.stream(
|
||||
ChatInput("session-stream", "开始", need_form_update=True)
|
||||
)
|
||||
]
|
||||
|
||||
assert events == [
|
||||
FormUpdate({"turn": 1}),
|
||||
TextDelta("<state>1002</state>第1轮"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_langgraph_backend_closes_owned_generator():
|
||||
generator = FakeResponseGenerator()
|
||||
backend = create_chat_backend(
|
||||
langgraph_settings(),
|
||||
response_generator=generator,
|
||||
)
|
||||
|
||||
await backend.aclose()
|
||||
|
||||
assert generator.closed
|
||||
|
||||
|
||||
def test_factory_keeps_fastgpt_as_default_compatible_backend():
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
environment="test",
|
||||
agent_backend="fastgpt",
|
||||
fastgpt_api_key="test-key",
|
||||
fastgpt_base_url="http://fastgpt.test",
|
||||
fastgpt_app_id="test-app",
|
||||
)
|
||||
fake_client = object()
|
||||
|
||||
backend = create_chat_backend(
|
||||
settings,
|
||||
fastgpt_client=fake_client,
|
||||
)
|
||||
|
||||
assert isinstance(backend, FastGPTBackend)
|
||||
@@ -1,200 +1,72 @@
|
||||
GET http://127.0.0.1:8000
|
||||
@baseUrl = http://101.89.108.122:8000
|
||||
@sessionId = a1103
|
||||
@timeStamp = 202603310303
|
||||
###
|
||||
|
||||
GET {{baseUrl}}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:07:21 GMT
|
||||
date: Thu, 18 Jun 2026 02:37:30 GMT
|
||||
server: uvicorn
|
||||
content-length: 32
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/chat
|
||||
POST {{baseUrl}}/chat?stream=true
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"text": "可以,继续"
|
||||
"sessionId": "{{sessionId}}",
|
||||
"timeStamp": "{{timeStamp}}",
|
||||
"text": "两车追尾",
|
||||
"needFormUpdate": true,
|
||||
"useTextChunk": true
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:18:14 GMT
|
||||
date: Thu, 18 Jun 2026 02:40:12 GMT
|
||||
server: uvicorn
|
||||
content-length: 169
|
||||
content-type: application/json
|
||||
content-type: text/event-stream; charset=utf-8
|
||||
connection: close
|
||||
transfer-encoding: chunked
|
||||
###
|
||||
POST http://127.0.0.1:8000/chat
|
||||
POST {{baseUrl}}/get_info
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"text": "没有"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:18:24 GMT
|
||||
server: uvicorn
|
||||
content-length: 187
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/chat
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"text": "没有"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:18:29 GMT
|
||||
server: uvicorn
|
||||
content-length: 178
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/chat
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"text": "是的"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:18:38 GMT
|
||||
server: uvicorn
|
||||
content-length: 196
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/chat
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"text": "十点半"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:19:07 GMT
|
||||
server: uvicorn
|
||||
content-length: 247
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/chat
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"text": "我在现场"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:19:27 GMT
|
||||
server: uvicorn
|
||||
content-length: 208
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/chat
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"text": "我闯红灯了"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:19:40 GMT
|
||||
server: uvicorn
|
||||
content-length: 231
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/chat
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"text": "【拍摄完成】"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 03:20:10 GMT
|
||||
server: uvicorn
|
||||
content-length: 210
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/get_info
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"key": "acdinfo"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 05:19:05 GMT
|
||||
server: uvicorn
|
||||
content-length: 271
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/set_info
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"key": "hphm1",
|
||||
"value": "沪A8938"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 05:30:03 GMT
|
||||
server: uvicorn
|
||||
content-length: 70
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
POST http://127.0.0.1:8000/get_info
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303",
|
||||
"sessionId": "{{sessionId}}",
|
||||
"timeStamp": "{{timeStamp}}",
|
||||
"key": "hphm1"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 05:30:27 GMT
|
||||
date: Thu, 18 Jun 2026 02:41:06 GMT
|
||||
server: uvicorn
|
||||
content-length: 97
|
||||
content-type: application/json
|
||||
connection: close
|
||||
###
|
||||
DELETE http://127.0.0.1:8000/delete_session
|
||||
POST {{baseUrl}}/set_info
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "a0009",
|
||||
"timeStamp": "202503310303"
|
||||
"sessionId": "{{sessionId}}",
|
||||
"timeStamp": "{{timeStamp}}",
|
||||
"key": "hphm1",
|
||||
"value": "沪A8939"
|
||||
}
|
||||
###
|
||||
DELETE {{baseUrl}}/delete_session
|
||||
content-type: application/json
|
||||
|
||||
{
|
||||
"sessionId": "{{sessionId}}",
|
||||
"timeStamp": "{{$timestamp}}"
|
||||
}
|
||||
|
||||
HTTP/1.1 200 - OK
|
||||
date: Fri, 20 Jun 2025 05:56:25 GMT
|
||||
date: Thu, 18 Jun 2026 02:37:48 GMT
|
||||
server: uvicorn
|
||||
content-length: 70
|
||||
content-length: 71
|
||||
content-type: application/json
|
||||
connection: close
|
||||
@@ -1,8 +1,11 @@
|
||||
GET http://101.89.151.141:3000
|
||||
@fastgptBaseUrl = http://127.0.0.1:3000
|
||||
@fastgptApiKey = replace-with-local-api-key
|
||||
|
||||
GET {{fastgptBaseUrl}}
|
||||
###
|
||||
POST http://101.89.151.141:3000/api/v1/chat/completions
|
||||
POST {{fastgptBaseUrl}}/api/v1/chat/completions
|
||||
content-type: application/json
|
||||
Authorization: Bearer fastgpt-xCH4CaEoNEyVtq7fkBEI5UP3O6sABKdpGszTtSYk4R2TVW5VgrPp1YPfuLX1iH
|
||||
Authorization: Bearer {{fastgptApiKey}}
|
||||
|
||||
{
|
||||
|
||||
@@ -23,4 +26,4 @@ content-type: application/json; charset=utf-8
|
||||
etag: "s14v22uu1g5f"
|
||||
content-length: 219
|
||||
date: Fri, 20 Jun 2025 02:37:16 GMT
|
||||
connection: close
|
||||
connection: close
|
||||
|
||||
70
test/api/test_chat_backend_boundary.py
Normal file
70
test/api/test_chat_backend_boundary.py
Normal file
@@ -0,0 +1,70 @@
|
||||
import pytest
|
||||
|
||||
from src.api.endpoints import chat
|
||||
from src.backends.chat import ChatInput, ChatResult, FormUpdate, TextDelta
|
||||
from src.schemas.models import ProcessRequest_chat
|
||||
|
||||
|
||||
class FakeBackend:
|
||||
def __init__(self):
|
||||
self.received = []
|
||||
|
||||
async def stream(self, chat_input: ChatInput):
|
||||
self.received.append(chat_input)
|
||||
yield TextDelta("<sta")
|
||||
yield TextDelta("te>1002</state>你")
|
||||
yield FormUpdate({"jdcsl": 2})
|
||||
yield TextDelta("好")
|
||||
|
||||
async def complete(self, chat_input: ChatInput):
|
||||
self.received.append(chat_input)
|
||||
return ChatResult(
|
||||
content="<state>1002</state>你好",
|
||||
status_code="1002",
|
||||
form_update={"jdcsl": 2},
|
||||
)
|
||||
|
||||
|
||||
def make_request():
|
||||
return ProcessRequest_chat(
|
||||
sessionId="session-001",
|
||||
timeStamp="20260725120000",
|
||||
text="发生了交通事故",
|
||||
needFormUpdate=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_chat_uses_backend_neutral_result():
|
||||
backend = FakeBackend()
|
||||
|
||||
response = await chat(make_request(), stream=False, backend=backend)
|
||||
|
||||
assert response.outputText == "你好"
|
||||
assert response.nextStageCode == "1002"
|
||||
assert response.formUpdate == {"jdcsl": 2}
|
||||
assert backend.received == [
|
||||
ChatInput(
|
||||
session_id="session-001",
|
||||
text="发生了交通事故",
|
||||
need_form_update=True,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_chat_keeps_state_prefix_buffering_in_fastapi_layer():
|
||||
backend = FakeBackend()
|
||||
|
||||
response = await chat(make_request(), stream=True, backend=backend)
|
||||
chunks = []
|
||||
async for chunk in response.body_iterator:
|
||||
chunks.append(chunk.decode() if isinstance(chunk, bytes) else chunk)
|
||||
body = "".join(chunks)
|
||||
|
||||
assert body.index("event: stage_code") < body.index("event: text_delta")
|
||||
assert '"nextStageCode": "1002"' in body
|
||||
assert '"text": "你"' in body
|
||||
assert '"text": "好"' in body
|
||||
assert "event: formUpdate" in body
|
||||
assert "event: done" in body
|
||||
116
test/api/test_chat_sse_contract.py
Normal file
116
test/api/test_chat_sse_contract.py
Normal file
@@ -0,0 +1,116 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from src.api.endpoints import chat
|
||||
from src.backends.chat import ChatInput, ChatResult, FormUpdate, TextDelta
|
||||
from src.schemas.models import ProcessRequest_chat
|
||||
|
||||
|
||||
def make_request(**overrides):
|
||||
payload = {
|
||||
"sessionId": "session-001",
|
||||
"timeStamp": "20260726120000",
|
||||
"text": "发生了交通事故",
|
||||
"needFormUpdate": True,
|
||||
}
|
||||
payload.update(overrides)
|
||||
return ProcessRequest_chat(**payload)
|
||||
|
||||
|
||||
async def response_text(response):
|
||||
chunks = []
|
||||
async for chunk in response.body_iterator:
|
||||
chunks.append(chunk.decode() if isinstance(chunk, bytes) else chunk)
|
||||
return "".join(chunks)
|
||||
|
||||
|
||||
def parse_sse(body):
|
||||
events = []
|
||||
for block in body.strip().split("\n\n"):
|
||||
lines = block.splitlines()
|
||||
event = lines[0].removeprefix("event: ")
|
||||
data = json.loads(lines[1].removeprefix("data: "))
|
||||
events.append((event, data))
|
||||
return events
|
||||
|
||||
|
||||
class OrderedBackend:
|
||||
async def stream(self, chat_input: ChatInput):
|
||||
yield TextDelta("<sta")
|
||||
yield TextDelta("te>1002</state>")
|
||||
yield FormUpdate({"jdcsl": 2})
|
||||
yield TextDelta("第一句。")
|
||||
yield TextDelta("第二句。")
|
||||
|
||||
async def complete(self, chat_input: ChatInput):
|
||||
return ChatResult("<state>1002</state>第一句。第二句。", "1002", {"jdcsl": 2})
|
||||
|
||||
|
||||
class MissingPrefixBackend:
|
||||
async def stream(self, chat_input: ChatInput):
|
||||
yield TextDelta("没有状态前缀")
|
||||
|
||||
async def complete(self, chat_input: ChatInput):
|
||||
return ChatResult("没有状态前缀")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sse_success_event_order_and_cardinality():
|
||||
response = await chat(make_request(), stream=True, backend=OrderedBackend())
|
||||
events = parse_sse(await response_text(response))
|
||||
names = [name for name, _ in events]
|
||||
|
||||
assert names == [
|
||||
"stage_code",
|
||||
"formUpdate",
|
||||
"text_delta",
|
||||
"text_delta",
|
||||
"done",
|
||||
]
|
||||
assert names.count("stage_code") == 1
|
||||
assert names.count("formUpdate") == 1
|
||||
assert names.count("done") == 1
|
||||
assert "error" not in names
|
||||
assert "".join(data["text"] for name, data in events if name == "text_delta") == (
|
||||
"第一句。第二句。"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_use_text_chunk_only_changes_delta_boundaries():
|
||||
request = make_request(useTextChunk=True)
|
||||
response = await chat(request, stream=True, backend=OrderedBackend())
|
||||
events = parse_sse(await response_text(response))
|
||||
|
||||
assert "".join(data["text"] for name, data in events if name == "text_delta") == (
|
||||
"第一句。第二句。"
|
||||
)
|
||||
assert [name for name, _ in events].count("done") == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_stream_prefix_characterizes_current_legacy_behavior():
|
||||
response = await chat(
|
||||
make_request(needFormUpdate=False),
|
||||
stream=True,
|
||||
backend=MissingPrefixBackend(),
|
||||
)
|
||||
events = parse_sse(await response_text(response))
|
||||
|
||||
assert [name for name, _ in events] == ["text_delta", "done"]
|
||||
assert events[0][1]["text"] == "没有状态前缀"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_non_stream_prefix_returns_compatible_business_error():
|
||||
response = await chat(
|
||||
make_request(needFormUpdate=False),
|
||||
stream=False,
|
||||
backend=MissingPrefixBackend(),
|
||||
)
|
||||
|
||||
assert response.code == "500"
|
||||
assert response.outputText == ""
|
||||
assert response.nextStageCode == ""
|
||||
assert response.msg == "大模型服务返回消息不完整"
|
||||
145
test/api/test_info_contract.py
Normal file
145
test/api/test_info_contract.py
Normal file
@@ -0,0 +1,145 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from src.api import endpoints
|
||||
from src.api.endpoints import get_info, set_info
|
||||
from src.schemas.models import ProcessRequest_get, ProcessRequest_set
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class FakeInfoClient:
|
||||
def __init__(self, state):
|
||||
self.state = state
|
||||
self.completion_calls = []
|
||||
|
||||
async def create_chat_completion(self, **kwargs):
|
||||
self.completion_calls.append(kwargs)
|
||||
if "variables" in kwargs:
|
||||
self.state = kwargs["variables"]["state"]
|
||||
return FakeResponse({"newVariables": {"state": self.state}})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def skip_helper_record_deletion(monkeypatch):
|
||||
calls = []
|
||||
|
||||
async def fake_delete(client, session_id):
|
||||
calls.append(session_id)
|
||||
|
||||
monkeypatch.setattr(endpoints, "delete_last_two_chat_records", fake_delete)
|
||||
return calls
|
||||
|
||||
|
||||
def make_set_request(**overrides):
|
||||
payload = {
|
||||
"sessionId": "session-001",
|
||||
"timeStamp": "20260726120000",
|
||||
"key": "hphm1",
|
||||
"value": "<PLATE_1>",
|
||||
}
|
||||
payload.update(overrides)
|
||||
return ProcessRequest_set(**payload)
|
||||
|
||||
|
||||
def make_get_request(**overrides):
|
||||
payload = {
|
||||
"sessionId": "session-001",
|
||||
"timeStamp": "20260726120000",
|
||||
"key": "all",
|
||||
}
|
||||
payload.update(overrides)
|
||||
return ProcessRequest_get(**payload)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_info_reads_then_writes_fastgpt_state(
|
||||
skip_helper_record_deletion,
|
||||
):
|
||||
client = FakeInfoClient({"hphm1": "<PLATE_OLD>", "jdcsl": 1})
|
||||
|
||||
response = await set_info(make_set_request(), client=client)
|
||||
|
||||
assert response.code == "200"
|
||||
assert client.state == {"hphm1": "<PLATE_1>", "jdcsl": 1}
|
||||
assert len(client.completion_calls) == 2
|
||||
assert client.completion_calls[1]["variables"]["state"] == client.state
|
||||
assert skip_helper_record_deletion == ["session-001", "session-001"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_info_include_input_info_keeps_legacy_magic_payload(
|
||||
skip_helper_record_deletion,
|
||||
):
|
||||
client = FakeInfoClient({})
|
||||
|
||||
await set_info(make_set_request(includeInputInfo=True), client=client)
|
||||
|
||||
message = client.completion_calls[0]["messages"][0]["content"]
|
||||
assert message == '<setInfo>{"key": "hphm1", "value": "<PLATE_1>"}</setInfo>'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_info_all_keeps_json_string_and_boolean_encoding(
|
||||
skip_helper_record_deletion,
|
||||
):
|
||||
client = FakeInfoClient(
|
||||
{
|
||||
"ywrysw": False,
|
||||
"ywfjdc": True,
|
||||
"jdcsl": 2,
|
||||
"xm1": "<PERSON_1>",
|
||||
"hphm1": "<PLATE_1>",
|
||||
"xm2": "<PERSON_2>",
|
||||
}
|
||||
)
|
||||
|
||||
response = await get_info(make_get_request(), client=client)
|
||||
value = json.loads(response.value)
|
||||
|
||||
assert response.code == "200"
|
||||
assert isinstance(response.value, str)
|
||||
assert value["acdinfo"]["ywrysw"] == "0"
|
||||
assert value["acdinfo"]["ywfjdc"] == "1"
|
||||
assert value["acdinfo"]["jdcsl"] == 2
|
||||
assert value["acdhuman1"]["xm1"] == "<PERSON_1>"
|
||||
assert value["acdhuman2"]["xm2"] == "<PERSON_2>"
|
||||
assert skip_helper_record_deletion == ["session-001"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_info_unknown_key_returns_json_encoded_empty_string(
|
||||
skip_helper_record_deletion,
|
||||
):
|
||||
client = FakeInfoClient({})
|
||||
|
||||
response = await get_info(
|
||||
make_get_request(key="unknown_legacy_key"),
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert response.code == "200"
|
||||
assert response.value == '""'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_info_fastgpt_shape_error_keeps_legacy_business_error():
|
||||
class InvalidClient:
|
||||
async def create_chat_completion(self, **kwargs):
|
||||
return FakeResponse({})
|
||||
|
||||
response = await get_info(make_get_request(), client=InvalidClient())
|
||||
|
||||
assert response.code == "500"
|
||||
assert response.value == ""
|
||||
assert response.msg == "大模型服务器无响应"
|
||||
195
test/api/test_public_schema_contract.py
Normal file
195
test/api/test_public_schema_contract.py
Normal file
@@ -0,0 +1,195 @@
|
||||
"""Characterization tests for the public HTTP API schemas.
|
||||
|
||||
These tests freeze the current contract before the FastGPT backend is replaced.
|
||||
They should change only when the teams integrating with this service agree to a
|
||||
contract change.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from src.schemas.models import (
|
||||
ProcessRequest_chat,
|
||||
ProcessRequest_get,
|
||||
ProcessRequest_set,
|
||||
ProcessResponse_chat,
|
||||
ProcessResponse_get,
|
||||
ProcessResponse_set,
|
||||
)
|
||||
|
||||
|
||||
SESSION_ID = "session-001"
|
||||
TIMESTAMP = "20260725120000"
|
||||
|
||||
|
||||
def chat_request_payload() -> dict[str, Any]:
|
||||
return {
|
||||
"sessionId": SESSION_ID,
|
||||
"timeStamp": TIMESTAMP,
|
||||
"text": "发生了交通事故",
|
||||
}
|
||||
|
||||
|
||||
def chat_response_payload() -> dict[str, Any]:
|
||||
return {
|
||||
"sessionId": SESSION_ID,
|
||||
"timeStamp": TIMESTAMP,
|
||||
"outputText": "请描述事故经过。",
|
||||
"nextStage": "通话中",
|
||||
"nextStageCode": "1002",
|
||||
"code": "200",
|
||||
}
|
||||
|
||||
|
||||
def get_request_payload() -> dict[str, Any]:
|
||||
return {
|
||||
"sessionId": SESSION_ID,
|
||||
"timeStamp": TIMESTAMP,
|
||||
"key": "acdinfo",
|
||||
}
|
||||
|
||||
|
||||
def get_response_payload() -> dict[str, Any]:
|
||||
return {
|
||||
"sessionId": SESSION_ID,
|
||||
"timeStamp": TIMESTAMP,
|
||||
"value": '{"jdcsl": "2"}',
|
||||
"code": "200",
|
||||
}
|
||||
|
||||
|
||||
def set_request_payload() -> dict[str, Any]:
|
||||
return {
|
||||
"sessionId": SESSION_ID,
|
||||
"timeStamp": TIMESTAMP,
|
||||
"key": "hphm1",
|
||||
"value": "沪A12345",
|
||||
}
|
||||
|
||||
|
||||
def set_response_payload() -> dict[str, Any]:
|
||||
return {
|
||||
"sessionId": SESSION_ID,
|
||||
"timeStamp": TIMESTAMP,
|
||||
"code": "200",
|
||||
}
|
||||
|
||||
|
||||
def test_chat_request_defaults_are_backward_compatible() -> None:
|
||||
request = ProcessRequest_chat(**chat_request_payload())
|
||||
|
||||
assert request.model_dump() == {
|
||||
**chat_request_payload(),
|
||||
"needFormUpdate": False,
|
||||
"useTextChunk": False,
|
||||
}
|
||||
|
||||
|
||||
def test_chat_response_shape_is_backward_compatible() -> None:
|
||||
response = ProcessResponse_chat(**chat_response_payload())
|
||||
|
||||
assert response.model_dump() == {
|
||||
**chat_response_payload(),
|
||||
"formUpdate": {},
|
||||
"msg": None,
|
||||
}
|
||||
|
||||
|
||||
def test_get_info_shapes_are_backward_compatible() -> None:
|
||||
request = ProcessRequest_get(**get_request_payload())
|
||||
response = ProcessResponse_get(**get_response_payload())
|
||||
|
||||
assert request.model_dump() == {
|
||||
**get_request_payload(),
|
||||
"includeInputInfo": False,
|
||||
}
|
||||
assert response.model_dump() == {
|
||||
**get_response_payload(),
|
||||
"msg": None,
|
||||
}
|
||||
|
||||
|
||||
def test_set_info_shapes_are_backward_compatible() -> None:
|
||||
request = ProcessRequest_set(**set_request_payload())
|
||||
response = ProcessResponse_set(**set_response_payload())
|
||||
|
||||
assert request.model_dump() == {
|
||||
**set_request_payload(),
|
||||
"includeInputInfo": False,
|
||||
}
|
||||
assert response.model_dump() == {
|
||||
**set_response_payload(),
|
||||
"msg": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "payload_factory", "required_field"),
|
||||
[
|
||||
(ProcessRequest_chat, chat_request_payload, "sessionId"),
|
||||
(ProcessRequest_chat, chat_request_payload, "timeStamp"),
|
||||
(ProcessRequest_chat, chat_request_payload, "text"),
|
||||
(ProcessRequest_get, get_request_payload, "key"),
|
||||
(ProcessRequest_set, set_request_payload, "key"),
|
||||
(ProcessRequest_set, set_request_payload, "value"),
|
||||
(ProcessResponse_chat, chat_response_payload, "nextStageCode"),
|
||||
(ProcessResponse_get, get_response_payload, "value"),
|
||||
],
|
||||
)
|
||||
def test_required_fields_remain_required(
|
||||
model: type[BaseModel],
|
||||
payload_factory: Callable[[], dict[str, Any]],
|
||||
required_field: str,
|
||||
) -> None:
|
||||
payload = payload_factory()
|
||||
payload.pop(required_field)
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
model(**payload)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "payload_factory", "field_name", "invalid_value"),
|
||||
[
|
||||
(ProcessRequest_chat, chat_request_payload, "sessionId", "s" * 65),
|
||||
(ProcessRequest_chat, chat_request_payload, "timeStamp", "t" * 33),
|
||||
(ProcessRequest_get, get_request_payload, "sessionId", "s" * 65),
|
||||
(ProcessRequest_set, set_request_payload, "timeStamp", "t" * 33),
|
||||
(ProcessResponse_chat, chat_response_payload, "nextStage", "n" * 33),
|
||||
(ProcessResponse_chat, chat_response_payload, "nextStageCode", "10020"),
|
||||
(ProcessResponse_chat, chat_response_payload, "code", "10000"),
|
||||
(ProcessResponse_get, get_response_payload, "code", "10000"),
|
||||
(ProcessResponse_set, set_response_payload, "code", "10000"),
|
||||
],
|
||||
)
|
||||
def test_public_length_limits_are_enforced(
|
||||
model: type[BaseModel],
|
||||
payload_factory: Callable[[], dict[str, Any]],
|
||||
field_name: str,
|
||||
invalid_value: str,
|
||||
) -> None:
|
||||
payload = payload_factory()
|
||||
payload[field_name] = invalid_value
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
model(**payload)
|
||||
|
||||
|
||||
def test_get_info_value_remains_a_json_encoded_string() -> None:
|
||||
response = ProcessResponse_get(**get_response_payload())
|
||||
|
||||
assert isinstance(response.value, str)
|
||||
assert response.value == '{"jdcsl": "2"}'
|
||||
|
||||
|
||||
def test_chat_form_update_remains_unstructured_for_compatibility() -> None:
|
||||
form_update = {"jdcsl": 2, "ywrysw": False}
|
||||
response = ProcessResponse_chat(
|
||||
**chat_response_payload(),
|
||||
formUpdate=form_update,
|
||||
)
|
||||
|
||||
assert response.formUpdate == form_update
|
||||
123
test/backends/test_fastgpt_backend.py
Normal file
123
test/backends/test_fastgpt_backend.py
Normal file
@@ -0,0 +1,123 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from src.backends.chat import ChatInput, FormUpdate, TextDelta
|
||||
from src.backends.fastgpt import FastGPTBackend
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, data):
|
||||
self._data = data
|
||||
self.raise_for_status_called = False
|
||||
|
||||
def raise_for_status(self):
|
||||
self.raise_for_status_called = True
|
||||
|
||||
def json(self):
|
||||
return self._data
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, response):
|
||||
self.response = response
|
||||
self.calls = []
|
||||
|
||||
async def create_chat_completion(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
return self.response
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_translates_fastgpt_response_to_neutral_result():
|
||||
response = FakeResponse(
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "<state>1002</state>请描述事故经过。"
|
||||
}
|
||||
}
|
||||
],
|
||||
"newVariables": {"status_code": "1002"},
|
||||
"responseData": [
|
||||
{
|
||||
"moduleName": "文本内容提取事故信息",
|
||||
"extractResult": {
|
||||
"formUpdate": '{"jdcsl": 2, "ywrysw": false}'
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
client = FakeClient(response)
|
||||
backend = FastGPTBackend(client)
|
||||
|
||||
result = await backend.complete(
|
||||
ChatInput(
|
||||
session_id="session-001",
|
||||
text="发生了交通事故",
|
||||
need_form_update=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert response.raise_for_status_called is True
|
||||
assert result.content == "<state>1002</state>请描述事故经过。"
|
||||
assert result.status_code == "1002"
|
||||
assert result.form_update == {"jdcsl": 2, "ywrysw": False}
|
||||
assert client.calls == [
|
||||
{
|
||||
"messages": [{"role": "user", "content": "发生了交通事故"}],
|
||||
"chatId": "session-001",
|
||||
"stream": False,
|
||||
"detail": True,
|
||||
"variables": {"needFormUpdate": True},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_translates_fastgpt_events_to_neutral_events(monkeypatch):
|
||||
response = object()
|
||||
client = FakeClient(response)
|
||||
backend = FastGPTBackend(client)
|
||||
|
||||
async def fake_aiter_stream_events(actual_response):
|
||||
assert actual_response is response
|
||||
yield SimpleNamespace(
|
||||
kind="answer",
|
||||
data={"choices": [{"delta": {"content": "<state>1002"}}]},
|
||||
)
|
||||
yield SimpleNamespace(
|
||||
kind="flowResponses",
|
||||
data=[
|
||||
{
|
||||
"moduleName": "文本内容提取事故信息",
|
||||
"extractResult": {"formUpdate": '{"jdcsl": 2}'},
|
||||
}
|
||||
],
|
||||
)
|
||||
yield SimpleNamespace(
|
||||
kind="answer",
|
||||
data={"choices": [{"delta": {"content": "</state>你好"}}]},
|
||||
)
|
||||
yield SimpleNamespace(kind="ignored", data={})
|
||||
|
||||
monkeypatch.setattr(
|
||||
"src.backends.fastgpt.aiter_stream_events",
|
||||
fake_aiter_stream_events,
|
||||
)
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in backend.stream(
|
||||
ChatInput(session_id="session-001", text="你好")
|
||||
)
|
||||
]
|
||||
|
||||
assert events == [
|
||||
TextDelta("<state>1002"),
|
||||
FormUpdate({"jdcsl": 2}),
|
||||
TextDelta("</state>你好"),
|
||||
]
|
||||
assert client.calls[0]["stream"] is True
|
||||
78
test/core/test_config.py
Normal file
78
test/core/test_config.py
Normal file
@@ -0,0 +1,78 @@
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from src.core.config import Settings
|
||||
|
||||
|
||||
def test_fastgpt_backend_accepts_legacy_environment_names():
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
ZNJJ_ENVIRONMENT="test",
|
||||
AGENT_BACKEND="fastgpt",
|
||||
ANALYSIS_AUTH_TOKEN="test-fastgpt-key",
|
||||
ANALYSIS_SERVICE_URL="http://fastgpt.test",
|
||||
APP_ID="test-app",
|
||||
)
|
||||
|
||||
assert settings.environment == "test"
|
||||
assert settings.agent_backend == "fastgpt"
|
||||
assert settings.has_fastgpt_config
|
||||
assert settings.fastgpt_api_key.get_secret_value() == "test-fastgpt-key"
|
||||
|
||||
|
||||
def test_fastgpt_backend_rejects_incomplete_configuration():
|
||||
with pytest.raises(ValidationError, match="FastGPT backend requires"):
|
||||
Settings(
|
||||
_env_file=None,
|
||||
environment="test",
|
||||
agent_backend="fastgpt",
|
||||
fastgpt_api_key="test-key",
|
||||
)
|
||||
|
||||
|
||||
def test_langgraph_backend_requires_only_minimal_llm_configuration():
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
environment="test",
|
||||
agent_backend="langgraph",
|
||||
langgraph_checkpointer="memory",
|
||||
llm_api_key="test-llm-key",
|
||||
llm_model="test-model",
|
||||
)
|
||||
|
||||
assert settings.agent_backend == "langgraph"
|
||||
assert settings.langgraph_checkpointer == "memory"
|
||||
assert not settings.has_fastgpt_config
|
||||
|
||||
|
||||
def test_langgraph_backend_rejects_missing_llm_configuration():
|
||||
with pytest.raises(ValidationError, match="LLM_API_KEY, LLM_MODEL"):
|
||||
Settings(
|
||||
_env_file=None,
|
||||
environment="test",
|
||||
agent_backend="langgraph",
|
||||
)
|
||||
|
||||
|
||||
def test_production_cannot_use_memory_checkpointer():
|
||||
with pytest.raises(ValidationError, match="cannot use the memory checkpointer"):
|
||||
Settings(
|
||||
_env_file=None,
|
||||
environment="production",
|
||||
agent_backend="langgraph",
|
||||
langgraph_checkpointer="memory",
|
||||
llm_api_key="test-llm-key",
|
||||
llm_model="test-model",
|
||||
)
|
||||
|
||||
|
||||
def test_secret_values_are_masked_in_settings_repr():
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
environment="test",
|
||||
agent_backend="langgraph",
|
||||
llm_api_key="never-print-this-key",
|
||||
llm_model="test-model",
|
||||
)
|
||||
|
||||
assert "never-print-this-key" not in repr(settings)
|
||||
162
test/fixtures/golden/accident-scenarios.json
vendored
Normal file
162
test/fixtures/golden/accident-scenarios.json
vendored
Normal file
@@ -0,0 +1,162 @@
|
||||
{
|
||||
"version": "2026-07-26",
|
||||
"description": "LangGraph 迁移的脱敏黄金场景。占位符不是可用个人信息。",
|
||||
"scenarios": [
|
||||
{
|
||||
"case_id": "single_vehicle_happy_path",
|
||||
"category": "end_to_end",
|
||||
"initial_stage": "1001",
|
||||
"turns": [
|
||||
{"event": "session_started", "input": "【继续办理】", "expected_stage": "1002"},
|
||||
{"event": "user_message", "input": "车辆倒车时碰到了固定物体", "expected_stage": "1002", "expected_patch": {"sgyy": "倒车碰到固定物体"}},
|
||||
{"event": "user_message", "input": "没有人员受伤", "expected_stage": "1002", "expected_patch": {"ywrysw": false}},
|
||||
{"event": "user_message", "input": "没有非机动车", "expected_stage": "1002", "expected_patch": {"ywfjdc": false}},
|
||||
{"event": "user_message", "input": "事故时间是十分钟前", "expected_stage": "1002"},
|
||||
{"event": "user_message", "input": "我还在现场", "expected_stage": "1002", "expected_patch": {"sfsgxc": true}},
|
||||
{"event": "user_message", "input": "只有一辆机动车", "expected_stage": "2000", "expected_patch": {"jdcsl": 1}},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2001"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2002"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2003"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2004"},
|
||||
{"event": "user_message", "input": "车牌正确", "expected_stage": "2005"},
|
||||
{"event": "user_message", "input": "车损在车辆前方", "expected_stage": "3001", "expected_patch": {"csbw1": "前方"}}
|
||||
],
|
||||
"expected_handoff_reason": null
|
||||
},
|
||||
{
|
||||
"case_id": "double_vehicle_happy_path",
|
||||
"category": "end_to_end",
|
||||
"initial_stage": "1002",
|
||||
"turns": [
|
||||
{"event": "user_message", "input": "两辆机动车发生追尾,没有人受伤,也没有非机动车", "expected_stage": "1002", "expected_patch": {"jdcsl": 2, "ywrysw": false, "ywfjdc": false, "sgyy": "追尾"}},
|
||||
{"event": "user_message", "input": "时间正确,我还在现场", "expected_stage": "2010", "expected_patch": {"sfsgxc": true}},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2011"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2012"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2013"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2014"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2015"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2016"},
|
||||
{"event": "user_message", "input": "车牌正确", "expected_stage": "3002"}
|
||||
],
|
||||
"expected_handoff_reason": null
|
||||
},
|
||||
{
|
||||
"case_id": "explicit_handoff_global",
|
||||
"category": "handoff",
|
||||
"initial_stage": "2002",
|
||||
"turns": [
|
||||
{"event": "user_message", "input": "请帮我转人工", "expected_stage": "0001"}
|
||||
],
|
||||
"expected_handoff_reason": "user_requested"
|
||||
},
|
||||
{
|
||||
"case_id": "injury_handoff",
|
||||
"category": "safety",
|
||||
"initial_stage": "1002",
|
||||
"turns": [
|
||||
{"event": "user_message", "input": "有人倒地并且不舒服", "expected_stage": "0003", "expected_patch": {"ywrysw": true}}
|
||||
],
|
||||
"expected_handoff_reason": "injury_or_complex"
|
||||
},
|
||||
{
|
||||
"case_id": "injury_negation_does_not_handoff",
|
||||
"category": "safety",
|
||||
"initial_stage": "1002",
|
||||
"turns": [
|
||||
{"event": "user_message", "input": "不是人受伤,是车受损,人没事", "expected_stage": "1002", "expected_patch": {"ywrysw": false}}
|
||||
],
|
||||
"expected_handoff_reason": null
|
||||
},
|
||||
{
|
||||
"case_id": "three_vehicle_complex_handoff",
|
||||
"category": "safety",
|
||||
"initial_stage": "1002",
|
||||
"turns": [
|
||||
{"event": "user_message", "input": "一共涉及三辆机动车", "expected_stage": "0003", "expected_patch": {"jdcsl": 3}}
|
||||
],
|
||||
"expected_handoff_reason": "complex_accident"
|
||||
},
|
||||
{
|
||||
"case_id": "photo_failure_single",
|
||||
"category": "deterministic_event",
|
||||
"initial_stage": "2001",
|
||||
"turns": [
|
||||
{"event": "photo_recognition_failed", "input": "【客户端连续3次拍摄识别失败:图片过于模糊】", "expected_stage": "0005"}
|
||||
],
|
||||
"expected_handoff_reason": "photo_recognition_failed"
|
||||
},
|
||||
{
|
||||
"case_id": "photo_failure_double_confirmation",
|
||||
"category": "deterministic_event",
|
||||
"initial_stage": "2016",
|
||||
"turns": [
|
||||
{"event": "photo_recognition_failed", "input": "【客户端连续3次拍摄识别失败:未识别到完整车牌】", "expected_stage": "0005"}
|
||||
],
|
||||
"expected_handoff_reason": "photo_recognition_failed"
|
||||
},
|
||||
{
|
||||
"case_id": "two_consecutive_no_responses",
|
||||
"category": "deterministic_event",
|
||||
"initial_stage": "1002",
|
||||
"turns": [
|
||||
{"event": "no_response", "input": "【用户无回复】", "expected_stage": "1002", "expected_no_response_count": 1},
|
||||
{"event": "no_response", "input": "【用户无回复】", "expected_stage": "0004", "expected_no_response_count": 2}
|
||||
],
|
||||
"expected_handoff_reason": "no_response"
|
||||
},
|
||||
{
|
||||
"case_id": "photo_step_cannot_skip",
|
||||
"category": "transition_guard",
|
||||
"initial_stage": "2011",
|
||||
"turns": [
|
||||
{"event": "user_message", "input": "后面的照片我都拍好了", "expected_stage": "2011"},
|
||||
{"event": "photo_completed", "input": "【拍摄完成】", "expected_stage": "2012"}
|
||||
],
|
||||
"expected_handoff_reason": null
|
||||
},
|
||||
{
|
||||
"case_id": "external_update_then_chat",
|
||||
"category": "state_consistency",
|
||||
"initial_stage": "1002",
|
||||
"initial_form": {"hphm1": "<PLATE_1>"},
|
||||
"turns": [
|
||||
{"event": "set_info", "input": {"key": "hphm1", "value": "<PLATE_CORRECTED>"}, "expected_stage": "1002", "expected_patch": {"hphm1": "<PLATE_CORRECTED>"}},
|
||||
{"event": "user_message", "input": "请继续办理", "expected_stage": "1002", "expected_state_contains": {"hphm1": "<PLATE_CORRECTED>"}}
|
||||
],
|
||||
"expected_handoff_reason": null
|
||||
},
|
||||
{
|
||||
"case_id": "invalid_external_internal_field",
|
||||
"category": "state_consistency",
|
||||
"initial_stage": "1002",
|
||||
"turns": [
|
||||
{"event": "set_info", "input": {"key": "stage_code", "value": "0000"}, "expected_error": "INVALID_FIELD", "expected_stage": "1002"}
|
||||
],
|
||||
"expected_handoff_reason": null
|
||||
},
|
||||
{
|
||||
"case_id": "prefix_split_across_chunks",
|
||||
"category": "prefix_parser",
|
||||
"initial_stage": "1002",
|
||||
"stream_chunks": ["<sta", "te>1002", "</sta", "te>请描述事故经过"],
|
||||
"expected_stage": "1002",
|
||||
"expected_text": "请描述事故经过"
|
||||
},
|
||||
{
|
||||
"case_id": "prefix_unknown_code",
|
||||
"category": "prefix_parser",
|
||||
"initial_stage": "1002",
|
||||
"model_output": "<state>9999</state>继续处理",
|
||||
"expected_error": "UNKNOWN_STAGE_CODE",
|
||||
"expected_stage": "1002"
|
||||
},
|
||||
{
|
||||
"case_id": "prefix_illegal_photo_jump",
|
||||
"category": "prefix_parser",
|
||||
"initial_stage": "2000",
|
||||
"model_output": "<state>2004</state>请确认车牌",
|
||||
"expected_error": "ILLEGAL_STAGE_TRANSITION",
|
||||
"expected_stage": "2000"
|
||||
}
|
||||
]
|
||||
}
|
||||
119
test/test_phase0_artifacts.py
Normal file
119
test/test_phase0_artifacts.py
Normal file
@@ -0,0 +1,119 @@
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DOMAIN = ROOT / "docs" / "domain"
|
||||
GOLDEN = ROOT / "test" / "fixtures" / "golden" / "accident-scenarios.json"
|
||||
|
||||
|
||||
def load_json(path):
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_stage_registry_and_transition_matrix_are_closed():
|
||||
stage_registry = load_json(DOMAIN / "stage-codes.json")
|
||||
transitions = load_json(DOMAIN / "stage-transitions.json")
|
||||
codes = {entry["code"] for entry in stage_registry["codes"]}
|
||||
|
||||
assert len(codes) == len(stage_registry["codes"])
|
||||
assert {"0000", "0004", "0005", "1001", "1002", "3001", "3002"} <= codes
|
||||
assert set(transitions["allowed"]) == codes
|
||||
|
||||
for source, targets in transitions["allowed"].items():
|
||||
assert set(targets) <= codes, source
|
||||
|
||||
for terminal in {"0000", "0001", "0002", "0003", "0004", "0005"}:
|
||||
assert transitions["allowed"][terminal] == []
|
||||
|
||||
|
||||
def test_photo_sequences_cannot_skip_steps():
|
||||
transitions = load_json(DOMAIN / "stage-transitions.json")
|
||||
|
||||
for sequence in transitions["photo_sequences"].values():
|
||||
for current, following in zip(sequence, sequence[1:]):
|
||||
assert following in transitions["allowed"][current]
|
||||
later_steps = set(sequence[sequence.index(following) + 1 :])
|
||||
assert later_steps.isdisjoint(transitions["allowed"][current])
|
||||
|
||||
|
||||
def test_field_registry_groups_are_complete_and_unique():
|
||||
registry = load_json(DOMAIN / "field-registry.json")
|
||||
fields = registry["fields"]
|
||||
keys = [field["key"] for field in fields]
|
||||
|
||||
assert len(keys) == len(set(keys))
|
||||
assert set(keys) == {
|
||||
key for group_keys in registry["groups"].values() for key in group_keys
|
||||
}
|
||||
assert {"sfzmhm1", "sjhm1", "hphm1", "sfzmhm2", "sjhm2", "hphm2"} <= {
|
||||
field["key"] for field in fields if field["sensitive"]
|
||||
}
|
||||
assert {"sfzmwh1", "sjwh1", "sfzmwh2", "sjwh2"} <= {
|
||||
field["key"] for field in fields if not field["external_write"]
|
||||
}
|
||||
|
||||
|
||||
def test_golden_scenarios_are_unique_and_use_known_stages():
|
||||
stages = {
|
||||
entry["code"]
|
||||
for entry in load_json(DOMAIN / "stage-codes.json")["codes"]
|
||||
}
|
||||
scenarios = load_json(GOLDEN)["scenarios"]
|
||||
case_ids = [case["case_id"] for case in scenarios]
|
||||
|
||||
assert len(scenarios) >= 12
|
||||
assert len(case_ids) == len(set(case_ids))
|
||||
for case in scenarios:
|
||||
assert case["initial_stage"] in stages
|
||||
if "expected_stage" in case:
|
||||
assert case["expected_stage"] in stages
|
||||
for turn in case.get("turns", []):
|
||||
if "expected_stage" in turn:
|
||||
assert turn["expected_stage"] in stages
|
||||
|
||||
|
||||
def test_golden_turn_sequences_follow_the_transition_matrix():
|
||||
allowed = load_json(DOMAIN / "stage-transitions.json")["allowed"]
|
||||
scenarios = load_json(GOLDEN)["scenarios"]
|
||||
|
||||
for case in scenarios:
|
||||
current = case["initial_stage"]
|
||||
for turn in case.get("turns", []):
|
||||
expected = turn.get("expected_stage")
|
||||
if expected is None:
|
||||
continue
|
||||
if "expected_error" not in turn:
|
||||
assert expected in allowed[current], (
|
||||
case["case_id"],
|
||||
current,
|
||||
expected,
|
||||
)
|
||||
current = expected
|
||||
|
||||
|
||||
def test_golden_fixture_contains_no_realistic_phone_or_national_id():
|
||||
raw = GOLDEN.read_text(encoding="utf-8")
|
||||
|
||||
assert not re.search(r"(?<!\d)1[3-9]\d{9}(?!\d)", raw)
|
||||
assert not re.search(r"(?<!\d)\d{17}[\dXx](?!\d)", raw)
|
||||
|
||||
|
||||
def test_repository_sources_and_examples_do_not_embed_fastgpt_tokens():
|
||||
token_pattern = re.compile(r"fastgpt-[A-Za-z0-9]{20,}")
|
||||
candidates = [ROOT / ".env.example"]
|
||||
|
||||
for directory in ("src", "test", "docs"):
|
||||
candidates.extend(
|
||||
path
|
||||
for path in (ROOT / directory).rglob("*")
|
||||
if path.is_file() and "__pycache__" not in path.parts
|
||||
)
|
||||
|
||||
for path in candidates:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
assert not token_pattern.search(text), path
|
||||
57
test/test_text_chunker.py
Normal file
57
test/test_text_chunker.py
Normal file
@@ -0,0 +1,57 @@
|
||||
from src.utils.text_chunker import SentenceTextChunker, SentenceTextChunkerConfig
|
||||
|
||||
|
||||
def test_chinese_sentence_chunks_wait_for_lookahead():
|
||||
chunker = SentenceTextChunker()
|
||||
chunks = []
|
||||
|
||||
for token in ["你好", "世界", "。", "下一", "句话", "。"]:
|
||||
chunks.extend(chunker.feed(token))
|
||||
|
||||
assert chunks == ["你好世界。"]
|
||||
assert chunker.flush() == "下一句话。"
|
||||
|
||||
|
||||
def test_flush_returns_pending_text():
|
||||
chunker = SentenceTextChunker()
|
||||
|
||||
assert chunker.feed("还没有句号") == []
|
||||
assert chunker.flush() == "还没有句号"
|
||||
assert chunker.flush() is None
|
||||
|
||||
|
||||
def test_decimal_point_does_not_split_sentence():
|
||||
chunker = SentenceTextChunker()
|
||||
|
||||
chunks = chunker.feed("价格是29.95元。下一句")
|
||||
|
||||
assert chunks == ["价格是29.95元。"]
|
||||
assert chunker.flush() == "下一句"
|
||||
|
||||
|
||||
def test_soft_break_after_max_chars():
|
||||
chunker = SentenceTextChunker(
|
||||
SentenceTextChunkerConfig(
|
||||
min_chars=1,
|
||||
max_chars=12,
|
||||
use_soft_breaks=True,
|
||||
)
|
||||
)
|
||||
|
||||
chunks = chunker.feed("这是一段比较长的话,需要先切一下继续播放")
|
||||
|
||||
assert chunks == ["这是一段比较长的话,"]
|
||||
assert chunker.flush() == "需要先切一下继续播放"
|
||||
|
||||
|
||||
def test_can_disable_soft_breaks():
|
||||
chunker = SentenceTextChunker(
|
||||
SentenceTextChunkerConfig(
|
||||
min_chars=1,
|
||||
max_chars=12,
|
||||
use_soft_breaks=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert chunker.feed("这是一段比较长的话,需要先切一下继续播放") == []
|
||||
assert chunker.flush() == "这是一段比较长的话,需要先切一下继续播放"
|
||||
4710
workflow/20251108/事故信息采集20251108.json
Normal file
4710
workflow/20251108/事故信息采集20251108.json
Normal file
File diff suppressed because one or more lines are too long
5185
workflow/20260726/事故信息采集20260726.json
Normal file
5185
workflow/20260726/事故信息采集20260726.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user