wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Inizialize simli_client on StartFrame; Follow variable naming scheme; Use logger instead of print statements;

Browse Source
This commit is contained in:
antonyesk601
2024-12-10 10:11:07 +00:00
parent bf40b4936b
commit 397342d0b9
2 changed files with 48 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
examples/foundational/25-simli-layer.py
View File

@@ -32,8 +32,7 @@ logger.add(sys.stderr, level="DEBUG")
async def main(): async def main():
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session) _, token = await configure(session)
print("Creating room") print("Creating room")
aiohttp_session = aiohttp.ClientSession() aiohttp_session = aiohttp.ClientSession()
daily_helper = DailyRESTHelper( daily_helper = DailyRESTHelper(
@@ -41,15 +40,11 @@ async def main():
daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
aiohttp_session=aiohttp_session, aiohttp_session=aiohttp_session,
) )
room = await daily_helper.create_room(DailyRoomParams()) room = await daily_helper.create_room(DailyRoomParams())
expiry_time: float = 60 * 60 expiry_time: float = 60 * 60
token = await daily_helper.get_token(room.url, expiry_time) token = await daily_helper.get_token(room.url, expiry_time)
print("Room created ", room.url) print("Room created ", room.url)
transport = DailyTransport( transport = DailyTransport(
room.url, room.url,
token, token,
@@ -84,7 +79,7 @@ async def main():
) )
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini")
messages = [ messages = [
{ {
"role": "system", "role": "system",
@@ -98,13 +93,9 @@ async def main():
# "content": "Eres Chatbot, un amigable y útil robot. Tu objetivo es demostrar tus capacidades de una manera breve. Tus respuestas se convertiran a audio así que nunca no debes incluir caracteres especiales. Contesta a lo que el usuario pregunte de una manera creativa, útil y breve. Empieza por presentarte a ti mismo.", # "content": "Eres Chatbot, un amigable y útil robot. Tu objetivo es demostrar tus capacidades de una manera breve. Tus respuestas se convertiran a audio así que nunca no debes incluir caracteres especiales. Contesta a lo que el usuario pregunte de una manera creativa, útil y breve. Empieza por presentarte a ti mismo.",
}, },
] ]
simliAi = SimliVideoService( simli_ai = SimliVideoService(
SimliConfig(os.getenv("SIMLI_API_KEY"), os.getenv("SIMLI_FACE_ID")) SimliConfig(os.getenv("SIMLI_API_KEY"), os.getenv("SIMLI_FACE_ID"))
) )
print("starting connection to simi")
await simliAi.startConnection()
print("connection started")
context = OpenAILLMContext(messages) context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context) context_aggregator = llm.create_context_aggregator(context)
@@ -114,7 +105,7 @@ async def main():
context_aggregator.user(), context_aggregator.user(),
llm, llm,
tts, tts,
simliAi, simli_ai,
transport.output(), transport.output(),
context_aggregator.assistant(), context_aggregator.assistant(),
] ]

92
src/pipecat/services/simli.py
View File

@@ -8,58 +8,57 @@ from pipecat.frames.frames import (
EndFrame, EndFrame,
CancelFrame, CancelFrame,
) )
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, StartFrame
import numpy as np import numpy as np
from av import AudioFrame from av import AudioFrame
from av.audio.resampler import AudioResampler from av.audio.resampler import AudioResampler
from simli import SimliClient, SimliConfig from simli import SimliClient, SimliConfig
from loguru import logger
class SimliVideoService(FrameProcessor): class SimliVideoService(FrameProcessor):
def __init__( def __init__(self, simli_config: SimliConfig, use_turn_server=False, latency_interval=0):
self, simliConfig: SimliConfig, useTurnServer=False, latencyInterval=60
):
super().__init__() super().__init__()
self.simliClient = SimliClient(simliConfig, useTurnServer, latencyInterval) self._simli_client = SimliClient(simli_config, use_turn_server, latency_interval)
self.pipecatResampler: AudioResampler = None self._ready = False
self.name = "SimliAi" self._pipecat_resampler: AudioResampler = None
self.ready = False self._simli_resampler = AudioResampler("s16", 1, 16000)
self.simliResampler = AudioResampler("s16", 1, 16000)
self.AudioTask: asyncio.Task = None
self.VideoTask: asyncio.Task = None
async def startConnection(self): self._audio_task: asyncio.Task = None
await self.simliClient.Initialize() self._video_task: asyncio.Task = None
self.ready = True
async def _start_connection(self):
await self._simli_client.Initialize()
self._ready = True
# Create task to consume and process audio and video # Create task to consume and process audio and video
self.AudioTask = asyncio.create_task(self.consume_and_process_audio()) self._audio_task = asyncio.create_task(self._consume_and_process_audio())
self.VideoTask = asyncio.create_task(self.consume_and_process_video()) self._video_task = asyncio.create_task(self._consume_and_process_video())
async def consume_and_process_audio(self): async def _consume_and_process_audio(self):
async for audio_frame in self.simliClient.getAudioStreamIterator(): while self._pipecat_resampler is None:
await asyncio.sleep(0.001)
async for audio_frame in self._simli_client.getAudioStreamIterator():
# Process the audio frame # Process the audio frame
try: try:
resampledFrames = self.pipecatResampler.resample(audio_frame) resampled_frames = self._pipecat_resampler.resample(audio_frame)
for resampled_frame in resampledFrames: for resampled_frame in resampled_frames:
await self.push_frame( await self.push_frame(
TTSAudioRawFrame( TTSAudioRawFrame(
audio=resampled_frame.to_ndarray().tobytes(), audio=resampled_frame.to_ndarray().tobytes(),
sample_rate=self.pipecatResampler.rate, sample_rate=self._pipecat_resampler.rate,
num_channels=1, num_channels=1,
), ),
) )
except Exception as e: except Exception as e:
print(e) logger.exception(f"{self} exception: {e}")
import traceback
traceback.print_exc() async def _consume_and_process_video(self):
while self._pipecat_resampler is None:
async def consume_and_process_video(self): await asyncio.sleep(0.001)
async for video_frame in self.simliClient.getVideoStreamIterator( async for video_frame in self._simli_client.getVideoStreamIterator(targetFormat="rgb24"):
targetFormat="rgb24"
):
# Process the video frame # Process the video frame
convertedFrame: OutputImageRawFrame = OutputImageRawFrame( convertedFrame: OutputImageRawFrame = OutputImageRawFrame(
image=video_frame.to_rgb().to_image().tobytes(), image=video_frame.to_rgb().to_image().tobytes(),
@@ -73,44 +72,41 @@ class SimliVideoService(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
if isinstance(frame, TTSAudioRawFrame): await self._start_connection()
elif isinstance(frame, TTSAudioRawFrame):
# Send audio frame to Simli # Send audio frame to Simli
try: try:
if self.ready: if self._ready:
AudioFrame
oldFrame = AudioFrame.from_ndarray( oldFrame = AudioFrame.from_ndarray(
np.frombuffer(frame.audio, dtype=np.int16)[None, :], np.frombuffer(frame.audio, dtype=np.int16)[None, :],
layout=frame.num_channels, layout=frame.num_channels,
) )
oldFrame.sample_rate = frame.sample_rate oldFrame.sample_rate = frame.sample_rate
if self.pipecatResampler is None: if self._pipecat_resampler is None:
self.pipecatResampler = AudioResampler( self._pipecat_resampler = AudioResampler(
"s16", oldFrame.layout, oldFrame.sample_rate "s16", oldFrame.layout, oldFrame.sample_rate
) )
resampledFrame = self.simliResampler.resample(oldFrame) resampledFrame = self._simli_resampler.resample(oldFrame)
for frame in resampledFrame: for frame in resampledFrame:
await self.simliClient.send( await self._simli_client.send(frame.to_ndarray().astype(np.int16).tobytes())
frame.to_ndarray().astype(np.int16).tobytes()
)
return return
else: else:
print( logger.warning(
"Simli Connection is not Initialized properly, passing audio to next processor" "Simli Connection is not Initialized properly, passing audio to next processor"
) )
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
except Exception as e: except Exception as e:
print(e) logger.exception(f"{self} exception: {e}")
import traceback
traceback.print_exc()
elif isinstance(frame, (EndFrame, CancelFrame)): elif isinstance(frame, (EndFrame, CancelFrame)):
await self.simliClient.stop() await self._simli_client.stop()
self.AudioTask.cancel() self._audio_task.cancel()
self.VideoTask.cancel() await self._audio_task
self._video_task.cancel()
await self._video_task
elif isinstance(frame, StartInterruptionFrame): elif isinstance(frame, StartInterruptionFrame):
await self.simliClient.clearBuffer() await self._simli_client.clearBuffer()
await self.push_frame(frame, direction) await self.push_frame(frame, direction)