improved unit tests / add a run_test function to test processors
This commit is contained in:
Aleix Conchillo Flaqué
0
tests/processors/__init__.py
Normal file
0
tests/processors/__init__.py
Normal file
0
tests/processors/aggregators/__init__.py
Normal file
0
tests/processors/aggregators/__init__.py
Normal file
370
tests/processors/aggregators/test_llm_response.py
Normal file
370
tests/processors/aggregators/test_llm_response.py
Normal file
@@ -0,0 +1,370 @@
|
||||
#
|
||||
# Copyright (c) 2024, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import unittest
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
BotInterruptionFrame,
|
||||
InterimTranscriptionFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
StartInterruptionFrame,
|
||||
StopInterruptionFrame,
|
||||
TextFrame,
|
||||
TranscriptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
)
|
||||
from pipecat.processors.aggregators.llm_response import (
|
||||
LLMAssistantContextAggregator,
|
||||
LLMFullResponseAggregator,
|
||||
LLMUserContextAggregator,
|
||||
)
|
||||
from pipecat.processors.aggregators.openai_llm_context import (
|
||||
OpenAILLMContext,
|
||||
OpenAILLMContextFrame,
|
||||
)
|
||||
from tests.utils import run_test
|
||||
|
||||
|
||||
class TestLLMUserContextAggregator(unittest.IsolatedAsyncioTestCase):
|
||||
# S E ->
|
||||
async def test_s_e(self):
|
||||
"""S E case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
]
|
||||
await run_test(context_aggregator, frames_to_send, expected_returned_frames)
|
||||
|
||||
# S T E -> T
|
||||
async def test_s_t_e(self):
|
||||
"""S T E case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
TranscriptionFrame("Hello", "", ""),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
await run_test(context_aggregator, frames_to_send, expected_returned_frames)
|
||||
|
||||
# S I T E -> T
|
||||
async def test_s_i_t_e(self):
|
||||
"""S I T E case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
InterimTranscriptionFrame("This", "", ""),
|
||||
TranscriptionFrame("This is a test", "", ""),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
await run_test(context_aggregator, frames_to_send, expected_returned_frames)
|
||||
|
||||
# S I E T -> T
|
||||
async def test_s_i_e_t(self):
|
||||
"""S I E T case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
InterimTranscriptionFrame("This", "", ""),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
TranscriptionFrame("This is a test", "", ""),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
await run_test(context_aggregator, frames_to_send, expected_returned_frames)
|
||||
|
||||
# S I E I T -> T
|
||||
async def test_s_i_e_i_t(self):
|
||||
"""S I E I T case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
InterimTranscriptionFrame("This", "", ""),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
InterimTranscriptionFrame("This is", "", ""),
|
||||
TranscriptionFrame("This is a test", "", ""),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
await run_test(context_aggregator, frames_to_send, expected_returned_frames)
|
||||
|
||||
# S E T -> T
|
||||
async def test_s_e_t(self):
|
||||
"""S E case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
TranscriptionFrame("This is a test", "", ""),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
await run_test(context_aggregator, frames_to_send, expected_returned_frames)
|
||||
|
||||
# S E I T -> T
|
||||
async def test_s_e_i_t(self):
|
||||
"""S E I T case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
InterimTranscriptionFrame("This", "", ""),
|
||||
TranscriptionFrame("This is a test", "", ""),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
await run_test(context_aggregator, frames_to_send, expected_returned_frames)
|
||||
|
||||
# S T1 I E S T2 E -> "T1 T2"
|
||||
async def test_s_t1_i_e_s_t2_e(self):
|
||||
"""S T1 I E S T2 E case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
TranscriptionFrame("T1", "", ""),
|
||||
InterimTranscriptionFrame("", "", ""),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
TranscriptionFrame("T2", "", ""),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
(received_down, _) = await run_test(
|
||||
context_aggregator, frames_to_send, expected_returned_frames
|
||||
)
|
||||
assert received_down[-1].context.messages[-1]["content"] == "T1 T2"
|
||||
|
||||
# S I E T1 I T2 -> T1 Interruption T2
|
||||
async def test_s_i_e_t1_i_t2(self):
|
||||
"""S I E T1 I T2 case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
InterimTranscriptionFrame("", "", ""),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
TranscriptionFrame("T1", "", ""),
|
||||
InterimTranscriptionFrame("", "", ""),
|
||||
TranscriptionFrame("T2", "", ""),
|
||||
]
|
||||
expected_down_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
expected_up_frames = [
|
||||
BotInterruptionFrame,
|
||||
]
|
||||
(received_down, _) = await run_test(
|
||||
context_aggregator, frames_to_send, expected_down_frames, expected_up_frames
|
||||
)
|
||||
assert received_down[-1].context.messages[-2]["content"] == "T1"
|
||||
assert received_down[-1].context.messages[-1]["content"] == "T2"
|
||||
|
||||
# S T1 E T2 -> T1 Interruption T2
|
||||
async def test_s_t1_e_t2(self):
|
||||
"""S T1 E T2 case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
TranscriptionFrame("T1", "", ""),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
TranscriptionFrame("T2", "", ""),
|
||||
]
|
||||
expected_down_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
expected_up_frames = [
|
||||
BotInterruptionFrame,
|
||||
]
|
||||
(received_down, _) = await run_test(
|
||||
context_aggregator, frames_to_send, expected_down_frames, expected_up_frames
|
||||
)
|
||||
assert received_down[-1].context.messages[-2]["content"] == "T1"
|
||||
assert received_down[-1].context.messages[-1]["content"] == "T2"
|
||||
|
||||
# S E T1 T2 -> T1 Interruption T2
|
||||
async def test_s_e_t1_t2(self):
|
||||
"""S E T1 T2 case"""
|
||||
context_aggregator = LLMUserContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
StartInterruptionFrame(),
|
||||
UserStartedSpeakingFrame(),
|
||||
StopInterruptionFrame(),
|
||||
UserStoppedSpeakingFrame(),
|
||||
TranscriptionFrame("T1", "", ""),
|
||||
TranscriptionFrame("T2", "", ""),
|
||||
]
|
||||
expected_down_frames = [
|
||||
StartInterruptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
StopInterruptionFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
OpenAILLMContextFrame,
|
||||
OpenAILLMContextFrame,
|
||||
]
|
||||
expected_up_frames = [
|
||||
BotInterruptionFrame,
|
||||
]
|
||||
(received_down, _) = await run_test(
|
||||
context_aggregator, frames_to_send, expected_down_frames, expected_up_frames
|
||||
)
|
||||
assert received_down[-1].context.messages[-2]["content"] == "T1"
|
||||
assert received_down[-1].context.messages[-1]["content"] == "T2"
|
||||
|
||||
|
||||
class TestLLMAssistantContextAggregator(unittest.IsolatedAsyncioTestCase):
|
||||
# S T E -> T
|
||||
async def test_s_t_e(self):
|
||||
"""S T E case"""
|
||||
context_aggregator = LLMAssistantContextAggregator(
|
||||
OpenAILLMContext(messages=[{"role": "", "content": ""}])
|
||||
)
|
||||
frames_to_send = [
|
||||
LLMFullResponseStartFrame(),
|
||||
TextFrame("Hello this is Pipecat speaking!"),
|
||||
TextFrame("How are you?"),
|
||||
LLMFullResponseEndFrame(),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
LLMFullResponseStartFrame,
|
||||
OpenAILLMContextFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
]
|
||||
(received_down, _) = await run_test(
|
||||
context_aggregator, frames_to_send, expected_returned_frames
|
||||
)
|
||||
assert (
|
||||
received_down[-2].context.messages[-1]["content"]
|
||||
== "Hello this is Pipecat speaking! How are you?"
|
||||
)
|
||||
|
||||
|
||||
class TestLLMFullResponseAggregator(unittest.IsolatedAsyncioTestCase):
|
||||
# S T E -> T
|
||||
async def test_s_t_e(self):
|
||||
"""S T E case"""
|
||||
response_aggregator = LLMFullResponseAggregator()
|
||||
frames_to_send = [
|
||||
LLMFullResponseStartFrame(),
|
||||
TextFrame("Hello "),
|
||||
TextFrame("this "),
|
||||
TextFrame("is "),
|
||||
TextFrame("Pipecat!"),
|
||||
LLMFullResponseEndFrame(),
|
||||
]
|
||||
expected_returned_frames = [
|
||||
LLMFullResponseStartFrame,
|
||||
TextFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
]
|
||||
(received_down, _) = await run_test(
|
||||
response_aggregator, frames_to_send, expected_returned_frames
|
||||
)
|
||||
assert received_down[-2].text == "Hello this is Pipecat!"
|
||||
0
tests/processors/frameworks/__init__.py
Normal file
0
tests/processors/frameworks/__init__.py
Normal file
99
tests/processors/frameworks/test_langchain.py
Normal file
99
tests/processors/frameworks/test_langchain.py
Normal file
@@ -0,0 +1,99 @@
|
||||
#
|
||||
# Copyright (c) 2024, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import unittest
|
||||
|
||||
from langchain.prompts import ChatPromptTemplate
|
||||
from langchain_core.language_models import FakeStreamingListLLM
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
EndFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
TextFrame,
|
||||
TranscriptionFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
UserStoppedSpeakingFrame,
|
||||
)
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.llm_response import (
|
||||
LLMAssistantResponseAggregator,
|
||||
LLMUserResponseAggregator,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameProcessor
|
||||
from pipecat.processors.frameworks.langchain import LangchainProcessor
|
||||
|
||||
|
||||
class TestLangchain(unittest.IsolatedAsyncioTestCase):
|
||||
class MockProcessor(FrameProcessor):
|
||||
def __init__(self, name):
|
||||
super().__init__()
|
||||
self.name = name
|
||||
self.token: list[str] = []
|
||||
# Start collecting tokens when we see the start frame
|
||||
self.start_collecting = False
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
async def process_frame(self, frame, direction):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, LLMFullResponseStartFrame):
|
||||
self.start_collecting = True
|
||||
elif isinstance(frame, TextFrame) and self.start_collecting:
|
||||
self.token.append(frame.text)
|
||||
elif isinstance(frame, LLMFullResponseEndFrame):
|
||||
self.start_collecting = False
|
||||
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
def setUp(self):
|
||||
self.expected_response = "Hello dear human"
|
||||
self.fake_llm = FakeStreamingListLLM(responses=[self.expected_response])
|
||||
|
||||
async def test_langchain(self):
|
||||
messages = [("system", "Say hello to {name}"), ("human", "{input}")]
|
||||
prompt = ChatPromptTemplate.from_messages(messages).partial(name="Thomas")
|
||||
chain = prompt | self.fake_llm
|
||||
proc = LangchainProcessor(chain=chain)
|
||||
self.mock_proc = self.MockProcessor("token_collector")
|
||||
|
||||
tma_in = LLMUserResponseAggregator(messages)
|
||||
tma_out = LLMAssistantResponseAggregator(messages)
|
||||
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
tma_in,
|
||||
proc,
|
||||
self.mock_proc,
|
||||
tma_out,
|
||||
]
|
||||
)
|
||||
|
||||
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=False))
|
||||
await task.queue_frames(
|
||||
[
|
||||
UserStartedSpeakingFrame(),
|
||||
TranscriptionFrame(text="Hi World", user_id="user", timestamp="now"),
|
||||
UserStoppedSpeakingFrame(),
|
||||
EndFrame(),
|
||||
]
|
||||
)
|
||||
|
||||
runner = PipelineRunner()
|
||||
await runner.run(task)
|
||||
self.assertEqual("".join(self.mock_proc.token), self.expected_response)
|
||||
# TODO: Address this issue
|
||||
# This next one would fail with:
|
||||
# AssertionError: ' H e l l o d e a r h u m a n' != 'Hello dear human'
|
||||
# self.assertEqual(tma_out.messages[-1]["content"], self.expected_response)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user