move WakeCheckFilter to processors/filters

This commit is contained in:
Aleix Conchillo Flaqué
2024-05-22 15:36:18 -07:00
parent 661aa79b7c
commit 0ddfa3de5b
4 changed files with 86 additions and 77 deletions

View File

@@ -8,14 +8,8 @@ import asyncio
import aiohttp import aiohttp
import os import os
import sys import sys
import re
import time
from enum import Enum
from pipecat.frames.frames import ( from pipecat.processors.filters.wake_check_filter import WakeCheckFilter
Frame, SystemFrame, TranscriptionFrame, TextFrame
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
@@ -37,73 +31,6 @@ logger.remove(0)
logger.add(sys.stderr, level="DEBUG") logger.add(sys.stderr, level="DEBUG")
class WakeCheckFilter(FrameProcessor):
"""
This filter looks for wake phrases in the transcription frames and only passes through frames
after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief
period of continued conversation after a wake phrase has been detected.
"""
class WakeState(Enum):
IDLE = 1
AWAKE = 2
class ParticipantState:
def __init__(self, participant_id: str):
self.participant_id = participant_id
self.state = WakeCheckFilter.WakeState.IDLE
self.wake_timer = 0
self.accumulator = ""
def __init__(self, wake_phrases: list[str], keepalive_timeout: float = 2):
super().__init__()
self._participant_states = {}
self._keepalive_timeout = keepalive_timeout
self._wake_patterns = []
for name in wake_phrases:
pattern = re.compile(r'\b' + r'\s*'.join(re.escape(word)
for word in name.split()) + r'\b', re.IGNORECASE)
self._wake_patterns.append(pattern)
async def process_frame(self, frame: Frame, direction: FrameDirection):
try:
if isinstance(frame, TranscriptionFrame):
p = self._participant_states.get(frame.user_id)
if p is None:
p = WakeCheckFilter.ParticipantState(frame.user_id)
self._participant_states[frame.user_id] = p
# If we have been AWAKE within the last keepalive_timeout seconds, pass
# the frame through
if p.state == WakeCheckFilter.WakeState.AWAKE:
if time.time() - p.wake_timer < self._keepalive_timeout:
logger.debug(
f"Wake phrase keepalive timeout has not expired. Passing frame through.")
p.wake_timer = time.time()
await self.push_frame(frame)
return
else:
p.state = WakeCheckFilter.WakeState.IDLE
p.accumulator += frame.text
for pattern in self._wake_patterns:
match = pattern.search(p.accumulator)
if match:
logger.debug(f"Wake phrase triggered: {match.group()}")
# Found the wake word. Discard from the accumulator up to the start of the match
# and modify the frame in place.
p.state = WakeCheckFilter.WakeState.AWAKE
p.wake_timer = time.time()
frame.text = p.accumulator[match.start():]
p.accumulator = ""
await self.push_frame(frame)
else:
pass
else:
await self.push_frame(frame, direction)
except Exception as e:
logger.error(f"Error in wake word filter: {e}")
async def main(room_url: str, token): async def main(room_url: str, token):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -152,7 +79,7 @@ async def main(room_url: str, token):
task = PipelineTask(pipeline, allow_interruptions=True) task = PipelineTask(pipeline, allow_interruptions=True)
@ transport.event_handler("on_first_participant_joined") @transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant): async def on_first_participant_joined(transport, participant):
transport.capture_participant_transcription(participant["id"]) transport.capture_participant_transcription(participant["id"])
await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.") await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.")

View File

@@ -10,7 +10,7 @@ from pipecat.frames.frames import AppFrame, ControlFrame, Frame, SystemFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class Filter(FrameProcessor): class FrameFilter(FrameProcessor):
def __init__(self, types: List[type]): def __init__(self, types: List[type]):
super().__init__() super().__init__()

View File

@@ -0,0 +1,82 @@
#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import re
import time
from enum import Enum
from pipecat.frames.frames import Frame, TranscriptionFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from loguru import logger
class WakeCheckFilter(FrameProcessor):
"""
This filter looks for wake phrases in the transcription frames and only passes through frames
after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief
period of continued conversation after a wake phrase has been detected.
"""
class WakeState(Enum):
IDLE = 1
AWAKE = 2
class ParticipantState:
def __init__(self, participant_id: str):
self.participant_id = participant_id
self.state = WakeCheckFilter.WakeState.IDLE
self.wake_timer = 0.0
self.accumulator = ""
def __init__(self, wake_phrases: list[str], keepalive_timeout: float = 2):
super().__init__()
self._participant_states = {}
self._keepalive_timeout = keepalive_timeout
self._wake_patterns = []
for name in wake_phrases:
pattern = re.compile(r'\b' + r'\s*'.join(re.escape(word)
for word in name.split()) + r'\b', re.IGNORECASE)
self._wake_patterns.append(pattern)
async def process_frame(self, frame: Frame, direction: FrameDirection):
try:
if isinstance(frame, TranscriptionFrame):
p = self._participant_states.get(frame.user_id)
if p is None:
p = WakeCheckFilter.ParticipantState(frame.user_id)
self._participant_states[frame.user_id] = p
# If we have been AWAKE within the last keepalive_timeout seconds, pass
# the frame through
if p.state == WakeCheckFilter.WakeState.AWAKE:
if time.time() - p.wake_timer < self._keepalive_timeout:
logger.debug(
"Wake phrase keepalive timeout has not expired. Passing frame through.")
p.wake_timer = time.time()
await self.push_frame(frame)
return
else:
p.state = WakeCheckFilter.WakeState.IDLE
p.accumulator += frame.text
for pattern in self._wake_patterns:
match = pattern.search(p.accumulator)
if match:
logger.debug(f"Wake phrase triggered: {match.group()}")
# Found the wake word. Discard from the accumulator up to the start of the match
# and modify the frame in place.
p.state = WakeCheckFilter.WakeState.AWAKE
p.wake_timer = time.time()
frame.text = p.accumulator[match.start():]
p.accumulator = ""
await self.push_frame(frame)
else:
pass
else:
await self.push_frame(frame, direction)
except Exception as e:
logger.error(f"Error in wake word filter: {e}")

View File

@@ -8,7 +8,7 @@ import asyncio
from asyncio import AbstractEventLoop from asyncio import AbstractEventLoop
from enum import Enum from enum import Enum
from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame from pipecat.frames.frames import ErrorFrame, Frame
from pipecat.utils.utils import obj_count, obj_id from pipecat.utils.utils import obj_count, obj_id
from loguru import logger from loguru import logger