wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #385 from pipecat-ai/mrkb/anthropic-beta-caching

Browse Source 
Mrkb/anthropic beta caching
This commit is contained in:
Mattie Ruth
2024-08-15 18:26:51 -04:00
committed by GitHub
parent 31577252f3 848db985fc
commit 4a6638f749
3 changed files with 16 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
linux-py3.10-requirements.txt
View File

@@ -16,7 +16,7 @@ aiosignal==1.3.1
# via aiohttp # via aiohttp
annotated-types==0.7.0 annotated-types==0.7.0
# via pydantic # via pydantic
anthropic==0.28.1 anthropic==0.34.0
# via # via
# openpipe # openpipe
# pipecat-ai (pyproject.toml) # pipecat-ai (pyproject.toml)

2
macos-py3.10-requirements.txt
View File

@@ -16,7 +16,7 @@ aiosignal==1.3.1
# via aiohttp # via aiohttp
annotated-types==0.7.0 annotated-types==0.7.0
# via pydantic # via pydantic
anthropic==0.28.1 anthropic==0.34.0
# via # via
# openpipe # openpipe
# pipecat-ai (pyproject.toml) # pipecat-ai (pyproject.toml)

18
src/pipecat/services/anthropic.py
View File

@@ -206,10 +206,13 @@ class AnthropicLLMService(LLMService):
finally: finally:
await self.stop_processing_metrics() await self.stop_processing_metrics()
await self.push_frame(LLMFullResponseEndFrame()) await self.push_frame(LLMFullResponseEndFrame())
comp_tokens = completion_tokens if not use_completion_tokens_estimate else completion_tokens_estimate
await self._report_usage_metrics( await self._report_usage_metrics(
prompt_tokens=prompt_tokens, prompt_tokens=prompt_tokens,
completion_tokens=(completion_tokens if not use_completion_tokens_estimate completion_tokens=comp_tokens,
else completion_tokens_estimate)) cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens
)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
@@ -244,13 +247,20 @@ class AnthropicLLMService(LLMService):
def _estimate_tokens(self, text: str) -> int: def _estimate_tokens(self, text: str) -> int:
return int(len(re.split(r'[^\w]+', text)) * 1.3) return int(len(re.split(r'[^\w]+', text)) * 1.3)
async def _report_usage_metrics(self, prompt_tokens: int, completion_tokens: int): async def _report_usage_metrics(
if prompt_tokens or completion_tokens: self,
prompt_tokens: int,
completion_tokens: int,
cache_creation_input_tokens: int,
cache_read_input_tokens: int):
if prompt_tokens or completion_tokens or cache_creation_input_tokens or cache_read_input_tokens:
tokens = { tokens = {
"processor": self.name, "processor": self.name,
"model": self._model, "model": self._model,
"prompt_tokens": prompt_tokens, "prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens, "completion_tokens": completion_tokens,
"cache_creation_input_tokens": cache_creation_input_tokens,
"cache_read_input_tokens": cache_read_input_tokens,
"total_tokens": prompt_tokens + completion_tokens "total_tokens": prompt_tokens + completion_tokens
} }
await self.start_llm_usage_metrics(tokens) await self.start_llm_usage_metrics(tokens)