Base OpenAI LLM service

This commit is contained in:
Moishe Lettvin
2024-03-08 11:09:16 -05:00
parent c75a3fb0d0
commit d9378e23ba
15 changed files with 558 additions and 221 deletions

View File

@@ -1,42 +1,7 @@
from openai import AsyncOpenAI
import json
from collections.abc import AsyncGenerator
from dailyai.services.ai_services import LLMService
from dailyai.services.openai_api_llm_service import BaseOpenAILLMService
class OLLamaLLMService(LLMService):
def __init__(self, model="llama2", base_url='http://localhost:11434/v1'):
super().__init__()
self._model = model
self._client = AsyncOpenAI(api_key="ollama", base_url=base_url)
class OLLamaLLMService(BaseOpenAILLMService):
async def get_response(self, messages, stream):
return await self._client.chat.completions.create(
stream=stream,
messages=messages,
model=self._model
)
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages)
async for chunk in chunks:
if len(chunk.choices) == 0:
continue
if chunk.choices[0].delta.content:
yield chunk.choices[0].delta.content
async def run_llm(self, messages) -> str | None:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages)
if response and len(response.choices) > 0:
return response.choices[0].message.content
else:
return None
def __init__(self, model="llama2", base_url="http://localhost:11434/v1"):
super().__init__(model=model, base_url=base_url, api_key="ollama")