Add NebiusLLMService with developer role and tool support fixes
- Add Nebius LLM service wrapping OpenAI-compatible Token Factory API - Set supports_developer_role = False (Nebius rejects developer role) - Default to openai/gpt-oss-120b model (supports function calling) - Add Nebius function-calling example and env.example entry - Fix Sarvam developer role support - Update examples to use developer role for intro messages
This commit is contained in:
@@ -1,13 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import sys
|
||||
|
||||
from pipecat.services import DeprecatedModuleProxy
|
||||
|
||||
from .llm import *
|
||||
|
||||
sys.modules[__name__] = DeprecatedModuleProxy(globals(), "nebius", "nebius.llm")
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
"""Nebius Token Factory LLM service implementation using OpenAI-compatible interface."""
|
||||
"""Nebius LLM service implementation using OpenAI-compatible interface."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
@@ -23,26 +23,16 @@ class NebiusLLMSettings(BaseOpenAILLMService.Settings):
|
||||
|
||||
|
||||
class NebiusLLMService(OpenAILLMService):
|
||||
"""A service for interacting with Nebius Token Factory's API using the OpenAI-compatible interface.
|
||||
"""A service for interacting with Nebius's API using the OpenAI-compatible interface.
|
||||
|
||||
This service extends OpenAILLMService to connect to Nebius Token Factory's API endpoint
|
||||
while maintaining full compatibility with OpenAI's interface and functionality.
|
||||
|
||||
Nebius Token Factory provides access to open-source models including Meta Llama,
|
||||
Qwen, and DeepSeek variants through an OpenAI-compatible REST API.
|
||||
|
||||
Set the ``NEBIUS_API_KEY`` environment variable or pass ``api_key`` directly.
|
||||
|
||||
Example::
|
||||
|
||||
service = NebiusLLMService(
|
||||
api_key="your-nebius-api-key",
|
||||
settings=NebiusLLMService.Settings(
|
||||
model="meta-llama/Meta-Llama-3.1-70B-Instruct",
|
||||
),
|
||||
)
|
||||
This service extends OpenAILLMService to connect to Nebius's API endpoint while
|
||||
maintaining full compatibility with OpenAI's interface and functionality.
|
||||
"""
|
||||
|
||||
# Nebius doesn't support the "developer" message role.
|
||||
# This value is used by BaseOpenAILLMService when calling the adapter.
|
||||
supports_developer_role = False
|
||||
|
||||
Settings = NebiusLLMSettings
|
||||
_settings: Settings
|
||||
|
||||
@@ -51,39 +41,32 @@ class NebiusLLMService(OpenAILLMService):
|
||||
*,
|
||||
api_key: str,
|
||||
base_url: str = "https://api.tokenfactory.nebius.com/v1/",
|
||||
model: Optional[str] = None,
|
||||
settings: Optional[Settings] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize the Nebius Token Factory LLM service.
|
||||
"""Initialize the Nebius LLM service.
|
||||
|
||||
Args:
|
||||
api_key: The API key for accessing Nebius Token Factory's API.
|
||||
api_key: The API key for accessing Nebius's API.
|
||||
base_url: The base URL for the Nebius API. Defaults to
|
||||
``"https://api.tokenfactory.nebius.com/v1/"``.
|
||||
model: The model identifier to use. Defaults to
|
||||
``"meta-llama/Meta-Llama-3.1-8B-Instruct"``.
|
||||
|
||||
.. deprecated:: 0.0.109
|
||||
Use ``settings=NebiusLLMService.Settings(model=...)`` instead.
|
||||
|
||||
settings: Runtime-updatable settings. When provided alongside deprecated
|
||||
parameters, ``settings`` values take precedence.
|
||||
**kwargs: Additional keyword arguments passed to OpenAILLMService.
|
||||
"""
|
||||
default_settings = self.Settings(model="meta-llama/Meta-Llama-3.1-8B-Instruct")
|
||||
|
||||
if model is not None:
|
||||
self._warn_init_param_moved_to_settings("model", "model")
|
||||
default_settings.model = model
|
||||
# Initialize default_settings with hardcoded defaults
|
||||
default_settings = self.Settings(
|
||||
model="openai/gpt-oss-120b",
|
||||
)
|
||||
|
||||
# Apply settings delta (canonical API, always wins)
|
||||
if settings is not None:
|
||||
default_settings.apply_update(settings)
|
||||
|
||||
super().__init__(api_key=api_key, base_url=base_url, settings=default_settings, **kwargs)
|
||||
|
||||
def create_client(self, api_key=None, base_url=None, **kwargs):
|
||||
"""Create OpenAI-compatible client for Nebius Token Factory API endpoint.
|
||||
"""Create OpenAI-compatible client for Nebius API endpoint.
|
||||
|
||||
Args:
|
||||
api_key: The API key for authentication. If None, uses instance default.
|
||||
@@ -91,7 +74,7 @@ class NebiusLLMService(OpenAILLMService):
|
||||
**kwargs: Additional keyword arguments for client configuration.
|
||||
|
||||
Returns:
|
||||
An OpenAI-compatible client configured for Nebius Token Factory's API.
|
||||
An OpenAI-compatible client configured for Nebius's API.
|
||||
"""
|
||||
logger.debug(f"Creating Nebius client with api {base_url}")
|
||||
return super().create_client(api_key, base_url, **kwargs)
|
||||
|
||||
@@ -42,6 +42,10 @@ class SarvamLLMService(OpenAILLMService):
|
||||
maintaining full compatibility with OpenAI's interface and functionality.
|
||||
"""
|
||||
|
||||
# Sarvam doesn't support the "developer" message role.
|
||||
# This value is used by BaseOpenAILLMService when calling the adapter.
|
||||
supports_developer_role = False
|
||||
|
||||
_SUPPORTED_MODELS = frozenset(
|
||||
{"sarvam-30b", "sarvam-30b-16k", "sarvam-105b", "sarvam-105b-32k"}
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user