wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #119 from daily-co/use-new-fal-client-library

Browse Source 
services: FalImageGenService now uses fal-client library
This commit is contained in:
Aleix Conchillo Flaqué
2024-04-12 05:59:58 +08:00
committed by GitHub
parent 1e83a405c0 cbc51babbe
commit f1b6b9f8e5
12 changed files with 67 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
dot-env.template
View File

@@ -22,8 +22,7 @@ ELEVENLABS_API_KEY=...
ELEVENLABS_VOICE_ID=... ELEVENLABS_VOICE_ID=...
# Fal # Fal
FAL_KEY_ID=... FAL_KEY=...
FAL_KEY_SECRET=...
# PlayHT # PlayHT
PLAY_HT_USER_ID=... PLAY_HT_USER_ID=...

3
examples/foundational/03-still-frame.py
View File

@@ -35,8 +35,7 @@ async def main(room_url):
image_size="square_hd" image_size="square_hd"
), ),
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key=os.getenv("FAL_KEY"),
key_secret=os.getenv("FAL_KEY_SECRET"),
) )
pipeline = Pipeline([imagegen]) pipeline = Pipeline([imagegen])

3
examples/foundational/03a-image-local.py
View File

@@ -39,8 +39,7 @@ async def main():
image_size="square_hd" image_size="square_hd"
), ),
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key=os.getenv("FAL_KEY"),
key_secret=os.getenv("FAL_KEY_SECRET"),
) )
pipeline = Pipeline([imagegen]) pipeline = Pipeline([imagegen])

3
examples/foundational/05-sync-speech-and-image.py
View File

@@ -89,8 +89,7 @@ async def main(room_url):
image_size="square_hd" image_size="square_hd"
), ),
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key=os.getenv("FAL_KEY"),
key_secret=os.getenv("FAL_KEY_SECRET"),
) )
gated_aggregator = GatedAggregator( gated_aggregator = GatedAggregator(

3
examples/foundational/05a-local-sync-speech-and-text.py
View File

@@ -49,8 +49,7 @@ async def main():
image_size="1024x1024" image_size="1024x1024"
), ),
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key=os.getenv("FAL_KEY"),
key_secret=os.getenv("FAL_KEY_SECRET"),
) )
# Get a complete audio chunk from the given text. Splitting this into its own # Get a complete audio chunk from the given text. Splitting this into its own

3
examples/foundational/08-bots-arguing.py
View File

@@ -55,8 +55,7 @@ async def main(room_url: str):
image_size="1024x1024" image_size="1024x1024"
), ),
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key=os.getenv("FAL_KEY"),
key_secret=os.getenv("FAL_KEY_SECRET"),
) )
bot1_messages = [ bot1_messages = [

3
examples/starter-apps/storybot.py
View File

@@ -208,8 +208,7 @@ async def main(room_url: str, token):
image_size = "1024x1024", image_size = "1024x1024",
}, },
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key=os.getenv("FAL_KEY"),
key_secret=os.getenv("FAL_KEY_SECRET"),
) )
lra = LLMAssistantResponseAggregator(messages) lra = LLMAssistantResponseAggregator(messages)
ura = LLMUserResponseAggregator(messages) ura = LLMUserResponseAggregator(messages)

126
linux-py3.10-requirements.txt
View File

@@ -4,10 +4,12 @@
# #
# pip-compile --all-extras pyproject.toml # pip-compile --all-extras pyproject.toml
# #
aiohttp==3.9.3 aiohttp==3.9.4
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
aiosignal==1.3.1 aiosignal==1.3.1
# via aiohttp # via aiohttp
annotated-types==0.6.0
# via pydantic
anthropic==0.20.0 anthropic==0.20.0
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
anyio==4.3.0 anyio==4.3.0
@@ -15,13 +17,10 @@ anyio==4.3.0
# anthropic # anthropic
# httpx # httpx
# openai # openai
# starlette
async-timeout==4.0.3 async-timeout==4.0.3
# via aiohttp # via aiohttp
attrs==23.2.0 attrs==23.2.0
# via # via aiohttp
# aiohttp
# fal
av==11.0.0 av==11.0.0
# via faster-whisper # via faster-whisper
azure-cognitiveservices-speech==1.36.0 azure-cognitiveservices-speech==1.36.0
@@ -33,31 +32,16 @@ certifi==2024.2.2
# httpcore # httpcore
# httpx # httpx
# requests # requests
cffi==1.16.0
# via cryptography
charset-normalizer==3.3.2 charset-normalizer==3.3.2
# via requests # via requests
click==8.1.7 click==8.1.7
# via # via flask
# fal
# flask
# rich-click
colorama==0.4.6
# via fal
coloredlogs==15.0.1 coloredlogs==15.0.1
# via onnxruntime # via onnxruntime
cryptography==42.0.5
# via pyjwt
ctranslate2==4.1.0 ctranslate2==4.1.0
# via faster-whisper # via faster-whisper
daily-python==0.7.3 daily-python==0.7.3
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
deprecated==1.2.14
# via opentelemetry-api
dill==0.3.7
# via fal
distlib==0.3.8
# via virtualenv
distro==1.9.0 distro==1.9.0
# via # via
# anthropic # anthropic
@@ -66,10 +50,8 @@ einops==0.7.0
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
exceptiongroup==1.2.0 exceptiongroup==1.2.0
# via anyio # via anyio
fal==0.12.7 fal-client==0.2.2
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
fastapi==0.99.1
# via fal
faster-whisper==1.0.1 faster-whisper==1.0.1
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
filelock==3.13.4 filelock==3.13.4
@@ -79,7 +61,6 @@ filelock==3.13.4
# torch # torch
# transformers # transformers
# triton # triton
# virtualenv
flask==3.0.3 flask==3.0.3
# via # via
# dailyai (pyproject.toml) # dailyai (pyproject.toml)
@@ -96,15 +77,8 @@ fsspec==2024.3.1
# via # via
# huggingface-hub # huggingface-hub
# torch # torch
grpc-interceptor==0.15.4
# via fal
grpcio==1.62.1 grpcio==1.62.1
# via # via pyht
# fal
# grpc-interceptor
# isolate
# isolate-proto
# pyht
h11==0.14.0 h11==0.14.0
# via httpcore # via httpcore
httpcore==1.0.5 httpcore==1.0.5
@@ -112,8 +86,10 @@ httpcore==1.0.5
httpx==0.27.0 httpx==0.27.0
# via # via
# anthropic # anthropic
# fal # fal-client
# openai # openai
httpx-sse==0.4.0
# via fal-client
huggingface-hub==0.22.2 huggingface-hub==0.22.2
# via # via
# faster-whisper # faster-whisper
@@ -122,38 +98,24 @@ huggingface-hub==0.22.2
# transformers # transformers
humanfriendly==10.0 humanfriendly==10.0
# via coloredlogs # via coloredlogs
idna==3.6 idna==3.7
# via # via
# anyio # anyio
# httpx # httpx
# requests # requests
# yarl # yarl
importlib-metadata==7.0.0
# via opentelemetry-api
isolate[build]==0.12.7
# via
# fal
# isolate-proto
isolate-proto==0.3.4
# via fal
itsdangerous==2.1.2 itsdangerous==2.1.2
# via flask # via flask
jinja2==3.1.3 jinja2==3.1.3
# via # via
# flask # flask
# torch # torch
markdown-it-py==3.0.0
# via rich
markupsafe==2.1.5 markupsafe==2.1.5
# via # via
# jinja2 # jinja2
# werkzeug # werkzeug
mdurl==0.1.2
# via markdown-it-py
mpmath==1.3.0 mpmath==1.3.0
# via sympy # via sympy
msgpack==1.0.8
# via fal
multidict==6.0.5 multidict==6.0.5
# via # via
# aiohttp # aiohttp
@@ -202,64 +164,35 @@ onnxruntime==1.17.1
# via faster-whisper # via faster-whisper
openai==1.14.3 openai==1.14.3
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
opentelemetry-api==1.24.0
# via
# fal
# opentelemetry-sdk
opentelemetry-sdk==1.24.0
# via fal
opentelemetry-semantic-conventions==0.45b0
# via opentelemetry-sdk
packaging==24.0 packaging==24.0
# via # via
# fal
# huggingface-hub # huggingface-hub
# onnxruntime # onnxruntime
# transformers # transformers
pathspec==0.11.2
# via fal
pillow==10.2.0 pillow==10.2.0
# via # via
# dailyai (pyproject.toml) # dailyai (pyproject.toml)
# fal
# torchvision # torchvision
platformdirs==4.2.0
# via
# isolate
# virtualenv
portalocker==2.8.2
# via fal
protobuf==4.25.3 protobuf==4.25.3
# via # via
# isolate
# isolate-proto
# onnxruntime # onnxruntime
# pyht # pyht
pyaudio==0.2.14 pyaudio==0.2.14
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
pycparser==2.22 pydantic==2.7.0
# via cffi
pydantic==1.10.15
# via # via
# anthropic # anthropic
# fal
# fastapi
# openai # openai
pygments==2.17.2 pydantic-core==2.18.1
# via rich # via pydantic
pyht==0.0.27 pyht==0.0.27
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
pyjwt[crypto]==2.8.0
# via fal
python-dateutil==2.9.0.post0
# via fal
python-dotenv==1.0.1 python-dotenv==1.0.1
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
pyyaml==6.0.1 pyyaml==6.0.1
# via # via
# ctranslate2 # ctranslate2
# huggingface-hub # huggingface-hub
# isolate
# timm # timm
# transformers # transformers
regex==2023.12.25 regex==2023.12.25
@@ -269,34 +202,20 @@ requests==2.31.0
# huggingface-hub # huggingface-hub
# pyht # pyht
# transformers # transformers
rich==13.7.1
# via
# fal
# rich-click
rich-click==1.7.4
# via fal
safetensors==0.4.2 safetensors==0.4.2
# via # via
# timm # timm
# transformers # transformers
six==1.16.0
# via python-dateutil
sniffio==1.3.1 sniffio==1.3.1
# via # via
# anthropic # anthropic
# anyio # anyio
# httpx # httpx
# openai # openai
starlette==0.27.0
# via fastapi
structlog==22.3.0
# via fal
sympy==1.12 sympy==1.12
# via # via
# onnxruntime # onnxruntime
# torch # torch
tblib==3.0.0
# via isolate
timm==0.9.16 timm==0.9.16
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
tokenizers==0.15.2 tokenizers==0.15.2
@@ -323,37 +242,24 @@ transformers==4.39.3
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
triton==2.2.0 triton==2.2.0
# via torch # via torch
types-python-dateutil==2.9.0.20240316
# via fal
typing-extensions==4.10.0 typing-extensions==4.10.0
# via # via
# anthropic # anthropic
# anyio # anyio
# dailyai (pyproject.toml) # dailyai (pyproject.toml)
# fal
# fastapi
# huggingface-hub # huggingface-hub
# openai # openai
# opentelemetry-sdk
# pydantic # pydantic
# rich-click # pydantic-core
# torch # torch
urllib3==2.2.1 urllib3==2.2.1
# via requests # via requests
virtualenv==20.25.1
# via isolate
websockets==12.0 websockets==12.0
# via # via dailyai (pyproject.toml)
# dailyai (pyproject.toml)
# fal
werkzeug==3.0.2 werkzeug==3.0.2
# via flask # via flask
wrapt==1.16.0
# via deprecated
yarl==1.9.4 yarl==1.9.4
# via aiohttp # via aiohttp
zipp==3.18.1
# via importlib-metadata
# The following packages are considered to be unsafe in a requirements file: # The following packages are considered to be unsafe in a requirements file:
# setuptools # setuptools

128
macos-py3.10-requirements.txt
View File

@@ -4,10 +4,12 @@
# #
# pip-compile --all-extras pyproject.toml # pip-compile --all-extras pyproject.toml
# #
aiohttp==3.9.3 aiohttp==3.9.4
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
aiosignal==1.3.1 aiosignal==1.3.1
# via aiohttp # via aiohttp
annotated-types==0.6.0
# via pydantic
anthropic==0.20.0 anthropic==0.20.0
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
anyio==4.3.0 anyio==4.3.0
@@ -15,13 +17,10 @@ anyio==4.3.0
# anthropic # anthropic
# httpx # httpx
# openai # openai
# starlette
async-timeout==4.0.3 async-timeout==4.0.3
# via aiohttp # via aiohttp
attrs==23.2.0 attrs==23.2.0
# via # via aiohttp
# aiohttp
# fal
av==11.0.0 av==11.0.0
# via faster-whisper # via faster-whisper
azure-cognitiveservices-speech==1.36.0 azure-cognitiveservices-speech==1.36.0
@@ -33,31 +32,16 @@ certifi==2024.2.2
# httpcore # httpcore
# httpx # httpx
# requests # requests
cffi==1.16.0
# via cryptography
charset-normalizer==3.3.2 charset-normalizer==3.3.2
# via requests # via requests
click==8.1.7 click==8.1.7
# via # via flask
# fal
# flask
# rich-click
colorama==0.4.6
# via fal
coloredlogs==15.0.1 coloredlogs==15.0.1
# via onnxruntime # via onnxruntime
cryptography==42.0.5 ctranslate2==4.2.0
# via pyjwt
ctranslate2==4.1.0
# via faster-whisper # via faster-whisper
daily-python==0.7.3 daily-python==0.7.3
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
deprecated==1.2.14
# via opentelemetry-api
dill==0.3.7
# via fal
distlib==0.3.8
# via virtualenv
distro==1.9.0 distro==1.9.0
# via # via
# anthropic # anthropic
@@ -66,10 +50,8 @@ einops==0.7.0
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
exceptiongroup==1.2.0 exceptiongroup==1.2.0
# via anyio # via anyio
fal==0.12.7 fal-client==0.2.2
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
fastapi==0.99.1
# via fal
faster-whisper==1.0.1 faster-whisper==1.0.1
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
filelock==3.13.4 filelock==3.13.4
@@ -78,7 +60,6 @@ filelock==3.13.4
# pyht # pyht
# torch # torch
# transformers # transformers
# virtualenv
flask==3.0.3 flask==3.0.3
# via # via
# dailyai (pyproject.toml) # dailyai (pyproject.toml)
@@ -95,15 +76,8 @@ fsspec==2024.3.1
# via # via
# huggingface-hub # huggingface-hub
# torch # torch
grpc-interceptor==0.15.4
# via fal
grpcio==1.62.1 grpcio==1.62.1
# via # via pyht
# fal
# grpc-interceptor
# isolate
# isolate-proto
# pyht
h11==0.14.0 h11==0.14.0
# via httpcore # via httpcore
httpcore==1.0.5 httpcore==1.0.5
@@ -111,8 +85,10 @@ httpcore==1.0.5
httpx==0.27.0 httpx==0.27.0
# via # via
# anthropic # anthropic
# fal # fal-client
# openai # openai
httpx-sse==0.4.0
# via fal-client
huggingface-hub==0.22.2 huggingface-hub==0.22.2
# via # via
# faster-whisper # faster-whisper
@@ -121,38 +97,24 @@ huggingface-hub==0.22.2
# transformers # transformers
humanfriendly==10.0 humanfriendly==10.0
# via coloredlogs # via coloredlogs
idna==3.6 idna==3.7
# via # via
# anyio # anyio
# httpx # httpx
# requests # requests
# yarl # yarl
importlib-metadata==7.0.0
# via opentelemetry-api
isolate[build]==0.12.7
# via
# fal
# isolate-proto
isolate-proto==0.3.4
# via fal
itsdangerous==2.1.2 itsdangerous==2.1.2
# via flask # via flask
jinja2==3.1.3 jinja2==3.1.3
# via # via
# flask # flask
# torch # torch
markdown-it-py==3.0.0
# via rich
markupsafe==2.1.5 markupsafe==2.1.5
# via # via
# jinja2 # jinja2
# werkzeug # werkzeug
mdurl==0.1.2
# via markdown-it-py
mpmath==1.3.0 mpmath==1.3.0
# via sympy # via sympy
msgpack==1.0.8
# via fal
multidict==6.0.5 multidict==6.0.5
# via # via
# aiohttp # aiohttp
@@ -170,64 +132,35 @@ onnxruntime==1.17.1
# via faster-whisper # via faster-whisper
openai==1.14.3 openai==1.14.3
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
opentelemetry-api==1.24.0
# via
# fal
# opentelemetry-sdk
opentelemetry-sdk==1.24.0
# via fal
opentelemetry-semantic-conventions==0.45b0
# via opentelemetry-sdk
packaging==24.0 packaging==24.0
# via # via
# fal
# huggingface-hub # huggingface-hub
# onnxruntime # onnxruntime
# transformers # transformers
pathspec==0.11.2
# via fal
pillow==10.2.0 pillow==10.2.0
# via # via
# dailyai (pyproject.toml) # dailyai (pyproject.toml)
# fal
# torchvision # torchvision
platformdirs==4.2.0
# via
# isolate
# virtualenv
portalocker==2.8.2
# via fal
protobuf==4.25.3 protobuf==4.25.3
# via # via
# isolate
# isolate-proto
# onnxruntime # onnxruntime
# pyht # pyht
pyaudio==0.2.14 pyaudio==0.2.14
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
pycparser==2.22 pydantic==2.7.0
# via cffi
pydantic==1.10.15
# via # via
# anthropic # anthropic
# fal
# fastapi
# openai # openai
pygments==2.17.2 pydantic-core==2.18.1
# via rich # via pydantic
pyht==0.0.27 pyht==0.0.27
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
pyjwt[crypto]==2.8.0
# via fal
python-dateutil==2.9.0.post0
# via fal
python-dotenv==1.0.1 python-dotenv==1.0.1
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
pyyaml==6.0.1 pyyaml==6.0.1
# via # via
# ctranslate2 # ctranslate2
# huggingface-hub # huggingface-hub
# isolate
# timm # timm
# transformers # transformers
regex==2023.12.25 regex==2023.12.25
@@ -237,34 +170,20 @@ requests==2.31.0
# huggingface-hub # huggingface-hub
# pyht # pyht
# transformers # transformers
rich==13.7.1
# via
# fal
# rich-click
rich-click==1.7.4
# via fal
safetensors==0.4.2 safetensors==0.4.2
# via # via
# timm # timm
# transformers # transformers
six==1.16.0
# via python-dateutil
sniffio==1.3.1 sniffio==1.3.1
# via # via
# anthropic # anthropic
# anyio # anyio
# httpx # httpx
# openai # openai
starlette==0.27.0
# via fastapi
structlog==22.3.0
# via fal
sympy==1.12 sympy==1.12
# via # via
# onnxruntime # onnxruntime
# torch # torch
tblib==3.0.0
# via isolate
timm==0.9.16 timm==0.9.16
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
tokenizers==0.15.2 tokenizers==0.15.2
@@ -289,37 +208,24 @@ tqdm==4.66.2
# transformers # transformers
transformers==4.39.3 transformers==4.39.3
# via dailyai (pyproject.toml) # via dailyai (pyproject.toml)
types-python-dateutil==2.9.0.20240316
# via fal
typing-extensions==4.10.0 typing-extensions==4.10.0
# via # via
# anthropic # anthropic
# anyio # anyio
# dailyai (pyproject.toml) # dailyai (pyproject.toml)
# fal
# fastapi
# huggingface-hub # huggingface-hub
# openai # openai
# opentelemetry-sdk
# pydantic # pydantic
# rich-click # pydantic-core
# torch # torch
urllib3==2.2.1 urllib3==2.2.1
# via requests # via requests
virtualenv==20.25.1
# via isolate
websockets==12.0 websockets==12.0
# via # via dailyai (pyproject.toml)
# dailyai (pyproject.toml)
# fal
werkzeug==3.0.2 werkzeug==3.0.2
# via flask # via flask
wrapt==1.16.0
# via deprecated
yarl==1.9.4 yarl==1.9.4
# via aiohttp # via aiohttp
zipp==3.18.1
# via importlib-metadata
# The following packages are considered to be unsafe in a requirements file: # The following packages are considered to be unsafe in a requirements file:
# setuptools # setuptools

2
pyproject.toml
View File

@@ -35,7 +35,7 @@ anthropic = [ "anthropic~=0.20.0" ]
azure = [ "azure-cognitiveservices-speech~=1.36.0" ] azure = [ "azure-cognitiveservices-speech~=1.36.0" ]
daily = [ "daily-python~=0.7.0" ] daily = [ "daily-python~=0.7.0" ]
examples = [ "python-dotenv~=1.0.0", "flask~=3.0.0", "flask_cors~=4.0.0" ] examples = [ "python-dotenv~=1.0.0", "flask~=3.0.0", "flask_cors~=4.0.0" ]
fal = [ "fal~=0.12.0" ] fal = [ "fal-client~=0.2.0" ]
local = [ "pyaudio~=0.2.0" ] local = [ "pyaudio~=0.2.0" ]
moondream = [ "einops~=0.7.0", "timm~=0.9.0", "transformers~=4.39.0" ] moondream = [ "einops~=0.7.0", "timm~=0.9.0", "transformers~=4.39.0" ]
openai = [ "openai~=1.14.0" ] openai = [ "openai~=1.14.0" ]

40
src/dailyai/services/fal_ai_services.py
View File

@@ -6,14 +6,15 @@ from PIL import Image
from pydantic import BaseModel from pydantic import BaseModel
from typing import Optional, Union, Dict from typing import Optional, Union, Dict
from dailyai.services.ai_services import ImageGenService from dailyai.services.ai_services import ImageGenService
try: try:
import fal import fal_client
except ModuleNotFoundError as e: except ModuleNotFoundError as e:
print(f"Exception: {e}") print(f"Exception: {e}")
print( print(
"In order to use Fal, you need to `pip install dailyai[fal]`. Also, set `FAL_KEY_ID` and `FAL_KEY_SECRET` environment variables.") "In order to use Fal, you need to `pip install dailyai[fal]`. Also, set `FAL_KEY` environment variable.")
raise Exception(f"Missing module: {e}") raise Exception(f"Missing module: {e}")
@@ -33,40 +34,25 @@ class FalImageGenService(ImageGenService):
aiohttp_session: aiohttp.ClientSession, aiohttp_session: aiohttp.ClientSession,
params: InputParams, params: InputParams,
model="fal-ai/fast-sdxl", model="fal-ai/fast-sdxl",
key_id=None, key=None,
key_secret=None
): ):
super().__init__() super().__init__()
self._model = model self._model = model
self._params = params self._params = params
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session
if key_id: if key:
os.environ["FAL_KEY_ID"] = key_id os.environ["FAL_KEY"] = key
if key_secret:
os.environ["FAL_KEY_SECRET"] = key_secret
async def run_image_gen(self, prompt: str) -> tuple[str, bytes, tuple[int, int]]: async def run_image_gen(self, prompt: str) -> tuple[str, bytes, tuple[int, int]]:
def get_image_url(prompt): response = await fal_client.run_async(
handler = fal.apps.submit( # type: ignore self._model,
self._model, arguments={"prompt": prompt, **self._params.dict()}
arguments={ )
"prompt": prompt,
**self._params.dict(),
},
)
for event in handler.iter_events():
if isinstance(event, fal.apps.InProgress): # type: ignore
pass
result = handler.get() image_url = response["images"][0]["url"] if response else None
image_url = result["images"][0]["url"] if result else None if not image_url:
if not image_url: raise Exception("Image generation failed")
raise Exception("Image generation failed")
return image_url
image_url = await asyncio.to_thread(get_image_url, prompt)
# Load the image from the url # Load the image from the url
async with self._aiohttp_session.get(image_url) as response: async with self._aiohttp_session.get(image_url) as response:

19
src/dailyai/services/moondream_ai_service.py
View File

@@ -1,3 +1,5 @@
import asyncio
from dailyai.pipeline.frames import ImageFrame, VisionImageFrame from dailyai.pipeline.frames import ImageFrame, VisionImageFrame
from dailyai.services.ai_services import VisionService from dailyai.services.ai_services import VisionService
@@ -43,10 +45,15 @@ class MoondreamService(VisionService):
self._model.eval() self._model.eval()
async def run_vision(self, frame: VisionImageFrame) -> str: async def run_vision(self, frame: VisionImageFrame) -> str:
image = Image.frombytes("RGB", (frame.size[0], frame.size[1]), frame.image) def get_image_description(frame: VisionImageFrame):
image_embeds = self._model.encode_image(image) image = Image.frombytes("RGB", (frame.size[0], frame.size[1]), frame.image)
description = self._model.answer_question( image_embeds = self._model.encode_image(image)
image_embeds=image_embeds, description = self._model.answer_question(
question=frame.text, image_embeds=image_embeds,
tokenizer=self._tokenizer) question=frame.text,
tokenizer=self._tokenizer)
return description
description = await asyncio.to_thread(get_image_description, frame)
return description return description