Skip to content

Commit bf745ac

Browse files
google-genai-botcopybara-github
authored andcommitted
feat(live): add ZIP_STORED visual frame pack and unpack helpers
Live video reaches ADK as `image/*` frames, alongside `audio/*` chunks. Saving each frame as its own artifact costs one storage operation per frame, while adding a batch method to `BaseArtifactService` would grow the abstract storage interface for every backend. This change adds `google.adk.live._visual_frames`, which packs a frame sequence into one uncompressed (`zipfile.ZIP_STORED`) `.zip` archive that the existing `save_artifact()` / `load_artifact()` methods store as is: - `VisualFrame`: a Pydantic model pairing a frame's `types.Blob` with its capture timestamp. - `pack_visual_frames()`: validates the frames (non-empty data, finite and non-decreasing timestamps), writes `frames/frame_NNNN.<ext>` members and a `metadata.json` timing manifest, and returns `(archive_bytes, manifest)`. - `unpack_visual_frames()`: the inverse; restores `(list[VisualFrame], manifest)` from the archive bytes. - `summarize_manifest()`: the manifest without its per-frame index, small enough for an artifact's `custom_metadata`. - `VISUAL_FRAMES_ZIP_MIME_TYPE`: the archive's MIME type, `application/zip`. NO_UNIT_GUIDE=Internal pack/unpack helper module for live visual frames PiperOrigin-RevId: 982888971
1 parent 3139855 commit bf745ac

6 files changed

Lines changed: 1069 additions & 25 deletions

File tree

‎src/google/adk/live/_live_llm_flow.py‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -155,15 +155,9 @@ async def send_to_model(
155155
types.LiveClientRealtimeInput(audio_stream_end=True) # type: ignore[arg-type]
156156
)
157157
elif live_request.blob:
158-
# Cache input audio chunks before flushing. The cache concatenates
159-
# every chunk into one audio file, so other blobs (e.g. video frames)
160-
# must stay out of it.
161-
if (
162-
run_config.save_live_blob
163-
and live_request.blob.mime_type
164-
and live_request.blob.mime_type.startswith('audio/')
165-
):
166-
realtime_cache_manager.cache_audio(
158+
# Cache input audio chunks before flushing.
159+
if run_config.save_live_blob:
160+
realtime_cache_manager.cache_blob(
167161
invocation_context, live_request.blob, cache_type='input'
168162
)
169163

@@ -316,24 +310,15 @@ def get_author_for_event(llm_response: LlmResponse) -> str:
316310
) as postprocess_agen:
317311
async for event in postprocess_agen:
318312
# Cache output audio chunks from model responses
319-
# TODO: support video data
320313
if (
321314
run_config.save_live_blob
322315
and event.content
323316
and event.content.parts
324317
):
325318
for part in event.content.parts:
326-
if (
327-
part.inline_data
328-
and part.inline_data.mime_type
329-
and part.inline_data.mime_type.startswith('audio/')
330-
):
331-
audio_blob = types.Blob(
332-
data=part.inline_data.data,
333-
mime_type=part.inline_data.mime_type,
334-
)
335-
realtime_cache_manager.cache_audio(
336-
invocation_context, audio_blob, cache_type='output'
319+
if part.inline_data:
320+
realtime_cache_manager.cache_blob(
321+
invocation_context, part.inline_data, cache_type='output'
337322
)
338323

339324
yield event

‎src/google/adk/live/_realtime_cache_manager.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,35 @@ def cache_audio(
119119
len(cache),
120120
)
121121

122+
def cache_blob(
123+
self,
124+
invocation_context: InvocationContext,
125+
blob: types.Blob,
126+
cache_type: str,
127+
) -> None:
128+
"""Cache an incoming user or outgoing model blob if it is audio.
129+
130+
The audio cache concatenates every chunk into one audio file, so other
131+
blobs (e.g. video frames) are skipped. Audio blobs are cached as copies,
132+
so later changes to the caller's blob do not change the cache.
133+
134+
Args:
135+
invocation_context: The current invocation context.
136+
blob: The blob to cache.
137+
cache_type: Type of data to cache, either 'input' or 'output'.
138+
139+
Raises:
140+
ValueError: If the blob is audio and cache_type is not 'input' or
141+
'output'.
142+
"""
143+
# TODO: cache image blobs once media caching lands.
144+
if blob.mime_type and blob.mime_type.startswith('audio/'):
145+
self.cache_audio(
146+
invocation_context,
147+
types.Blob(data=blob.data, mime_type=blob.mime_type),
148+
cache_type,
149+
)
150+
122151
async def flush_caches(
123152
self,
124153
invocation_context: InvocationContext,

0 commit comments

Comments
 (0)