Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
6fc558e
feat: dockerized e2e ai pipeline, pgvector relationships, and ocr fixes
Prakhar-Sethi012 Jul 9, 2026
5f47fb2
Merge branch 'Changes_Prakhar' of https://github.com/Prakhar-Sethi012…
Prakhar-Sethi012 Jul 9, 2026
13a51ed
chore: added alembic migration for pgvector and relationship tables
Prakhar-Sethi012 Jul 10, 2026
cf5c5be
fix: removed duplicate migration file to prevent table recreation
Prakhar-Sethi012 Jul 11, 2026
cfe265d
perf: implemented singleton model cache to eliminate indexing cold-st…
Prakhar-Sethi012 Jul 11, 2026
614f69c
Merge branch 'dev' into Changes_Prakhar
Prakhar-Sethi012 Jul 13, 2026
9f26515
feat: updated LLM expansion prompt
Prakhar-Sethi012 Jul 14, 2026
88657ff
Merge branch 'GDGVIT:dev' into Changes_Prakhar
Prakhar-Sethi012 Jul 14, 2026
ab2782a
No change
Prakhar-Sethi012 Aug 6, 2026
25182cf
feat: upgrade to background processing with redis and celery
Prakhar-Sethi012 Aug 15, 2026
e420cfc
fix: resolved merge conflict in files route
Prakhar-Sethi012 Aug 15, 2026
8a2ef54
feat/fix: finalize async AI processing architecture and patch critica…
Prakhar-Sethi012 Aug 16, 2026
8b30462
Merge branch 'GDGVIT:dev' into Changes_Prakhar
Prakhar-Sethi012 Aug 21, 2026
fdc5b43
Merge branch 'GDGVIT:dev' into Changes_Prakhar
Prakhar-Sethi012 Aug 22, 2026
237a86c
fix(backend): stabilize Jina CLIP upgrade, resolve build failures, an…
Prakhar-Sethi012 Aug 22, 2026
9324586
Merge branch 'GDGVIT:dev' into Changes_Prakhar
Prakhar-Sethi012 Aug 24, 2026
256d767
fix(multimodal): enable native image encoding for jina-clip-v1
Prakhar-Sethi012 Aug 24, 2026
2dcb9e4
Merge branch 'Changes_Prakhar' of https://github.com/Prakhar-Sethi012…
Prakhar-Sethi012 Aug 24, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions app/processing/embeddings.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,27 @@ def embed_chunks(

# 2. Embed image chunks
if images:
image_vectors = model.encode(
images=images,
batch_size=batch_size,
convert_to_numpy=True,
normalize_embeddings=True,
)
# Dynamically hunt down the underlying Jina CLIP AutoModel inside the wrapper
jina_hf_model = next((m for m in model.modules() if hasattr(m, "encode_image")), None)

if not jina_hf_model:
raise RuntimeError("Could not find the Jina Vision encoder inside the model!")

import torch
import numpy as np

# Disable gradients to save RAM during the forward pass
with torch.no_grad():
image_vectors = jina_hf_model.encode_image(images)

# Convert to a numpy array if it returned a PyTorch tensor
if isinstance(image_vectors, torch.Tensor):
image_vectors = image_vectors.cpu().numpy()

# Normalize the vectors to ensure cosine similarity matches the text vectors
norms = np.linalg.norm(image_vectors, axis=1, keepdims=True)
image_vectors = image_vectors / np.maximum(norms, 1e-12)

for list_pos, chunk_idx in enumerate(image_indices):
chunks[chunk_idx]["embedding"] = image_vectors[list_pos].tolist()
del chunks[chunk_idx]["content_image"]
Expand Down
36 changes: 24 additions & 12 deletions app/processing/extractors.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import logging
import zipfile
from pathlib import Path
from typing import TypedDict
from typing import TypedDict, Any

from .ocr import ocr_image_bytes

Expand All @@ -38,9 +38,10 @@
# Shared type
# ---------------------------------------------------------------------------

class ChunkData(TypedDict):
class ChunkData(TypedDict, total=False):
chunk_index: int
content_text: str
content_image: Any # Accepts the PIL Image object for Jina CLIP
# embedding and keyword_tokens intentionally absent here;
# they are injected by the embedding pipeline.

Expand Down Expand Up @@ -396,20 +397,31 @@ def extract_xlsx(path: Path) -> list[ChunkData]:


# ---------------------------------------------------------------------------
# Standalone image extractor (OCR only)
# Standalone image extractor (OCR AND Vision Encoding)
# ---------------------------------------------------------------------------

def extract_image(path: Path) -> list[ChunkData]:
"""Single chunk: full OCR output of the image file."""
img_bytes = path.read_bytes()
ocr_text = ocr_image_bytes(img_bytes)
if not ocr_text:
"""Single chunk: full OCR output AND the PIL Image for Jina Vision."""
from PIL import Image

# 1. Load the raw image for the Jina CLIP Vision Encoder
try:
# Convert to RGB to safely drop alpha channels (transparency)
img_obj = Image.open(path).convert("RGB")
except Exception as exc:
logger.error("[image] Failed to open image %s: %s", path, exc)
return []
# Images rarely exceed one chunk but run through rolling chunker for safety
return [
ChunkData(chunk_index=i, content_text=c)
for i, c in enumerate(_rolling_chunks(_clean(ocr_text)))
]

# 2. Extract OCR text (optional, just in case there is text in the photo)
img_bytes = path.read_bytes()
ocr_text = ocr_image_bytes(img_bytes) or ""

# 3. Return a single chunk containing BOTH the text and the image!
return [{
"chunk_index": 0,
"content_text": ocr_text,
"content_image": img_obj
}]


# ---------------------------------------------------------------------------
Expand Down
Loading