first commit

app/core/__init__.py

@@ -0,0 +1,4 @@
+"""Embedding service — uses Ollama qwen3-embedding:8b."""
+from app.core.embedding_engine import get_embedding_sync
+
+get_embedding = get_embedding_sync

app/core/embedding_engine.py

@@ -0,0 +1,79 @@
+"""Embedding generation via Ollama's qwen3-embedding:8b model."""
+import hashlib
+import numpy as np
+
+
+def get_embedding_sync(text: str, ollama_url: str = "http://10.0.1.127:11434") -> list[float]:
+    """Synchronous embedding call via Ollama /qwen3-embedding:8b."""
+    try:
+        import httpx
+        with httpx.Client(timeout=30) as client:
+            resp = client.post(
+                f"{ollama_url}/api/embed",
+                json={"model": "qwen3-embedding:8b", "input": text},
+            )
+            resp.raise_for_status()
+            data = resp.json()
+            vectors = data.get("embeddings", [])
+            if vectors:
+                # Ollama may return multiple inputs; use first
+                emb = vectors[0] if isinstance(vectors[0], list) else vectors
+                return emb
+    except Exception:
+        pass
+    # Fallback to deterministic feature vector if Ollama unavailable
+    return _hash_embedding(text)
+
+
+def get_embedding(text: str, ollama_url: str = "http://10.0.1.127:11434") -> list[float]:
+    """Generate an embedding — calls Ollama qwen3-embedding:8b."""
+    return get_embedding_sync(text, ollama_url)
+
+
+async def get_embedding_async(text: str, ollama_url: str = "http://10.0.1.127:11434") -> list[float]:
+    """Async embedding call via Ollama /qwen3-embedding:8b."""
+    try:
+        import httpx
+        async with httpx.AsyncClient(timeout=30) as client:
+            resp = await client.post(
+                f"{ollama_url}/api/embed",
+                json={"model": "qwen3-embedding:8b", "input": text},
+            )
+            resp.raise_for_status()
+            data = resp.json()
+            vectors = data.get("embeddings", [])
+            if vectors:
+                emb = vectors[0] if isinstance(vectors[0], list) else vectors
+                return emb
+    except Exception:
+        pass
+    return _hash_embedding(text)
+
+
+def _hash_embedding(text: str) -> list[float]:
+    """Deterministic 4096-dim feature vector fallback (no Ollama needed)."""
+    feature_dim = 4096
+    vec = np.zeros(feature_dim, dtype=np.float32)
+    for n in [1, 2, 3, 4]:
+        tokens = [text[i:i+n] for i in range(len(text)-n+1)]
+        for token in tokens[:200]:
+            h = hashlib.md5(token.encode()).hexdigest()
+            for i in range(0, 12, 3):
+                val = (int(h[i:i+2], 16) - 128) / 128.0
+                feature_idx = (int(h[i+2:i+4], 16) * 37) % feature_dim
+                vec[feature_idx] += val
+    norm = np.linalg.norm(vec)
+    if norm > 0:
+        vec /= norm
+    return vec.tolist()
+
+
+def compute_similarity(vec1: list[float], vec2: list[float]) -> float:
+    """Cosine similarity between two vectors."""
+    v1 = np.array(vec1, dtype=np.float32)
+    v2 = np.array(vec2, dtype=np.float32)
+    if v1.shape[0] != v2.shape[0]:
+        return 0.0
+    v1 /= np.linalg.norm(v1) + 1e-8
+    v2 /= np.linalg.norm(v2) + 1e-8
+    return float(np.dot(v1, v2))

app/core/processor.py

@@ -0,0 +1,110 @@
+"""Marker/OCR integration for document processing."""
+from __future__ import annotations
+
+import json
+import asyncio
+import io
+import re
+from pathlib import Path
+from typing import Any
+import httpx
+
+from app.config import get_settings
+import app.db.database as _database_mod
+
+
+class MarkerProcessor:
+    """Handles OCR via Marker API with polygon output."""
+
+    def __init__(self):
+        self.settings = get_settings()
+
+    async def process_pdf(self, file_data: bytes, filename: str) -> dict[str, Any]:
+        """Send PDF to Marker API for OCR with polygon extraction."""
+        async with httpx.AsyncClient(timeout=300) as client:
+            resp = await client.post(
+                f"{self.settings.marker_api_url}/convert",
+                files={"file": (filename, file_data, "application/pdf")},
+            )
+            resp.raise_for_status()
+            return resp.json()
+
+    async def process_pdf_url(self, pdf_url: str, filename: str) -> dict[str, Any]:
+        """Process PDF from URL via Marker API."""
+        async with httpx.AsyncClient(timeout=300) as client:
+            resp = await client.post(
+                f"{self.settings.marker_api_url}/convert",
+                json={"pdf_url": pdf_url},
+            )
+            resp.raise_for_status()
+            return resp.json()
+
+    async def parse_marker_json(self, marker_output: dict, doc_id: str) -> list[dict]:
+        """Parse Marker JSON output into vectorized chunks with polygon data."""
+        pages = marker_output.get("pages", [])
+        chunks = []
+        
+        for page in pages:
+            page_num = page.get("meta", {}).get("page_num", page.get("page", 0))
+            text_lines = page.get("text_lines", [])
+            
+            for block_idx, tl in enumerate(text_lines):
+                content = tl.get("text", "")
+                polygon = tl.get("bbox") or tl.get("polygon")
+                block_type = tl.get("type", "text")
+                
+                if not content or not isinstance(content, str) or not content.strip():
+                    continue
+                
+                chunks.append({
+                    "content": content.strip(),
+                    "page_num": page_num,
+                    "block_index": block_idx,
+                    "polygon": polygon,
+                    "chunk_type": block_type,
+                })
+
+        if chunks:
+            await _database_mod.db.batch_chunk(doc_id, chunks)
+        
+        return chunks
+
+    async def process_document_file(
+        self, file_data: bytes, filename: str, doc_id: str
+    ) -> dict[str, Any]:
+        """Process document file and return structured result."""
+        ext = Path(filename).suffix.lower()
+        
+        if ext == ".pdf":
+            result = await self.process_pdf(file_data, filename)
+            pages_data = result.get("pages", [])
+            
+            chunks = await self.parse_marker_json({"pages": pages_data}, doc_id)
+            return {
+                "success": result.get("success", True),
+                "doc_id": doc_id,
+                "page_count": result.get("page_count", len(pages_data)),
+                "chunks": len(chunks),
+                "ocr_model": result.get("ocr_model", "deepseek-ocr"),
+            }
+        
+        elif ext in (".txt", ".pdf", ".md"):
+            text = file_data.decode("utf-8", errors="replace")
+            paragraphs = re.split(r'\n\s*\n', text)
+            chunks = []
+            for i, para in enumerate(paragraphs):
+                if len(para.strip()) > 20:
+                    chunks.append({
+                        "content": para.strip(),
+                        "page_num": 0,
+                        "block_index": i,
+                        "polygon": None,
+                        "chunk_type": "text",
+                    })
+            
+            if chunks:
+                await _database_mod.db.batch_chunk(doc_id, chunks)
+            
+            return {"success": True, "doc_id": doc_id, "chunks": len(chunks), "page_count": 1}
+
+        return {"success": False, "error": f"Unsupported file type: {ext}"}