fix(files): add OCR fallback for image-only PDFs

hindsight-api-slim/hindsight_api/engine/parsers/markitdown.py

@@ -33,6 +33,8 @@
     ".html",
     ".htm",
 }
+_PDF_EXTENSION = ".pdf"
+_PDF_OCR_DPI = 200
 
 
 @dataclass(frozen=True)
@@ -165,17 +167,38 @@ def _convert_sync(self, file_data: bytes, filename: str) -> str:
             tmp.write(file_data)
             tmp_path = tmp.name
 
+        pdf_ocr_attempted = False
         try:
             # Parse using markitdown, passing an explicit charset hint for text
             # files to avoid markitdown's sample-based (and crash-prone) detection.
             result = self._markitdown.convert(tmp_path, stream_info=self._utf8_stream_info(file_data, filename))
 
-            if not result or not result.text_content:
+            if (
+                not result
+                or not result.text_content
+                or (Path(filename).suffix.lower() == _PDF_EXTENSION and not result.text_content.strip())
+            ):
+                if Path(filename).suffix.lower() == _PDF_EXTENSION:
+                    pdf_ocr_attempted = True
+                    return self._ocr_pdf_sync(file_data, filename)
                 raise RuntimeError(f"No content extracted from '{filename}'")
 
             return result.text_content
 
         except Exception as e:
+            if Path(filename).suffix.lower() == _PDF_EXTENSION and not pdf_ocr_attempted:
+                # Some PDF converters raise instead of returning an empty result
+                # for image-only PDFs; give the OCR fallback the same opportunity.
+                try:
+                    return self._ocr_pdf_sync(file_data, filename)
+                except Exception as ocr_error:
+                    logger.error(
+                        "MarkItDown PDF parsing and OCR failed for %s: %s; OCR error: %s",
+                        filename,
+                        e,
+                        ocr_error,
+                    )
+                    raise RuntimeError(f"{e}; PDF OCR fallback failed: {ocr_error}") from ocr_error
             logger.error(f"Markitdown parsing failed for {filename}: {e}")
             raise RuntimeError(f"Failed to parse '{filename}': {e}") from e
 
@@ -186,6 +209,55 @@ def _convert_sync(self, file_data: bytes, filename: str) -> str:
             except Exception:
                 pass
 
+    def _ocr_pdf_sync(self, file_data: bytes, filename: str) -> str:
+        """Render an image-only PDF page-by-page and run the configured image OCR."""
+        if not self._ocr_enabled:
+            raise RuntimeError(
+                f"PDF '{filename}' appears to be scanned images without selectable text. "
+                "Configure MarkItDown OCR or choose an OCR-capable parser."
+            )
+
+        import pypdfium2 as pdfium
+
+        page_text: list[str] = []
+        try:
+            document = pdfium.PdfDocument(bytes(file_data))
+            try:
+                for page_number, page in enumerate(document):
+                    bitmap = page.render(scale=_PDF_OCR_DPI / 72)
+                    with tempfile.NamedTemporaryFile(suffix=f"-{page_number + 1}.png", delete=False) as tmp:
+                        image_path = tmp.name
+                    try:
+                        bitmap.to_pil().save(image_path, format="PNG")
+                        try:
+                            result = self._markitdown.convert(image_path)
+                            text = result.text_content if result else ""
+                            if text and text.strip():
+                                page_text.append(text.strip())
+                        except Exception as page_error:
+                            # A damaged page should not discard useful OCR from the rest
+                            # of the document. If every page fails, the empty-result error
+                            # below advances the configured parser fallback chain.
+                            logger.warning(
+                                "MarkItDown OCR failed for page %d of '%s': %s",
+                                page_number + 1,
+                                filename,
+                                page_error,
+                            )
+                    finally:
+                        try:
+                            Path(image_path).unlink()
+                        except OSError:
+                            pass
+            finally:
+                document.close()
+        except Exception as e:
+            raise RuntimeError(f"OCR failed for scanned PDF '{filename}': {e}") from e
+
+        if not page_text:
+            raise RuntimeError(f"No OCR content extracted from scanned PDF '{filename}'")
+        return "\n\n".join(page_text)
+
     @staticmethod
     def _utf8_stream_info(file_data: bytes, filename: str) -> "StreamInfo | None":
         """Return a UTF-8 charset hint for text files that decode cleanly as UTF-8.

hindsight-api-slim/pyproject.toml

@@ -75,6 +75,7 @@ dependencies = [
     # wheels (tracked upstream in BerriAI/litellm#31261).
     "litellm>=1.91.3,<1.92; sys_platform == 'darwin'",
     "markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4", # File to markdown conversion
+    "pypdfium2>=5.4.0", # Render scanned PDF pages for OCR fallback
     "obstore>=0.4.0", # S3/GCS/Azure object storage client (Rust-backed)
     "winloop>=0.1.0; sys_platform == 'win32'",
     "uvloop>=0.22.1; sys_platform != 'win32'",

hindsight-api-slim/tests/test_file_retain.py

@@ -5,6 +5,7 @@
 import asyncio
 import io
 import json
+import sys
 from datetime import datetime, timezone
 
 import pytest
@@ -519,6 +520,271 @@ def convert(self, path):
         await parser.convert(b"\x89PNG\r\n\x1a\n", "screenshot.png")
 
 
+@pytest.mark.asyncio
+async def test_markitdown_scanned_pdf_uses_page_ocr(monkeypatch):
+    """An empty PDF conversion is rasterized and OCR'd one page at a time."""
+    import markitdown
+
+    from hindsight_api.engine.parsers import MarkitdownParser
+
+    class FakeResult:
+        def __init__(self, text_content):
+            self.text_content = text_content
+
+    class FakeMarkItDown:
+        def __init__(self, **kwargs):
+            self.calls = []
+
+        def convert(self, path, **kwargs):
+            self.calls.append(path)
+            return FakeResult("" if path.endswith(".pdf") else f"OCR {len(self.calls)}")
+
+    class FakeBitmap:
+        def to_pil(self):
+            return type("Image", (), {"save": lambda self, path, format: None})()
+
+    class FakePage:
+        def render(self, **kwargs):
+            assert kwargs == {"scale": 200 / 72}
+            return FakeBitmap()
+
+    class FakeDocument:
+        def __iter__(self):
+            return iter([FakePage(), FakePage()])
+
+        def close(self):
+            pass
+
+    class FakePdfium:
+        @staticmethod
+        def PdfDocument(file_data):
+            assert file_data == b"pdf"
+            return FakeDocument()
+
+    monkeypatch.setattr(markitdown, "MarkItDown", FakeMarkItDown)
+    monkeypatch.setitem(sys.modules, "pypdfium2", FakePdfium)
+
+    parser = MarkitdownParser(
+        ocr_enabled=True, ocr_api_key="key", ocr_base_url="https://example.test/v1", ocr_model="vision"
+    )
+    assert await parser.convert(b"pdf", "scan.pdf") == "OCR 2\n\nOCR 3"
+
+
+@pytest.mark.asyncio
+async def test_markitdown_pdf_parse_error_uses_page_ocr(monkeypatch):
+    """A PDF converter exception still gets an OCR fallback attempt."""
+    import markitdown
+
+    from hindsight_api.engine.parsers import MarkitdownParser
+
+    class FakeResult:
+        def __init__(self, text_content):
+            self.text_content = text_content
+
+    class FakeMarkItDown:
+        def __init__(self, **kwargs):
+            self.image_calls = 0
+
+        def convert(self, path, **kwargs):
+            if path.endswith(".pdf"):
+                raise RuntimeError("pdf text extraction failed")
+            self.image_calls += 1
+            return FakeResult("Recovered from OCR")
+
+    class FakePage:
+        def render(self, **kwargs):
+            return type(
+                "Bitmap", (), {"to_pil": lambda self: type("Image", (), {"save": lambda self, path, format: None})()}
+            )()
+
+    class FakeDocument:
+        def __iter__(self):
+            return iter([FakePage()])
+
+        def close(self):
+            pass
+
+    class FakePdfium:
+        @staticmethod
+        def PdfDocument(file_data):
+            return FakeDocument()
+
+    monkeypatch.setattr(markitdown, "MarkItDown", FakeMarkItDown)
+    monkeypatch.setitem(sys.modules, "pypdfium2", FakePdfium)
+
+    parser = MarkitdownParser(
+        ocr_enabled=True,
+        ocr_api_key="key",
+        ocr_base_url="https://example.test/v1",
+        ocr_model="vision",
+    )
+    assert await parser.convert(b"pdf", "scan.pdf") == "Recovered from OCR"
+
+
+@pytest.mark.asyncio
+async def test_markitdown_scanned_pdf_without_ocr_has_actionable_error(monkeypatch):
+    """A scanned PDF without OCR explains the remediation and can trigger outer fallback."""
+    import markitdown
+
+    from hindsight_api.engine.parsers import MarkitdownParser
+
+    class FakeResult:
+        text_content = ""
+
+    class FakeMarkItDown:
+        def __init__(self, **kwargs):
+            pass
+
+        def convert(self, path, **kwargs):
+            return FakeResult()
+
+    monkeypatch.setattr(markitdown, "MarkItDown", FakeMarkItDown)
+    parser = MarkitdownParser()
+    with pytest.raises(RuntimeError, match="scanned images without selectable text"):
+        await parser.convert(b"pdf", "scan.pdf")
+
+
+@pytest.mark.asyncio
+async def test_markitdown_scanned_pdf_skips_failed_ocr_page(monkeypatch):
+    """One damaged page does not discard OCR content extracted from other pages."""
+    import markitdown
+
+    from hindsight_api.engine.parsers import MarkitdownParser
+
+    class FakeResult:
+        def __init__(self, text_content):
+            self.text_content = text_content
+
+    class FakeMarkItDown:
+        def __init__(self, **kwargs):
+            self.image_calls = 0
+
+        def convert(self, path, **kwargs):
+            if path.endswith(".pdf"):
+                return FakeResult("")
+            self.image_calls += 1
+            if self.image_calls == 1:
+                raise RuntimeError("damaged page")
+            return FakeResult("Recovered page")
+
+    class FakePage:
+        def render(self, **kwargs):
+            return type(
+                "Bitmap", (), {"to_pil": lambda self: type("Image", (), {"save": lambda self, path, format: None})()}
+            )()
+
+    class FakeDocument:
+        def __iter__(self):
+            return iter([FakePage(), FakePage()])
+
+        def close(self):
+            pass
+
+    class FakePdfium:
+        @staticmethod
+        def PdfDocument(file_data):
+            return FakeDocument()
+
+    monkeypatch.setattr(markitdown, "MarkItDown", FakeMarkItDown)
+    monkeypatch.setitem(sys.modules, "pypdfium2", FakePdfium)
+
+    parser = MarkitdownParser(
+        ocr_enabled=True,
+        ocr_api_key="key",
+        ocr_base_url="https://example.test/v1",
+        ocr_model="vision",
+    )
+    assert await parser.convert(b"pdf", "scan.pdf") == "Recovered page"
+
+
+@pytest.mark.asyncio
+async def test_markitdown_scanned_pdf_ocr_runs_once_on_failure(monkeypatch):
+    """A failed PDF OCR fallback is not retried by the parser's error wrapper."""
+    import markitdown
+
+    from hindsight_api.engine.parsers import MarkitdownParser
+
+    class FakeResult:
+        text_content = ""
+
+    pdf_calls = 0
+
+    class FakeMarkItDown:
+        def __init__(self, **kwargs):
+            pass
+
+        def convert(self, path, **kwargs):
+            if path.endswith(".pdf"):
+                nonlocal pdf_calls
+                pdf_calls += 1
+                return FakeResult()
+            raise RuntimeError("OCR unavailable")
+
+    class FakePage:
+        def render(self, **kwargs):
+            return type(
+                "Bitmap", (), {"to_pil": lambda self: type("Image", (), {"save": lambda self, path, format: None})()}
+            )()
+
+    class FakeDocument:
+        def __iter__(self):
+            return iter([FakePage()])
+
+        def close(self):
+            pass
+
+    class FakePdfium:
+        @staticmethod
+        def PdfDocument(file_data):
+            return FakeDocument()
+
+    monkeypatch.setattr(markitdown, "MarkItDown", FakeMarkItDown)
+    monkeypatch.setitem(sys.modules, "pypdfium2", FakePdfium)
+
+    parser = MarkitdownParser(
+        ocr_enabled=True,
+        ocr_api_key="key",
+        ocr_base_url="https://example.test/v1",
+        ocr_model="vision",
+    )
+    with pytest.raises(RuntimeError, match="Failed to parse"):
+        await parser.convert(b"pdf", "scan.pdf")
+    assert pdf_calls == 1
+
+
+@pytest.mark.asyncio
+async def test_scanned_pdf_ocr_failure_advances_parser_chain():
+    """A failed MarkItDown PDF OCR attempt advances to the next configured parser."""
+    from hindsight_api.engine.parsers import FileParserRegistry
+    from hindsight_api.engine.parsers.base import FileParser
+
+    class EmptyMarkitdownParser(FileParser):
+        async def convert(self, file_data: bytes, filename: str) -> str:
+            raise RuntimeError("No OCR content extracted from scanned PDF")
+
+        def name(self) -> str:
+            return "markitdown"
+
+    class OcrFallbackParser(FileParser):
+        async def convert(self, file_data: bytes, filename: str) -> str:
+            return "Fallback OCR text"
+
+        def name(self) -> str:
+            return "llama_parse"
+
+    registry = FileParserRegistry()
+    registry.register(EmptyMarkitdownParser())
+    registry.register(OcrFallbackParser())
+
+    result = await registry.convert_with_fallback(
+        parsers=["markitdown", "llama_parse"],
+        file_data=b"pdf",
+        filename="scan.pdf",
+    )
+    assert result.content == "Fallback OCR text"
+    assert result.parser_name == "llama_parse"
+
+
 def test_markitdown_converter_can_enable_ocr(monkeypatch):
     """When enabled, Markitdown receives an OpenAI-compatible client, model, and OCR prompt."""
     import markitdown