Initial marker-api benchmark setup: Containerfile, app.py, convert_both.py, compare_conversions.py

2026-06-05 13:26:25 +02:00
commit 460fc0003b
33 changed files with 58290 additions and 0 deletions
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Full conversion of test docs under both configs."""
+import os
+import time
+import requests
+
+API_URL = "http://localhost:8000/v1/files/convert"
+
+TEST_FILES = [
+    "/home/oval/marker-api/test_files/enisa/enisa-international-strategy-2026.pdf",
+    "/home/oval/marker-api/test_files/enisa/nis-investments-2025.pdf",
+    "/home/oval/marker-api/test_files/enisa/enisa-nis360-2026.pdf",
+    "/home/oval/marker-api/test_files/enisa/enisa-stakeholder-strategy-2026-2028.pdf",
+    "/home/oval/marker-api/test_files/enisa/nis2-technical-implementation-guidance.pdf",
+    "/home/oval/marker-api/test_files/downloaded/attention.pdf",
+]
+
+OUTPUT_BASE = "/home/oval/marker-api/test_files/conversion_comparison"
+
+def convert(filepath, use_llm):
+    if not os.path.exists(filepath):
+        return None
+    files = {'file': (os.path.basename(filepath), open(filepath, 'rb'), 'application/pdf')}
+    data = {
+        'page_range': '0',
+        'use_llm': 'true' if use_llm else 'false',
+        'output_format': 'markdown',
+    }
+    start = time.time()
+    resp = requests.post(API_URL, files=files, data=data, timeout=300)
+    elapsed = time.time() - start
+    if resp.status_code == 200:
+        return {"output": resp.json().get("output", ""), "time": elapsed}
+    print(f"  [ERROR] {resp.status_code}: {resp.text}")
+    return None
+
+def main():
+    for use_llm in [False, True]:
+        label = "llm" if use_llm else "non_llm"
+        out_dir = os.path.join(OUTPUT_BASE, label)
+        os.makedirs(out_dir, exist_ok=True)
+        print(f"\n{'='*70}")
+        print(f"  Converting with {'LLM' if use_llm else 'No LLM'} -> {out_dir}")
+        print(f"{'='*70}")
+        for path in TEST_FILES:
+            basename = os.path.splitext(os.path.basename(path))[0]
+            print(f"\n[{label}] {os.path.basename(path)}...")
+            result = convert(path, use_llm=use_llm)
+            if result and result["output"]:
+                out_path = os.path.join(out_dir, f"{label}_{basename}.md")
+                with open(out_path, "w", encoding="utf-8") as f:
+                    f.write(result["output"])
+                sz = len(result["output"])
+                print(f"  -> {out_path}  ({sz} chars, {result['time']:.1f}s)")
+            else:
+                print(f"  -> FAILED")
+
+    print(f"\nDone. Output in {OUTPUT_BASE}/")
+
+if __name__ == "__main__":
+    main()