marker_api/convert_both.py

#!/usr/bin/env python3
"""Full conversion of test docs under both configs."""
import os
import time
import requests

API_URL = os.environ.get("API_URL", "http://localhost:8001/v1/files/convert")

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_FILES = [
    os.path.join(SCRIPT_DIR, "test_files", "enisa", "enisa-international-strategy-2026.pdf"),
    os.path.join(SCRIPT_DIR, "test_files", "enisa", "nis-investments-2025.pdf"),
    os.path.join(SCRIPT_DIR, "test_files", "enisa", "enisa-nis360-2026.pdf"),
    os.path.join(SCRIPT_DIR, "test_files", "enisa", "enisa-stakeholder-strategy-2026-2028.pdf"),
    os.path.join(SCRIPT_DIR, "test_files", "enisa", "nis2-technical-implementation-guidance.pdf"),
    os.path.join(SCRIPT_DIR, "test_files", "downloaded", "attention.pdf"),
]

OUTPUT_BASE = os.path.join(SCRIPT_DIR, "test_files", "conversion_comparison")

def convert(filepath, use_llm):
    if not os.path.exists(filepath):
        return None
    files = {'file': (os.path.basename(filepath), open(filepath, 'rb'), 'application/pdf')}
    data = {
        'page_range': '0',
        'use_llm': 'true' if use_llm else 'false',
        'output_format': 'markdown',
    }
    start = time.time()
    resp = requests.post(API_URL, files=files, data=data, timeout=300)
    elapsed = time.time() - start
    if resp.status_code == 200:
        return {"output": resp.json().get("output", ""), "time": elapsed}
    print(f"  [ERROR] {resp.status_code}: {resp.text}")
    return None

def main():
    for use_llm in [False, True]:
        label = "llm" if use_llm else "non_llm"
        out_dir = os.path.join(OUTPUT_BASE, label)
        os.makedirs(out_dir, exist_ok=True)
        print(f"\n{'='*70}")
        print(f"  Converting with {'LLM' if use_llm else 'No LLM'} -> {out_dir}")
        print(f"{'='*70}")
        for path in TEST_FILES:
            basename = os.path.splitext(os.path.basename(path))[0]
            print(f"\n[{label}] {os.path.basename(path)}...")
            result = convert(path, use_llm=use_llm)
            if result and result["output"]:
                out_path = os.path.join(out_dir, f"{label}_{basename}.md")
                with open(out_path, "w", encoding="utf-8") as f:
                    f.write(result["output"])
                sz = len(result["output"])
                print(f"  -> {out_path}  ({sz} chars, {result['time']:.1f}s)")
            else:
                print(f"  -> FAILED")

    print(f"\nDone. Output in {OUTPUT_BASE}/")

if __name__ == "__main__":
    main()