oval
df8d0d6b74
- docker-compose.yml: default TORCH_DEVICE=cpu, PORT=8001, named volumes for model cache (marker-cache) and results (marker-results) with :Z flag - convert_both.py: API_URL defaults to port 8001, overridable via env - TODO.md: reflect final decisions (iGPU 30x slower than CPU, --privileged required for GPU, CPU mode is default)
62 lines
2.4 KiB
Python
62 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Full conversion of test docs under both configs."""
|
|
import os
|
|
import time
|
|
import requests
|
|
|
|
API_URL = os.environ.get("API_URL", "http://localhost:8001/v1/files/convert")
|
|
|
|
TEST_FILES = [
|
|
"/home/oval/marker-api/test_files/enisa/enisa-international-strategy-2026.pdf",
|
|
"/home/oval/marker-api/test_files/enisa/nis-investments-2025.pdf",
|
|
"/home/oval/marker-api/test_files/enisa/enisa-nis360-2026.pdf",
|
|
"/home/oval/marker-api/test_files/enisa/enisa-stakeholder-strategy-2026-2028.pdf",
|
|
"/home/oval/marker-api/test_files/enisa/nis2-technical-implementation-guidance.pdf",
|
|
"/home/oval/marker-api/test_files/downloaded/attention.pdf",
|
|
]
|
|
|
|
OUTPUT_BASE = "/home/oval/marker-api/test_files/conversion_comparison"
|
|
|
|
def convert(filepath, use_llm):
|
|
if not os.path.exists(filepath):
|
|
return None
|
|
files = {'file': (os.path.basename(filepath), open(filepath, 'rb'), 'application/pdf')}
|
|
data = {
|
|
'page_range': '0',
|
|
'use_llm': 'true' if use_llm else 'false',
|
|
'output_format': 'markdown',
|
|
}
|
|
start = time.time()
|
|
resp = requests.post(API_URL, files=files, data=data, timeout=300)
|
|
elapsed = time.time() - start
|
|
if resp.status_code == 200:
|
|
return {"output": resp.json().get("output", ""), "time": elapsed}
|
|
print(f" [ERROR] {resp.status_code}: {resp.text}")
|
|
return None
|
|
|
|
def main():
|
|
for use_llm in [False, True]:
|
|
label = "llm" if use_llm else "non_llm"
|
|
out_dir = os.path.join(OUTPUT_BASE, label)
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
print(f"\n{'='*70}")
|
|
print(f" Converting with {'LLM' if use_llm else 'No LLM'} -> {out_dir}")
|
|
print(f"{'='*70}")
|
|
for path in TEST_FILES:
|
|
basename = os.path.splitext(os.path.basename(path))[0]
|
|
print(f"\n[{label}] {os.path.basename(path)}...")
|
|
result = convert(path, use_llm=use_llm)
|
|
if result and result["output"]:
|
|
out_path = os.path.join(out_dir, f"{label}_{basename}.md")
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
f.write(result["output"])
|
|
sz = len(result["output"])
|
|
print(f" -> {out_path} ({sz} chars, {result['time']:.1f}s)")
|
|
else:
|
|
print(f" -> FAILED")
|
|
|
|
print(f"\nDone. Output in {OUTPUT_BASE}/")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|