oval
33997778f6
- Add .gitignore excluding .env, __pycache__, test_files (copyrighted PDFs), session logs, marker/ submodule, and other generated/private files - Add .env with local config (gitignored); expand .env.example with all supported environment variables - docker-compose.yml: all hardcoded values now use default pattern - Containerfile: install marker-pdf from GitHub instead of local submodule; fix AMD_COMPUTE default to false; sanitize default OLLAMA_HOST - Remove marker/ submodule from git tracking (keep local checkout via .gitignore) - Remove session transcript and copyrighted ENISA PDFs from git tracking - Sanitize local IPs in README, kubernetes deployment, deploy-podman.sh - Fix hardcoded paths in convert_both.py and compare_conversions.py - Update requirements.txt to reference GitHub instead of local --editable
63 lines
2.5 KiB
Python
63 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Full conversion of test docs under both configs."""
|
|
import os
|
|
import time
|
|
import requests
|
|
|
|
API_URL = os.environ.get("API_URL", "http://localhost:8001/v1/files/convert")
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
TEST_FILES = [
|
|
os.path.join(SCRIPT_DIR, "test_files", "enisa", "enisa-international-strategy-2026.pdf"),
|
|
os.path.join(SCRIPT_DIR, "test_files", "enisa", "nis-investments-2025.pdf"),
|
|
os.path.join(SCRIPT_DIR, "test_files", "enisa", "enisa-nis360-2026.pdf"),
|
|
os.path.join(SCRIPT_DIR, "test_files", "enisa", "enisa-stakeholder-strategy-2026-2028.pdf"),
|
|
os.path.join(SCRIPT_DIR, "test_files", "enisa", "nis2-technical-implementation-guidance.pdf"),
|
|
os.path.join(SCRIPT_DIR, "test_files", "downloaded", "attention.pdf"),
|
|
]
|
|
|
|
OUTPUT_BASE = os.path.join(SCRIPT_DIR, "test_files", "conversion_comparison")
|
|
|
|
def convert(filepath, use_llm):
|
|
if not os.path.exists(filepath):
|
|
return None
|
|
files = {'file': (os.path.basename(filepath), open(filepath, 'rb'), 'application/pdf')}
|
|
data = {
|
|
'page_range': '0',
|
|
'use_llm': 'true' if use_llm else 'false',
|
|
'output_format': 'markdown',
|
|
}
|
|
start = time.time()
|
|
resp = requests.post(API_URL, files=files, data=data, timeout=300)
|
|
elapsed = time.time() - start
|
|
if resp.status_code == 200:
|
|
return {"output": resp.json().get("output", ""), "time": elapsed}
|
|
print(f" [ERROR] {resp.status_code}: {resp.text}")
|
|
return None
|
|
|
|
def main():
|
|
for use_llm in [False, True]:
|
|
label = "llm" if use_llm else "non_llm"
|
|
out_dir = os.path.join(OUTPUT_BASE, label)
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
print(f"\n{'='*70}")
|
|
print(f" Converting with {'LLM' if use_llm else 'No LLM'} -> {out_dir}")
|
|
print(f"{'='*70}")
|
|
for path in TEST_FILES:
|
|
basename = os.path.splitext(os.path.basename(path))[0]
|
|
print(f"\n[{label}] {os.path.basename(path)}...")
|
|
result = convert(path, use_llm=use_llm)
|
|
if result and result["output"]:
|
|
out_path = os.path.join(out_dir, f"{label}_{basename}.md")
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
f.write(result["output"])
|
|
sz = len(result["output"])
|
|
print(f" -> {out_path} ({sz} chars, {result['time']:.1f}s)")
|
|
else:
|
|
print(f" -> FAILED")
|
|
|
|
print(f"\nDone. Output in {OUTPUT_BASE}/")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|