From 0c4bcdc7d22426943f6aea1305f15cc72c2c490a Mon Sep 17 00:00:00 2001 From: oval Date: Sun, 7 Jun 2026 22:39:28 +0200 Subject: [PATCH] Switch from Vega20 (gfx906) base to ROCm 7.2 for gfx1151; set local ollama :11435 --- Containerfile | 21 ++++++++++++--------- TODO.md | 23 +++++++++++++++++++++++ app.py | 3 +-- docker-compose.yml | 11 ++++++++--- 4 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 TODO.md diff --git a/Containerfile b/Containerfile index a093b2b..37f5a83 100644 --- a/Containerfile +++ b/Containerfile @@ -1,9 +1,10 @@ -# ===== containerfile (podman & docker) for marker-api on powermac ===== +# ===== containerfile (podman & docker) for marker-api on this host ===== # -# Targets 2x AMD Vega 20 (GFX906) with ROCm using gfx906-patched PyTorch. -# deepseek-ocr and GPT services are on the same host at :8082 / :8080. +# Targets AMD Radeon 8060S (GFX1151) with ROCm 7.2 native PyTorch 2.9.1. +# Ollama for OCR is on localhost:11435. +# Removed gfx906 base (Vega20) — now uses rocm/pytorch which natively supports gfx1151. -FROM mixa3607/pytorch-gfx906:v2.7.1-rocm-6.3.3 +FROM rocm/pytorch:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.9.1 # install system deps & remove system-installed python3-blinker to avoid pip upgrade conflicts RUN apt-get update && \ @@ -23,9 +24,11 @@ WORKDIR /app # ---- deps ---- COPY marker/ /app/marker/ -# We use --break-system-packages and make sure we do NOT use --ignore-installed so that pre-installed -# gfx906 patched torch, torchvision, and torchaudio are preserved and NOT overwritten by PyPI releases. -RUN pip install --no-cache-dir --break-system-packages -e "/app/marker[full]" && \ +# rocm/pytorch base already includes ROCm torch 2.9.1. +# Use --extra-index-url so pip prefers ROCm builds if torch gets reinstalled. +RUN pip install --no-cache-dir --break-system-packages \ + --extra-index-url https://download.pytorch.org/whl/rocm7.2 \ + -e "/app/marker[full]" && \ pip install --no-cache-dir --break-system-packages flask gunicorn # ---- font: download GoNotoCurrent-Regular.ttf at build time ---- @@ -47,7 +50,7 @@ ENV \ PYTHONDONTWRITEBYTECODE=1 \ MARKER_OUTPUT_DIR=/app/conversion_results \ UPLOAD_DIR=/app/uploads \ - OLLAMA_HOST=http://localhost:8082 \ + OLLAMA_HOST=http://localhost:11435 \ DEESEEK_OCR_MODEL=deepseek-ocr \ AMD_COMPUTE=true \ TORCH_DEVICE= \ @@ -56,7 +59,7 @@ ENV \ HOST=0.0.0.0 \ LLM_SERVICE=marker.services.openai.OpenAIService \ USE_LLM=false \ - OPENAI_BASE_URL=http://localhost:8082/v1 \ + OPENAI_BASE_URL=http://localhost:11435/v1 \ OPENAI_MODEL= \ GUNICORN_WORKERS=1 diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..3fe4052 --- /dev/null +++ b/TODO.md @@ -0,0 +1,23 @@ +# TODO: marker-api Local Deployment + +## Priority 1: Verify local GPU compute works +- [x] PyTorch ROCm sees gfx1151 on host +- [x] Ollama runs locally on port 11435 with ROCm +- [x] Start marker-api container with rocm/pytorch base +- [ ] Verify `/health` shows `torch_device: cuda` +- [ ] Run `convert_both.py` — ensure `non_llm` outputs are produced + +## Priority 2: Fix LLM correction pipeline +- [ ] Remove Vega-specific `HSA_OVERRIDE_GFX_VERSION` from `app.py` ✅ +- [ ] Confirm `build_options()` fix (rm `use_llm` dup) resolves Click warnings ✅ +- [ ] Run `convert_both.py` and verify `non_llm` vs `llm` outputs differ +- [ ] Confirm LLM requests reach local ollama container + +## Priority 3: Hardening +- [ ] Add health check for LLM connectivity +- [ ] Ensure `marker` user can write to `conversion_results` directory +- [ ] Expose LLM metrics/status in /health endpoint + +## Known issues +- GPUs are considered "iGPU" by ollama (system memory, not VRAM) +- `LD_LIBRARY_PATH` in entrypoint may need /opt/rocm/lib from the rocm/pytorch image diff --git a/app.py b/app.py index 3953eb1..b16f7de 100644 --- a/app.py +++ b/app.py @@ -22,7 +22,7 @@ from marker.providers.registry import load_extensions from marker.settings import settings as marker_settings # Ollama / OCR fallback configuration -OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://10.0.1.127:11434") +OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11435") DEESEEK_OCR_MODEL = os.environ.get("DEESEEK_OCR_MODEL", "deepseek-ocr") AMD_COMPUTE = os.environ.get("AMD_COMPUTE", "false").lower() in ("true", "1", "yes") @@ -50,7 +50,6 @@ def _configure_env(): if AMD_COMPUTE and not TORCH_DEVICE: os.environ["TORCH_DEVICE"] = "cuda" os.environ["TORCH_DEVICE_MODEL"] = "cuda" - os.environ["HSA_OVERRIDE_GFX_VERSION"] = "9.0.6" if MODEL_DTYPE == "bfloat16": os.environ["MODEL_DTYPE"] = "bfloat16" if torch_device_override := os.environ.get("TORCH_DEVICE"): diff --git a/docker-compose.yml b/docker-compose.yml index db21f54..f2420c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,11 +10,16 @@ services: env_file: - .env # optional; remove if no .env exists environment: - - OLLAMA_HOST=${OLLAMA_HOST:-http://10.0.1.127:11434} + - OLLAMA_HOST=${OLLAMA_HOST:-http://localhost:11435} - DEESEEK_OCR_MODEL=${DEESEEK_OCR_MODEL:-deepseek-ocr} - - AMD_COMPUTE=${AMD_COMPUTE:-false} - - TORCH_DEVICE=${TORCH_DEVICE:-} + - AMD_COMPUTE=${AMD_COMPUTE:-true} + - TORCH_DEVICE=${TORCH_DEVICE:-cuda} - MODEL_DTYPE=${MODEL_DTYPE:-float32} + - LLM_SERVICE=marker.services.openai.OpenAIService + - USE_LLM=false + - OPENAI_BASE_URL=http://localhost:11435/v1 + - OPENAI_API_KEY=not-needed + - OPENAI_MODEL= - PORT=8000 - HOST=0.0.0.0 - GUNICORN_WORKERS=${GUNICORN_WORKERS:-2}