Switch from Vega20 (gfx906) base to ROCm 7.2 for gfx1151; set local ollama :11435

Containerfile

@@ -1,9 +1,10 @@
-# ===== containerfile (podman & docker) for marker-api on powermac  =====
+# ===== containerfile (podman & docker) for marker-api on this host  =====
 #
-# Targets 2x AMD Vega 20 (GFX906) with ROCm using gfx906-patched PyTorch.
+# Targets AMD Radeon 8060S (GFX1151) with ROCm 7.2 native PyTorch 2.9.1.
-# deepseek-ocr and GPT services are on the same host at :8082 / :8080.
+# Ollama for OCR is on localhost:11435.
+# Removed gfx906 base (Vega20) — now uses rocm/pytorch which natively supports gfx1151.
 
-FROM mixa3607/pytorch-gfx906:v2.7.1-rocm-6.3.3
+FROM rocm/pytorch:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.9.1
 
 # install system deps & remove system-installed python3-blinker to avoid pip upgrade conflicts
 RUN apt-get update && \
@@ -23,9 +24,11 @@ WORKDIR /app
 # ---- deps ----
 COPY marker/ /app/marker/
 
-# We use --break-system-packages and make sure we do NOT use --ignore-installed so that pre-installed
+# rocm/pytorch base already includes ROCm torch 2.9.1.
-# gfx906 patched torch, torchvision, and torchaudio are preserved and NOT overwritten by PyPI releases.
+# Use --extra-index-url so pip prefers ROCm builds if torch gets reinstalled.
-RUN pip install --no-cache-dir --break-system-packages -e "/app/marker[full]" && \
+RUN pip install --no-cache-dir --break-system-packages \
+    --extra-index-url https://download.pytorch.org/whl/rocm7.2 \
+    -e "/app/marker[full]" && \
     pip install --no-cache-dir --break-system-packages flask gunicorn
 
 # ---- font: download GoNotoCurrent-Regular.ttf at build time ----
@@ -47,7 +50,7 @@ ENV \
     PYTHONDONTWRITEBYTECODE=1 \
     MARKER_OUTPUT_DIR=/app/conversion_results \
     UPLOAD_DIR=/app/uploads \
-    OLLAMA_HOST=http://localhost:8082 \
+    OLLAMA_HOST=http://localhost:11435 \
     DEESEEK_OCR_MODEL=deepseek-ocr \
     AMD_COMPUTE=true \
     TORCH_DEVICE= \
@@ -56,7 +59,7 @@ ENV \
     HOST=0.0.0.0 \
     LLM_SERVICE=marker.services.openai.OpenAIService \
     USE_LLM=false \
-    OPENAI_BASE_URL=http://localhost:8082/v1 \
+    OPENAI_BASE_URL=http://localhost:11435/v1 \
     OPENAI_MODEL= \
     GUNICORN_WORKERS=1
 

TODO.md

@@ -0,0 +1,23 @@
+# TODO: marker-api Local Deployment
+
+## Priority 1: Verify local GPU compute works
+- [x] PyTorch ROCm sees gfx1151 on host
+- [x] Ollama runs locally on port 11435 with ROCm
+- [x] Start marker-api container with rocm/pytorch base
+- [ ] Verify `/health` shows `torch_device: cuda`
+- [ ] Run `convert_both.py` — ensure `non_llm` outputs are produced
+
+## Priority 2: Fix LLM correction pipeline
+- [ ] Remove Vega-specific `HSA_OVERRIDE_GFX_VERSION` from `app.py` ✅
+- [ ] Confirm `build_options()` fix (rm `use_llm` dup) resolves Click warnings ✅
+- [ ] Run `convert_both.py` and verify `non_llm` vs `llm` outputs differ
+- [ ] Confirm LLM requests reach local ollama container
+
+## Priority 3: Hardening
+- [ ] Add health check for LLM connectivity
+- [ ] Ensure `marker` user can write to `conversion_results` directory
+- [ ] Expose LLM metrics/status in /health endpoint
+
+## Known issues
+- GPUs are considered "iGPU" by ollama (system memory, not VRAM)
+- `LD_LIBRARY_PATH` in entrypoint may need /opt/rocm/lib from the rocm/pytorch image

app.py

@@ -22,7 +22,7 @@ from marker.providers.registry import load_extensions
 from marker.settings import settings as marker_settings
 
 # Ollama / OCR fallback configuration
-OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://10.0.1.127:11434")
+OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11435")
 DEESEEK_OCR_MODEL = os.environ.get("DEESEEK_OCR_MODEL", "deepseek-ocr")
 
 AMD_COMPUTE = os.environ.get("AMD_COMPUTE", "false").lower() in ("true", "1", "yes")
@@ -50,7 +50,6 @@ def _configure_env():
     if AMD_COMPUTE and not TORCH_DEVICE:
         os.environ["TORCH_DEVICE"] = "cuda"
         os.environ["TORCH_DEVICE_MODEL"] = "cuda"
-        os.environ["HSA_OVERRIDE_GFX_VERSION"] = "9.0.6"
     if MODEL_DTYPE == "bfloat16":
         os.environ["MODEL_DTYPE"] = "bfloat16"
     if torch_device_override := os.environ.get("TORCH_DEVICE"):

docker-compose.yml

@@ -10,11 +10,16 @@ services:
     env_file:
       - .env          # optional; remove if no .env exists
     environment:
-      - OLLAMA_HOST=${OLLAMA_HOST:-http://10.0.1.127:11434}
+      - OLLAMA_HOST=${OLLAMA_HOST:-http://localhost:11435}
       - DEESEEK_OCR_MODEL=${DEESEEK_OCR_MODEL:-deepseek-ocr}
-      - AMD_COMPUTE=${AMD_COMPUTE:-false}
+      - AMD_COMPUTE=${AMD_COMPUTE:-true}
-      - TORCH_DEVICE=${TORCH_DEVICE:-}
+      - TORCH_DEVICE=${TORCH_DEVICE:-cuda}
       - MODEL_DTYPE=${MODEL_DTYPE:-float32}
+      - LLM_SERVICE=marker.services.openai.OpenAIService
+      - USE_LLM=false
+      - OPENAI_BASE_URL=http://localhost:11435/v1
+      - OPENAI_API_KEY=not-needed
+      - OPENAI_MODEL=
       - PORT=8000
       - HOST=0.0.0.0
       - GUNICORN_WORKERS=${GUNICORN_WORKERS:-2}