Initial marker-api benchmark setup: Containerfile, app.py, convert_both.py, compare_conversions.py

2026-06-05 13:26:25 +02:00
commit 460fc0003b
33 changed files with 58290 additions and 0 deletions
@@ -0,0 +1,444 @@
+#requires -Version 5.1
+<#
+.SYNOPSIS
+    Recursively convert supported documents to markdown via marker-api.
+.DESCRIPTION
+    Scans a target folder for PDFs, DOCX, XLSX, PPTX, EPUB, images, and HTML.
+    Sends each file to the marker-api /marker endpoint (multipart/form-data)
+    and writes the resulting .md file beside the original.
+.EXAMPLE
+    .\marker-convert.ps1 -TargetFolder .\documents
+    .\marker-convert.ps1 .\documents -Force -MaxConcurrency 8
+    .\marker-convert.ps1 .\documents -ApiUrl http://10.0.0.5:8000/marker
+#>
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true, Position = 0)]
+    [string]$TargetFolder,
+
+    [Uri]$ApiUrl = "http://localhost:8000/marker",
+
+    [ValidateSet("markdown", "json", "html", "chunks")]
+    [string]$OutputFormat = "markdown",
+
+    [switch]$Force,
+
+    [ValidateRange(1, 32)]
+    [int]$MaxConcurrency = 4,
+
+    [ValidateRange(30, 1800)]
+    [int]$Timeout = 300,
+
+    [string]$PageRange = "",
+
+    [switch]$ForceOcr,
+
+    [switch]$DisableImageExtraction,
+
+    [switch]$UseLlm,
+
+    [string]$LlmService = "marker.services.ollama.OllamaService",
+
+    [string]$Processors = "",
+
+    [string]$ConfigJson = "",
+
+    [string]$ConverterCls = ""
+)
+
+$ErrorActionPreference = "Stop"
+
+# ════════════ 1. Validate ════════════
+$TargetFolder = (Resolve-Path $TargetFolder).Path
+if (-not (Test-Path $TargetFolder)) {
+    Write-Error "Target folder does not exist: $TargetFolder"
+    exit 1
+}
+
+Write-Host "====================================================" -ForegroundColor Cyan
+Write-Host "  Force overwrite: $(if ($Force) { 'Yes' } else { 'No' })" -ForegroundColor White
+Write-Host "  Force OCR      : $(if ($ForceOcr) { 'Yes' } else { 'No' })" -ForegroundColor White
+Write-Host "  Use LLM        : $(if ($UseLlm) { 'Yes' } else { 'No' })" -ForegroundColor White
+Write-Host "  LLM Service    : $LlmService" -ForegroundColor White
+Write-Host "  Disable ImgExt : $(if ($DisableImageExtraction) { 'Yes' } else { 'No' })" -ForegroundColor White
+if ($PageRange) { Write-Host "  Page range     : $PageRange" -ForegroundColor White }
+if ($Processors) { Write-Host "  Processors     : $Processors" -ForegroundColor White }
+if ($ConverterCls) { Write-Host "  Converter      : $ConverterCls" -ForegroundColor White }
+Write-Host "====================================================" -ForegroundColor Cyan
+Write-Host ""
+
+# ════════════ 2. Discover files ════════════
+$supportedExtensions = @('.pdf', '.docx', '.xlsx', '.pptx', '.epub',
+                         '.png', '.jpg', '.jpeg', '.bmp', '.gif',
+                         '.tiff', '.tif', '.webp', '.heic', '.html', '.htm')
+
+$files = Get-ChildItem -Path $TargetFolder -Recurse -File |
+    Where-Object { $_.Extension -in $supportedExtensions } |
+    Sort-Object FullName
+
+if ($files.Count -eq 0) {
+    Write-Host "No supported files found in $TargetFolder." -ForegroundColor Yellow
+    exit 0
+}
+
+Write-Host "Found $($files.Count) document(s) to convert." -ForegroundColor Green
+Write-Host ""
+
+# ════════════ 3. Worker function ════════════
+# This script block runs inside Start-Job for each file.
+$ScriptBlock = {
+    param(
+        [Parameter(Mandatory=$true)]
+        [string]$FilePath,
+
+        [Parameter(Mandatory=$true)]
+        [Uri]$ApiUrl,
+
+        [Parameter(Mandatory=$true)]
+        [string]$Ofmt,
+
+        [Parameter(Mandatory=$true)]
+        [bool]$Force,
+
+        [Parameter(Mandatory=$true)]
+        [int]$Timeout,
+
+        [string]$PageRange,
+
+        [bool]$ForceOcr,
+
+        [bool]$DisableImageExtraction,
+
+        [bool]$UseLlm,
+
+        [string]$LlmService,
+
+        [string]$Processors,
+
+        [string]$ConfigJson,
+
+        [string]$ConverterCls
+    )
+
+    # Build output path (same directory, same base name with .md extension)
+    $baseName = [System.IO.Path]::GetFileNameWithoutExtension($FilePath)
+    $dirPath  = Split-Path $FilePath
+    $mdPath   = Join-Path $dirPath "$baseName.md"
+
+    $result = [ordered]@{}
+    $result.File       = (Split-Path $FilePath -Leaf)
+    $result.SizeBytes  = (Get-Item $FilePath).Length
+    $result.Status     = "pending"
+    $result.ElapsedMs  = 0
+    $result.Error      = ""
+    $result.OutputFile = ""
+
+    # Skip if output already exists and -Force was not used
+    if ((Test-Path $mdPath) -and ($Force -eq $false)) {
+        $result.Status = "skipped"
+        $result.OutputFile = $mdPath
+        return $result
+    }
+
+    $sw = [System.Diagnostics.Stopwatch]::StartNew()
+
+    try {
+        $bin = [System.IO.File]::ReadAllBytes($FilePath)
+        $name = [System.IO.Path]::GetFileName($FilePath)
+        $boundary = "----marker-convert-$(Get-Random -Minimum 100000 -Maximum 999999)"
+        $enc = [System.Text.Encoding]::UTF8
+
+        # ── Build multipart body ──
+        $contentParts = [System.Collections.ArrayList]::new()
+
+        # Part 1: file
+        $fileHeader = $enc.GetBytes(
+            "--$boundary`r`n" +
+            "Content-Disposition: form-data; name=`"file`"; filename=`"$name`"`r`n" +
+            "Content-Type: application/octet-stream`r`n`r`n"
+        )
+        $contentParts.Add($fileHeader) | Out-Null
+        $contentParts.Add($bin) | Out-Null
+        $contentParts.Add($enc.GetBytes("`r`n")) | Out-Null
+
+        # Part 2: output_format
+        $fmtPart = $enc.GetBytes(
+            "--$boundary`r`n" +
+            "Content-Disposition: form-data; name=`"output_format`"`r`n`r`n" +
+            "$Ofmt`r`n"
+        )
+        $contentParts.Add($fmtPart) | Out-Null
+
+        # Part 3: force_ocr
+        $ocrPart = $enc.GetBytes(
+            "--$boundary`r`n" +
+            "Content-Disposition: form-data; name=`"force_ocr`"`r`n`r`n" +
+            "$(if ($ForceOcr) { 'true' } else { 'false' })`r`n"
+        )
+        $contentParts.Add($ocrPart) | Out-Null
+
+        # Part 4: paginate_output (default false)
+        $pagPart = $enc.GetBytes(
+            "--$boundary`r`n" +
+            "Content-Disposition: form-data; name=`"paginate_output`"`r`n`r`n" +
+            "false`r`n"
+        )
+        $contentParts.Add($pagPart) | Out-Null
+
+        # Part 5: use_llm
+        $llmPart = $enc.GetBytes(
+            "--$boundary`r`n" +
+            "Content-Disposition: form-data; name=`"use_llm`"`r`n`r`n" +
+            "$(if ($UseLlm) { 'true' } else { 'false' })`r`n"
+        )
+        $contentParts.Add($llmPart) | Out-Null
+
+        # Part 6a: llm_service (only if UseLlm or custom service)
+        if ($UseLlm -or $LlmService -ne "marker.services.ollama.OllamaService") {
+            $svcPart = $enc.GetBytes(
+                "--$boundary`r`n" +
+                "Content-Disposition: form-data; name=`"llm_service`"`r`n`r`n" +
+                "$LlmService`r`n"
+            )
+            $contentParts.Add($svcPart) | Out-Null
+        }
+
+        # Part 6b: page_range
+        if ($PageRange) {
+            $prPart = $enc.GetBytes(
+                "--$boundary`r`n" +
+                "Content-Disposition: form-data; name=`"page_range`"`r`n`r`n" +
+                "$PageRange`r`n"
+            )
+            $contentParts.Add($prPart) | Out-Null
+        }
+
+        # Part 6c: disable_image_extraction
+        if ($DisableImageExtraction) {
+            $diePart = $enc.GetBytes(
+                "--$boundary`r`n" +
+                "Content-Disposition: form-data; name=`"disable_image_extraction`"`r`n`r`n" +
+                "true`r`n"
+            )
+            $contentParts.Add($diePart) | Out-Null
+        }
+
+        # Part 6d: processors
+        if ($Processors) {
+            $prcPart = $enc.GetBytes(
+                "--$boundary`r`n" +
+                "Content-Disposition: form-data; name=`"processors`"`r`n`r`n" +
+                "$Processors`r`n"
+            )
+            $contentParts.Add($prcPart) | Out-Null
+        }
+
+        # Part 6e: config_json
+        if ($ConfigJson) {
+            $cfgPart = $enc.GetBytes(
+                "--$boundary`r`n" +
+                "Content-Disposition: form-data; name=`"config_json`"`r`n`r`n" +
+                "$ConfigJson`r`n"
+            )
+            $contentParts.Add($cfgPart) | Out-Null
+        }
+
+        # Part 6f: converter_cls
+        if ($ConverterCls) {
+            $ccPart = $enc.GetBytes(
+                "--$boundary`r`n" +
+                "Content-Disposition: form-data; name=`"converter_cls`"`r`n`r`n" +
+                "$ConverterCls`r`n"
+            )
+            $contentParts.Add($ccPart) | Out-Null
+        }
+
+        # Closing boundary
+        $closing = $enc.GetBytes("--$boundary--`r`n")
+
+        # Calculate total content-length
+        $contentLength = 0
+        foreach ($p in $contentParts) {
+            $contentLength += $p.Length
+        }
+        $contentLength += $closing.Length
+
+        # ── Create HTTP request ──
+        $request = [System.Net.HttpWebRequest]::Create($ApiUrl)
+        $request.Method              = "POST"
+        $request.ContentType         = "multipart/form-data; boundary=$boundary"
+        $request.ContentLength       = $contentLength
+        $request.Timeout             = $Timeout * 1000
+        $request.KeepAlive           = $true
+        $request.ProtocolVersion     = [System.Net.HttpVersion]::Version11
+
+        $reqStream = $request.GetRequestStream()
+
+        # Write content parts
+        foreach ($p in $contentParts) {
+            $reqStream.Write($p, 0, $p.Length)
+        }
+        # Write closing boundary
+        $reqStream.Write($closing, 0, $closing.Length)
+        $reqStream.Flush()
+        $reqStream.Close()
+
+        # ── Get response ──
+        $response = $request.GetResponse()
+        $resStream = $response.GetResponseStream()
+        $reader = [System.IO.StreamReader]::new($resStream, $enc)
+        $content = $reader.ReadToEnd()
+        $reader.Close()
+        $resStream.Close()
+        $response.Close()
+
+        # ── Write output file ──
+        [System.IO.File]::WriteAllText($mdPath, $content, $enc)
+
+        $sw.Stop()
+        $result.Status   = "success"
+        $result.ElapsedMs = $sw.ElapsedMilliseconds
+        $result.OutputFile = $mdPath
+        $result.ResponseSize = $content.Length
+
+    } catch {
+        $sw.Stop()
+        $result.Status     = "failed"
+        $result.ElapsedMs  = $sw.ElapsedMilliseconds
+        $result.Error      = $_.Exception.Message
+        if ($_.Exception.Response) {
+            $result.Error += "`n  Status: $([int]$_.Exception.Response.StatusCode)"
+        }
+    }
+
+    return $result
+}
+
+# ════════════ 4. Dispatch ════════════
+Write-Host "Dispatching $($files.Count) conversion(s) with $($MaxConcurrency) worker(s)..." -ForegroundColor Yellow
+
+$allResults = @()
+$runningJobs = @()
+
+for ($i = 0; $i -lt $files.Count; $i++) {
+    # If we've reached concurrency limit, wait for at least one job to complete
+    while ($runningJobs.Count -ge $MaxConcurrency) {
+        $completed = $runningJobs | Where-Object { $_.State -in @('Completed','Failed') }
+        if ($completed) {
+            foreach ($job in $completed) {
+                try {
+                    $allResults += Receive-Job $job
+                } catch {
+                    $r = [ordered]@{}
+                    $r.File       = "unknown"
+                    $r.SizeBytes  = 0
+                    $r.Status     = "failed"
+                    $r.ElapsedMs  = 0
+                    $r.Error      = $_.Exception.Message
+                    $r.OutputFile = ""
+                    $allResults += $r
+                }
+                $runningJobs = @($runningJobs | Where-Object { $_.Id -ne $job.Id })
+                Remove-Job $job 2>&1 | Out-Null
+            }
+        } else {
+            Start-Sleep -Milliseconds 200
+        }
+    }
+
+    # Launch new job
+    $f = $files[$i]
+    $safeName = [System.Security.SecurityElement]::Escape($f.Name)
+    $jobName = "mk_$safeName"
+
+    $job = Start-Job `
+        -Name $jobName `
+        -ScriptBlock $ScriptBlock `
+        -ArgumentList @(
+            $f.FullName
+            $ApiUrl
+            $OutputFormat
+            [bool]$Force
+            $Timeout
+            $PageRange
+            [bool]$ForceOcr
+            [bool]$DisableImageExtraction
+            [bool]$UseLlm
+            $LlmService
+            $Processors
+            $ConfigJson
+            $ConverterCls
+        )
+
+    $runningJobs += $job
+    Write-Host "  [$($i + 1)/$($files.Count)] Queued: $($f.Name)" -ForegroundColor Gray
+}
+
+# Wait for remaining jobs
+while ($runningJobs.Count -gt 0) {
+    $completed = $runningJobs | Where-Object { $_.State -in @('Completed','Failed') }
+    if ($completed) {
+        foreach ($job in $completed) {
+            try {
+                $allResults += Receive-Job $job
+            } catch {
+                $r = [ordered]@{}
+                $r.File       = "unknown"
+                $r.SizeBytes  = 0
+                $r.Status     = "failed"
+                $r.ElapsedMs  = 0
+                $r.Error      = $_.Exception.Message
+                $r.OutputFile = ""
+                $allResults += $r
+            }
+            $runningJobs = @($runningJobs | Where-Object { $_.Id -ne $job.Id })
+            Remove-Job $job 2>&1 | Out-Null
+        }
+    } else {
+        Start-Sleep -Milliseconds 200
+    }
+}
+
+# ════════════ 5. Report ════════════
+$ok   = @($allResults | Where-Object { $_.Status -eq "success" })
+$fail = @($allResults | Where-Object { $_.Status -eq "failed"   })
+$skip = @($allResults | Where-Object { $_.Status -eq "skipped"  })
+
+Write-Host ""
+Write-Host "====== Conversion Results ======" -ForegroundColor Cyan
+Write-Host "  Total files    : $($files.Count)" -ForegroundColor White
+Write-Host "  Successful     : $($ok.Count)     " -ForegroundColor Green
+Write-Host "  Failed         : $($fail.Count)    " -ForegroundColor Red
+Write-Host "  Skipped        : $($skip.Count)    " -ForegroundColor Yellow
+
+if ($ok.Count -gt 0) {
+    $avgMs = [Math]::Round(($ok | Measure-Object -Property ElapsedMs -Sum).Sum / $ok.Count)
+    Write-Host "  Avg time       : $avgMs ms" -ForegroundColor Green
+}
+
+if ($fail.Count -gt 0) {
+    Write-Host ""
+    Write-Host "Errors:" -ForegroundColor Red
+    foreach ($r in $fail) {
+        Write-Host "  [FAIL] $($r.File)" -ForegroundColor DarkRed
+        Write-Host "         $($r.Error)" -ForegroundColor DarkGray
+    }
+}
+
+# Save CSV report
+$csvPath = Join-Path $TargetFolder "_marker_convert_results.csv"
+$resultsForCsv = @($allResults | ForEach-Object {
+    [PSCustomObject]@{
+        FileName   = $_.File
+        OutputFile = if ($_.OutputFile) { $_.OutputFile } else { "N/A" }
+        Status     = $_.Status
+        SizeBytes  = $_.SizeBytes
+        ElapsedMs  = $_.ElapsedMs
+        Error      = $_.Error
+    }
+})
+$resultsForCsv | Sort-Object Status, File | Export-Csv $csvPath -NoTypeInformation -Encoding UTF8
+
+Write-Host ""
+Write-Host "CSV report: $csvPath" -ForegroundColor Magenta
+Write-Host "====================================================" -ForegroundColor Cyan