#requires -Version 5.1
<#
.SYNOPSIS
    Recursively convert supported documents to markdown via marker-api.
.DESCRIPTION
    Scans a target folder for PDFs, DOCX, XLSX, PPTX, EPUB, images, and HTML.
    Sends each file to the marker-api /marker endpoint (multipart/form-data)
    and writes the resulting .md file beside the original.
.EXAMPLE
    .\marker-convert.ps1 -TargetFolder .\documents
    .\marker-convert.ps1 .\documents -Force -MaxConcurrency 8
    .\marker-convert.ps1 .\documents -ApiUrl http://10.0.0.5:8000/marker
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory = $true, Position = 0)]
    [string]$TargetFolder,

    [Uri]$ApiUrl = "http://localhost:8000/marker",

    [ValidateSet("markdown", "json", "html", "chunks")]
    [string]$OutputFormat = "markdown",

    [switch]$Force,

    [ValidateRange(1, 32)]
    [int]$MaxConcurrency = 4,

    [ValidateRange(30, 1800)]
    [int]$Timeout = 300,

    [string]$PageRange = "",

    [switch]$ForceOcr,

    [switch]$DisableImageExtraction,

    [switch]$UseLlm,

    [string]$LlmService = "marker.services.ollama.OllamaService",

    [string]$Processors = "",

    [string]$ConfigJson = "",

    [string]$ConverterCls = ""
)

$ErrorActionPreference = "Stop"

# ════════════ 1. Validate ════════════
$TargetFolder = (Resolve-Path $TargetFolder).Path
if (-not (Test-Path $TargetFolder)) {
    Write-Error "Target folder does not exist: $TargetFolder"
    exit 1
}

Write-Host "====================================================" -ForegroundColor Cyan
Write-Host "  Force overwrite: $(if ($Force) { 'Yes' } else { 'No' })" -ForegroundColor White
Write-Host "  Force OCR      : $(if ($ForceOcr) { 'Yes' } else { 'No' })" -ForegroundColor White
Write-Host "  Use LLM        : $(if ($UseLlm) { 'Yes' } else { 'No' })" -ForegroundColor White
Write-Host "  LLM Service    : $LlmService" -ForegroundColor White
Write-Host "  Disable ImgExt : $(if ($DisableImageExtraction) { 'Yes' } else { 'No' })" -ForegroundColor White
if ($PageRange) { Write-Host "  Page range     : $PageRange" -ForegroundColor White }
if ($Processors) { Write-Host "  Processors     : $Processors" -ForegroundColor White }
if ($ConverterCls) { Write-Host "  Converter      : $ConverterCls" -ForegroundColor White }
Write-Host "====================================================" -ForegroundColor Cyan
Write-Host ""

# ════════════ 2. Discover files ════════════
$supportedExtensions = @('.pdf', '.docx', '.xlsx', '.pptx', '.epub',
                         '.png', '.jpg', '.jpeg', '.bmp', '.gif',
                         '.tiff', '.tif', '.webp', '.heic', '.html', '.htm')

$files = Get-ChildItem -Path $TargetFolder -Recurse -File |
    Where-Object { $_.Extension -in $supportedExtensions } |
    Sort-Object FullName

if ($files.Count -eq 0) {
    Write-Host "No supported files found in $TargetFolder." -ForegroundColor Yellow
    exit 0
}

Write-Host "Found $($files.Count) document(s) to convert." -ForegroundColor Green
Write-Host ""

# ════════════ 3. Worker function ════════════
# This script block runs inside Start-Job for each file.
$ScriptBlock = {
    param(
        [Parameter(Mandatory=$true)]
        [string]$FilePath,

        [Parameter(Mandatory=$true)]
        [Uri]$ApiUrl,

        [Parameter(Mandatory=$true)]
        [string]$Ofmt,

        [Parameter(Mandatory=$true)]
        [bool]$Force,

        [Parameter(Mandatory=$true)]
        [int]$Timeout,

        [string]$PageRange,

        [bool]$ForceOcr,

        [bool]$DisableImageExtraction,

        [bool]$UseLlm,

        [string]$LlmService,

        [string]$Processors,

        [string]$ConfigJson,

        [string]$ConverterCls
    )

    # Build output path (same directory, same base name with .md extension)
    $baseName = [System.IO.Path]::GetFileNameWithoutExtension($FilePath)
    $dirPath  = Split-Path $FilePath
    $mdPath   = Join-Path $dirPath "$baseName.md"

    $result = [ordered]@{}
    $result.File       = (Split-Path $FilePath -Leaf)
    $result.SizeBytes  = (Get-Item $FilePath).Length
    $result.Status     = "pending"
    $result.ElapsedMs  = 0
    $result.Error      = ""
    $result.OutputFile = ""

    # Skip if output already exists and -Force was not used
    if ((Test-Path $mdPath) -and ($Force -eq $false)) {
        $result.Status = "skipped"
        $result.OutputFile = $mdPath
        return $result
    }

    $sw = [System.Diagnostics.Stopwatch]::StartNew()

    try {
        $bin = [System.IO.File]::ReadAllBytes($FilePath)
        $name = [System.IO.Path]::GetFileName($FilePath)
        $boundary = "----marker-convert-$(Get-Random -Minimum 100000 -Maximum 999999)"
        $enc = [System.Text.Encoding]::UTF8

        # ── Build multipart body ──
        $contentParts = [System.Collections.ArrayList]::new()

        # Part 1: file
        $fileHeader = $enc.GetBytes(
            "--$boundary`r`n" +
            "Content-Disposition: form-data; name=`"file`"; filename=`"$name`"`r`n" +
            "Content-Type: application/octet-stream`r`n`r`n"
        )
        $contentParts.Add($fileHeader) | Out-Null
        $contentParts.Add($bin) | Out-Null
        $contentParts.Add($enc.GetBytes("`r`n")) | Out-Null

        # Part 2: output_format
        $fmtPart = $enc.GetBytes(
            "--$boundary`r`n" +
            "Content-Disposition: form-data; name=`"output_format`"`r`n`r`n" +
            "$Ofmt`r`n"
        )
        $contentParts.Add($fmtPart) | Out-Null

        # Part 3: force_ocr
        $ocrPart = $enc.GetBytes(
            "--$boundary`r`n" +
            "Content-Disposition: form-data; name=`"force_ocr`"`r`n`r`n" +
            "$(if ($ForceOcr) { 'true' } else { 'false' })`r`n"
        )
        $contentParts.Add($ocrPart) | Out-Null

        # Part 4: paginate_output (default false)
        $pagPart = $enc.GetBytes(
            "--$boundary`r`n" +
            "Content-Disposition: form-data; name=`"paginate_output`"`r`n`r`n" +
            "false`r`n"
        )
        $contentParts.Add($pagPart) | Out-Null

        # Part 5: use_llm
        $llmPart = $enc.GetBytes(
            "--$boundary`r`n" +
            "Content-Disposition: form-data; name=`"use_llm`"`r`n`r`n" +
            "$(if ($UseLlm) { 'true' } else { 'false' })`r`n"
        )
        $contentParts.Add($llmPart) | Out-Null

        # Part 6a: llm_service (only if UseLlm or custom service)
        if ($UseLlm -or $LlmService -ne "marker.services.ollama.OllamaService") {
            $svcPart = $enc.GetBytes(
                "--$boundary`r`n" +
                "Content-Disposition: form-data; name=`"llm_service`"`r`n`r`n" +
                "$LlmService`r`n"
            )
            $contentParts.Add($svcPart) | Out-Null
        }

        # Part 6b: page_range
        if ($PageRange) {
            $prPart = $enc.GetBytes(
                "--$boundary`r`n" +
                "Content-Disposition: form-data; name=`"page_range`"`r`n`r`n" +
                "$PageRange`r`n"
            )
            $contentParts.Add($prPart) | Out-Null
        }

        # Part 6c: disable_image_extraction
        if ($DisableImageExtraction) {
            $diePart = $enc.GetBytes(
                "--$boundary`r`n" +
                "Content-Disposition: form-data; name=`"disable_image_extraction`"`r`n`r`n" +
                "true`r`n"
            )
            $contentParts.Add($diePart) | Out-Null
        }

        # Part 6d: processors
        if ($Processors) {
            $prcPart = $enc.GetBytes(
                "--$boundary`r`n" +
                "Content-Disposition: form-data; name=`"processors`"`r`n`r`n" +
                "$Processors`r`n"
            )
            $contentParts.Add($prcPart) | Out-Null
        }

        # Part 6e: config_json
        if ($ConfigJson) {
            $cfgPart = $enc.GetBytes(
                "--$boundary`r`n" +
                "Content-Disposition: form-data; name=`"config_json`"`r`n`r`n" +
                "$ConfigJson`r`n"
            )
            $contentParts.Add($cfgPart) | Out-Null
        }

        # Part 6f: converter_cls
        if ($ConverterCls) {
            $ccPart = $enc.GetBytes(
                "--$boundary`r`n" +
                "Content-Disposition: form-data; name=`"converter_cls`"`r`n`r`n" +
                "$ConverterCls`r`n"
            )
            $contentParts.Add($ccPart) | Out-Null
        }

        # Closing boundary
        $closing = $enc.GetBytes("--$boundary--`r`n")

        # Calculate total content-length
        $contentLength = 0
        foreach ($p in $contentParts) {
            $contentLength += $p.Length
        }
        $contentLength += $closing.Length

        # ── Create HTTP request ──
        $request = [System.Net.HttpWebRequest]::Create($ApiUrl)
        $request.Method              = "POST"
        $request.ContentType         = "multipart/form-data; boundary=$boundary"
        $request.ContentLength       = $contentLength
        $request.Timeout             = $Timeout * 1000
        $request.KeepAlive           = $true
        $request.ProtocolVersion     = [System.Net.HttpVersion]::Version11

        $reqStream = $request.GetRequestStream()

        # Write content parts
        foreach ($p in $contentParts) {
            $reqStream.Write($p, 0, $p.Length)
        }
        # Write closing boundary
        $reqStream.Write($closing, 0, $closing.Length)
        $reqStream.Flush()
        $reqStream.Close()

        # ── Get response ──
        $response = $request.GetResponse()
        $resStream = $response.GetResponseStream()
        $reader = [System.IO.StreamReader]::new($resStream, $enc)
        $content = $reader.ReadToEnd()
        $reader.Close()
        $resStream.Close()
        $response.Close()

        # ── Write output file ──
        [System.IO.File]::WriteAllText($mdPath, $content, $enc)

        $sw.Stop()
        $result.Status   = "success"
        $result.ElapsedMs = $sw.ElapsedMilliseconds
        $result.OutputFile = $mdPath
        $result.ResponseSize = $content.Length

    } catch {
        $sw.Stop()
        $result.Status     = "failed"
        $result.ElapsedMs  = $sw.ElapsedMilliseconds
        $result.Error      = $_.Exception.Message
        if ($_.Exception.Response) {
            $result.Error += "`n  Status: $([int]$_.Exception.Response.StatusCode)"
        }
    }

    return $result
}

# ════════════ 4. Dispatch ════════════
Write-Host "Dispatching $($files.Count) conversion(s) with $($MaxConcurrency) worker(s)..." -ForegroundColor Yellow

$allResults = @()
$runningJobs = @()

for ($i = 0; $i -lt $files.Count; $i++) {
    # If we've reached concurrency limit, wait for at least one job to complete
    while ($runningJobs.Count -ge $MaxConcurrency) {
        $completed = $runningJobs | Where-Object { $_.State -in @('Completed','Failed') }
        if ($completed) {
            foreach ($job in $completed) {
                try {
                    $allResults += Receive-Job $job
                } catch {
                    $r = [ordered]@{}
                    $r.File       = "unknown"
                    $r.SizeBytes  = 0
                    $r.Status     = "failed"
                    $r.ElapsedMs  = 0
                    $r.Error      = $_.Exception.Message
                    $r.OutputFile = ""
                    $allResults += $r
                }
                $runningJobs = @($runningJobs | Where-Object { $_.Id -ne $job.Id })
                Remove-Job $job 2>&1 | Out-Null
            }
        } else {
            Start-Sleep -Milliseconds 200
        }
    }

    # Launch new job
    $f = $files[$i]
    $safeName = [System.Security.SecurityElement]::Escape($f.Name)
    $jobName = "mk_$safeName"

    $job = Start-Job `
        -Name $jobName `
        -ScriptBlock $ScriptBlock `
        -ArgumentList @(
            $f.FullName
            $ApiUrl
            $OutputFormat
            [bool]$Force
            $Timeout
            $PageRange
            [bool]$ForceOcr
            [bool]$DisableImageExtraction
            [bool]$UseLlm
            $LlmService
            $Processors
            $ConfigJson
            $ConverterCls
        )

    $runningJobs += $job
    Write-Host "  [$($i + 1)/$($files.Count)] Queued: $($f.Name)" -ForegroundColor Gray
}

# Wait for remaining jobs
while ($runningJobs.Count -gt 0) {
    $completed = $runningJobs | Where-Object { $_.State -in @('Completed','Failed') }
    if ($completed) {
        foreach ($job in $completed) {
            try {
                $allResults += Receive-Job $job
            } catch {
                $r = [ordered]@{}
                $r.File       = "unknown"
                $r.SizeBytes  = 0
                $r.Status     = "failed"
                $r.ElapsedMs  = 0
                $r.Error      = $_.Exception.Message
                $r.OutputFile = ""
                $allResults += $r
            }
            $runningJobs = @($runningJobs | Where-Object { $_.Id -ne $job.Id })
            Remove-Job $job 2>&1 | Out-Null
        }
    } else {
        Start-Sleep -Milliseconds 200
    }
}

# ════════════ 5. Report ════════════
$ok   = @($allResults | Where-Object { $_.Status -eq "success" })
$fail = @($allResults | Where-Object { $_.Status -eq "failed"   })
$skip = @($allResults | Where-Object { $_.Status -eq "skipped"  })

Write-Host ""
Write-Host "====== Conversion Results ======" -ForegroundColor Cyan
Write-Host "  Total files    : $($files.Count)" -ForegroundColor White
Write-Host "  Successful     : $($ok.Count)     " -ForegroundColor Green
Write-Host "  Failed         : $($fail.Count)    " -ForegroundColor Red
Write-Host "  Skipped        : $($skip.Count)    " -ForegroundColor Yellow

if ($ok.Count -gt 0) {
    $avgMs = [Math]::Round(($ok | Measure-Object -Property ElapsedMs -Sum).Sum / $ok.Count)
    Write-Host "  Avg time       : $avgMs ms" -ForegroundColor Green
}

if ($fail.Count -gt 0) {
    Write-Host ""
    Write-Host "Errors:" -ForegroundColor Red
    foreach ($r in $fail) {
        Write-Host "  [FAIL] $($r.File)" -ForegroundColor DarkRed
        Write-Host "         $($r.Error)" -ForegroundColor DarkGray
    }
}

# Save CSV report
$csvPath = Join-Path $TargetFolder "_marker_convert_results.csv"
$resultsForCsv = @($allResults | ForEach-Object {
    [PSCustomObject]@{
        FileName   = $_.File
        OutputFile = if ($_.OutputFile) { $_.OutputFile } else { "N/A" }
        Status     = $_.Status
        SizeBytes  = $_.SizeBytes
        ElapsedMs  = $_.ElapsedMs
        Error      = $_.Error
    }
})
$resultsForCsv | Sort-Object Status, File | Export-Csv $csvPath -NoTypeInformation -Encoding UTF8

Write-Host ""
Write-Host "CSV report: $csvPath" -ForegroundColor Magenta
Write-Host "====================================================" -ForegroundColor Cyan