oval
445 lines
15 KiB
PowerShell
445 lines
15 KiB
PowerShell
#requires -Version 5.1
|
|
<#
|
|
.SYNOPSIS
|
|
Recursively convert supported documents to markdown via marker-api.
|
|
.DESCRIPTION
|
|
Scans a target folder for PDFs, DOCX, XLSX, PPTX, EPUB, images, and HTML.
|
|
Sends each file to the marker-api /marker endpoint (multipart/form-data)
|
|
and writes the resulting .md file beside the original.
|
|
.EXAMPLE
|
|
.\marker-convert.ps1 -TargetFolder .\documents
|
|
.\marker-convert.ps1 .\documents -Force -MaxConcurrency 8
|
|
.\marker-convert.ps1 .\documents -ApiUrl http://10.0.0.5:8000/marker
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true, Position = 0)]
|
|
[string]$TargetFolder,
|
|
|
|
[Uri]$ApiUrl = "http://localhost:8000/marker",
|
|
|
|
[ValidateSet("markdown", "json", "html", "chunks")]
|
|
[string]$OutputFormat = "markdown",
|
|
|
|
[switch]$Force,
|
|
|
|
[ValidateRange(1, 32)]
|
|
[int]$MaxConcurrency = 4,
|
|
|
|
[ValidateRange(30, 1800)]
|
|
[int]$Timeout = 300,
|
|
|
|
[string]$PageRange = "",
|
|
|
|
[switch]$ForceOcr,
|
|
|
|
[switch]$DisableImageExtraction,
|
|
|
|
[switch]$UseLlm,
|
|
|
|
[string]$LlmService = "marker.services.ollama.OllamaService",
|
|
|
|
[string]$Processors = "",
|
|
|
|
[string]$ConfigJson = "",
|
|
|
|
[string]$ConverterCls = ""
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
# ════════════ 1. Validate ════════════
|
|
$TargetFolder = (Resolve-Path $TargetFolder).Path
|
|
if (-not (Test-Path $TargetFolder)) {
|
|
Write-Error "Target folder does not exist: $TargetFolder"
|
|
exit 1
|
|
}
|
|
|
|
Write-Host "====================================================" -ForegroundColor Cyan
|
|
Write-Host " Force overwrite: $(if ($Force) { 'Yes' } else { 'No' })" -ForegroundColor White
|
|
Write-Host " Force OCR : $(if ($ForceOcr) { 'Yes' } else { 'No' })" -ForegroundColor White
|
|
Write-Host " Use LLM : $(if ($UseLlm) { 'Yes' } else { 'No' })" -ForegroundColor White
|
|
Write-Host " LLM Service : $LlmService" -ForegroundColor White
|
|
Write-Host " Disable ImgExt : $(if ($DisableImageExtraction) { 'Yes' } else { 'No' })" -ForegroundColor White
|
|
if ($PageRange) { Write-Host " Page range : $PageRange" -ForegroundColor White }
|
|
if ($Processors) { Write-Host " Processors : $Processors" -ForegroundColor White }
|
|
if ($ConverterCls) { Write-Host " Converter : $ConverterCls" -ForegroundColor White }
|
|
Write-Host "====================================================" -ForegroundColor Cyan
|
|
Write-Host ""
|
|
|
|
# ════════════ 2. Discover files ════════════
|
|
$supportedExtensions = @('.pdf', '.docx', '.xlsx', '.pptx', '.epub',
|
|
'.png', '.jpg', '.jpeg', '.bmp', '.gif',
|
|
'.tiff', '.tif', '.webp', '.heic', '.html', '.htm')
|
|
|
|
$files = Get-ChildItem -Path $TargetFolder -Recurse -File |
|
|
Where-Object { $_.Extension -in $supportedExtensions } |
|
|
Sort-Object FullName
|
|
|
|
if ($files.Count -eq 0) {
|
|
Write-Host "No supported files found in $TargetFolder." -ForegroundColor Yellow
|
|
exit 0
|
|
}
|
|
|
|
Write-Host "Found $($files.Count) document(s) to convert." -ForegroundColor Green
|
|
Write-Host ""
|
|
|
|
# ════════════ 3. Worker function ════════════
|
|
# This script block runs inside Start-Job for each file.
|
|
$ScriptBlock = {
|
|
param(
|
|
[Parameter(Mandatory=$true)]
|
|
[string]$FilePath,
|
|
|
|
[Parameter(Mandatory=$true)]
|
|
[Uri]$ApiUrl,
|
|
|
|
[Parameter(Mandatory=$true)]
|
|
[string]$Ofmt,
|
|
|
|
[Parameter(Mandatory=$true)]
|
|
[bool]$Force,
|
|
|
|
[Parameter(Mandatory=$true)]
|
|
[int]$Timeout,
|
|
|
|
[string]$PageRange,
|
|
|
|
[bool]$ForceOcr,
|
|
|
|
[bool]$DisableImageExtraction,
|
|
|
|
[bool]$UseLlm,
|
|
|
|
[string]$LlmService,
|
|
|
|
[string]$Processors,
|
|
|
|
[string]$ConfigJson,
|
|
|
|
[string]$ConverterCls
|
|
)
|
|
|
|
# Build output path (same directory, same base name with .md extension)
|
|
$baseName = [System.IO.Path]::GetFileNameWithoutExtension($FilePath)
|
|
$dirPath = Split-Path $FilePath
|
|
$mdPath = Join-Path $dirPath "$baseName.md"
|
|
|
|
$result = [ordered]@{}
|
|
$result.File = (Split-Path $FilePath -Leaf)
|
|
$result.SizeBytes = (Get-Item $FilePath).Length
|
|
$result.Status = "pending"
|
|
$result.ElapsedMs = 0
|
|
$result.Error = ""
|
|
$result.OutputFile = ""
|
|
|
|
# Skip if output already exists and -Force was not used
|
|
if ((Test-Path $mdPath) -and ($Force -eq $false)) {
|
|
$result.Status = "skipped"
|
|
$result.OutputFile = $mdPath
|
|
return $result
|
|
}
|
|
|
|
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
|
|
|
try {
|
|
$bin = [System.IO.File]::ReadAllBytes($FilePath)
|
|
$name = [System.IO.Path]::GetFileName($FilePath)
|
|
$boundary = "----marker-convert-$(Get-Random -Minimum 100000 -Maximum 999999)"
|
|
$enc = [System.Text.Encoding]::UTF8
|
|
|
|
# ── Build multipart body ──
|
|
$contentParts = [System.Collections.ArrayList]::new()
|
|
|
|
# Part 1: file
|
|
$fileHeader = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"file`"; filename=`"$name`"`r`n" +
|
|
"Content-Type: application/octet-stream`r`n`r`n"
|
|
)
|
|
$contentParts.Add($fileHeader) | Out-Null
|
|
$contentParts.Add($bin) | Out-Null
|
|
$contentParts.Add($enc.GetBytes("`r`n")) | Out-Null
|
|
|
|
# Part 2: output_format
|
|
$fmtPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"output_format`"`r`n`r`n" +
|
|
"$Ofmt`r`n"
|
|
)
|
|
$contentParts.Add($fmtPart) | Out-Null
|
|
|
|
# Part 3: force_ocr
|
|
$ocrPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"force_ocr`"`r`n`r`n" +
|
|
"$(if ($ForceOcr) { 'true' } else { 'false' })`r`n"
|
|
)
|
|
$contentParts.Add($ocrPart) | Out-Null
|
|
|
|
# Part 4: paginate_output (default false)
|
|
$pagPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"paginate_output`"`r`n`r`n" +
|
|
"false`r`n"
|
|
)
|
|
$contentParts.Add($pagPart) | Out-Null
|
|
|
|
# Part 5: use_llm
|
|
$llmPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"use_llm`"`r`n`r`n" +
|
|
"$(if ($UseLlm) { 'true' } else { 'false' })`r`n"
|
|
)
|
|
$contentParts.Add($llmPart) | Out-Null
|
|
|
|
# Part 6a: llm_service (only if UseLlm or custom service)
|
|
if ($UseLlm -or $LlmService -ne "marker.services.ollama.OllamaService") {
|
|
$svcPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"llm_service`"`r`n`r`n" +
|
|
"$LlmService`r`n"
|
|
)
|
|
$contentParts.Add($svcPart) | Out-Null
|
|
}
|
|
|
|
# Part 6b: page_range
|
|
if ($PageRange) {
|
|
$prPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"page_range`"`r`n`r`n" +
|
|
"$PageRange`r`n"
|
|
)
|
|
$contentParts.Add($prPart) | Out-Null
|
|
}
|
|
|
|
# Part 6c: disable_image_extraction
|
|
if ($DisableImageExtraction) {
|
|
$diePart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"disable_image_extraction`"`r`n`r`n" +
|
|
"true`r`n"
|
|
)
|
|
$contentParts.Add($diePart) | Out-Null
|
|
}
|
|
|
|
# Part 6d: processors
|
|
if ($Processors) {
|
|
$prcPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"processors`"`r`n`r`n" +
|
|
"$Processors`r`n"
|
|
)
|
|
$contentParts.Add($prcPart) | Out-Null
|
|
}
|
|
|
|
# Part 6e: config_json
|
|
if ($ConfigJson) {
|
|
$cfgPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"config_json`"`r`n`r`n" +
|
|
"$ConfigJson`r`n"
|
|
)
|
|
$contentParts.Add($cfgPart) | Out-Null
|
|
}
|
|
|
|
# Part 6f: converter_cls
|
|
if ($ConverterCls) {
|
|
$ccPart = $enc.GetBytes(
|
|
"--$boundary`r`n" +
|
|
"Content-Disposition: form-data; name=`"converter_cls`"`r`n`r`n" +
|
|
"$ConverterCls`r`n"
|
|
)
|
|
$contentParts.Add($ccPart) | Out-Null
|
|
}
|
|
|
|
# Closing boundary
|
|
$closing = $enc.GetBytes("--$boundary--`r`n")
|
|
|
|
# Calculate total content-length
|
|
$contentLength = 0
|
|
foreach ($p in $contentParts) {
|
|
$contentLength += $p.Length
|
|
}
|
|
$contentLength += $closing.Length
|
|
|
|
# ── Create HTTP request ──
|
|
$request = [System.Net.HttpWebRequest]::Create($ApiUrl)
|
|
$request.Method = "POST"
|
|
$request.ContentType = "multipart/form-data; boundary=$boundary"
|
|
$request.ContentLength = $contentLength
|
|
$request.Timeout = $Timeout * 1000
|
|
$request.KeepAlive = $true
|
|
$request.ProtocolVersion = [System.Net.HttpVersion]::Version11
|
|
|
|
$reqStream = $request.GetRequestStream()
|
|
|
|
# Write content parts
|
|
foreach ($p in $contentParts) {
|
|
$reqStream.Write($p, 0, $p.Length)
|
|
}
|
|
# Write closing boundary
|
|
$reqStream.Write($closing, 0, $closing.Length)
|
|
$reqStream.Flush()
|
|
$reqStream.Close()
|
|
|
|
# ── Get response ──
|
|
$response = $request.GetResponse()
|
|
$resStream = $response.GetResponseStream()
|
|
$reader = [System.IO.StreamReader]::new($resStream, $enc)
|
|
$content = $reader.ReadToEnd()
|
|
$reader.Close()
|
|
$resStream.Close()
|
|
$response.Close()
|
|
|
|
# ── Write output file ──
|
|
[System.IO.File]::WriteAllText($mdPath, $content, $enc)
|
|
|
|
$sw.Stop()
|
|
$result.Status = "success"
|
|
$result.ElapsedMs = $sw.ElapsedMilliseconds
|
|
$result.OutputFile = $mdPath
|
|
$result.ResponseSize = $content.Length
|
|
|
|
} catch {
|
|
$sw.Stop()
|
|
$result.Status = "failed"
|
|
$result.ElapsedMs = $sw.ElapsedMilliseconds
|
|
$result.Error = $_.Exception.Message
|
|
if ($_.Exception.Response) {
|
|
$result.Error += "`n Status: $([int]$_.Exception.Response.StatusCode)"
|
|
}
|
|
}
|
|
|
|
return $result
|
|
}
|
|
|
|
# ════════════ 4. Dispatch ════════════
|
|
Write-Host "Dispatching $($files.Count) conversion(s) with $($MaxConcurrency) worker(s)..." -ForegroundColor Yellow
|
|
|
|
$allResults = @()
|
|
$runningJobs = @()
|
|
|
|
for ($i = 0; $i -lt $files.Count; $i++) {
|
|
# If we've reached concurrency limit, wait for at least one job to complete
|
|
while ($runningJobs.Count -ge $MaxConcurrency) {
|
|
$completed = $runningJobs | Where-Object { $_.State -in @('Completed','Failed') }
|
|
if ($completed) {
|
|
foreach ($job in $completed) {
|
|
try {
|
|
$allResults += Receive-Job $job
|
|
} catch {
|
|
$r = [ordered]@{}
|
|
$r.File = "unknown"
|
|
$r.SizeBytes = 0
|
|
$r.Status = "failed"
|
|
$r.ElapsedMs = 0
|
|
$r.Error = $_.Exception.Message
|
|
$r.OutputFile = ""
|
|
$allResults += $r
|
|
}
|
|
$runningJobs = @($runningJobs | Where-Object { $_.Id -ne $job.Id })
|
|
Remove-Job $job 2>&1 | Out-Null
|
|
}
|
|
} else {
|
|
Start-Sleep -Milliseconds 200
|
|
}
|
|
}
|
|
|
|
# Launch new job
|
|
$f = $files[$i]
|
|
$safeName = [System.Security.SecurityElement]::Escape($f.Name)
|
|
$jobName = "mk_$safeName"
|
|
|
|
$job = Start-Job `
|
|
-Name $jobName `
|
|
-ScriptBlock $ScriptBlock `
|
|
-ArgumentList @(
|
|
$f.FullName
|
|
$ApiUrl
|
|
$OutputFormat
|
|
[bool]$Force
|
|
$Timeout
|
|
$PageRange
|
|
[bool]$ForceOcr
|
|
[bool]$DisableImageExtraction
|
|
[bool]$UseLlm
|
|
$LlmService
|
|
$Processors
|
|
$ConfigJson
|
|
$ConverterCls
|
|
)
|
|
|
|
$runningJobs += $job
|
|
Write-Host " [$($i + 1)/$($files.Count)] Queued: $($f.Name)" -ForegroundColor Gray
|
|
}
|
|
|
|
# Wait for remaining jobs
|
|
while ($runningJobs.Count -gt 0) {
|
|
$completed = $runningJobs | Where-Object { $_.State -in @('Completed','Failed') }
|
|
if ($completed) {
|
|
foreach ($job in $completed) {
|
|
try {
|
|
$allResults += Receive-Job $job
|
|
} catch {
|
|
$r = [ordered]@{}
|
|
$r.File = "unknown"
|
|
$r.SizeBytes = 0
|
|
$r.Status = "failed"
|
|
$r.ElapsedMs = 0
|
|
$r.Error = $_.Exception.Message
|
|
$r.OutputFile = ""
|
|
$allResults += $r
|
|
}
|
|
$runningJobs = @($runningJobs | Where-Object { $_.Id -ne $job.Id })
|
|
Remove-Job $job 2>&1 | Out-Null
|
|
}
|
|
} else {
|
|
Start-Sleep -Milliseconds 200
|
|
}
|
|
}
|
|
|
|
# ════════════ 5. Report ════════════
|
|
$ok = @($allResults | Where-Object { $_.Status -eq "success" })
|
|
$fail = @($allResults | Where-Object { $_.Status -eq "failed" })
|
|
$skip = @($allResults | Where-Object { $_.Status -eq "skipped" })
|
|
|
|
Write-Host ""
|
|
Write-Host "====== Conversion Results ======" -ForegroundColor Cyan
|
|
Write-Host " Total files : $($files.Count)" -ForegroundColor White
|
|
Write-Host " Successful : $($ok.Count) " -ForegroundColor Green
|
|
Write-Host " Failed : $($fail.Count) " -ForegroundColor Red
|
|
Write-Host " Skipped : $($skip.Count) " -ForegroundColor Yellow
|
|
|
|
if ($ok.Count -gt 0) {
|
|
$avgMs = [Math]::Round(($ok | Measure-Object -Property ElapsedMs -Sum).Sum / $ok.Count)
|
|
Write-Host " Avg time : $avgMs ms" -ForegroundColor Green
|
|
}
|
|
|
|
if ($fail.Count -gt 0) {
|
|
Write-Host ""
|
|
Write-Host "Errors:" -ForegroundColor Red
|
|
foreach ($r in $fail) {
|
|
Write-Host " [FAIL] $($r.File)" -ForegroundColor DarkRed
|
|
Write-Host " $($r.Error)" -ForegroundColor DarkGray
|
|
}
|
|
}
|
|
|
|
# Save CSV report
|
|
$csvPath = Join-Path $TargetFolder "_marker_convert_results.csv"
|
|
$resultsForCsv = @($allResults | ForEach-Object {
|
|
[PSCustomObject]@{
|
|
FileName = $_.File
|
|
OutputFile = if ($_.OutputFile) { $_.OutputFile } else { "N/A" }
|
|
Status = $_.Status
|
|
SizeBytes = $_.SizeBytes
|
|
ElapsedMs = $_.ElapsedMs
|
|
Error = $_.Error
|
|
}
|
|
})
|
|
$resultsForCsv | Sort-Object Status, File | Export-Csv $csvPath -NoTypeInformation -Encoding UTF8
|
|
|
|
Write-Host ""
|
|
Write-Host "CSV report: $csvPath" -ForegroundColor Magenta
|
|
Write-Host "====================================================" -ForegroundColor Cyan
|