oval
6851b26923
FastAPI + Celery + Svelte app for English→Danish video translation with voice cloning. Includes diarization, STT, translation, TTS, and audio mixing pipeline.
111 lines
2.9 KiB
Python
111 lines
2.9 KiB
Python
import subprocess
|
|
import os
|
|
import numpy as np
|
|
import soundfile as sf
|
|
from typing import Optional
|
|
from ..config import settings
|
|
|
|
|
|
def mix_audio(
|
|
video_path: str,
|
|
segments: list[dict],
|
|
output_path: str,
|
|
):
|
|
temp_dir = settings.temp_dir
|
|
|
|
original_audio_path = os.path.join(temp_dir, "original_audio.wav")
|
|
_extract_audio(video_path, original_audio_path)
|
|
|
|
original_audio, orig_sr = sf.read(original_audio_path)
|
|
if len(original_audio.shape) > 1:
|
|
original_audio = original_audio.mean(axis=1)
|
|
|
|
target_sr = 24000
|
|
if orig_sr != target_sr:
|
|
import librosa
|
|
|
|
original_audio = librosa.resample(
|
|
original_audio, orig_sr=orig_sr, target_sr=target_sr
|
|
)
|
|
|
|
total_samples = len(original_audio)
|
|
danish_track = np.zeros(total_samples, dtype=np.float32)
|
|
|
|
for seg in segments:
|
|
danish_audio = seg.get("danish_audio")
|
|
if danish_audio is None or len(danish_audio) == 0:
|
|
continue
|
|
|
|
start_sample = int(seg["start_time"] * target_sr)
|
|
end_sample = min(start_sample + len(danish_audio), total_samples)
|
|
actual_len = end_sample - start_sample
|
|
danish_track[start_sample:end_sample] += danish_audio[:actual_len]
|
|
|
|
volume_envelope = np.ones(total_samples, dtype=np.float32)
|
|
for seg in segments:
|
|
start_sample = int(seg["start_time"] * target_sr)
|
|
end_sample = int(seg["end_time"] * target_sr)
|
|
volume_envelope[start_sample:end_sample] = 0.23
|
|
|
|
orig_adjusted = original_audio * volume_envelope
|
|
|
|
mixed = orig_adjusted + danish_track
|
|
peak = np.max(np.abs(mixed))
|
|
if peak > 0.99:
|
|
mixed = mixed / peak * 0.95
|
|
|
|
mixed_path = os.path.join(temp_dir, "mixed_audio.wav")
|
|
sf.write(mixed_path, mixed, target_sr)
|
|
|
|
_replace_audio(video_path, mixed_path, output_path)
|
|
|
|
for f in [original_audio_path, mixed_path]:
|
|
if os.path.exists(f):
|
|
os.remove(f)
|
|
|
|
|
|
def _extract_audio(video_path: str, output_path: str):
|
|
cmd = [
|
|
"ffmpeg",
|
|
"-y",
|
|
"-i",
|
|
video_path,
|
|
"-vn",
|
|
"-acodec",
|
|
"pcm_s16le",
|
|
"-ar",
|
|
"24000",
|
|
"-ac",
|
|
"1",
|
|
output_path,
|
|
]
|
|
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"Audio extraction failed: {result.stderr}")
|
|
|
|
|
|
def _replace_audio(video_path: str, audio_path: str, output_path: str):
|
|
cmd = [
|
|
"ffmpeg",
|
|
"-y",
|
|
"-i",
|
|
video_path,
|
|
"-i",
|
|
audio_path,
|
|
"-c:v",
|
|
"copy",
|
|
"-c:a",
|
|
"aac",
|
|
"-b:a",
|
|
"192k",
|
|
"-map",
|
|
"0:v:0",
|
|
"-map",
|
|
"1:a:0",
|
|
"-shortest",
|
|
output_path,
|
|
]
|
|
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"Audio replacement failed: {result.stderr}")
|