Initial commit: Børnetime app scaffold
FastAPI + Celery + Svelte app for English→Danish video translation with voice cloning. Includes diarization, STT, translation, TTS, and audio mixing pipeline.
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
import logging
|
||||
import torch
|
||||
from typing import Optional
|
||||
from pyannote.audio import Pipeline
|
||||
from ..config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_diarization_pipeline: Optional[Pipeline] = None
|
||||
|
||||
|
||||
def get_diarization_pipeline() -> Pipeline:
|
||||
global _diarization_pipeline
|
||||
if _diarization_pipeline is None:
|
||||
import os
|
||||
|
||||
hf_token = settings.hf_token or os.environ.get("HUGGINGFACE_TOKEN")
|
||||
logger.info("Loading diarization model on %s", settings.diarization_device)
|
||||
_diarization_pipeline = Pipeline.from_pretrained(
|
||||
settings.diarization_model,
|
||||
use_auth_token=hf_token,
|
||||
)
|
||||
_diarization_pipeline.to(torch.device(settings.diarization_device))
|
||||
return _diarization_pipeline
|
||||
|
||||
|
||||
def diarize(audio_path: str) -> list[dict]:
|
||||
pipeline = get_diarization_pipeline()
|
||||
diarization = pipeline(audio_path)
|
||||
|
||||
segments = []
|
||||
for turn, _, speaker in diarization.itertracks(yield_label=True):
|
||||
segments.append(
|
||||
{
|
||||
"speaker_label": speaker,
|
||||
"start_time": turn.start,
|
||||
"end_time": turn.end,
|
||||
}
|
||||
)
|
||||
|
||||
segments.sort(key=lambda s: s["start_time"])
|
||||
segments = _merge_adjacent_same_speaker(segments)
|
||||
segments = _relabel_speakers(segments)
|
||||
|
||||
return segments
|
||||
|
||||
|
||||
def _merge_adjacent_same_speaker(
|
||||
segments: list[dict], max_gap: float = 0.5
|
||||
) -> list[dict]:
|
||||
if not segments:
|
||||
return []
|
||||
|
||||
merged = [segments[0]]
|
||||
for seg in segments[1:]:
|
||||
prev = merged[-1]
|
||||
if (
|
||||
prev["speaker_label"] == seg["speaker_label"]
|
||||
and (seg["start_time"] - prev["end_time"]) <= max_gap
|
||||
):
|
||||
prev["end_time"] = seg["end_time"]
|
||||
else:
|
||||
merged.append(seg)
|
||||
return merged
|
||||
|
||||
|
||||
def _relabel_speakers(segments: list[dict]) -> list[dict]:
|
||||
label_map = {}
|
||||
counter = 0
|
||||
for seg in segments:
|
||||
spk = seg["speaker_label"]
|
||||
if spk not in label_map:
|
||||
label = chr(65 + counter)
|
||||
label_map[spk] = f"Speaker {label}"
|
||||
counter += 1
|
||||
seg["speaker_label"] = label_map[spk]
|
||||
return segments
|
||||
|
||||
|
||||
def unload_model():
|
||||
global _diarization_pipeline
|
||||
_diarization_pipeline = None
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
@@ -0,0 +1,110 @@
|
||||
import subprocess
|
||||
import os
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from typing import Optional
|
||||
from ..config import settings
|
||||
|
||||
|
||||
def mix_audio(
|
||||
video_path: str,
|
||||
segments: list[dict],
|
||||
output_path: str,
|
||||
):
|
||||
temp_dir = settings.temp_dir
|
||||
|
||||
original_audio_path = os.path.join(temp_dir, "original_audio.wav")
|
||||
_extract_audio(video_path, original_audio_path)
|
||||
|
||||
original_audio, orig_sr = sf.read(original_audio_path)
|
||||
if len(original_audio.shape) > 1:
|
||||
original_audio = original_audio.mean(axis=1)
|
||||
|
||||
target_sr = 24000
|
||||
if orig_sr != target_sr:
|
||||
import librosa
|
||||
|
||||
original_audio = librosa.resample(
|
||||
original_audio, orig_sr=orig_sr, target_sr=target_sr
|
||||
)
|
||||
|
||||
total_samples = len(original_audio)
|
||||
danish_track = np.zeros(total_samples, dtype=np.float32)
|
||||
|
||||
for seg in segments:
|
||||
danish_audio = seg.get("danish_audio")
|
||||
if danish_audio is None or len(danish_audio) == 0:
|
||||
continue
|
||||
|
||||
start_sample = int(seg["start_time"] * target_sr)
|
||||
end_sample = min(start_sample + len(danish_audio), total_samples)
|
||||
actual_len = end_sample - start_sample
|
||||
danish_track[start_sample:end_sample] += danish_audio[:actual_len]
|
||||
|
||||
volume_envelope = np.ones(total_samples, dtype=np.float32)
|
||||
for seg in segments:
|
||||
start_sample = int(seg["start_time"] * target_sr)
|
||||
end_sample = int(seg["end_time"] * target_sr)
|
||||
volume_envelope[start_sample:end_sample] = 0.23
|
||||
|
||||
orig_adjusted = original_audio * volume_envelope
|
||||
|
||||
mixed = orig_adjusted + danish_track
|
||||
peak = np.max(np.abs(mixed))
|
||||
if peak > 0.99:
|
||||
mixed = mixed / peak * 0.95
|
||||
|
||||
mixed_path = os.path.join(temp_dir, "mixed_audio.wav")
|
||||
sf.write(mixed_path, mixed, target_sr)
|
||||
|
||||
_replace_audio(video_path, mixed_path, output_path)
|
||||
|
||||
for f in [original_audio_path, mixed_path]:
|
||||
if os.path.exists(f):
|
||||
os.remove(f)
|
||||
|
||||
|
||||
def _extract_audio(video_path: str, output_path: str):
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
video_path,
|
||||
"-vn",
|
||||
"-acodec",
|
||||
"pcm_s16le",
|
||||
"-ar",
|
||||
"24000",
|
||||
"-ac",
|
||||
"1",
|
||||
output_path,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Audio extraction failed: {result.stderr}")
|
||||
|
||||
|
||||
def _replace_audio(video_path: str, audio_path: str, output_path: str):
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
video_path,
|
||||
"-i",
|
||||
audio_path,
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"1:a:0",
|
||||
"-shortest",
|
||||
output_path,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Audio replacement failed: {result.stderr}")
|
||||
@@ -0,0 +1,141 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import librosa
|
||||
import re
|
||||
import logging
|
||||
from functools import partial
|
||||
from dataclasses import asdict
|
||||
from typing import Optional
|
||||
from transformers import (
|
||||
AutoConfig,
|
||||
AutoModel,
|
||||
AutoTokenizer,
|
||||
WhisperProcessor,
|
||||
)
|
||||
from boson_multimodal.data_collator.higgs_audio_collator import HiggsAudioSampleCollator
|
||||
from boson_multimodal.data_types import ChatMLSample, AudioContent, Message
|
||||
from boson_multimodal.dataset.chatml_dataset import (
|
||||
ChatMLDatasetSample,
|
||||
prepare_chatml_sample_qwen,
|
||||
)
|
||||
|
||||
from ..config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_model: Optional[AutoModel] = None
|
||||
_tokenizer: Optional[AutoTokenizer] = None
|
||||
_collator: Optional[HiggsAudioSampleCollator] = None
|
||||
_config: Optional[AutoConfig] = None
|
||||
|
||||
|
||||
def _load_model():
|
||||
global _model, _tokenizer, _collator, _config
|
||||
if _model is not None:
|
||||
return
|
||||
|
||||
logger.info("Loading Higgs STT on %s", settings.stt_device)
|
||||
dtype = torch.bfloat16 if settings.stt_dtype == "bfloat16" else torch.float16
|
||||
|
||||
_config = AutoConfig.from_pretrained(settings.stt_model_id, trust_remote_code=True)
|
||||
_model = AutoModel.from_pretrained(
|
||||
settings.stt_model_id,
|
||||
torch_dtype=dtype,
|
||||
trust_remote_code=True,
|
||||
attn_implementation="eager",
|
||||
device_map=settings.stt_device,
|
||||
)
|
||||
_model.eval()
|
||||
_tokenizer = AutoTokenizer.from_pretrained(settings.stt_model_id)
|
||||
_model.audio_out_bos_token_id = _tokenizer.convert_tokens_to_ids(
|
||||
"<|audio_out_bos|>"
|
||||
)
|
||||
_model.audio_eos_token_id = _tokenizer.convert_tokens_to_ids("<|audio_eos|>")
|
||||
|
||||
whisper_proc = WhisperProcessor.from_pretrained("openai/whisper-large-v3")
|
||||
_collator = HiggsAudioSampleCollator(
|
||||
whisper_processor=whisper_proc,
|
||||
audio_in_token_id=_config.audio_in_token_idx,
|
||||
audio_out_token_id=_config.audio_out_token_idx,
|
||||
audio_stream_bos_id=_config.audio_stream_bos_id,
|
||||
audio_stream_eos_id=_config.audio_stream_eos_id,
|
||||
encode_whisper_embed=_config.encode_whisper_embed,
|
||||
pad_token_id=_config.pad_token_id,
|
||||
return_audio_in_tokens=_config.encode_audio_in_tokens,
|
||||
use_delay_pattern=_config.use_delay_pattern,
|
||||
round_to=1,
|
||||
audio_num_codebooks=_config.audio_num_codebooks,
|
||||
chunk_size_seconds=getattr(_config, "chunk_size_seconds", 30),
|
||||
encoder_padding_method=getattr(_config, "encoder_padding_method", "max_length"),
|
||||
)
|
||||
|
||||
|
||||
def transcribe_segment(audio_path: str, start_time: float, end_time: float) -> str:
|
||||
_load_model()
|
||||
|
||||
audio_np, sr = sf.read(audio_path)
|
||||
if sr != 16000:
|
||||
audio_np = librosa.resample(audio_np, orig_sr=sr, target_sr=16000)
|
||||
|
||||
start_sample = int(start_time * 16000)
|
||||
end_sample = int(end_time * 16000)
|
||||
segment_audio = audio_np[start_sample:end_sample]
|
||||
|
||||
if len(segment_audio) == 0:
|
||||
return ""
|
||||
|
||||
prompt = "Transcribe the speech. Output only the spoken words in lowercase with no punctuation."
|
||||
messages = [
|
||||
Message(role="user", content=[prompt, AudioContent(audio_url="placeholder")])
|
||||
]
|
||||
chatml = ChatMLSample(messages=messages)
|
||||
prep_fn = partial(prepare_chatml_sample_qwen, enable_thinking=True)
|
||||
input_tokens, _, _, _ = prep_fn(chatml, _tokenizer, add_generation_prompt=True)
|
||||
|
||||
sample = ChatMLDatasetSample(
|
||||
input_ids=torch.LongTensor(input_tokens),
|
||||
label_ids=None,
|
||||
audio_ids_concat=None,
|
||||
audio_ids_start=None,
|
||||
audio_waveforms_concat=torch.tensor(segment_audio, dtype=torch.float32),
|
||||
audio_waveforms_start=torch.tensor([0]),
|
||||
audio_sample_rate=torch.tensor([16000]),
|
||||
audio_speaker_indices=torch.tensor([0]),
|
||||
)
|
||||
|
||||
batch = asdict(_collator([sample]))
|
||||
device = next(_model.parameters()).device
|
||||
batch = {
|
||||
k: v.to(device).contiguous() if isinstance(v, torch.Tensor) else v
|
||||
for k, v in batch.items()
|
||||
}
|
||||
|
||||
with torch.inference_mode():
|
||||
outputs = _model.generate(
|
||||
**batch,
|
||||
max_new_tokens=1024,
|
||||
use_cache=True,
|
||||
do_sample=False,
|
||||
stop_strings=["<|im_end|>", "<|endoftext|>"],
|
||||
tokenizer=_tokenizer,
|
||||
)
|
||||
|
||||
output_ids = outputs[0] if isinstance(outputs, tuple) else outputs
|
||||
full_text = _tokenizer.decode(output_ids[0], skip_special_tokens=False)
|
||||
|
||||
parts = full_text.split("assistant\n")
|
||||
hyp = parts[-1] if len(parts) > 1 else full_text
|
||||
hyp = re.sub(r"<think>.*?</think>", "", hyp, flags=re.DOTALL)
|
||||
hyp = re.sub(r"<\|.*?\|>", "", hyp).strip()
|
||||
|
||||
return hyp
|
||||
|
||||
|
||||
def unload_model():
|
||||
global _model, _tokenizer, _collator, _config
|
||||
_model = None
|
||||
_tokenizer = None
|
||||
_collator = None
|
||||
_config = None
|
||||
torch.cuda.empty_cache()
|
||||
@@ -0,0 +1,60 @@
|
||||
import logging
|
||||
from ..config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_llm = None
|
||||
|
||||
|
||||
def _load_model():
|
||||
global _llm
|
||||
if _llm is not None:
|
||||
return
|
||||
|
||||
from llama_cpp import Llama
|
||||
|
||||
logger.info("Loading TranslateGemma 12B from %s", settings.translate_model_path)
|
||||
_llm = Llama(
|
||||
model_path=settings.translate_model_path,
|
||||
n_gpu_layers=settings.translate_n_gpu_layers,
|
||||
n_ctx=2048,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
|
||||
def translate(text: str) -> str:
|
||||
_load_model()
|
||||
|
||||
if not text.strip():
|
||||
return ""
|
||||
|
||||
lines = [
|
||||
"Translate the following English text to Danish.",
|
||||
"Preserve the original meaning, tone, and style.",
|
||||
"Output only the Danish translation, nothing else.",
|
||||
"",
|
||||
f"English: {text}",
|
||||
"",
|
||||
"Danish:",
|
||||
]
|
||||
prompt = "\n".join(lines)
|
||||
|
||||
stop_tokens = ["\n\n", "English:", "User:"]
|
||||
output = _llm(
|
||||
prompt,
|
||||
max_tokens=1024,
|
||||
temperature=0.1,
|
||||
stop=stop_tokens,
|
||||
echo=False,
|
||||
)
|
||||
|
||||
result = output["choices"][0]["text"].strip()
|
||||
return result
|
||||
|
||||
|
||||
def unload_model():
|
||||
global _llm
|
||||
_llm = None
|
||||
import torch
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
@@ -0,0 +1,256 @@
|
||||
import io
|
||||
import os
|
||||
import logging
|
||||
import numpy as np
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from ..config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_model = None
|
||||
_tokenizer = None
|
||||
_use_http = True
|
||||
|
||||
|
||||
def _load_http_mode():
|
||||
global _use_http
|
||||
import requests
|
||||
|
||||
try:
|
||||
resp = requests.get(
|
||||
settings.tts_api_url.replace("/v1/audio/speech", "/health"), timeout=5
|
||||
)
|
||||
if resp.ok:
|
||||
_use_http = True
|
||||
logger.info("TTS sidecar available at %s", settings.tts_api_url)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
_use_http = False
|
||||
return False
|
||||
|
||||
|
||||
def _load_model():
|
||||
global _model, _tokenizer
|
||||
if _model is not None:
|
||||
return
|
||||
|
||||
logger.info("Loading Higgs TTS 3 on %s", settings.tts_device)
|
||||
from transformers import AutoModel, AutoTokenizer
|
||||
|
||||
_model = AutoModel.from_pretrained(
|
||||
settings.tts_model_id,
|
||||
torch_dtype=torch.bfloat16,
|
||||
trust_remote_code=True,
|
||||
device_map=settings.tts_device,
|
||||
)
|
||||
_model.eval()
|
||||
|
||||
_tokenizer = AutoTokenizer.from_pretrained(settings.tts_model_id)
|
||||
logger.info("Higgs TTS 3 loaded successfully")
|
||||
|
||||
|
||||
def synthesize_segment(
|
||||
text: str,
|
||||
reference_audio_path: Optional[str] = None,
|
||||
reference_text: Optional[str] = None,
|
||||
speaker_label: Optional[str] = None,
|
||||
) -> bytes:
|
||||
if not text.strip():
|
||||
return b""
|
||||
|
||||
input_text = text
|
||||
if speaker_label:
|
||||
input_text = f"<|emotion:contentment|><|prosody:expressive_low|>{text}"
|
||||
|
||||
if _use_http:
|
||||
try:
|
||||
return _synthesize_http(input_text, reference_audio_path, reference_text)
|
||||
except Exception as e:
|
||||
logger.warning("HTTP TTS failed, falling back to direct: %s", e)
|
||||
|
||||
return _synthesize_direct(input_text, reference_audio_path, reference_text)
|
||||
|
||||
|
||||
def _synthesize_http(
|
||||
text: str,
|
||||
reference_audio_path: Optional[str] = None,
|
||||
reference_text: Optional[str] = None,
|
||||
) -> bytes:
|
||||
import requests
|
||||
|
||||
payload = {
|
||||
"input": text,
|
||||
"model": settings.tts_model_id,
|
||||
"temperature": 0.8,
|
||||
"top_k": 50,
|
||||
"max_new_tokens": 2048,
|
||||
}
|
||||
|
||||
if reference_audio_path and reference_text:
|
||||
import base64
|
||||
|
||||
with open(reference_audio_path, "rb") as f:
|
||||
audio_b64 = base64.b64encode(f.read()).decode()
|
||||
payload["references"] = [
|
||||
{
|
||||
"audio": audio_b64,
|
||||
"text": reference_text,
|
||||
}
|
||||
]
|
||||
|
||||
resp = requests.post(settings.tts_api_url, json=payload, timeout=120)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
|
||||
def _synthesize_direct(
|
||||
text: str,
|
||||
reference_audio_path: Optional[str] = None,
|
||||
reference_text: Optional[str] = None,
|
||||
) -> bytes:
|
||||
_load_model()
|
||||
|
||||
from boson_multimodal.data_types import ChatMLSample, Message
|
||||
from boson_multimodal.dataset.chatml_dataset import (
|
||||
ChatMLDatasetSample,
|
||||
prepare_chatml_sample_qwen,
|
||||
)
|
||||
from boson_multimodal.data_collator.higgs_audio_collator import (
|
||||
HiggsAudioSampleCollator,
|
||||
)
|
||||
from functools import partial
|
||||
from dataclasses import asdict
|
||||
|
||||
prompt = f"Generate speech for the following text. Output only the spoken words."
|
||||
messages = [Message(role="user", content=[prompt])]
|
||||
chatml = ChatMLSample(messages=messages)
|
||||
prep_fn = partial(prepare_chatml_sample_qwen, enable_thinking=False)
|
||||
input_tokens, _, _, _ = prep_fn(chatml, _tokenizer, add_generation_prompt=True)
|
||||
|
||||
ref_audio = None
|
||||
if reference_audio_path:
|
||||
import soundfile as sf
|
||||
|
||||
ref_audio, ref_sr = sf.read(reference_audio_path)
|
||||
if ref_sr != 16000:
|
||||
import librosa
|
||||
|
||||
ref_audio = librosa.resample(ref_audio, orig_sr=ref_sr, target_sr=16000)
|
||||
ref_audio = torch.tensor(ref_audio, dtype=torch.float32)
|
||||
|
||||
sample = ChatMLDatasetSample(
|
||||
input_ids=torch.LongTensor(input_tokens),
|
||||
label_ids=None,
|
||||
audio_ids_concat=None,
|
||||
audio_ids_start=None,
|
||||
audio_waveforms_concat=ref_audio if ref_audio is not None else torch.zeros(1),
|
||||
audio_waveforms_start=torch.tensor([0]),
|
||||
audio_sample_rate=torch.tensor([16000]),
|
||||
audio_speaker_indices=torch.tensor([0]),
|
||||
)
|
||||
|
||||
collator = HiggsAudioSampleCollator(
|
||||
whisper_processor=None,
|
||||
audio_in_token_id=getattr(_model.config, "audio_in_token_idx", 0),
|
||||
audio_out_token_id=getattr(_model.config, "audio_out_token_idx", 1),
|
||||
audio_stream_bos_id=getattr(_model.config, "audio_stream_bos_id", 2),
|
||||
audio_stream_eos_id=getattr(_model.config, "audio_stream_eos_id", 3),
|
||||
encode_whisper_embed=getattr(_model.config, "encode_whisper_embed", False),
|
||||
pad_token_id=_tokenizer.pad_token_id or 0,
|
||||
return_audio_in_tokens=getattr(_model.config, "encode_audio_in_tokens", False),
|
||||
use_delay_pattern=getattr(_model.config, "use_delay_pattern", True),
|
||||
round_to=1,
|
||||
audio_num_codebooks=getattr(_model.config, "audio_num_codebooks", 8),
|
||||
chunk_size_seconds=getattr(_model.config, "chunk_size_seconds", 30),
|
||||
encoder_padding_method=getattr(
|
||||
_model.config, "encoder_padding_method", "max_length"
|
||||
),
|
||||
)
|
||||
|
||||
batch = asdict(collator([sample]))
|
||||
device = next(_model.parameters()).device
|
||||
batch = {
|
||||
k: v.to(device).contiguous() if isinstance(v, torch.Tensor) else v
|
||||
for k, v in batch.items()
|
||||
}
|
||||
|
||||
with torch.inference_mode():
|
||||
outputs = _model.generate(
|
||||
**batch,
|
||||
max_new_tokens=2048,
|
||||
temperature=0.8,
|
||||
top_k=50,
|
||||
do_sample=True,
|
||||
use_cache=True,
|
||||
)
|
||||
|
||||
output_ids = outputs[0] if isinstance(outputs, tuple) else outputs
|
||||
|
||||
if hasattr(_tokenizer, "decode_audio"):
|
||||
audio_data = _tokenizer.decode_audio(output_ids)
|
||||
if isinstance(audio_data, bytes):
|
||||
return audio_data
|
||||
return audio_data.tobytes()
|
||||
|
||||
if hasattr(_model, "decode_audio"):
|
||||
audio_data = _model.decode_audio(output_ids)
|
||||
if isinstance(audio_data, bytes):
|
||||
return audio_data
|
||||
if isinstance(audio_data, torch.Tensor):
|
||||
audio_np = audio_data.cpu().float().numpy()
|
||||
return _numpy_to_wav_bytes(audio_np, 24000)
|
||||
|
||||
if isinstance(output_ids, torch.Tensor):
|
||||
audio_np = output_ids.cpu().float().numpy()
|
||||
if audio_np.ndim > 1:
|
||||
audio_np = audio_np.flatten()
|
||||
return _numpy_to_wav_bytes(audio_np, 24000)
|
||||
|
||||
return b""
|
||||
|
||||
|
||||
def _numpy_to_wav_bytes(audio: np.ndarray, sample_rate: int = 24000) -> bytes:
|
||||
import wave
|
||||
import struct
|
||||
|
||||
audio = np.clip(audio, -1.0, 1.0)
|
||||
audio_int16 = (audio * 32767).astype(np.int16)
|
||||
|
||||
buf = io.BytesIO()
|
||||
with wave.open(buf, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(sample_rate)
|
||||
wf.writeframes(audio_int16.tobytes())
|
||||
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _bytes_to_numpy(
|
||||
audio_bytes: bytes, target_sr: int = 24000
|
||||
) -> tuple[np.ndarray, int]:
|
||||
import soundfile as sf
|
||||
|
||||
with io.BytesIO(audio_bytes) as buf:
|
||||
data, sr = sf.read(buf)
|
||||
|
||||
if len(data.shape) > 1:
|
||||
data = data.mean(axis=1)
|
||||
|
||||
if sr != target_sr:
|
||||
import librosa
|
||||
|
||||
data = librosa.resample(data, orig_sr=sr, target_sr=target_sr)
|
||||
|
||||
return data.astype(np.float32), target_sr
|
||||
|
||||
|
||||
def unload_model():
|
||||
global _model, _tokenizer
|
||||
_model = None
|
||||
_tokenizer = None
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
@@ -0,0 +1,47 @@
|
||||
import subprocess
|
||||
import os
|
||||
from typing import Optional
|
||||
from ..config import settings
|
||||
|
||||
|
||||
def get_video_duration(path: str) -> Optional[float]:
|
||||
cmd = [
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
"default=noprint_wrappers=1:nokey=1",
|
||||
path,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return float(result.stdout.strip())
|
||||
return None
|
||||
|
||||
|
||||
def extract_audio(video_path: str, output_path: str, sample_rate: int = 16000) -> str:
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
video_path,
|
||||
"-vn",
|
||||
"-acodec",
|
||||
"pcm_s16le",
|
||||
"-ar",
|
||||
str(sample_rate),
|
||||
"-ac",
|
||||
"1",
|
||||
output_path,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Audio extraction failed: {result.stderr}")
|
||||
return output_path
|
||||
|
||||
|
||||
def build_output_path(video_path: str) -> str:
|
||||
base, ext = os.path.splitext(os.path.basename(video_path))
|
||||
return f"{base}_en_da_audio{ext}"
|
||||
Reference in New Issue
Block a user