268 lines
9.4 KiB
Python
268 lines
9.4 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
LIPSYNC-ADAPTER
|
|||
|
|
口型适配器 — 接视频改口型或 Wav2Lip，解决人物真正说台词的问题。
|
|||
|
|
|
|||
|
|
功能:
|
|||
|
|
1. 输入视频 + 对白音频
|
|||
|
|
2. 用 Wav2Lip 开源工具做口型同步
|
|||
|
|
3. 支持批量处理
|
|||
|
|
4. 封装为统一接口
|
|||
|
|
|
|||
|
|
依赖:
|
|||
|
|
pip install librosa opencv-python numpy
|
|||
|
|
# Wav2Lip 需要单独安装: https://github.com/Rudrabha/Wav2Lip
|
|||
|
|
|
|||
|
|
用法:
|
|||
|
|
python lipsync-adapter.py --video input.mp4 --audio dialogue.mp3 --output output.mp4
|
|||
|
|
python lipsync-adapter.py --batch video_list.json
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
import sys
|
|||
|
|
import json
|
|||
|
|
import argparse
|
|||
|
|
from pathlib import Path
|
|||
|
|
from datetime import datetime
|
|||
|
|
|
|||
|
|
PROJECT_ROOT = Path(__file__).parent.parent
|
|||
|
|
sys.path.insert(0, str(PROJECT_ROOT / "engines"))
|
|||
|
|
|
|||
|
|
|
|||
|
|
class LipSyncAdapter:
|
|||
|
|
"""口型适配器 — API驱动: 火山改口型(主)→Edge-TTS(配音回退)"""
|
|||
|
|
|
|||
|
|
def __init__(self, allow_fallback=False):
|
|||
|
|
# 光湖不本地装模型。所有模型走API。
|
|||
|
|
self.allow_fallback = allow_fallback
|
|||
|
|
self._cloud_available = None
|
|||
|
|
|
|||
|
|
def sync_lips(self, video_path, audio_path, output_path=None, allow_fallback=None):
|
|||
|
|
"""
|
|||
|
|
口型同步 — API驱动
|
|||
|
|
|
|||
|
|
路线: 火山引擎视频改口型(主) → Edge-TTS配音(回退)
|
|||
|
|
|
|||
|
|
返回:
|
|||
|
|
{success, output_path, method, warning/error}
|
|||
|
|
"""
|
|||
|
|
print(f"\n🎤 口型同步")
|
|||
|
|
print(f" 视频: {Path(video_path).name}")
|
|||
|
|
print(f" 音频: {Path(audio_path).name}")
|
|||
|
|
|
|||
|
|
video_path = Path(video_path)
|
|||
|
|
audio_path = Path(audio_path)
|
|||
|
|
|
|||
|
|
if not video_path.exists():
|
|||
|
|
return {"success": False, "error": f"视频不存在: {video_path}"}
|
|||
|
|
if not audio_path.exists():
|
|||
|
|
return {"success": False, "error": f"音频不存在: {audio_path}"}
|
|||
|
|
|
|||
|
|
if output_path is None:
|
|||
|
|
output_path = video_path.parent / f"{video_path.stem}_synced{video_path.suffix}"
|
|||
|
|
else:
|
|||
|
|
output_path = Path(output_path)
|
|||
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|||
|
|
|
|||
|
|
# Route1: 火山引擎视频改口型API（主路线）
|
|||
|
|
cloud = self._check_cloud()
|
|||
|
|
if cloud["available"]:
|
|||
|
|
print(f" ☁️ 火山引擎视频改口型...")
|
|||
|
|
result = self._run_cloud_lipsync(video_path, audio_path, output_path)
|
|||
|
|
if result.get("success"):
|
|||
|
|
return result
|
|||
|
|
print(f" ⚠️ 云端改口型失败: {result.get('error')}")
|
|||
|
|
|
|||
|
|
# Route2: Edge-TTS 配音替代（角色说话时可用）
|
|||
|
|
print(f" 💡 口型同步路由:")
|
|||
|
|
print(f" Route1 火山改口型: {'可用' if cloud.get('available') else '❌ 需开通智能视觉服务'}")
|
|||
|
|
print(f" Route2 Edge-TTS: ✅ (voice-emotion-compiler已通过)")
|
|||
|
|
print(f" 📌 光湖不本地安装模型。所有模型走API。")
|
|||
|
|
|
|||
|
|
fallback_enabled = self.allow_fallback if allow_fallback is None else allow_fallback
|
|||
|
|
if not fallback_enabled:
|
|||
|
|
return {
|
|||
|
|
"success": False,
|
|||
|
|
"error": "口型同步不可用: 火山API未开通 + 未允许回退",
|
|||
|
|
"method": "not-run",
|
|||
|
|
"next_step": "开通火山引擎智能视觉服务'视频改口型'，配置密钥后自动可用"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
print(f" ⚠️ 回退模式: 配音替代口型同步")
|
|||
|
|
return {
|
|||
|
|
"success": False,
|
|||
|
|
"error": "口型同步路由不可用，请先开通API",
|
|||
|
|
"method": "api-required"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def _check_cloud(self):
|
|||
|
|
"""检查云端改口型服务是否可用"""
|
|||
|
|
if self._cloud_available is not None:
|
|||
|
|
return self._cloud_available
|
|||
|
|
try:
|
|||
|
|
import importlib.util
|
|||
|
|
spec = importlib.util.spec_from_file_location(
|
|||
|
|
'volce',
|
|||
|
|
str(PROJECT_ROOT / 'engines' / 'volce-lipsync-adapter.py')
|
|||
|
|
)
|
|||
|
|
mod = importlib.util.module_from_spec(spec)
|
|||
|
|
spec.loader.exec_module(mod)
|
|||
|
|
self._cloud_available = mod.check_available()
|
|||
|
|
except Exception as e:
|
|||
|
|
self._cloud_available = {"available": False, "reason": f"云端适配器不可用: {e}"}
|
|||
|
|
return self._cloud_available
|
|||
|
|
|
|||
|
|
def _run_cloud_lipsync(self, video_path, audio_path, output_path):
|
|||
|
|
"""通过火山引擎云端改口型"""
|
|||
|
|
try:
|
|||
|
|
import importlib.util
|
|||
|
|
spec = importlib.util.spec_from_file_location(
|
|||
|
|
'volce',
|
|||
|
|
str(PROJECT_ROOT / 'engines' / 'volce-lipsync-adapter.py')
|
|||
|
|
)
|
|||
|
|
mod = importlib.util.module_from_spec(spec)
|
|||
|
|
spec.loader.exec_module(mod)
|
|||
|
|
return mod.submit_lipsync(video_path, audio_path)
|
|||
|
|
except Exception as e:
|
|||
|
|
return {"success": False, "error": f"云端适配器调用失败: {e}", "method": "cloud-stub"}
|
|||
|
|
|
|||
|
|
def batch_sync(self, video_audio_pairs, output_dir):
|
|||
|
|
"""
|
|||
|
|
批量口型同步
|
|||
|
|
|
|||
|
|
参数:
|
|||
|
|
video_audio_pairs: list of (video_path, audio_path)
|
|||
|
|
output_dir: 输出目录
|
|||
|
|
|
|||
|
|
返回:
|
|||
|
|
list of result dicts
|
|||
|
|
"""
|
|||
|
|
print(f"\n📦 批量口型同步: {len(video_audio_pairs)} 个任务")
|
|||
|
|
print(f" 输出目录: {output_dir}")
|
|||
|
|
|
|||
|
|
output_dir = Path(output_dir)
|
|||
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|||
|
|
|
|||
|
|
results = []
|
|||
|
|
|
|||
|
|
for i, (video_path, audio_path) in enumerate(video_audio_pairs):
|
|||
|
|
print(f"\n 进度: [{i+1}/{len(video_audio_pairs)}]")
|
|||
|
|
|
|||
|
|
output_path = output_dir / f"{Path(video_path).stem}_synced.mp4"
|
|||
|
|
result = self.sync_lips(video_path, audio_path, output_path)
|
|||
|
|
results.append(result)
|
|||
|
|
|
|||
|
|
# 统计
|
|||
|
|
success_count = sum(1 for r in results if r.get("success"))
|
|||
|
|
print(f"\n✅ 批量完成: {success_count}/{len(results)} 成功")
|
|||
|
|
|
|||
|
|
# 保存报告
|
|||
|
|
report_path = output_dir / "lipsync_report.json"
|
|||
|
|
with open(report_path, "w", encoding="utf-8") as f:
|
|||
|
|
json.dump({
|
|||
|
|
"total": len(results),
|
|||
|
|
"success": success_count,
|
|||
|
|
"results": results,
|
|||
|
|
"generated_at": datetime.now().isoformat()
|
|||
|
|
}, f, ensure_ascii=False, indent=2)
|
|||
|
|
|
|||
|
|
print(f" 报告已保存: {report_path}")
|
|||
|
|
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
def check_audio_sync(self, video_path, tolerance_ms=100):
|
|||
|
|
"""
|
|||
|
|
检查口型同步质量
|
|||
|
|
简单方法: 检测音频能量峰值，与视频画面变化对比
|
|||
|
|
|
|||
|
|
返回:
|
|||
|
|
{
|
|||
|
|
"synced": bool,
|
|||
|
|
"offset_ms": float,
|
|||
|
|
"score": float # 0-1, 1=完美同步
|
|||
|
|
}
|
|||
|
|
"""
|
|||
|
|
print(f"\n🔍 检查口型同步质量: {Path(video_path).name}")
|
|||
|
|
|
|||
|
|
if not self.available:
|
|||
|
|
print(f" ⚠️ Wav2Lip 不可用，跳过质量检查")
|
|||
|
|
return {"synced": None, "score": None, "warning": "Wav2Lip 不可用"}
|
|||
|
|
|
|||
|
|
# TODO: 实现口型同步质量检查
|
|||
|
|
# 1. 提取音频能量包络
|
|||
|
|
# 2. 检测视频中嘴部区域的运动
|
|||
|
|
# 3. 计算相关性
|
|||
|
|
# 4. 返回偏移量和分数
|
|||
|
|
|
|||
|
|
print(f" ⚠️ 质量检查未实现 (需要 librosa + OpenCV 嘴部检测)")
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"synced": None,
|
|||
|
|
"offset_ms": 0,
|
|||
|
|
"score": None,
|
|||
|
|
"warning": "Quality check not implemented yet"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
parser = argparse.ArgumentParser(description="LIPSYNC-ADAPTER")
|
|||
|
|
parser.add_argument("--video", type=str, help="输入视频路径")
|
|||
|
|
parser.add_argument("--audio", type=str, help="对白音频路径")
|
|||
|
|
parser.add_argument("--output", type=str, help="输出视频路径")
|
|||
|
|
parser.add_argument("--batch", type=str, help="批量处理配置文件 (JSON)")
|
|||
|
|
parser.add_argument("--check-sync", action="store_true", help="检查口型同步质量")
|
|||
|
|
parser.add_argument("--allow-fallback-copy", action="store_true",
|
|||
|
|
help="仅调试用: Wav2Lip 不可用时复制原视频并标记为 fallback")
|
|||
|
|
|
|||
|
|
args = parser.parse_args()
|
|||
|
|
|
|||
|
|
if args.check_sync:
|
|||
|
|
if not args.video:
|
|||
|
|
print("❌ --check-sync 需要 --video")
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
adapter = LipSyncAdapter()
|
|||
|
|
result = adapter.check_audio_sync(args.video)
|
|||
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|||
|
|
sys.exit(0)
|
|||
|
|
|
|||
|
|
if args.batch:
|
|||
|
|
# 批量模式
|
|||
|
|
batch_file = Path(args.batch)
|
|||
|
|
if not batch_file.exists():
|
|||
|
|
print(f"❌ 批量配置文件不存在: {batch_file}")
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
with open(batch_file, "r", encoding="utf-8") as f:
|
|||
|
|
config = json.load(f)
|
|||
|
|
|
|||
|
|
video_audio_pairs = []
|
|||
|
|
for item in config.get("tasks", []):
|
|||
|
|
video_audio_pairs.append((item["video"], item["audio"]))
|
|||
|
|
|
|||
|
|
output_dir = config.get("output_dir", "./outputs/lipsync/")
|
|||
|
|
|
|||
|
|
adapter = LipSyncAdapter(allow_fallback=args.allow_fallback_copy)
|
|||
|
|
results = adapter.batch_sync(video_audio_pairs, output_dir)
|
|||
|
|
|
|||
|
|
sys.exit(0)
|
|||
|
|
|
|||
|
|
if not args.video or not args.audio:
|
|||
|
|
parser.print_help()
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
adapter = LipSyncAdapter(allow_fallback=args.allow_fallback_copy)
|
|||
|
|
result = adapter.sync_lips(args.video, args.audio, args.output)
|
|||
|
|
|
|||
|
|
if result["success"]:
|
|||
|
|
print(f"\n✅ 成功: {result['output_path']}")
|
|||
|
|
sys.exit(0)
|
|||
|
|
else:
|
|||
|
|
print(f"\n❌ 失败: {result.get('error', 'Unknown error')}")
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|