feat(asr): add pluggable ONNX streaming backend and model registry

Add a second backend (sherpa-onnx) alongside llama.cpp, so native
streaming models can be selected by ID from a JSON registry. All such
models share one adapter: only the model files and a few options differ.
Models now live under the app data directory and are fetched by
scripts/fetch-models.sh. Adding a model is a data change, not code.

Measured on the same audio: the streaming backend is ~35x cheaper in CPU
than Qwen3-ASR (0.10-0.12 vs 4.27 core-seconds per audio second), shows
the first caption at 0.48-1.44s, and matches its accuracy for zh/en.

新增 sherpa-onnx 流式后端,与 llama.cpp 并存;模型按注册表 id 从应用
数据目录加载。同族模型共用一个适配器,新增模型只改 data/models.json。
实测流式后端 CPU 约为 Qwen3-ASR 的 1/35,首个字幕 0.48~1.44s,中英
准确率持平。注意预编译库是旧 C++ ABI,必须走其 C API。
This commit is contained in:
blumia committed 2026-10-12 16:06:51 +08:00
1 parent 6d64987d64
commit bd921b1ac5
15 files changed
+869 -18

No files matched your search

+43 -3
View File
@@ -62,6 +62,26 @@ if(NOT LC_MTMD_LIB)
endif()
message(STATUS "mtmd: ${LC_MTMD_LIB}")
# ---------------------------------------------------------------------------
# sherpa-onnx(X-ASR / zipformer / Nemotron 等流式 ONNX 模型)
#
# 注意:官方预编译库是用旧 C++ ABI 编译的,所以只能用它的 C API。
# 安装:把官方的 sherpa-onnx-vX.Y.Z-linux-x64-shared.tar.bz2 解到某个目录,
# 用 -DSHERPA_ONNX_ROOT=<dir> 指定(默认见下)。
# ---------------------------------------------------------------------------
set(SHERPA_ONNX_ROOT "$ENV{HOME}/.local/opt/sherpa-onnx" CACHE PATH "sherpa-onnx 安装目录")
set(LC_HAS_SHERPA OFF)
if(EXISTS "${SHERPA_ONNX_ROOT}/include/sherpa-onnx/c-api/c-api.h")
find_library(LC_SHERPA_C_API NAMES sherpa-onnx-c-api
HINTS "${SHERPA_ONNX_ROOT}/lib" REQUIRED)
find_library(LC_SHERPA_ORT NAMES onnxruntime
HINTS "${SHERPA_ONNX_ROOT}/lib" REQUIRED)
set(LC_HAS_SHERPA ON)
message(STATUS "sherpa-onnx: ${SHERPA_ONNX_ROOT}")
else()
message(WARNING "未找到 sherpa-onnx(${SHERPA_ONNX_ROOT}),流式 ONNX 后端将不可用")
endif()
# ---------------------------------------------------------------------------
# core:全部业务逻辑,刻意不依赖 Qt(方便测试与复用)
# ---------------------------------------------------------------------------
@@ -72,6 +92,12 @@ add_library(livecaption_core STATIC
src/core/streamingasr.cpp
src/core/livepipeline.cpp
)
if(LC_HAS_SHERPA)
target_sources(livecaption_core PRIVATE src/core/sherpaasr.cpp)
target_compile_definitions(livecaption_core PUBLIC LC_HAVE_SHERPA=1)
target_include_directories(livecaption_core PRIVATE "${SHERPA_ONNX_ROOT}/include")
target_link_libraries(livecaption_core PUBLIC ${LC_SHERPA_C_API} ${LC_SHERPA_ORT})
endif()
target_include_directories(livecaption_core PUBLIC src/core)
target_link_libraries(livecaption_core PUBLIC
Threads::Threads
@@ -81,6 +107,20 @@ target_link_libraries(livecaption_core PUBLIC
${LC_MTMD_LIB}
)
# ---------------------------------------------------------------------------
# models:模型注册表与存储(Qt;注册表编译进二进制)
# ---------------------------------------------------------------------------
add_library(livecaption_models STATIC
src/models/modelstore.cpp
)
target_include_directories(livecaption_models PUBLIC src/models)
target_link_libraries(livecaption_models PUBLIC Qt6::Core)
qt_add_resources(livecaption_models "registry"
PREFIX "/"
BASE "${CMAKE_CURRENT_SOURCE_DIR}"
FILES data/models.json)
# ---------------------------------------------------------------------------
# ui:悬浮窗与 Qt 适配层
# ---------------------------------------------------------------------------
@@ -97,15 +137,15 @@ target_link_libraries(livecaption_ui PUBLIC livecaption_core Qt6::Widgets)
# GUI:悬浮字幕窗
add_executable(live-caption apps/gui/main.cpp)
target_include_directories(live-caption PRIVATE apps/common)
target_link_libraries(live-caption PRIVATE livecaption_ui)
target_link_libraries(live-caption PRIVATE livecaption_ui livecaption_models)
# CLI:文件 / 流式文件 / 命令行实时;不链接 Qt Widgets
add_executable(live-caption-cli apps/cli/main.cpp)
target_include_directories(live-caption-cli PRIVATE apps/common)
target_link_libraries(live-caption-cli PRIVATE livecaption_core Qt6::Core)
target_link_libraries(live-caption-cli PRIVATE livecaption_core livecaption_models)
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
foreach(_t livecaption_core livecaption_ui live-caption live-caption-cli)
foreach(_t livecaption_core livecaption_ui livecaption_models live-caption live-caption-cli)
target_compile_options(${_t} PRIVATE -Wall -Wextra -Wno-unused-parameter)
endforeach()
endif()