From bd921b1ac5f5889da96ce44b3bcd6c405cb93e93 Mon Sep 17 00:00:00 2001 From: Wang Zichong Date: Mon, 12 Oct 2026 16:06:51 +0800 Subject: [PATCH] feat(asr): add pluggable ONNX streaming backend and model registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a second backend (sherpa-onnx) alongside llama.cpp, so native streaming models can be selected by ID from a JSON registry. All such models share one adapter: only the model files and a few options differ. Models now live under the app data directory and are fetched by scripts/fetch-models.sh. Adding a model is a data change, not code. Measured on the same audio: the streaming backend is ~35x cheaper in CPU than Qwen3-ASR (0.10-0.12 vs 4.27 core-seconds per audio second), shows the first caption at 0.48-1.44s, and matches its accuracy for zh/en. 新增 sherpa-onnx 流式后端，与 llama.cpp 并存；模型按注册表 id 从应用 数据目录加载。同族模型共用一个适配器，新增模型只改 data/models.json。 实测流式后端 CPU 约为 Qwen3-ASR 的 1/35，首个字幕 0.48~1.44s，中英 准确率持平。注意预编译库是旧 C++ ABI，必须走其 C API。 --- CMakeLists.txt | 46 ++++++++++++- README.md | 16 +++++ apps/cli/main.cpp | 28 +++++--- apps/common/appcli.h | 83 +++++++++++++++++++++++- apps/gui/main.cpp | 19 ++++-- data/models.json | 88 +++++++++++++++++++++++++ docs/asr-backends.md | 90 ++++++++++++++++++++++++++ scripts/fetch-models.sh | 74 +++++++++++++++++++++ src/core/iasr.h | 14 ++++ src/core/sherpaasr.cpp | 131 ++++++++++++++++++++++++++++++++++++++ src/core/sherpaasr.h | 56 ++++++++++++++++ src/core/streamingasr.cpp | 75 ++++++++++++++++++++++ src/core/streamingasr.h | 1 + src/models/modelstore.cpp | 110 ++++++++++++++++++++++++++++++++ src/models/modelstore.h | 56 ++++++++++++++++ 15 files changed, 869 insertions(+), 18 deletions(-) create mode 100644 data/models.json create mode 100644 docs/asr-backends.md create mode 100755 scripts/fetch-models.sh create mode 100644 src/core/sherpaasr.cpp create mode 100644 src/core/sherpaasr.h create mode 100644 src/models/modelstore.cpp create mode 100644 src/models/modelstore.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 52448e2..c59d1c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,26 @@ if(NOT LC_MTMD_LIB) endif() message(STATUS "mtmd: ${LC_MTMD_LIB}") +# --------------------------------------------------------------------------- +# sherpa-onnx（X-ASR / zipformer / Nemotron 等流式 ONNX 模型） +# +# 注意：官方预编译库是用旧 C++ ABI 编译的，所以只能用它的 C API。 +# 安装：把官方的 sherpa-onnx-vX.Y.Z-linux-x64-shared.tar.bz2 解到某个目录， +# 用 -DSHERPA_ONNX_ROOT= 指定（默认见下）。 +# --------------------------------------------------------------------------- +set(SHERPA_ONNX_ROOT "$ENV{HOME}/.local/opt/sherpa-onnx" CACHE PATH "sherpa-onnx 安装目录") +set(LC_HAS_SHERPA OFF) +if(EXISTS "${SHERPA_ONNX_ROOT}/include/sherpa-onnx/c-api/c-api.h") + find_library(LC_SHERPA_C_API NAMES sherpa-onnx-c-api + HINTS "${SHERPA_ONNX_ROOT}/lib" REQUIRED) + find_library(LC_SHERPA_ORT NAMES onnxruntime + HINTS "${SHERPA_ONNX_ROOT}/lib" REQUIRED) + set(LC_HAS_SHERPA ON) + message(STATUS "sherpa-onnx: ${SHERPA_ONNX_ROOT}") +else() + message(WARNING "未找到 sherpa-onnx（${SHERPA_ONNX_ROOT}），流式 ONNX 后端将不可用") +endif() + # --------------------------------------------------------------------------- # core：全部业务逻辑，刻意不依赖 Qt（方便测试与复用） # --------------------------------------------------------------------------- @@ -72,6 +92,12 @@ add_library(livecaption_core STATIC src/core/streamingasr.cpp src/core/livepipeline.cpp ) +if(LC_HAS_SHERPA) + target_sources(livecaption_core PRIVATE src/core/sherpaasr.cpp) + target_compile_definitions(livecaption_core PUBLIC LC_HAVE_SHERPA=1) + target_include_directories(livecaption_core PRIVATE "${SHERPA_ONNX_ROOT}/include") + target_link_libraries(livecaption_core PUBLIC ${LC_SHERPA_C_API} ${LC_SHERPA_ORT}) +endif() target_include_directories(livecaption_core PUBLIC src/core) target_link_libraries(livecaption_core PUBLIC Threads::Threads @@ -81,6 +107,20 @@ target_link_libraries(livecaption_core PUBLIC ${LC_MTMD_LIB} ) +# --------------------------------------------------------------------------- +# models：模型注册表与存储（Qt；注册表编译进二进制） +# --------------------------------------------------------------------------- +add_library(livecaption_models STATIC + src/models/modelstore.cpp +) +target_include_directories(livecaption_models PUBLIC src/models) +target_link_libraries(livecaption_models PUBLIC Qt6::Core) +qt_add_resources(livecaption_models "registry" + PREFIX "/" + BASE "${CMAKE_CURRENT_SOURCE_DIR}" + FILES data/models.json) + + # --------------------------------------------------------------------------- # ui：悬浮窗与 Qt 适配层 # --------------------------------------------------------------------------- @@ -97,15 +137,15 @@ target_link_libraries(livecaption_ui PUBLIC livecaption_core Qt6::Widgets) # GUI：悬浮字幕窗 add_executable(live-caption apps/gui/main.cpp) target_include_directories(live-caption PRIVATE apps/common) -target_link_libraries(live-caption PRIVATE livecaption_ui) +target_link_libraries(live-caption PRIVATE livecaption_ui livecaption_models) # CLI：文件 / 流式文件 / 命令行实时；不链接 Qt Widgets add_executable(live-caption-cli apps/cli/main.cpp) target_include_directories(live-caption-cli PRIVATE apps/common) -target_link_libraries(live-caption-cli PRIVATE livecaption_core Qt6::Core) +target_link_libraries(live-caption-cli PRIVATE livecaption_core livecaption_models) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - foreach(_t livecaption_core livecaption_ui live-caption live-caption-cli) + foreach(_t livecaption_core livecaption_ui livecaption_models live-caption live-caption-cli) target_compile_options(${_t} PRIVATE -Wall -Wextra -Wno-unused-parameter) endforeach() endif() diff --git a/README.md b/README.md index f0c3102..025fd7f 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,22 @@ cmake --build build -j ctest --test-dir build --output-on-failure ``` +## 推理后端 + +支持两条后端，用 `--asr-backend` 切换（详见 [docs/asr-backends.md](docs/asr-backends.md)）： + +- **`llama`（默认）**：llama.cpp/ggml + Qwen3-ASR 的 GGUF 模型 —— 准确率高、52 种语言，但每步成本随段落增长。 +- **`sherpa`**：sherpa-onnx + 流式 ONNX 模型（X-ASR / zipformer / Nemotron） —— 原生流式，每步成本恒定，CPU 约为前者的 1/35，中英准确率持平。 + +模型都放在 `~/.local/share/live-caption/models/`，注册表是 `data/models.json` +（**加模型只加一条数据，不改代码**），用 `scripts/fetch-models.sh` 下载。 + +```bash +# 流式后端 +./build/live-caption-cli --asr-backend sherpa --asr-model zipformer-zh-2025 +./build/live-caption-cli --asr-backend sherpa --asr-model nemotron35-560ms --language ja +``` + ## 代码结构 ``` diff --git a/apps/cli/main.cpp b/apps/cli/main.cpp index bc95e3d..e0c9c1c 100644 --- a/apps/cli/main.cpp +++ b/apps/cli/main.cpp @@ -15,6 +15,7 @@ #include "appcli.h" #include "audio.h" #include "livepipeline.h" +#include "iasr.h" #include "qwen3asr.h" #include "streamingasr.h" #include "transcript.h" @@ -25,7 +26,7 @@ std::atomic g_stop{false}; extern "C" void handleSigInt(int) { g_stop = true; } -int runFile(lc::Qwen3Asr & asr, const QString & path) +int runFile(lc::IAsr & asr, const QString & path) { std::vector pcm; std::string err; @@ -54,7 +55,7 @@ int runFile(lc::Qwen3Asr & asr, const QString & path) return 0; } -int runStreamFile(lc::Qwen3Asr & asr, const QString & path, const lc::StreamOptions & sop) +int runStreamFile(lc::IAsr & asr, const QString & path, const lc::StreamOptions & sop) { std::vector pcm; std::string err; @@ -94,7 +95,7 @@ int runStreamFile(lc::Qwen3Asr & asr, const QString & path, const lc::StreamOpti return 0; } -int runCliLive(lc::Qwen3Asr & asr, const lc::StreamOptions & sop) +int runCliLive(lc::IAsr & asr, const lc::StreamOptions & sop) { lc::LivePipelineOptions popt; popt.stream = sop; @@ -135,19 +136,28 @@ int main(int argc, char ** argv) parser.addHelpOption(); lc::app::addModelOptions(parser); lc::app::addStreamOptions(parser); + lc::app::addBackendOptions(parser); parser.addOption(QCommandLineOption("file", "转写一个 wav 文件后退出", "path")); parser.addOption(QCommandLineOption("stream-file", "以流式模式转写 wav 文件后退出", "path")); parser.addOption(QCommandLineOption("cli-live", "实时转写字幕，只用命令行输出（不开窗口）")); parser.process(app); const lc::AsrOptions asrOpt = lc::app::asrOptions(parser); - const lc::StreamOptions sop = lc::app::streamOptions(parser); + lc::StreamOptions sop = lc::app::streamOptions(parser); - lc::Qwen3Asr asr; - if (!lc::app::loadModel(asr, asrOpt)) return 1; + std::string err; + std::unique_ptr asr = lc::app::makeAsr(parser, asrOpt, &err); + if (!asr) { + std::fprintf(stderr, "%s\n", err.c_str()); + return 1; + } + // 原生流式识别器很便宜，用更细的更新粒度换低延迟 + if (asr->kind() == lc::AsrKind::NativeStreaming && !parser.isSet("stream-chunk")) { + sop.chunkSec = 0.32; + } - if (parser.isSet("file")) return runFile(asr, parser.value("file")); - if (parser.isSet("stream-file")) return runStreamFile(asr, parser.value("stream-file"), sop); + if (parser.isSet("file")) return runFile(*asr, parser.value("file")); + if (parser.isSet("stream-file")) return runStreamFile(*asr, parser.value("stream-file"), sop); - return runCliLive(asr, sop); + return runCliLive(*asr, sop); } diff --git a/apps/common/appcli.h b/apps/common/appcli.h index 3bc215b..6295e5b 100644 --- a/apps/common/appcli.h +++ b/apps/common/appcli.h @@ -1,13 +1,19 @@ #pragma once #include +#include #include #include #include -#include "qwen3asr.h" +#include "iasr.h" +#include "modelstore.h" #include "streamingasr.h" +#include "qwen3asr.h" +#if defined(LC_HAVE_SHERPA) +#include "sherpaasr.h" +#endif // 两个可执行程序（GUI / CLI）共用的命令行选项定义与解析。 namespace lc::app { @@ -87,4 +93,79 @@ inline bool loadModel(Qwen3Asr & asr, const AsrOptions & opt) return true; } +// ---- 推理后端选择 ---- +// +// llama 后端：Qwen3-ASR 的 GGUF 模型（--model / --mmproj） +// sherpa 后端：流式 ONNX 模型，按注册表 id 从模型存储目录取（--asr-model ） +inline void addBackendOptions(QCommandLineParser & p) +{ + p.addOption(makeOption("asr-backend", "推理后端: llama（Qwen3-ASR GGUF）或 sherpa（流式 ONNX）", + "kind", "llama")); + p.addOption(makeOption("asr-model", "模型注册表里的 id（sherpa 后端用它定位文件）", "id")); + p.addOption(makeOption("model-dir", "覆盖模型存储目录（默认 ~/.local/share/live-caption/models）", "dir")); + p.addOption(makeOption("language", "语言提示，多语言模型用（zh / en / ja …）；留空=自动", "lang")); +} + +// 按命令行选择并加载后端；失败时返回 nullptr 并把原因写进 err +inline std::unique_ptr makeAsr(const QCommandLineParser & p, + const AsrOptions & asrOpt, std::string * err) +{ + const QString dir = p.value("model-dir"); + if (!dir.isEmpty()) models::ModelStore::setRootOverride(dir); + + const QString backend = p.value("asr-backend"); + const QString modelId = p.value("asr-model"); + const QString lang = p.value("language"); + + const models::ModelInfo * info = + modelId.isEmpty() ? nullptr : models::ModelStore::find(modelId); + if (!modelId.isEmpty() && !info) { + if (err) *err = "注册表里没有模型: " + modelId.toStdString(); + return nullptr; + } + + const bool useSherpa = (backend == QLatin1String("sherpa")) + || (info && info->backend == models::Backend::SherpaOnline); + + if (useSherpa) { +#if defined(LC_HAVE_SHERPA) + if (!info) { + if (err) *err = "sherpa 后端需要 --asr-model （见 data/models.json）"; + return nullptr; + } + QStringList missing; + if (!models::ModelStore::isInstalled(*info, &missing)) { + if (err) { + *err = "模型未安装，缺失: " + missing.join(QStringLiteral(", ")).toStdString() + + "\n 可运行: scripts/fetch-models.sh " + info->id.toStdString(); + } + return nullptr; + } + SherpaAsr::Options o; + o.encoder = info->filePath(QStringLiteral("encoder")).toStdString(); + o.decoder = info->filePath(QStringLiteral("decoder")).toStdString(); + o.joiner = info->filePath(QStringLiteral("joiner")).toStdString(); + o.tokens = info->filePath(QStringLiteral("tokens")).toStdString(); + o.threads = 1; // 这类模型单线程最快（多线程反而被同步开销拖慢） + if (info->languages.size() > 1 && !lang.isEmpty()) { + o.language = lang.toStdString(); // 多语言模型（Nemotron）需要语言提示 + } + auto a = std::make_unique(); + if (!a->load(o, err)) return nullptr; + return a; +#else + if (err) *err = "编译时未找到 sherpa-onnx（-DSHERPA_ONNX_ROOT=...）"; + return nullptr; +#endif + } + + auto a = std::make_unique(); + if (asrOpt.modelPath.empty() || asrOpt.mmprojPath.empty()) { + if (err) *err = "llama 后端需要 --model 和 --mmproj（或改用 --asr-backend sherpa --asr-model ）"; + return nullptr; + } + if (!a->load(asrOpt, err)) return nullptr; + return a; +} + } // namespace lc::app diff --git a/apps/gui/main.cpp b/apps/gui/main.cpp index f7c5d58..36bf1d5 100644 --- a/apps/gui/main.cpp +++ b/apps/gui/main.cpp @@ -10,13 +10,14 @@ #include "appcli.h" #include "captionwindow.h" #include "livetranscriber.h" +#include "iasr.h" #include "qwen3asr.h" #include "streamingasr.h" #include "transcript.h" namespace { -int runGui(lc::Qwen3Asr & asr, const lc::StreamOptions & sop, +int runGui(lc::IAsr & asr, const lc::StreamOptions & sop, const QString & screenshotPath, int screenshotDelaySec, int opacityPercent) { lc::CaptionWindow window; @@ -84,18 +85,26 @@ int main(int argc, char ** argv) parser.addHelpOption(); lc::app::addModelOptions(parser); lc::app::addStreamOptions(parser); + lc::app::addBackendOptions(parser); parser.addOption(lc::app::makeOption("opacity", "字幕背景不透明度，取值 20-100", "n", "84")); parser.addOption(lc::app::makeOption("screenshot", "启动后自动截图并退出（验证渲染用）", "path")); parser.addOption(lc::app::makeOption("screenshot-delay", "截图前等待秒数", "sec", "6")); parser.process(app); const lc::AsrOptions asrOpt = lc::app::asrOptions(parser); - const lc::StreamOptions sop = lc::app::streamOptions(parser); + lc::StreamOptions sop = lc::app::streamOptions(parser); - lc::Qwen3Asr asr; - if (!lc::app::loadModel(asr, asrOpt)) return 1; + std::string err; + std::unique_ptr asr = lc::app::makeAsr(parser, asrOpt, &err); + if (!asr) { + std::fprintf(stderr, "%s\n", err.c_str()); + return 1; + } + if (asr->kind() == lc::AsrKind::NativeStreaming && !parser.isSet("stream-chunk")) { + sop.chunkSec = 0.32; + } - return runGui(asr, sop, + return runGui(*asr, sop, parser.value("screenshot"), parser.value("screenshot-delay").toInt(), parser.value("opacity").toInt()); diff --git a/data/models.json b/data/models.json new file mode 100644 index 0000000..8977066 --- /dev/null +++ b/data/models.json @@ -0,0 +1,88 @@ +{ + "version": 1, + "comment": "live-caption 的模型注册表。新增模型 = 在这里加一条，不需要改代码。", + "models": [ + { + "id": "zipformer-zh-2025", + "name": "中文流式（k2-fsa 官方 2025-06-30）", + "backend": "sherpa-online", + "languages": ["zh"], + "license": "Apache-2.0", + "note": "154MB，实测中文逐字正确；首个字幕 0.48s，约 0.10 核·秒/秒音频", + "files": { + "encoder": { + "name": "encoder.int8.onnx", + "url": "https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30/resolve/main/encoder.int8.onnx" + }, + "decoder": { + "name": "decoder.onnx", + "url": "https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30/resolve/main/decoder.onnx" + }, + "joiner": { + "name": "joiner.int8.onnx", + "url": "https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30/resolve/main/joiner.int8.onnx" + }, + "tokens": { + "name": "tokens.txt", + "url": "https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30/resolve/main/tokens.txt" + } + }, + "options": {} + }, + { + "id": "xasr-zh-en-480", + "name": "中英流式（X-ASR-zh-en, 480ms）", + "backend": "sherpa-online", + "languages": ["zh", "en"], + "license": "Apache-2.0", + "note": "587MB，实测中英准确率与 Qwen3-ASR 持平；首个字幕 1.44s", + "files": { + "encoder": { + "name": "encoder-480ms.onnx", + "url": "https://hf-mirror.com/GilgameshWind/X-ASR-zh-en/resolve/main/deployment/models/chunk-480ms-model/encoder-480ms.onnx" + }, + "decoder": { + "name": "decoder-480ms.onnx", + "url": "https://hf-mirror.com/GilgameshWind/X-ASR-zh-en/resolve/main/deployment/models/chunk-480ms-model/decoder-480ms.onnx" + }, + "joiner": { + "name": "joiner-480ms.onnx", + "url": "https://hf-mirror.com/GilgameshWind/X-ASR-zh-en/resolve/main/deployment/models/chunk-480ms-model/joiner-480ms.onnx" + }, + "tokens": { + "name": "tokens.txt", + "url": "https://hf-mirror.com/GilgameshWind/X-ASR-zh-en/resolve/main/deployment/models/chunk-480ms-model/tokens.txt" + } + }, + "options": {} + }, + { + "id": "nemotron35-560ms", + "name": "多语言流式（Nemotron 3.5, 560ms, 含日语）", + "backend": "sherpa-online", + "languages": ["en", "ja", "ko", "vi", "ru", "ar", "hi", "fr", "de", "es", "it", "pt", "nl", "tr", "uk", "zh"], + "license": "OpenMDW-1.1", + "note": "651MB，40 种语言 + 内置语言识别；实测中英日都可用但都有瑕疵", + "optionsHint": "language", + "files": { + "encoder": { + "name": "encoder.int8.onnx", + "url": "https://hf-mirror.com/csukuangfj2/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/resolve/main/encoder.int8.onnx" + }, + "decoder": { + "name": "decoder.int8.onnx", + "url": "https://hf-mirror.com/csukuangfj2/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/resolve/main/decoder.int8.onnx" + }, + "joiner": { + "name": "joiner.int8.onnx", + "url": "https://hf-mirror.com/csukuangfj2/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/resolve/main/joiner.int8.onnx" + }, + "tokens": { + "name": "tokens.txt", + "url": "https://hf-mirror.com/csukuangfj2/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/resolve/main/tokens.txt" + } + }, + "options": {} + } + ] +} diff --git a/docs/asr-backends.md b/docs/asr-backends.md new file mode 100644 index 0000000..670a28b --- /dev/null +++ b/docs/asr-backends.md @@ -0,0 +1,90 @@ +# 推理后端与模型管理 + +## 结构：2 个实现覆盖所有模型 + +实测确认了一件很重要的事：**不同模型不需要各写一套代码**。所有流式 ONNX 模型走的是同一套 API，差别只是「指向哪个模型文件 + 几行选项」。 + +``` +IAsr（src/core/iasr.h） +├─ Qwen3Asr llama.cpp / ggml GGUF 模型（Qwen3-ASR） AsrKind::PrefixRollback +└─ SherpaAsr sherpa-onnx / onnxruntime + ├─ X-ASR-zh-en 中英流式 + ├─ zipformer-zh-2025 中文流式（k2-fsa 官方） + └─ Nemotron-3.5 40 语言流式（含日语） AsrKind::NativeStreaming +``` + +两条路径的区别（`StreamingAsr` 里按 `IAsr::kind()` 分流）： + +| | PrefixRollback | NativeStreaming | +|---|---|---| +| 模型类型 | 离线模型（LLM 解码器） | 有状态流式识别器 | +| 上层要做的 | 切块 + 前缀回滚 + KV 复用 | 只喂音频，取当前整句文本 | +| 每步成本 | 随句子长度增长 | 恒定 | +| 已稳定/不稳定 | 用 token 回滚算 | 用「与上次的公共前缀」算 | + +## 模型注册表与存储 + +**新增模型 = 加一条数据，不改代码。** + +- 注册表：`data/models.json`（编译进二进制，同时也给脚本用） +- 存储位置：`~/.local/share/live-caption/models//` + （即 `QStandardPaths::GenericDataLocation + /live-caption/models`） +- 下载：`scripts/fetch-models.sh [id...]` / `--all`；已存在的文件会跳过 + +```bash +./scripts/fetch-models.sh # 列出模型与安装状态 +./scripts/fetch-models.sh zipformer-zh-2025 +``` + +## 用法 + +```bash +# 默认后端：llama.cpp + Qwen3-ASR GGUF +./build/live-caption-cli --model M.gguf --mmproj MM.gguf + +# 流式 ONNX 后端：按注册表 id 从模型目录取 +./build/live-caption-cli --asr-backend sherpa --asr-model zipformer-zh-2025 +./build/live-caption-cli --asr-backend sherpa --asr-model nemotron35-560ms --language ja + +# 覆盖模型目录 / 语言提示 +./build/live-caption-cli --asr-backend sherpa --asr-model ... --model-dir /path/to/models +``` + +## 实测对比（同一批音频，1 线程） + +| 模型 | 中文 | 英文 | 日语 | RTF | 核·秒/秒音频 | 首个字幕 | 体积 | 许可 | +|---|---|---|---|---|---|---|---|---| +| Qwen3-ASR 0.6B（llama） | ✅ | ✅ | ✅ | 0.56 | 4.27 | ~2.5s | 973MB | Apache-2.0 | +| zipformer-zh-2025 | ✅ | — | — | 0.098 | **0.10** | **0.48s** | 167MB | Apache-2.0 | +| X-ASR-zh-en 480ms | ✅ | ✅ | — | 0.122 | 0.12 | 1.44s | 587MB | Apache-2.0 | +| Nemotron 3.5 560ms | ⚠️ 漏字 | ⚠️ 若干错词 | ⚠️ 有错字 | 0.35 | 0.35 | 1.12s | 651MB | OpenMDW-1.1 | + +> 静态模型（zipformer-zh-2025）的多线程是**负优化**：onnxruntime 对小模型开多线程， +> 同步/自旋开销会让它更慢（8 线程比 1 线程慢 2.6 倍、CPU 高 28 倍）。所以适配器里固定 +> `threads = 1`。 + +## 两个坑 + +1. **sherpa-onnx 的预编译库是旧 C++ ABI**（`_GLIBCXX_USE_CXX11_ABI=0`），和 Qt 的新 ABI 冲突， + C++ API 链接不上。**必须用它的 C API**（`sherpa-onnx/c-api/c-api.h`，`extern "C"`，无名字改写）。 +2. **结束时要补一小段静音**（`streamFinish()` 里补 0.3s）再 `InputFinished`，否则最后一个字会被截掉。 + +## 自动语言识别 + +| 模型 | 自带 LID | +|---|---| +| Qwen3-ASR | ✅ 输出 `language X`（实测中英都准）| +| Nemotron 3.5 | ✅ `language=auto`，但实测会串（日语开头吐韩文）| +| X-ASR / zipformer | ❌ 固定语言对，不出语言标签 | + +LID 不必由 ASR 模型提供：Chrome 就带了个独立的 2.5MB `ONDEVICE_langid.tflite`（我们在它的语言包目录里看到了）， +Windows 同样。sherpa-onnx 也有专门的语种识别模型。所以**「按语言装包 + 用户选语言或小 LID 模型」**是可行路线。 + +## 推荐组合（按语言装包） + +| 语言 | 实时预览 | 定稿 | +|---|---|---| +| 中文 | `zipformer-zh-2025`（最快最准） | Qwen3-ASR | +| 英文 | `xasr-zh-en-480` | Qwen3-ASR | +| 日语 | `nemotron35-560ms --language ja`（质量一般） | Qwen3-ASR（开源日语第一） | +| 其他 | — | Qwen3-ASR（52 语言兜底） | diff --git a/scripts/fetch-models.sh b/scripts/fetch-models.sh new file mode 100755 index 0000000..1fa09b1 --- /dev/null +++ b/scripts/fetch-models.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# 把模型注册表里的模型下载到应用数据目录（QStandardPaths::AppDataLocation）。 +# +# 用法: +# ./fetch-models.sh # 列出所有模型和安装状态 +# ./fetch-models.sh [...] # 下载指定模型 +# ./fetch-models.sh --all # 下载全部 +# +# 目录规则（与程序里 ModelStore 保持一致）： +# ${XDG_DATA_HOME:-~/.local/share}/live-caption/models// +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REG="$ROOT/data/models.json" +DEST="${XDG_DATA_HOME:-$HOME/.local/share}/live-caption/models" + +[ -f "$REG" ] || { echo "找不到注册表: $REG" >&2; exit 1; } + +ids=$(python3 -c " +import json,sys +d=json.load(open('$REG')) +print(' '.join(m['id'] for m in d['models']))") + +show_status() { + python3 - "$REG" "$DEST" <<'PY' +import json, os, sys +reg, dest = sys.argv[1], sys.argv[2] +d = json.load(open(reg)) +for m in d['models']: + mid = m['id']; mdir = os.path.join(dest, mid) + files = m['files'] + have = sum(1 for f in files.values() if os.path.exists(os.path.join(mdir, f['name']))) + total = len(files) + mark = "已安装" if have == total else ("部分" if have else "未安装") + langs = ",".join(m['languages'][:4]) + ("…" if len(m['languages']) > 4 else "") + print(f" [{have}/{total} {mark}] {mid:22s} {langs:18s} {m.get('license','')}") + print(f" {m['name']}") +PY +} + +download_one() { + local id="$1" + local dir="$DEST/$id" + mkdir -p "$dir" + echo "==> $id → $dir" + python3 - "$REG" "$id" <<'PY' | while IFS=$'\t' read -r name url; do +import json, sys +d = json.load(open(sys.argv[1])); mid = sys.argv[2] +m = next(x for x in d['models'] if x['id'] == mid) +for f in m['files'].values(): + print(f"{f['name']}\t{f['url']}") +PY + if [ -f "$dir/$name" ]; then + echo " 已存在，跳过: $name" + continue + fi + echo " 下载 $name" + if command -v aria2c >/dev/null 2>&1; then + aria2c -x16 -s16 -k4M --summary-interval=0 -d "$dir" -o "$name" "$url" >/dev/null + else + curl -fL --progress-bar -o "$dir/$name" "$url" + fi + done + echo " 完成" +} + +case "${1:-}" in + "" ) echo "模型存储目录: $DEST"; echo; show_status ;; + --all) shift; for id in $ids; do download_one "$id"; done ;; + *) for id in "$@"; do + echo "$ids" | tr ' ' '\n' | grep -qx "$id" || { echo "未知模型: $id" >&2; exit 1; } + download_one "$id" + done ;; +esac diff --git a/src/core/iasr.h b/src/core/iasr.h index 2bb2d3a..6373770 100644 --- a/src/core/iasr.h +++ b/src/core/iasr.h @@ -30,15 +30,29 @@ struct AsrStreamStep { double inferSeconds = 0.0; // 以上合计 }; +// 推理后端的种类，决定上层用哪条流式路径 +enum class AsrKind { + PrefixRollback, // 离线模型 + 前缀回滚（Qwen3-ASR / llama.cpp） + NativeStreaming, // 有状态流式识别器（sherpa-onnx 的 transducer 系） +}; + // 把推理后端抽象出来，好处是流式策略（StreamingAsr）和实时管线（LivePipeline） // 可以用一个假的实现来测，不必加载真模型。 class IAsr { public: virtual ~IAsr() = default; + virtual AsrKind kind() const { return AsrKind::PrefixRollback; } + // 期望的输入采样率（Qwen3-ASR 为 16000） virtual int sampleRate() const = 0; + // ---- NativeStreaming 专用 ---- + // 喂入音频，返回"这一句到目前为止"的完整文本（识别器会随时修正前面的词） + virtual std::string streamFeed(const float * /*pcm*/, size_t /*n*/) { return {}; } + // 结束这一句（把尾部 flush 出来），返回最终文本 + virtual std::string streamFinish() { return {}; } + // 整段转写：把 prefix 当作"已经生成的 assistant 文本"，从那里继续生成 virtual AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples, const std::string & prefix) = 0; diff --git a/src/core/sherpaasr.cpp b/src/core/sherpaasr.cpp new file mode 100644 index 0000000..b65c69e --- /dev/null +++ b/src/core/sherpaasr.cpp @@ -0,0 +1,131 @@ +#include "sherpaasr.h" + +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +namespace lc { + +struct SherpaAsr::Impl { + Options opt; + const SherpaOnnxOnlineRecognizer * rec = nullptr; + const SherpaOnnxOnlineStream * stream = nullptr; + + std::string resultText() const + { + if (!rec || !stream) return {}; + const SherpaOnnxOnlineRecognizerResult * r = + SherpaOnnxGetOnlineStreamResult(rec, stream); + if (!r) return {}; + std::string text = r->text ? r->text : ""; + SherpaOnnxDestroyOnlineRecognizerResult(r); + return text; + } + + void drain() + { + while (rec && stream && SherpaOnnxIsOnlineStreamReady(rec, stream)) { + SherpaOnnxDecodeOnlineStream(rec, stream); + } + } +}; + +SherpaAsr::SherpaAsr() : d(new Impl) {} + +SherpaAsr::~SherpaAsr() +{ + if (!d) return; + if (d->stream) SherpaOnnxDestroyOnlineStream(d->stream); + if (d->rec) SherpaOnnxDestroyOnlineRecognizer(d->rec); + delete d; +} + +bool SherpaAsr::load(const Options & opt, std::string * error) +{ + const auto fail = [error](const std::string & msg) { + if (error) *error = msg; + return false; + }; + + d->opt = opt; + + SherpaOnnxOnlineRecognizerConfig cfg; + std::memset(&cfg, 0, sizeof(cfg)); + cfg.feat_config.sample_rate = 16000; + cfg.feat_config.feature_dim = 80; + cfg.model_config.transducer.encoder = opt.encoder.c_str(); + cfg.model_config.transducer.decoder = opt.decoder.c_str(); + cfg.model_config.transducer.joiner = opt.joiner.c_str(); + cfg.model_config.tokens = opt.tokens.c_str(); + cfg.model_config.num_threads = opt.threads; + cfg.model_config.provider = "cpu"; + cfg.model_config.modeling_unit = ""; + cfg.decoding_method = "greedy_search"; + cfg.max_active_paths = 4; + cfg.enable_endpoint = 0; // 分段由我们的 VAD 负责 + + d->rec = SherpaOnnxCreateOnlineRecognizer(&cfg); + if (!d->rec) return fail("创建 sherpa-onnx 识别器失败（检查模型文件）"); + + d->stream = SherpaOnnxCreateOnlineStream(d->rec); + if (!d->stream) return fail("创建 sherpa-onnx 流失败"); + if (!opt.language.empty()) { + SherpaOnnxOnlineStreamSetOption(d->stream, "language", opt.language.c_str()); + } + return true; +} + +std::string SherpaAsr::streamFeed(const float * pcm, size_t n) +{ + if (!d->rec || !d->stream) return {}; + SherpaOnnxOnlineStreamAcceptWaveform(d->stream, 16000, pcm, (int32_t) n); + d->drain(); + return d->resultText(); +} + +std::string SherpaAsr::streamFinish() +{ + if (!d->rec || !d->stream) return {}; + // 先补一小段静音再结束，否则最后一个字常常来不及输出 + const size_t pad = (size_t) (16000 * 0.3); + const std::vector silence(pad, 0.0f); + SherpaOnnxOnlineStreamAcceptWaveform(d->stream, 16000, silence.data(), (int32_t) pad); + d->drain(); + // 通知不再有输入，把尾部一并解出来 + SherpaOnnxOnlineStreamInputFinished(d->stream); + d->drain(); + return d->resultText(); +} + +void SherpaAsr::streamReset() +{ + if (!d->rec || !d->stream) return; + SherpaOnnxOnlineStreamReset(d->rec, d->stream); +} + +AsrResult SherpaAsr::transcribeWithPrefix(const float *, size_t, const std::string &) +{ + AsrResult r; + r.ok = false; + r.error = "该后端只支持原生流式（NativeStreaming），不支持整段/前缀回滚模式"; + return r; +} + +AsrStreamStep SherpaAsr::streamAppend(const float *, size_t, const std::string &, int) +{ + AsrStreamStep s; + s.ok = false; + s.error = "该后端只支持原生流式（NativeStreaming），请用 streamFeed()"; + return s; +} + +AsrStreamStep SherpaAsr::streamAppendTurn(const float *, size_t, int) +{ + AsrStreamStep s; + s.ok = false; + s.error = "该后端只支持原生流式（NativeStreaming），请用 streamFeed()"; + return s; +} + +} // namespace lc diff --git a/src/core/sherpaasr.h b/src/core/sherpaasr.h new file mode 100644 index 0000000..6848471 --- /dev/null +++ b/src/core/sherpaasr.h @@ -0,0 +1,56 @@ +#pragma once + +#include + +#include "iasr.h" + +struct SherpaOnnxOnlineRecognizer; +struct SherpaOnnxOnlineStream; + +namespace lc { + +// sherpa-onnx 的流式识别器（X-ASR / zipformer / Nemotron …）。 +// +// 用的是它的 **C API** 而不是 C++ API：预编译库是用旧 C++ ABI +// （_GLIBCXX_USE_CXX11_ABI=0）编的，和 Qt 的新 ABI 不兼容；C API 是 extern "C"， +// 不经过 C++ 名字改写，没有这个问题。 +// +// 这类模型是**有状态流式**：喂音频 → 随时取当前整句文本（会修正前面的词）。 +class SherpaAsr : public IAsr { +public: + struct Options { + std::string encoder; + std::string decoder; + std::string joiner; + std::string tokens; + int threads = 1; // 注意：这类模型单线程反而最快 + std::string language; // 多语言模型（如 Nemotron）用；空 = 不设置 + }; + + SherpaAsr(); + ~SherpaAsr() override; + + SherpaAsr(const SherpaAsr &) = delete; + SherpaAsr & operator=(const SherpaAsr &) = delete; + + bool load(const Options & opt, std::string * error); + + AsrKind kind() const override { return AsrKind::NativeStreaming; } + int sampleRate() const override { return 16000; } + + std::string streamFeed(const float * pcm, size_t n) override; + std::string streamFinish() override; + void streamReset() override; + + // 这些只对 PrefixRollback 后端有意义；这里给出明确的失败信息 + AsrResult transcribeWithPrefix(const float *, size_t, const std::string &) override; + AsrStreamStep streamAppend(const float *, size_t, const std::string &, int) override; + AsrStreamStep streamAppendTurn(const float *, size_t, int) override; + std::string dropLastTokens(const std::string & text, int) const override { return text; } + +private: + struct Impl; + Impl * d = nullptr; +}; + +} // namespace lc diff --git a/src/core/streamingasr.cpp b/src/core/streamingasr.cpp index b7b8d0d..bbaf949 100644 --- a/src/core/streamingasr.cpp +++ b/src/core/streamingasr.cpp @@ -1,9 +1,37 @@ #include "streamingasr.h" +#include +#include +#include + #include "transcript.h" namespace lc { +namespace { + +// 去掉首尾空白（含末尾空格），用于 sherpa 这类可能带前导空格的输出 +std::string trimAscii(const std::string & s) +{ + const auto notSpace = [](unsigned char c) { return !std::isspace(c); }; + auto b = std::find_if(s.begin(), s.end(), notSpace); + auto e = std::find_if(s.rbegin(), s.rend(), notSpace).base(); + if (b >= e) return {}; + return std::string(b, e); +} + +// 两个字符串的最长公共前缀，切在 UTF-8 字符边界上 +size_t utf8Lcp(const std::string & a, const std::string & b) +{ + const size_t n = std::min(a.size(), b.size()); + size_t i = 0; + while (i < n && a[i] == b[i]) ++i; + while (i > 0 && (static_cast(a[i]) & 0xC0) == 0x80) --i; + return i; +} + +} // namespace + StreamingAsr::StreamingAsr(IAsr & asr, StreamOptions opt) : asr_(asr), opt_(opt) { @@ -18,6 +46,7 @@ void StreamingAsr::reset() totalSeconds_ = 0.0; committedRaw_.clear(); lastRawFull_.clear(); + previousFull_.clear(); language_.clear(); chunkIndex_ = 0; } @@ -50,6 +79,14 @@ void StreamingAsr::feed(const float * pcm, size_t n, void StreamingAsr::flush(const std::function & onStep) { + // 原生流式：把尾部 flush 出来即收尾（streamFinish 会补齐最后一个字） + if (asr_.kind() == AsrKind::NativeStreaming) { + std::vector chunk; + chunk.swap(pending_); + runStep(chunk, true, onStep); + return; + } + if (!pending_.empty()) { std::vector chunk; chunk.swap(pending_); @@ -86,6 +123,44 @@ void StreamingAsr::flush(const std::function & onStep) void StreamingAsr::runStep(const std::vector & audio, bool finalStep, const std::function & onStep) { + // ---- 原生流式（sherpa-onnx 这类有状态识别器）---- + // 它自己维护状态并随时给出整句文本，会修正前面的词；我们只负责： + // 1) 把音频喂进去 + // 2) 用"与上一次的公共前缀"作为已稳定部分，其余算不稳定尾部 + if (asr_.kind() == AsrKind::NativeStreaming) { + Step s; + s.chunkIndex = chunkIndex_; + + const auto t0 = std::chrono::steady_clock::now(); + std::string cur; + if (!audio.empty()) cur = asr_.streamFeed(audio.data(), audio.size()); + if (finalStep) cur = asr_.streamFinish(); + const auto t1 = std::chrono::steady_clock::now(); + + s.ok = true; + s.inferSeconds = std::chrono::duration(t1 - t0).count(); + s.full = trimAscii(cur); + + const std::string stable = s.full.substr(0, utf8Lcp(previousFull_, s.full)); + previousFull_ = s.full; + s.committed = stable; + s.tail = s.full.substr(stable.size()); + s.language = language_; + + totalSeconds_ += (double) audio.size() / asr_.sampleRate(); + s.windowSeconds = totalSeconds_; + + ++chunkIndex_; + if (onStep) onStep(s); + + if (finalStep) { + // 这一句结束：并入历史，等下一次 reset() + historyText_ += s.full; + previousFull_.clear(); + } + return; + } + // 冷启动阶段不给前缀；之后把"已稳定文本"回喂给模型 const std::string prefix = (chunkIndex_ >= opt_.unfixedChunkNum) ? committedRaw_ : std::string(); diff --git a/src/core/streamingasr.h b/src/core/streamingasr.h index 27de93c..20757db 100644 --- a/src/core/streamingasr.h +++ b/src/core/streamingasr.h @@ -85,6 +85,7 @@ private: double totalSeconds_ = 0.0; std::string committedRaw_; // 当前段的原始已提交文本（含 language 头，用于回喂） std::string lastRawFull_; // 上一次完整的原始文本（含不稳定尾部） + std::string previousFull_; // 原生流式：上一次的完整文本（用来算已稳定的前缀） std::string historyText_; // 此前各段已提交的正文（已剥离 language 头） std::string language_; int chunkIndex_ = 0; diff --git a/src/models/modelstore.cpp b/src/models/modelstore.cpp new file mode 100644 index 0000000..0a422fa --- /dev/null +++ b/src/models/modelstore.cpp @@ -0,0 +1,110 @@ +#include "modelstore.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace lc::models { + +namespace { + +QString g_rootOverride; + +const std::vector & parseRegistry() +{ + static std::vector cache; + static bool loaded = false; + if (loaded) return cache; + loaded = true; + + QFile f(QStringLiteral(":/data/models.json")); + if (!f.open(QIODevice::ReadOnly)) { + qWarning("无法读取内嵌的模型注册表"); + return cache; + } + const QJsonDocument doc = QJsonDocument::fromJson(f.readAll()); + for (const QJsonValue & v : doc.object().value(QStringLiteral("models")).toArray()) { + const QJsonObject o = v.toObject(); + ModelInfo m; + m.id = o.value(QStringLiteral("id")).toString(); + m.name = o.value(QStringLiteral("name")).toString(); + m.license = o.value(QStringLiteral("license")).toString(); + m.note = o.value(QStringLiteral("note")).toString(); + m.backend = (o.value(QStringLiteral("backend")).toString() == QLatin1String("llama-cpp")) + ? Backend::LlamaCpp + : Backend::SherpaOnline; + for (const QJsonValue & l : o.value(QStringLiteral("languages")).toArray()) { + m.languages << l.toString(); + } + const QJsonObject files = o.value(QStringLiteral("files")).toObject(); + for (auto it = files.begin(); it != files.end(); ++it) { + const QJsonObject fo = it.value().toObject(); + m.files.push_back({it.key(), + fo.value(QStringLiteral("name")).toString(), + fo.value(QStringLiteral("url")).toString()}); + } + cache.push_back(std::move(m)); + } + return cache; +} + +} // namespace + +void ModelStore::setRootOverride(const QString & dir) { g_rootOverride = dir; } + +QString ModelStore::root() +{ + if (!g_rootOverride.isEmpty()) return g_rootOverride; + // 用 GenericDataLocation（~/.local/share）+ 固定名，保证 CLI/GUI 一致， + // 也和 scripts/fetch-models.sh 的目录规则对得上。 + const QString base = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation); + return base + QStringLiteral("/live-caption/models"); +} + +QString ModelStore::dirFor(const QString & id) { return root() + QLatin1Char('/') + id; } + +QString ModelInfo::filePath(const QString & key) const +{ + for (const ModelFile & f : files) { + if (f.key == key) return ModelStore::dirFor(id) + QLatin1Char('/') + f.name; + } + return {}; +} + +const std::vector & ModelStore::registry() { return parseRegistry(); } + +const ModelInfo * ModelStore::find(const QString & id) +{ + for (const ModelInfo & m : registry()) { + if (m.id == id) return &m; + } + return nullptr; +} + +bool ModelStore::isInstalled(const ModelInfo & m, QStringList * missing) +{ + bool ok = true; + for (const ModelFile & f : m.files) { + const QString p = ModelStore::dirFor(m.id) + QLatin1Char('/') + f.name; + if (!QFileInfo::exists(p) || QFileInfo(p).size() == 0) { + ok = false; + if (missing) *missing << f.name; + } + } + return ok; +} + +const ModelInfo * ModelStore::pickForLanguage(const QString & lang) +{ + // 先找语言列表里明确包含该语言、且已安装的 + for (const ModelInfo & m : registry()) { + if (m.languages.contains(lang) && isInstalled(m)) return &m; + } + return nullptr; +} + +} // namespace lc::models diff --git a/src/models/modelstore.h b/src/models/modelstore.h new file mode 100644 index 0000000..3e33ae0 --- /dev/null +++ b/src/models/modelstore.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +#include + +// 模型注册表与存储。 +// +// 存储位置遵循 XDG：/models//，即 +// ~/.local/share/live-caption/models// +// 注册表是 data/models.json（编译进二进制），新增模型只加一条数据，不改代码。 +namespace lc::models { + +enum class Backend { + SherpaOnline, // sherpa-onnx 的流式识别器（X-ASR / zipformer / Nemotron …） + LlamaCpp, // ggml / llama.cpp（GGUF，例如 Qwen3-ASR） +}; + +struct ModelFile { + QString key; // encoder / decoder / joiner / tokens / model / mmproj + QString name; // 文件名 + QString url; // 下载地址 +}; + +struct ModelInfo { + QString id; + QString name; + QString license; + QString note; + Backend backend = Backend::SherpaOnline; + QStringList languages; + std::vector files; + + QString filePath(const QString & key) const; // // +}; + +class ModelStore { +public: + static QString root(); // 模型根目录 + static QString dirFor(const QString & id); + + static const std::vector & registry(); + static const ModelInfo * find(const QString & id); + + // 文件是否齐全；missing 返回缺失文件名 + static bool isInstalled(const ModelInfo & m, QStringList * missing = nullptr); + + // 按语言挑一个已安装的模型（优先语言列表里排前的） + static const ModelInfo * pickForLanguage(const QString & lang); + + // 手工指定模型目录时，用这个覆盖 root() + static void setRootOverride(const QString & dir); +}; + +} // namespace lc::models