feat(asr): add pluggable ONNX streaming backend and model registry
Add a second backend (sherpa-onnx) alongside llama.cpp, so native streaming models can be selected by ID from a JSON registry. All such models share one adapter: only the model files and a few options differ. Models now live under the app data directory and are fetched by scripts/fetch-models.sh. Adding a model is a data change, not code. Measured on the same audio: the streaming backend is ~35x cheaper in CPU than Qwen3-ASR (0.10-0.12 vs 4.27 core-seconds per audio second), shows the first caption at 0.48-1.44s, and matches its accuracy for zh/en. 新增 sherpa-onnx 流式后端，与 llama.cpp 并存；模型按注册表 id 从应用 数据目录加载。同族模型共用一个适配器，新增模型只改 data/models.json。 实测流式后端 CPU 约为 Qwen3-ASR 的 1/35，首个字幕 0.48~1.44s，中英 准确率持平。注意预编译库是旧 C++ ABI，必须走其 C API。
This commit is contained in:
blumia committed
1 parent
6d64987d64
commit
bd921b1ac5
15 files changed
+869
-18
No files matched your search
+43
-3
@@ -62,6 +62,26 @@ if(NOT LC_MTMD_LIB)
|
|||||||
endif()
|
endif()
|
||||||
message(STATUS "mtmd: ${LC_MTMD_LIB}")
|
message(STATUS "mtmd: ${LC_MTMD_LIB}")
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# sherpa-onnx（X-ASR / zipformer / Nemotron 等流式 ONNX 模型）
|
||||||
|
#
|
||||||
|
# 注意：官方预编译库是用旧 C++ ABI 编译的，所以只能用它的 C API。
|
||||||
|
# 安装：把官方的 sherpa-onnx-vX.Y.Z-linux-x64-shared.tar.bz2 解到某个目录，
|
||||||
|
# 用 -DSHERPA_ONNX_ROOT=<dir> 指定（默认见下）。
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
set(SHERPA_ONNX_ROOT "$ENV{HOME}/.local/opt/sherpa-onnx" CACHE PATH "sherpa-onnx 安装目录")
|
||||||
|
set(LC_HAS_SHERPA OFF)
|
||||||
|
if(EXISTS "${SHERPA_ONNX_ROOT}/include/sherpa-onnx/c-api/c-api.h")
|
||||||
|
find_library(LC_SHERPA_C_API NAMES sherpa-onnx-c-api
|
||||||
|
HINTS "${SHERPA_ONNX_ROOT}/lib" REQUIRED)
|
||||||
|
find_library(LC_SHERPA_ORT NAMES onnxruntime
|
||||||
|
HINTS "${SHERPA_ONNX_ROOT}/lib" REQUIRED)
|
||||||
|
set(LC_HAS_SHERPA ON)
|
||||||
|
message(STATUS "sherpa-onnx: ${SHERPA_ONNX_ROOT}")
|
||||||
|
else()
|
||||||
|
message(WARNING "未找到 sherpa-onnx（${SHERPA_ONNX_ROOT}），流式 ONNX 后端将不可用")
|
||||||
|
endif()
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# core：全部业务逻辑，刻意不依赖 Qt（方便测试与复用）
|
# core：全部业务逻辑，刻意不依赖 Qt（方便测试与复用）
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -72,6 +92,12 @@ add_library(livecaption_core STATIC
|
|||||||
src/core/streamingasr.cpp
|
src/core/streamingasr.cpp
|
||||||
src/core/livepipeline.cpp
|
src/core/livepipeline.cpp
|
||||||
)
|
)
|
||||||
|
if(LC_HAS_SHERPA)
|
||||||
|
target_sources(livecaption_core PRIVATE src/core/sherpaasr.cpp)
|
||||||
|
target_compile_definitions(livecaption_core PUBLIC LC_HAVE_SHERPA=1)
|
||||||
|
target_include_directories(livecaption_core PRIVATE "${SHERPA_ONNX_ROOT}/include")
|
||||||
|
target_link_libraries(livecaption_core PUBLIC ${LC_SHERPA_C_API} ${LC_SHERPA_ORT})
|
||||||
|
endif()
|
||||||
target_include_directories(livecaption_core PUBLIC src/core)
|
target_include_directories(livecaption_core PUBLIC src/core)
|
||||||
target_link_libraries(livecaption_core PUBLIC
|
target_link_libraries(livecaption_core PUBLIC
|
||||||
Threads::Threads
|
Threads::Threads
|
||||||
@@ -81,6 +107,20 @@ target_link_libraries(livecaption_core PUBLIC
|
|||||||
${LC_MTMD_LIB}
|
${LC_MTMD_LIB}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# models：模型注册表与存储（Qt；注册表编译进二进制）
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
add_library(livecaption_models STATIC
|
||||||
|
src/models/modelstore.cpp
|
||||||
|
)
|
||||||
|
target_include_directories(livecaption_models PUBLIC src/models)
|
||||||
|
target_link_libraries(livecaption_models PUBLIC Qt6::Core)
|
||||||
|
qt_add_resources(livecaption_models "registry"
|
||||||
|
PREFIX "/"
|
||||||
|
BASE "${CMAKE_CURRENT_SOURCE_DIR}"
|
||||||
|
FILES data/models.json)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# ui：悬浮窗与 Qt 适配层
|
# ui：悬浮窗与 Qt 适配层
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -97,15 +137,15 @@ target_link_libraries(livecaption_ui PUBLIC livecaption_core Qt6::Widgets)
|
|||||||
# GUI：悬浮字幕窗
|
# GUI：悬浮字幕窗
|
||||||
add_executable(live-caption apps/gui/main.cpp)
|
add_executable(live-caption apps/gui/main.cpp)
|
||||||
target_include_directories(live-caption PRIVATE apps/common)
|
target_include_directories(live-caption PRIVATE apps/common)
|
||||||
target_link_libraries(live-caption PRIVATE livecaption_ui)
|
target_link_libraries(live-caption PRIVATE livecaption_ui livecaption_models)
|
||||||
|
|
||||||
# CLI：文件 / 流式文件 / 命令行实时；不链接 Qt Widgets
|
# CLI：文件 / 流式文件 / 命令行实时；不链接 Qt Widgets
|
||||||
add_executable(live-caption-cli apps/cli/main.cpp)
|
add_executable(live-caption-cli apps/cli/main.cpp)
|
||||||
target_include_directories(live-caption-cli PRIVATE apps/common)
|
target_include_directories(live-caption-cli PRIVATE apps/common)
|
||||||
target_link_libraries(live-caption-cli PRIVATE livecaption_core Qt6::Core)
|
target_link_libraries(live-caption-cli PRIVATE livecaption_core livecaption_models)
|
||||||
|
|
||||||
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
|
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
|
||||||
foreach(_t livecaption_core livecaption_ui live-caption live-caption-cli)
|
foreach(_t livecaption_core livecaption_ui livecaption_models live-caption live-caption-cli)
|
||||||
target_compile_options(${_t} PRIVATE -Wall -Wextra -Wno-unused-parameter)
|
target_compile_options(${_t} PRIVATE -Wall -Wextra -Wno-unused-parameter)
|
||||||
endforeach()
|
endforeach()
|
||||||
endif()
|
endif()
|
||||||
|
|||||||
@@ -47,6 +47,22 @@ cmake --build build -j
|
|||||||
ctest --test-dir build --output-on-failure
|
ctest --test-dir build --output-on-failure
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## 推理后端
|
||||||
|
|
||||||
|
支持两条后端，用 `--asr-backend` 切换（详见 [docs/asr-backends.md](docs/asr-backends.md)）：
|
||||||
|
|
||||||
|
- **`llama`（默认）**：llama.cpp/ggml + Qwen3-ASR 的 GGUF 模型 —— 准确率高、52 种语言，但每步成本随段落增长。
|
||||||
|
- **`sherpa`**：sherpa-onnx + 流式 ONNX 模型（X-ASR / zipformer / Nemotron） —— 原生流式，每步成本恒定，CPU 约为前者的 1/35，中英准确率持平。
|
||||||
|
|
||||||
|
模型都放在 `~/.local/share/live-caption/models/`，注册表是 `data/models.json`
|
||||||
|
（**加模型只加一条数据，不改代码**），用 `scripts/fetch-models.sh` 下载。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 流式后端
|
||||||
|
./build/live-caption-cli --asr-backend sherpa --asr-model zipformer-zh-2025
|
||||||
|
./build/live-caption-cli --asr-backend sherpa --asr-model nemotron35-560ms --language ja
|
||||||
|
```
|
||||||
|
|
||||||
## 代码结构
|
## 代码结构
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
+19
-9
@@ -15,6 +15,7 @@
|
|||||||
#include "appcli.h"
|
#include "appcli.h"
|
||||||
#include "audio.h"
|
#include "audio.h"
|
||||||
#include "livepipeline.h"
|
#include "livepipeline.h"
|
||||||
|
#include "iasr.h"
|
||||||
#include "qwen3asr.h"
|
#include "qwen3asr.h"
|
||||||
#include "streamingasr.h"
|
#include "streamingasr.h"
|
||||||
#include "transcript.h"
|
#include "transcript.h"
|
||||||
@@ -25,7 +26,7 @@ std::atomic<bool> g_stop{false};
|
|||||||
|
|
||||||
extern "C" void handleSigInt(int) { g_stop = true; }
|
extern "C" void handleSigInt(int) { g_stop = true; }
|
||||||
|
|
||||||
int runFile(lc::Qwen3Asr & asr, const QString & path)
|
int runFile(lc::IAsr & asr, const QString & path)
|
||||||
{
|
{
|
||||||
std::vector<float> pcm;
|
std::vector<float> pcm;
|
||||||
std::string err;
|
std::string err;
|
||||||
@@ -54,7 +55,7 @@ int runFile(lc::Qwen3Asr & asr, const QString & path)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int runStreamFile(lc::Qwen3Asr & asr, const QString & path, const lc::StreamOptions & sop)
|
int runStreamFile(lc::IAsr & asr, const QString & path, const lc::StreamOptions & sop)
|
||||||
{
|
{
|
||||||
std::vector<float> pcm;
|
std::vector<float> pcm;
|
||||||
std::string err;
|
std::string err;
|
||||||
@@ -94,7 +95,7 @@ int runStreamFile(lc::Qwen3Asr & asr, const QString & path, const lc::StreamOpti
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int runCliLive(lc::Qwen3Asr & asr, const lc::StreamOptions & sop)
|
int runCliLive(lc::IAsr & asr, const lc::StreamOptions & sop)
|
||||||
{
|
{
|
||||||
lc::LivePipelineOptions popt;
|
lc::LivePipelineOptions popt;
|
||||||
popt.stream = sop;
|
popt.stream = sop;
|
||||||
@@ -135,19 +136,28 @@ int main(int argc, char ** argv)
|
|||||||
parser.addHelpOption();
|
parser.addHelpOption();
|
||||||
lc::app::addModelOptions(parser);
|
lc::app::addModelOptions(parser);
|
||||||
lc::app::addStreamOptions(parser);
|
lc::app::addStreamOptions(parser);
|
||||||
|
lc::app::addBackendOptions(parser);
|
||||||
parser.addOption(QCommandLineOption("file", "转写一个 wav 文件后退出", "path"));
|
parser.addOption(QCommandLineOption("file", "转写一个 wav 文件后退出", "path"));
|
||||||
parser.addOption(QCommandLineOption("stream-file", "以流式模式转写 wav 文件后退出", "path"));
|
parser.addOption(QCommandLineOption("stream-file", "以流式模式转写 wav 文件后退出", "path"));
|
||||||
parser.addOption(QCommandLineOption("cli-live", "实时转写字幕，只用命令行输出（不开窗口）"));
|
parser.addOption(QCommandLineOption("cli-live", "实时转写字幕，只用命令行输出（不开窗口）"));
|
||||||
parser.process(app);
|
parser.process(app);
|
||||||
|
|
||||||
const lc::AsrOptions asrOpt = lc::app::asrOptions(parser);
|
const lc::AsrOptions asrOpt = lc::app::asrOptions(parser);
|
||||||
const lc::StreamOptions sop = lc::app::streamOptions(parser);
|
lc::StreamOptions sop = lc::app::streamOptions(parser);
|
||||||
|
|
||||||
lc::Qwen3Asr asr;
|
std::string err;
|
||||||
if (!lc::app::loadModel(asr, asrOpt)) return 1;
|
std::unique_ptr<lc::IAsr> asr = lc::app::makeAsr(parser, asrOpt, &err);
|
||||||
|
if (!asr) {
|
||||||
|
std::fprintf(stderr, "%s\n", err.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
// 原生流式识别器很便宜，用更细的更新粒度换低延迟
|
||||||
|
if (asr->kind() == lc::AsrKind::NativeStreaming && !parser.isSet("stream-chunk")) {
|
||||||
|
sop.chunkSec = 0.32;
|
||||||
|
}
|
||||||
|
|
||||||
if (parser.isSet("file")) return runFile(asr, parser.value("file"));
|
if (parser.isSet("file")) return runFile(*asr, parser.value("file"));
|
||||||
if (parser.isSet("stream-file")) return runStreamFile(asr, parser.value("stream-file"), sop);
|
if (parser.isSet("stream-file")) return runStreamFile(*asr, parser.value("stream-file"), sop);
|
||||||
|
|
||||||
return runCliLive(asr, sop);
|
return runCliLive(*asr, sop);
|
||||||
}
|
}
|
||||||
+82
-1
@@ -1,13 +1,19 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <cstdio>
|
#include <cstdio>
|
||||||
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include <QCommandLineOption>
|
#include <QCommandLineOption>
|
||||||
#include <QCommandLineParser>
|
#include <QCommandLineParser>
|
||||||
|
|
||||||
#include "qwen3asr.h"
|
#include "iasr.h"
|
||||||
|
#include "modelstore.h"
|
||||||
#include "streamingasr.h"
|
#include "streamingasr.h"
|
||||||
|
#include "qwen3asr.h"
|
||||||
|
#if defined(LC_HAVE_SHERPA)
|
||||||
|
#include "sherpaasr.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
// 两个可执行程序（GUI / CLI）共用的命令行选项定义与解析。
|
// 两个可执行程序（GUI / CLI）共用的命令行选项定义与解析。
|
||||||
namespace lc::app {
|
namespace lc::app {
|
||||||
@@ -87,4 +93,79 @@ inline bool loadModel(Qwen3Asr & asr, const AsrOptions & opt)
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- 推理后端选择 ----
|
||||||
|
//
|
||||||
|
// llama 后端：Qwen3-ASR 的 GGUF 模型（--model / --mmproj）
|
||||||
|
// sherpa 后端：流式 ONNX 模型，按注册表 id 从模型存储目录取（--asr-model <id>）
|
||||||
|
inline void addBackendOptions(QCommandLineParser & p)
|
||||||
|
{
|
||||||
|
p.addOption(makeOption("asr-backend", "推理后端: llama（Qwen3-ASR GGUF）或 sherpa（流式 ONNX）",
|
||||||
|
"kind", "llama"));
|
||||||
|
p.addOption(makeOption("asr-model", "模型注册表里的 id（sherpa 后端用它定位文件）", "id"));
|
||||||
|
p.addOption(makeOption("model-dir", "覆盖模型存储目录（默认 ~/.local/share/live-caption/models）", "dir"));
|
||||||
|
p.addOption(makeOption("language", "语言提示，多语言模型用（zh / en / ja …）；留空=自动", "lang"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 按命令行选择并加载后端；失败时返回 nullptr 并把原因写进 err
|
||||||
|
inline std::unique_ptr<IAsr> makeAsr(const QCommandLineParser & p,
|
||||||
|
const AsrOptions & asrOpt, std::string * err)
|
||||||
|
{
|
||||||
|
const QString dir = p.value("model-dir");
|
||||||
|
if (!dir.isEmpty()) models::ModelStore::setRootOverride(dir);
|
||||||
|
|
||||||
|
const QString backend = p.value("asr-backend");
|
||||||
|
const QString modelId = p.value("asr-model");
|
||||||
|
const QString lang = p.value("language");
|
||||||
|
|
||||||
|
const models::ModelInfo * info =
|
||||||
|
modelId.isEmpty() ? nullptr : models::ModelStore::find(modelId);
|
||||||
|
if (!modelId.isEmpty() && !info) {
|
||||||
|
if (err) *err = "注册表里没有模型: " + modelId.toStdString();
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bool useSherpa = (backend == QLatin1String("sherpa"))
|
||||||
|
|| (info && info->backend == models::Backend::SherpaOnline);
|
||||||
|
|
||||||
|
if (useSherpa) {
|
||||||
|
#if defined(LC_HAVE_SHERPA)
|
||||||
|
if (!info) {
|
||||||
|
if (err) *err = "sherpa 后端需要 --asr-model <id>（见 data/models.json）";
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
QStringList missing;
|
||||||
|
if (!models::ModelStore::isInstalled(*info, &missing)) {
|
||||||
|
if (err) {
|
||||||
|
*err = "模型未安装，缺失: " + missing.join(QStringLiteral(", ")).toStdString()
|
||||||
|
+ "\n 可运行: scripts/fetch-models.sh " + info->id.toStdString();
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
SherpaAsr::Options o;
|
||||||
|
o.encoder = info->filePath(QStringLiteral("encoder")).toStdString();
|
||||||
|
o.decoder = info->filePath(QStringLiteral("decoder")).toStdString();
|
||||||
|
o.joiner = info->filePath(QStringLiteral("joiner")).toStdString();
|
||||||
|
o.tokens = info->filePath(QStringLiteral("tokens")).toStdString();
|
||||||
|
o.threads = 1; // 这类模型单线程最快（多线程反而被同步开销拖慢）
|
||||||
|
if (info->languages.size() > 1 && !lang.isEmpty()) {
|
||||||
|
o.language = lang.toStdString(); // 多语言模型（Nemotron）需要语言提示
|
||||||
|
}
|
||||||
|
auto a = std::make_unique<SherpaAsr>();
|
||||||
|
if (!a->load(o, err)) return nullptr;
|
||||||
|
return a;
|
||||||
|
#else
|
||||||
|
if (err) *err = "编译时未找到 sherpa-onnx（-DSHERPA_ONNX_ROOT=...）";
|
||||||
|
return nullptr;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
auto a = std::make_unique<Qwen3Asr>();
|
||||||
|
if (asrOpt.modelPath.empty() || asrOpt.mmprojPath.empty()) {
|
||||||
|
if (err) *err = "llama 后端需要 --model 和 --mmproj（或改用 --asr-backend sherpa --asr-model <id>）";
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
if (!a->load(asrOpt, err)) return nullptr;
|
||||||
|
return a;
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace lc::app
|
} // namespace lc::app
|
||||||
+14
-5
@@ -10,13 +10,14 @@
|
|||||||
#include "appcli.h"
|
#include "appcli.h"
|
||||||
#include "captionwindow.h"
|
#include "captionwindow.h"
|
||||||
#include "livetranscriber.h"
|
#include "livetranscriber.h"
|
||||||
|
#include "iasr.h"
|
||||||
#include "qwen3asr.h"
|
#include "qwen3asr.h"
|
||||||
#include "streamingasr.h"
|
#include "streamingasr.h"
|
||||||
#include "transcript.h"
|
#include "transcript.h"
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
int runGui(lc::Qwen3Asr & asr, const lc::StreamOptions & sop,
|
int runGui(lc::IAsr & asr, const lc::StreamOptions & sop,
|
||||||
const QString & screenshotPath, int screenshotDelaySec, int opacityPercent)
|
const QString & screenshotPath, int screenshotDelaySec, int opacityPercent)
|
||||||
{
|
{
|
||||||
lc::CaptionWindow window;
|
lc::CaptionWindow window;
|
||||||
@@ -84,18 +85,26 @@ int main(int argc, char ** argv)
|
|||||||
parser.addHelpOption();
|
parser.addHelpOption();
|
||||||
lc::app::addModelOptions(parser);
|
lc::app::addModelOptions(parser);
|
||||||
lc::app::addStreamOptions(parser);
|
lc::app::addStreamOptions(parser);
|
||||||
|
lc::app::addBackendOptions(parser);
|
||||||
parser.addOption(lc::app::makeOption("opacity", "字幕背景不透明度，取值 20-100", "n", "84"));
|
parser.addOption(lc::app::makeOption("opacity", "字幕背景不透明度，取值 20-100", "n", "84"));
|
||||||
parser.addOption(lc::app::makeOption("screenshot", "启动后自动截图并退出（验证渲染用）", "path"));
|
parser.addOption(lc::app::makeOption("screenshot", "启动后自动截图并退出（验证渲染用）", "path"));
|
||||||
parser.addOption(lc::app::makeOption("screenshot-delay", "截图前等待秒数", "sec", "6"));
|
parser.addOption(lc::app::makeOption("screenshot-delay", "截图前等待秒数", "sec", "6"));
|
||||||
parser.process(app);
|
parser.process(app);
|
||||||
|
|
||||||
const lc::AsrOptions asrOpt = lc::app::asrOptions(parser);
|
const lc::AsrOptions asrOpt = lc::app::asrOptions(parser);
|
||||||
const lc::StreamOptions sop = lc::app::streamOptions(parser);
|
lc::StreamOptions sop = lc::app::streamOptions(parser);
|
||||||
|
|
||||||
lc::Qwen3Asr asr;
|
std::string err;
|
||||||
if (!lc::app::loadModel(asr, asrOpt)) return 1;
|
std::unique_ptr<lc::IAsr> asr = lc::app::makeAsr(parser, asrOpt, &err);
|
||||||
|
if (!asr) {
|
||||||
|
std::fprintf(stderr, "%s\n", err.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (asr->kind() == lc::AsrKind::NativeStreaming && !parser.isSet("stream-chunk")) {
|
||||||
|
sop.chunkSec = 0.32;
|
||||||
|
}
|
||||||
|
|
||||||
return runGui(asr, sop,
|
return runGui(*asr, sop,
|
||||||
parser.value("screenshot"),
|
parser.value("screenshot"),
|
||||||
parser.value("screenshot-delay").toInt(),
|
parser.value("screenshot-delay").toInt(),
|
||||||
parser.value("opacity").toInt());
|
parser.value("opacity").toInt());
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
{
|
||||||
|
"version": 1,
|
||||||
|
"comment": "live-caption 的模型注册表。新增模型 = 在这里加一条，不需要改代码。",
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"id": "zipformer-zh-2025",
|
||||||
|
"name": "中文流式（k2-fsa 官方 2025-06-30）",
|
||||||
|
"backend": "sherpa-online",
|
||||||
|
"languages": ["zh"],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"note": "154MB，实测中文逐字正确；首个字幕 0.48s，约 0.10 核·秒/秒音频",
|
||||||
|
"files": {
|
||||||
|
"encoder": {
|
||||||
|
"name": "encoder.int8.onnx",
|
||||||
|
"url": "https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30/resolve/main/encoder.int8.onnx"
|
||||||
|
},
|
||||||
|
"decoder": {
|
||||||
|
"name": "decoder.onnx",
|
||||||
|
"url": "https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30/resolve/main/decoder.onnx"
|
||||||
|
},
|
||||||
|
"joiner": {
|
||||||
|
"name": "joiner.int8.onnx",
|
||||||
|
"url": "https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30/resolve/main/joiner.int8.onnx"
|
||||||
|
},
|
||||||
|
"tokens": {
|
||||||
|
"name": "tokens.txt",
|
||||||
|
"url": "https://hf-mirror.com/csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30/resolve/main/tokens.txt"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "xasr-zh-en-480",
|
||||||
|
"name": "中英流式（X-ASR-zh-en, 480ms）",
|
||||||
|
"backend": "sherpa-online",
|
||||||
|
"languages": ["zh", "en"],
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"note": "587MB，实测中英准确率与 Qwen3-ASR 持平；首个字幕 1.44s",
|
||||||
|
"files": {
|
||||||
|
"encoder": {
|
||||||
|
"name": "encoder-480ms.onnx",
|
||||||
|
"url": "https://hf-mirror.com/GilgameshWind/X-ASR-zh-en/resolve/main/deployment/models/chunk-480ms-model/encoder-480ms.onnx"
|
||||||
|
},
|
||||||
|
"decoder": {
|
||||||
|
"name": "decoder-480ms.onnx",
|
||||||
|
"url": "https://hf-mirror.com/GilgameshWind/X-ASR-zh-en/resolve/main/deployment/models/chunk-480ms-model/decoder-480ms.onnx"
|
||||||
|
},
|
||||||
|
"joiner": {
|
||||||
|
"name": "joiner-480ms.onnx",
|
||||||
|
"url": "https://hf-mirror.com/GilgameshWind/X-ASR-zh-en/resolve/main/deployment/models/chunk-480ms-model/joiner-480ms.onnx"
|
||||||
|
},
|
||||||
|
"tokens": {
|
||||||
|
"name": "tokens.txt",
|
||||||
|
"url": "https://hf-mirror.com/GilgameshWind/X-ASR-zh-en/resolve/main/deployment/models/chunk-480ms-model/tokens.txt"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "nemotron35-560ms",
|
||||||
|
"name": "多语言流式（Nemotron 3.5, 560ms, 含日语）",
|
||||||
|
"backend": "sherpa-online",
|
||||||
|
"languages": ["en", "ja", "ko", "vi", "ru", "ar", "hi", "fr", "de", "es", "it", "pt", "nl", "tr", "uk", "zh"],
|
||||||
|
"license": "OpenMDW-1.1",
|
||||||
|
"note": "651MB，40 种语言 + 内置语言识别；实测中英日都可用但都有瑕疵",
|
||||||
|
"optionsHint": "language",
|
||||||
|
"files": {
|
||||||
|
"encoder": {
|
||||||
|
"name": "encoder.int8.onnx",
|
||||||
|
"url": "https://hf-mirror.com/csukuangfj2/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/resolve/main/encoder.int8.onnx"
|
||||||
|
},
|
||||||
|
"decoder": {
|
||||||
|
"name": "decoder.int8.onnx",
|
||||||
|
"url": "https://hf-mirror.com/csukuangfj2/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/resolve/main/decoder.int8.onnx"
|
||||||
|
},
|
||||||
|
"joiner": {
|
||||||
|
"name": "joiner.int8.onnx",
|
||||||
|
"url": "https://hf-mirror.com/csukuangfj2/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/resolve/main/joiner.int8.onnx"
|
||||||
|
},
|
||||||
|
"tokens": {
|
||||||
|
"name": "tokens.txt",
|
||||||
|
"url": "https://hf-mirror.com/csukuangfj2/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/resolve/main/tokens.txt"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
# 推理后端与模型管理
|
||||||
|
|
||||||
|
## 结构：2 个实现覆盖所有模型
|
||||||
|
|
||||||
|
实测确认了一件很重要的事：**不同模型不需要各写一套代码**。所有流式 ONNX 模型走的是同一套 API，差别只是「指向哪个模型文件 + 几行选项」。
|
||||||
|
|
||||||
|
```
|
||||||
|
IAsr（src/core/iasr.h）
|
||||||
|
├─ Qwen3Asr llama.cpp / ggml GGUF 模型（Qwen3-ASR） AsrKind::PrefixRollback
|
||||||
|
└─ SherpaAsr sherpa-onnx / onnxruntime
|
||||||
|
├─ X-ASR-zh-en 中英流式
|
||||||
|
├─ zipformer-zh-2025 中文流式（k2-fsa 官方）
|
||||||
|
└─ Nemotron-3.5 40 语言流式（含日语） AsrKind::NativeStreaming
|
||||||
|
```
|
||||||
|
|
||||||
|
两条路径的区别（`StreamingAsr` 里按 `IAsr::kind()` 分流）：
|
||||||
|
|
||||||
|
| | PrefixRollback | NativeStreaming |
|
||||||
|
|---|---|---|
|
||||||
|
| 模型类型 | 离线模型（LLM 解码器） | 有状态流式识别器 |
|
||||||
|
| 上层要做的 | 切块 + 前缀回滚 + KV 复用 | 只喂音频，取当前整句文本 |
|
||||||
|
| 每步成本 | 随句子长度增长 | 恒定 |
|
||||||
|
| 已稳定/不稳定 | 用 token 回滚算 | 用「与上次的公共前缀」算 |
|
||||||
|
|
||||||
|
## 模型注册表与存储
|
||||||
|
|
||||||
|
**新增模型 = 加一条数据，不改代码。**
|
||||||
|
|
||||||
|
- 注册表：`data/models.json`（编译进二进制，同时也给脚本用）
|
||||||
|
- 存储位置：`~/.local/share/live-caption/models/<id>/`
|
||||||
|
（即 `QStandardPaths::GenericDataLocation + /live-caption/models`）
|
||||||
|
- 下载：`scripts/fetch-models.sh [id...]` / `--all`；已存在的文件会跳过
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/fetch-models.sh # 列出模型与安装状态
|
||||||
|
./scripts/fetch-models.sh zipformer-zh-2025
|
||||||
|
```
|
||||||
|
|
||||||
|
## 用法
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 默认后端：llama.cpp + Qwen3-ASR GGUF
|
||||||
|
./build/live-caption-cli --model M.gguf --mmproj MM.gguf
|
||||||
|
|
||||||
|
# 流式 ONNX 后端：按注册表 id 从模型目录取
|
||||||
|
./build/live-caption-cli --asr-backend sherpa --asr-model zipformer-zh-2025
|
||||||
|
./build/live-caption-cli --asr-backend sherpa --asr-model nemotron35-560ms --language ja
|
||||||
|
|
||||||
|
# 覆盖模型目录 / 语言提示
|
||||||
|
./build/live-caption-cli --asr-backend sherpa --asr-model ... --model-dir /path/to/models
|
||||||
|
```
|
||||||
|
|
||||||
|
## 实测对比（同一批音频，1 线程）
|
||||||
|
|
||||||
|
| 模型 | 中文 | 英文 | 日语 | RTF | 核·秒/秒音频 | 首个字幕 | 体积 | 许可 |
|
||||||
|
|---|---|---|---|---|---|---|---|---|
|
||||||
|
| Qwen3-ASR 0.6B（llama） | ✅ | ✅ | ✅ | 0.56 | 4.27 | ~2.5s | 973MB | Apache-2.0 |
|
||||||
|
| zipformer-zh-2025 | ✅ | — | — | 0.098 | **0.10** | **0.48s** | 167MB | Apache-2.0 |
|
||||||
|
| X-ASR-zh-en 480ms | ✅ | ✅ | — | 0.122 | 0.12 | 1.44s | 587MB | Apache-2.0 |
|
||||||
|
| Nemotron 3.5 560ms | ⚠️ 漏字 | ⚠️ 若干错词 | ⚠️ 有错字 | 0.35 | 0.35 | 1.12s | 651MB | OpenMDW-1.1 |
|
||||||
|
|
||||||
|
> 静态模型（zipformer-zh-2025）的多线程是**负优化**：onnxruntime 对小模型开多线程，
|
||||||
|
> 同步/自旋开销会让它更慢（8 线程比 1 线程慢 2.6 倍、CPU 高 28 倍）。所以适配器里固定
|
||||||
|
> `threads = 1`。
|
||||||
|
|
||||||
|
## 两个坑
|
||||||
|
|
||||||
|
1. **sherpa-onnx 的预编译库是旧 C++ ABI**（`_GLIBCXX_USE_CXX11_ABI=0`），和 Qt 的新 ABI 冲突，
|
||||||
|
C++ API 链接不上。**必须用它的 C API**（`sherpa-onnx/c-api/c-api.h`，`extern "C"`，无名字改写）。
|
||||||
|
2. **结束时要补一小段静音**（`streamFinish()` 里补 0.3s）再 `InputFinished`，否则最后一个字会被截掉。
|
||||||
|
|
||||||
|
## 自动语言识别
|
||||||
|
|
||||||
|
| 模型 | 自带 LID |
|
||||||
|
|---|---|
|
||||||
|
| Qwen3-ASR | ✅ 输出 `language X`（实测中英都准）|
|
||||||
|
| Nemotron 3.5 | ✅ `language=auto`，但实测会串（日语开头吐韩文）|
|
||||||
|
| X-ASR / zipformer | ❌ 固定语言对，不出语言标签 |
|
||||||
|
|
||||||
|
LID 不必由 ASR 模型提供：Chrome 就带了个独立的 2.5MB `ONDEVICE_langid.tflite`（我们在它的语言包目录里看到了），
|
||||||
|
Windows 同样。sherpa-onnx 也有专门的语种识别模型。所以**「按语言装包 + 用户选语言或小 LID 模型」**是可行路线。
|
||||||
|
|
||||||
|
## 推荐组合（按语言装包）
|
||||||
|
|
||||||
|
| 语言 | 实时预览 | 定稿 |
|
||||||
|
|---|---|---|
|
||||||
|
| 中文 | `zipformer-zh-2025`（最快最准） | Qwen3-ASR |
|
||||||
|
| 英文 | `xasr-zh-en-480` | Qwen3-ASR |
|
||||||
|
| 日语 | `nemotron35-560ms --language ja`（质量一般） | Qwen3-ASR（开源日语第一） |
|
||||||
|
| 其他 | — | Qwen3-ASR（52 语言兜底） |
|
||||||
Executable
+74
@@ -0,0 +1,74 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# 把模型注册表里的模型下载到应用数据目录（QStandardPaths::AppDataLocation）。
|
||||||
|
#
|
||||||
|
# 用法:
|
||||||
|
# ./fetch-models.sh # 列出所有模型和安装状态
|
||||||
|
# ./fetch-models.sh <id> [...] # 下载指定模型
|
||||||
|
# ./fetch-models.sh --all # 下载全部
|
||||||
|
#
|
||||||
|
# 目录规则（与程序里 ModelStore 保持一致）：
|
||||||
|
# ${XDG_DATA_HOME:-~/.local/share}/live-caption/models/<id>/
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
REG="$ROOT/data/models.json"
|
||||||
|
DEST="${XDG_DATA_HOME:-$HOME/.local/share}/live-caption/models"
|
||||||
|
|
||||||
|
[ -f "$REG" ] || { echo "找不到注册表: $REG" >&2; exit 1; }
|
||||||
|
|
||||||
|
ids=$(python3 -c "
|
||||||
|
import json,sys
|
||||||
|
d=json.load(open('$REG'))
|
||||||
|
print(' '.join(m['id'] for m in d['models']))")
|
||||||
|
|
||||||
|
show_status() {
|
||||||
|
python3 - "$REG" "$DEST" <<'PY'
|
||||||
|
import json, os, sys
|
||||||
|
reg, dest = sys.argv[1], sys.argv[2]
|
||||||
|
d = json.load(open(reg))
|
||||||
|
for m in d['models']:
|
||||||
|
mid = m['id']; mdir = os.path.join(dest, mid)
|
||||||
|
files = m['files']
|
||||||
|
have = sum(1 for f in files.values() if os.path.exists(os.path.join(mdir, f['name'])))
|
||||||
|
total = len(files)
|
||||||
|
mark = "已安装" if have == total else ("部分" if have else "未安装")
|
||||||
|
langs = ",".join(m['languages'][:4]) + ("…" if len(m['languages']) > 4 else "")
|
||||||
|
print(f" [{have}/{total} {mark}] {mid:22s} {langs:18s} {m.get('license','')}")
|
||||||
|
print(f" {m['name']}")
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
download_one() {
|
||||||
|
local id="$1"
|
||||||
|
local dir="$DEST/$id"
|
||||||
|
mkdir -p "$dir"
|
||||||
|
echo "==> $id → $dir"
|
||||||
|
python3 - "$REG" "$id" <<'PY' | while IFS=$'\t' read -r name url; do
|
||||||
|
import json, sys
|
||||||
|
d = json.load(open(sys.argv[1])); mid = sys.argv[2]
|
||||||
|
m = next(x for x in d['models'] if x['id'] == mid)
|
||||||
|
for f in m['files'].values():
|
||||||
|
print(f"{f['name']}\t{f['url']}")
|
||||||
|
PY
|
||||||
|
if [ -f "$dir/$name" ]; then
|
||||||
|
echo " 已存在，跳过: $name"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo " 下载 $name"
|
||||||
|
if command -v aria2c >/dev/null 2>&1; then
|
||||||
|
aria2c -x16 -s16 -k4M --summary-interval=0 -d "$dir" -o "$name" "$url" >/dev/null
|
||||||
|
else
|
||||||
|
curl -fL --progress-bar -o "$dir/$name" "$url"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo " 完成"
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
"" ) echo "模型存储目录: $DEST"; echo; show_status ;;
|
||||||
|
--all) shift; for id in $ids; do download_one "$id"; done ;;
|
||||||
|
*) for id in "$@"; do
|
||||||
|
echo "$ids" | tr ' ' '\n' | grep -qx "$id" || { echo "未知模型: $id" >&2; exit 1; }
|
||||||
|
download_one "$id"
|
||||||
|
done ;;
|
||||||
|
esac
|
||||||
@@ -30,15 +30,29 @@ struct AsrStreamStep {
|
|||||||
double inferSeconds = 0.0; // 以上合计
|
double inferSeconds = 0.0; // 以上合计
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// 推理后端的种类，决定上层用哪条流式路径
|
||||||
|
enum class AsrKind {
|
||||||
|
PrefixRollback, // 离线模型 + 前缀回滚（Qwen3-ASR / llama.cpp）
|
||||||
|
NativeStreaming, // 有状态流式识别器（sherpa-onnx 的 transducer 系）
|
||||||
|
};
|
||||||
|
|
||||||
// 把推理后端抽象出来，好处是流式策略（StreamingAsr）和实时管线（LivePipeline）
|
// 把推理后端抽象出来，好处是流式策略（StreamingAsr）和实时管线（LivePipeline）
|
||||||
// 可以用一个假的实现来测，不必加载真模型。
|
// 可以用一个假的实现来测，不必加载真模型。
|
||||||
class IAsr {
|
class IAsr {
|
||||||
public:
|
public:
|
||||||
virtual ~IAsr() = default;
|
virtual ~IAsr() = default;
|
||||||
|
|
||||||
|
virtual AsrKind kind() const { return AsrKind::PrefixRollback; }
|
||||||
|
|
||||||
// 期望的输入采样率（Qwen3-ASR 为 16000）
|
// 期望的输入采样率（Qwen3-ASR 为 16000）
|
||||||
virtual int sampleRate() const = 0;
|
virtual int sampleRate() const = 0;
|
||||||
|
|
||||||
|
// ---- NativeStreaming 专用 ----
|
||||||
|
// 喂入音频，返回"这一句到目前为止"的完整文本（识别器会随时修正前面的词）
|
||||||
|
virtual std::string streamFeed(const float * /*pcm*/, size_t /*n*/) { return {}; }
|
||||||
|
// 结束这一句（把尾部 flush 出来），返回最终文本
|
||||||
|
virtual std::string streamFinish() { return {}; }
|
||||||
|
|
||||||
// 整段转写：把 prefix 当作"已经生成的 assistant 文本"，从那里继续生成
|
// 整段转写：把 prefix 当作"已经生成的 assistant 文本"，从那里继续生成
|
||||||
virtual AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples,
|
virtual AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples,
|
||||||
const std::string & prefix) = 0;
|
const std::string & prefix) = 0;
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
#include "sherpaasr.h"
|
||||||
|
|
||||||
|
#include <cstring>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "sherpa-onnx/c-api/c-api.h"
|
||||||
|
|
||||||
|
namespace lc {
|
||||||
|
|
||||||
|
struct SherpaAsr::Impl {
|
||||||
|
Options opt;
|
||||||
|
const SherpaOnnxOnlineRecognizer * rec = nullptr;
|
||||||
|
const SherpaOnnxOnlineStream * stream = nullptr;
|
||||||
|
|
||||||
|
std::string resultText() const
|
||||||
|
{
|
||||||
|
if (!rec || !stream) return {};
|
||||||
|
const SherpaOnnxOnlineRecognizerResult * r =
|
||||||
|
SherpaOnnxGetOnlineStreamResult(rec, stream);
|
||||||
|
if (!r) return {};
|
||||||
|
std::string text = r->text ? r->text : "";
|
||||||
|
SherpaOnnxDestroyOnlineRecognizerResult(r);
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
void drain()
|
||||||
|
{
|
||||||
|
while (rec && stream && SherpaOnnxIsOnlineStreamReady(rec, stream)) {
|
||||||
|
SherpaOnnxDecodeOnlineStream(rec, stream);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
SherpaAsr::SherpaAsr() : d(new Impl) {}
|
||||||
|
|
||||||
|
SherpaAsr::~SherpaAsr()
|
||||||
|
{
|
||||||
|
if (!d) return;
|
||||||
|
if (d->stream) SherpaOnnxDestroyOnlineStream(d->stream);
|
||||||
|
if (d->rec) SherpaOnnxDestroyOnlineRecognizer(d->rec);
|
||||||
|
delete d;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool SherpaAsr::load(const Options & opt, std::string * error)
|
||||||
|
{
|
||||||
|
const auto fail = [error](const std::string & msg) {
|
||||||
|
if (error) *error = msg;
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
d->opt = opt;
|
||||||
|
|
||||||
|
SherpaOnnxOnlineRecognizerConfig cfg;
|
||||||
|
std::memset(&cfg, 0, sizeof(cfg));
|
||||||
|
cfg.feat_config.sample_rate = 16000;
|
||||||
|
cfg.feat_config.feature_dim = 80;
|
||||||
|
cfg.model_config.transducer.encoder = opt.encoder.c_str();
|
||||||
|
cfg.model_config.transducer.decoder = opt.decoder.c_str();
|
||||||
|
cfg.model_config.transducer.joiner = opt.joiner.c_str();
|
||||||
|
cfg.model_config.tokens = opt.tokens.c_str();
|
||||||
|
cfg.model_config.num_threads = opt.threads;
|
||||||
|
cfg.model_config.provider = "cpu";
|
||||||
|
cfg.model_config.modeling_unit = "";
|
||||||
|
cfg.decoding_method = "greedy_search";
|
||||||
|
cfg.max_active_paths = 4;
|
||||||
|
cfg.enable_endpoint = 0; // 分段由我们的 VAD 负责
|
||||||
|
|
||||||
|
d->rec = SherpaOnnxCreateOnlineRecognizer(&cfg);
|
||||||
|
if (!d->rec) return fail("创建 sherpa-onnx 识别器失败（检查模型文件）");
|
||||||
|
|
||||||
|
d->stream = SherpaOnnxCreateOnlineStream(d->rec);
|
||||||
|
if (!d->stream) return fail("创建 sherpa-onnx 流失败");
|
||||||
|
if (!opt.language.empty()) {
|
||||||
|
SherpaOnnxOnlineStreamSetOption(d->stream, "language", opt.language.c_str());
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string SherpaAsr::streamFeed(const float * pcm, size_t n)
|
||||||
|
{
|
||||||
|
if (!d->rec || !d->stream) return {};
|
||||||
|
SherpaOnnxOnlineStreamAcceptWaveform(d->stream, 16000, pcm, (int32_t) n);
|
||||||
|
d->drain();
|
||||||
|
return d->resultText();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string SherpaAsr::streamFinish()
|
||||||
|
{
|
||||||
|
if (!d->rec || !d->stream) return {};
|
||||||
|
// 先补一小段静音再结束，否则最后一个字常常来不及输出
|
||||||
|
const size_t pad = (size_t) (16000 * 0.3);
|
||||||
|
const std::vector<float> silence(pad, 0.0f);
|
||||||
|
SherpaOnnxOnlineStreamAcceptWaveform(d->stream, 16000, silence.data(), (int32_t) pad);
|
||||||
|
d->drain();
|
||||||
|
// 通知不再有输入，把尾部一并解出来
|
||||||
|
SherpaOnnxOnlineStreamInputFinished(d->stream);
|
||||||
|
d->drain();
|
||||||
|
return d->resultText();
|
||||||
|
}
|
||||||
|
|
||||||
|
void SherpaAsr::streamReset()
|
||||||
|
{
|
||||||
|
if (!d->rec || !d->stream) return;
|
||||||
|
SherpaOnnxOnlineStreamReset(d->rec, d->stream);
|
||||||
|
}
|
||||||
|
|
||||||
|
AsrResult SherpaAsr::transcribeWithPrefix(const float *, size_t, const std::string &)
|
||||||
|
{
|
||||||
|
AsrResult r;
|
||||||
|
r.ok = false;
|
||||||
|
r.error = "该后端只支持原生流式（NativeStreaming），不支持整段/前缀回滚模式";
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
AsrStreamStep SherpaAsr::streamAppend(const float *, size_t, const std::string &, int)
|
||||||
|
{
|
||||||
|
AsrStreamStep s;
|
||||||
|
s.ok = false;
|
||||||
|
s.error = "该后端只支持原生流式（NativeStreaming），请用 streamFeed()";
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
AsrStreamStep SherpaAsr::streamAppendTurn(const float *, size_t, int)
|
||||||
|
{
|
||||||
|
AsrStreamStep s;
|
||||||
|
s.ok = false;
|
||||||
|
s.error = "该后端只支持原生流式（NativeStreaming），请用 streamFeed()";
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace lc
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "iasr.h"
|
||||||
|
|
||||||
|
struct SherpaOnnxOnlineRecognizer;
|
||||||
|
struct SherpaOnnxOnlineStream;
|
||||||
|
|
||||||
|
namespace lc {
|
||||||
|
|
||||||
|
// sherpa-onnx 的流式识别器（X-ASR / zipformer / Nemotron …）。
|
||||||
|
//
|
||||||
|
// 用的是它的 **C API** 而不是 C++ API：预编译库是用旧 C++ ABI
|
||||||
|
// （_GLIBCXX_USE_CXX11_ABI=0）编的，和 Qt 的新 ABI 不兼容；C API 是 extern "C"，
|
||||||
|
// 不经过 C++ 名字改写，没有这个问题。
|
||||||
|
//
|
||||||
|
// 这类模型是**有状态流式**：喂音频 → 随时取当前整句文本（会修正前面的词）。
|
||||||
|
class SherpaAsr : public IAsr {
|
||||||
|
public:
|
||||||
|
struct Options {
|
||||||
|
std::string encoder;
|
||||||
|
std::string decoder;
|
||||||
|
std::string joiner;
|
||||||
|
std::string tokens;
|
||||||
|
int threads = 1; // 注意：这类模型单线程反而最快
|
||||||
|
std::string language; // 多语言模型（如 Nemotron）用；空 = 不设置
|
||||||
|
};
|
||||||
|
|
||||||
|
SherpaAsr();
|
||||||
|
~SherpaAsr() override;
|
||||||
|
|
||||||
|
SherpaAsr(const SherpaAsr &) = delete;
|
||||||
|
SherpaAsr & operator=(const SherpaAsr &) = delete;
|
||||||
|
|
||||||
|
bool load(const Options & opt, std::string * error);
|
||||||
|
|
||||||
|
AsrKind kind() const override { return AsrKind::NativeStreaming; }
|
||||||
|
int sampleRate() const override { return 16000; }
|
||||||
|
|
||||||
|
std::string streamFeed(const float * pcm, size_t n) override;
|
||||||
|
std::string streamFinish() override;
|
||||||
|
void streamReset() override;
|
||||||
|
|
||||||
|
// 这些只对 PrefixRollback 后端有意义；这里给出明确的失败信息
|
||||||
|
AsrResult transcribeWithPrefix(const float *, size_t, const std::string &) override;
|
||||||
|
AsrStreamStep streamAppend(const float *, size_t, const std::string &, int) override;
|
||||||
|
AsrStreamStep streamAppendTurn(const float *, size_t, int) override;
|
||||||
|
std::string dropLastTokens(const std::string & text, int) const override { return text; }
|
||||||
|
|
||||||
|
private:
|
||||||
|
struct Impl;
|
||||||
|
Impl * d = nullptr;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace lc
|
||||||
@@ -1,9 +1,37 @@
|
|||||||
#include "streamingasr.h"
|
#include "streamingasr.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cctype>
|
||||||
|
#include <chrono>
|
||||||
|
|
||||||
#include "transcript.h"
|
#include "transcript.h"
|
||||||
|
|
||||||
namespace lc {
|
namespace lc {
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
// 去掉首尾空白（含末尾空格），用于 sherpa 这类可能带前导空格的输出
|
||||||
|
std::string trimAscii(const std::string & s)
|
||||||
|
{
|
||||||
|
const auto notSpace = [](unsigned char c) { return !std::isspace(c); };
|
||||||
|
auto b = std::find_if(s.begin(), s.end(), notSpace);
|
||||||
|
auto e = std::find_if(s.rbegin(), s.rend(), notSpace).base();
|
||||||
|
if (b >= e) return {};
|
||||||
|
return std::string(b, e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 两个字符串的最长公共前缀，切在 UTF-8 字符边界上
|
||||||
|
size_t utf8Lcp(const std::string & a, const std::string & b)
|
||||||
|
{
|
||||||
|
const size_t n = std::min(a.size(), b.size());
|
||||||
|
size_t i = 0;
|
||||||
|
while (i < n && a[i] == b[i]) ++i;
|
||||||
|
while (i > 0 && (static_cast<unsigned char>(a[i]) & 0xC0) == 0x80) --i;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
StreamingAsr::StreamingAsr(IAsr & asr, StreamOptions opt)
|
StreamingAsr::StreamingAsr(IAsr & asr, StreamOptions opt)
|
||||||
: asr_(asr), opt_(opt)
|
: asr_(asr), opt_(opt)
|
||||||
{
|
{
|
||||||
@@ -18,6 +46,7 @@ void StreamingAsr::reset()
|
|||||||
totalSeconds_ = 0.0;
|
totalSeconds_ = 0.0;
|
||||||
committedRaw_.clear();
|
committedRaw_.clear();
|
||||||
lastRawFull_.clear();
|
lastRawFull_.clear();
|
||||||
|
previousFull_.clear();
|
||||||
language_.clear();
|
language_.clear();
|
||||||
chunkIndex_ = 0;
|
chunkIndex_ = 0;
|
||||||
}
|
}
|
||||||
@@ -50,6 +79,14 @@ void StreamingAsr::feed(const float * pcm, size_t n,
|
|||||||
|
|
||||||
void StreamingAsr::flush(const std::function<void(const Step &)> & onStep)
|
void StreamingAsr::flush(const std::function<void(const Step &)> & onStep)
|
||||||
{
|
{
|
||||||
|
// 原生流式：把尾部 flush 出来即收尾（streamFinish 会补齐最后一个字）
|
||||||
|
if (asr_.kind() == AsrKind::NativeStreaming) {
|
||||||
|
std::vector<float> chunk;
|
||||||
|
chunk.swap(pending_);
|
||||||
|
runStep(chunk, true, onStep);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (!pending_.empty()) {
|
if (!pending_.empty()) {
|
||||||
std::vector<float> chunk;
|
std::vector<float> chunk;
|
||||||
chunk.swap(pending_);
|
chunk.swap(pending_);
|
||||||
@@ -86,6 +123,44 @@ void StreamingAsr::flush(const std::function<void(const Step &)> & onStep)
|
|||||||
void StreamingAsr::runStep(const std::vector<float> & audio, bool finalStep,
|
void StreamingAsr::runStep(const std::vector<float> & audio, bool finalStep,
|
||||||
const std::function<void(const Step &)> & onStep)
|
const std::function<void(const Step &)> & onStep)
|
||||||
{
|
{
|
||||||
|
// ---- 原生流式（sherpa-onnx 这类有状态识别器）----
|
||||||
|
// 它自己维护状态并随时给出整句文本，会修正前面的词；我们只负责：
|
||||||
|
// 1) 把音频喂进去
|
||||||
|
// 2) 用"与上一次的公共前缀"作为已稳定部分，其余算不稳定尾部
|
||||||
|
if (asr_.kind() == AsrKind::NativeStreaming) {
|
||||||
|
Step s;
|
||||||
|
s.chunkIndex = chunkIndex_;
|
||||||
|
|
||||||
|
const auto t0 = std::chrono::steady_clock::now();
|
||||||
|
std::string cur;
|
||||||
|
if (!audio.empty()) cur = asr_.streamFeed(audio.data(), audio.size());
|
||||||
|
if (finalStep) cur = asr_.streamFinish();
|
||||||
|
const auto t1 = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
|
s.ok = true;
|
||||||
|
s.inferSeconds = std::chrono::duration<double>(t1 - t0).count();
|
||||||
|
s.full = trimAscii(cur);
|
||||||
|
|
||||||
|
const std::string stable = s.full.substr(0, utf8Lcp(previousFull_, s.full));
|
||||||
|
previousFull_ = s.full;
|
||||||
|
s.committed = stable;
|
||||||
|
s.tail = s.full.substr(stable.size());
|
||||||
|
s.language = language_;
|
||||||
|
|
||||||
|
totalSeconds_ += (double) audio.size() / asr_.sampleRate();
|
||||||
|
s.windowSeconds = totalSeconds_;
|
||||||
|
|
||||||
|
++chunkIndex_;
|
||||||
|
if (onStep) onStep(s);
|
||||||
|
|
||||||
|
if (finalStep) {
|
||||||
|
// 这一句结束：并入历史，等下一次 reset()
|
||||||
|
historyText_ += s.full;
|
||||||
|
previousFull_.clear();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// 冷启动阶段不给前缀；之后把"已稳定文本"回喂给模型
|
// 冷启动阶段不给前缀；之后把"已稳定文本"回喂给模型
|
||||||
const std::string prefix =
|
const std::string prefix =
|
||||||
(chunkIndex_ >= opt_.unfixedChunkNum) ? committedRaw_ : std::string();
|
(chunkIndex_ >= opt_.unfixedChunkNum) ? committedRaw_ : std::string();
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ private:
|
|||||||
double totalSeconds_ = 0.0;
|
double totalSeconds_ = 0.0;
|
||||||
std::string committedRaw_; // 当前段的原始已提交文本（含 language 头，用于回喂）
|
std::string committedRaw_; // 当前段的原始已提交文本（含 language 头，用于回喂）
|
||||||
std::string lastRawFull_; // 上一次完整的原始文本（含不稳定尾部）
|
std::string lastRawFull_; // 上一次完整的原始文本（含不稳定尾部）
|
||||||
|
std::string previousFull_; // 原生流式：上一次的完整文本（用来算已稳定的前缀）
|
||||||
std::string historyText_; // 此前各段已提交的正文（已剥离 language 头）
|
std::string historyText_; // 此前各段已提交的正文（已剥离 language 头）
|
||||||
std::string language_;
|
std::string language_;
|
||||||
int chunkIndex_ = 0;
|
int chunkIndex_ = 0;
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
#include "modelstore.h"
|
||||||
|
|
||||||
|
#include <QDir>
|
||||||
|
#include <QFile>
|
||||||
|
#include <QFileInfo>
|
||||||
|
#include <QJsonArray>
|
||||||
|
#include <QJsonDocument>
|
||||||
|
#include <QJsonObject>
|
||||||
|
#include <QStandardPaths>
|
||||||
|
|
||||||
|
namespace lc::models {
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
QString g_rootOverride;
|
||||||
|
|
||||||
|
const std::vector<ModelInfo> & parseRegistry()
|
||||||
|
{
|
||||||
|
static std::vector<ModelInfo> cache;
|
||||||
|
static bool loaded = false;
|
||||||
|
if (loaded) return cache;
|
||||||
|
loaded = true;
|
||||||
|
|
||||||
|
QFile f(QStringLiteral(":/data/models.json"));
|
||||||
|
if (!f.open(QIODevice::ReadOnly)) {
|
||||||
|
qWarning("无法读取内嵌的模型注册表");
|
||||||
|
return cache;
|
||||||
|
}
|
||||||
|
const QJsonDocument doc = QJsonDocument::fromJson(f.readAll());
|
||||||
|
for (const QJsonValue & v : doc.object().value(QStringLiteral("models")).toArray()) {
|
||||||
|
const QJsonObject o = v.toObject();
|
||||||
|
ModelInfo m;
|
||||||
|
m.id = o.value(QStringLiteral("id")).toString();
|
||||||
|
m.name = o.value(QStringLiteral("name")).toString();
|
||||||
|
m.license = o.value(QStringLiteral("license")).toString();
|
||||||
|
m.note = o.value(QStringLiteral("note")).toString();
|
||||||
|
m.backend = (o.value(QStringLiteral("backend")).toString() == QLatin1String("llama-cpp"))
|
||||||
|
? Backend::LlamaCpp
|
||||||
|
: Backend::SherpaOnline;
|
||||||
|
for (const QJsonValue & l : o.value(QStringLiteral("languages")).toArray()) {
|
||||||
|
m.languages << l.toString();
|
||||||
|
}
|
||||||
|
const QJsonObject files = o.value(QStringLiteral("files")).toObject();
|
||||||
|
for (auto it = files.begin(); it != files.end(); ++it) {
|
||||||
|
const QJsonObject fo = it.value().toObject();
|
||||||
|
m.files.push_back({it.key(),
|
||||||
|
fo.value(QStringLiteral("name")).toString(),
|
||||||
|
fo.value(QStringLiteral("url")).toString()});
|
||||||
|
}
|
||||||
|
cache.push_back(std::move(m));
|
||||||
|
}
|
||||||
|
return cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
void ModelStore::setRootOverride(const QString & dir) { g_rootOverride = dir; }
|
||||||
|
|
||||||
|
QString ModelStore::root()
|
||||||
|
{
|
||||||
|
if (!g_rootOverride.isEmpty()) return g_rootOverride;
|
||||||
|
// 用 GenericDataLocation（~/.local/share）+ 固定名，保证 CLI/GUI 一致，
|
||||||
|
// 也和 scripts/fetch-models.sh 的目录规则对得上。
|
||||||
|
const QString base = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation);
|
||||||
|
return base + QStringLiteral("/live-caption/models");
|
||||||
|
}
|
||||||
|
|
||||||
|
QString ModelStore::dirFor(const QString & id) { return root() + QLatin1Char('/') + id; }
|
||||||
|
|
||||||
|
QString ModelInfo::filePath(const QString & key) const
|
||||||
|
{
|
||||||
|
for (const ModelFile & f : files) {
|
||||||
|
if (f.key == key) return ModelStore::dirFor(id) + QLatin1Char('/') + f.name;
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::vector<ModelInfo> & ModelStore::registry() { return parseRegistry(); }
|
||||||
|
|
||||||
|
const ModelInfo * ModelStore::find(const QString & id)
|
||||||
|
{
|
||||||
|
for (const ModelInfo & m : registry()) {
|
||||||
|
if (m.id == id) return &m;
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ModelStore::isInstalled(const ModelInfo & m, QStringList * missing)
|
||||||
|
{
|
||||||
|
bool ok = true;
|
||||||
|
for (const ModelFile & f : m.files) {
|
||||||
|
const QString p = ModelStore::dirFor(m.id) + QLatin1Char('/') + f.name;
|
||||||
|
if (!QFileInfo::exists(p) || QFileInfo(p).size() == 0) {
|
||||||
|
ok = false;
|
||||||
|
if (missing) *missing << f.name;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ModelInfo * ModelStore::pickForLanguage(const QString & lang)
|
||||||
|
{
|
||||||
|
// 先找语言列表里明确包含该语言、且已安装的
|
||||||
|
for (const ModelInfo & m : registry()) {
|
||||||
|
if (m.languages.contains(lang) && isInstalled(m)) return &m;
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace lc::models
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <QString>
|
||||||
|
#include <QStringList>
|
||||||
|
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
// 模型注册表与存储。
|
||||||
|
//
|
||||||
|
// 存储位置遵循 XDG：<AppDataLocation>/models/<id>/，即
|
||||||
|
// ~/.local/share/live-caption/models/<id>/
|
||||||
|
// 注册表是 data/models.json（编译进二进制），新增模型只加一条数据，不改代码。
|
||||||
|
namespace lc::models {
|
||||||
|
|
||||||
|
enum class Backend {
|
||||||
|
SherpaOnline, // sherpa-onnx 的流式识别器（X-ASR / zipformer / Nemotron …）
|
||||||
|
LlamaCpp, // ggml / llama.cpp（GGUF，例如 Qwen3-ASR）
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ModelFile {
|
||||||
|
QString key; // encoder / decoder / joiner / tokens / model / mmproj
|
||||||
|
QString name; // 文件名
|
||||||
|
QString url; // 下载地址
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ModelInfo {
|
||||||
|
QString id;
|
||||||
|
QString name;
|
||||||
|
QString license;
|
||||||
|
QString note;
|
||||||
|
Backend backend = Backend::SherpaOnline;
|
||||||
|
QStringList languages;
|
||||||
|
std::vector<ModelFile> files;
|
||||||
|
|
||||||
|
QString filePath(const QString & key) const; // <store>/<id>/<name>
|
||||||
|
};
|
||||||
|
|
||||||
|
class ModelStore {
|
||||||
|
public:
|
||||||
|
static QString root(); // 模型根目录
|
||||||
|
static QString dirFor(const QString & id);
|
||||||
|
|
||||||
|
static const std::vector<ModelInfo> & registry();
|
||||||
|
static const ModelInfo * find(const QString & id);
|
||||||
|
|
||||||
|
// 文件是否齐全；missing 返回缺失文件名
|
||||||
|
static bool isInstalled(const ModelInfo & m, QStringList * missing = nullptr);
|
||||||
|
|
||||||
|
// 按语言挑一个已安装的模型（优先语言列表里排前的）
|
||||||
|
static const ModelInfo * pickForLanguage(const QString & lang);
|
||||||
|
|
||||||
|
// 手工指定模型目录时，用这个覆盖 root()
|
||||||
|
static void setRootOverride(const QString & dir);
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace lc::models
|
||||||
Reference in new issue
Block a user