feat(asr): add pluggable ONNX streaming backend and model registry

Add a second backend (sherpa-onnx) alongside llama.cpp, so native
streaming models can be selected by ID from a JSON registry. All such
models share one adapter: only the model files and a few options differ.
Models now live under the app data directory and are fetched by
scripts/fetch-models.sh. Adding a model is a data change, not code.

Measured on the same audio: the streaming backend is ~35x cheaper in CPU
than Qwen3-ASR (0.10-0.12 vs 4.27 core-seconds per audio second), shows
the first caption at 0.48-1.44s, and matches its accuracy for zh/en.

新增 sherpa-onnx 流式后端,与 llama.cpp 并存;模型按注册表 id 从应用
数据目录加载。同族模型共用一个适配器,新增模型只改 data/models.json。
实测流式后端 CPU 约为 Qwen3-ASR 的 1/35,首个字幕 0.48~1.44s,中英
准确率持平。注意预编译库是旧 C++ ABI,必须走其 C API。
This commit is contained in:
blumia committed 2026-10-12 16:06:51 +08:00
1 parent 6d64987d64
commit bd921b1ac5
15 files changed
+869 -18

No files matched your search

+82 -1
View File
@@ -1,13 +1,19 @@
#pragma once
#include <cstdio>
#include <memory>
#include <string>
#include <QCommandLineOption>
#include <QCommandLineParser>
#include "qwen3asr.h"
#include "iasr.h"
#include "modelstore.h"
#include "streamingasr.h"
#include "qwen3asr.h"
#if defined(LC_HAVE_SHERPA)
#include "sherpaasr.h"
#endif
// 两个可执行程序(GUI / CLI)共用的命令行选项定义与解析。
namespace lc::app {
@@ -87,4 +93,79 @@ inline bool loadModel(Qwen3Asr & asr, const AsrOptions & opt)
return true;
}
// ---- 推理后端选择 ----
//
// llama 后端:Qwen3-ASR 的 GGUF 模型(--model / --mmproj)
// sherpa 后端:流式 ONNX 模型,按注册表 id 从模型存储目录取(--asr-model <id>)
inline void addBackendOptions(QCommandLineParser & p)
{
p.addOption(makeOption("asr-backend", "推理后端: llama(Qwen3-ASR GGUF)或 sherpa(流式 ONNX)",
"kind", "llama"));
p.addOption(makeOption("asr-model", "模型注册表里的 id(sherpa 后端用它定位文件)", "id"));
p.addOption(makeOption("model-dir", "覆盖模型存储目录(默认 ~/.local/share/live-caption/models)", "dir"));
p.addOption(makeOption("language", "语言提示,多语言模型用(zh / en / ja …);留空=自动", "lang"));
}
// 按命令行选择并加载后端;失败时返回 nullptr 并把原因写进 err
inline std::unique_ptr<IAsr> makeAsr(const QCommandLineParser & p,
const AsrOptions & asrOpt, std::string * err)
{
const QString dir = p.value("model-dir");
if (!dir.isEmpty()) models::ModelStore::setRootOverride(dir);
const QString backend = p.value("asr-backend");
const QString modelId = p.value("asr-model");
const QString lang = p.value("language");
const models::ModelInfo * info =
modelId.isEmpty() ? nullptr : models::ModelStore::find(modelId);
if (!modelId.isEmpty() && !info) {
if (err) *err = "注册表里没有模型: " + modelId.toStdString();
return nullptr;
}
const bool useSherpa = (backend == QLatin1String("sherpa"))
|| (info && info->backend == models::Backend::SherpaOnline);
if (useSherpa) {
#if defined(LC_HAVE_SHERPA)
if (!info) {
if (err) *err = "sherpa 后端需要 --asr-model <id>(见 data/models.json)";
return nullptr;
}
QStringList missing;
if (!models::ModelStore::isInstalled(*info, &missing)) {
if (err) {
*err = "模型未安装,缺失: " + missing.join(QStringLiteral(", ")).toStdString()
+ "\n 可运行: scripts/fetch-models.sh " + info->id.toStdString();
}
return nullptr;
}
SherpaAsr::Options o;
o.encoder = info->filePath(QStringLiteral("encoder")).toStdString();
o.decoder = info->filePath(QStringLiteral("decoder")).toStdString();
o.joiner = info->filePath(QStringLiteral("joiner")).toStdString();
o.tokens = info->filePath(QStringLiteral("tokens")).toStdString();
o.threads = 1; // 这类模型单线程最快(多线程反而被同步开销拖慢)
if (info->languages.size() > 1 && !lang.isEmpty()) {
o.language = lang.toStdString(); // 多语言模型(Nemotron)需要语言提示
}
auto a = std::make_unique<SherpaAsr>();
if (!a->load(o, err)) return nullptr;
return a;
#else
if (err) *err = "编译时未找到 sherpa-onnx(-DSHERPA_ONNX_ROOT=...)";
return nullptr;
#endif
}
auto a = std::make_unique<Qwen3Asr>();
if (asrOpt.modelPath.empty() || asrOpt.mmprojPath.empty()) {
if (err) *err = "llama 后端需要 --model 和 --mmproj(或改用 --asr-backend sherpa --asr-model <id>)";
return nullptr;
}
if (!a->load(asrOpt, err)) return nullptr;
return a;
}
} // namespace lc::app