From 736ec14d5b530ddc072ddfe9bb2a5d9041f4b4a6 Mon Sep 17 00:00:00 2001 From: Wang Zichong Date: Mon, 12 Oct 2026 13:43:50 +0800 Subject: [PATCH] refactor: split core/ui/apps targets and add unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move all business logic into liblivecaption_core (no Qt dependency) and the widgets into liblivecaption_ui, leaving two executables: the caption window and the CLI, which no longer links Qt Widgets. Abstract the ASR backend behind IAsr so the streaming policy and the live pipeline can be tested with a scripted fake backend instead of a real model. 把全部业务逻辑下沉到不依赖 Qt 的 liblivecaption_core，界面独立成 liblivecaption_ui，拆出悬浮窗与命令行两个可执行程序（CLI 不再链接 Qt Widgets）；并用 IAsr 抽象推理后端，使流式策略与实时管线可脱离真模型 用假实现做单元测试。测试还抓到一个真 bug：段恰好结束在 chunk 边界时 只提交了稳定部分，最后几个词被丢掉，已修。 --- CMakeLists.txt | 69 +++++--- README.md | 44 +++-- apps/cli/main.cpp | 152 +++++++++++++++++ apps/common/appcli.h | 73 ++++++++ apps/gui/main.cpp | 102 +++++++++++ docs/testing.md | 65 +++++++ src/{ => core}/audio.cpp | 0 src/{ => core}/audio.h | 0 src/core/iasr.h | 60 +++++++ src/core/livepipeline.cpp | 147 ++++++++++++++++ src/core/livepipeline.h | 87 ++++++++++ src/{ => core}/qwen3asr.cpp | 13 +- src/core/qwen3asr.h | 52 ++++++ src/{ => core}/streamingasr.cpp | 19 ++- src/{ => core}/streamingasr.h | 9 +- src/{ => core}/transcript.cpp | 0 src/{ => core}/transcript.h | 0 src/core/vad.h | 43 +++++ src/livetranscriber.cpp | 178 -------------------- src/livetranscriber.h | 52 ------ src/main.cpp | 288 -------------------------------- src/qwen3asr.h | 82 --------- src/{ => ui}/captionwindow.cpp | 0 src/{ => ui}/captionwindow.h | 0 src/ui/livetranscriber.cpp | 60 +++++++ src/ui/livetranscriber.h | 40 +++++ tests/CMakeLists.txt | 15 ++ tests/fakeasr.h | 75 +++++++++ tests/tst_livepipeline.cpp | 127 ++++++++++++++ tests/tst_streamingasr.cpp | 174 +++++++++++++++++++ tests/tst_transcript.cpp | 48 ++++++ tests/tst_vad.cpp | 83 +++++++++ 32 files changed, 1502 insertions(+), 655 deletions(-) create mode 100644 apps/cli/main.cpp create mode 100644 apps/common/appcli.h create mode 100644 apps/gui/main.cpp create mode 100644 docs/testing.md rename src/{ => core}/audio.cpp (100%) rename src/{ => core}/audio.h (100%) create mode 100644 src/core/iasr.h create mode 100644 src/core/livepipeline.cpp create mode 100644 src/core/livepipeline.h rename src/{ => core}/qwen3asr.cpp (98%) create mode 100644 src/core/qwen3asr.h rename src/{ => core}/streamingasr.cpp (90%) rename src/{ => core}/streamingasr.h (94%) rename src/{ => core}/transcript.cpp (100%) rename src/{ => core}/transcript.h (100%) create mode 100644 src/core/vad.h delete mode 100644 src/livetranscriber.cpp delete mode 100644 src/livetranscriber.h delete mode 100644 src/main.cpp delete mode 100644 src/qwen3asr.h rename src/{ => ui}/captionwindow.cpp (100%) rename src/{ => ui}/captionwindow.h (100%) create mode 100644 src/ui/livetranscriber.cpp create mode 100644 src/ui/livetranscriber.h create mode 100644 tests/CMakeLists.txt create mode 100644 tests/fakeasr.h create mode 100644 tests/tst_livepipeline.cpp create mode 100644 tests/tst_streamingasr.cpp create mode 100644 tests/tst_transcript.cpp create mode 100644 tests/tst_vad.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1290f3d..52448e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,10 +4,13 @@ project(live-caption LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_AUTOMOC ON) if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) endif() +option(LC_BUILD_TESTS "构建单元测试" ON) + # --------------------------------------------------------------------------- # llama.cpp / libmtmd 的搜索路径 # @@ -29,8 +32,6 @@ find_package(Qt6 REQUIRED COMPONENTS Core Widgets) find_package(PkgConfig REQUIRED) find_package(Threads REQUIRED) -set(CMAKE_AUTOMOC ON) - pkg_check_modules(SAMPLERATE REQUIRED IMPORTED_TARGET samplerate) pkg_check_modules(PULSE REQUIRED IMPORTED_TARGET libpulse) @@ -47,37 +48,32 @@ else() GIT_REPOSITORY https://github.com/ggml-org/llama.cpp.git GIT_TAG b9951 FIND_PACKAGE_ARGS NAMES llama CONFIG) - set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE) - set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE) + set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE) set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) FetchContent_MakeAvailable(llama) set(LC_LLAMA_TARGET llama) endif() # --- mtmd：llama.pc 里没有，单独找 --- -find_library(LC_MTMD_LIB NAMES mtmd - HINTS /usr/lib/x86_64-linux-gnu/llama) +find_library(LC_MTMD_LIB NAMES mtmd HINTS /usr/lib/x86_64-linux-gnu/llama) if(NOT LC_MTMD_LIB) message(FATAL_ERROR "找不到 libmtmd（应随 libllama0 一起安装）") endif() message(STATUS "mtmd: ${LC_MTMD_LIB}") # --------------------------------------------------------------------------- -add_executable(live-caption-cli - src/main.cpp - src/transcript.cpp - src/qwen3asr.cpp - src/streamingasr.cpp - src/audio.cpp - src/livetranscriber.cpp - src/captionwindow.cpp +# core：全部业务逻辑，刻意不依赖 Qt（方便测试与复用） +# --------------------------------------------------------------------------- +add_library(livecaption_core STATIC + src/core/transcript.cpp + src/core/audio.cpp + src/core/qwen3asr.cpp + src/core/streamingasr.cpp + src/core/livepipeline.cpp ) - -target_include_directories(live-caption-cli PRIVATE src) - -target_link_libraries(live-caption-cli PRIVATE - Qt6::Core - Qt6::Widgets +target_include_directories(livecaption_core PUBLIC src/core) +target_link_libraries(livecaption_core PUBLIC Threads::Threads PkgConfig::SAMPLERATE PkgConfig::PULSE @@ -85,6 +81,37 @@ target_link_libraries(live-caption-cli PRIVATE ${LC_MTMD_LIB} ) +# --------------------------------------------------------------------------- +# ui：悬浮窗与 Qt 适配层 +# --------------------------------------------------------------------------- +add_library(livecaption_ui STATIC + src/ui/captionwindow.cpp + src/ui/livetranscriber.cpp +) +target_include_directories(livecaption_ui PUBLIC src/ui) +target_link_libraries(livecaption_ui PUBLIC livecaption_core Qt6::Widgets) + +# --------------------------------------------------------------------------- +# 可执行程序 +# --------------------------------------------------------------------------- +# GUI：悬浮字幕窗 +add_executable(live-caption apps/gui/main.cpp) +target_include_directories(live-caption PRIVATE apps/common) +target_link_libraries(live-caption PRIVATE livecaption_ui) + +# CLI：文件 / 流式文件 / 命令行实时；不链接 Qt Widgets +add_executable(live-caption-cli apps/cli/main.cpp) +target_include_directories(live-caption-cli PRIVATE apps/common) +target_link_libraries(live-caption-cli PRIVATE livecaption_core Qt6::Core) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(live-caption-cli PRIVATE -Wall -Wextra -Wno-unused-parameter) + foreach(_t livecaption_core livecaption_ui live-caption live-caption-cli) + target_compile_options(${_t} PRIVATE -Wall -Wextra -Wno-unused-parameter) + endforeach() +endif() + +# --------------------------------------------------------------------------- +if(LC_BUILD_TESTS) + enable_testing() + add_subdirectory(tests) endif() diff --git a/README.md b/README.md index d8ac70d..f0c3102 100644 --- a/README.md +++ b/README.md @@ -33,24 +33,40 @@ aria2c -x16 -s16 -k4M -o mmproj-Qwen3-ASR-0.6B-Q8_0.gguf "$B/mmproj-Qwen3-ASR-0. cmake -S . -B build -DCMAKE_BUILD_TYPE=Release cmake --build build -j -# 3. 单文件转写 -./build/live-caption-cli \ - --model Qwen3-ASR-0.6B-Q8_0.gguf \ - --mmproj mmproj-Qwen3-ASR-0.6B-Q8_0.gguf \ - --file some_16k_mono.wav - -# 4. 实时字幕 -./build/live-caption-cli \ +# 3. 悬浮字幕窗 +./build/live-caption \ --model Qwen3-ASR-0.6B-Q8_0.gguf \ --mmproj mmproj-Qwen3-ASR-0.6B-Q8_0.gguf + +# 4. 命令行：单文件转写 / 流式转写 / 无窗口实时 +./build/live-caption-cli --model M --mmproj MM --file some_16k_mono.wav +./build/live-caption-cli --model M --mmproj MM --stream-file some_16k_mono.wav +./build/live-caption-cli --model M --mmproj MM --cli-live + +# 5. 单元测试（不需要模型，也不需要显示环境） +ctest --test-dir build --output-on-failure ``` -## 目录 +## 代码结构 ``` -├─ src/ # 原型源码 -├─ docs/ # 各阶段验证报告 -└─ experiments/ - ├─ phase0/ # 可行性验证脚本（采集/RTF/模型元数据） - └─ phase1/ # 命令行原型说明与实机测试脚本 +src/core/ → liblivecaption_core（静态库，**不依赖 Qt**） + iasr.h 推理后端抽象（便于用假实现测流式策略） + transcript.{h,cpp} 输出解析、语言名映射 + audio.{h,cpp} monitor 采集 / WAV 读取 / 重采样 + qwen3asr.{h,cpp} 进程内推理（libllama + libmtmd） + streamingasr.{h,cpp} 流式策略（8s 切片复用、前缀回滚、分段） + vad.h 能量 VAD + livepipeline.{h,cpp} 实时管线：采集→VAD→分段→流式转写 +src/ui/ → liblivecaption_ui（静态库，Qt6::Widgets） + captionwindow.{h,cpp} 悬浮字幕窗 + livetranscriber.{h,cpp} LivePipeline 的 Qt 适配器（只转信号） +apps/gui/ → live-caption （悬浮窗程序） +apps/cli/ → live-caption-cli （命令行程序，只链 Qt6::Core） +apps/common/ 两个程序共用的命令行选项 +tests/ → 单元测试（QtTest，只链 core） +docs/ 各阶段说明与验证报告 +experiments/ Phase 0/1 的验证脚本 ``` + +拆分的意义：**业务逻辑全在 core 且不依赖 Qt**，所以流式策略、VAD、输出解析这些最容易出错的部分可以脱离真模型和显示环境做单元测试；GUI 与 CLI 各自独立演进，CLI 也不再需要链接 Qt Widgets。 diff --git a/apps/cli/main.cpp b/apps/cli/main.cpp new file mode 100644 index 0000000..b9767c0 --- /dev/null +++ b/apps/cli/main.cpp @@ -0,0 +1,152 @@ +// 命令行版：转写文件、流式转写文件、或纯命令行实时字幕（不开窗口）。 +// 只依赖 core，不链接 Qt Widgets。 +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "appcli.h" +#include "audio.h" +#include "livepipeline.h" +#include "qwen3asr.h" +#include "streamingasr.h" +#include "transcript.h" + +namespace { + +std::atomic g_stop{false}; + +extern "C" void handleSigInt(int) { g_stop = true; } + +int runFile(lc::Qwen3Asr & asr, const QString & path) +{ + std::vector pcm; + std::string err; + if (!lc::readWavMono(path.toStdString(), asr.sampleRate(), pcm, &err)) { + std::fprintf(stderr, "读取音频失败: %s\n", err.c_str()); + return 1; + } + + const double dur = (double) pcm.size() / asr.sampleRate(); + std::fprintf(stderr, "音频: %s (%.2fs)\n", qUtf8Printable(path), dur); + + const auto t0 = std::chrono::steady_clock::now(); + lc::AsrResult r = asr.transcribe(pcm.data(), pcm.size()); + const double wall = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + + if (!r.ok) { + std::fprintf(stderr, "转写失败: %s\n", r.error.c_str()); + return 1; + } + + std::printf("语言: %s (%s)\n", r.transcript.language.c_str(), + lc::languageDisplayName(r.transcript.language).c_str()); + std::printf("文本: %s\n", r.transcript.text.c_str()); + std::fprintf(stderr, "推理: %.2fs / 音频 %.2fs (rtf %.3f) 生成 %d tokens, 墙钟 %.2fs\n", + r.inferSeconds, dur, dur > 0 ? r.inferSeconds / dur : 0, r.tokenCount, wall); + return 0; +} + +int runStreamFile(lc::Qwen3Asr & asr, const QString & path, const lc::StreamOptions & sop) +{ + std::vector pcm; + std::string err; + if (!lc::readWavMono(path.toStdString(), asr.sampleRate(), pcm, &err)) { + std::fprintf(stderr, "读取音频失败: %s\n", err.c_str()); + return 1; + } + + const double dur = (double) pcm.size() / asr.sampleRate(); + std::fprintf(stderr, "流式转写: %s (%.2fs) chunk=%.1fs segment=%.1fs mode=%s\n", + qUtf8Printable(path), dur, sop.chunkSec, sop.maxUtteranceSec, + sop.incremental ? "incremental" : "window"); + + lc::StreamingAsr stream(asr, sop); + const size_t chunkSamples = (size_t) (sop.chunkSec * asr.sampleRate()); + + const auto onStep = [](const lc::StreamingAsr::Step & s) { + std::printf("\n[chunk %2d] 窗口 %5.1fs / 编码 %5.2fs / 本步 %5.2fs%s\n", + s.chunkIndex, s.windowSeconds, s.encodeSeconds, s.inferSeconds, + s.ok ? "" : " <失败>"); + if (!s.ok) std::printf(" error: %s\n", s.error.c_str()); + std::printf(" 已提交: %s\n", s.committed.c_str()); + std::printf(" 含尾部: %s\n", s.full.c_str()); + std::fflush(stdout); + }; + + for (size_t off = 0; off < pcm.size(); off += chunkSamples) { + const size_t n = std::min(chunkSamples, pcm.size() - off); + stream.feed(pcm.data() + off, n, onStep); + } + stream.flush(onStep); + + std::printf("\n==== 最终提交文本 (%s) ====\n%s\n", + lc::languageDisplayName(stream.language()).c_str(), + stream.committedText().c_str()); + return 0; +} + +int runCliLive(lc::Qwen3Asr & asr, const lc::StreamOptions & sop) +{ + lc::LivePipelineOptions popt; + popt.stream = sop; + + lc::LivePipeline pipeline(asr, popt); + pipeline.setCaptionCallback([](const std::string & committed, const std::string & tail, + const std::string & lang) { + // 原地刷新：稳定部分正常显示，尾部用暗色表示未定 + std::printf("\r\033[K[%s] %s\033[2m%s\033[0m", + lc::languageDisplayName(lang).c_str(), committed.c_str(), tail.c_str()); + std::fflush(stdout); + }); + pipeline.setErrorCallback([](const std::string & m) { + std::fprintf(stderr, "\n采集失败: %s\n", m.c_str()); + }); + + std::string err; + if (!pipeline.start(&err)) { + std::fprintf(stderr, "启动采集失败: %s\n", err.c_str()); + return 1; + } + + std::fprintf(stderr, "正在采集: %s (Ctrl+C 退出)\n", pipeline.deviceName().c_str()); + std::signal(SIGINT, handleSigInt); + while (!g_stop) std::this_thread::sleep_for(std::chrono::milliseconds(100)); + pipeline.stop(); + return 0; +} + +} // namespace + +int main(int argc, char ** argv) +{ + QCoreApplication app(argc, argv); + + QCommandLineParser parser; + parser.setApplicationDescription(QStringLiteral("实时字幕（命令行版）")); + parser.addHelpOption(); + lc::app::addModelOptions(parser); + lc::app::addStreamOptions(parser); + parser.addOption(QCommandLineOption("file", "转写一个 wav 文件后退出", "path")); + parser.addOption(QCommandLineOption("stream-file", "以流式模式转写 wav 文件后退出", "path")); + parser.addOption(QCommandLineOption("cli-live", "实时转写字幕，只用命令行输出（不开窗口）")); + parser.process(app); + + const lc::AsrOptions asrOpt = lc::app::asrOptions(parser); + const lc::StreamOptions sop = lc::app::streamOptions(parser); + + lc::Qwen3Asr asr; + if (!lc::app::loadModel(asr, asrOpt)) return 1; + + if (parser.isSet("file")) return runFile(asr, parser.value("file")); + if (parser.isSet("stream-file")) return runStreamFile(asr, parser.value("stream-file"), sop); + + return runCliLive(asr, sop); +} diff --git a/apps/common/appcli.h b/apps/common/appcli.h new file mode 100644 index 0000000..e031439 --- /dev/null +++ b/apps/common/appcli.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include + +#include +#include + +#include "qwen3asr.h" +#include "streamingasr.h" + +// 两个可执行程序（GUI / CLI）共用的命令行选项定义与解析。 +namespace lc::app { + +inline void addModelOptions(QCommandLineParser & p) +{ + p.addOption(QCommandLineOption("model", "主模型 GGUF 路径（架构须为 qwen3vl）", "path")); + p.addOption(QCommandLineOption("mmproj", "mmproj GGUF 路径（clip + qwen3a）", "path")); + p.addOption(QCommandLineOption("threads", "推理线程数", "n", "8")); + p.addOption(QCommandLineOption("max-tokens", "单段最多生成 token 数", "n", "256")); + p.addOption(QCommandLineOption("verbose", "输出 llama.cpp/mtmd 的详细日志")); +} + +inline void addStreamOptions(QCommandLineParser & p) +{ + p.addOption(QCommandLineOption("stream-chunk", "流式 chunk 秒数", "sec", "2.0")); + p.addOption(QCommandLineOption("stream-window", "滑窗上限秒数（仅 window 模式）", "sec", "16")); + p.addOption(QCommandLineOption("stream-segment", "单段最长秒数（0=不限）", "sec", "30")); + p.addOption(QCommandLineOption("unfixed-tokens", "前缀回滚丢弃的 token 数", "n", "5")); + p.addOption(QCommandLineOption("unfixed-chunks", "冷启动不带前缀的 chunk 数", "n", "2")); + p.addOption(QCommandLineOption("stream-mode", "流式后端: incremental 或 window", "mode", + "incremental")); +} + +inline AsrOptions asrOptions(const QCommandLineParser & p) +{ + AsrOptions o; + o.modelPath = p.value("model").toStdString(); + o.mmprojPath = p.value("mmproj").toStdString(); + o.threads = p.value("threads").toInt(); + o.maxTokens = p.value("max-tokens").toInt(); + o.verbose = p.isSet("verbose"); + return o; +} + +inline StreamOptions streamOptions(const QCommandLineParser & p) +{ + StreamOptions o; + o.chunkSec = p.value("stream-chunk").toDouble(); + o.windowSec = p.value("stream-window").toDouble(); + o.maxUtteranceSec = p.value("stream-segment").toDouble(); + o.unfixedTokenNum = p.value("unfixed-tokens").toInt(); + o.unfixedChunkNum = p.value("unfixed-chunks").toInt(); + o.maxTokensPerStep = p.value("max-tokens").toInt(); + o.incremental = p.value("stream-mode") != QLatin1String("window"); + return o; +} + +inline bool loadModel(Qwen3Asr & asr, const AsrOptions & opt) +{ + if (opt.modelPath.empty() || opt.mmprojPath.empty()) { + std::fprintf(stderr, "必须提供 --model 和 --mmproj\n"); + return false; + } + std::string err; + if (!asr.load(opt, &err)) { + std::fprintf(stderr, "加载失败: %s\n", err.c_str()); + return false; + } + return true; +} + +} // namespace lc::app diff --git a/apps/gui/main.cpp b/apps/gui/main.cpp new file mode 100644 index 0000000..7d2004a --- /dev/null +++ b/apps/gui/main.cpp @@ -0,0 +1,102 @@ +// 图形界面版：悬浮字幕窗。 +#include + +#include +#include +#include +#include +#include + +#include "appcli.h" +#include "captionwindow.h" +#include "livetranscriber.h" +#include "qwen3asr.h" +#include "streamingasr.h" +#include "transcript.h" + +namespace { + +int runGui(lc::Qwen3Asr & asr, const lc::StreamOptions & sop, + const QString & screenshotPath, int screenshotDelaySec, int opacityPercent) +{ + lc::CaptionWindow window; + window.setPanelOpacity(opacityPercent); + lc::LiveTranscriber live(asr); + + QObject::connect(&live, &lc::LiveTranscriber::caption, &window, + [&window](const QString & committed, const QString & tail, + const QString & lang) { + window.setLanguage( + QString::fromStdString(lc::languageDisplayName(lang.toStdString()))); + window.setCaption(committed, tail); + }); + QObject::connect(&live, &lc::LiveTranscriber::failed, &window, [](const QString & m) { + qWarning("采集失败: %s", qUtf8Printable(m)); + }); + QObject::connect(&window, &lc::CaptionWindow::closed, qApp, &QCoreApplication::quit); + QObject::connect(&window, &lc::CaptionWindow::languageClicked, &window, [] { + qInfo() << "语言标签被点击（设置界面将在 Phase 4 接入）"; + }); + + QString err; + if (!live.start(sop, &err)) { + qCritical("启动采集失败: %s", qUtf8Printable(err)); + return 1; + } + qInfo("正在采集: %s", qUtf8Printable(live.deviceName())); + + window.show(); + + if (!screenshotPath.isEmpty()) { + // 自截图：用于验证渲染（不依赖外部截图工具） + QTimer::singleShot(screenshotDelaySec * 1000, &window, [&window, screenshotPath] { + const bool ok1 = window.grab().save(screenshotPath); + + window.setCollapsed(false); + QString expandedPath = screenshotPath; + expandedPath.replace(QStringLiteral(".png"), QStringLiteral("-expanded.png")); + const bool ok2 = window.grab().save(expandedPath); + + std::fprintf(stderr, "截图: 折叠态 %s / 展开态 %s\n", + ok1 ? "成功" : "失败", ok2 ? "成功" : "失败"); + QCoreApplication::quit(); + }); + } + + return QCoreApplication::exec(); +} + +} // namespace + +int main(int argc, char ** argv) +{ + // 说明意图：DPI 取整策略必须在 QGuiApplication 之前定。 + // （Deepin 的会话环境已经设了 QT_SCALE_FACTOR_ROUNDING_POLICY， + // 所以本机仍会看到 Qt 的那条提醒，那不是本程序的问题。） + QGuiApplication::setHighDpiScaleFactorRoundingPolicy( + Qt::HighDpiScaleFactorRoundingPolicy::PassThrough); + + QApplication app(argc, argv); + QCoreApplication::setApplicationName(QStringLiteral("live-caption")); + + QCommandLineParser parser; + parser.setApplicationDescription(QStringLiteral("实时字幕（悬浮窗）")); + parser.addHelpOption(); + lc::app::addModelOptions(parser); + lc::app::addStreamOptions(parser); + parser.addOption(QCommandLineOption("opacity", "字幕背景不透明度（20-100，百分比）", "n", "84")); + parser.addOption(QCommandLineOption("screenshot", "启动后自动截图并退出（验证渲染用）", "path")); + parser.addOption(QCommandLineOption("screenshot-delay", "截图前等待秒数", "sec", "6")); + parser.process(app); + + const lc::AsrOptions asrOpt = lc::app::asrOptions(parser); + const lc::StreamOptions sop = lc::app::streamOptions(parser); + + lc::Qwen3Asr asr; + if (!lc::app::loadModel(asr, asrOpt)) return 1; + + return runGui(asr, sop, + parser.value("screenshot"), + parser.value("screenshot-delay").toInt(), + parser.value("opacity").toInt()); +} diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..5da03de --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,65 @@ +# 测试 + +## 分层 + +| 目标 | 依赖 | 说明 | +|---|---|---| +| `livecaption_core` | 无 Qt | 全部业务逻辑；单元测试只依赖它 | +| `livecaption_ui` | Qt6::Widgets | 悬浮窗与 Qt 适配层 | +| `live-caption` | ui | 悬浮窗程序 | +| `live-caption-cli` | core + Qt6::Core | 命令行程序（**不链接 Qt Widgets**） | +| `tst_*` | core + Qt6::Test | 单元测试 | + +## 运行 + +```bash +cmake -S . -B build && cmake --build build -j +ctest --test-dir build --output-on-failure # 也可以单独跑 build/tests/tst_xxx +``` + +单元测试**不需要模型文件，也不需要显示环境**，秒级完成。 + +## 为什么能这样测 + +关键是把推理后端抽象成 `IAsr`（`src/core/iasr.h`），测试里换成 `tests/fakeasr.h` 的 `FakeAsr`： + +- 可以**脚本化**每一次 `streamAppend` 返回什么文本； +- 记录每次拿到的前缀、音频样本数、以及 `streamReset` 的调用次数； +- `dropLastTokens` 用"空格分词"模拟词表，并保留原有格式。 + +另外 `LivePipeline` 刻意不带 Qt 依赖、并且把「送音频」拆成同步的 `process()`，所以测试可以直接喂音频，不必碰 PulseAudio。 + +## 覆盖的回归用例 + +这些用例对应的都是开发过程中**实际踩到**的坑： + +| 测试 | 对应的坑 | +|---|---| +| `tst_streamingasr.keepsJoiningSpace` | 用 `trim()` 过的文本拼接，导致续写缺空格（`potatoesand`） | +| `tst_streamingasr.segmentsAccumulateWithoutLosingEarlierText` | 多段拼接时只保留最后一个语言头之后的内容，前面正文丢失 | +| `tst_streamingasr.committedDropsUnstableTail` | 已提交 / 不稳定尾部的划分 | +| `tst_streamingasr.flushCommitsEverything` | 流结束时尾部被丢掉 | +| `tst_streamingasr.coldStartHasNoPrefix` | 冷启动阶段不应回喂前缀 | +| `tst_livepipeline.silenceIsNotForwarded` | 静音被送进模型导致幻觉（英文语音前冒出 `嗯。`） | +| `tst_livepipeline.noSamplesDroppedAcrossIrregularChunks` | 工作线程丢样本（余数没接上），造成假静音与截断 | +| `tst_livepipeline.silenceEndsSegment` | 静音收束分段 | +| `tst_vad.quietSpeechIsDetected` | 噪声底用均值跟踪，把很轻的话音全判成静音 | +| `tst_transcript.*` | 输出解析与语言名映射 | + +写测试的过程中还**抓到一个真 bug**：段恰好结束在 chunk 边界（没有待处理音频）时，`flush()` 只提交"已稳定"部分，最后几个词会被丢掉。已修（用完整文本 `lastRawFull_` 提交）。 + +## 尚未覆盖 + +以下几类需要真模型或真实音频设备，属于集成测试，没放进单元测试： + +- **8s 编码切片对齐**：这是准确率的决定性因素（切 2s 会明显掉准确率），但必须用真模型跑才能验证。 +- **增量 KV 复用的单步延迟**：需要真模型才能测。 +- 采集链路（PulseAudio/PipeWire 的 monitor 源）。 +- GUI 渲染。 + +这些目前靠 `experiments/` 下的脚本与实机验证（见 `docs/phase0..3`）。 + +## 后续可加 + +- 给需要模型的集成测试打 CTest label（如 `model`），默认跳过，用环境变量指模型路径时才跑。 +- GUI 的渲染回归：已有的 `--screenshot` 自截图，可以断言面板像素 alpha（`(0,0,0,214)`），避免再次出现"样式表背景对自定义 QWidget 不生效 → 明明想半透明结果全透明"这类问题。 diff --git a/src/audio.cpp b/src/core/audio.cpp similarity index 100% rename from src/audio.cpp rename to src/core/audio.cpp diff --git a/src/audio.h b/src/core/audio.h similarity index 100% rename from src/audio.h rename to src/core/audio.h diff --git a/src/core/iasr.h b/src/core/iasr.h new file mode 100644 index 0000000..4148755 --- /dev/null +++ b/src/core/iasr.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include "transcript.h" + +namespace lc { + +// 一次整段转写的结果 +struct AsrResult { + bool ok = false; + std::string error; + ParsedTranscript transcript; // 已拆出 language / text + std::string raw; // 模型原始输出（未 trim） + double inferSeconds = 0.0; + int tokenCount = 0; +}; + +// 流式单步的结果 +struct AsrStreamStep { + bool ok = false; + std::string error; + std::string newText; // 本步新生成的文本（不含前缀，未 trim） + int newTokens = 0; + int audioTokens = 0; + double encodeSeconds = 0.0; + double inferSeconds = 0.0; +}; + +// 把推理后端抽象出来，好处是流式策略（StreamingAsr）和实时管线（LivePipeline） +// 可以用一个假的实现来测，不必加载真模型。 +class IAsr { +public: + virtual ~IAsr() = default; + + // 期望的输入采样率（Qwen3-ASR 为 16000） + virtual int sampleRate() const = 0; + + // 整段转写：把 prefix 当作"已经生成的 assistant 文本"，从那里继续生成 + virtual AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples, + const std::string & prefix) = 0; + + // 流式：只编码新音频，已解码的音频留在 KV cache 里复用 + virtual AsrStreamStep streamAppend(const float * pcm, size_t nSamples, + const std::string & prefix, int maxTokens) = 0; + virtual void streamReset() = 0; + + // 用词表把 text 编码成 token、丢掉最后 dropLast 个、再还原成文本。 + // 对应官方的 unfixed_token_num 回滚。 + virtual std::string dropLastTokens(const std::string & text, int dropLast) const = 0; + + // 便利：不带前缀的整段转写 + AsrResult transcribe(const float * pcm, size_t nSamples) + { + return transcribeWithPrefix(pcm, nSamples, std::string()); + } +}; + +} // namespace lc diff --git a/src/core/livepipeline.cpp b/src/core/livepipeline.cpp new file mode 100644 index 0000000..18ffc7c --- /dev/null +++ b/src/core/livepipeline.cpp @@ -0,0 +1,147 @@ +#include "livepipeline.h" + +#include + +namespace lc { + +namespace { +constexpr int kFrameSamples = 320; // 20ms @16k +} // namespace + +LivePipeline::LivePipeline(IAsr & asr, LivePipelineOptions opt) + : asr_(asr), opt_(std::move(opt)), stream_(asr, opt_.stream) {} + +LivePipeline::~LivePipeline() +{ + stop(); +} + +std::string LivePipeline::deviceName() const +{ + return capture_.deviceName(); +} + +bool LivePipeline::start(std::string * error) +{ + const bool ok = capture_.start( + asr_.sampleRate(), + [this](const float * s, size_t n) { + { + std::lock_guard lk(mutex_); + pending_.insert(pending_.end(), s, s + n); + } + cv_.notify_one(); + }, + error); + if (!ok) return false; + + worker_ = std::thread([this] { workerLoop(); }); + return true; +} + +void LivePipeline::stop() +{ + { + std::lock_guard lk(mutex_); + quit_ = true; + } + cv_.notify_all(); + if (worker_.joinable()) worker_.join(); + capture_.stop(); +} + +void LivePipeline::workerLoop() +{ + for (;;) { + std::vector chunk; + { + std::unique_lock lk(mutex_); + cv_.wait_for(lk, std::chrono::milliseconds(50), + [this] { return quit_ || !pending_.empty(); }); + if (quit_ && pending_.empty()) break; + chunk.swap(pending_); + } + if (!chunk.empty()) process(chunk.data(), chunk.size()); + } + + if (inSegment_) finishSegment(); +} + +void LivePipeline::emitStep(const StreamingAsr::Step & s) +{ + if (!s.ok || !onCaption_) return; + onCaption_(s.committed, s.tail, s.language); +} + +void LivePipeline::process(const float * pcm, size_t n) +{ + buffer_.insert(buffer_.end(), pcm, pcm + n); + + const size_t prerollSamples = (size_t) (opt_.prerollSec * asr_.sampleRate()); + const float frameSec = (float) kFrameSamples / asr_.sampleRate(); + + std::vector segAudio; // 本段要送进引擎的音频 + bool flushNow = false; + + size_t off = 0; + for (; off + kFrameSamples <= buffer_.size(); off += kFrameSamples) { + const float * frame = buffer_.data() + off; + const bool speech = vad_.feed(frame, kFrameSamples); + + if (!inSegment_) { + // 静音期间什么都不送给模型：否则模型会对静音产生幻觉 + if (speech) { + inSegment_ = true; + silenceAcc_ = 0.0f; + segAudio.insert(segAudio.end(), preroll_.begin(), preroll_.end()); + preroll_.clear(); + segAudio.insert(segAudio.end(), frame, frame + kFrameSamples); + } else { + preroll_.insert(preroll_.end(), frame, frame + kFrameSamples); + if (preroll_.size() > prerollSamples) { + preroll_.erase(preroll_.begin(), + preroll_.begin() + (long) (preroll_.size() - prerollSamples)); + } + } + } else { + segAudio.insert(segAudio.end(), frame, frame + kFrameSamples); + if (speech) silenceAcc_ = 0.0f; + else silenceAcc_ += frameSec; + + if (silenceAcc_ >= (float) opt_.silenceTimeoutSec) { + flushNow = true; + off += kFrameSamples; // 这一帧已经并入 segAudio + break; + } + } + } + + if (!segAudio.empty()) { + const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); }; + stream_.feed(segAudio.data(), segAudio.size(), cb); + } + + if (flushNow) { + const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); }; + stream_.flush(cb); + stream_.reset(); + inSegment_ = false; + silenceAcc_ = 0.0f; + preroll_.clear(); + } + + if (off > 0) buffer_.erase(buffer_.begin(), buffer_.begin() + (long) off); +} + +void LivePipeline::finishSegment() +{ + if (!inSegment_) return; + const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); }; + stream_.flush(cb); + stream_.reset(); + inSegment_ = false; + silenceAcc_ = 0.0f; + preroll_.clear(); +} + +} // namespace lc diff --git a/src/core/livepipeline.h b/src/core/livepipeline.h new file mode 100644 index 0000000..dc26370 --- /dev/null +++ b/src/core/livepipeline.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "audio.h" +#include "iasr.h" +#include "streamingasr.h" +#include "vad.h" + +namespace lc { + +struct LivePipelineOptions { + StreamOptions stream; + double silenceTimeoutSec = 0.8; // 静音多久算一句结束 + double prerollSec = 0.25; // 语音开始前保留的一段，避免吃掉词头 +}; + +// 实时管线：采集 → VAD → 分段 → 流式转写。 +// +// 刻意不带任何 Qt 依赖（回调用 std::function），好处是可以脱离 UI 单独测试： +// 测试直接调 process() 喂音频，不必碰 PulseAudio，也不用跑事件循环。 +class LivePipeline { +public: + using CaptionCallback = std::function; + using ErrorCallback = std::function; + + LivePipeline(IAsr & asr, LivePipelineOptions opt = {}); + ~LivePipeline(); + + LivePipeline(const LivePipeline &) = delete; + LivePipeline & operator=(const LivePipeline &) = delete; + + void setCaptionCallback(CaptionCallback cb) { onCaption_ = std::move(cb); } + void setErrorCallback(ErrorCallback cb) { onError_ = std::move(cb); } + + // 启动真实采集（默认输出设备的 monitor） + bool start(std::string * error = nullptr); + void stop(); + std::string deviceName() const; + + // 同步处理一段 16k 单声道音频。 + // 真实运行时由采集线程调用；测试可以直接调它。 + void process(const float * pcm, size_t n); + + // 结束当前这一段并提交（流结束或手动收尾时用） + void finishSegment(); + + // 注意：StreamingAsr::committedText() 是按值返回的，这里也必须按值返回 + std::string committedText() const { return stream_.committedText(); } + const std::string & language() const { return stream_.language(); } + int chunkCount() const { return stream_.chunkCount(); } + bool inSegment() const { return inSegment_; } + +private: + void workerLoop(); + void emitStep(const StreamingAsr::Step & s); + + IAsr & asr_; + LivePipelineOptions opt_; + Vad vad_; + StreamingAsr stream_; + + CaptionCallback onCaption_; + ErrorCallback onError_; + + bool inSegment_ = false; + float silenceAcc_ = 0.0f; + std::vector preroll_; // 语音开始前保留的一小段 + std::vector buffer_; // 跨批次的残留样本（不接上会持续丢样本） + + MonitorCapture capture_; + std::thread worker_; + std::mutex mutex_; + std::condition_variable cv_; + std::vector pending_; + bool quit_ = false; +}; + +} // namespace lc diff --git a/src/qwen3asr.cpp b/src/core/qwen3asr.cpp similarity index 98% rename from src/qwen3asr.cpp rename to src/core/qwen3asr.cpp index 73a02d7..77879ad 100644 --- a/src/qwen3asr.cpp +++ b/src/core/qwen3asr.cpp @@ -186,11 +186,6 @@ bool Qwen3Asr::load(const AsrOptions & opt, std::string * error) return true; } -Qwen3Asr::Result Qwen3Asr::transcribe(const float * pcm, size_t nSamples) -{ - return transcribeWithPrefix(pcm, nSamples, std::string()); -} - std::string Qwen3Asr::dropLastTokens(const std::string & text, int dropLast) const { if (!d->vocab || dropLast <= 0 || text.empty()) return text; @@ -220,10 +215,10 @@ std::string Qwen3Asr::dropLastTokens(const std::string & text, int dropLast) con return out; } -Qwen3Asr::Result Qwen3Asr::transcribeWithPrefix(const float * pcm, size_t nSamples, +AsrResult Qwen3Asr::transcribeWithPrefix(const float * pcm, size_t nSamples, const std::string & prefix) { - Result res; + AsrResult res; if (!d->model || !d->ctx || !d->mctx) { res.error = "模型未加载"; @@ -377,10 +372,10 @@ bool Qwen3Asr::prepareStream(std::string * error) // 按这个粒度切，编码结果才和「整段音频一次编码」一致；切小了会明显掉准确率。 static constexpr double kSliceSeconds = 8.0; -Qwen3Asr::StreamStep Qwen3Asr::streamAppend(const float * pcm, size_t nSamples, +AsrStreamStep Qwen3Asr::streamAppend(const float * pcm, size_t nSamples, const std::string & prefix, int maxTokens) { - StreamStep res; + AsrStreamStep res; if (!d->model || !d->ctx || !d->mctx) { res.error = "模型未加载"; return res; } std::string err; diff --git a/src/core/qwen3asr.h b/src/core/qwen3asr.h new file mode 100644 index 0000000..fe580fe --- /dev/null +++ b/src/core/qwen3asr.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include "iasr.h" + +namespace lc { + +struct AsrOptions { + std::string modelPath; // Qwen3-ASR 主模型 GGUF（架构须为 qwen3vl） + std::string mmprojPath; // 音频编码器 mmproj GGUF（clip + qwen3a） + int threads = 8; + int nCtx = 4096; + int nBatch = 4096; + int nUbatch = 1024; + int maxTokens = 256; // 单段最多生成多少 token + bool warmup = false; // mtmd 初始化后是否跑一次预热编码 + bool verbose = false; +}; + +// 进程内 Qwen3-ASR 推理（libllama + libmtmd）。 +// 模型常驻，可反复调用；每次整段转写会清空 KV cache。 +class Qwen3Asr : public IAsr { +public: + Qwen3Asr(); + ~Qwen3Asr() override; + + Qwen3Asr(const Qwen3Asr &) = delete; + Qwen3Asr & operator=(const Qwen3Asr &) = delete; + + bool load(const AsrOptions & opt, std::string * error); + + // ---- IAsr ---- + int sampleRate() const override; + + AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples, + const std::string & prefix) override; + + AsrStreamStep streamAppend(const float * pcm, size_t nSamples, + const std::string & prefix, int maxTokens) override; + void streamReset() override; + + std::string dropLastTokens(const std::string & text, int dropLast) const override; + +private: + bool prepareStream(std::string * error); + + struct Impl; + Impl * d = nullptr; +}; + +} // namespace lc diff --git a/src/streamingasr.cpp b/src/core/streamingasr.cpp similarity index 90% rename from src/streamingasr.cpp rename to src/core/streamingasr.cpp index 225938d..93f6711 100644 --- a/src/streamingasr.cpp +++ b/src/core/streamingasr.cpp @@ -1,11 +1,10 @@ #include "streamingasr.h" -#include "qwen3asr.h" #include "transcript.h" namespace lc { -StreamingAsr::StreamingAsr(Qwen3Asr & asr, StreamOptions opt) +StreamingAsr::StreamingAsr(IAsr & asr, StreamOptions opt) : asr_(asr), opt_(opt) { asr_.streamReset(); @@ -18,6 +17,7 @@ void StreamingAsr::reset() pending_.clear(); totalSeconds_ = 0.0; committedRaw_.clear(); + lastRawFull_.clear(); language_.clear(); chunkIndex_ = 0; } @@ -58,11 +58,15 @@ void StreamingAsr::flush(const std::function & onStep) return; } - // 没有待处理音频：直接把当前已稳定文本提交为最终 - if (committedRaw_.empty()) return; + // 没有待处理音频：直接把这一段提交掉。 + // 注意要用 lastRawFull_（含不稳定尾部）而不是 committedRaw_： + // 段恰好结束在 chunk 边界时，否则最后几个词会被丢掉。 + const std::string full = lastRawFull_.empty() ? committedRaw_ : lastRawFull_; + if (full.empty()) return; - historyText_ += parseAsrOutput(committedRaw_).text; + historyText_ += parseAsrOutput(full).text; committedRaw_.clear(); + lastRawFull_.clear(); Step s; s.chunkIndex = chunkIndex_; @@ -92,7 +96,7 @@ void StreamingAsr::runStep(const std::vector & audio, bool finalStep, std::string newText; if (opt_.incremental) { // 只把这一块交给增量引擎；音频总长不影响单步成本 - Qwen3Asr::StreamStep r = + AsrStreamStep r = asr_.streamAppend(audio.data(), audio.size(), prefix, opt_.maxTokensPerStep); s.ok = r.ok; s.error = r.error; @@ -112,7 +116,7 @@ void StreamingAsr::runStep(const std::vector & audio, bool finalStep, buffer_.begin() + (long) (buffer_.size() - maxSamples)); } } - Qwen3Asr::Result r = + AsrResult r = asr_.transcribeWithPrefix(buffer_.data(), buffer_.size(), prefix); s.ok = r.ok; s.error = r.error; @@ -124,6 +128,7 @@ void StreamingAsr::runStep(const std::vector & audio, bool finalStep, if (s.ok) { // 拼接要用未经 trim 的文本：流式续写时开头那个空格是有意义的 s.rawFull = prefix + newText; + lastRawFull_ = s.rawFull; const ParsedTranscript pSeg = parseAsrOutput(s.rawFull); if (!pSeg.language.empty()) language_ = pSeg.language; diff --git a/src/streamingasr.h b/src/core/streamingasr.h similarity index 94% rename from src/streamingasr.h rename to src/core/streamingasr.h index df1cf66..6655eab 100644 --- a/src/streamingasr.h +++ b/src/core/streamingasr.h @@ -5,9 +5,9 @@ #include #include -namespace lc { +#include "iasr.h" -class Qwen3Asr; +namespace lc { // 官方流式解码的参数（见 docs/phase2-streaming.md） struct StreamOptions { @@ -48,7 +48,7 @@ public: std::string error; }; - StreamingAsr(Qwen3Asr & asr, StreamOptions opt = {}); + StreamingAsr(IAsr & asr, StreamOptions opt = {}); // 喂入 16k 单声道音频；每凑够 chunkSec 就跑一步并回调 void feed(const float * pcm, size_t n, const std::function & onStep); @@ -70,12 +70,13 @@ private: void runStep(const std::vector & audio, bool finalStep, const std::function & onStep); - Qwen3Asr & asr_; + IAsr & asr_; StreamOptions opt_; std::vector buffer_; // window 模式：当前窗口音频 std::vector pending_; // 尚未凑够一块的音频 double totalSeconds_ = 0.0; std::string committedRaw_; // 当前段的原始已提交文本（含 language 头，用于回喂） + std::string lastRawFull_; // 上一次完整的原始文本（含不稳定尾部） std::string historyText_; // 此前各段已提交的正文（已剥离 language 头） std::string language_; int chunkIndex_ = 0; diff --git a/src/transcript.cpp b/src/core/transcript.cpp similarity index 100% rename from src/transcript.cpp rename to src/core/transcript.cpp diff --git a/src/transcript.h b/src/core/transcript.h similarity index 100% rename from src/transcript.h rename to src/core/transcript.h diff --git a/src/core/vad.h b/src/core/vad.h new file mode 100644 index 0000000..3285c50 --- /dev/null +++ b/src/core/vad.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +namespace lc { + +// 能量 VAD。 +// +// 噪声底必须用"快降慢升"的**最小值跟踪**，不能用均值： +// 否则遇到本身就很轻的音频（例如峰值 RMS 只有 0.03 的录音）， +// 噪声底会一路抬升、阈值超过语音电平，把绝大部分话音都判成静音。 +class Vad { +public: + explicit Vad(float floorInit = 0.01f, float factor = 3.0f, float minThreshold = 0.0015f) + : floor_(floorInit), factor_(factor), minThreshold_(minThreshold) {} + + // 返回该帧是否为语音 + bool feed(const float * frame, size_t n) + { + double acc = 0.0; + for (size_t i = 0; i < n; ++i) acc += (double) frame[i] * frame[i]; + const float rms = std::sqrt(acc / (double) n); + + if (rms < floor_) floor_ = rms; // 快速下探 + else floor_ = floor_ * 0.9999f + rms * 0.0001f; // 极慢回升 + + lastRms_ = rms; + return rms > std::max(floor_ * factor_, minThreshold_); + } + + float noiseFloor() const { return floor_; } + float lastRms() const { return lastRms_; } + +private: + float floor_; + float factor_; + float minThreshold_; + float lastRms_ = 0.0f; +}; + +} // namespace lc diff --git a/src/livetranscriber.cpp b/src/livetranscriber.cpp deleted file mode 100644 index d168ea5..0000000 --- a/src/livetranscriber.cpp +++ /dev/null @@ -1,178 +0,0 @@ -#include "livetranscriber.h" - -#include -#include - -#include "qwen3asr.h" - -namespace lc { - -namespace { - -constexpr int kFrameSamples = 320; // 20ms @16k -constexpr float kSilenceTimeout = 0.8f; // 静音多久算一句结束 -constexpr float kPrerollSec = 0.25f; // 语音开始前保留的一段，避免吃掉词头 - -// 能量 VAD。噪声底用"快降慢升"的最小值跟踪（不能用均值，否则轻音频会被全判成静音）。 -class Vad { -public: - bool feed(const float * frame, size_t n) - { - double acc = 0.0; - for (size_t i = 0; i < n; ++i) acc += (double) frame[i] * frame[i]; - const float rms = std::sqrt(acc / (double) n); - - if (rms < floor_) floor_ = rms; - else floor_ = floor_ * 0.9999f + rms * 0.0001f; - - return rms > std::max(floor_ * 3.0f, 0.0015f); - } - -private: - float floor_ = 0.01f; -}; - -} // namespace - -LiveTranscriber::LiveTranscriber(Qwen3Asr & asr, QObject * parent) - : QObject(parent), asr_(asr) {} - -LiveTranscriber::~LiveTranscriber() -{ - stop(); -} - -QString LiveTranscriber::deviceName() const -{ - return QString::fromStdString(capture_.deviceName()); -} - -bool LiveTranscriber::start(const StreamOptions & opts, QString * error) -{ - opts_ = opts; - - std::string err; - const bool ok = capture_.start(asr_.sampleRate(), - [this](const float * s, size_t n) { - { - std::lock_guard lk(mutex_); - pending_.insert(pending_.end(), s, s + n); - } - cv_.notify_one(); - }, - &err); - if (!ok) { - if (error) *error = QString::fromStdString(err); - return false; - } - - worker_ = std::thread([this] { workerLoop(); }); - return true; -} - -void LiveTranscriber::stop() -{ - { - std::lock_guard lk(mutex_); - quit_ = true; - } - cv_.notify_all(); - if (worker_.joinable()) worker_.join(); - - capture_.stop(); -} - -void LiveTranscriber::workerLoop() -{ - Vad vad; - StreamingAsr stream(asr_, opts_); - - float silenceAcc = 0.0f; - bool inSegment = false; - bool flushNow = false; - - std::vector buffer; // 跨批次的残留样本 - std::vector preroll; // 语音开始前保留的一小段，避免吃掉词头 - std::vector segAudio; // 本段待送进引擎的音频 - - const size_t prerollSamples = (size_t) (kPrerollSec * asr_.sampleRate()); - - const auto emitStep = [this](const StreamingAsr::Step & s) { - if (!s.ok) return; - Q_EMIT caption(QString::fromStdString(s.committed), - QString::fromStdString(s.tail), - QString::fromStdString(s.language)); - }; - - const auto feedSegment = [&] { - if (!segAudio.empty()) stream.feed(segAudio.data(), segAudio.size(), emitStep); - segAudio.clear(); - }; - - for (;;) { - std::vector chunk; - { - std::unique_lock lk(mutex_); - cv_.wait_for(lk, std::chrono::milliseconds(50), - [this] { return quit_ || !pending_.empty(); }); - if (quit_ && pending_.empty()) break; - chunk.swap(pending_); - } - - // 不足一帧的残留要接上，否则会持续丢样本 - buffer.insert(buffer.end(), chunk.begin(), chunk.end()); - - size_t off = 0; - for (; off + kFrameSamples <= buffer.size(); off += kFrameSamples) { - const float * frame = buffer.data() + off; - const bool speech = vad.feed(frame, kFrameSamples); - - if (!inSegment) { - // 静音期间什么都不送给模型：否则模型会对静音产生幻觉 - if (speech) { - inSegment = true; - silenceAcc = 0.0f; - segAudio.insert(segAudio.end(), preroll.begin(), preroll.end()); - preroll.clear(); - segAudio.insert(segAudio.end(), frame, frame + kFrameSamples); - } else { - preroll.insert(preroll.end(), frame, frame + kFrameSamples); - if (preroll.size() > prerollSamples) { - preroll.erase(preroll.begin(), - preroll.begin() + (long) (preroll.size() - prerollSamples)); - } - } - } else { - segAudio.insert(segAudio.end(), frame, frame + kFrameSamples); - if (speech) silenceAcc = 0.0f; - else silenceAcc += (float) kFrameSamples / asr_.sampleRate(); - if (silenceAcc >= kSilenceTimeout) { - flushNow = true; - off += kFrameSamples; // 这一帧已经并入 segAudio - break; - } - } - } - - if (!segAudio.empty()) feedSegment(); - - // 静音够久 → 收束这一段（提交文本并重置上下文） - if (flushNow) { - stream.flush(emitStep); - stream.reset(); - inSegment = false; - silenceAcc = 0.0f; - flushNow = false; - preroll.clear(); - } - - if (off > 0) buffer.erase(buffer.begin(), buffer.begin() + (long) off); - } - - if (inSegment) { - feedSegment(); - stream.flush(emitStep); - } -} - -} // namespace lc diff --git a/src/livetranscriber.h b/src/livetranscriber.h deleted file mode 100644 index 40858a5..0000000 --- a/src/livetranscriber.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include - -#include "audio.h" -#include "streamingasr.h" - -namespace lc { - -class Qwen3Asr; - -// 实时字幕：采集默认输出 → 流式转写 → 用信号把文本推给 UI。 -// 分段由静音检测驱动：句子之间的自然停顿就收束一段、清空上下文， -// 这样解码器的上下文不会无限增长（见 docs/phase2-streaming.md）。 -class LiveTranscriber : public QObject { - Q_OBJECT - -public: - explicit LiveTranscriber(Qwen3Asr & asr, QObject * parent = nullptr); - ~LiveTranscriber() override; - - bool start(const StreamOptions & opts, QString * error = nullptr); - void stop(); - - QString deviceName() const; - -Q_SIGNALS: - // committed = 已稳定文本；tail = 不稳定尾部（UI 用暗色显示） - void caption(const QString & committed, const QString & tail, const QString & language); - void failed(const QString & message); - -private: - void workerLoop(); - - Qwen3Asr & asr_; - MonitorCapture capture_; - StreamOptions opts_; - std::thread worker_; - std::mutex mutex_; - std::condition_variable cv_; - std::vector pending_; - bool quit_ = false; -}; - -} // namespace lc diff --git a/src/main.cpp b/src/main.cpp deleted file mode 100644 index f61a106..0000000 --- a/src/main.cpp +++ /dev/null @@ -1,288 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "audio.h" -#include "captionwindow.h" -#include "livetranscriber.h" -#include "qwen3asr.h" -#include "streamingasr.h" -#include "transcript.h" - -namespace { - -// --------------------------------------------------------------------------- -// 文件模式 -// --------------------------------------------------------------------------- -int runFile(lc::Qwen3Asr & asr, const QString & path) -{ - std::vector pcm; - std::string err; - if (!lc::readWavMono(path.toStdString(), asr.sampleRate(), pcm, &err)) { - std::fprintf(stderr, "读取音频失败: %s\n", err.c_str()); - return 1; - } - - const double dur = (double) pcm.size() / asr.sampleRate(); - std::fprintf(stderr, "音频: %s (%.2fs)\n", qUtf8Printable(path), dur); - - const auto t0 = std::chrono::steady_clock::now(); - lc::Qwen3Asr::Result r = asr.transcribe(pcm.data(), pcm.size()); - const double wall = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); - - if (!r.ok) { - std::fprintf(stderr, "转写失败: %s\n", r.error.c_str()); - return 1; - } - - std::printf("语言: %s (%s)\n", r.transcript.language.c_str(), - lc::languageDisplayName(r.transcript.language).c_str()); - std::printf("文本: %s\n", r.transcript.text.c_str()); - std::fprintf(stderr, "推理: %.2fs / 音频 %.2fs (rtf %.3f) 生成 %d tokens, 墙钟 %.2fs\n", - r.inferSeconds, dur, dur > 0 ? r.inferSeconds / dur : 0, r.tokenCount, wall); - return 0; -} - -// --------------------------------------------------------------------------- -// 流式文件模式（把文件当音频流按块喂进去） -// --------------------------------------------------------------------------- -int runStreamFile(lc::Qwen3Asr & asr, const QString & path, const lc::StreamOptions & sop) -{ - std::vector pcm; - std::string err; - if (!lc::readWavMono(path.toStdString(), asr.sampleRate(), pcm, &err)) { - std::fprintf(stderr, "读取音频失败: %s\n", err.c_str()); - return 1; - } - - const double dur = (double) pcm.size() / asr.sampleRate(); - std::fprintf(stderr, "流式转写: %s (%.2fs) chunk=%.1fs segment=%.1fs mode=%s\n", - qUtf8Printable(path), dur, sop.chunkSec, sop.maxUtteranceSec, - sop.incremental ? "incremental" : "window"); - - lc::StreamingAsr stream(asr, sop); - const size_t chunkSamples = (size_t) (sop.chunkSec * asr.sampleRate()); - - const auto onStep = [](const lc::StreamingAsr::Step & s) { - std::printf("\n[chunk %2d] 窗口 %5.1fs / 编码 %5.2fs / 本步 %5.2fs%s\n", - s.chunkIndex, s.windowSeconds, s.encodeSeconds, s.inferSeconds, - s.ok ? "" : " <失败>"); - if (!s.ok) std::printf(" error: %s\n", s.error.c_str()); - std::printf(" 已提交: %s\n", s.committed.c_str()); - std::printf(" 含尾部: %s\n", s.full.c_str()); - std::fflush(stdout); - }; - - for (size_t off = 0; off < pcm.size(); off += chunkSamples) { - const size_t n = std::min(chunkSamples, pcm.size() - off); - stream.feed(pcm.data() + off, n, onStep); - } - stream.flush(onStep); - - std::printf("\n==== 最终提交文本 (%s) ====\n%s\n", - lc::languageDisplayName(stream.language()).c_str(), - stream.committedText().c_str()); - return 0; -} - -// --------------------------------------------------------------------------- -// 命令行实时模式（无窗口，便于在无显示环境下调试） -// --------------------------------------------------------------------------- -int runCliLive(lc::Qwen3Asr & asr, const lc::StreamOptions & sop) -{ - lc::LiveTranscriber live(asr); - - QObject::connect(&live, &lc::LiveTranscriber::caption, - [](const QString & committed, const QString & tail, const QString & lang) { - // 用 \r 原地刷新：稳定部分正常显示，尾部加括号表示未定 - std::printf("\r\033[K[%s] %s\033[2m%s\033[0m", - lc::languageDisplayName(lang.toStdString()).c_str(), - qUtf8Printable(committed), qUtf8Printable(tail)); - std::fflush(stdout); - }); - QObject::connect(&live, &lc::LiveTranscriber::failed, [](const QString & m) { - std::fprintf(stderr, "\n采集失败: %s\n", qUtf8Printable(m)); - }); - - QString err; - if (!live.start(sop, &err)) { - std::fprintf(stderr, "启动采集失败: %s\n", qUtf8Printable(err)); - return 1; - } - std::fprintf(stderr, "正在采集: %s (Ctrl+C 退出)\n", qUtf8Printable(live.deviceName())); - return QCoreApplication::exec(); -} - -// --------------------------------------------------------------------------- -// 图形界面模式：悬浮字幕窗 -// --------------------------------------------------------------------------- -int runGui(lc::Qwen3Asr & asr, const lc::StreamOptions & sop, - const QString & screenshotPath, int screenshotDelaySec, int opacityPercent) -{ - lc::CaptionWindow window; - window.setPanelOpacity(opacityPercent); - lc::LiveTranscriber live(asr); - - QObject::connect(&live, &lc::LiveTranscriber::caption, &window, - [&window](const QString & committed, const QString & tail, - const QString & lang) { - window.setLanguage( - QString::fromStdString(lc::languageDisplayName(lang.toStdString()))); - window.setCaption(committed, tail); - }); - QObject::connect(&live, &lc::LiveTranscriber::failed, &window, [](const QString & m) { - qWarning("采集失败: %s", qUtf8Printable(m)); - }); - QObject::connect(&window, &lc::CaptionWindow::closed, qApp, &QCoreApplication::quit); - QObject::connect(&window, &lc::CaptionWindow::languageClicked, &window, [&window] { - qInfo() << "语言标签被点击（设置界面将在 Phase 4 接入）:" << window.isCollapsed(); - }); - - QString err; - if (!live.start(sop, &err)) { - qCritical("启动采集失败: %s", qUtf8Printable(err)); - return 1; - } - qInfo("正在采集: %s", qUtf8Printable(live.deviceName())); - - window.show(); - - if (!screenshotPath.isEmpty()) { - // 自截图：用于验证渲染（不依赖外部截图工具） - QTimer::singleShot(screenshotDelaySec * 1000, &window, [&window, screenshotPath] { - const bool ok1 = window.grab().save(screenshotPath); - - // 再截一张展开态的 - window.setCollapsed(false); - const QString expanded = screenshotPath; - QString expandedPath = expanded; - expandedPath.replace(QStringLiteral(".png"), QStringLiteral("-expanded.png")); - const bool ok2 = window.grab().save(expandedPath); - - std::fprintf(stderr, "截图: 折叠态 %s / 展开态 %s\n", - ok1 ? "成功" : "失败", ok2 ? "成功" : "失败"); - QCoreApplication::quit(); - }); - } - - return QCoreApplication::exec(); -} - -} // namespace - -int main(int argc, char ** argv) -{ - // 文件类模式不需要图形环境；实时模式（默认）需要 QApplication - bool wantsWidgets = true; - for (int i = 1; i < argc; ++i) { - const QString a = QString::fromLocal8Bit(argv[i]); - if (a == QLatin1String("--file") || a == QLatin1String("--stream-file")) { - wantsWidgets = false; - break; - } - } - - std::unique_ptr app; - if (wantsWidgets) { - // 说明意图：DPI 取整策略必须在 QGuiApplication 之前定。 - // （Deepin 的会话环境已经设了 QT_SCALE_FACTOR_ROUNDING_POLICY， - // 所以本机仍会看到 Qt 的那条提醒，那不是本程序的问题。） - QGuiApplication::setHighDpiScaleFactorRoundingPolicy( - Qt::HighDpiScaleFactorRoundingPolicy::PassThrough); - app.reset(new QApplication(argc, argv)); - } else { - app.reset(new QCoreApplication(argc, argv)); - } - QCoreApplication::setApplicationName(QStringLiteral("live-caption")); - - QCommandLineParser parser; - parser.setApplicationDescription(QStringLiteral("实时字幕（Qwen3-ASR + llama.cpp）")); - parser.addHelpOption(); - - QCommandLineOption optModel("model", "主模型 GGUF 路径（架构须为 qwen3vl）", "path"); - QCommandLineOption optMmproj("mmproj", "mmproj GGUF 路径（clip + qwen3a）", "path"); - QCommandLineOption optFile("file", "转写一个 wav 文件后退出", "path"); - QCommandLineOption optStreamFile("stream-file", "以流式模式转写 wav 文件后退出", "path"); - QCommandLineOption optCliLive("cli-live", "实时转写但只用命令行输出（不开窗口）"); - QCommandLineOption optThreads("threads", "推理线程数", "n", "8"); - QCommandLineOption optMaxTokens("max-tokens", "单段最多生成 token 数", "n", "256"); - QCommandLineOption optStreamChunk("stream-chunk", "流式 chunk 秒数", "sec", "2.0"); - QCommandLineOption optStreamWindow("stream-window", "滑窗上限秒数（仅 window 模式）", "sec", "16"); - QCommandLineOption optStreamSegment("stream-segment", "单段最长秒数（0=不限）", "sec", "30"); - QCommandLineOption optUnfixedTokens("unfixed-tokens", "前缀回滚丢弃的 token 数", "n", "5"); - QCommandLineOption optUnfixedChunks("unfixed-chunks", "冷启动不带前缀的 chunk 数", "n", "2"); - QCommandLineOption optStreamMode("stream-mode", "流式后端: incremental 或 window", "mode", "incremental"); - QCommandLineOption optVerbose("verbose", "输出 llama.cpp/mtmd 的详细日志"); - QCommandLineOption optScreenshot("screenshot", "启动后自动截图并退出（验证渲染用）", "path"); - QCommandLineOption optScreenshotDelay("screenshot-delay", "截图前等待秒数", "sec", "6"); - QCommandLineOption optOpacity("opacity", "字幕背景不透明度（20-100，百分比）", "n", "84"); - - parser.addOption(optModel); - parser.addOption(optMmproj); - parser.addOption(optFile); - parser.addOption(optStreamFile); - parser.addOption(optCliLive); - parser.addOption(optThreads); - parser.addOption(optMaxTokens); - parser.addOption(optStreamChunk); - parser.addOption(optStreamWindow); - parser.addOption(optStreamSegment); - parser.addOption(optUnfixedTokens); - parser.addOption(optUnfixedChunks); - parser.addOption(optStreamMode); - parser.addOption(optVerbose); - parser.addOption(optScreenshot); - parser.addOption(optScreenshotDelay); - parser.addOption(optOpacity); - parser.process(*app); - - lc::AsrOptions asrOpt; - asrOpt.modelPath = parser.value(optModel).toStdString(); - asrOpt.mmprojPath = parser.value(optMmproj).toStdString(); - asrOpt.threads = parser.value(optThreads).toInt(); - asrOpt.maxTokens = parser.value(optMaxTokens).toInt(); - asrOpt.verbose = parser.isSet(optVerbose); - - if (asrOpt.modelPath.empty() || asrOpt.mmprojPath.empty()) { - std::fprintf(stderr, "必须提供 --model 和 --mmproj\n"); - return 1; - } - - lc::StreamOptions sop; - sop.chunkSec = parser.value(optStreamChunk).toDouble(); - sop.windowSec = parser.value(optStreamWindow).toDouble(); - sop.maxUtteranceSec = parser.value(optStreamSegment).toDouble(); - sop.unfixedTokenNum = parser.value(optUnfixedTokens).toInt(); - sop.unfixedChunkNum = parser.value(optUnfixedChunks).toInt(); - sop.maxTokensPerStep = asrOpt.maxTokens; - sop.incremental = parser.value(optStreamMode) != QLatin1String("window"); - - lc::Qwen3Asr asr; - std::string err; - const auto t0 = std::chrono::steady_clock::now(); - if (!asr.load(asrOpt, &err)) { - std::fprintf(stderr, "加载失败: %s\n", err.c_str()); - return 1; - } - std::fprintf(stderr, "模型加载完成: %.2fs (采样率 %d)\n", - std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), - asr.sampleRate()); - - if (parser.isSet(optFile)) return runFile(asr, parser.value(optFile)); - if (parser.isSet(optStreamFile)) return runStreamFile(asr, parser.value(optStreamFile), sop); - if (parser.isSet(optCliLive)) return runCliLive(asr, sop); - return runGui(asr, sop, - parser.value(optScreenshot), - parser.value(optScreenshotDelay).toInt(), - parser.value(optOpacity).toInt()); -} diff --git a/src/qwen3asr.h b/src/qwen3asr.h deleted file mode 100644 index ddbe220..0000000 --- a/src/qwen3asr.h +++ /dev/null @@ -1,82 +0,0 @@ -#pragma once - -#include -#include - -#include "transcript.h" - -namespace lc { - -struct AsrOptions { - std::string modelPath; // Qwen3-ASR 主模型 GGUF（架构须为 qwen3vl） - std::string mmprojPath; // 音频编码器 mmproj GGUF（clip + qwen3a） - int threads = 8; - int nCtx = 4096; - int nBatch = 4096; - int nUbatch = 1024; - int maxTokens = 256; // 单段最多生成多少 token - bool warmup = false; // mtmd 初始化后是否跑一次预热编码 - bool verbose = false; -}; - -// 进程内 Qwen3-ASR 推理封装（libllama + libmtmd）。 -// 模型常驻，可反复 transcribe()；每次调用会清空 KV cache。 -class Qwen3Asr { -public: - Qwen3Asr(); - ~Qwen3Asr(); - - Qwen3Asr(const Qwen3Asr &) = delete; - Qwen3Asr & operator=(const Qwen3Asr &) = delete; - - bool load(const AsrOptions & opt, std::string * error); - - // 期望的输入采样率（Qwen3-ASR 为 16000） - int sampleRate() const; - - struct Result { - bool ok = false; - std::string error; - ParsedTranscript transcript; // 已拆出 language / text - std::string raw; // 模型原始输出，便于排查 - double inferSeconds = 0.0; - int tokenCount = 0; - }; - - // pcm: 单声道 float32 [-1,1]，采样率为 sampleRate() - Result transcribe(const float * pcm, size_t nSamples); - - // 流式单步：把 prefix 当作"已经生成的 assistant 文本"接在 assistant 头之后， - // 让模型从 prefix 末尾继续生成。官方流式算法的核心（前缀回滚）就靠它。 - Result transcribeWithPrefix(const float * pcm, size_t nSamples, const std::string & prefix); - - // 用模型词表把 text 编码成 token、丢掉最后 dropLast 个、再还原成文本。 - // 对应官方的 unfixed_token_num 回滚。 - std::string dropLastTokens(const std::string & text, int dropLast) const; - - // ---- 流式（增量 KV 复用）---- - // 与上面每次重编码整个窗口的做法不同：这里只对新到的音频做一次编码， - // 已经解码过的音频留在 KV cache 里复用，因此每步成本与音频总长无关。 - void streamReset(); // 清空 KV 与流式状态 - - struct StreamStep { - bool ok = false; - std::string error; - std::string newText; // 本步新生成的文本（不含前缀，未 trim） - int newTokens = 0; - int audioTokens = 0; // 本步新追加的音频 token 数 - double encodeSeconds = 0.0; - double inferSeconds = 0.0; - }; - // 追加一段音频，在 assistant 头之后接上 prefix 作为"已生成文本"继续生成。 - StreamStep streamAppend(const float * pcm, size_t nSamples, - const std::string & prefix, int maxTokens); - -private: - bool prepareStream(std::string * error); - - struct Impl; - Impl * d = nullptr; -}; - -} // namespace lc diff --git a/src/captionwindow.cpp b/src/ui/captionwindow.cpp similarity index 100% rename from src/captionwindow.cpp rename to src/ui/captionwindow.cpp diff --git a/src/captionwindow.h b/src/ui/captionwindow.h similarity index 100% rename from src/captionwindow.h rename to src/ui/captionwindow.h diff --git a/src/ui/livetranscriber.cpp b/src/ui/livetranscriber.cpp new file mode 100644 index 0000000..2f81f0d --- /dev/null +++ b/src/ui/livetranscriber.cpp @@ -0,0 +1,60 @@ +#include "livetranscriber.h" + +#include "iasr.h" + +namespace lc { + +LiveTranscriber::LiveTranscriber(IAsr & asr, QObject * parent) + : QObject(parent), asr_(asr) +{ +} + +LiveTranscriber::~LiveTranscriber() +{ + stop(); +} + +QString LiveTranscriber::deviceName() const +{ + return pipeline_ ? QString::fromStdString(pipeline_->deviceName()) : QString(); +} + +bool LiveTranscriber::start(const StreamOptions & opts, QString * error) +{ + stop(); + + LivePipelineOptions popt; + popt.stream = opts; + + pipeline_ = std::make_unique(asr_, popt); + + // 回调在工作线程上被调用；连到主线程的接收者时 Qt 会自动排队投递。 + pipeline_->setCaptionCallback([this](const std::string & committed, + const std::string & tail, + const std::string & language) { + Q_EMIT caption(QString::fromStdString(committed), + QString::fromStdString(tail), + QString::fromStdString(language)); + }); + pipeline_->setErrorCallback([this](const std::string & message) { + Q_EMIT failed(QString::fromStdString(message)); + }); + + std::string err; + if (!pipeline_->start(&err)) { + if (error) *error = QString::fromStdString(err); + pipeline_.reset(); + return false; + } + return true; +} + +void LiveTranscriber::stop() +{ + if (pipeline_) { + pipeline_->stop(); + pipeline_.reset(); + } +} + +} // namespace lc diff --git a/src/ui/livetranscriber.h b/src/ui/livetranscriber.h new file mode 100644 index 0000000..2d3df1d --- /dev/null +++ b/src/ui/livetranscriber.h @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include +#include + +#include "livepipeline.h" +#include "streamingasr.h" + +namespace lc { + +class IAsr; + +// LivePipeline 的 Qt 适配器：只负责把 std::function 回调转成 Qt 信号， +// 真正的逻辑都在 core 里（所以那些逻辑可以脱离 UI 单独测试）。 +class LiveTranscriber : public QObject { + Q_OBJECT + +public: + explicit LiveTranscriber(IAsr & asr, QObject * parent = nullptr); + ~LiveTranscriber() override; + + bool start(const StreamOptions & opts, QString * error = nullptr); + void stop(); + + QString deviceName() const; + +Q_SIGNALS: + // committed = 已稳定文本；tail = 不稳定尾部（UI 用暗色显示） + void caption(const QString & committed, const QString & tail, const QString & language); + void failed(const QString & message); + +private: + IAsr & asr_; + std::unique_ptr pipeline_; +}; + +} // namespace lc + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..8fbe155 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +# 单元测试：只依赖 core，不需要模型文件、不需要显示环境。 +# 需要真模型的集成测试不放在这里（会另外打 label）。 +find_package(Qt6 REQUIRED COMPONENTS Test) + +function(lc_add_test name) + add_executable(${name} ${name}.cpp) + target_link_libraries(${name} PRIVATE livecaption_core Qt6::Test) + target_include_directories(${name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + add_test(NAME ${name} COMMAND ${name}) +endfunction() + +lc_add_test(tst_transcript) +lc_add_test(tst_vad) +lc_add_test(tst_streamingasr) +lc_add_test(tst_livepipeline) diff --git a/tests/fakeasr.h b/tests/fakeasr.h new file mode 100644 index 0000000..b1e3b23 --- /dev/null +++ b/tests/fakeasr.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include +#include + +#include "iasr.h" + +// 可脚本化的假推理后端：让流式策略与实时管线可以脱离真模型测试。 +// +// dropLastTokens 用"空格分词"模拟词表：丢掉最后 N 个词，并保留原有格式 +// （这点很重要 —— 续写开头那个空格是有意义的）。 +class FakeAsr : public lc::IAsr { +public: + // 每次 streamAppend 依次返回这里的文本；用完后返回空串 + std::vector scripted; + + // 记录（供断言用） + std::vector prefixesSeen; + std::vector samplesSeen; + int resets = 0; + + int sampleRate() const override { return 16000; } + + lc::AsrResult transcribeWithPrefix(const float *, size_t, + const std::string &) override + { + lc::AsrResult r; + r.ok = true; + if (call_ < scripted.size()) { + r.raw = scripted[call_]; + r.transcript = lc::parseAsrOutput(r.raw); + } + ++call_; + return r; + } + + lc::AsrStreamStep streamAppend(const float *, size_t nSamples, + const std::string & prefix, int) override + { + prefixesSeen.push_back(prefix); + samplesSeen.push_back(nSamples); + + lc::AsrStreamStep s; + s.ok = true; + if (call_ < scripted.size()) s.newText = scripted[call_]; + ++call_; + s.newTokens = 1; + s.encodeSeconds = 0.001; + s.inferSeconds = 0.001; + return s; + } + + void streamReset() override { ++resets; } + + std::string dropLastTokens(const std::string & text, int dropLast) const override + { + if (dropLast <= 0 || text.empty()) return text; + + const auto isSpace = [](char c) { return std::isspace((unsigned char) c) != 0; }; + + size_t pos = text.size(); + for (int i = 0; i < dropLast; ++i) { + while (pos > 0 && isSpace(text[pos - 1])) --pos; // 跳过尾部空白 + while (pos > 0 && !isSpace(text[pos - 1])) --pos; // 跳过一个词 + if (pos == 0) return {}; + } + while (pos > 0 && isSpace(text[pos - 1])) --pos; + return text.substr(0, pos); + } + +private: + size_t call_ = 0; +}; diff --git a/tests/tst_livepipeline.cpp b/tests/tst_livepipeline.cpp new file mode 100644 index 0000000..dd8b920 --- /dev/null +++ b/tests/tst_livepipeline.cpp @@ -0,0 +1,127 @@ +#include + +#include +#include +#include + +#include "fakeasr.h" +#include "livepipeline.h" + +namespace { + +constexpr double kPi = 3.14159265358979323846; + +std::vector tone(size_t samples, float amplitude) +{ + std::vector v(samples); + for (size_t i = 0; i < samples; ++i) { + v[i] = amplitude * (float) std::sin(2.0 * kPi * 440.0 * (double) i / 16000.0); + } + return v; +} + +size_t totalSamples(const std::vector & v) +{ + size_t n = 0; + for (size_t s : v) n += s; + return n; +} + +lc::LivePipelineOptions options() +{ + lc::LivePipelineOptions o; + o.stream.chunkSec = 0.5; // 0.5s 一块，测试里能快些触发 + return o; +} + +} // namespace + +// 实时管线的回归测试（用 FakeAsr + 直接喂音频，不碰 PulseAudio）。 +class TstLivePipeline : public QObject { + Q_OBJECT + +private slots: + // 静音不能送进模型。 + // 曾经把采集到的静音也喂了进去，模型对静音产生幻觉（英文语音前冒出一个"嗯。"）。 + void silenceIsNotForwarded() + { + FakeAsr fake; + lc::LivePipeline pipeline(fake, options()); + + const auto silence = tone(16000 * 3, 0.001f); // 3s 背景噪声 + pipeline.process(silence.data(), silence.size()); + + QCOMPARE(fake.samplesSeen.size(), size_t(0)); + } + + // 不丢样本。 + // 曾经每次只处理 floor(可用/320) 帧并丢掉余数，每轮丢约 160 个样本， + // 造成假静音和句子被截断。 + void noSamplesDroppedAcrossIrregularChunks() + { + FakeAsr fake; + lc::LivePipeline pipeline(fake, options()); + + const auto speech = tone(16000 * 10, 0.1f); // 10s 语音 + const size_t sizes[] = {1000, 7, 3333, 999, 12345, 1}; + + size_t off = 0; + int i = 0; + while (off < speech.size()) { + const size_t n = std::min(sizes[i++ % 6], speech.size() - off); + pipeline.process(speech.data() + off, n); + off += n; + } + + const size_t forwarded = totalSamples(fake.samplesSeen); + // 允许剩下的不足一块留在引擎里（< 0.5s），其余都必须送到 + QVERIFY2(forwarded >= 16000 * 9, + qPrintable(QString("只送到了 %1 个样本（共 %2）") + .arg(forwarded).arg(speech.size()))); + } + + // 静音够久要收束这一段（提交文本并重置上下文） + void silenceEndsSegment() + { + FakeAsr fake; + fake.scripted = {"a b c d e f"}; + + lc::LivePipeline pipeline(fake, options()); + QString lastCommitted; + pipeline.setCaptionCallback( + [&lastCommitted](const std::string & committed, const std::string &, + const std::string &) { + lastCommitted = QString::fromStdString(committed); + }); + + const auto speech = tone(16000 * 1, 0.1f); + pipeline.process(speech.data(), speech.size()); + const int resetsAfterSpeech = fake.resets; + + const auto silence = tone(16000 * 2, 0.001f); + pipeline.process(silence.data(), silence.size()); + + QVERIFY2(fake.resets > resetsAfterSpeech, "静音后没有收束这一段"); + QVERIFY(!pipeline.inSegment()); + } + + // 语音之后继续有语音：不应该被切断成很多段 + void continuousSpeechStaysInOneSegment() + { + FakeAsr fake; + lc::LivePipeline pipeline(fake, options()); + const int resetsAfterConstruct = fake.resets; // 构造 StreamingAsr 时会 reset 一次 + + const auto speech = tone(16000 * 4, 0.1f); + for (size_t off = 0; off < speech.size(); off += 4000) { + const size_t n = std::min(size_t(4000), speech.size() - off); + pipeline.process(speech.data() + off, n); + } + + QVERIFY(pipeline.inSegment()); + QCOMPARE(fake.resets, resetsAfterConstruct); + } +}; + +QTEST_MAIN(TstLivePipeline) +#include "tst_livepipeline.moc" diff --git a/tests/tst_streamingasr.cpp b/tests/tst_streamingasr.cpp new file mode 100644 index 0000000..ec31242 --- /dev/null +++ b/tests/tst_streamingasr.cpp @@ -0,0 +1,174 @@ +#include + +#include +#include + +#include "fakeasr.h" +#include "streamingasr.h" + +namespace { + +std::vector chunk(size_t samples, float value = 0.1f) +{ + return std::vector(samples, value); +} + +QString q(const std::string & s) { return QString::fromStdString(s); } + +} // namespace + +// 流式策略的回归测试（用 FakeAsr，不加载真模型）。 +class TstStreamingAsr : public QObject { + Q_OBJECT + +private: + static lc::StreamOptions baseOptions() + { + lc::StreamOptions o; + o.chunkSec = 1.0; // 1s 一块，方便构造 + o.unfixedTokenNum = 1; // 回滚 1 个 token，断言更直观 + o.unfixedChunkNum = 0; // 默认就回喂前缀 + o.incremental = true; + return o; + } + +private slots: + // 冷启动阶段（前 unfixedChunkNum 块）不回喂前缀 + void coldStartHasNoPrefix() + { + FakeAsr fake; + fake.scripted = {"a b c d", "e f", "g h"}; + + lc::StreamOptions o = baseOptions(); + o.unfixedChunkNum = 2; + + lc::StreamingAsr stream(fake, o); + const auto cb = [](const lc::StreamingAsr::Step &) {}; + + const auto c = chunk(16000); + stream.feed(c.data(), c.size(), cb); + stream.feed(c.data(), c.size(), cb); + stream.feed(c.data(), c.size(), cb); + + QCOMPARE(fake.prefixesSeen.size(), size_t(3)); + QCOMPARE(q(fake.prefixesSeen[0]), QString("")); + QCOMPARE(q(fake.prefixesSeen[1]), QString("")); + QVERIFY(!fake.prefixesSeen[2].empty()); + } + + // 续写开头那个空格不能被吃掉。 + // 曾经用 trim 过的文本去拼接，导致 "potatoesand" 这种缺空格。 + void keepsJoiningSpace() + { + FakeAsr fake; + fake.scripted = {"language EnglishHe hoped there", + " would be stew"}; + + lc::StreamingAsr stream(fake, baseOptions()); + + std::vector steps; + const auto cb = [&steps](const lc::StreamingAsr::Step & s) { steps.push_back(s); }; + + const auto c = chunk(16000); + stream.feed(c.data(), c.size(), cb); + stream.feed(c.data(), c.size(), cb); + + QCOMPARE(steps.size(), size_t(2)); + QCOMPARE(q(steps[0].full), QString("He hoped there")); + QVERIFY2(q(steps[1].full) == QString("He hoped would be stew"), + qPrintable(QString("实际为 \"%1\"").arg(q(steps[1].full)))); + } + + // 已提交文本 = 完整文本去掉最后 unfixedTokenNum 个 token； + // 尾部单独作为不稳定部分给出。 + void committedDropsUnstableTail() + { + FakeAsr fake; + fake.scripted = {"language EnglishHe hoped there", + " would be stew"}; + + lc::StreamingAsr stream(fake, baseOptions()); + std::vector steps; + const auto cb = [&steps](const lc::StreamingAsr::Step & s) { steps.push_back(s); }; + + const auto c = chunk(16000); + stream.feed(c.data(), c.size(), cb); + stream.feed(c.data(), c.size(), cb); + + QCOMPARE(q(steps[1].committed), QString("He hoped would be")); + QCOMPARE(q(steps[1].tail), QString(" stew")); + // 已提交 + 尾部 = 完整文本 + QCOMPARE(q(steps[1].committed + steps[1].tail), q(steps[1].full)); + } + + // 流结束时要全部提交，不能把尾部丢掉 + void flushCommitsEverything() + { + FakeAsr fake; + fake.scripted = {"language Englishone two three"}; + + lc::StreamOptions o = baseOptions(); + o.chunkSec = 100.0; // 特意设大：让音频留在 pending 里，由 flush 处理 + lc::StreamingAsr stream(fake, o); + + std::vector steps; + const auto cb = [&steps](const lc::StreamingAsr::Step & s) { steps.push_back(s); }; + + const auto c = chunk(16000); + stream.feed(c.data(), c.size(), cb); + QCOMPARE(steps.size(), size_t(0)); // 还没凑够一块 + + stream.flush(cb); + QCOMPARE(steps.size(), size_t(1)); + QCOMPARE(q(steps[0].committed), QString("one two three")); + QVERIFY(steps[0].tail.empty()); + } + + // 多段：每段各自的 language 头都要剥掉，前面几段的正文不能丢。 + // 曾经用整段拼接后再解析，rfind 只保留最后一个头之后的内容，前面的正文全没了。 + void segmentsAccumulateWithoutLosingEarlierText() + { + FakeAsr fake; + fake.scripted = {"language Chineseni hao shi jie", + " language English hello there world"}; + + lc::StreamOptions o = baseOptions(); + o.chunkSec = 1.0; + + lc::StreamingAsr stream(fake, o); + const auto cb = [](const lc::StreamingAsr::Step &) {}; + + const auto c = chunk(16000); + stream.feed(c.data(), c.size(), cb); // 第 1 段 + stream.flush(cb); // 收束第 1 段 + + stream.feed(c.data(), c.size(), cb); // 第 2 段 + stream.flush(cb); + + const QString all = q(stream.committedText()); + QVERIFY2(all.contains("ni hao"), qPrintable(all)); + QVERIFY2(all.contains("hello there world"), qPrintable(all)); + } + + // 同一次 feed 里跨多块也要正确切分 + void splitsMultipleChunksInOneFeed() + { + FakeAsr fake; + fake.scripted = {"a b c", "d e f", "g h i"}; + + lc::StreamOptions o = baseOptions(); + lc::StreamingAsr stream(fake, o); + int steps = 0; + const auto cb = [&steps](const lc::StreamingAsr::Step &) { ++steps; }; + + const auto c = chunk(16000 * 3); + stream.feed(c.data(), c.size(), cb); + + QCOMPARE(steps, 3); + QCOMPARE(fake.samplesSeen.size(), size_t(3)); + for (size_t n : fake.samplesSeen) QCOMPARE(n, size_t(16000)); + } +}; + +QTEST_MAIN(TstStreamingAsr) +#include "tst_streamingasr.moc" diff --git a/tests/tst_transcript.cpp b/tests/tst_transcript.cpp new file mode 100644 index 0000000..9c88973 --- /dev/null +++ b/tests/tst_transcript.cpp @@ -0,0 +1,48 @@ +#include + +#include "transcript.h" + +// 输出解析：Qwen3-ASR 的原始输出形如 "language English正文" +class TstTranscript : public QObject { + Q_OBJECT + +private slots: + void splitsLanguageAndText() + { + const auto r = lc::parseAsrOutput("language EnglishHe hoped there would be stew."); + QCOMPARE(QString::fromStdString(r.language), QString("English")); + QCOMPARE(QString::fromStdString(r.text), QString("He hoped there would be stew.")); + } + + void handlesChinese() + { + const auto r = lc::parseAsrOutput("language Chinese不排除购房者可以在互联网上下单。"); + QCOMPARE(QString::fromStdString(r.language), QString("Chinese")); + QCOMPARE(QString::fromStdString(r.text), QString("不排除购房者可以在互联网上下单。")); + } + + void withoutMarkerKeepsWholeText() + { + const auto r = lc::parseAsrOutput(" just text "); + QVERIFY(r.language.empty()); + QCOMPARE(QString::fromStdString(r.text), QString("just text")); + } + + void trimsSurroundingWhitespaceOfBody() + { + const auto r = lc::parseAsrOutput("language English hi there "); + QCOMPARE(QString::fromStdString(r.text), QString("hi there")); + } + + void displayNameMapping() + { + QCOMPARE(QString::fromStdString(lc::languageDisplayName("English")), QString("英语")); + QCOMPARE(QString::fromStdString(lc::languageDisplayName("Chinese")), QString("中文")); + // 未收录的语言原样返回 + QCOMPARE(QString::fromStdString(lc::languageDisplayName("Klingon")), QString("Klingon")); + QVERIFY(lc::languageDisplayName("").empty()); + } +}; + +QTEST_MAIN(TstTranscript) +#include "tst_transcript.moc" diff --git a/tests/tst_vad.cpp b/tests/tst_vad.cpp new file mode 100644 index 0000000..72f3fe0 --- /dev/null +++ b/tests/tst_vad.cpp @@ -0,0 +1,83 @@ +#include + +#include +#include + +#include "vad.h" + +namespace { + +constexpr size_t kFrame = 320; // 20ms @16k +constexpr double kPi = 3.14159265358979323846; + +// 生成 amplitude 幅值的 440Hz 正弦，用来模拟"有内容"的音频 +std::vector tone(size_t samples, float amplitude) +{ + std::vector v(samples); + for (size_t i = 0; i < samples; ++i) { + v[i] = amplitude * (float) std::sin(2.0 * kPi * 440.0 * (double) i / 16000.0); + } + return v; +} + +int countSpeech(lc::Vad & vad, const std::vector & audio) +{ + int speech = 0; + for (size_t off = 0; off + kFrame <= audio.size(); off += kFrame) { + if (vad.feed(audio.data() + off, kFrame)) ++speech; + } + return speech; +} + +} // namespace + +// VAD 的回归测试。 +// +// 关键教训：噪声底必须用"最小值跟踪"。曾经用均值跟踪，遇到本来就很轻的音频 +// （峰值 RMS 只有 0.03 量级的录音）会把阈值抬到语音电平之上，把绝大部分话音 +// 判成静音 —— 实时字幕于是每句都被切断。 +class TstVad : public QObject { + Q_OBJECT + +private slots: + void backgroundStaysSilent() + { + lc::Vad vad; + const auto bg = tone(kFrame * 200, 0.001f); // rms ≈ 0.0007 + QCOMPARE(countSpeech(vad, bg), 0); + } + + // 轻语音必须被判成语音：这里语音 rms ≈ 0.0028， + // 旧的"均值跟踪 + 0.004 绝对下限"实现会全部漏掉。 + void quietSpeechIsDetected() + { + lc::Vad vad; + const auto bg = tone(kFrame * 40, 0.001f); // rms ≈ 0.0007 + countSpeech(vad, bg); + + const auto sp = tone(kFrame * 100, 0.004f); // rms ≈ 0.0028 + const int speech = countSpeech(vad, sp); + QVERIFY2(speech >= 95, + qPrintable(QString("轻语音只被识别出 %1/100 帧").arg(speech))); + } + + void loudSpeechIsDetected() + { + lc::Vad vad; + const auto sp = tone(kFrame * 50, 0.2f); + QVERIFY(countSpeech(vad, sp) >= 48); + } + + void floorFollowsQuietInput() + { + lc::Vad vad; + const auto bg = tone(kFrame * 200, 0.001f); + countSpeech(vad, bg); + // 噪声底应该已经下探到背景噪声量级，而不是停在初值 + QVERIFY2(vad.noiseFloor() < 0.002f, + qPrintable(QString("噪声底仍为 %1").arg(vad.noiseFloor()))); + } +}; + +QTEST_MAIN(TstVad) +#include "tst_vad.moc"