refactor: split core/ui/apps targets and add unit tests
Move all business logic into liblivecaption_core (no Qt dependency) and the widgets into liblivecaption_ui, leaving two executables: the caption window and the CLI, which no longer links Qt Widgets. Abstract the ASR backend behind IAsr so the streaming policy and the live pipeline can be tested with a scripted fake backend instead of a real model. 把全部业务逻辑下沉到不依赖 Qt 的 liblivecaption_core，界面独立成 liblivecaption_ui，拆出悬浮窗与命令行两个可执行程序（CLI 不再链接 Qt Widgets）；并用 IAsr 抽象推理后端，使流式策略与实时管线可脱离真模型 用假实现做单元测试。测试还抓到一个真 bug：段恰好结束在 chunk 边界时 只提交了稳定部分，最后几个词被丢掉，已修。
This commit is contained in:
blumia committed
1 parent
4ed4e23282
commit
736ec14d5b
32 files changed
+1502
-655
No files matched your search
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,60 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
#include "transcript.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
// 一次整段转写的结果
|
||||
struct AsrResult {
|
||||
bool ok = false;
|
||||
std::string error;
|
||||
ParsedTranscript transcript; // 已拆出 language / text
|
||||
std::string raw; // 模型原始输出（未 trim）
|
||||
double inferSeconds = 0.0;
|
||||
int tokenCount = 0;
|
||||
};
|
||||
|
||||
// 流式单步的结果
|
||||
struct AsrStreamStep {
|
||||
bool ok = false;
|
||||
std::string error;
|
||||
std::string newText; // 本步新生成的文本（不含前缀，未 trim）
|
||||
int newTokens = 0;
|
||||
int audioTokens = 0;
|
||||
double encodeSeconds = 0.0;
|
||||
double inferSeconds = 0.0;
|
||||
};
|
||||
|
||||
// 把推理后端抽象出来，好处是流式策略（StreamingAsr）和实时管线（LivePipeline）
|
||||
// 可以用一个假的实现来测，不必加载真模型。
|
||||
class IAsr {
|
||||
public:
|
||||
virtual ~IAsr() = default;
|
||||
|
||||
// 期望的输入采样率（Qwen3-ASR 为 16000）
|
||||
virtual int sampleRate() const = 0;
|
||||
|
||||
// 整段转写：把 prefix 当作"已经生成的 assistant 文本"，从那里继续生成
|
||||
virtual AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix) = 0;
|
||||
|
||||
// 流式：只编码新音频，已解码的音频留在 KV cache 里复用
|
||||
virtual AsrStreamStep streamAppend(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix, int maxTokens) = 0;
|
||||
virtual void streamReset() = 0;
|
||||
|
||||
// 用词表把 text 编码成 token、丢掉最后 dropLast 个、再还原成文本。
|
||||
// 对应官方的 unfixed_token_num 回滚。
|
||||
virtual std::string dropLastTokens(const std::string & text, int dropLast) const = 0;
|
||||
|
||||
// 便利：不带前缀的整段转写
|
||||
AsrResult transcribe(const float * pcm, size_t nSamples)
|
||||
{
|
||||
return transcribeWithPrefix(pcm, nSamples, std::string());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,147 @@
|
||||
#include "livepipeline.h"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
namespace lc {
|
||||
|
||||
namespace {
|
||||
constexpr int kFrameSamples = 320; // 20ms @16k
|
||||
} // namespace
|
||||
|
||||
LivePipeline::LivePipeline(IAsr & asr, LivePipelineOptions opt)
|
||||
: asr_(asr), opt_(std::move(opt)), stream_(asr, opt_.stream) {}
|
||||
|
||||
LivePipeline::~LivePipeline()
|
||||
{
|
||||
stop();
|
||||
}
|
||||
|
||||
std::string LivePipeline::deviceName() const
|
||||
{
|
||||
return capture_.deviceName();
|
||||
}
|
||||
|
||||
bool LivePipeline::start(std::string * error)
|
||||
{
|
||||
const bool ok = capture_.start(
|
||||
asr_.sampleRate(),
|
||||
[this](const float * s, size_t n) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mutex_);
|
||||
pending_.insert(pending_.end(), s, s + n);
|
||||
}
|
||||
cv_.notify_one();
|
||||
},
|
||||
error);
|
||||
if (!ok) return false;
|
||||
|
||||
worker_ = std::thread([this] { workerLoop(); });
|
||||
return true;
|
||||
}
|
||||
|
||||
void LivePipeline::stop()
|
||||
{
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mutex_);
|
||||
quit_ = true;
|
||||
}
|
||||
cv_.notify_all();
|
||||
if (worker_.joinable()) worker_.join();
|
||||
capture_.stop();
|
||||
}
|
||||
|
||||
void LivePipeline::workerLoop()
|
||||
{
|
||||
for (;;) {
|
||||
std::vector<float> chunk;
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cv_.wait_for(lk, std::chrono::milliseconds(50),
|
||||
[this] { return quit_ || !pending_.empty(); });
|
||||
if (quit_ && pending_.empty()) break;
|
||||
chunk.swap(pending_);
|
||||
}
|
||||
if (!chunk.empty()) process(chunk.data(), chunk.size());
|
||||
}
|
||||
|
||||
if (inSegment_) finishSegment();
|
||||
}
|
||||
|
||||
void LivePipeline::emitStep(const StreamingAsr::Step & s)
|
||||
{
|
||||
if (!s.ok || !onCaption_) return;
|
||||
onCaption_(s.committed, s.tail, s.language);
|
||||
}
|
||||
|
||||
void LivePipeline::process(const float * pcm, size_t n)
|
||||
{
|
||||
buffer_.insert(buffer_.end(), pcm, pcm + n);
|
||||
|
||||
const size_t prerollSamples = (size_t) (opt_.prerollSec * asr_.sampleRate());
|
||||
const float frameSec = (float) kFrameSamples / asr_.sampleRate();
|
||||
|
||||
std::vector<float> segAudio; // 本段要送进引擎的音频
|
||||
bool flushNow = false;
|
||||
|
||||
size_t off = 0;
|
||||
for (; off + kFrameSamples <= buffer_.size(); off += kFrameSamples) {
|
||||
const float * frame = buffer_.data() + off;
|
||||
const bool speech = vad_.feed(frame, kFrameSamples);
|
||||
|
||||
if (!inSegment_) {
|
||||
// 静音期间什么都不送给模型：否则模型会对静音产生幻觉
|
||||
if (speech) {
|
||||
inSegment_ = true;
|
||||
silenceAcc_ = 0.0f;
|
||||
segAudio.insert(segAudio.end(), preroll_.begin(), preroll_.end());
|
||||
preroll_.clear();
|
||||
segAudio.insert(segAudio.end(), frame, frame + kFrameSamples);
|
||||
} else {
|
||||
preroll_.insert(preroll_.end(), frame, frame + kFrameSamples);
|
||||
if (preroll_.size() > prerollSamples) {
|
||||
preroll_.erase(preroll_.begin(),
|
||||
preroll_.begin() + (long) (preroll_.size() - prerollSamples));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
segAudio.insert(segAudio.end(), frame, frame + kFrameSamples);
|
||||
if (speech) silenceAcc_ = 0.0f;
|
||||
else silenceAcc_ += frameSec;
|
||||
|
||||
if (silenceAcc_ >= (float) opt_.silenceTimeoutSec) {
|
||||
flushNow = true;
|
||||
off += kFrameSamples; // 这一帧已经并入 segAudio
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!segAudio.empty()) {
|
||||
const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); };
|
||||
stream_.feed(segAudio.data(), segAudio.size(), cb);
|
||||
}
|
||||
|
||||
if (flushNow) {
|
||||
const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); };
|
||||
stream_.flush(cb);
|
||||
stream_.reset();
|
||||
inSegment_ = false;
|
||||
silenceAcc_ = 0.0f;
|
||||
preroll_.clear();
|
||||
}
|
||||
|
||||
if (off > 0) buffer_.erase(buffer_.begin(), buffer_.begin() + (long) off);
|
||||
}
|
||||
|
||||
void LivePipeline::finishSegment()
|
||||
{
|
||||
if (!inSegment_) return;
|
||||
const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); };
|
||||
stream_.flush(cb);
|
||||
stream_.reset();
|
||||
inSegment_ = false;
|
||||
silenceAcc_ = 0.0f;
|
||||
preroll_.clear();
|
||||
}
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,87 @@
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "audio.h"
|
||||
#include "iasr.h"
|
||||
#include "streamingasr.h"
|
||||
#include "vad.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
struct LivePipelineOptions {
|
||||
StreamOptions stream;
|
||||
double silenceTimeoutSec = 0.8; // 静音多久算一句结束
|
||||
double prerollSec = 0.25; // 语音开始前保留的一段，避免吃掉词头
|
||||
};
|
||||
|
||||
// 实时管线：采集 → VAD → 分段 → 流式转写。
|
||||
//
|
||||
// 刻意不带任何 Qt 依赖（回调用 std::function），好处是可以脱离 UI 单独测试：
|
||||
// 测试直接调 process() 喂音频，不必碰 PulseAudio，也不用跑事件循环。
|
||||
class LivePipeline {
|
||||
public:
|
||||
using CaptionCallback = std::function<void(const std::string & committed,
|
||||
const std::string & tail,
|
||||
const std::string & language)>;
|
||||
using ErrorCallback = std::function<void(const std::string & message)>;
|
||||
|
||||
LivePipeline(IAsr & asr, LivePipelineOptions opt = {});
|
||||
~LivePipeline();
|
||||
|
||||
LivePipeline(const LivePipeline &) = delete;
|
||||
LivePipeline & operator=(const LivePipeline &) = delete;
|
||||
|
||||
void setCaptionCallback(CaptionCallback cb) { onCaption_ = std::move(cb); }
|
||||
void setErrorCallback(ErrorCallback cb) { onError_ = std::move(cb); }
|
||||
|
||||
// 启动真实采集（默认输出设备的 monitor）
|
||||
bool start(std::string * error = nullptr);
|
||||
void stop();
|
||||
std::string deviceName() const;
|
||||
|
||||
// 同步处理一段 16k 单声道音频。
|
||||
// 真实运行时由采集线程调用；测试可以直接调它。
|
||||
void process(const float * pcm, size_t n);
|
||||
|
||||
// 结束当前这一段并提交（流结束或手动收尾时用）
|
||||
void finishSegment();
|
||||
|
||||
// 注意：StreamingAsr::committedText() 是按值返回的，这里也必须按值返回
|
||||
std::string committedText() const { return stream_.committedText(); }
|
||||
const std::string & language() const { return stream_.language(); }
|
||||
int chunkCount() const { return stream_.chunkCount(); }
|
||||
bool inSegment() const { return inSegment_; }
|
||||
|
||||
private:
|
||||
void workerLoop();
|
||||
void emitStep(const StreamingAsr::Step & s);
|
||||
|
||||
IAsr & asr_;
|
||||
LivePipelineOptions opt_;
|
||||
Vad vad_;
|
||||
StreamingAsr stream_;
|
||||
|
||||
CaptionCallback onCaption_;
|
||||
ErrorCallback onError_;
|
||||
|
||||
bool inSegment_ = false;
|
||||
float silenceAcc_ = 0.0f;
|
||||
std::vector<float> preroll_; // 语音开始前保留的一小段
|
||||
std::vector<float> buffer_; // 跨批次的残留样本（不接上会持续丢样本）
|
||||
|
||||
MonitorCapture capture_;
|
||||
std::thread worker_;
|
||||
std::mutex mutex_;
|
||||
std::condition_variable cv_;
|
||||
std::vector<float> pending_;
|
||||
bool quit_ = false;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -186,11 +186,6 @@ bool Qwen3Asr::load(const AsrOptions & opt, std::string * error)
|
||||
return true;
|
||||
}
|
||||
|
||||
Qwen3Asr::Result Qwen3Asr::transcribe(const float * pcm, size_t nSamples)
|
||||
{
|
||||
return transcribeWithPrefix(pcm, nSamples, std::string());
|
||||
}
|
||||
|
||||
std::string Qwen3Asr::dropLastTokens(const std::string & text, int dropLast) const
|
||||
{
|
||||
if (!d->vocab || dropLast <= 0 || text.empty()) return text;
|
||||
@@ -220,10 +215,10 @@ std::string Qwen3Asr::dropLastTokens(const std::string & text, int dropLast) con
|
||||
return out;
|
||||
}
|
||||
|
||||
Qwen3Asr::Result Qwen3Asr::transcribeWithPrefix(const float * pcm, size_t nSamples,
|
||||
AsrResult Qwen3Asr::transcribeWithPrefix(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix)
|
||||
{
|
||||
Result res;
|
||||
AsrResult res;
|
||||
|
||||
if (!d->model || !d->ctx || !d->mctx) {
|
||||
res.error = "模型未加载";
|
||||
@@ -377,10 +372,10 @@ bool Qwen3Asr::prepareStream(std::string * error)
|
||||
// 按这个粒度切，编码结果才和「整段音频一次编码」一致；切小了会明显掉准确率。
|
||||
static constexpr double kSliceSeconds = 8.0;
|
||||
|
||||
Qwen3Asr::StreamStep Qwen3Asr::streamAppend(const float * pcm, size_t nSamples,
|
||||
AsrStreamStep Qwen3Asr::streamAppend(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix, int maxTokens)
|
||||
{
|
||||
StreamStep res;
|
||||
AsrStreamStep res;
|
||||
if (!d->model || !d->ctx || !d->mctx) { res.error = "模型未加载"; return res; }
|
||||
|
||||
std::string err;
|
||||
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "iasr.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
struct AsrOptions {
|
||||
std::string modelPath; // Qwen3-ASR 主模型 GGUF（架构须为 qwen3vl）
|
||||
std::string mmprojPath; // 音频编码器 mmproj GGUF（clip + qwen3a）
|
||||
int threads = 8;
|
||||
int nCtx = 4096;
|
||||
int nBatch = 4096;
|
||||
int nUbatch = 1024;
|
||||
int maxTokens = 256; // 单段最多生成多少 token
|
||||
bool warmup = false; // mtmd 初始化后是否跑一次预热编码
|
||||
bool verbose = false;
|
||||
};
|
||||
|
||||
// 进程内 Qwen3-ASR 推理（libllama + libmtmd）。
|
||||
// 模型常驻，可反复调用；每次整段转写会清空 KV cache。
|
||||
class Qwen3Asr : public IAsr {
|
||||
public:
|
||||
Qwen3Asr();
|
||||
~Qwen3Asr() override;
|
||||
|
||||
Qwen3Asr(const Qwen3Asr &) = delete;
|
||||
Qwen3Asr & operator=(const Qwen3Asr &) = delete;
|
||||
|
||||
bool load(const AsrOptions & opt, std::string * error);
|
||||
|
||||
// ---- IAsr ----
|
||||
int sampleRate() const override;
|
||||
|
||||
AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix) override;
|
||||
|
||||
AsrStreamStep streamAppend(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix, int maxTokens) override;
|
||||
void streamReset() override;
|
||||
|
||||
std::string dropLastTokens(const std::string & text, int dropLast) const override;
|
||||
|
||||
private:
|
||||
bool prepareStream(std::string * error);
|
||||
|
||||
struct Impl;
|
||||
Impl * d = nullptr;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -1,11 +1,10 @@
|
||||
#include "streamingasr.h"
|
||||
|
||||
#include "qwen3asr.h"
|
||||
#include "transcript.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
StreamingAsr::StreamingAsr(Qwen3Asr & asr, StreamOptions opt)
|
||||
StreamingAsr::StreamingAsr(IAsr & asr, StreamOptions opt)
|
||||
: asr_(asr), opt_(opt)
|
||||
{
|
||||
asr_.streamReset();
|
||||
@@ -18,6 +17,7 @@ void StreamingAsr::reset()
|
||||
pending_.clear();
|
||||
totalSeconds_ = 0.0;
|
||||
committedRaw_.clear();
|
||||
lastRawFull_.clear();
|
||||
language_.clear();
|
||||
chunkIndex_ = 0;
|
||||
}
|
||||
@@ -58,11 +58,15 @@ void StreamingAsr::flush(const std::function<void(const Step &)> & onStep)
|
||||
return;
|
||||
}
|
||||
|
||||
// 没有待处理音频：直接把当前已稳定文本提交为最终
|
||||
if (committedRaw_.empty()) return;
|
||||
// 没有待处理音频：直接把这一段提交掉。
|
||||
// 注意要用 lastRawFull_（含不稳定尾部）而不是 committedRaw_：
|
||||
// 段恰好结束在 chunk 边界时，否则最后几个词会被丢掉。
|
||||
const std::string full = lastRawFull_.empty() ? committedRaw_ : lastRawFull_;
|
||||
if (full.empty()) return;
|
||||
|
||||
historyText_ += parseAsrOutput(committedRaw_).text;
|
||||
historyText_ += parseAsrOutput(full).text;
|
||||
committedRaw_.clear();
|
||||
lastRawFull_.clear();
|
||||
|
||||
Step s;
|
||||
s.chunkIndex = chunkIndex_;
|
||||
@@ -92,7 +96,7 @@ void StreamingAsr::runStep(const std::vector<float> & audio, bool finalStep,
|
||||
std::string newText;
|
||||
if (opt_.incremental) {
|
||||
// 只把这一块交给增量引擎；音频总长不影响单步成本
|
||||
Qwen3Asr::StreamStep r =
|
||||
AsrStreamStep r =
|
||||
asr_.streamAppend(audio.data(), audio.size(), prefix, opt_.maxTokensPerStep);
|
||||
s.ok = r.ok;
|
||||
s.error = r.error;
|
||||
@@ -112,7 +116,7 @@ void StreamingAsr::runStep(const std::vector<float> & audio, bool finalStep,
|
||||
buffer_.begin() + (long) (buffer_.size() - maxSamples));
|
||||
}
|
||||
}
|
||||
Qwen3Asr::Result r =
|
||||
AsrResult r =
|
||||
asr_.transcribeWithPrefix(buffer_.data(), buffer_.size(), prefix);
|
||||
s.ok = r.ok;
|
||||
s.error = r.error;
|
||||
@@ -124,6 +128,7 @@ void StreamingAsr::runStep(const std::vector<float> & audio, bool finalStep,
|
||||
if (s.ok) {
|
||||
// 拼接要用未经 trim 的文本：流式续写时开头那个空格是有意义的
|
||||
s.rawFull = prefix + newText;
|
||||
lastRawFull_ = s.rawFull;
|
||||
|
||||
const ParsedTranscript pSeg = parseAsrOutput(s.rawFull);
|
||||
if (!pSeg.language.empty()) language_ = pSeg.language;
|
||||
@@ -5,9 +5,9 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace lc {
|
||||
#include "iasr.h"
|
||||
|
||||
class Qwen3Asr;
|
||||
namespace lc {
|
||||
|
||||
// 官方流式解码的参数（见 docs/phase2-streaming.md）
|
||||
struct StreamOptions {
|
||||
@@ -48,7 +48,7 @@ public:
|
||||
std::string error;
|
||||
};
|
||||
|
||||
StreamingAsr(Qwen3Asr & asr, StreamOptions opt = {});
|
||||
StreamingAsr(IAsr & asr, StreamOptions opt = {});
|
||||
|
||||
// 喂入 16k 单声道音频；每凑够 chunkSec 就跑一步并回调
|
||||
void feed(const float * pcm, size_t n, const std::function<void(const Step &)> & onStep);
|
||||
@@ -70,12 +70,13 @@ private:
|
||||
void runStep(const std::vector<float> & audio, bool finalStep,
|
||||
const std::function<void(const Step &)> & onStep);
|
||||
|
||||
Qwen3Asr & asr_;
|
||||
IAsr & asr_;
|
||||
StreamOptions opt_;
|
||||
std::vector<float> buffer_; // window 模式：当前窗口音频
|
||||
std::vector<float> pending_; // 尚未凑够一块的音频
|
||||
double totalSeconds_ = 0.0;
|
||||
std::string committedRaw_; // 当前段的原始已提交文本（含 language 头，用于回喂）
|
||||
std::string lastRawFull_; // 上一次完整的原始文本（含不稳定尾部）
|
||||
std::string historyText_; // 此前各段已提交的正文（已剥离 language 头）
|
||||
std::string language_;
|
||||
int chunkIndex_ = 0;
|
||||
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,43 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
|
||||
namespace lc {
|
||||
|
||||
// 能量 VAD。
|
||||
//
|
||||
// 噪声底必须用"快降慢升"的**最小值跟踪**，不能用均值：
|
||||
// 否则遇到本身就很轻的音频（例如峰值 RMS 只有 0.03 的录音），
|
||||
// 噪声底会一路抬升、阈值超过语音电平，把绝大部分话音都判成静音。
|
||||
class Vad {
|
||||
public:
|
||||
explicit Vad(float floorInit = 0.01f, float factor = 3.0f, float minThreshold = 0.0015f)
|
||||
: floor_(floorInit), factor_(factor), minThreshold_(minThreshold) {}
|
||||
|
||||
// 返回该帧是否为语音
|
||||
bool feed(const float * frame, size_t n)
|
||||
{
|
||||
double acc = 0.0;
|
||||
for (size_t i = 0; i < n; ++i) acc += (double) frame[i] * frame[i];
|
||||
const float rms = std::sqrt(acc / (double) n);
|
||||
|
||||
if (rms < floor_) floor_ = rms; // 快速下探
|
||||
else floor_ = floor_ * 0.9999f + rms * 0.0001f; // 极慢回升
|
||||
|
||||
lastRms_ = rms;
|
||||
return rms > std::max(floor_ * factor_, minThreshold_);
|
||||
}
|
||||
|
||||
float noiseFloor() const { return floor_; }
|
||||
float lastRms() const { return lastRms_; }
|
||||
|
||||
private:
|
||||
float floor_;
|
||||
float factor_;
|
||||
float minThreshold_;
|
||||
float lastRms_ = 0.0f;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -1,178 +0,0 @@
|
||||
#include "livetranscriber.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
|
||||
#include "qwen3asr.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int kFrameSamples = 320; // 20ms @16k
|
||||
constexpr float kSilenceTimeout = 0.8f; // 静音多久算一句结束
|
||||
constexpr float kPrerollSec = 0.25f; // 语音开始前保留的一段，避免吃掉词头
|
||||
|
||||
// 能量 VAD。噪声底用"快降慢升"的最小值跟踪（不能用均值，否则轻音频会被全判成静音）。
|
||||
class Vad {
|
||||
public:
|
||||
bool feed(const float * frame, size_t n)
|
||||
{
|
||||
double acc = 0.0;
|
||||
for (size_t i = 0; i < n; ++i) acc += (double) frame[i] * frame[i];
|
||||
const float rms = std::sqrt(acc / (double) n);
|
||||
|
||||
if (rms < floor_) floor_ = rms;
|
||||
else floor_ = floor_ * 0.9999f + rms * 0.0001f;
|
||||
|
||||
return rms > std::max(floor_ * 3.0f, 0.0015f);
|
||||
}
|
||||
|
||||
private:
|
||||
float floor_ = 0.01f;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
LiveTranscriber::LiveTranscriber(Qwen3Asr & asr, QObject * parent)
|
||||
: QObject(parent), asr_(asr) {}
|
||||
|
||||
LiveTranscriber::~LiveTranscriber()
|
||||
{
|
||||
stop();
|
||||
}
|
||||
|
||||
QString LiveTranscriber::deviceName() const
|
||||
{
|
||||
return QString::fromStdString(capture_.deviceName());
|
||||
}
|
||||
|
||||
bool LiveTranscriber::start(const StreamOptions & opts, QString * error)
|
||||
{
|
||||
opts_ = opts;
|
||||
|
||||
std::string err;
|
||||
const bool ok = capture_.start(asr_.sampleRate(),
|
||||
[this](const float * s, size_t n) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mutex_);
|
||||
pending_.insert(pending_.end(), s, s + n);
|
||||
}
|
||||
cv_.notify_one();
|
||||
},
|
||||
&err);
|
||||
if (!ok) {
|
||||
if (error) *error = QString::fromStdString(err);
|
||||
return false;
|
||||
}
|
||||
|
||||
worker_ = std::thread([this] { workerLoop(); });
|
||||
return true;
|
||||
}
|
||||
|
||||
void LiveTranscriber::stop()
|
||||
{
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mutex_);
|
||||
quit_ = true;
|
||||
}
|
||||
cv_.notify_all();
|
||||
if (worker_.joinable()) worker_.join();
|
||||
|
||||
capture_.stop();
|
||||
}
|
||||
|
||||
void LiveTranscriber::workerLoop()
|
||||
{
|
||||
Vad vad;
|
||||
StreamingAsr stream(asr_, opts_);
|
||||
|
||||
float silenceAcc = 0.0f;
|
||||
bool inSegment = false;
|
||||
bool flushNow = false;
|
||||
|
||||
std::vector<float> buffer; // 跨批次的残留样本
|
||||
std::vector<float> preroll; // 语音开始前保留的一小段，避免吃掉词头
|
||||
std::vector<float> segAudio; // 本段待送进引擎的音频
|
||||
|
||||
const size_t prerollSamples = (size_t) (kPrerollSec * asr_.sampleRate());
|
||||
|
||||
const auto emitStep = [this](const StreamingAsr::Step & s) {
|
||||
if (!s.ok) return;
|
||||
Q_EMIT caption(QString::fromStdString(s.committed),
|
||||
QString::fromStdString(s.tail),
|
||||
QString::fromStdString(s.language));
|
||||
};
|
||||
|
||||
const auto feedSegment = [&] {
|
||||
if (!segAudio.empty()) stream.feed(segAudio.data(), segAudio.size(), emitStep);
|
||||
segAudio.clear();
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
std::vector<float> chunk;
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cv_.wait_for(lk, std::chrono::milliseconds(50),
|
||||
[this] { return quit_ || !pending_.empty(); });
|
||||
if (quit_ && pending_.empty()) break;
|
||||
chunk.swap(pending_);
|
||||
}
|
||||
|
||||
// 不足一帧的残留要接上，否则会持续丢样本
|
||||
buffer.insert(buffer.end(), chunk.begin(), chunk.end());
|
||||
|
||||
size_t off = 0;
|
||||
for (; off + kFrameSamples <= buffer.size(); off += kFrameSamples) {
|
||||
const float * frame = buffer.data() + off;
|
||||
const bool speech = vad.feed(frame, kFrameSamples);
|
||||
|
||||
if (!inSegment) {
|
||||
// 静音期间什么都不送给模型：否则模型会对静音产生幻觉
|
||||
if (speech) {
|
||||
inSegment = true;
|
||||
silenceAcc = 0.0f;
|
||||
segAudio.insert(segAudio.end(), preroll.begin(), preroll.end());
|
||||
preroll.clear();
|
||||
segAudio.insert(segAudio.end(), frame, frame + kFrameSamples);
|
||||
} else {
|
||||
preroll.insert(preroll.end(), frame, frame + kFrameSamples);
|
||||
if (preroll.size() > prerollSamples) {
|
||||
preroll.erase(preroll.begin(),
|
||||
preroll.begin() + (long) (preroll.size() - prerollSamples));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
segAudio.insert(segAudio.end(), frame, frame + kFrameSamples);
|
||||
if (speech) silenceAcc = 0.0f;
|
||||
else silenceAcc += (float) kFrameSamples / asr_.sampleRate();
|
||||
if (silenceAcc >= kSilenceTimeout) {
|
||||
flushNow = true;
|
||||
off += kFrameSamples; // 这一帧已经并入 segAudio
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!segAudio.empty()) feedSegment();
|
||||
|
||||
// 静音够久 → 收束这一段（提交文本并重置上下文）
|
||||
if (flushNow) {
|
||||
stream.flush(emitStep);
|
||||
stream.reset();
|
||||
inSegment = false;
|
||||
silenceAcc = 0.0f;
|
||||
flushNow = false;
|
||||
preroll.clear();
|
||||
}
|
||||
|
||||
if (off > 0) buffer.erase(buffer.begin(), buffer.begin() + (long) off);
|
||||
}
|
||||
|
||||
if (inSegment) {
|
||||
feedSegment();
|
||||
stream.flush(emitStep);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace lc
|
||||
@@ -1,52 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <QObject>
|
||||
#include <QString>
|
||||
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "audio.h"
|
||||
#include "streamingasr.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
class Qwen3Asr;
|
||||
|
||||
// 实时字幕：采集默认输出 → 流式转写 → 用信号把文本推给 UI。
|
||||
// 分段由静音检测驱动：句子之间的自然停顿就收束一段、清空上下文，
|
||||
// 这样解码器的上下文不会无限增长（见 docs/phase2-streaming.md）。
|
||||
class LiveTranscriber : public QObject {
|
||||
Q_OBJECT
|
||||
|
||||
public:
|
||||
explicit LiveTranscriber(Qwen3Asr & asr, QObject * parent = nullptr);
|
||||
~LiveTranscriber() override;
|
||||
|
||||
bool start(const StreamOptions & opts, QString * error = nullptr);
|
||||
void stop();
|
||||
|
||||
QString deviceName() const;
|
||||
|
||||
Q_SIGNALS:
|
||||
// committed = 已稳定文本；tail = 不稳定尾部（UI 用暗色显示）
|
||||
void caption(const QString & committed, const QString & tail, const QString & language);
|
||||
void failed(const QString & message);
|
||||
|
||||
private:
|
||||
void workerLoop();
|
||||
|
||||
Qwen3Asr & asr_;
|
||||
MonitorCapture capture_;
|
||||
StreamOptions opts_;
|
||||
std::thread worker_;
|
||||
std::mutex mutex_;
|
||||
std::condition_variable cv_;
|
||||
std::vector<float> pending_;
|
||||
bool quit_ = false;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
-288
@@ -1,288 +0,0 @@
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <QApplication>
|
||||
#include <QCommandLineParser>
|
||||
#include <QCoreApplication>
|
||||
#include <QGuiApplication>
|
||||
#include <QScopedPointer>
|
||||
#include <QTimer>
|
||||
|
||||
#include "audio.h"
|
||||
#include "captionwindow.h"
|
||||
#include "livetranscriber.h"
|
||||
#include "qwen3asr.h"
|
||||
#include "streamingasr.h"
|
||||
#include "transcript.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 文件模式
|
||||
// ---------------------------------------------------------------------------
|
||||
int runFile(lc::Qwen3Asr & asr, const QString & path)
|
||||
{
|
||||
std::vector<float> pcm;
|
||||
std::string err;
|
||||
if (!lc::readWavMono(path.toStdString(), asr.sampleRate(), pcm, &err)) {
|
||||
std::fprintf(stderr, "读取音频失败: %s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
const double dur = (double) pcm.size() / asr.sampleRate();
|
||||
std::fprintf(stderr, "音频: %s (%.2fs)\n", qUtf8Printable(path), dur);
|
||||
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
lc::Qwen3Asr::Result r = asr.transcribe(pcm.data(), pcm.size());
|
||||
const double wall = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
|
||||
|
||||
if (!r.ok) {
|
||||
std::fprintf(stderr, "转写失败: %s\n", r.error.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::printf("语言: %s (%s)\n", r.transcript.language.c_str(),
|
||||
lc::languageDisplayName(r.transcript.language).c_str());
|
||||
std::printf("文本: %s\n", r.transcript.text.c_str());
|
||||
std::fprintf(stderr, "推理: %.2fs / 音频 %.2fs (rtf %.3f) 生成 %d tokens, 墙钟 %.2fs\n",
|
||||
r.inferSeconds, dur, dur > 0 ? r.inferSeconds / dur : 0, r.tokenCount, wall);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 流式文件模式（把文件当音频流按块喂进去）
|
||||
// ---------------------------------------------------------------------------
|
||||
int runStreamFile(lc::Qwen3Asr & asr, const QString & path, const lc::StreamOptions & sop)
|
||||
{
|
||||
std::vector<float> pcm;
|
||||
std::string err;
|
||||
if (!lc::readWavMono(path.toStdString(), asr.sampleRate(), pcm, &err)) {
|
||||
std::fprintf(stderr, "读取音频失败: %s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
const double dur = (double) pcm.size() / asr.sampleRate();
|
||||
std::fprintf(stderr, "流式转写: %s (%.2fs) chunk=%.1fs segment=%.1fs mode=%s\n",
|
||||
qUtf8Printable(path), dur, sop.chunkSec, sop.maxUtteranceSec,
|
||||
sop.incremental ? "incremental" : "window");
|
||||
|
||||
lc::StreamingAsr stream(asr, sop);
|
||||
const size_t chunkSamples = (size_t) (sop.chunkSec * asr.sampleRate());
|
||||
|
||||
const auto onStep = [](const lc::StreamingAsr::Step & s) {
|
||||
std::printf("\n[chunk %2d] 窗口 %5.1fs / 编码 %5.2fs / 本步 %5.2fs%s\n",
|
||||
s.chunkIndex, s.windowSeconds, s.encodeSeconds, s.inferSeconds,
|
||||
s.ok ? "" : " <失败>");
|
||||
if (!s.ok) std::printf(" error: %s\n", s.error.c_str());
|
||||
std::printf(" 已提交: %s\n", s.committed.c_str());
|
||||
std::printf(" 含尾部: %s\n", s.full.c_str());
|
||||
std::fflush(stdout);
|
||||
};
|
||||
|
||||
for (size_t off = 0; off < pcm.size(); off += chunkSamples) {
|
||||
const size_t n = std::min(chunkSamples, pcm.size() - off);
|
||||
stream.feed(pcm.data() + off, n, onStep);
|
||||
}
|
||||
stream.flush(onStep);
|
||||
|
||||
std::printf("\n==== 最终提交文本 (%s) ====\n%s\n",
|
||||
lc::languageDisplayName(stream.language()).c_str(),
|
||||
stream.committedText().c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 命令行实时模式（无窗口，便于在无显示环境下调试）
|
||||
// ---------------------------------------------------------------------------
|
||||
int runCliLive(lc::Qwen3Asr & asr, const lc::StreamOptions & sop)
|
||||
{
|
||||
lc::LiveTranscriber live(asr);
|
||||
|
||||
QObject::connect(&live, &lc::LiveTranscriber::caption,
|
||||
[](const QString & committed, const QString & tail, const QString & lang) {
|
||||
// 用 \r 原地刷新：稳定部分正常显示，尾部加括号表示未定
|
||||
std::printf("\r\033[K[%s] %s\033[2m%s\033[0m",
|
||||
lc::languageDisplayName(lang.toStdString()).c_str(),
|
||||
qUtf8Printable(committed), qUtf8Printable(tail));
|
||||
std::fflush(stdout);
|
||||
});
|
||||
QObject::connect(&live, &lc::LiveTranscriber::failed, [](const QString & m) {
|
||||
std::fprintf(stderr, "\n采集失败: %s\n", qUtf8Printable(m));
|
||||
});
|
||||
|
||||
QString err;
|
||||
if (!live.start(sop, &err)) {
|
||||
std::fprintf(stderr, "启动采集失败: %s\n", qUtf8Printable(err));
|
||||
return 1;
|
||||
}
|
||||
std::fprintf(stderr, "正在采集: %s (Ctrl+C 退出)\n", qUtf8Printable(live.deviceName()));
|
||||
return QCoreApplication::exec();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 图形界面模式：悬浮字幕窗
|
||||
// ---------------------------------------------------------------------------
|
||||
int runGui(lc::Qwen3Asr & asr, const lc::StreamOptions & sop,
|
||||
const QString & screenshotPath, int screenshotDelaySec, int opacityPercent)
|
||||
{
|
||||
lc::CaptionWindow window;
|
||||
window.setPanelOpacity(opacityPercent);
|
||||
lc::LiveTranscriber live(asr);
|
||||
|
||||
QObject::connect(&live, &lc::LiveTranscriber::caption, &window,
|
||||
[&window](const QString & committed, const QString & tail,
|
||||
const QString & lang) {
|
||||
window.setLanguage(
|
||||
QString::fromStdString(lc::languageDisplayName(lang.toStdString())));
|
||||
window.setCaption(committed, tail);
|
||||
});
|
||||
QObject::connect(&live, &lc::LiveTranscriber::failed, &window, [](const QString & m) {
|
||||
qWarning("采集失败: %s", qUtf8Printable(m));
|
||||
});
|
||||
QObject::connect(&window, &lc::CaptionWindow::closed, qApp, &QCoreApplication::quit);
|
||||
QObject::connect(&window, &lc::CaptionWindow::languageClicked, &window, [&window] {
|
||||
qInfo() << "语言标签被点击（设置界面将在 Phase 4 接入）:" << window.isCollapsed();
|
||||
});
|
||||
|
||||
QString err;
|
||||
if (!live.start(sop, &err)) {
|
||||
qCritical("启动采集失败: %s", qUtf8Printable(err));
|
||||
return 1;
|
||||
}
|
||||
qInfo("正在采集: %s", qUtf8Printable(live.deviceName()));
|
||||
|
||||
window.show();
|
||||
|
||||
if (!screenshotPath.isEmpty()) {
|
||||
// 自截图：用于验证渲染（不依赖外部截图工具）
|
||||
QTimer::singleShot(screenshotDelaySec * 1000, &window, [&window, screenshotPath] {
|
||||
const bool ok1 = window.grab().save(screenshotPath);
|
||||
|
||||
// 再截一张展开态的
|
||||
window.setCollapsed(false);
|
||||
const QString expanded = screenshotPath;
|
||||
QString expandedPath = expanded;
|
||||
expandedPath.replace(QStringLiteral(".png"), QStringLiteral("-expanded.png"));
|
||||
const bool ok2 = window.grab().save(expandedPath);
|
||||
|
||||
std::fprintf(stderr, "截图: 折叠态 %s / 展开态 %s\n",
|
||||
ok1 ? "成功" : "失败", ok2 ? "成功" : "失败");
|
||||
QCoreApplication::quit();
|
||||
});
|
||||
}
|
||||
|
||||
return QCoreApplication::exec();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
// 文件类模式不需要图形环境；实时模式（默认）需要 QApplication
|
||||
bool wantsWidgets = true;
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
const QString a = QString::fromLocal8Bit(argv[i]);
|
||||
if (a == QLatin1String("--file") || a == QLatin1String("--stream-file")) {
|
||||
wantsWidgets = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<QCoreApplication> app;
|
||||
if (wantsWidgets) {
|
||||
// 说明意图：DPI 取整策略必须在 QGuiApplication 之前定。
|
||||
// （Deepin 的会话环境已经设了 QT_SCALE_FACTOR_ROUNDING_POLICY，
|
||||
// 所以本机仍会看到 Qt 的那条提醒，那不是本程序的问题。）
|
||||
QGuiApplication::setHighDpiScaleFactorRoundingPolicy(
|
||||
Qt::HighDpiScaleFactorRoundingPolicy::PassThrough);
|
||||
app.reset(new QApplication(argc, argv));
|
||||
} else {
|
||||
app.reset(new QCoreApplication(argc, argv));
|
||||
}
|
||||
QCoreApplication::setApplicationName(QStringLiteral("live-caption"));
|
||||
|
||||
QCommandLineParser parser;
|
||||
parser.setApplicationDescription(QStringLiteral("实时字幕（Qwen3-ASR + llama.cpp）"));
|
||||
parser.addHelpOption();
|
||||
|
||||
QCommandLineOption optModel("model", "主模型 GGUF 路径（架构须为 qwen3vl）", "path");
|
||||
QCommandLineOption optMmproj("mmproj", "mmproj GGUF 路径（clip + qwen3a）", "path");
|
||||
QCommandLineOption optFile("file", "转写一个 wav 文件后退出", "path");
|
||||
QCommandLineOption optStreamFile("stream-file", "以流式模式转写 wav 文件后退出", "path");
|
||||
QCommandLineOption optCliLive("cli-live", "实时转写但只用命令行输出（不开窗口）");
|
||||
QCommandLineOption optThreads("threads", "推理线程数", "n", "8");
|
||||
QCommandLineOption optMaxTokens("max-tokens", "单段最多生成 token 数", "n", "256");
|
||||
QCommandLineOption optStreamChunk("stream-chunk", "流式 chunk 秒数", "sec", "2.0");
|
||||
QCommandLineOption optStreamWindow("stream-window", "滑窗上限秒数（仅 window 模式）", "sec", "16");
|
||||
QCommandLineOption optStreamSegment("stream-segment", "单段最长秒数（0=不限）", "sec", "30");
|
||||
QCommandLineOption optUnfixedTokens("unfixed-tokens", "前缀回滚丢弃的 token 数", "n", "5");
|
||||
QCommandLineOption optUnfixedChunks("unfixed-chunks", "冷启动不带前缀的 chunk 数", "n", "2");
|
||||
QCommandLineOption optStreamMode("stream-mode", "流式后端: incremental 或 window", "mode", "incremental");
|
||||
QCommandLineOption optVerbose("verbose", "输出 llama.cpp/mtmd 的详细日志");
|
||||
QCommandLineOption optScreenshot("screenshot", "启动后自动截图并退出（验证渲染用）", "path");
|
||||
QCommandLineOption optScreenshotDelay("screenshot-delay", "截图前等待秒数", "sec", "6");
|
||||
QCommandLineOption optOpacity("opacity", "字幕背景不透明度（20-100，百分比）", "n", "84");
|
||||
|
||||
parser.addOption(optModel);
|
||||
parser.addOption(optMmproj);
|
||||
parser.addOption(optFile);
|
||||
parser.addOption(optStreamFile);
|
||||
parser.addOption(optCliLive);
|
||||
parser.addOption(optThreads);
|
||||
parser.addOption(optMaxTokens);
|
||||
parser.addOption(optStreamChunk);
|
||||
parser.addOption(optStreamWindow);
|
||||
parser.addOption(optStreamSegment);
|
||||
parser.addOption(optUnfixedTokens);
|
||||
parser.addOption(optUnfixedChunks);
|
||||
parser.addOption(optStreamMode);
|
||||
parser.addOption(optVerbose);
|
||||
parser.addOption(optScreenshot);
|
||||
parser.addOption(optScreenshotDelay);
|
||||
parser.addOption(optOpacity);
|
||||
parser.process(*app);
|
||||
|
||||
lc::AsrOptions asrOpt;
|
||||
asrOpt.modelPath = parser.value(optModel).toStdString();
|
||||
asrOpt.mmprojPath = parser.value(optMmproj).toStdString();
|
||||
asrOpt.threads = parser.value(optThreads).toInt();
|
||||
asrOpt.maxTokens = parser.value(optMaxTokens).toInt();
|
||||
asrOpt.verbose = parser.isSet(optVerbose);
|
||||
|
||||
if (asrOpt.modelPath.empty() || asrOpt.mmprojPath.empty()) {
|
||||
std::fprintf(stderr, "必须提供 --model 和 --mmproj\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
lc::StreamOptions sop;
|
||||
sop.chunkSec = parser.value(optStreamChunk).toDouble();
|
||||
sop.windowSec = parser.value(optStreamWindow).toDouble();
|
||||
sop.maxUtteranceSec = parser.value(optStreamSegment).toDouble();
|
||||
sop.unfixedTokenNum = parser.value(optUnfixedTokens).toInt();
|
||||
sop.unfixedChunkNum = parser.value(optUnfixedChunks).toInt();
|
||||
sop.maxTokensPerStep = asrOpt.maxTokens;
|
||||
sop.incremental = parser.value(optStreamMode) != QLatin1String("window");
|
||||
|
||||
lc::Qwen3Asr asr;
|
||||
std::string err;
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
if (!asr.load(asrOpt, &err)) {
|
||||
std::fprintf(stderr, "加载失败: %s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
std::fprintf(stderr, "模型加载完成: %.2fs (采样率 %d)\n",
|
||||
std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(),
|
||||
asr.sampleRate());
|
||||
|
||||
if (parser.isSet(optFile)) return runFile(asr, parser.value(optFile));
|
||||
if (parser.isSet(optStreamFile)) return runStreamFile(asr, parser.value(optStreamFile), sop);
|
||||
if (parser.isSet(optCliLive)) return runCliLive(asr, sop);
|
||||
return runGui(asr, sop,
|
||||
parser.value(optScreenshot),
|
||||
parser.value(optScreenshotDelay).toInt(),
|
||||
parser.value(optOpacity).toInt());
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
#include "transcript.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
struct AsrOptions {
|
||||
std::string modelPath; // Qwen3-ASR 主模型 GGUF（架构须为 qwen3vl）
|
||||
std::string mmprojPath; // 音频编码器 mmproj GGUF（clip + qwen3a）
|
||||
int threads = 8;
|
||||
int nCtx = 4096;
|
||||
int nBatch = 4096;
|
||||
int nUbatch = 1024;
|
||||
int maxTokens = 256; // 单段最多生成多少 token
|
||||
bool warmup = false; // mtmd 初始化后是否跑一次预热编码
|
||||
bool verbose = false;
|
||||
};
|
||||
|
||||
// 进程内 Qwen3-ASR 推理封装（libllama + libmtmd）。
|
||||
// 模型常驻，可反复 transcribe()；每次调用会清空 KV cache。
|
||||
class Qwen3Asr {
|
||||
public:
|
||||
Qwen3Asr();
|
||||
~Qwen3Asr();
|
||||
|
||||
Qwen3Asr(const Qwen3Asr &) = delete;
|
||||
Qwen3Asr & operator=(const Qwen3Asr &) = delete;
|
||||
|
||||
bool load(const AsrOptions & opt, std::string * error);
|
||||
|
||||
// 期望的输入采样率（Qwen3-ASR 为 16000）
|
||||
int sampleRate() const;
|
||||
|
||||
struct Result {
|
||||
bool ok = false;
|
||||
std::string error;
|
||||
ParsedTranscript transcript; // 已拆出 language / text
|
||||
std::string raw; // 模型原始输出，便于排查
|
||||
double inferSeconds = 0.0;
|
||||
int tokenCount = 0;
|
||||
};
|
||||
|
||||
// pcm: 单声道 float32 [-1,1]，采样率为 sampleRate()
|
||||
Result transcribe(const float * pcm, size_t nSamples);
|
||||
|
||||
// 流式单步：把 prefix 当作"已经生成的 assistant 文本"接在 assistant 头之后，
|
||||
// 让模型从 prefix 末尾继续生成。官方流式算法的核心（前缀回滚）就靠它。
|
||||
Result transcribeWithPrefix(const float * pcm, size_t nSamples, const std::string & prefix);
|
||||
|
||||
// 用模型词表把 text 编码成 token、丢掉最后 dropLast 个、再还原成文本。
|
||||
// 对应官方的 unfixed_token_num 回滚。
|
||||
std::string dropLastTokens(const std::string & text, int dropLast) const;
|
||||
|
||||
// ---- 流式（增量 KV 复用）----
|
||||
// 与上面每次重编码整个窗口的做法不同：这里只对新到的音频做一次编码，
|
||||
// 已经解码过的音频留在 KV cache 里复用，因此每步成本与音频总长无关。
|
||||
void streamReset(); // 清空 KV 与流式状态
|
||||
|
||||
struct StreamStep {
|
||||
bool ok = false;
|
||||
std::string error;
|
||||
std::string newText; // 本步新生成的文本（不含前缀，未 trim）
|
||||
int newTokens = 0;
|
||||
int audioTokens = 0; // 本步新追加的音频 token 数
|
||||
double encodeSeconds = 0.0;
|
||||
double inferSeconds = 0.0;
|
||||
};
|
||||
// 追加一段音频，在 assistant 头之后接上 prefix 作为"已生成文本"继续生成。
|
||||
StreamStep streamAppend(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix, int maxTokens);
|
||||
|
||||
private:
|
||||
bool prepareStream(std::string * error);
|
||||
|
||||
struct Impl;
|
||||
Impl * d = nullptr;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,60 @@
|
||||
#include "livetranscriber.h"
|
||||
|
||||
#include "iasr.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
LiveTranscriber::LiveTranscriber(IAsr & asr, QObject * parent)
|
||||
: QObject(parent), asr_(asr)
|
||||
{
|
||||
}
|
||||
|
||||
LiveTranscriber::~LiveTranscriber()
|
||||
{
|
||||
stop();
|
||||
}
|
||||
|
||||
QString LiveTranscriber::deviceName() const
|
||||
{
|
||||
return pipeline_ ? QString::fromStdString(pipeline_->deviceName()) : QString();
|
||||
}
|
||||
|
||||
bool LiveTranscriber::start(const StreamOptions & opts, QString * error)
|
||||
{
|
||||
stop();
|
||||
|
||||
LivePipelineOptions popt;
|
||||
popt.stream = opts;
|
||||
|
||||
pipeline_ = std::make_unique<LivePipeline>(asr_, popt);
|
||||
|
||||
// 回调在工作线程上被调用；连到主线程的接收者时 Qt 会自动排队投递。
|
||||
pipeline_->setCaptionCallback([this](const std::string & committed,
|
||||
const std::string & tail,
|
||||
const std::string & language) {
|
||||
Q_EMIT caption(QString::fromStdString(committed),
|
||||
QString::fromStdString(tail),
|
||||
QString::fromStdString(language));
|
||||
});
|
||||
pipeline_->setErrorCallback([this](const std::string & message) {
|
||||
Q_EMIT failed(QString::fromStdString(message));
|
||||
});
|
||||
|
||||
std::string err;
|
||||
if (!pipeline_->start(&err)) {
|
||||
if (error) *error = QString::fromStdString(err);
|
||||
pipeline_.reset();
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void LiveTranscriber::stop()
|
||||
{
|
||||
if (pipeline_) {
|
||||
pipeline_->stop();
|
||||
pipeline_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,40 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include <QObject>
|
||||
#include <QString>
|
||||
|
||||
#include "livepipeline.h"
|
||||
#include "streamingasr.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
class IAsr;
|
||||
|
||||
// LivePipeline 的 Qt 适配器：只负责把 std::function 回调转成 Qt 信号，
|
||||
// 真正的逻辑都在 core 里（所以那些逻辑可以脱离 UI 单独测试）。
|
||||
class LiveTranscriber : public QObject {
|
||||
Q_OBJECT
|
||||
|
||||
public:
|
||||
explicit LiveTranscriber(IAsr & asr, QObject * parent = nullptr);
|
||||
~LiveTranscriber() override;
|
||||
|
||||
bool start(const StreamOptions & opts, QString * error = nullptr);
|
||||
void stop();
|
||||
|
||||
QString deviceName() const;
|
||||
|
||||
Q_SIGNALS:
|
||||
// committed = 已稳定文本；tail = 不稳定尾部（UI 用暗色显示）
|
||||
void caption(const QString & committed, const QString & tail, const QString & language);
|
||||
void failed(const QString & message);
|
||||
|
||||
private:
|
||||
IAsr & asr_;
|
||||
std::unique_ptr<LivePipeline> pipeline_;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
|
||||
Reference in new issue
Block a user