refactor: split core/ui/apps targets and add unit tests
Move all business logic into liblivecaption_core (no Qt dependency) and the widgets into liblivecaption_ui, leaving two executables: the caption window and the CLI, which no longer links Qt Widgets. Abstract the ASR backend behind IAsr so the streaming policy and the live pipeline can be tested with a scripted fake backend instead of a real model. 把全部业务逻辑下沉到不依赖 Qt 的 liblivecaption_core，界面独立成 liblivecaption_ui，拆出悬浮窗与命令行两个可执行程序（CLI 不再链接 Qt Widgets）；并用 IAsr 抽象推理后端，使流式策略与实时管线可脱离真模型 用假实现做单元测试。测试还抓到一个真 bug：段恰好结束在 chunk 边界时 只提交了稳定部分，最后几个词被丢掉，已修。
This commit is contained in:
blumia committed
1 parent
4ed4e23282
commit
736ec14d5b
32 files changed
+1502
-655
No files matched your search
@@ -0,0 +1,381 @@
|
||||
#include "audio.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
|
||||
#include <samplerate.h>
|
||||
#include <pulse/pulseaudio.h>
|
||||
|
||||
namespace lc {
|
||||
|
||||
namespace {
|
||||
|
||||
// 单声道流式重采样器（libsamplerate）
|
||||
class Resampler {
|
||||
public:
|
||||
~Resampler() { if (st_) src_delete(st_); }
|
||||
|
||||
bool init(int fromRate, int toRate)
|
||||
{
|
||||
int err = 0;
|
||||
st_ = src_new(SRC_SINC_FASTEST, 1, &err);
|
||||
ratio_ = (double) toRate / (double) fromRate;
|
||||
return st_ != nullptr;
|
||||
}
|
||||
|
||||
// 输入 frames 个单声道样本，追加到 out
|
||||
bool process(const float * in, size_t frames, std::vector<float> & out)
|
||||
{
|
||||
if (!st_) return false;
|
||||
if (frames == 0) return true;
|
||||
|
||||
// 输出上限：比例 + 余量
|
||||
const size_t maxOut = (size_t) (frames * ratio_) + 64;
|
||||
const size_t base = out.size();
|
||||
out.resize(base + maxOut);
|
||||
|
||||
SRC_DATA d{};
|
||||
d.data_in = in;
|
||||
d.input_frames = (long) frames;
|
||||
d.data_out = out.data() + base;
|
||||
d.output_frames = (long) maxOut;
|
||||
d.src_ratio = ratio_;
|
||||
d.end_of_input = 0;
|
||||
|
||||
if (src_process(st_, &d) != 0) return false;
|
||||
out.resize(base + (size_t) d.output_frames_gen);
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
SRC_STATE * st_ = nullptr;
|
||||
double ratio_ = 1.0;
|
||||
};
|
||||
|
||||
// 读小端整数
|
||||
uint32_t rd32(const uint8_t * p) { return p[0] | (p[1] << 8) | (p[2] << 16) | ((uint32_t) p[3] << 24); }
|
||||
uint16_t rd16(const uint8_t * p) { return (uint16_t) (p[0] | (p[1] << 8)); }
|
||||
|
||||
} // namespace
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WAV 读取
|
||||
// ---------------------------------------------------------------------------
|
||||
bool readWavMono(const std::string & path, int targetRate,
|
||||
std::vector<float> & out, std::string * error)
|
||||
{
|
||||
const auto fail = [error](const std::string & m) {
|
||||
if (error) *error = m;
|
||||
return false;
|
||||
};
|
||||
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
if (!f) return fail("无法打开文件: " + path);
|
||||
|
||||
std::vector<uint8_t> buf((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
|
||||
if (buf.size() < 44 || std::memcmp(buf.data(), "RIFF", 4) != 0 || std::memcmp(buf.data() + 8, "WAVE", 4) != 0) {
|
||||
return fail("不是合法的 WAV 文件: " + path);
|
||||
}
|
||||
|
||||
uint16_t audioFormat = 0, channels = 0, bitsPerSample = 0;
|
||||
uint32_t sampleRate = 0;
|
||||
const uint8_t * dataPtr = nullptr;
|
||||
size_t dataLen = 0;
|
||||
|
||||
size_t pos = 12;
|
||||
while (pos + 8 <= buf.size()) {
|
||||
const uint8_t * chunk = buf.data() + pos;
|
||||
const uint32_t chunkSize = rd32(chunk + 4);
|
||||
const uint8_t * body = chunk + 8;
|
||||
if (std::memcmp(chunk, "fmt ", 4) == 0) {
|
||||
if (chunkSize < 16) return fail("fmt 块过短");
|
||||
audioFormat = rd16(body + 0);
|
||||
channels = rd16(body + 2);
|
||||
sampleRate = rd32(body + 4);
|
||||
bitsPerSample = rd16(body + 14);
|
||||
if (audioFormat == 0xFFFE && chunkSize >= 40) {
|
||||
audioFormat = rd16(body + 24); // WAVE_FORMAT_EXTENSIBLE 的 subformat 前两字节
|
||||
}
|
||||
} else if (std::memcmp(chunk, "data", 4) == 0) {
|
||||
dataPtr = body;
|
||||
dataLen = chunkSize;
|
||||
if (pos + 8 + chunkSize > buf.size()) dataLen = buf.size() - (pos + 8);
|
||||
}
|
||||
pos += 8 + chunkSize + (chunkSize & 1); // 块按偶数字节对齐
|
||||
}
|
||||
|
||||
if (!dataPtr || channels == 0 || sampleRate == 0) return fail("WAV 缺少 fmt 或 data 块");
|
||||
|
||||
const bool isFloat = (audioFormat == 3);
|
||||
const int bytesPerSample = bitsPerSample / 8;
|
||||
if (bytesPerSample <= 0) return fail("不支持的位深");
|
||||
const size_t frameBytes = (size_t) bytesPerSample * channels;
|
||||
const size_t frames = dataLen / frameBytes;
|
||||
|
||||
// 降混为单声道 float
|
||||
std::vector<float> mono(frames);
|
||||
for (size_t i = 0; i < frames; ++i) {
|
||||
double acc = 0.0;
|
||||
const uint8_t * p = dataPtr + i * frameBytes;
|
||||
for (uint16_t c = 0; c < channels; ++c, p += bytesPerSample) {
|
||||
if (isFloat) {
|
||||
float v;
|
||||
std::memcpy(&v, p, 4);
|
||||
acc += v;
|
||||
} else if (bitsPerSample == 16) {
|
||||
acc += (int16_t) rd16(p) / 32768.0;
|
||||
} else if (bitsPerSample == 24) {
|
||||
int32_t v = p[0] | (p[1] << 8) | (p[2] << 16);
|
||||
if (v & 0x800000) v |= ~0xFFFFFF;
|
||||
acc += v / 8388608.0;
|
||||
} else if (bitsPerSample == 32) {
|
||||
acc += (int32_t) rd32(p) / 2147483648.0;
|
||||
}
|
||||
}
|
||||
mono[i] = (float) (acc / channels);
|
||||
}
|
||||
|
||||
if ((int) sampleRate == targetRate) {
|
||||
out = std::move(mono);
|
||||
return true;
|
||||
}
|
||||
|
||||
Resampler rs;
|
||||
if (!rs.init((int) sampleRate, targetRate)) return fail("重采样器初始化失败");
|
||||
out.clear();
|
||||
return rs.process(mono.data(), mono.size(), out);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 实时采集：默认 sink 的 monitor
|
||||
// ---------------------------------------------------------------------------
|
||||
struct MonitorCapture::Impl {
|
||||
pa_threaded_mainloop * ml = nullptr;
|
||||
pa_context * ctx = nullptr;
|
||||
pa_stream * stream = nullptr;
|
||||
ChunkCallback callback;
|
||||
int targetRate = 16000;
|
||||
int deviceRate = 48000;
|
||||
int channels = 2;
|
||||
Resampler resampler;
|
||||
std::string deviceName;
|
||||
bool ready = false;
|
||||
};
|
||||
|
||||
namespace {
|
||||
|
||||
void storeMonitor(pa_context * /*c*/, const pa_server_info * info, void * userdata)
|
||||
{
|
||||
auto * d = static_cast<MonitorCapture::Impl *>(userdata);
|
||||
if (!info || !info->default_sink_name) return;
|
||||
d->deviceName = std::string(info->default_sink_name) + ".monitor";
|
||||
pa_threaded_mainloop_signal(d->ml, 0);
|
||||
}
|
||||
|
||||
void streamState(pa_stream * s, void * userdata)
|
||||
{
|
||||
auto * d = static_cast<MonitorCapture::Impl *>(userdata);
|
||||
switch (pa_stream_get_state(s)) {
|
||||
case PA_STREAM_READY:
|
||||
case PA_STREAM_FAILED:
|
||||
case PA_STREAM_TERMINATED:
|
||||
pa_threaded_mainloop_signal(d->ml, 0);
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
|
||||
void streamRead(pa_stream * s, size_t /*nbytes*/, void * userdata)
|
||||
{
|
||||
auto * d = static_cast<MonitorCapture::Impl *>(userdata);
|
||||
while (pa_stream_readable_size(s) > 0) {
|
||||
const void * data = nullptr;
|
||||
size_t bytes = 0;
|
||||
if (pa_stream_peek(s, &data, &bytes) < 0 || !data || bytes == 0) {
|
||||
pa_stream_drop(s);
|
||||
continue;
|
||||
}
|
||||
|
||||
const float * interleaved = static_cast<const float *>(data);
|
||||
const size_t frames = bytes / (sizeof(float) * (size_t) d->channels);
|
||||
|
||||
// 降混单声道
|
||||
std::vector<float> mono(frames);
|
||||
for (size_t i = 0; i < frames; ++i) {
|
||||
double acc = 0.0;
|
||||
for (int c = 0; c < d->channels; ++c) acc += interleaved[i * d->channels + c];
|
||||
mono[i] = (float) (acc / d->channels);
|
||||
}
|
||||
|
||||
// 重采样到目标采样率
|
||||
std::vector<float> resampled;
|
||||
if (d->deviceRate == d->targetRate) {
|
||||
resampled = std::move(mono);
|
||||
} else if (!d->resampler.process(mono.data(), mono.size(), resampled)) {
|
||||
pa_stream_drop(s);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (d->callback && !resampled.empty()) {
|
||||
d->callback(resampled.data(), resampled.size());
|
||||
}
|
||||
pa_stream_drop(s);
|
||||
}
|
||||
}
|
||||
|
||||
void contextState(pa_context * c, void * userdata)
|
||||
{
|
||||
auto * d = static_cast<MonitorCapture::Impl *>(userdata);
|
||||
switch (pa_context_get_state(c)) {
|
||||
case PA_CONTEXT_READY: d->ready = true; pa_threaded_mainloop_signal(d->ml, 0); break;
|
||||
case PA_CONTEXT_FAILED:
|
||||
case PA_CONTEXT_TERMINATED:
|
||||
pa_threaded_mainloop_signal(d->ml, 0);
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
MonitorCapture::MonitorCapture() : d(new Impl) {}
|
||||
|
||||
MonitorCapture::~MonitorCapture()
|
||||
{
|
||||
stop();
|
||||
delete d;
|
||||
}
|
||||
|
||||
std::string MonitorCapture::deviceName() const { return d->deviceName; }
|
||||
|
||||
bool MonitorCapture::start(int targetRate, ChunkCallback callback, std::string * error)
|
||||
{
|
||||
const auto fail = [error](const std::string & m) {
|
||||
if (error) *error = m;
|
||||
return false;
|
||||
};
|
||||
|
||||
d->targetRate = targetRate;
|
||||
d->callback = std::move(callback);
|
||||
|
||||
d->ml = pa_threaded_mainloop_new();
|
||||
if (!d->ml) return fail("pa_threaded_mainloop_new 失败");
|
||||
|
||||
d->ctx = pa_context_new(pa_threaded_mainloop_get_api(d->ml), "live-caption");
|
||||
if (!d->ctx) return fail("pa_context_new 失败");
|
||||
|
||||
pa_context_set_state_callback(d->ctx, contextState, d);
|
||||
|
||||
if (pa_threaded_mainloop_start(d->ml) < 0) return fail("pa_threaded_mainloop_start 失败");
|
||||
|
||||
pa_threaded_mainloop_lock(d->ml);
|
||||
|
||||
if (pa_context_connect(d->ctx, nullptr, PA_CONTEXT_NOFLAGS, nullptr) < 0) {
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
return fail("pa_context_connect 失败（PulseAudio/PipeWire 未运行？）");
|
||||
}
|
||||
|
||||
// 等 context READY
|
||||
for (;;) {
|
||||
const pa_context_state_t st = pa_context_get_state(d->ctx);
|
||||
if (st == PA_CONTEXT_READY) break;
|
||||
if (!PA_CONTEXT_IS_GOOD(st)) {
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
return fail("无法连接到 PulseAudio/PipeWire 服务");
|
||||
}
|
||||
pa_threaded_mainloop_wait(d->ml);
|
||||
}
|
||||
|
||||
// 查默认 sink，拼出它的 monitor 源
|
||||
d->deviceName.clear();
|
||||
pa_operation * op = pa_context_get_server_info(d->ctx, storeMonitor, d);
|
||||
while (pa_operation_get_state(op) == PA_OPERATION_RUNNING) {
|
||||
pa_threaded_mainloop_wait(d->ml);
|
||||
}
|
||||
pa_operation_unref(op);
|
||||
|
||||
if (d->deviceName.empty()) {
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
return fail("拿不到默认输出设备名");
|
||||
}
|
||||
|
||||
// 以设备实际速率请求（默认 48k）
|
||||
pa_sample_spec ss{};
|
||||
ss.format = PA_SAMPLE_FLOAT32LE;
|
||||
ss.rate = 48000;
|
||||
ss.channels = 2;
|
||||
|
||||
d->stream = pa_stream_new(d->ctx, "live-caption-monitor", &ss, nullptr);
|
||||
if (!d->stream) {
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
return fail("pa_stream_new 失败");
|
||||
}
|
||||
|
||||
pa_stream_set_read_callback(d->stream, streamRead, d);
|
||||
pa_stream_set_state_callback(d->stream, streamState, d);
|
||||
|
||||
pa_buffer_attr attr{};
|
||||
attr.maxlength = (uint32_t) -1;
|
||||
attr.tlength = (uint32_t) -1;
|
||||
attr.prebuf = (uint32_t) -1;
|
||||
attr.minreq = (uint32_t) -1;
|
||||
attr.fragsize = (uint32_t) (pa_frame_size(&ss) * 1024); // ~21ms
|
||||
|
||||
if (pa_stream_connect_record(d->stream, d->deviceName.c_str(), &attr, PA_STREAM_ADJUST_LATENCY) < 0) {
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
return fail("连接 monitor 源失败: " + d->deviceName);
|
||||
}
|
||||
|
||||
// 等 stream READY，才能确认 monitor 源确实可用
|
||||
for (;;) {
|
||||
const pa_stream_state_t st = pa_stream_get_state(d->stream);
|
||||
if (st == PA_STREAM_READY) break;
|
||||
if (st == PA_STREAM_FAILED || st == PA_STREAM_TERMINATED) {
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
return fail("monitor 源不可用: " + d->deviceName + " (" +
|
||||
std::string(pa_strerror(pa_context_errno(d->ctx))) + ")");
|
||||
}
|
||||
pa_threaded_mainloop_wait(d->ml);
|
||||
}
|
||||
|
||||
// 更新为实际生效的格式
|
||||
const pa_sample_spec * actual = pa_stream_get_sample_spec(d->stream);
|
||||
if (actual) {
|
||||
d->deviceRate = (int) actual->rate;
|
||||
d->channels = (int) actual->channels;
|
||||
}
|
||||
if (!d->resampler.init(d->deviceRate, d->targetRate)) {
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
return fail("重采样器初始化失败");
|
||||
}
|
||||
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
return true;
|
||||
}
|
||||
|
||||
void MonitorCapture::stop()
|
||||
{
|
||||
if (!d->ml) return;
|
||||
|
||||
pa_threaded_mainloop_lock(d->ml);
|
||||
if (d->stream) {
|
||||
pa_stream_disconnect(d->stream);
|
||||
pa_stream_unref(d->stream);
|
||||
d->stream = nullptr;
|
||||
}
|
||||
if (d->ctx) {
|
||||
pa_context_disconnect(d->ctx);
|
||||
pa_context_unref(d->ctx);
|
||||
d->ctx = nullptr;
|
||||
}
|
||||
pa_threaded_mainloop_unlock(d->ml);
|
||||
|
||||
pa_threaded_mainloop_stop(d->ml);
|
||||
pa_threaded_mainloop_free(d->ml);
|
||||
d->ml = nullptr;
|
||||
d->callback = nullptr;
|
||||
}
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,40 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace lc {
|
||||
|
||||
// 读取 WAV（PCM 16/24/32 位或 float32），降混为单声道并重采样到 targetRate。
|
||||
bool readWavMono(const std::string & path, int targetRate,
|
||||
std::vector<float> & out, std::string * error);
|
||||
|
||||
// 采集默认输出设备的 monitor（即"系统正在播放的声音"），
|
||||
// 输出 targetRate 单声道 float32。回调在 PulseAudio 线程上调用，必须快速返回。
|
||||
class MonitorCapture {
|
||||
public:
|
||||
using ChunkCallback = std::function<void(const float * mono, size_t nSamples)>;
|
||||
|
||||
MonitorCapture();
|
||||
~MonitorCapture();
|
||||
|
||||
MonitorCapture(const MonitorCapture &) = delete;
|
||||
MonitorCapture & operator=(const MonitorCapture &) = delete;
|
||||
|
||||
bool start(int targetRate, ChunkCallback callback, std::string * error);
|
||||
void stop();
|
||||
|
||||
// 实际采集的 monitor 源名（start 成功后有效）
|
||||
std::string deviceName() const;
|
||||
|
||||
// 前向声明（定义在 .cpp）。公开仅为让文件内的 C 回调能引用该类型，
|
||||
// 头文件里它始终是不完整类型。
|
||||
struct Impl;
|
||||
|
||||
private:
|
||||
Impl * d = nullptr;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,60 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
#include "transcript.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
// 一次整段转写的结果
|
||||
struct AsrResult {
|
||||
bool ok = false;
|
||||
std::string error;
|
||||
ParsedTranscript transcript; // 已拆出 language / text
|
||||
std::string raw; // 模型原始输出（未 trim）
|
||||
double inferSeconds = 0.0;
|
||||
int tokenCount = 0;
|
||||
};
|
||||
|
||||
// 流式单步的结果
|
||||
struct AsrStreamStep {
|
||||
bool ok = false;
|
||||
std::string error;
|
||||
std::string newText; // 本步新生成的文本（不含前缀，未 trim）
|
||||
int newTokens = 0;
|
||||
int audioTokens = 0;
|
||||
double encodeSeconds = 0.0;
|
||||
double inferSeconds = 0.0;
|
||||
};
|
||||
|
||||
// 把推理后端抽象出来，好处是流式策略（StreamingAsr）和实时管线（LivePipeline）
|
||||
// 可以用一个假的实现来测，不必加载真模型。
|
||||
class IAsr {
|
||||
public:
|
||||
virtual ~IAsr() = default;
|
||||
|
||||
// 期望的输入采样率（Qwen3-ASR 为 16000）
|
||||
virtual int sampleRate() const = 0;
|
||||
|
||||
// 整段转写：把 prefix 当作"已经生成的 assistant 文本"，从那里继续生成
|
||||
virtual AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix) = 0;
|
||||
|
||||
// 流式：只编码新音频，已解码的音频留在 KV cache 里复用
|
||||
virtual AsrStreamStep streamAppend(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix, int maxTokens) = 0;
|
||||
virtual void streamReset() = 0;
|
||||
|
||||
// 用词表把 text 编码成 token、丢掉最后 dropLast 个、再还原成文本。
|
||||
// 对应官方的 unfixed_token_num 回滚。
|
||||
virtual std::string dropLastTokens(const std::string & text, int dropLast) const = 0;
|
||||
|
||||
// 便利：不带前缀的整段转写
|
||||
AsrResult transcribe(const float * pcm, size_t nSamples)
|
||||
{
|
||||
return transcribeWithPrefix(pcm, nSamples, std::string());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,147 @@
|
||||
#include "livepipeline.h"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
namespace lc {
|
||||
|
||||
namespace {
|
||||
constexpr int kFrameSamples = 320; // 20ms @16k
|
||||
} // namespace
|
||||
|
||||
LivePipeline::LivePipeline(IAsr & asr, LivePipelineOptions opt)
|
||||
: asr_(asr), opt_(std::move(opt)), stream_(asr, opt_.stream) {}
|
||||
|
||||
LivePipeline::~LivePipeline()
|
||||
{
|
||||
stop();
|
||||
}
|
||||
|
||||
std::string LivePipeline::deviceName() const
|
||||
{
|
||||
return capture_.deviceName();
|
||||
}
|
||||
|
||||
bool LivePipeline::start(std::string * error)
|
||||
{
|
||||
const bool ok = capture_.start(
|
||||
asr_.sampleRate(),
|
||||
[this](const float * s, size_t n) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mutex_);
|
||||
pending_.insert(pending_.end(), s, s + n);
|
||||
}
|
||||
cv_.notify_one();
|
||||
},
|
||||
error);
|
||||
if (!ok) return false;
|
||||
|
||||
worker_ = std::thread([this] { workerLoop(); });
|
||||
return true;
|
||||
}
|
||||
|
||||
void LivePipeline::stop()
|
||||
{
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mutex_);
|
||||
quit_ = true;
|
||||
}
|
||||
cv_.notify_all();
|
||||
if (worker_.joinable()) worker_.join();
|
||||
capture_.stop();
|
||||
}
|
||||
|
||||
void LivePipeline::workerLoop()
|
||||
{
|
||||
for (;;) {
|
||||
std::vector<float> chunk;
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cv_.wait_for(lk, std::chrono::milliseconds(50),
|
||||
[this] { return quit_ || !pending_.empty(); });
|
||||
if (quit_ && pending_.empty()) break;
|
||||
chunk.swap(pending_);
|
||||
}
|
||||
if (!chunk.empty()) process(chunk.data(), chunk.size());
|
||||
}
|
||||
|
||||
if (inSegment_) finishSegment();
|
||||
}
|
||||
|
||||
void LivePipeline::emitStep(const StreamingAsr::Step & s)
|
||||
{
|
||||
if (!s.ok || !onCaption_) return;
|
||||
onCaption_(s.committed, s.tail, s.language);
|
||||
}
|
||||
|
||||
void LivePipeline::process(const float * pcm, size_t n)
|
||||
{
|
||||
buffer_.insert(buffer_.end(), pcm, pcm + n);
|
||||
|
||||
const size_t prerollSamples = (size_t) (opt_.prerollSec * asr_.sampleRate());
|
||||
const float frameSec = (float) kFrameSamples / asr_.sampleRate();
|
||||
|
||||
std::vector<float> segAudio; // 本段要送进引擎的音频
|
||||
bool flushNow = false;
|
||||
|
||||
size_t off = 0;
|
||||
for (; off + kFrameSamples <= buffer_.size(); off += kFrameSamples) {
|
||||
const float * frame = buffer_.data() + off;
|
||||
const bool speech = vad_.feed(frame, kFrameSamples);
|
||||
|
||||
if (!inSegment_) {
|
||||
// 静音期间什么都不送给模型：否则模型会对静音产生幻觉
|
||||
if (speech) {
|
||||
inSegment_ = true;
|
||||
silenceAcc_ = 0.0f;
|
||||
segAudio.insert(segAudio.end(), preroll_.begin(), preroll_.end());
|
||||
preroll_.clear();
|
||||
segAudio.insert(segAudio.end(), frame, frame + kFrameSamples);
|
||||
} else {
|
||||
preroll_.insert(preroll_.end(), frame, frame + kFrameSamples);
|
||||
if (preroll_.size() > prerollSamples) {
|
||||
preroll_.erase(preroll_.begin(),
|
||||
preroll_.begin() + (long) (preroll_.size() - prerollSamples));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
segAudio.insert(segAudio.end(), frame, frame + kFrameSamples);
|
||||
if (speech) silenceAcc_ = 0.0f;
|
||||
else silenceAcc_ += frameSec;
|
||||
|
||||
if (silenceAcc_ >= (float) opt_.silenceTimeoutSec) {
|
||||
flushNow = true;
|
||||
off += kFrameSamples; // 这一帧已经并入 segAudio
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!segAudio.empty()) {
|
||||
const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); };
|
||||
stream_.feed(segAudio.data(), segAudio.size(), cb);
|
||||
}
|
||||
|
||||
if (flushNow) {
|
||||
const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); };
|
||||
stream_.flush(cb);
|
||||
stream_.reset();
|
||||
inSegment_ = false;
|
||||
silenceAcc_ = 0.0f;
|
||||
preroll_.clear();
|
||||
}
|
||||
|
||||
if (off > 0) buffer_.erase(buffer_.begin(), buffer_.begin() + (long) off);
|
||||
}
|
||||
|
||||
void LivePipeline::finishSegment()
|
||||
{
|
||||
if (!inSegment_) return;
|
||||
const auto cb = [this](const StreamingAsr::Step & s) { emitStep(s); };
|
||||
stream_.flush(cb);
|
||||
stream_.reset();
|
||||
inSegment_ = false;
|
||||
silenceAcc_ = 0.0f;
|
||||
preroll_.clear();
|
||||
}
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,87 @@
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "audio.h"
|
||||
#include "iasr.h"
|
||||
#include "streamingasr.h"
|
||||
#include "vad.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
struct LivePipelineOptions {
|
||||
StreamOptions stream;
|
||||
double silenceTimeoutSec = 0.8; // 静音多久算一句结束
|
||||
double prerollSec = 0.25; // 语音开始前保留的一段，避免吃掉词头
|
||||
};
|
||||
|
||||
// 实时管线：采集 → VAD → 分段 → 流式转写。
|
||||
//
|
||||
// 刻意不带任何 Qt 依赖（回调用 std::function），好处是可以脱离 UI 单独测试：
|
||||
// 测试直接调 process() 喂音频，不必碰 PulseAudio，也不用跑事件循环。
|
||||
class LivePipeline {
|
||||
public:
|
||||
using CaptionCallback = std::function<void(const std::string & committed,
|
||||
const std::string & tail,
|
||||
const std::string & language)>;
|
||||
using ErrorCallback = std::function<void(const std::string & message)>;
|
||||
|
||||
LivePipeline(IAsr & asr, LivePipelineOptions opt = {});
|
||||
~LivePipeline();
|
||||
|
||||
LivePipeline(const LivePipeline &) = delete;
|
||||
LivePipeline & operator=(const LivePipeline &) = delete;
|
||||
|
||||
void setCaptionCallback(CaptionCallback cb) { onCaption_ = std::move(cb); }
|
||||
void setErrorCallback(ErrorCallback cb) { onError_ = std::move(cb); }
|
||||
|
||||
// 启动真实采集（默认输出设备的 monitor）
|
||||
bool start(std::string * error = nullptr);
|
||||
void stop();
|
||||
std::string deviceName() const;
|
||||
|
||||
// 同步处理一段 16k 单声道音频。
|
||||
// 真实运行时由采集线程调用；测试可以直接调它。
|
||||
void process(const float * pcm, size_t n);
|
||||
|
||||
// 结束当前这一段并提交（流结束或手动收尾时用）
|
||||
void finishSegment();
|
||||
|
||||
// 注意：StreamingAsr::committedText() 是按值返回的，这里也必须按值返回
|
||||
std::string committedText() const { return stream_.committedText(); }
|
||||
const std::string & language() const { return stream_.language(); }
|
||||
int chunkCount() const { return stream_.chunkCount(); }
|
||||
bool inSegment() const { return inSegment_; }
|
||||
|
||||
private:
|
||||
void workerLoop();
|
||||
void emitStep(const StreamingAsr::Step & s);
|
||||
|
||||
IAsr & asr_;
|
||||
LivePipelineOptions opt_;
|
||||
Vad vad_;
|
||||
StreamingAsr stream_;
|
||||
|
||||
CaptionCallback onCaption_;
|
||||
ErrorCallback onError_;
|
||||
|
||||
bool inSegment_ = false;
|
||||
float silenceAcc_ = 0.0f;
|
||||
std::vector<float> preroll_; // 语音开始前保留的一小段
|
||||
std::vector<float> buffer_; // 跨批次的残留样本（不接上会持续丢样本）
|
||||
|
||||
MonitorCapture capture_;
|
||||
std::thread worker_;
|
||||
std::mutex mutex_;
|
||||
std::condition_variable cv_;
|
||||
std::vector<float> pending_;
|
||||
bool quit_ = false;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,478 @@
|
||||
#include "qwen3asr.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <vector>
|
||||
|
||||
#include <llama.h>
|
||||
#include <mtmd.h>
|
||||
#include <mtmd-helper.h>
|
||||
|
||||
namespace lc {
|
||||
|
||||
namespace {
|
||||
|
||||
std::string defaultQwenChatPrompt(const std::string & content)
|
||||
{
|
||||
// Qwen3-ASR 的 chat template 就是标准 ChatML（见模型 GGUF 的 tokenizer.chat_template）
|
||||
return "<|im_start|>user\n" + content + "<|im_end|>\n<|im_start|>assistant\n";
|
||||
}
|
||||
|
||||
void silentLog(ggml_log_level /*level*/, const char * /*text*/, void * /*user*/) {}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct Qwen3Asr::Impl {
|
||||
AsrOptions opt;
|
||||
llama_model * model = nullptr;
|
||||
llama_context * ctx = nullptr;
|
||||
mtmd_context * mctx = nullptr;
|
||||
const llama_vocab * vocab = nullptr;
|
||||
int audioRate = 16000;
|
||||
|
||||
// 流式（增量）状态
|
||||
bool streamReady = false;
|
||||
std::vector<llama_token> headTokens; // 音频之前的 prompt 文本
|
||||
std::vector<llama_token> tailTokens; // 音频之后的 prompt 文本（含 assistant 头）
|
||||
std::vector<float> streamAudio; // 当前这段的全部音频（不滑动）
|
||||
size_t stableSamples = 0; // 已固化进 KV 的音频样本数（8s 对齐）
|
||||
llama_pos nPastStable = 0; // stable 音频末尾的 KV 位置
|
||||
|
||||
std::string buildPrompt(const std::string & marker) const
|
||||
{
|
||||
const char * tmpl = llama_model_chat_template(model, nullptr);
|
||||
if (tmpl) {
|
||||
llama_chat_message msg{"user", marker.c_str()};
|
||||
std::vector<char> buf(marker.size() * 2 + 4096);
|
||||
const int32_t n = llama_chat_apply_template(tmpl, &msg, 1, true, buf.data(), (int32_t) buf.size());
|
||||
if (n > 0 && (size_t) n <= buf.size()) {
|
||||
return std::string(buf.data(), (size_t) n);
|
||||
}
|
||||
}
|
||||
return defaultQwenChatPrompt(marker);
|
||||
}
|
||||
|
||||
// 用显式位置解码一批 token（便于在增量流式里精确控制位置）
|
||||
bool decodeTokens(const llama_token * tokens, int32_t n, llama_pos startPos, bool wantLogits)
|
||||
{
|
||||
if (n <= 0) return true;
|
||||
llama_batch b = llama_batch_init(n, 0, 1);
|
||||
for (int32_t i = 0; i < n; ++i) {
|
||||
b.token[i] = tokens[i];
|
||||
b.pos[i] = startPos + i;
|
||||
b.n_seq_id[i] = 1;
|
||||
b.seq_id[i][0] = 0;
|
||||
b.logits[i] = (wantLogits && i == n - 1) ? 1 : 0;
|
||||
}
|
||||
b.n_tokens = n;
|
||||
const int32_t rc = llama_decode(ctx, b);
|
||||
llama_batch_free(b);
|
||||
return rc == 0;
|
||||
}
|
||||
|
||||
// 把一段音频编码后追加到 KV 的 nPast 处
|
||||
bool appendAudio(const float * pcm, size_t nSamples, llama_pos & nPast, std::string * err)
|
||||
{
|
||||
if (nSamples == 0) return true;
|
||||
|
||||
mtmd_bitmap * bmp = mtmd_bitmap_init_from_audio(nSamples, pcm);
|
||||
if (!bmp) { if (err) *err = "构造音频 bitmap 失败"; return false; }
|
||||
|
||||
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
|
||||
const std::string marker = mtmd_default_marker();
|
||||
const mtmd_input_text itext{ marker.c_str(), false, true };
|
||||
const mtmd_bitmap * bms[1] = { bmp };
|
||||
|
||||
if (mtmd_tokenize(mctx, chunks, &itext, bms, 1) != 0) {
|
||||
mtmd_bitmap_free(bmp);
|
||||
mtmd_input_chunks_free(chunks);
|
||||
if (err) *err = "音频 mtmd_tokenize 失败";
|
||||
return false;
|
||||
}
|
||||
|
||||
const mtmd_input_chunk * audioChunk = nullptr;
|
||||
for (size_t i = 0; i < mtmd_input_chunks_size(chunks); ++i) {
|
||||
const mtmd_input_chunk * c = mtmd_input_chunks_get(chunks, i);
|
||||
if (mtmd_input_chunk_get_type(c) == MTMD_INPUT_CHUNK_TYPE_AUDIO) { audioChunk = c; break; }
|
||||
}
|
||||
if (!audioChunk) {
|
||||
mtmd_bitmap_free(bmp);
|
||||
mtmd_input_chunks_free(chunks);
|
||||
if (err) *err = "tokenize 结果里没有音频块";
|
||||
return false;
|
||||
}
|
||||
|
||||
llama_pos np = nPast;
|
||||
const int32_t rc = mtmd_helper_eval_chunk_single(mctx, ctx, audioChunk, nPast, /*seq_id*/ 0,
|
||||
opt.nBatch, /*logits_last*/ false, &np);
|
||||
mtmd_bitmap_free(bmp);
|
||||
mtmd_input_chunks_free(chunks);
|
||||
if (rc != 0) { if (err) *err = "音频编码/解码失败"; return false; }
|
||||
|
||||
nPast = np;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
Qwen3Asr::Qwen3Asr() : d(new Impl) {}
|
||||
|
||||
Qwen3Asr::~Qwen3Asr()
|
||||
{
|
||||
if (!d) return;
|
||||
if (d->mctx) mtmd_free(d->mctx);
|
||||
if (d->ctx) llama_free(d->ctx);
|
||||
if (d->model) llama_model_free(d->model);
|
||||
delete d;
|
||||
}
|
||||
|
||||
int Qwen3Asr::sampleRate() const { return d->audioRate; }
|
||||
|
||||
bool Qwen3Asr::load(const AsrOptions & opt, std::string * error)
|
||||
{
|
||||
const auto fail = [error](const std::string & msg) {
|
||||
if (error) *error = msg;
|
||||
return false;
|
||||
};
|
||||
|
||||
d->opt = opt;
|
||||
|
||||
if (!opt.verbose) {
|
||||
llama_log_set(silentLog, nullptr);
|
||||
mtmd_log_set(silentLog, nullptr);
|
||||
}
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
llama_model_params mparams = llama_model_default_params();
|
||||
mparams.n_gpu_layers = 0; // 本机无独显，强制 CPU
|
||||
mparams.use_mmap = true;
|
||||
|
||||
d->model = llama_model_load_from_file(opt.modelPath.c_str(), mparams);
|
||||
if (!d->model) {
|
||||
return fail("加载主模型失败: " + opt.modelPath);
|
||||
}
|
||||
d->vocab = llama_model_get_vocab(d->model);
|
||||
|
||||
llama_context_params cparams = llama_context_default_params();
|
||||
cparams.n_ctx = (uint32_t) opt.nCtx;
|
||||
cparams.n_batch = (uint32_t) opt.nBatch;
|
||||
cparams.n_ubatch = (uint32_t) opt.nUbatch;
|
||||
cparams.n_threads = opt.threads;
|
||||
cparams.n_threads_batch = opt.threads;
|
||||
|
||||
d->ctx = llama_init_from_model(d->model, cparams);
|
||||
if (!d->ctx) {
|
||||
return fail("创建 llama 上下文失败");
|
||||
}
|
||||
|
||||
mtmd_context_params mctxParams = mtmd_context_params_default();
|
||||
mctxParams.use_gpu = false;
|
||||
mctxParams.n_threads = opt.threads;
|
||||
mctxParams.print_timings = false;
|
||||
mctxParams.warmup = opt.warmup;
|
||||
|
||||
d->mctx = mtmd_init_from_file(opt.mmprojPath.c_str(), d->model, mctxParams);
|
||||
if (!d->mctx) {
|
||||
return fail("加载 mmproj 失败: " + opt.mmprojPath);
|
||||
}
|
||||
|
||||
if (!mtmd_support_audio(d->mctx)) {
|
||||
return fail("该 mmproj 不支持音频输入（需要 clip.projector_type = qwen3a 的版本）");
|
||||
}
|
||||
|
||||
const int rate = mtmd_get_audio_sample_rate(d->mctx);
|
||||
if (rate > 0) d->audioRate = rate;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string Qwen3Asr::dropLastTokens(const std::string & text, int dropLast) const
|
||||
{
|
||||
if (!d->vocab || dropLast <= 0 || text.empty()) return text;
|
||||
|
||||
std::vector<llama_token> tokens(text.size() + 16);
|
||||
int32_t n = llama_tokenize(d->vocab, text.c_str(), (int32_t) text.size(),
|
||||
tokens.data(), (int32_t) tokens.size(),
|
||||
/*add_special*/ false, /*parse_special*/ true);
|
||||
if (n < 0) { // 缓冲区不够，按返回的所需长度重来
|
||||
tokens.resize((size_t) -n);
|
||||
n = llama_tokenize(d->vocab, text.c_str(), (int32_t) text.size(),
|
||||
tokens.data(), (int32_t) tokens.size(), false, true);
|
||||
if (n < 0) return text;
|
||||
}
|
||||
tokens.resize((size_t) n);
|
||||
if ((int) tokens.size() <= dropLast) return {};
|
||||
|
||||
tokens.resize(tokens.size() - (size_t) dropLast);
|
||||
|
||||
// special=true：让 <asr_text> 这类特殊标记以字面形式保留，前缀才能原样回喂
|
||||
std::string out;
|
||||
char buf[512];
|
||||
for (llama_token id : tokens) {
|
||||
const int32_t m = llama_token_to_piece(d->vocab, id, buf, sizeof(buf), 0, /*special*/ true);
|
||||
if (m > 0) out.append(buf, (size_t) m);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
AsrResult Qwen3Asr::transcribeWithPrefix(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix)
|
||||
{
|
||||
AsrResult res;
|
||||
|
||||
if (!d->model || !d->ctx || !d->mctx) {
|
||||
res.error = "模型未加载";
|
||||
return res;
|
||||
}
|
||||
if (nSamples == 0) {
|
||||
res.error = "音频为空";
|
||||
return res;
|
||||
}
|
||||
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
|
||||
// 每段独立转写，先清空 KV cache
|
||||
llama_memory_clear(llama_get_memory(d->ctx), true);
|
||||
|
||||
const std::string marker = mtmd_default_marker();
|
||||
// 关键：assistant 头之后接上"已生成文本"，让模型从那里继续生成（前缀回滚）
|
||||
const std::string prompt = d->buildPrompt(marker) + prefix;
|
||||
|
||||
mtmd_bitmap * bitmap = mtmd_bitmap_init_from_audio(nSamples, pcm);
|
||||
if (!bitmap) {
|
||||
res.error = "构造音频 bitmap 失败";
|
||||
return res;
|
||||
}
|
||||
|
||||
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
|
||||
const mtmd_input_text text{ prompt.c_str(), /*add_special*/ true, /*parse_special*/ true };
|
||||
const mtmd_bitmap * bitmaps[1] = { bitmap };
|
||||
|
||||
const int32_t tokRes = mtmd_tokenize(d->mctx, chunks, &text, bitmaps, 1);
|
||||
if (tokRes != 0) {
|
||||
mtmd_bitmap_free(bitmap);
|
||||
mtmd_input_chunks_free(chunks);
|
||||
res.error = "mtmd_tokenize 失败 (code " + std::to_string(tokRes) + ")";
|
||||
return res;
|
||||
}
|
||||
|
||||
llama_pos nPast = 0;
|
||||
const int32_t evalRes = mtmd_helper_eval_chunks(
|
||||
d->mctx, d->ctx, chunks,
|
||||
/*n_past*/ 0, /*seq_id*/ 0, /*n_batch*/ d->opt.nBatch,
|
||||
/*logits_last*/ true, &nPast);
|
||||
mtmd_bitmap_free(bitmap);
|
||||
if (evalRes != 0) {
|
||||
mtmd_input_chunks_free(chunks);
|
||||
res.error = "mtmd_helper_eval_chunks 失败 (code " + std::to_string(evalRes) + ")";
|
||||
return res;
|
||||
}
|
||||
mtmd_input_chunks_free(chunks);
|
||||
|
||||
// 贪心解码
|
||||
llama_sampler * smpl = llama_sampler_chain_init(llama_sampler_chain_default_params());
|
||||
llama_sampler_chain_add(smpl, llama_sampler_init_greedy());
|
||||
|
||||
std::string out;
|
||||
for (int i = 0; i < d->opt.maxTokens; ++i) {
|
||||
llama_token id = llama_sampler_sample(smpl, d->ctx, -1);
|
||||
if (llama_vocab_is_eog(d->vocab, id)) break;
|
||||
|
||||
char buf[256];
|
||||
const int32_t n = llama_token_to_piece(d->vocab, id, buf, sizeof(buf), 0, /*special*/ true);
|
||||
if (n > 0) out.append(buf, (size_t) n);
|
||||
++res.tokenCount;
|
||||
|
||||
llama_batch batch = llama_batch_get_one(&id, 1);
|
||||
if (llama_decode(d->ctx, batch) != 0) {
|
||||
res.error = "llama_decode 失败";
|
||||
break;
|
||||
}
|
||||
}
|
||||
llama_sampler_free(smpl);
|
||||
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
res.inferSeconds = std::chrono::duration<double>(t1 - t0).count();
|
||||
res.raw = out;
|
||||
res.transcript = parseAsrOutput(out);
|
||||
res.ok = res.error.empty();
|
||||
return res;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 流式（增量 KV 复用）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
void Qwen3Asr::streamReset()
|
||||
{
|
||||
if (!d->ctx) return;
|
||||
llama_memory_clear(llama_get_memory(d->ctx), true);
|
||||
d->streamReady = false;
|
||||
d->headTokens.clear();
|
||||
d->tailTokens.clear();
|
||||
d->streamAudio.clear();
|
||||
d->stableSamples = 0;
|
||||
d->nPastStable = 0;
|
||||
}
|
||||
|
||||
bool Qwen3Asr::prepareStream(std::string * error)
|
||||
{
|
||||
if (d->streamReady) return true;
|
||||
|
||||
// 用一个占位音频 tokenize 一次，只为切出「音频之前」和「音频之后」的 prompt 文本 token。
|
||||
// 占位音频本身不会被解码。
|
||||
std::vector<float> dummy(1600, 0.0f); // 0.1s
|
||||
mtmd_bitmap * bmp = mtmd_bitmap_init_from_audio(dummy.size(), dummy.data());
|
||||
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
|
||||
|
||||
const std::string marker = mtmd_default_marker();
|
||||
const std::string prompt = d->buildPrompt(marker);
|
||||
const mtmd_input_text itext{ prompt.c_str(), true, true };
|
||||
const mtmd_bitmap * bms[1] = { bmp };
|
||||
|
||||
if (mtmd_tokenize(d->mctx, chunks, &itext, bms, 1) != 0) {
|
||||
mtmd_bitmap_free(bmp);
|
||||
mtmd_input_chunks_free(chunks);
|
||||
if (error) *error = "prepareStream: mtmd_tokenize 失败";
|
||||
return false;
|
||||
}
|
||||
mtmd_bitmap_free(bmp);
|
||||
|
||||
bool seenAudio = false;
|
||||
for (size_t i = 0; i < mtmd_input_chunks_size(chunks); ++i) {
|
||||
const mtmd_input_chunk * c = mtmd_input_chunks_get(chunks, i);
|
||||
if (mtmd_input_chunk_get_type(c) == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
|
||||
seenAudio = true;
|
||||
continue;
|
||||
}
|
||||
size_t n = 0;
|
||||
const llama_token * toks = mtmd_input_chunk_get_tokens_text(c, &n);
|
||||
if (!toks || n == 0) continue;
|
||||
std::vector<llama_token> & dst = seenAudio ? d->tailTokens : d->headTokens;
|
||||
dst.insert(dst.end(), toks, toks + n);
|
||||
}
|
||||
mtmd_input_chunks_free(chunks);
|
||||
|
||||
if (d->headTokens.empty() || d->tailTokens.empty()) {
|
||||
if (error) *error = "prepareStream: 没能切出音频前后的 prompt 文本";
|
||||
return false;
|
||||
}
|
||||
|
||||
llama_memory_clear(llama_get_memory(d->ctx), true);
|
||||
if (!d->decodeTokens(d->headTokens.data(), (int32_t) d->headTokens.size(), 0, false)) {
|
||||
if (error) *error = "prepareStream: 解码 prompt 头部失败";
|
||||
return false;
|
||||
}
|
||||
d->nPastStable = (llama_pos) d->headTokens.size();
|
||||
d->streamReady = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 8 秒对齐的音频切片：这是编码器的注意力窗口（n_window_infer = 800 帧 @100Hz）。
|
||||
// 按这个粒度切，编码结果才和「整段音频一次编码」一致；切小了会明显掉准确率。
|
||||
static constexpr double kSliceSeconds = 8.0;
|
||||
|
||||
AsrStreamStep Qwen3Asr::streamAppend(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix, int maxTokens)
|
||||
{
|
||||
AsrStreamStep res;
|
||||
if (!d->model || !d->ctx || !d->mctx) { res.error = "模型未加载"; return res; }
|
||||
|
||||
std::string err;
|
||||
if (!prepareStream(&err)) { res.error = err; return res; }
|
||||
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
|
||||
d->streamAudio.insert(d->streamAudio.end(), pcm, pcm + nSamples);
|
||||
|
||||
const size_t sliceSamples = (size_t) (kSliceSeconds * d->audioRate);
|
||||
const size_t targetStable = (d->streamAudio.size() / sliceSamples) * sliceSamples;
|
||||
|
||||
const llama_pos nPastBefore = d->nPastStable;
|
||||
|
||||
// 1) 已排满 8s 的切片固化进 KV，之后永远复用
|
||||
if (targetStable > d->stableSamples) {
|
||||
llama_memory_seq_rm(llama_get_memory(d->ctx), 0, d->nPastStable, -1);
|
||||
llama_pos np = d->nPastStable;
|
||||
if (!d->appendAudio(d->streamAudio.data() + d->stableSamples,
|
||||
targetStable - d->stableSamples, np, &err)) {
|
||||
res.error = "固化音频切片失败: " + err;
|
||||
return res;
|
||||
}
|
||||
d->nPastStable = np;
|
||||
d->stableSamples = targetStable;
|
||||
}
|
||||
|
||||
// 2) 尾部不满 8s 的部分：每次重编码，但只是临时的（下一步会被丢掉）
|
||||
llama_memory_seq_rm(llama_get_memory(d->ctx), 0, d->nPastStable, -1);
|
||||
llama_pos nPast = d->nPastStable;
|
||||
if (d->streamAudio.size() > d->stableSamples) {
|
||||
if (!d->appendAudio(d->streamAudio.data() + d->stableSamples,
|
||||
d->streamAudio.size() - d->stableSamples, nPast, &err)) {
|
||||
res.error = "编码尾部音频失败: " + err;
|
||||
return res;
|
||||
}
|
||||
}
|
||||
res.audioTokens = (int) (nPast - nPastBefore);
|
||||
const auto tEnc = std::chrono::steady_clock::now();
|
||||
|
||||
// 3) 音频之后的 prompt 文本 + 前缀
|
||||
std::vector<llama_token> prefixToks;
|
||||
if (!prefix.empty()) {
|
||||
prefixToks.resize(prefix.size() + 16);
|
||||
int32_t n = llama_tokenize(d->vocab, prefix.c_str(), (int32_t) prefix.size(),
|
||||
prefixToks.data(), (int32_t) prefixToks.size(),
|
||||
/*add_special*/ false, /*parse_special*/ true);
|
||||
if (n < 0) {
|
||||
prefixToks.resize((size_t) -n);
|
||||
n = llama_tokenize(d->vocab, prefix.c_str(), (int32_t) prefix.size(),
|
||||
prefixToks.data(), (int32_t) prefixToks.size(), false, true);
|
||||
}
|
||||
if (n < 0) { res.error = "前缀分词失败"; return res; }
|
||||
prefixToks.resize((size_t) n);
|
||||
}
|
||||
|
||||
llama_pos p = nPast;
|
||||
const bool prefixLast = prefixToks.empty();
|
||||
if (!d->decodeTokens(d->tailTokens.data(), (int32_t) d->tailTokens.size(), p, prefixLast)) {
|
||||
res.error = "解码 tail 失败";
|
||||
return res;
|
||||
}
|
||||
p += (llama_pos) d->tailTokens.size();
|
||||
|
||||
if (!prefixToks.empty()) {
|
||||
if (!d->decodeTokens(prefixToks.data(), (int32_t) prefixToks.size(), p, true)) {
|
||||
res.error = "解码前缀失败";
|
||||
return res;
|
||||
}
|
||||
p += (llama_pos) prefixToks.size();
|
||||
}
|
||||
|
||||
// 4) 贪心生成（每个 token 的 logits 都要留给下一次采样）
|
||||
llama_sampler * smpl = llama_sampler_chain_init(llama_sampler_chain_default_params());
|
||||
llama_sampler_chain_add(smpl, llama_sampler_init_greedy());
|
||||
|
||||
std::string out;
|
||||
for (int i = 0; i < maxTokens; ++i) {
|
||||
llama_token id = llama_sampler_sample(smpl, d->ctx, -1);
|
||||
if (llama_vocab_is_eog(d->vocab, id)) break;
|
||||
|
||||
char buf[256];
|
||||
const int32_t n = llama_token_to_piece(d->vocab, id, buf, sizeof(buf), 0, /*special*/ true);
|
||||
if (n > 0) out.append(buf, (size_t) n);
|
||||
++res.newTokens;
|
||||
|
||||
if (!d->decodeTokens(&id, 1, p, true)) { res.error = "生成时解码失败"; break; }
|
||||
++p;
|
||||
}
|
||||
llama_sampler_free(smpl);
|
||||
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
res.encodeSeconds = std::chrono::duration<double>(tEnc - t0).count();
|
||||
res.inferSeconds = std::chrono::duration<double>(t1 - t0).count();
|
||||
res.newText = out;
|
||||
res.ok = res.error.empty();
|
||||
return res;
|
||||
}
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "iasr.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
struct AsrOptions {
|
||||
std::string modelPath; // Qwen3-ASR 主模型 GGUF（架构须为 qwen3vl）
|
||||
std::string mmprojPath; // 音频编码器 mmproj GGUF（clip + qwen3a）
|
||||
int threads = 8;
|
||||
int nCtx = 4096;
|
||||
int nBatch = 4096;
|
||||
int nUbatch = 1024;
|
||||
int maxTokens = 256; // 单段最多生成多少 token
|
||||
bool warmup = false; // mtmd 初始化后是否跑一次预热编码
|
||||
bool verbose = false;
|
||||
};
|
||||
|
||||
// 进程内 Qwen3-ASR 推理（libllama + libmtmd）。
|
||||
// 模型常驻，可反复调用；每次整段转写会清空 KV cache。
|
||||
class Qwen3Asr : public IAsr {
|
||||
public:
|
||||
Qwen3Asr();
|
||||
~Qwen3Asr() override;
|
||||
|
||||
Qwen3Asr(const Qwen3Asr &) = delete;
|
||||
Qwen3Asr & operator=(const Qwen3Asr &) = delete;
|
||||
|
||||
bool load(const AsrOptions & opt, std::string * error);
|
||||
|
||||
// ---- IAsr ----
|
||||
int sampleRate() const override;
|
||||
|
||||
AsrResult transcribeWithPrefix(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix) override;
|
||||
|
||||
AsrStreamStep streamAppend(const float * pcm, size_t nSamples,
|
||||
const std::string & prefix, int maxTokens) override;
|
||||
void streamReset() override;
|
||||
|
||||
std::string dropLastTokens(const std::string & text, int dropLast) const override;
|
||||
|
||||
private:
|
||||
bool prepareStream(std::string * error);
|
||||
|
||||
struct Impl;
|
||||
Impl * d = nullptr;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,175 @@
|
||||
#include "streamingasr.h"
|
||||
|
||||
#include "transcript.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
StreamingAsr::StreamingAsr(IAsr & asr, StreamOptions opt)
|
||||
: asr_(asr), opt_(opt)
|
||||
{
|
||||
asr_.streamReset();
|
||||
}
|
||||
|
||||
void StreamingAsr::reset()
|
||||
{
|
||||
asr_.streamReset();
|
||||
buffer_.clear();
|
||||
pending_.clear();
|
||||
totalSeconds_ = 0.0;
|
||||
committedRaw_.clear();
|
||||
lastRawFull_.clear();
|
||||
language_.clear();
|
||||
chunkIndex_ = 0;
|
||||
}
|
||||
|
||||
void StreamingAsr::resetAll()
|
||||
{
|
||||
reset();
|
||||
historyText_.clear();
|
||||
}
|
||||
|
||||
std::string StreamingAsr::committedText() const
|
||||
{
|
||||
return historyText_ + parseAsrOutput(committedRaw_).text;
|
||||
}
|
||||
|
||||
void StreamingAsr::feed(const float * pcm, size_t n,
|
||||
const std::function<void(const Step &)> & onStep)
|
||||
{
|
||||
pending_.insert(pending_.end(), pcm, pcm + n);
|
||||
|
||||
const size_t chunkSamples = (size_t) (opt_.chunkSec * asr_.sampleRate());
|
||||
if (chunkSamples == 0) return;
|
||||
|
||||
while (pending_.size() >= chunkSamples) {
|
||||
std::vector<float> chunk(pending_.begin(), pending_.begin() + (long) chunkSamples);
|
||||
pending_.erase(pending_.begin(), pending_.begin() + (long) chunkSamples);
|
||||
runStep(chunk, false, onStep);
|
||||
}
|
||||
}
|
||||
|
||||
void StreamingAsr::flush(const std::function<void(const Step &)> & onStep)
|
||||
{
|
||||
if (!pending_.empty()) {
|
||||
std::vector<float> chunk;
|
||||
chunk.swap(pending_);
|
||||
// 流结束：没有更多音频可以修正尾部了，这一步全部提交
|
||||
runStep(chunk, true, onStep);
|
||||
return;
|
||||
}
|
||||
|
||||
// 没有待处理音频：直接把这一段提交掉。
|
||||
// 注意要用 lastRawFull_（含不稳定尾部）而不是 committedRaw_：
|
||||
// 段恰好结束在 chunk 边界时，否则最后几个词会被丢掉。
|
||||
const std::string full = lastRawFull_.empty() ? committedRaw_ : lastRawFull_;
|
||||
if (full.empty()) return;
|
||||
|
||||
historyText_ += parseAsrOutput(full).text;
|
||||
committedRaw_.clear();
|
||||
lastRawFull_.clear();
|
||||
|
||||
Step s;
|
||||
s.chunkIndex = chunkIndex_;
|
||||
s.language = language_;
|
||||
s.ok = true;
|
||||
s.full = historyText_;
|
||||
s.committed = historyText_;
|
||||
if (onStep) onStep(s);
|
||||
|
||||
// 收束这一段
|
||||
asr_.streamReset();
|
||||
buffer_.clear();
|
||||
totalSeconds_ = 0.0;
|
||||
chunkIndex_ = 0;
|
||||
}
|
||||
|
||||
void StreamingAsr::runStep(const std::vector<float> & audio, bool finalStep,
|
||||
const std::function<void(const Step &)> & onStep)
|
||||
{
|
||||
// 冷启动阶段不给前缀；之后把"已稳定文本"回喂给模型
|
||||
const std::string prefix =
|
||||
(chunkIndex_ >= opt_.unfixedChunkNum) ? committedRaw_ : std::string();
|
||||
|
||||
Step s;
|
||||
s.chunkIndex = chunkIndex_;
|
||||
|
||||
std::string newText;
|
||||
if (opt_.incremental) {
|
||||
// 只把这一块交给增量引擎；音频总长不影响单步成本
|
||||
AsrStreamStep r =
|
||||
asr_.streamAppend(audio.data(), audio.size(), prefix, opt_.maxTokensPerStep);
|
||||
s.ok = r.ok;
|
||||
s.error = r.error;
|
||||
s.encodeSeconds = r.encodeSeconds;
|
||||
s.inferSeconds = r.inferSeconds;
|
||||
s.audioTokens = r.audioTokens;
|
||||
newText = r.newText;
|
||||
totalSeconds_ += (double) audio.size() / asr_.sampleRate();
|
||||
s.windowSeconds = totalSeconds_;
|
||||
} else {
|
||||
// window 模式：累积到 buffer，滑动窗口后重编码整个窗口
|
||||
buffer_.insert(buffer_.end(), audio.begin(), audio.end());
|
||||
if (opt_.windowSec > 0) {
|
||||
const size_t maxSamples = (size_t) (opt_.windowSec * asr_.sampleRate());
|
||||
if (buffer_.size() > maxSamples) {
|
||||
buffer_.erase(buffer_.begin(),
|
||||
buffer_.begin() + (long) (buffer_.size() - maxSamples));
|
||||
}
|
||||
}
|
||||
AsrResult r =
|
||||
asr_.transcribeWithPrefix(buffer_.data(), buffer_.size(), prefix);
|
||||
s.ok = r.ok;
|
||||
s.error = r.error;
|
||||
s.inferSeconds = r.inferSeconds;
|
||||
s.windowSeconds = (double) buffer_.size() / asr_.sampleRate();
|
||||
newText = r.raw;
|
||||
}
|
||||
|
||||
if (s.ok) {
|
||||
// 拼接要用未经 trim 的文本：流式续写时开头那个空格是有意义的
|
||||
s.rawFull = prefix + newText;
|
||||
lastRawFull_ = s.rawFull;
|
||||
|
||||
const ParsedTranscript pSeg = parseAsrOutput(s.rawFull);
|
||||
if (!pSeg.language.empty()) language_ = pSeg.language;
|
||||
|
||||
// 这一段是否到此为止（流结束，或超过单段时长上限）
|
||||
const bool endSegment =
|
||||
finalStep || (opt_.maxUtteranceSec > 0 && totalSeconds_ >= opt_.maxUtteranceSec);
|
||||
|
||||
if (endSegment) {
|
||||
// 没有更多音频可以修正尾部了，全部提交，并把正文并入历史
|
||||
committedRaw_ = s.rawFull;
|
||||
historyText_ += pSeg.text;
|
||||
s.full = historyText_;
|
||||
s.committed = historyText_;
|
||||
s.tail.clear();
|
||||
} else {
|
||||
// 最后 unfixedTokenNum 个 token 视为不稳定，不提交也不回喂
|
||||
committedRaw_ = asr_.dropLastTokens(s.rawFull, opt_.unfixedTokenNum);
|
||||
const std::string committedSeg = parseAsrOutput(committedRaw_).text;
|
||||
s.full = historyText_ + pSeg.text;
|
||||
s.committed = historyText_ + committedSeg;
|
||||
// 不稳定尾部 = 本段文本去掉已稳定的那部分
|
||||
s.tail = s.full.size() > s.committed.size()
|
||||
? s.full.substr(s.committed.size())
|
||||
: std::string();
|
||||
}
|
||||
s.language = language_;
|
||||
}
|
||||
|
||||
++chunkIndex_;
|
||||
if (onStep) onStep(s);
|
||||
|
||||
if (s.ok && !finalStep &&
|
||||
opt_.maxUtteranceSec > 0 && totalSeconds_ >= opt_.maxUtteranceSec) {
|
||||
// 收束这一段：保留历史正文，清空 KV 重新开始
|
||||
asr_.streamReset();
|
||||
buffer_.clear();
|
||||
totalSeconds_ = 0.0;
|
||||
committedRaw_.clear();
|
||||
chunkIndex_ = 0;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "iasr.h"
|
||||
|
||||
namespace lc {
|
||||
|
||||
// 官方流式解码的参数（见 docs/phase2-streaming.md）
|
||||
struct StreamOptions {
|
||||
double chunkSec = 2.0; // 累积多少秒音频跑一步
|
||||
double windowSec = 16.0; // 仅 window 模式使用：滑窗上限，<=0 表示不滑窗
|
||||
int unfixedTokenNum = 5; // 前缀回滚：丢掉最后 N 个 token
|
||||
int unfixedChunkNum = 2; // 前 N 个 chunk 不带前缀（冷启动）
|
||||
int maxTokensPerStep = 256;
|
||||
|
||||
// true = 增量 KV 复用：只编码新音频，已解码音频留在 KV（推荐）
|
||||
// false = 每次重编码整个窗口（Phase 2 早期的做法，仅用于对比）
|
||||
bool incremental = true;
|
||||
|
||||
// 单段最长时长（秒）：超过就提交并重置上下文（0 = 不限）。
|
||||
// 真实场景应由静音检测决定分段；这里是一个兜底上界，
|
||||
// 因为解码器对 KV 的注意力随总长增长，段太长单步会超预算。
|
||||
double maxUtteranceSec = 30.0;
|
||||
};
|
||||
|
||||
// 把「累积音频 → 带前缀继续生成 → 前缀回滚」这套策略封装起来。
|
||||
// 输出分两部分：
|
||||
// committed —— 已稳定文本（就是下一步要回喂的前缀）
|
||||
// full —— 含不稳定尾部（最后 unfixedTokenNum 个 token）
|
||||
class StreamingAsr {
|
||||
public:
|
||||
struct Step {
|
||||
int chunkIndex = 0;
|
||||
std::string language;
|
||||
std::string committed; // 稳定部分
|
||||
std::string tail; // 不稳定尾部（committed 之后的部分）
|
||||
std::string full; // 含不稳定尾部
|
||||
std::string rawFull; // 未 trim 的完整文本（含 language 头）
|
||||
double windowSeconds = 0.0;
|
||||
double encodeSeconds = 0.0;
|
||||
double inferSeconds = 0.0;
|
||||
int audioTokens = 0;
|
||||
bool ok = true;
|
||||
std::string error;
|
||||
};
|
||||
|
||||
StreamingAsr(IAsr & asr, StreamOptions opt = {});
|
||||
|
||||
// 喂入 16k 单声道音频；每凑够 chunkSec 就跑一步并回调
|
||||
void feed(const float * pcm, size_t n, const std::function<void(const Step &)> & onStep);
|
||||
// 处理不足一块的残留音频，并作为最后一步提交全部文本。
|
||||
// 若没有待处理音频，也把当前已稳定文本提交为最终（静音分段时会用到）。
|
||||
void flush(const std::function<void(const Step &)> & onStep);
|
||||
|
||||
// 结束当前这段（例如检测到静音）：清空 KV 与状态，下一段重新开始
|
||||
void reset();
|
||||
// 重置全部状态（含历史文本）
|
||||
void resetAll();
|
||||
|
||||
// 全部已提交文本（含此前各段，已剥离 language 头）
|
||||
std::string committedText() const;
|
||||
const std::string & language() const { return language_; }
|
||||
int chunkCount() const { return chunkIndex_; }
|
||||
|
||||
private:
|
||||
void runStep(const std::vector<float> & audio, bool finalStep,
|
||||
const std::function<void(const Step &)> & onStep);
|
||||
|
||||
IAsr & asr_;
|
||||
StreamOptions opt_;
|
||||
std::vector<float> buffer_; // window 模式：当前窗口音频
|
||||
std::vector<float> pending_; // 尚未凑够一块的音频
|
||||
double totalSeconds_ = 0.0;
|
||||
std::string committedRaw_; // 当前段的原始已提交文本（含 language 头，用于回喂）
|
||||
std::string lastRawFull_; // 上一次完整的原始文本（含不稳定尾部）
|
||||
std::string historyText_; // 此前各段已提交的正文（已剥离 language 头）
|
||||
std::string language_;
|
||||
int chunkIndex_ = 0;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,79 @@
|
||||
#include "transcript.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
|
||||
namespace lc {
|
||||
|
||||
namespace {
|
||||
|
||||
std::string trim(const std::string & s)
|
||||
{
|
||||
const auto notSpace = [](unsigned char c) { return !std::isspace(c); };
|
||||
auto begin = std::find_if(s.begin(), s.end(), notSpace);
|
||||
auto end = std::find_if(s.rbegin(), s.rend(), notSpace).base();
|
||||
if (begin >= end) return {};
|
||||
return std::string(begin, end);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ParsedTranscript parseAsrOutput(const std::string & raw)
|
||||
{
|
||||
ParsedTranscript out;
|
||||
|
||||
static const std::string kMarker = "<asr_text>";
|
||||
const std::string trimmed = trim(raw);
|
||||
|
||||
const auto markerPos = trimmed.rfind(kMarker);
|
||||
std::string head;
|
||||
if (markerPos != std::string::npos) {
|
||||
head = trimmed.substr(0, markerPos);
|
||||
out.text = trimmed.substr(markerPos + kMarker.size());
|
||||
} else {
|
||||
head = trimmed;
|
||||
out.text = trimmed;
|
||||
}
|
||||
|
||||
// 头部里找 "language <Name>"
|
||||
// 允许前面有空白，语言名只取连续的字母
|
||||
const std::string key = "language";
|
||||
auto pos = head.find(key);
|
||||
if (pos != std::string::npos) {
|
||||
pos += key.size();
|
||||
while (pos < head.size() && std::isspace(static_cast<unsigned char>(head[pos]))) ++pos;
|
||||
size_t end = pos;
|
||||
while (end < head.size() && std::isalpha(static_cast<unsigned char>(head[end]))) ++end;
|
||||
out.language = head.substr(pos, end - pos);
|
||||
}
|
||||
|
||||
out.text = trim(out.text);
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string languageDisplayName(const std::string & lang)
|
||||
{
|
||||
if (lang.empty()) return {};
|
||||
|
||||
static const struct { const char * key; const char * display; } kMap[] = {
|
||||
{ "English", "英语" },
|
||||
{ "Chinese", "中文" },
|
||||
{ "Japanese", "日语" },
|
||||
{ "Korean", "韩语" },
|
||||
{ "French", "法语" },
|
||||
{ "German", "德语" },
|
||||
{ "Spanish", "西班牙语" },
|
||||
{ "Russian", "俄语" },
|
||||
{ "Italian", "意大利语" },
|
||||
{ "Portuguese", "葡萄牙语" },
|
||||
{ "Arabic", "阿拉伯语" },
|
||||
{ "Cantonese", "粤语" },
|
||||
};
|
||||
|
||||
for (const auto & e : kMap) {
|
||||
if (lang == e.key) return e.display;
|
||||
}
|
||||
return lang;
|
||||
}
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,20 @@
|
||||
#pragma once
|
||||
#include <string>
|
||||
|
||||
namespace lc {
|
||||
|
||||
// Qwen3-ASR 原始输出形如：
|
||||
// language English<asr_text>He hoped there would be stew for dinner...
|
||||
// language Chinese<asr_text>不排除购房者可以在互联网上下单。
|
||||
// 本结构把语言名和正文分开：语言名给 UI 的语言标签，正文才上屏。
|
||||
struct ParsedTranscript {
|
||||
std::string language; // 原始语言名，如 "English" / "Chinese"；无则空
|
||||
std::string text; // 转写正文
|
||||
};
|
||||
|
||||
ParsedTranscript parseAsrOutput(const std::string & raw);
|
||||
|
||||
// "English" -> "英语"，"Chinese" -> "中文"；未知则原样返回
|
||||
std::string languageDisplayName(const std::string & lang);
|
||||
|
||||
} // namespace lc
|
||||
@@ -0,0 +1,43 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
|
||||
namespace lc {
|
||||
|
||||
// 能量 VAD。
|
||||
//
|
||||
// 噪声底必须用"快降慢升"的**最小值跟踪**，不能用均值：
|
||||
// 否则遇到本身就很轻的音频（例如峰值 RMS 只有 0.03 的录音），
|
||||
// 噪声底会一路抬升、阈值超过语音电平，把绝大部分话音都判成静音。
|
||||
class Vad {
|
||||
public:
|
||||
explicit Vad(float floorInit = 0.01f, float factor = 3.0f, float minThreshold = 0.0015f)
|
||||
: floor_(floorInit), factor_(factor), minThreshold_(minThreshold) {}
|
||||
|
||||
// 返回该帧是否为语音
|
||||
bool feed(const float * frame, size_t n)
|
||||
{
|
||||
double acc = 0.0;
|
||||
for (size_t i = 0; i < n; ++i) acc += (double) frame[i] * frame[i];
|
||||
const float rms = std::sqrt(acc / (double) n);
|
||||
|
||||
if (rms < floor_) floor_ = rms; // 快速下探
|
||||
else floor_ = floor_ * 0.9999f + rms * 0.0001f; // 极慢回升
|
||||
|
||||
lastRms_ = rms;
|
||||
return rms > std::max(floor_ * factor_, minThreshold_);
|
||||
}
|
||||
|
||||
float noiseFloor() const { return floor_; }
|
||||
float lastRms() const { return lastRms_; }
|
||||
|
||||
private:
|
||||
float floor_;
|
||||
float factor_;
|
||||
float minThreshold_;
|
||||
float lastRms_ = 0.0f;
|
||||
};
|
||||
|
||||
} // namespace lc
|
||||
Reference in new issue
Block a user