patrick/Vosklet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More testing

Browse Source
This commit is contained in:
msqr1
2024-03-25 23:01:24 -07:00
parent 1340e1430f
commit df6b240bde
12 changed files with 6150 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/bindings.cc
View File

@@ -21,6 +21,7 @@ EMSCRIPTEN_BINDINGS() {
.constructor<int, float, genericModel*>(allow_raw_pointers())
.constructor<int, float, genericModel*, genericModel*>(allow_raw_pointers())
.constructor<int, float, genericModel*, std::string, int>(allow_raw_pointers())
.function("acceptWaveform", &recognizer::acceptWaveform, allow_raw_pointers())
.function("reset", &recognizer::reset, allow_raw_pointers())
.function("setEndpointerMode", &recognizer::setEndpointerMode, allow_raw_pointers())
.function("setEndpointerDelays", &recognizer::setEndpointerDelays, allow_raw_pointers())

2
src/genericModel.cc
View File

@@ -49,7 +49,7 @@ void genericModel::extractAndLoad(int tarStart, int tarSize) {
archive_read_free(src);
if(normalMdl) mdl = vosk_model_new(storepath.c_str());
else vosk_spk_model_new(storepath.c_str());
emscripten_console_log("Loading finished!");
emscripten_console_log("Model loaded!");
if(normalMdl ? std::get<0>(mdl) == nullptr : std::get<1>(mdl) == nullptr) fireEv(index, "Unable to load model for recognition");
else fireEv(index, "0");
};

11
src/link.cc
View File

@@ -1,6 +1,6 @@
#include "link.h"
pthread_t dstThrd{pthread_self()};
auto dstThrd = pthread_self();
ProxyingQueue glbQ{};
void fireEv(int index, const char* content, const char* type) {
auto proxy{[index, content, type](){
@@ -11,12 +11,3 @@ void fireEv(int index, const char* content, const char* type) {
if(dstThrd == pthread_self()) proxy();
else glbQ.proxySync(dstThrd, proxy);
}
void fireEv(int index, std::atomic_int* state, float* dataBuf) {
auto proxy{[index, state, dataBuf](){
EM_ASM({
objs[$0].dispatchEvent(new CustomEvent("0", { "detail" : $1 + "," + $2}));
}, index, state, dataBuf);
}};
if(dstThrd == pthread_self()) proxy();
else glbQ.proxySync(dstThrd, proxy);
}

3
src/link.h
View File

@@ -1,7 +1,8 @@
#pragma once
#include <thread>
#include <emscripten/em_asm.h>
#include <emscripten/proxying.h>
using namespace emscripten;
void fireEv(int index, const char* content, const char* type = nullptr); // Normal
void fireEv(int index, std::atomic_int* state, float* dataBuf); // For recognizer's success initialization

41
src/pre.js
View File

@@ -29,13 +29,9 @@ class genericModel extends EventTarget {
let mdl = new genericModel(url, storepath, id, normalMdl)
let result = new Promise((resolve, reject) => {
mdl.addEventListener("0", ev => {
switch(ev.detail) {
case "0":
return resolve(mdl)
default:
mdl.delete()
reject(ev.detail)
}
if(ev.detail === "0") return resolve(mdl)
mdl.delete()
reject(ev.detail)
}, { once : true })
})
let tar
@@ -93,12 +89,7 @@ class Recognizer extends EventTarget {
let rec = new Recognizer()
let result = new Promise((resolve, reject) => {
rec.addEventListener("0", ev => {
if(ev.detail.indexOf(",") !== -1) {
let loadInfo = ev.detail.split(",")
rec.state = Module.HEAP32.subarray(parseInt(loadInfo[0]), parseInt(loadInfo[0]) + 1) // State is an array with 1 element, there is no other way to get a reference to a single element
rec.dataBuf = Module.HEAPF32.subarray(parseInt(loadInfo[1]), parseInt(loadInfo[1]) + 128)
return resolve(rec)
}
if(ev.detail === "0") return resolve(rec)
rec.delete()
reject(ev.detail)
}, { once : true })
@@ -115,16 +106,15 @@ class Recognizer extends EventTarget {
}
return result
}
async getNode(ctx, channelIndex = 0) {
async getNode(ctx) {
if(typeof this.node === "undefined") {
await ctx.audioWorklet.addModule(processorUrl)
this.node = new AudioWorkletNode(ctx, 'VoskletProcessor', { channelCountMode: "max", numberOfInputs: 1, numberOfOutputs: 0, processorOptions: { dataBuf: this.dataBuf, state: this.state, channel: channelIndex }})
await ctx.audioWorklet.addModule("../src/processor.js", { credentials : "omit"})
this.node = new AudioWorkletNode(ctx, 'VoskletProcessor', { channelCountMode: "explicit", channelCount: 1, numberOfInputs: 1, numberOfOutputs: 1, processorOptions: { dataBuf: this.dataBuf, state: this.state }})
}
return this.node
}
recognize(buf, channelIndex = 0) {
Module.HEAPF32.set(buf.getChannelData(channelIndex).subarray(0, 512), this.ptr)
this.obj.acceptWaveForm()
recognize(buf) {
Module.HEAPF32.set(buf.getChannelData(0).subarray(0, 512), this.ptr)
}
delete() {
if (this.obj) this.obj.delete()
@@ -158,7 +148,7 @@ Module.makeRecognizerWithSpkModel = (model, sampleRate, spkModel) => {
Module.makeRecognizerWithGrm = (model, sampleRate, grammar) => {
return Recognizer._init(model.obj, sampleRate, 3, grammar, null)
}
let processorUrl = URL.createObjectURL(new Blob(['(',
/*let processorURL = URL.createObjectURL(new Blob(['(',
(() => {
registerProcessor("VoskletProcessor", class extends AudioWorkletProcessor {
constructor(options) {
@@ -167,15 +157,14 @@ let processorUrl = URL.createObjectURL(new Blob(['(',
this.state = options.processorOptions.state
}
process(inputs, outputs, params) {
Atomics.wait(state, 0)
inputs.copyFromChannel(this.dataBuf, this.channelIndex)
while(state[0])
inputs.copyFromChannel(this.dataBuf, 0)
return true
}
})
}).toString()
, ')()'], {type : "text/javascript"}))
/*let pthreadUrl = URL.createObjectURL(new Blob(['(',
}).toString(),
')()'], {type : "text/javascript"}))
let pthreadURL = URL.createObjectURL(new Blob(['(',
(() => {
{ PTHREAD_SCRIPT }
}).toString()

25
src/recognizer.cc
View File

@@ -1,39 +1,36 @@
#include "recognizer.h"
recognizer::recognizer(int index, float sampleRate, genericModel* model) : index{index}, sampleRate{sampleRate}, rec{vosk_recognizer_new(std::get<0>(model->mdl),sampleRate)} {
recognizer::recognizer(int index, float sampleRate, genericModel* model) : index{index}, rec{vosk_recognizer_new(std::get<0>(model->mdl),sampleRate)} {
finishConstruction(model);
}
recognizer::recognizer(int index, float sampleRate, genericModel* model, genericModel* spkModel) : index(index), sampleRate{sampleRate}, rec{vosk_recognizer_new_spk(std::get<0>(model->mdl), sampleRate, std::get<1>(spkModel->mdl))} {
recognizer::recognizer(int index, float sampleRate, genericModel* model, genericModel* spkModel) : index{index}, rec{vosk_recognizer_new_spk(std::get<0>(model->mdl), sampleRate, std::get<1>(spkModel->mdl))} {
finishConstruction(model, spkModel);
}
recognizer::recognizer(int index, float sampleRate, genericModel* model, const std::string& grm, int dummy) : index{index}, sampleRate{sampleRate}, rec{vosk_recognizer_new_grm(std::get<0>(model->mdl), sampleRate, grm.c_str())} {
recognizer::recognizer(int index, float sampleRate, genericModel* model, const std::string& grm, int dummy) : index{index}, rec{vosk_recognizer_new_grm(std::get<0>(model->mdl), sampleRate, grm.c_str())} {
finishConstruction(model);
}
recognizer::~recognizer() {
done = true;
vosk_recognizer_free(rec);
}
void recognizer::reset() {
vosk_recognizer_reset(rec);
}
void recognizer::finishConstruction(genericModel* model, genericModel* spkModel) {
if(rec == nullptr) {
fireEv(index, "Unable to initialize recognizer");
return;
}
auto main {[this](){
emscripten_console_log("Recognizer loaded!");
fireEv(index, &state, dataBuf);
while(!done) {
switch(vosk_recognizer_accept_waveform_f(rec, dataBuf, 512)) {
while(!dataQ.empty()) {
switch(vosk_recognizer_accept_waveform_f(rec, dataQ.front().data, dataQ.front().len)) {
case 0:
fireEv(index, vosk_recognizer_result(rec), "result");
break;
case 1:
fireEv(index, vosk_recognizer_partial_result(rec), "partialResult");
}
free(dataQ.front().data);
dataQ.pop();
}
state = 0;
state.wait(0, std::memory_order_relaxed);
}
}};
if(!model->resourceUsed) {
@@ -54,6 +51,12 @@ void recognizer::finishConstruction(genericModel* model, genericModel* spkModel)
std::thread t{main};
t.detach();
}
void recognizer::acceptWaveform(int start, int len) {
dataQ.emplace(start, len);
}
void recognizer::reset() {
vosk_recognizer_reset(rec);
}
void recognizer::setEndpointerMode(VoskEndpointerMode mode) {
vosk_recognizer_set_endpointer_mode(rec, mode);
}

13
src/recognizer.h
View File

@@ -1,12 +1,16 @@
#pragma once
#include "genericModel.h"
#include <condition_variable>
#include <condition_variable>
#include <queue>
struct audioData {
float* data;
int len;
audioData(int start, int len) : data{reinterpret_cast<float*>(start)}, len{len} {}
};
struct recognizer {
std::atomic_bool done;
std::atomic_int state; // 0: Copying data from JS, 1: Processing from C++
float dataBuf[128];
float sampleRate;
std::queue<audioData> dataQ{};
int index;
VoskRecognizer* rec;
recognizer(int index, float sampleRate, genericModel* model);
@@ -14,6 +18,7 @@ struct recognizer {
recognizer(int index, float sampleRate, genericModel* model, const std::string& grm, int dummy);
~recognizer();
void finishConstruction(genericModel* model, genericModel* spkModel = nullptr);
void acceptWaveform(int start, int len);
void reset();
void setEndpointerMode(VoskEndpointerMode mode);
void setEndpointerDelays(float tStartMax, float tEnd, float tMax);