patrick/Vosklet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Large commit

Browse Source
This commit is contained in:
msqr1
2024-03-24 00:58:13 -07:00
parent 638de69e67
commit 8bb9733cd1
14 changed files with 4203 additions and 4966 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/Makefile
View File

@@ -20,10 +20,9 @@ CLAPACK_WASM:=$(realpath clapack-wasm)
Vosklet.js: | vosk libarchive
cd $(SRC) && \
em++ -O3 global.cc genericModel.cc model.cc spkModel.cc recognizer.cc bindings.cc -sWASMFS \ -sWASM_BIGINT -sSINGLE_FILE -sMODULARIZE -sEMBIND_STD_STRING_IS_UTF8 -sPTHREAD_POOL_DELAY_LOAD -sTEXTDECODER=2 -sPTHREAD_POOL_SIZE_STRICT=2 -sINITIAL_MEMORY=$(MAX_MEMORY) -sPTHREAD_POOL_SIZE=$(MAX_THREADS) -sPOLYFILL=0 -sSUPPORT_LONGJMP=0 -sEXPORTED_FUNCTIONS=_malloc -sEXPORT_NAME=loadVosklet -sMALLOC=emmalloc -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8OnStack -sENVIRONMENT=web,worker -I. -I$(LIBARCHIVE)/include -I$(VOSK)/src -L$(LIBARCHIVE)/lib -larchive -L$(KALDI)/src -l:online2/kaldi-online2.a -l:decoder/kaldi-decoder.a -l:ivector/kaldi-ivector.a -l:gmm/kaldi-gmm.a -l:tree/kaldi-tree.a -l:feat/kaldi-feat.a -l:cudamatrix/kaldi-cudamatrix.a -l:lat/kaldi-lat.a -l:lm/kaldi-lm.a -l:rnnlm/kaldi-rnnlm.a -l:hmm/kaldi-hmm.a -l:nnet3/kaldi-nnet3.a -l:transform/kaldi-transform.a -l:matrix/kaldi-matrix.a -l:fstext/kaldi-fstext.a -l:util/kaldi-util.a -l:base/kaldi-base.a -L$(OPENFST)/lib -l:libfst.a -l:libfstngram.a -L$(CLAPACK_WASM) -l:CBLAS/lib/cblas.a -l:CLAPACK-3.2.1/lapack.a -l:CLAPACK-3.2.1/libcblaswr.a -l:f2c_BLAS-3.8.0/blas.a -l:libf2c/libf2c.a -L$(VOSK)/src -l:vosk.a -lopfs.js -lembind -pthread -flto -msimd128 --pre-js pre.js -o ../Vosklet.js && \
em++ -O3 proxier.cc genericModel.cc recognizer.cc bindings.cc -sWASMFS \ -sWASM_BIGINT -sSINGLE_FILE -sGZIP_EMBEDDINGS -sMODULARIZE -sEMBIND_STD_STRING_IS_UTF8 -sPTHREAD_POOL_DELAY_LOAD -sTEXTDECODER=2 -sPTHREAD_POOL_SIZE_STRICT=2 -sINITIAL_MEMORY=$(MAX_MEMORY) -sPTHREAD_POOL_SIZE=$(MAX_THREADS) -sPOLYFILL=0 -sEXIT_RUNTIME=0, -sINVOKE_RUN=0 -sSUPPORT_LONGJMP=0 -sEXPORTED_FUNCTIONS=_malloc -sEXPORT_NAME=loadVosklet -sMALLOC=emmalloc -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8OnStack -sENVIRONMENT=web,worker -I. -I$(LIBARCHIVE)/include -I$(VOSK)/src -L$(LIBARCHIVE)/lib -larchive -L$(KALDI)/src -l:online2/kaldi-online2.a -l:decoder/kaldi-decoder.a -l:ivector/kaldi-ivector.a -l:gmm/kaldi-gmm.a -l:tree/kaldi-tree.a -l:feat/kaldi-feat.a -l:cudamatrix/kaldi-cudamatrix.a -l:lat/kaldi-lat.a -l:lm/kaldi-lm.a -l:rnnlm/kaldi-rnnlm.a -l:hmm/kaldi-hmm.a -l:nnet3/kaldi-nnet3.a -l:transform/kaldi-transform.a -l:matrix/kaldi-matrix.a -l:fstext/kaldi-fstext.a -l:util/kaldi-util.a -l:base/kaldi-base.a -L$(OPENFST)/lib -l:libfst.a -l:libfstngram.a -L$(CLAPACK_WASM) -l:CBLAS/lib/cblas.a -l:CLAPACK-3.2.1/lapack.a -l:CLAPACK-3.2.1/libcblaswr.a -l:f2c_BLAS-3.8.0/blas.a -l:libf2c/libf2c.a -L$(VOSK)/src -l:vosk.a -lopfs.js -lembind -pthread -flto -msimd128 --pre-js pre.js -o ../Vosklet.js && \
cd .. && \
rm -f Vosklet.worker.js && \
sed -i 's/locateFile("Vosklet.worker.js")/pthreadUrl/g' Vosklet.js && \
prepare:
sudo apt install shtool libtool autogen autotools-dev pkg-config make && \

7
src/bindings.cc
View File

@@ -6,6 +6,13 @@ using namespace emscripten;
EMSCRIPTEN_BINDINGS() {
function("setLogLevel", &vosk_set_log_level, allow_raw_pointers());
enum_<VoskEndpointerMode>("epMode")
.value("ANSWER_DEFAULT", VOSK_EP_ANSWER_DEFAULT)
.value("ANSWER_SHORT", VOSK_EP_ANSWER_SHORT)
.value("ANSWER_LONG", VOSK_EP_ANSWER_LONG)
.value("ANSWER_VERY_LONG", VOSK_EP_ANSWER_VERY_LONG);
class_<genericModel>("genericModel")
.constructor<int, bool, std::string, std::string>(allow_raw_pointers())
.function("extractAndLoad", &genericModel::extractAndLoad, allow_raw_pointers());

19
src/genericModel.cc
View File

@@ -5,20 +5,22 @@ void genericModel::extractAndLoad(int tarStart, int tarSize) {
static fs::path path{};
static int fd{};
thrd.addTask([this, tarStart, tarSize](){
emscripten_console_log("Untaring...");
archive* src {archive_read_new()};
archive_read_support_format_tar(src);
archive_read_open_memory(src, (void*)tarStart, tarSize);
free((void*)tarStart);
archive_read_open_memory(src, reinterpret_cast<void*>(tarStart), tarSize);
if(archive_errno(src) != 0) {
emscripten_console_logf("Unable to open tar in WASM memory: %s", archive_error_string(src));
free(reinterpret_cast<void*>(tarStart));
fireEv(index, "Unable to open tar in WASM memory");
return;
}
while(1) {
if(archive_read_next_header2(src, entry) != ARCHIVE_OK) {
emscripten_console_logf("Tar header read failed: %s", archive_error_string(src));
fireEv(index, "Tar header read failed");
return;
int headerRes {archive_read_next_header2(src, entry)};
if(headerRes == ARCHIVE_EOF) break;
if(headerRes < ARCHIVE_OK) {
fireEv(index, archive_error_string(src));
break;
}
path = archive_entry_pathname(entry);
path = storepath + path.generic_string().substr(path.generic_string().find("/"));
@@ -41,13 +43,14 @@ void genericModel::extractAndLoad(int tarStart, int tarSize) {
return;
}
}
free(reinterpret_cast<void*>(tarStart));
fs::remove(storepath + "/README");
archive_read_free(src);
/*if(normalMdl) mdl = vosk_model_new(storepath.c_str());
if(normalMdl) mdl = vosk_model_new(storepath.c_str());
else vosk_spk_model_new(storepath.c_str());
emscripten_console_log("Loading finished!");
if(normalMdl ? std::get<0>(mdl) == nullptr : std::get<1>(mdl) == nullptr) fireEv(index, "Unable to load model for recognition");
else fireEv(index, "0");*/
else fireEv(index, "0");
});
}
genericModel::~genericModel() {

5
src/genericModel.h
View File

@@ -1,10 +1,10 @@
#pragma once
#include "proxier.h"
#include "jsLink.h"
#include <filesystem>
#include <fstream>
#include <fcntl.h>
#include <variant>
#include <fcntl.h>
#include <vosk_api.h>
#include <archive.h>
@@ -17,7 +17,6 @@ struct genericModel {
std::string storepath;
std::string id;
std::variant<VoskModel*, VoskSpkModel*> mdl;
reusableThrd thrd;
archive_entry* entry;
void extractAndLoad(int tarStart, int tarSize);
genericModel(int index, bool normalMdl, std::string storepath, std::string id);

20
src/proxier.cc → src/jsLink.cc
View File

@@ -1,4 +1,4 @@
#include "proxier.h"
#include <jsLink.h>
pthread_t dstThrd{pthread_self()};
ProxyingQueue glbQ{};
@@ -12,22 +12,4 @@ void fireEv(int index, const char* content, const char* type) {
}};
if(dstThrd == pthread_self()) proxy();
else glbQ.proxySync(dstThrd, proxy);
}
reusableThrd::reusableThrd() {
std::thread thrd{[this](){
while(!done) {
while(!queue.empty()) {
emscripten_console_log("==========Executing task===========");
queue.front()();
queue.pop();
}
}
}};
thrd.detach();
}
void reusableThrd::addTask(std::function<void()>&& task) {
queue.emplace(task);
}
reusableThrd::~reusableThrd() {
done = true;
}

10
src/proxier.h → src/jsLink.h
View File

@@ -9,14 +9,4 @@
#include <emscripten/proxying.h>
using namespace emscripten;
extern pthread_t dstThrd;
extern ProxyingQueue glbQ;
void fireEv(int index, const char* content, const char* type = nullptr);
struct reusableThrd {
std::queue<std::function<void()>> queue{};
bool done{};
void addTask(std::function<void()>&& task);
reusableThrd();
~reusableThrd();
};

79
src/pre.js
View File

@@ -27,22 +27,26 @@ class genericModel extends EventTarget {
}
static async _init(url, storepath, id, normalMdl) {
let mdl = new genericModel(url, storepath, id, normalMdl)
let result = new Promise((resolve, reject) => {
mdl.addEventListener("0", ev => {
switch(ev.detail) {
case "0":
return resolve(mdl)
default:
mdl.delete()
reject(ev.detail)
}
}, { once : true })
})
let tar
mdl.addEventListener("0", async (ev) => {
switch(ev.detail) {
case "0":
return mdl
default:
mdl.delete()
throw ev.detail
}
}, { once : true })
mdl.obj = new Module.genericModel(objs.length - 1, normalMdl, "/" + storepath, id)
try {
console.log("Getting Data file")
let dataFile = await (await getFileHandle(storepath + "/model.tgz")).getFile()
console.log("Getting ID file")
let idFile = await (await getFileHandle(storepath + "/id")).getFile()
if(await idFile.text() !== id) throw ""
tar = await new Response(dataFile.stream().pipeThrough(new DecompressionStream("gzip"))).arrayBuffer()
tar = dataFile.stream()
}
catch {
try {
@@ -50,22 +54,25 @@ class genericModel extends EventTarget {
if(!res.ok) {
throw "Unable to download model"
}
let teedBody = res.body.tee()
let newDataFile = await (await getFileHandle(storepath + "/model.tgz", true)).createWritable()
tar = await new Response(res.body.pipeThrough(new DecompressionStream("gzip"))).arrayBuffer()
await newDataFile.write(tar)
await newDataFile.write(await new Response(teedBody[0]).arrayBuffer())
await newDataFile.close()
let newIDFile = await (await getFileHandle(storepath + "/id", true)).createWritable()
await newIDFile.write(id)
await newIDFile.close()
tar = teedBody[1]
}
catch(e) {
mdl.obj.delete()
throw e
}
}
tar = await new Response(tar.pipeThrough(new DecompressionStream("gzip"))).arrayBuffer()
let tarStart = Module._malloc(tar.byteLength)
Module.HEAPU8.set(new Uint8Array(tar), tarStart)
mdl.obj.extractAndLoad(tarStart, tar.byteLength)
return result
}
delete() {
if (this.obj) this.obj.delete()
@@ -84,33 +91,34 @@ class Recognizer extends EventTarget {
}
static async _init(model, sampleRate, mode, grammar, spkModel) {
let rec = new Recognizer()
rec.addEventListener("0", (ev) => {
if(ev.detail === "0") {
rec.ptr = Module._malloc(512)
return rec
}
rec.delete()
throw ev.detail
}, { once : true })
let result = new Promise((resolve, reject) => {
rec.addEventListener("0", ev => {
if(ev.detail.indexOf(",") === -1) {
let loadInfo = ev.detail.split(",")
rec.dataBuf = Module.HEAPF32.subarray(parseInt(loadInfo[0]), parseInt(loadInfo[0]) + 128)
rec.state = Module.HEAP8.subarray(parseInt(loadInfo[1]), parseInt(loadInfo[1]) + 1) // State is an array with 1 element, there is no other way to get a reference to a single element
return resolve(rec)
}
rec.delete()
reject(rec)
}, { once : true })
})
switch(mode) {
case 1:
rec.obj = new Module.recognizer(model, sampleRate, objs.length-1)
rec.obj = new Module.recognizer(objs.length - 1, sampleRate, model)
break
case 2:
rec.obj = new Module.recognizer(model, spkModel, sampleRate, objs.length-1)
rec.obj = new Module.recognizer(objs.length -1, sampleRate, model, spkModel)
break
default:
rec.obj = new Module.recognizer(model, grammar, sampleRate, objs.length-1, 0)
rec.obj = new Module.recognizer(objs.length - 1, sampleRate, grammar, 0)
}
return result
}
async getNode(ctx, channelIndex = 0) {
if(typeof this.node === "undefined") {
let msgChannel = new MessageChannel()
await ctx.audioWorklet.addModule(processorUrl)
this.node = new AudioWorkletNode(ctx, 'BRProcessor', { channelCountMode: "max", numberOfInputs: 1, numberOfOutputs: 1, processorOptions: { ptr: this.ptr, channel: channelIndex, recognizerPort: msgChannel.port1 } })
msgChannel.port1.onmessage = () => {
this.obj.acceptWaveForm()
}
this.node = new AudioWorkletNode(ctx, 'VoskletProcessor', { channelCountMode: "max", numberOfInputs: 1, numberOfOutputs: 0, processorOptions: { dataBuf: this.dataBuf, state: this.state, channel: channelIndex }})
}
return this.node
}
@@ -152,19 +160,16 @@ Module.makeRecognizerWithGrm = (model, sampleRate, grammar) => {
}
let processorUrl = URL.createObjectURL(new Blob(['(',
(() => {
registerProcessor("BRProcessor", class extends AudioWorkletProcessor {
registerProcessor("VoskletProcessor", class extends AudioWorkletProcessor {
constructor(options) {
this.done = false
this.port.onmessage = (ev) => this.done = true
this.ptr = options.processorOptions.ptr
this.recognizerPort = options.processorOptions.recognizerPort
this.channelIndex = options.processorOptions.channelIndex
this.dataBuf = options.processorOptions.dataBuf
this.state = options.processorOptions.state
}
process(inputs, outputs, params) {
if(this.done) return false
this.wasmMem.set(inputs[0].getChannelData(this.channelIndex))
this.recognizerPort.postMessage("0")
outputs = inputs
while(state[0])
inputs.copyFromChannel(this.dataBuf, this.channelIndex)
state[0] = 1
return true
}
})

26
src/recognizer.cc
View File

@@ -1,19 +1,16 @@
#include "recognizer.h"
recognizer::recognizer(int index, float sampleRate, genericModel* model) : index(index) {
emscripten_console_log("Recognizer constructor called...");
rec = vosk_recognizer_new(std::get<0>(model->mdl),sampleRate);
finishConstruction(model, nullptr);
finishConstruction(model);
}
recognizer::recognizer(int index, float sampleRate, genericModel* model, genericModel* spkMdl) : index(index) {
emscripten_console_log("Recognizer constructor called...");
rec = vosk_recognizer_new_spk(std::get<0>(model->mdl), sampleRate, std::get<1>(spkMdl->mdl));
finishConstruction(model, spkMdl);
}
recognizer::recognizer(int index, float sampleRate, genericModel* model, const std::string& grm, int dummy) : index(index) {
emscripten_console_log("Recognizer constructor called...");
rec = vosk_recognizer_new_grm(std::get<0>(model->mdl), sampleRate, grm.c_str());
finishConstruction(model, nullptr);
finishConstruction(model);
}
recognizer::~recognizer() {
done.test_and_set(std::memory_order_relaxed);
@@ -21,27 +18,30 @@ recognizer::~recognizer() {
controller.test_and_set(std::memory_order_relaxed);
controller.notify_one();
vosk_recognizer_free(rec);
free(dataPtr);
}
void recognizer:: finishConstruction(genericModel* model, genericModel* spkModel) {
void recognizer::finishConstruction(genericModel* model, genericModel* spkModel) {
if(rec == nullptr) {
fireEv(index, "Unable to initialize recognizer");
return;
}
auto main {[this](){
emscripten_console_log("Recognizer loaded!");
fireEv(index, "0");
std::string loadInfo{};
loadInfo += reinterpret_cast<int>(dataBuf);
loadInfo += ",";
loadInfo += reinterpret_cast<int>(&state);
fireEv(index, loadInfo.c_str());
while(!done.test(std::memory_order_relaxed)) {
controller.wait(!done.test(std::memory_order_relaxed), std::memory_order_relaxed);
controller.clear(std::memory_order_relaxed);
if(done.test(std::memory_order_relaxed)) continue;
switch(vosk_recognizer_accept_waveform_f(rec, dataPtr, 512)) {
switch(vosk_recognizer_accept_waveform_f(rec, dataBuf, 512)) {
case 0:
fireEv(index, vosk_recognizer_result(rec), "result");
break;
case 1:
fireEv(index, vosk_recognizer_partial_result(rec), "partialResult");
}
state = 0;
}
}};
if(!model->recognizerUsedThrd) {
@@ -60,6 +60,12 @@ void recognizer:: finishConstruction(genericModel* model, genericModel* spkModel
std::thread t{main};
t.detach();
}
void recognizer::setEndpointerMode(VoskEndpointerMode mode) {
vosk_recognizer_set_endpointer_mode(rec, mode);
}
void recognizer::setEndpointerDelays(float tStartMax, float tEnd, float tMax) {
vosk_recognizer_set_endpointer_delays(rec, tStartMax, tEnd, tMax);
}
void recognizer::acceptWaveForm() {
controller.test_and_set(std::memory_order_relaxed);
controller.notify_one();

9
src/recognizer.h
View File

@@ -4,14 +4,17 @@
struct recognizer {
std::atomic_flag done{};
std::atomic_flag controller{};
float* dataPtr;
float dataBuf[128];
char state; // 0: Copying data from JS, 1: Processing from C++
int index;
VoskRecognizer* rec;
recognizer(int index, float sampleRate, genericModel* model);
recognizer(int index, float sampleRate, genericModel* model, genericModel* spkModel);
recognizer(int index, float sampleRate, genericModel* model, const std::string& grm, int dummy);
recognizer(int index, float sampleRate, genericModel* model, const std::string& grm, int dummy);
~recognizer();
void finishConstruction(genericModel* model, genericModel* spkModel);
void finishConstruction(genericModel* model, genericModel* spkModel = nullptr);
void setEndpointerMode(VoskEndpointerMode mode);
void setEndpointerDelays(float tStartMax, float tEnd, float tMax);
void acceptWaveForm();
void setSpkModel(genericModel* model);
void setGrm(const std::string& grm);