From e92ddadc992246bdab4bf1db1d279922dab09d55 Mon Sep 17 00:00:00 2001
From: msqr1 <repulseshipp@gmail.com>
Date: Wed, 24 Jan 2024 20:30:27 -0800
Subject: [PATCH] Add documents and optimize build

---
 README.md         | 45 ++++++++++++++++++++++++++++-----------------
 install.sh        | 32 +++++++++++++++++++++++++-------
 src/kaldi.patch   | 12 ++++++++++++
 src/post.js       |  1 -
 src/pre.js        | 15 +++++++++------
 src/processor.js  | 23 +++++++++++++++++++++++
 src/recognizer.cc | 31 +++++++++++++++++++------------
 src/recognizer.h  |  8 +++++++-
 src/vosk.patch    | 16 ++++++++++++++++
 9 files changed, 139 insertions(+), 44 deletions(-)
 delete mode 100644 src/post.js
 create mode 100644 src/processor.js
 create mode 100644 src/vosk.patch

diff --git a/README.md b/README.md
index eff29d9..0ca6a86 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
-# Browser-recognizer- 
+# Browser-recognizer
 - A speech recognizer built on Vosk that can be run on the browser, inspired by [vosk-browser](https://github.com/ccoreilly/vosk-browser), but built from scratch and no code taken!
-- Browser-recognizer can run both in the browser main thread and web workers.
+- Browser-recognizer can run both in the browser main thread and web workers
+- The API also designed to have strong exception safety
 ## Global and all objects' common interface 
 | Function signature (global) | Description |
 |---|---|
@@ -10,11 +11,11 @@
 
 | Function signature (all objects) | Description
 |---|---|
-| ```delete()``` | Delete this Object
+| ```delete()``` | Delete (call C++ destructor on) this object
 ## ```Recognizer``` object
 | Function signature | Description |
 |---|---|
-| ```processAudio(audio: AudioBuffer)``` | Recognize an audio chunk, 
+| ```processAudio(ctx: AudioContext)``` | Recognize an audio chunk 
 | ```setPartialWords(partialWords: bool)``` | Return words' information in a partialResult event (default: false) |
 | ```setWords(words: bool)``` | Return words' information in a result event (default: false) |
 | ```setNLSML(nlsml: bool)``` | Return result and partialResult in NLSML form (default: false) |
@@ -26,23 +27,33 @@
 |---|---|
 | ```partialResult``` | There is a partial recognition result, check the event's "details" property |
 | ```result``` | There is a full recognition result, check the event's "details" property |
-## Other key points
-- If an error occurs, no changes was made.
-- Fixed memory size at 300MB, changing it require recompilation (because the use of pthread will lead)
-### Additions to vosk-browser:
-- Multiple models support
-- Speaker model (SpkModel) support
-- Storage path management (when many models are required)
-- Model ID management (when model updates are required)
-### This requires SharedArrayBuffer, so set the response headers:
+## Compilation
+Changing any setting to non-default values requires recompilation
+```
+git clone --depth=1 https://github.com/msqr1/Browser-recognizer &&
+cd Browser-recognizer &&
+[Name]=[Value]... ./install.sh
+```
+| Name | Description | Default value |
+|---|---|---|
+| MAX_MEMORY | Set max memory (suffix mb, gb, or none for bytes) | 300mb, as [recommended](https://alphacephei.com/vosk/models) |
+| MAX_THREAD | Set the max number of thread (min: 2) | 2 (1 OPFS thread + 1 recognizer thread) |
+| EMSDK | Set EMSDK's path (will install EMSDK in root folder if unset) | **.** |
+## Response headers
+Browser-recognizer require SharedArrayBuffer, so these response headers must be set:
 - ***Cross-Origin-Embedder-Policy*** ---> ***require-corp***
 - ***Cross-Origin-Opener-Policy*** ---> ***same-origin***
-### If you can't set these headers, you can use a VERY HACKY workaround at *src/addCOI.js*.
 
-## Demo
+If you can't set them, you may use a VERY HACKY workaround at *src/addCOI.js*.
+## Additions to vosk-browser:
+- Can download multiple models
+- Model storage path management (when many models are required)
+- Model ID management (when model updates are required)
+
+## Usage
 ```
 <!--Load this from a script tag-->
-<script src="BrowserRecognizer.js" type="module"></script>
+<script src="BrowserRecognizer.js"></script>
 <!-->
 <script>
   // Select name
@@ -58,7 +69,7 @@
     console.log("Partial result: ",e.details)
   })
 
-  // Recognize
+  // Process audio
   media = await navigator.mediaDevices.getUserMedia({
     video: false,
     audio: {
diff --git a/install.sh b/install.sh
index ba0fd10..9e43861 100755
--- a/install.sh
+++ b/install.sh
@@ -8,9 +8,10 @@
 # Total build time is around 45 minutes, mostly from building Kaldi
 
 sudo apt install shtool libtool autogen autotools-dev pkg-config make &&
-wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash &&
-nvm install 20.11.0 &&
 
+MAX_MEMORY?=300mb &&
+MAX_THREAD?=2 &&
+EMSDK?=$(realpath .) &&
 SRC=$(realpath src) &&
 KALDI=$(realpath kaldi) &&
 VOSK=$(realpath vosk-api) &&
@@ -19,13 +20,29 @@ LIBARCHIVE=$(realpath libarchive) &&
 ZSTD=$(realpath zstd) && 
 CLAPACK_WASM=$(realpath clapack-wasm) &&
 
-source ../../emsdk/emsdk_env.sh &&
-export PATH=:$PATH:$(realpath ../../emsdk/upstream/bin) &&
+if [[! -d $EMSDK_PATH] ]
+  echo "Invalid EMSDK path"
+  exit 1
+if [[! $MAX_MEMORY =~ [0-9]+(mb|gb)* ]]
+  echo "Invalid MAX_MEMORY value"
+  exit 1
+if [ $MAX_THREAD < 2 ] 
+  echo "MAX_THREAD is less than 2"
+  exit 1
+if [$(realpath $EMSDK) = $(realpath .)]; then
+  git clone --depth=1 https://github.com/emscripten-core/emsdk.git &&
+  cd emsdk &&
+  ./emsdk install 3.1.51 &&
+  ./emsdk activate 3.1.51 &&
+
+source $EMSDK/emsdk_env.sh &&
+export PATH=:$PATH:$EMSDK/upstream/bin &&
 
 rm -rf /tmp/zstd &&
 rm -rf /tmp/libarchive &&
 rm -rf /tmp/openfst &&
 
+cd $SRC &&
 git clone -b v1.5.5 --depth=1 https://github.com/facebook/zstd /tmp/zstd &&
 git clone -b v3.7.2 --depth=1 https://github.com/libarchive/libarchive /tmp/libarchive &&
 git clone --depth=1 https://gitlab.inria.fr/multispeech/kaldi.web/clapack-wasm.git &&
@@ -54,13 +71,14 @@ echo "PACKAGE_VERSION = 1.8.0" >> $OPENFST/Makefile &&
 
 cd $KALDI/src &&
 git apply $SRC/kaldi.patch &&
-CXXFLAGS="-O3 -msimd128 -UHAVE_EXECINFO_H -pthread -flto" LDFLAGS="-O3 -sERROR_ON_UNDEFINED_SYMBOLS=0 -lembind -pthread -flto" emconfigure ./configure --use-cuda=no --with-cudadecoder=no --static --static-math=yes --static-fst=yes  --debug-level=0 --clapack-root=$CLAPACK_WASM --host=WASM && 
+CXXFLAGS="-O3 -msimd128 -UHAVE_EXECINFO_H -pthread -flto" LDFLAGS="-O3 -sERROR_ON_UNDEFINED_SYMBOLS=0 -lembind -pthread -flto" emconfigure ./configure --use-cuda=no --with-cudadecoder=no --static --static-math=yes --static-fst=yes  --debug-level=0 --double-precision=yes --clapack-root=$CLAPACK_WASM --host=WASM && 
 emmake make online2 lm rnnlm &&
 
 cd $VOSK/src &&
+git apply $SRC/vosk.patch &&
 VOSK_FILES="recognizer.cc language_model.cc model.cc spk_model.cc vosk_api.cc" &&
-em++ -pthread -O3 -flto -I. -I$KALDI/src -I$OPENFST/include $VOSK_FILES -c &&
+em++ -pthread -O3 -flto -Wno-deprecated -I. -I$KALDI/src -I$OPENFST/include $VOSK_FILES -c &&
 emar -rcs vosk.a ${VOSK_FILES//.cc/.o} &&
 
 cd $SRC &&
-em++ -O3 genericModel.cc model.cc spkModel.cc recognizer.cc bindings.cc -sWASMFS -sWASM_BIGINT -sSUPPORT_BIG_ENDIAN -sSINGLE_FILE -sEMBIND_AOT -sWASM_WORKER -sAUDIO_WORKLET -sEMBIND_STD_STRING_IS_UTF8 -sSUPPORT_LONGJMP=0 -sMODULARIZE -sEXPORT_NAME=loadBR -sEXPORT_ES6 -sENVIRONMENT=web,worker -sINITIAL_MEMORY=300mb -sASYNCIFY -sPTHREAD_POOL_SIZE=2 --pre-js pre.js --extern-post-js post.js -pthread -flto -I. -I$LIBARCHIVE/include -I$VOSK/src -L$LIBARCHIVE/lib -larchive -L$ZSTD/lib -lzstd -L$KALDI/src -l:online2/kaldi-online2.a -l:decoder/kaldi-decoder.a -l:ivector/kaldi-ivector.a -l:gmm/kaldi-gmm.a -l:tree/kaldi-tree.a -l:feat/kaldi-feat.a -l:cudamatrix/kaldi-cudamatrix.a -l:lat/kaldi-lat.a -l:lm/kaldi-lm.a -l:rnnlm/kaldi-rnnlm.a -l:hmm/kaldi-hmm.a -l:nnet3/kaldi-nnet3.a -l:transform/kaldi-transform.a -l:matrix/kaldi-matrix.a -l:fstext/kaldi-fstext.a -l:util/kaldi-util.a -l:base/kaldi-base.a -L$OPENFST/lib -l:libfst.a -l:libfstngram.a -L$CLAPACK_WASM -l:CBLAS/lib/cblas.a -l:CLAPACK-3.2.1/lapack.a -l:CLAPACK-3.2.1/libcblaswr.a -l:f2c_BLAS-3.8.0/blas.a -l:libf2c/libf2c.a -L$VOSK/src -l:vosk.a -lopfs.js -lembind -lopenal -o ../BrowserRecognizer.js  
+em++ -O3 genericModel.cc model.cc spkModel.cc recognizer.cc bindings.cc -sWASMFS -sWASM_BIGINT -sSINGLE_FILE -sEMBIND_STD_STRING_IS_UTF8 -sSUPPORT_LONGJMP=0 -sMODULARIZE -sEXPORT_NAME=loadBR -sENVIRONMENT=web,worker -sINITIAL_MAX_MEMORY=$MAX_MEMORY -sASYNCIFY -sPTHREAD_POOL_SIZE=$MAX_THREAD -sPTHREAD_POOL_SIZE_STRICT -sPTHREAD_POOL_DELAY_LOAD -sASYNCIFY_ONLY=['emscripten_wget'] -sALLOW_BLOCKING_ON_MAIN_THREAD=0 -sPOLYFILL=0 --pre-js pre.js  -I. -I$LIBARCHIVE/include -I$VOSK/src -L$LIBARCHIVE/lib -larchive -L$ZSTD/lib -lzstd -L$KALDI/src -l:online2/kaldi-online2.a -l:decoder/kaldi-decoder.a -l:ivector/kaldi-ivector.a -l:gmm/kaldi-gmm.a -l:tree/kaldi-tree.a -l:feat/kaldi-feat.a -l:cudamatrix/kaldi-cudamatrix.a -l:lat/kaldi-lat.a -l:lm/kaldi-lm.a -l:rnnlm/kaldi-rnnlm.a -l:hmm/kaldi-hmm.a -l:nnet3/kaldi-nnet3.a -l:transform/kaldi-transform.a -l:matrix/kaldi-matrix.a -l:fstext/kaldi-fstext.a -l:util/kaldi-util.a -l:base/kaldi-base.a -L$OPENFST/lib -l:libfst.a -l:libfstngram.a -L$CLAPACK_WASM -l:CBLAS/lib/cblas.a -l:CLAPACK-3.2.1/lapack.a -l:CLAPACK-3.2.1/libcblaswr.a -l:f2c_BLAS-3.8.0/blas.a -l:libf2c/libf2c.a -L$VOSK/src -l:vosk.a -lopfs.js -lembind -pthread -flto -o BrowserRecognizer.js  
diff --git a/src/kaldi.patch b/src/kaldi.patch
index 15de933..bfe280a 100644
--- a/src/kaldi.patch
+++ b/src/kaldi.patch
@@ -39,3 +39,15 @@ index c3a122281..71d37256d 100644
      TaskSequencer<IvectorExtractorComputeDerivedVarsClass> sequencer(
          sequencer_opts);
      for (int32 i = 0; i < NumGauss(); i++)
+diff --git a/src/base/kaldi-types.h b/src/base/kaldi-types.h
+index 7ebf4f8..2f5979e 100644
+--- a/src/base/kaldi-types.h
++++ b/src/base/kaldi-types.h
+@@ -20,6 +20,7 @@
+ 
+ #ifndef KALDI_BASE_KALDI_TYPES_H_
+ #define KALDI_BASE_KALDI_TYPES_H_ 1
++#define KALDI_DOUBLEPRECISION 1
+ 
+ namespace kaldi {
+ // TYPEDEFS ..................................................................
diff --git a/src/post.js b/src/post.js
deleted file mode 100644
index b1e371c..0000000
--- a/src/post.js
+++ /dev/null
@@ -1 +0,0 @@
-window.loadBR = loadBR
\ No newline at end of file
diff --git a/src/pre.js b/src/pre.js
index 6e5ecb9..b890136 100644
--- a/src/pre.js
+++ b/src/pre.js
@@ -1,14 +1,14 @@
-var objs =  []
+let objs =  []
 class recognizer extends EventTarget {
   constructor(rec) {
     super()
     this.obj = rec
     objs.push(this)
   }
-  processAudio(buffer) {
-    if(buffer.numberOfChannels < 1) throw Error("Buffer has ",buffer.numberOfChannels, " channel")
-    let data = buffer.getChannelData(0);
-    if(!(data instanceof Float32Array)) throw Error("Channel data isn't a Float32Array");
+  processAudio(ctx) {
+    let ptr = Module._malloc(512);
+
+    this.obj.acceptWaveForm(ptr)
   }
   delete() {
     this.obj.delete()
@@ -32,7 +32,10 @@ class recognizer extends EventTarget {
     this.obj.setMaxAlternatives(alts)
   }
 }
-Module.deleteAll = () => objs.forEach(obj => obj.delete())
+Module.deleteAll = () => {
+  objs.forEach(obj => obj.delete())
+  ctx.close()
+}
 Module.makeModel = async (url, path, id) => {
   let mdl
   try {
diff --git a/src/processor.js b/src/processor.js
new file mode 100644
index 0000000..9d1d65a
--- /dev/null
+++ b/src/processor.js
@@ -0,0 +1,23 @@
+registerProcessor("wasmMemCpy", class extends AudioWorkletProcessor {
+  constructor(options) {
+    super(options)
+    this.retval = true
+    this.port.onmessage = (ev) => {
+      switch(ev.cmd) {
+        case "init":
+          this.ptr = ev.ptr
+          this.wasmMem = ev.wasmMem
+          this.recognizerPort = ev.ports[0]
+          break
+        case "deinit":
+          this.retval = false
+          break
+      }
+    }
+  }
+  process(input, output, param) {
+    const data = input[0][0]
+    this.recognizerPort.postMessage("done")
+    return this.retval
+  }
+})
\ No newline at end of file
diff --git a/src/recognizer.cc b/src/recognizer.cc
index 41952fe..e0c7ce4 100644
--- a/src/recognizer.cc
+++ b/src/recognizer.cc
@@ -1,9 +1,25 @@
 #include "recognizer.h" 
+audioData::audioData(int addr, int len) : addr(reinterpret_cast<float*>(addr)), len(len) {}
 recognizer::recognizer(model* mdl, float sampleRate, int index) : index(index) {
   rec = vosk_recognizer_new(mdl->mdl,sampleRate);
   if(rec == nullptr) {
     throwJS("Unable to initialize recognizer");
-  } 
+    return;
+  }
+  std::thread t{[this](){
+    while(!queue.empty()) {
+      audioData data {queue.front()};
+      queue.pop();
+      switch(vosk_recognizer_accept_waveform_f(rec, data.addr, data.len)) {
+      case 0:
+      fireEv("result", vosk_recognizer_result(rec));
+      break;
+      case 1:
+      fireEv("partialResult", vosk_recognizer_partial_result(rec));
+      }
+    }
+  }};
+  t.detach();
 }
 recognizer::~recognizer() {
   vosk_recognizer_free(rec);
@@ -13,17 +29,8 @@ void recognizer::fireEv(const char *type, const char *content) {
     recognizers[$0].dispatchEvent(new CustomEvent(UTF8ToString($1), {"details" : UTF8ToString($2)}));
   },this->index, type, content);
 }
-void recognizer::acceptWaveForm(float* data, int len) {
-  switch(vosk_recognizer_accept_waveform_f(rec, data, len)) {
-    case 0:
-    fireEv("result", vosk_recognizer_result(rec));
-    break;
-    case 1:
-    fireEv("partialResult", vosk_recognizer_partial_result(rec));
-    break;
-    default:
-    throwJS("acceptWaveForm error (from C++)", true);
-  }
+void recognizer::acceptWaveForm(int addr, int len) {
+  queue.emplace(addr, len);
 }
 void recognizer::setGrm(const std::string& grm) {
   vosk_recognizer_set_grm(rec, grm.c_str());
diff --git a/src/recognizer.h b/src/recognizer.h
index 7486f42..9de5a80 100644
--- a/src/recognizer.h
+++ b/src/recognizer.h
@@ -5,6 +5,7 @@
 #include <filesystem>
 #include <atomic>
 #include <thread>
+#include <queue>
 
 #include <emscripten/wasmfs.h>
 #include <emscripten/webaudio.h>
@@ -15,12 +16,17 @@
 extern void throwJS(const char* msg, bool err = false);
 namespace fs = std::filesystem;
 
+struct audioData {
+  float* addr{};
+  int len{};
+  audioData(int addr, int len);
+};
 struct recognizer {
   int index{};
   VoskRecognizer* rec{};
   recognizer(model* model, float sampleRate, int index);
   ~recognizer();
-  void acceptWaveForm(float* data, int len);
+  void acceptWaveForm(int addr, int len);
   void fireEv(const char* type, const char* content);
   void setSpkModel(spkModel* model);
   void setGrm(const std::string& grm);
diff --git a/src/vosk.patch b/src/vosk.patch
new file mode 100644
index 0000000..e2e0aca
--- /dev/null
+++ b/src/vosk.patch
@@ -0,0 +1,16 @@
+diff --git a/src/recognizer.cc b/src/recognizer.cc
+index 1da6e6b..5e0652a 100644
+--- a/src/recognizer.cc
++++ b/src/recognizer.cc
+@@ -384,8 +384,10 @@ bool Recognizer::AcceptWaveform(const float *fdata, int len)
+ {
+     Vector<BaseFloat> wave;
+     wave.Resize(len, kUndefined);
+-    for (int i = 0; i < len; i++)
++    for (int i = 0; i < len; i++) {
+         wave(i) = fdata[i];
++        wave(i) *= 32768;
++    }
+     return AcceptWaveform(wave);
+ }
+