Add documents and optimize build

2024-01-24 20:30:27 -08:00
parent 6da9a662a1
commit e92ddadc99
9 changed files with 139 additions and 44 deletions
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
-# Browser-recognizer- 
+# Browser-recognizer
 - A speech recognizer built on Vosk that can be run on the browser, inspired by [vosk-browser](https://github.com/ccoreilly/vosk-browser), but built from scratch and no code taken!
- Browser-recognizer can run both in the browser main thread and web workers.
+- Browser-recognizer can run both in the browser main thread and web workers
+- The API also designed to have strong exception safety
 ## Global and all objects' common interface 
 | Function signature (global) | Description |
 |---|---|
@@ -10,11 +11,11 @@

 | Function signature (all objects) | Description
 |---|---|
-| ```delete()``` | Delete this Object
+| ```delete()``` | Delete (call C++ destructor on) this object
 ## ```Recognizer``` object
 | Function signature | Description |
 |---|---|
-| ```processAudio(audio: AudioBuffer)``` | Recognize an audio chunk, 
+| ```processAudio(ctx: AudioContext)``` | Recognize an audio chunk 
 | ```setPartialWords(partialWords: bool)``` | Return words' information in a partialResult event (default: false) |
 | ```setWords(words: bool)``` | Return words' information in a result event (default: false) |
 | ```setNLSML(nlsml: bool)``` | Return result and partialResult in NLSML form (default: false) |
@@ -26,23 +27,33 @@
 |---|---|
 | ```partialResult``` | There is a partial recognition result, check the event's "details" property |
 | ```result``` | There is a full recognition result, check the event's "details" property |
-## Other key points
- If an error occurs, no changes was made.
- Fixed memory size at 300MB, changing it require recompilation (because the use of pthread will lead)
-### Additions to vosk-browser:
- Multiple models support
- Speaker model (SpkModel) support
- Storage path management (when many models are required)
- Model ID management (when model updates are required)
-### This requires SharedArrayBuffer, so set the response headers:
+## Compilation
+Changing any setting to non-default values requires recompilation
+```
+git clone --depth=1 https://github.com/msqr1/Browser-recognizer &&
+cd Browser-recognizer &&
+[Name]=[Value]... ./install.sh
+```
+| Name | Description | Default value |
+|---|---|---|
+| MAX_MEMORY | Set max memory (suffix mb, gb, or none for bytes) | 300mb, as [recommended](https://alphacephei.com/vosk/models) |
+| MAX_THREAD | Set the max number of thread (min: 2) | 2 (1 OPFS thread + 1 recognizer thread) |
+| EMSDK | Set EMSDK's path (will install EMSDK in root folder if unset) | **.** |
+## Response headers
+Browser-recognizer require SharedArrayBuffer, so these response headers must be set:
 - ***Cross-Origin-Embedder-Policy*** ---> ***require-corp***
 - ***Cross-Origin-Opener-Policy*** ---> ***same-origin***
-### If you can't set these headers, you can use a VERY HACKY workaround at *src/addCOI.js*.

-## Demo
+If you can't set them, you may use a VERY HACKY workaround at *src/addCOI.js*.
+## Additions to vosk-browser:
+- Can download multiple models
+- Model storage path management (when many models are required)
+- Model ID management (when model updates are required)
+
+## Usage
 ```
 <!--Load this from a script tag-->
-<script src="BrowserRecognizer.js" type="module"></script>
+<script src="BrowserRecognizer.js"></script>
 <!-->
 <script>
  // Select name
@@ -58,7 +69,7 @@
    console.log("Partial result: ",e.details)
  })

-  // Recognize
+  // Process audio
  media = await navigator.mediaDevices.getUserMedia({
    video: false,
    audio: {
--- a/install.sh
+++ b/install.sh
@@ -8,9 +8,10 @@
 # Total build time is around 45 minutes, mostly from building Kaldi

 sudo apt install shtool libtool autogen autotools-dev pkg-config make &&
-wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash &&
-nvm install 20.11.0 &&

+MAX_MEMORY?=300mb &&
+MAX_THREAD?=2 &&
+EMSDK?=$(realpath .) &&
 SRC=$(realpath src) &&
 KALDI=$(realpath kaldi) &&
 VOSK=$(realpath vosk-api) &&
@@ -19,13 +20,29 @@ LIBARCHIVE=$(realpath libarchive) &&
 ZSTD=$(realpath zstd) && 
 CLAPACK_WASM=$(realpath clapack-wasm) &&

-source ../../emsdk/emsdk_env.sh &&
-export PATH=:$PATH:$(realpath ../../emsdk/upstream/bin) &&
+if [[! -d $EMSDK_PATH] ]
+  echo "Invalid EMSDK path"
+  exit 1
+if [[! $MAX_MEMORY =~ [0-9]+(mb|gb)* ]]
+  echo "Invalid MAX_MEMORY value"
+  exit 1
+if [ $MAX_THREAD < 2 ] 
+  echo "MAX_THREAD is less than 2"
+  exit 1
+if [$(realpath $EMSDK) = $(realpath .)]; then
+  git clone --depth=1 https://github.com/emscripten-core/emsdk.git &&
+  cd emsdk &&
+  ./emsdk install 3.1.51 &&
+  ./emsdk activate 3.1.51 &&
+
+source $EMSDK/emsdk_env.sh &&
+export PATH=:$PATH:$EMSDK/upstream/bin &&

 rm -rf /tmp/zstd &&
 rm -rf /tmp/libarchive &&
 rm -rf /tmp/openfst &&

+cd $SRC &&
 git clone -b v1.5.5 --depth=1 https://github.com/facebook/zstd /tmp/zstd &&
 git clone -b v3.7.2 --depth=1 https://github.com/libarchive/libarchive /tmp/libarchive &&
 git clone --depth=1 https://gitlab.inria.fr/multispeech/kaldi.web/clapack-wasm.git &&
@@ -54,13 +71,14 @@ echo "PACKAGE_VERSION = 1.8.0" >> $OPENFST/Makefile &&

 cd $KALDI/src &&
 git apply $SRC/kaldi.patch &&
-CXXFLAGS="-O3 -msimd128 -UHAVE_EXECINFO_H -pthread -flto" LDFLAGS="-O3 -sERROR_ON_UNDEFINED_SYMBOLS=0 -lembind -pthread -flto" emconfigure ./configure --use-cuda=no --with-cudadecoder=no --static --static-math=yes --static-fst=yes  --debug-level=0 --clapack-root=$CLAPACK_WASM --host=WASM && 
+CXXFLAGS="-O3 -msimd128 -UHAVE_EXECINFO_H -pthread -flto" LDFLAGS="-O3 -sERROR_ON_UNDEFINED_SYMBOLS=0 -lembind -pthread -flto" emconfigure ./configure --use-cuda=no --with-cudadecoder=no --static --static-math=yes --static-fst=yes  --debug-level=0 --double-precision=yes --clapack-root=$CLAPACK_WASM --host=WASM && 
 emmake make online2 lm rnnlm &&

 cd $VOSK/src &&
+git apply $SRC/vosk.patch &&
 VOSK_FILES="recognizer.cc language_model.cc model.cc spk_model.cc vosk_api.cc" &&
-em++ -pthread -O3 -flto -I. -I$KALDI/src -I$OPENFST/include $VOSK_FILES -c &&
+em++ -pthread -O3 -flto -Wno-deprecated -I. -I$KALDI/src -I$OPENFST/include $VOSK_FILES -c &&
 emar -rcs vosk.a ${VOSK_FILES//.cc/.o} &&

 cd $SRC &&
-em++ -O3 genericModel.cc model.cc spkModel.cc recognizer.cc bindings.cc -sWASMFS -sWASM_BIGINT -sSUPPORT_BIG_ENDIAN -sSINGLE_FILE -sEMBIND_AOT -sWASM_WORKER -sAUDIO_WORKLET -sEMBIND_STD_STRING_IS_UTF8 -sSUPPORT_LONGJMP=0 -sMODULARIZE -sEXPORT_NAME=loadBR -sEXPORT_ES6 -sENVIRONMENT=web,worker -sINITIAL_MEMORY=300mb -sASYNCIFY -sPTHREAD_POOL_SIZE=2 --pre-js pre.js --extern-post-js post.js -pthread -flto -I. -I$LIBARCHIVE/include -I$VOSK/src -L$LIBARCHIVE/lib -larchive -L$ZSTD/lib -lzstd -L$KALDI/src -l:online2/kaldi-online2.a -l:decoder/kaldi-decoder.a -l:ivector/kaldi-ivector.a -l:gmm/kaldi-gmm.a -l:tree/kaldi-tree.a -l:feat/kaldi-feat.a -l:cudamatrix/kaldi-cudamatrix.a -l:lat/kaldi-lat.a -l:lm/kaldi-lm.a -l:rnnlm/kaldi-rnnlm.a -l:hmm/kaldi-hmm.a -l:nnet3/kaldi-nnet3.a -l:transform/kaldi-transform.a -l:matrix/kaldi-matrix.a -l:fstext/kaldi-fstext.a -l:util/kaldi-util.a -l:base/kaldi-base.a -L$OPENFST/lib -l:libfst.a -l:libfstngram.a -L$CLAPACK_WASM -l:CBLAS/lib/cblas.a -l:CLAPACK-3.2.1/lapack.a -l:CLAPACK-3.2.1/libcblaswr.a -l:f2c_BLAS-3.8.0/blas.a -l:libf2c/libf2c.a -L$VOSK/src -l:vosk.a -lopfs.js -lembind -lopenal -o ../BrowserRecognizer.js  
+em++ -O3 genericModel.cc model.cc spkModel.cc recognizer.cc bindings.cc -sWASMFS -sWASM_BIGINT -sSINGLE_FILE -sEMBIND_STD_STRING_IS_UTF8 -sSUPPORT_LONGJMP=0 -sMODULARIZE -sEXPORT_NAME=loadBR -sENVIRONMENT=web,worker -sINITIAL_MAX_MEMORY=$MAX_MEMORY -sASYNCIFY -sPTHREAD_POOL_SIZE=$MAX_THREAD -sPTHREAD_POOL_SIZE_STRICT -sPTHREAD_POOL_DELAY_LOAD -sASYNCIFY_ONLY=['emscripten_wget'] -sALLOW_BLOCKING_ON_MAIN_THREAD=0 -sPOLYFILL=0 --pre-js pre.js  -I. -I$LIBARCHIVE/include -I$VOSK/src -L$LIBARCHIVE/lib -larchive -L$ZSTD/lib -lzstd -L$KALDI/src -l:online2/kaldi-online2.a -l:decoder/kaldi-decoder.a -l:ivector/kaldi-ivector.a -l:gmm/kaldi-gmm.a -l:tree/kaldi-tree.a -l:feat/kaldi-feat.a -l:cudamatrix/kaldi-cudamatrix.a -l:lat/kaldi-lat.a -l:lm/kaldi-lm.a -l:rnnlm/kaldi-rnnlm.a -l:hmm/kaldi-hmm.a -l:nnet3/kaldi-nnet3.a -l:transform/kaldi-transform.a -l:matrix/kaldi-matrix.a -l:fstext/kaldi-fstext.a -l:util/kaldi-util.a -l:base/kaldi-base.a -L$OPENFST/lib -l:libfst.a -l:libfstngram.a -L$CLAPACK_WASM -l:CBLAS/lib/cblas.a -l:CLAPACK-3.2.1/lapack.a -l:CLAPACK-3.2.1/libcblaswr.a -l:f2c_BLAS-3.8.0/blas.a -l:libf2c/libf2c.a -L$VOSK/src -l:vosk.a -lopfs.js -lembind -pthread -flto -o BrowserRecognizer.js  
--- a/src/kaldi.patch
+++ b/src/kaldi.patch
@@ -39,3 +39,15 @@ index c3a122281..71d37256d 100644
     TaskSequencer<IvectorExtractorComputeDerivedVarsClass> sequencer(
         sequencer_opts);
     for (int32 i = 0; i < NumGauss(); i++)
+diff --git a/src/base/kaldi-types.h b/src/base/kaldi-types.h
+index 7ebf4f8..2f5979e 100644
+--- a/src/base/kaldi-types.h
+++ b/src/base/kaldi-types.h
+@@ -20,6 +20,7 @@
+ 
+ #ifndef KALDI_BASE_KALDI_TYPES_H_
+ #define KALDI_BASE_KALDI_TYPES_H_ 1
+#define KALDI_DOUBLEPRECISION 1
+ 
+ namespace kaldi {
+ // TYPEDEFS ..................................................................
--- a/src/post.js
+++ b/src/post.js
@@ -1 +0,0 @@
-window.loadBR = loadBR
--- a/src/pre.js
+++ b/src/pre.js
@@ -1,14 +1,14 @@
-var objs =  []
+let objs =  []
 class recognizer extends EventTarget {
  constructor(rec) {
    super()
    this.obj = rec
    objs.push(this)
  }
-  processAudio(buffer) {
-    if(buffer.numberOfChannels < 1) throw Error("Buffer has ",buffer.numberOfChannels, " channel")
-    let data = buffer.getChannelData(0);
-    if(!(data instanceof Float32Array)) throw Error("Channel data isn't a Float32Array");
+  processAudio(ctx) {
+    let ptr = Module._malloc(512);
+
+    this.obj.acceptWaveForm(ptr)
  }
  delete() {
    this.obj.delete()
@@ -32,7 +32,10 @@ class recognizer extends EventTarget {
    this.obj.setMaxAlternatives(alts)
  }
 }
-Module.deleteAll = () => objs.forEach(obj => obj.delete())
+Module.deleteAll = () => {
+  objs.forEach(obj => obj.delete())
+  ctx.close()
+}
 Module.makeModel = async (url, path, id) => {
  let mdl
  try {
--- a/src/processor.js
+++ b/src/processor.js
@@ -0,0 +1,23 @@
+registerProcessor("wasmMemCpy", class extends AudioWorkletProcessor {
+  constructor(options) {
+    super(options)
+    this.retval = true
+    this.port.onmessage = (ev) => {
+      switch(ev.cmd) {
+        case "init":
+          this.ptr = ev.ptr
+          this.wasmMem = ev.wasmMem
+          this.recognizerPort = ev.ports[0]
+          break
+        case "deinit":
+          this.retval = false
+          break
+      }
+    }
+  }
+  process(input, output, param) {
+    const data = input[0][0]
+    this.recognizerPort.postMessage("done")
+    return this.retval
+  }
+})
--- a/src/recognizer.cc
+++ b/src/recognizer.cc
@@ -1,9 +1,25 @@
 #include "recognizer.h" 
+audioData::audioData(int addr, int len) : addr(reinterpret_cast<float*>(addr)), len(len) {}
 recognizer::recognizer(model* mdl, float sampleRate, int index) : index(index) {
  rec = vosk_recognizer_new(mdl->mdl,sampleRate);
  if(rec == nullptr) {
    throwJS("Unable to initialize recognizer");
+    return;
  }
+  std::thread t{[this](){
+    while(!queue.empty()) {
+      audioData data {queue.front()};
+      queue.pop();
+      switch(vosk_recognizer_accept_waveform_f(rec, data.addr, data.len)) {
+      case 0:
+      fireEv("result", vosk_recognizer_result(rec));
+      break;
+      case 1:
+      fireEv("partialResult", vosk_recognizer_partial_result(rec));
+      }
+    }
+  }};
+  t.detach();
 }
 recognizer::~recognizer() {
  vosk_recognizer_free(rec);
@@ -13,17 +29,8 @@ void recognizer::fireEv(const char *type, const char *content) {
    recognizers[$0].dispatchEvent(new CustomEvent(UTF8ToString($1), {"details" : UTF8ToString($2)}));
  },this->index, type, content);
 }
-void recognizer::acceptWaveForm(float* data, int len) {
-  switch(vosk_recognizer_accept_waveform_f(rec, data, len)) {
-    case 0:
-    fireEv("result", vosk_recognizer_result(rec));
-    break;
-    case 1:
-    fireEv("partialResult", vosk_recognizer_partial_result(rec));
-    break;
-    default:
-    throwJS("acceptWaveForm error (from C++)", true);
-  }
+void recognizer::acceptWaveForm(int addr, int len) {
+  queue.emplace(addr, len);
 }
 void recognizer::setGrm(const std::string& grm) {
  vosk_recognizer_set_grm(rec, grm.c_str());
--- a/src/recognizer.h
+++ b/src/recognizer.h
@@ -5,6 +5,7 @@
 #include <filesystem>
 #include <atomic>
 #include <thread>
+#include <queue>

 #include <emscripten/wasmfs.h>
 #include <emscripten/webaudio.h>
@@ -15,12 +16,17 @@
 extern void throwJS(const char* msg, bool err = false);
 namespace fs = std::filesystem;

+struct audioData {
+  float* addr{};
+  int len{};
+  audioData(int addr, int len);
+};
 struct recognizer {
  int index{};
  VoskRecognizer* rec{};
  recognizer(model* model, float sampleRate, int index);
  ~recognizer();
-  void acceptWaveForm(float* data, int len);
+  void acceptWaveForm(int addr, int len);
  void fireEv(const char* type, const char* content);
  void setSpkModel(spkModel* model);
  void setGrm(const std::string& grm);
--- a/src/vosk.patch
+++ b/src/vosk.patch
@@ -0,0 +1,16 @@
+diff --git a/src/recognizer.cc b/src/recognizer.cc
+index 1da6e6b..5e0652a 100644
+--- a/src/recognizer.cc
+++ b/src/recognizer.cc
+@@ -384,8 +384,10 @@ bool Recognizer::AcceptWaveform(const float *fdata, int len)
+ {
+     Vector<BaseFloat> wave;
+     wave.Resize(len, kUndefined);
+-    for (int i = 0; i < len; i++)
+    for (int i = 0; i < len; i++) {
+         wave(i) = fdata[i];
+        wave(i) *= 32768;
+    }
+     return AcceptWaveform(wave);
+ }
+