Add more debug info, untested

README.md

@@ -1,18 +1,21 @@
 # Overview
-- **STILL UNDER DEVELOPMENT!**
 - A speech recognizer built on Vosk that can be run on the browser, inspired by [vosk-browser](https://github.com/ccoreilly/vosk-browser), but built from scratch and no code taken!
 - Designed with basic/nothrow exception safety
 - See the *usage* folder's README.md for API documentation, notes.
 - See the *devel* folder for the newest build (not guaranteed to work) and the JS build script
 
 # Additions to vosk-browser:
-- Download multiple models
-- Model storage path management (for multiple models)
+- Support multiple models
+- Model storage path management
 - Model ID management (for model updates)
 - Smaller JS size (>3.1MB vs 1.4MB)
 - All related files (pthread worker, worklet processor,...) are merged
 - Shorter from-scratch build time
 
-# Basic usage
-- Use the prebuilt binary through JsDelivr.
+# Disclaimer:
+- Will log empty statement into JS console, see *global.cc* line 34
+- Model loading can be slow due due to slow OPFS reading speed
+
+# Basic usage
+- Use the prebuilt binary through JsDelivr
 

devel/Vosklet.worker.js

@@ -1 +1,165 @@
-"use strict";var Module={};var initializedJS=false;function threadPrintErr(...args){var text=args.join(" ");console.error(text)}function threadAlert(...args){var text=args.join(" ");postMessage({cmd:"alert",text:text,threadId:Module["_pthread_self"]()})}var err=threadPrintErr;self.alert=threadAlert;Module["instantiateWasm"]=(info,receiveInstance)=>{var module=Module["wasmModule"];Module["wasmModule"]=null;var instance=new WebAssembly.Instance(module,info);return receiveInstance(instance)};self.onunhandledrejection=e=>{throw e.reason||e};function handleMessage(e){try{if(e.data.cmd==="load"){let messageQueue=[];self.onmessage=e=>messageQueue.push(e);self.startWorker=instance=>{Module=instance;postMessage({"cmd":"loaded"});for(let msg of messageQueue){handleMessage(msg)}self.onmessage=handleMessage};Module["wasmModule"]=e.data.wasmModule;for(const handler of e.data.handlers){Module[handler]=(...args)=>{postMessage({cmd:"callHandler",handler:handler,args:args})}}Module["wasmMemory"]=e.data.wasmMemory;Module["buffer"]=Module["wasmMemory"].buffer;Module["ENVIRONMENT_IS_PTHREAD"]=true;if(typeof e.data.urlOrBlob=="string"){importScripts(e.data.urlOrBlob)}else{var objectUrl=URL.createObjectURL(e.data.urlOrBlob);importScripts(objectUrl);URL.revokeObjectURL(objectUrl)}loadVosklet(Module)}else if(e.data.cmd==="run"){Module["__emscripten_thread_init"](e.data.pthread_ptr,0,0,1);Module["__emscripten_thread_mailbox_await"](e.data.pthread_ptr);Module["establishStackSpace"]();Module["PThread"].receiveObjectTransfer(e.data);Module["PThread"].threadInitTLS();if(!initializedJS){Module["__embind_initialize_bindings"]();initializedJS=true}try{Module["invokeEntryPoint"](e.data.start_routine,e.data.arg)}catch(ex){if(ex!="unwind"){throw ex}}}else if(e.data.cmd==="cancel"){if(Module["_pthread_self"]()){Module["__emscripten_thread_exit"](-1)}}else if(e.data.target==="setimmediate"){}else if(e.data.cmd==="checkMailbox"){if(initializedJS){Module["checkMailbox"]()}}else if(e.data.cmd){err(`worker.js received unknown command ${e.data.cmd}`);err(e.data)}}catch(ex){Module["__emscripten_thread_crashed"]?.();throw ex}}self.onmessage=handleMessage;
+/**
+ * @license
+ * Copyright 2015 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+// Pthread Web Worker startup routine:
+// This is the entry point file that is loaded first by each Web Worker
+// that executes pthreads on the Emscripten application.
+
+'use strict';
+
+var Module = {};
+
+// Thread-local guard variable for one-time init of the JS state
+var initializedJS = false;
+
+function assert(condition, text) {
+  if (!condition) abort('Assertion failed: ' + text);
+}
+
+function threadPrintErr(...args) {
+  var text = args.join(' ');
+  console.error(text);
+}
+function threadAlert(...args) {
+  var text = args.join(' ');
+  postMessage({cmd: 'alert', text, threadId: Module['_pthread_self']()});
+}
+// We don't need out() for now, but may need to add it if we want to use it
+// here. Or, if this code all moves into the main JS, that problem will go
+// away. (For now, adding it here increases code size for no benefit.)
+var out = () => { throw 'out() is not defined in worker.js.'; }
+var err = threadPrintErr;
+self.alert = threadAlert;
+var dbg = threadPrintErr;
+
+Module['instantiateWasm'] = (info, receiveInstance) => {
+  // Instantiate from the module posted from the main thread.
+  // We can just use sync instantiation in the worker.
+  var module = Module['wasmModule'];
+  // We don't need the module anymore; new threads will be spawned from the main thread.
+  Module['wasmModule'] = null;
+  var instance = new WebAssembly.Instance(module, info);
+  // TODO: Due to Closure regression https://github.com/google/closure-compiler/issues/3193,
+  // the above line no longer optimizes out down to the following line.
+  // When the regression is fixed, we can remove this if/else.
+  return receiveInstance(instance);
+}
+
+// Turn unhandled rejected promises into errors so that the main thread will be
+// notified about them.
+self.onunhandledrejection = (e) => {
+  throw e.reason || e;
+};
+
+function handleMessage(e) {
+  try {
+    if (e.data.cmd === 'load') { // Preload command that is called once per worker to parse and load the Emscripten code.
+
+    // Until we initialize the runtime, queue up any further incoming messages.
+    let messageQueue = [];
+    self.onmessage = (e) => messageQueue.push(e);
+
+    // And add a callback for when the runtime is initialized.
+    self.startWorker = (instance) => {
+      Module = instance;
+      // Notify the main thread that this thread has loaded.
+      postMessage({ 'cmd': 'loaded' });
+      // Process any messages that were queued before the thread was ready.
+      for (let msg of messageQueue) {
+        handleMessage(msg);
+      }
+      // Restore the real message handler.
+      self.onmessage = handleMessage;
+    };
+
+      // Module and memory were sent from main thread
+      Module['wasmModule'] = e.data.wasmModule;
+
+      // Use `const` here to ensure that the variable is scoped only to
+      // that iteration, allowing safe reference from a closure.
+      for (const handler of e.data.handlers) {
+        Module[handler] = (...args) => {
+          dbg(`calling handler on main thread: ${handler}`);
+          postMessage({ cmd: 'callHandler', handler, args: args });
+        }
+      }
+
+      Module['wasmMemory'] = e.data.wasmMemory;
+
+      Module['wasmOffsetData'] = e.data.wasmOffsetConverter;
+
+      Module['buffer'] = Module['wasmMemory'].buffer;
+
+      Module['workerID'] = e.data.workerID;
+
+      Module['ENVIRONMENT_IS_PTHREAD'] = true;
+
+      if (typeof e.data.urlOrBlob == 'string') {
+        importScripts(e.data.urlOrBlob);
+      } else {
+        var objectUrl = URL.createObjectURL(e.data.urlOrBlob);
+        importScripts(objectUrl);
+        URL.revokeObjectURL(objectUrl);
+      }
+      loadVosklet(Module);
+    } else if (e.data.cmd === 'run') {
+      // Pass the thread address to wasm to store it for fast access.
+      Module['__emscripten_thread_init'](e.data.pthread_ptr, /*is_main=*/0, /*is_runtime=*/0, /*can_block=*/1);
+
+      // Await mailbox notifications with `Atomics.waitAsync` so we can start
+      // using the fast `Atomics.notify` notification path.
+      Module['__emscripten_thread_mailbox_await'](e.data.pthread_ptr);
+
+      assert(e.data.pthread_ptr);
+      // Also call inside JS module to set up the stack frame for this pthread in JS module scope
+      Module['establishStackSpace']();
+      Module['PThread'].receiveObjectTransfer(e.data);
+      Module['PThread'].threadInitTLS();
+
+      if (!initializedJS) {
+        // Embind must initialize itself on all threads, as it generates support JS.
+        // We only do this once per worker since they get reused
+        Module['__embind_initialize_bindings']();
+        initializedJS = true;
+      }
+
+      try {
+        Module['invokeEntryPoint'](e.data.start_routine, e.data.arg);
+      } catch(ex) {
+        if (ex != 'unwind') {
+          // The pthread "crashed".  Do not call `_emscripten_thread_exit` (which
+          // would make this thread joinable).  Instead, re-throw the exception
+          // and let the top level handler propagate it back to the main thread.
+          throw ex;
+        }
+        dbg(`Pthread 0x${Module['_pthread_self']().toString(16)} completed its main entry point with an 'unwind', keeping the worker alive for asynchronous operation.`);
+      }
+    } else if (e.data.cmd === 'cancel') { // Main thread is asking for a pthread_cancel() on this thread.
+      if (Module['_pthread_self']()) {
+        Module['__emscripten_thread_exit'](-1);
+      }
+    } else if (e.data.target === 'setimmediate') {
+      // no-op
+    } else if (e.data.cmd === 'checkMailbox') {
+      if (initializedJS) {
+        Module['checkMailbox']();
+      }
+    } else if (e.data.cmd) {
+      // The received message looks like something that should be handled by this message
+      // handler, (since there is a e.data.cmd field present), but is not one of the
+      // recognized commands:
+      err(`worker.js received unknown command ${e.data.cmd}`);
+      err(e.data);
+    }
+  } catch(ex) {
+    err(`worker.js onmessage() captured an uncaught exception: ${ex}`);
+    if (ex?.stack) err(ex.stack);
+    Module['__emscripten_thread_crashed']?.();
+    throw ex;
+  }
+};
+
+self.onmessage = handleMessage;

devel/index.html

@@ -4,9 +4,27 @@
     <script src="Vosklet.js"></script>
     <script>
       window.onload = async () => {
-        const module = await loadVosklet();
-        const model = await module.makeModel("../usage/en-model.tgz","model","ID");
-        const recognizer = await module.makeRecognizer()
+        window.ctx = new AudioContext()
+        let micNode = ctx.createMediaStreamSource(await navigator.mediaDevices.getUserMedia({
+          video: false,
+          audio: {
+            echoCancellation: true,
+            noiseSuppression: true,
+            channelCount: 1,
+            sampleRate: 16000
+          },
+        }))
+        window.module = await loadVosklet()
+        window.model = await module.makeModel("../usage/en-model.tgz","model","ID")
+        window.recognizer = await module.makeRecognizer(model, ctx.sampleRate)
+        recognizer.addEventListener("result", e => {
+          console.log("Result: ", e.detail)
+        })
+        recognizer.addEventListener("partialResult", e => {
+          console.log("Partial result: ", e.detail)
+        })
+        //let recNode = recognizer.getNode(ctx);
+        
       }
     </script>
   </head>

devel/test.sh

@@ -8,7 +8,7 @@ OPENFST=$(realpath openfst) &&
 LIBARCHIVE=$(realpath libarchive) &&
 CLAPACK_WASM=$(realpath clapack-wasm) &&
 
-MAX_MEMORY=${MAX_MEMORY:-300mb} &&
+MAX_MEMORY=${MAX_MEMORY:-500mb} &&
 MAX_THREADS=${MAX_THREADS:-2} &&
 EMSDK=${EMSDK:-$(realpath emsdk)} &&
 
@@ -28,7 +28,7 @@ fi
 . $EMSDK/emsdk_env.sh &&
 
 cd $SRC &&
-MODE=1 && # 0: Ultra debug info, 1: Extremely optimized release, else custom
+MODE=2 && # 0: Ultra debug info, 1: Extremely optimized release, else custom
 echo "Mode = $MODE" &&
 if [ $MODE = 0 ]; then
   em++ -O0 global.cc genericModel.cc recognizer.cc bindings.cc -sWASMFS -sWASM_BIGINT -sSINGLE_FILE -sGZIP_EMBEDDINGS -sMODULARIZE -sEMBIND_STD_STRING_IS_UTF8 -sPTHREAD_POOL_DELAY_LOAD  -sRUNTIME_DEBUG -sSTACK_OVERFLOW_CHECK=2 -sTEXTDECODER=2 -sPTHREAD_POOL_SIZE_STRICT=2 -sASSERTIONS=2 -sINITIAL_MEMORY=$MAX_MEMORY -sPTHREAD_POOL_SIZE=$MAX_THREADS -sDISABLE_EXCEPTION_CATCHING=0 -sEXIT_RUNTIME=0 -sINVOKE_RUN=0 -sPOLYFILL=0 -sEXPORTED_FUNCTIONS=_malloc -sEXPORT_NAME=loadVosklet -sMALLOC=emmalloc -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8OnStack -sENVIRONMENT=web,worker -I. -I$LIBARCHIVE/include -I$VOSK/src -L$LIBARCHIVE/lib -larchive -L$KALDI/src -l:online2/kaldi-online2.a -l:decoder/kaldi-decoder.a -l:ivector/kaldi-ivector.a -l:gmm/kaldi-gmm.a -l:tree/kaldi-tree.a -l:feat/kaldi-feat.a -l:cudamatrix/kaldi-cudamatrix.a -l:lat/kaldi-lat.a -l:lm/kaldi-lm.a -l:rnnlm/kaldi-rnnlm.a -l:hmm/kaldi-hmm.a -l:nnet3/kaldi-nnet3.a -l:transform/kaldi-transform.a -l:matrix/kaldi-matrix.a -l:fstext/kaldi-fstext.a -l:util/kaldi-util.a -l:base/kaldi-base.a -L$OPENFST/lib -l:libfst.a -l:libfstngram.a -L$CLAPACK_WASM -l:CBLAS/lib/cblas.a -l:CLAPACK-3.2.1/lapack.a -l:CLAPACK-3.2.1/libcblaswr.a -l:f2c_BLAS-3.8.0/blas.a -l:libf2c/libf2c.a -L$VOSK/src -l:vosk.a -lopfs.js -lembind -pthread -flto -msimd128 --embind-emit-tsd Vosklet.d.ts -fsanitize=undefined -fsanitize=address -fsanitize=leak -g3 --pre-js pre.js -o ../devel/Vosklet.js 

src/genericModel.cc

@@ -5,30 +5,31 @@ void genericModel::load() {
   thrd.addTask([this](){
     if(normalMdl) mdl = vosk_model_new(storepath.c_str());
     else vosk_spk_model_new(storepath.c_str());
-    if(normalMdl && std::get<0>(mdl) == nullptr || std::get<1>(mdl) == nullptr) fireEv(index, "Unable to load model for recognition");
-    else fireEv(index, nullptr);
+    emscripten_console_log("Loading finished!");
+    if(normalMdl ? std::get<0>(mdl) == nullptr : std::get<1>(mdl) == nullptr) fireEv(index, "Unable to load model for recognition");
+    else fireEv(index, "0");
   });
 }
 bool genericModel::checkFiles() {
   if(normalMdl) {
-    return fs::exists(storepath + "/am/final.mdl", tank) &&
-      fs::exists(storepath + "/conf/mfcc.conf", tank) &&
-      fs::exists(storepath + "/conf/model.conf", tank) &&
-      fs::exists(storepath + "/graph/phones/word_boundary.int", tank) &&
-      fs::exists(storepath + "/graph/Gr.fst", tank) &&
-      fs::exists(storepath + "/graph/HCLr.fst", tank) &&
-      fs::exists(storepath + "/graph/disambig_tid.int", tank) &&
-      fs::exists(storepath + "/ivector/final.dubm", tank) &&
-      fs::exists(storepath + "/ivector/final.ie", tank) &&
-      fs::exists(storepath + "/ivector/final.mat", tank) &&
-      fs::exists(storepath + "/ivector/global_cmvn.stats", tank) && 
-      fs::exists(storepath + "/ivector/online_cmvn.conf", tank) &&
-      fs::exists(storepath + "/ivector/splice.conf", tank);
+    return fs::exists(storepath + "/am/final.mdl") &&
+      fs::exists(storepath + "/conf/mfcc.conf") &&
+      fs::exists(storepath + "/conf/model.conf") &&
+      fs::exists(storepath + "/graph/phones/word_boundary.int") &&
+      fs::exists(storepath + "/graph/Gr.fst") &&
+      fs::exists(storepath + "/graph/HCLr.fst") &&
+      fs::exists(storepath + "/graph/disambig_tid.int") &&
+      fs::exists(storepath + "/ivector/final.dubm") &&
+      fs::exists(storepath + "/ivector/final.ie") &&
+      fs::exists(storepath + "/ivector/final.mat") &&
+      fs::exists(storepath + "/ivector/global_cmvn.stats") && 
+      fs::exists(storepath + "/ivector/online_cmvn.conf") &&
+      fs::exists(storepath + "/ivector/splice.conf");
   }
-  return fs::exists(storepath + "/mfcc.conf", tank) && 
-  fs::exists(storepath + "/final.ext.raw", tank) && 
-  fs::exists(storepath + "/mean.vec", tank) && 
-  fs::exists(storepath + "/transform.mat", tank);
+  return fs::exists(storepath + "/mfcc.conf") && 
+  fs::exists(storepath + "/final.ext.raw") && 
+  fs::exists(storepath + "/mean.vec") && 
+  fs::exists(storepath + "/transform.mat");
 }
 genericModel::~genericModel() {
   archive_entry_free(entry);
@@ -53,8 +54,8 @@ void genericModel::check() {
       fireEv(index, "OPFS initialization failed");
       return;
     }
-    if(!checkFiles() && !fs::exists(storepath + "/id", tank)) {
-      emscripten_console_log("Model is not available, fetching...");
+    if(!checkFiles() && !fs::exists(storepath + "/id")) {
+      emscripten_console_log("Model is not available or incorrect file, fetching...");
       fireEv(index, "1");
       return;
     }
@@ -80,8 +81,8 @@ void genericModel::afterFetch() {
   thrd.addTask([this](){
     emscripten_console_log("Trying to extract...");
     if(!extract()) {
-      fs::remove(storepath + ".tar",tank);
-      fs::remove_all(storepath, tank);
+      fs::remove(storepath + ".tar");
+      fs::remove_all(storepath);
       emscripten_console_log("Unable to extract model");
       fireEv(index, "Unable to extract model");
       return;
@@ -97,16 +98,16 @@ void genericModel::afterFetch() {
     if(idFd == -1) {
       emscripten_console_log("Unable to create ID file");
       fireEv(index, "Unable to create ID file");
-      fs::remove(storepath + ".tar",tank);
-      fs::remove_all(storepath, tank);
+      fs::remove(storepath + ".tar");
+      fs::remove_all(storepath);
       return;
     }
     emscripten_console_log("Writing ID...");
     if(write(idFd,id.data(), id.size()) == -1) {
       emscripten_console_log("Unable to write new ID");
       fireEv(index, "Unable to write new ID");
-      fs::remove(storepath + ".tar",tank);
-      fs::remove_all(storepath, tank);
+      fs::remove(storepath + ".tar");
+      fs::remove_all(storepath);
       close(idFd);
       return;
     };
@@ -148,8 +149,8 @@ bool genericModel::extract() {
       return false;
     }
   }
-  fs::remove(storepath + "/README",tank);
-  fs::remove(storepath + ".tar" ,tank);
+  fs::remove(storepath + "/README");
+  fs::remove(storepath + ".tar" );
   archive_read_free(src);
   return true;
 }

src/genericModel.h

@@ -14,7 +14,7 @@ namespace fs = std::filesystem;
 
 struct genericModel {
   bool normalMdl;
-  bool recognizerUsedThrd;
+  bool recognizerUsedThrd{};
   int index;
   std::string storepath;
   std::string id;

src/global.cc

@@ -1,7 +1,6 @@
 #include "global.h"
 
 pthread_t dstThrd{pthread_self()};
-std::error_code tank{};
 bool OPFSOk{};
 bool OPFSTried{};
 ProxyingQueue glbQ{};
@@ -19,9 +18,10 @@ void fireEv(int index, const char* content, const char* type) {
 reusableThrd::reusableThrd() {
   std::thread thrd{[this](){
     while(!done.test(std::memory_order_relaxed)) {
-      blocker.wait(done.test(std::memory_order_relaxed) || queue.empty(), std::memory_order_relaxed);
+      blocker.wait(queue.empty() || done.test(std::memory_order_relaxed),  std::memory_order_relaxed);
       blocker.clear(std::memory_order_relaxed);
       while(!queue.empty()) {
+        emscripten_console_log("==========Executing task===========");
         queue.front()();
         queue.pop();
       }
@@ -31,6 +31,8 @@ reusableThrd::reusableThrd() {
 }
 void reusableThrd::addTask(std::function<void()>&& task) {
   queue.emplace(task);
+  // The following line will make -O3 magically work. Yes, I know, it is a god dang PRINT STATEMENT.
+  //emscripten_console_log("");
   blocker.test_and_set(std::memory_order_relaxed);
   blocker.notify_one();
 }

src/recognizer.cc

@@ -1,14 +1,17 @@
 #include "recognizer.h" 
 
 recognizer::recognizer(genericModel* model, float sampleRate, int index) : index(index) {
+  emscripten_console_log("Recognizer constructor called...");
   rec = vosk_recognizer_new(std::get<0>(model->mdl),sampleRate);
   finishConstruction(model, nullptr);
 }
-recognizer::recognizer(genericModel* model, genericModel* spkMdl, float sampleRate, int index) {
+recognizer::recognizer(genericModel* model, genericModel* spkMdl, float sampleRate, int index) : index(index) {
+  emscripten_console_log("Recognizer constructor called...");
   rec = vosk_recognizer_new_spk(std::get<0>(model->mdl), sampleRate, std::get<1>(spkMdl->mdl));
   finishConstruction(model, spkMdl);
 }
-recognizer::recognizer(genericModel* model, const std::string& grm, float sampleRate, int index, int dummy) {
+recognizer::recognizer(genericModel* model, const std::string& grm, float sampleRate, int index, int dummy) : index(index) {
+  emscripten_console_log("Recognizer constructor called...");
   rec = vosk_recognizer_new_grm(std::get<0>(model->mdl), sampleRate, grm.c_str());
   finishConstruction(model, nullptr);
 }
@@ -26,6 +29,7 @@ void recognizer::finishConstruction(genericModel* model, genericModel* spkModel)
     return;
   }
   auto main {[this](){
+    emscripten_console_log("Recognizer loaded!");
     fireEv(index, "0");
     while(!done.test(std::memory_order_relaxed)) {
       controller.wait(!done.test(std::memory_order_relaxed), std::memory_order_relaxed);
@@ -43,12 +47,16 @@ void recognizer::finishConstruction(genericModel* model, genericModel* spkModel)
   if(!model->recognizerUsedThrd) {
     model->recognizerUsedThrd = true;
     model->thrd.addTask(main);
+    emscripten_console_log("Adding task to model thread...");
     return;
   }
   if(spkModel != nullptr && !spkModel->recognizerUsedThrd) {
     spkModel->recognizerUsedThrd = true;
     spkModel->thrd.addTask(main);
+    emscripten_console_log("Adding task to speaker model thread...");
+    return;
   }
+  emscripten_console_log("New recognizer thread");
   std::thread t{main};
   t.detach();
 }