patrick/Vosklet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove zstd and use JS decompressionStream => support gzip, 4.4MB -> 3.9MB, change example model.

Browse Source
This commit is contained in:
msqr1
2024-02-06 19:29:27 -08:00
parent 31ee35e1f7
commit 6fd7c7e191
8 changed files with 99 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/Makefile
View File

@@ -50,19 +50,12 @@ prepare:
. $(EMSDK)/emsdk_env.sh && \
export PATH=:$$PATH:$(EMSDK)/upstream/bin
zstd: prepare
rm -rf /tmp/zstd && \
git clone -b v1.5.5 --depth=1 https://github.com/facebook/zstd /tmp/zstd && \
cd /tmp/zstd && \
HAVE_THREAD=0 ZSTD_LEGACY_SUPPORT=0 HAVE_ZLIB=0 HAVE_LZMA=0 HAVE_LZ4=0 ZSTD_NOBENCH=1 ZSTD_NODICT=1 ZSTD_NOCOMPRESS=1 BACKTRACE=0 PREFIX=$(ZSTD) CPPFLAGS="-O3 -flto" LDFLAGS="-O3 -flto" emmake make -j$(COMPILE_JOBS) install && \
rm -rf /tmp/zstd
libarchive: | zstd
libarchive: prepare
rm -rf /tmp/libarchive && \
git clone -b v3.7.2 --depth=1 https://github.com/libarchive/libarchive /tmp/libarchive && \
cd /tmp/libarchive && \
build/autogen.sh && \
CPPFLAGS="-I$(ZSTD)/include -flto" LDFLAGS="-L$(ZSTD)/lib -flto" emconfigure ./configure --prefix=$(LIBARCHIVE) --without-lz4 --without-lzma --without-zlib --without-bz2lib --without-xml2 --without-expat --without-cng --without-openssl --without-libb2 --disable-bsdunzip --disable-xattr --disable-acl --disable-bsdcpio --disable-bsdcat --disable-rpath --disable-maintainer-mode --disable-dependency-tracking --enable-static --disable-shared && \
CPPFLAGS="-O3 -flto" LDFLAGS="-O3 -flto" emconfigure ./configure --prefix=$(LIBARCHIVE) --without-lz4 --without-lzma --without-zlib --without-bz2lib --without-xml2 --without-expat --without-cng --without-openssl --without-libb2 --without-zstd --disable-bsdunzip --disable-xattr --disable-acl --disable-bsdcpio --disable-bsdcat --disable-rpath --disable-maintainer-mode --disable-dependency-tracking --enable-static --disable-shared && \
emmake make -j$(COMPILE_JOBS) install && \
rm -rf /tmp/libarchive

34
src/genericModel.cc
View File

@@ -28,13 +28,15 @@ bool genericModel::checkModel() {
}
void genericModel::afterFetch() {
thrd.setTask1([this](){
if(!extractModel() && fs::remove("/opfs/m0dEl.tzst",tank)) {
fs::current_path("/opfs");
fs::remove_all(storepath);
if(!extractModel()) {
fs::remove("/opfs/m0dEl.tar",tank);
fs::current_path("/opfs", tank);
fs::remove_all(storepath, tank);
fireEv("_continue", "Unable to extract model", index);
return;
}
fs::remove("/opfs/m0dEl.tzst",tank);
fs::remove("/opfs/m0dEl.tar",tank);
fs::remove("README",tank);
std::ofstream idFile("id");
if(!idFile.is_open()) {
fs::current_path("/opfs");
@@ -55,21 +57,25 @@ void genericModel::afterFetch() {
});
}
bool genericModel::extractModel() {
std::string path{};
static std::string path{};
archive* src {archive_read_new()};
archive_entry* entry {};
archive_read_support_filter_all(src);
archive_read_support_format_all(src);
archive_read_open_filename(src, "/opfs/m0dEl.tzst", 10240);
archive* dst{archive_write_disk_new()};
static archive_entry* entry{};
archive_read_support_format_tar(src);
archive_read_open_filename(src, "/opfs/m0dEl.tar", 10240);
archive_write_disk_set_standard_lookup(dst);
archive_write_disk_set_options(dst, ARCHIVE_EXTRACT_NO_AUTODIR | ARCHIVE_EXTRACT_UNLINK);
if(archive_errno(src) != 0) return false;
while (archive_read_next_header(src, &entry) == ARCHIVE_OK) {
if(archive_errno(dst) != 0) return false;
while(archive_read_next_header2(src, entry) == ARCHIVE_OK) {
path = archive_entry_pathname(entry);
// Strip first component, keep relative path
path = "." + path.substr(path.find("/"));
archive_entry_set_pathname(entry, path.c_str());
archive_read_extract(src, entry, ARCHIVE_EXTRACT_UNLINK | ARCHIVE_EXTRACT_NO_AUTODIR);
path = "." + path.substr(path.find("/")); // Strip 1st component
emscripten_console_log(archive_entry_pathname(entry));
archive_read_extract2(src, entry, dst);
if(archive_errno(src) != 0) return false;
if(archive_errno(dst) != 0) return false;
}
archive_read_free(src);
archive_write_free(dst);
return true;
}

10
src/pre.js
View File

@@ -1,4 +1,5 @@
let objs = []
let dStream = new DecompressionStream("gzip")
Module.revokeURLs = () => {
URL.revokeObjectURL(pthreadUrl)
URL.revokeObjectURL(processorUrl)
@@ -96,8 +97,13 @@ Module.makeModel = async (url, storepath, id) => {
if(!res.ok) {
return reject("Unable to download model")
}
let wStream = await (await (await navigator.storage.getDirectory()).getFileHandle("m0dEl.tzst", {create : true})).createWritable()
await wStream.write(await res.arrayBuffer())
let wStream = await (await (await navigator.storage.getDirectory()).getFileHandle("m0dEl.tar", {create : true})).createWritable()
let tarReader = res.body.pipeThrough(dStream).getReader()
while(true) {
let readRes = await tarReader.read()
if(!readRes.done) await wStream.write(readRes.value)
else break
}
await wStream.close()
mdl.obj.afterFetch()
})()