From d04fb30048a0dd63db71a9f3ef0d4f5af518de48 Mon Sep 17 00:00:00 2001
From: nick-delirium <nikita@openreplay.com>
Date: Mon, 4 Dec 2023 15:54:14 +0100
Subject: [PATCH] feat(ui): compression zstd support

---
 frontend/app/player/web/MessageLoader.ts     | 58 ++++++++++++--------
 frontend/app/player/web/network/loadFiles.ts |  6 +-
 frontend/package.json                        |  1 +
 frontend/yarn.lock                           |  8 +++
 third-party.md                               |  1 +
 5 files changed, 49 insertions(+), 25 deletions(-)

diff --git a/frontend/app/player/web/MessageLoader.ts b/frontend/app/player/web/MessageLoader.ts
index c935d1017..9e82401cc 100644
--- a/frontend/app/player/web/MessageLoader.ts
+++ b/frontend/app/player/web/MessageLoader.ts
@@ -7,6 +7,7 @@ import type {
   Message,
 } from './messages';
 import logger from 'App/logger';
+import * as fzstd from 'fzstd';
 
 
 interface State {
@@ -37,31 +38,44 @@ export default class MessageLoader {
       ? (b: Uint8Array) => decryptSessionBytes(b, this.session.fileKey!)
       : (b: Uint8Array) => Promise.resolve(b)
     // Each time called - new fileReader created
-    const fileReader = new MFileReader(new Uint8Array(), this.session.startedAt)
-    return (b: Uint8Array) => decrypt(b).then(b => {
-      toggleStatus?.(true);
-      fileReader.append(b)
-      fileReader.checkForIndexes()
-      const msgs: Array<Message & { _index?: number }> = []
-      for (let msg = fileReader.readNext();msg !== null;msg = fileReader.readNext()) {
-        msgs.push(msg)
+    const unarchived = (b: Uint8Array) => {
+      const isZstd = b[0] === 0x28 && b[1] === 0xb5 && b[2] === 0x2f && b[3] === 0xfd
+      // zstd magical numbers 40 181 47 253
+      console.log(isZstd, b[0], b[1], b[2], b[3])
+      if (isZstd) {
+       return fzstd.decompress(b)
+      } else {
+        return b
       }
-      const sorted = msgs.sort((m1, m2) => {
-        return m1.time - m2.time
-      })
+    }
+    const fileReader = new MFileReader(new Uint8Array(), this.session.startedAt)
+    return (b: Uint8Array) => {
+      decrypt(b).then(b => {
+        const data = unarchived(b)
+        toggleStatus?.(true);
+        fileReader.append(data)
+        fileReader.checkForIndexes()
+        const msgs: Array<Message & { _index?: number }> = []
+        for (let msg = fileReader.readNext();msg !== null;msg = fileReader.readNext()) {
+          msgs.push(msg)
+        }
+        const sorted = msgs.sort((m1, m2) => {
+          return m1.time - m2.time
+        })
 
-      sorted.forEach(msg => {
-        this.messageManager.distributeMessage(msg)
-      })
-      logger.info("Messages count: ", msgs.length, sorted, file)
+        sorted.forEach(msg => {
+          this.messageManager.distributeMessage(msg)
+        })
+        logger.info("Messages count: ", msgs.length, sorted, file)
 
-      this.messageManager._sortMessagesHack(sorted)
-      toggleStatus?.(false);
-      this.messageManager.setMessagesLoading(false)
-    }).catch(e => {
-      console.error(e)
-      this.uiErrorHandler?.error('Error parsing file: ' + e.message)
-    })
+        this.messageManager._sortMessagesHack(sorted)
+        toggleStatus?.(false);
+        this.messageManager.setMessagesLoading(false)
+      }).catch(e => {
+        console.error(e)
+        this.uiErrorHandler?.error('Error parsing file: ' + e.message)
+      })
+    }
   }
 
   loadDomFiles(urls: string[], parser: (b: Uint8Array) => Promise<void>) {
diff --git a/frontend/app/player/web/network/loadFiles.ts b/frontend/app/player/web/network/loadFiles.ts
index 273c27ec6..084f54780 100644
--- a/frontend/app/player/web/network/loadFiles.ts
+++ b/frontend/app/player/web/network/loadFiles.ts
@@ -58,13 +58,13 @@ async function requestEFSMobFile(filename: string) {
 }
 
 const processAPIStreamResponse = (response: Response, skippable: boolean) => {
-  return new Promise<ArrayBuffer>((res, rej) => {
+  return new Promise<Blob>((res, rej) => {
     if (response.status === 404 && skippable) {
       return rej(ALLOWED_404)
     }
     if (response.status >= 400) {
       return rej(`Bad file status code ${response.status}. Url: ${response.url}`)
     }
-    res(response.arrayBuffer())
-  }).then(buffer => new Uint8Array(buffer))
+    res(response.blob())
+  }).then(async blob => new Uint8Array(await blob.arrayBuffer()))
 }
diff --git a/frontend/package.json b/frontend/package.json
index 1ef811e19..b5eef6c49 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -37,6 +37,7 @@
     "country-flag-icons": "^1.5.7",
     "deep-diff": "^1.0.2",
     "fflate": "^0.7.4",
+    "fzstd": "^0.1.0",
     "html-to-image": "^1.9.0",
     "html2canvas": "^1.4.1",
     "immutable": "^4.0.0-rc.12",
diff --git a/frontend/yarn.lock b/frontend/yarn.lock
index 46621984e..156721e02 100644
--- a/frontend/yarn.lock
+++ b/frontend/yarn.lock
@@ -12293,6 +12293,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"fzstd@npm:^0.1.0":
+  version: 0.1.0
+  resolution: "fzstd@npm:0.1.0"
+  checksum: 0d3f0e499d6eb285e390cdcccd4ee8fb56c83834542153e47fc60f41971ee485dde8e0569db01c6ae6afe7c1335236ebcc612563771f20f4ec9b82c6d11fb5c7
+  languageName: node
+  linkType: hard
+
 "gauge@npm:^3.0.0":
   version: 3.0.2
   resolution: "gauge@npm:3.0.2"
@@ -18130,6 +18137,7 @@ __metadata:
     fflate: ^0.7.4
     file-loader: ^6.2.0
     flow-bin: ^0.115.0
+    fzstd: ^0.1.0
     html-to-image: ^1.9.0
     html-webpack-plugin: ^5.5.0
     html2canvas: ^1.4.1
diff --git a/third-party.md b/third-party.md
index 5c373781f..48a7f824a 100644
--- a/third-party.md
+++ b/third-party.md
@@ -121,3 +121,4 @@ Below is the list of dependencies used in OpenReplay software. Licenses may chan
 | eget | MIT | Infrastructure |
 | @medv/finder | MIT | JavaScript |
 | fflate | MIT | JavaScript |
+| fzstd | MIT | JavaScript |