feat(ui): switch canvas player to images (#1925)

* feat(ui): switch canvas player to images

* feat(ui): complete player

* fix(ui): expect undefined .tar for canvas

* fix(ui): improve error handling

* fix(ui): file format

frontend/app/components/shared/SessionSettings/components/CaptureRate.tsx

@@ -42,7 +42,7 @@ function CaptureRate(props: Props) {
   }, [projectId]);
 
   React.useEffect(() => {
-    setConditions(captureConditions.map((condition: any) => new Conditions(condition, true)));
+    setConditions(captureConditions.map((condition: any) => new Conditions(condition, true, isMobile)));
   }, [captureConditions]);
 
   const onCaptureRateChange = (input: string) => {

frontend/app/mstore/types/FeatureFlag.ts

@@ -7,12 +7,12 @@ export class Conditions {
   filter = new Filter().fromJson({ name: 'Rollout conditions', filters: [] });
   name = 'Condition Set';
 
-  constructor(data?: Record<string, any>, isConditional?: boolean) {
+  constructor(data?: Record<string, any>, isConditional?: boolean, isMobile?: boolean) {
     makeAutoObservable(this);
     this.name = data?.name;
     if (data && (data.rolloutPercentage || data.captureRate)) {
       this.rolloutPercentage = data.rolloutPercentage ?? data.captureRate;
-      this.filter = new Filter(isConditional).fromJson(data);
+      this.filter = new Filter(isConditional, isMobile).fromJson(data);
     }
   }
 

frontend/app/mstore/types/filter.ts

@@ -16,7 +16,7 @@ export default class Filter {
     page: number = 1
     limit: number = 10
 
-    constructor(private readonly isConditional = false) {
+    constructor(private readonly isConditional = false, private readonly isMobile = false) {
         makeAutoObservable(this, {
             filters: observable,
             eventsOrder: observable,
@@ -64,7 +64,7 @@ export default class Filter {
     fromJson(json: any) {
         this.name = json.name
         this.filters = json.filters.map((i: Record<string, any>) =>
-          new FilterItem(undefined, this.isConditional).fromJson(i)
+          new FilterItem(undefined, this.isConditional, this.isMobile).fromJson(i)
         );
         this.eventsOrder = json.eventsOrder
         return this

frontend/app/mstore/types/filterItem.ts

@@ -1,6 +1,6 @@
 import { makeAutoObservable, observable, action } from 'mobx';
-import { FilterKey, FilterType, FilterCategory } from 'Types/filter/filterType';
-import { filtersMap, conditionalFiltersMap } from 'Types/filter/newFilter';
+import { FilterKey, FilterType, FilterCategory }                          from 'Types/filter/filterType';
+import { filtersMap, conditionalFiltersMap, mobileConditionalFiltersMap } from 'Types/filter/newFilter';
 
 export default class FilterItem {
   type: string = '';
@@ -21,7 +21,11 @@ export default class FilterItem {
   completed: number = 0;
   dropped: number = 0;
 
-  constructor(data: any = {}, private readonly isConditional?: boolean) {
+  constructor(
+    data: any = {},
+    private readonly isConditional?: boolean,
+    private readonly isMobile?: boolean
+) {
     makeAutoObservable(this, {
       type: observable,
       key: observable,
@@ -61,7 +65,11 @@ export default class FilterItem {
     const isMetadata = json.type === FilterKey.METADATA;
     let _filter: any = (isMetadata ? filtersMap['_' + json.source] : filtersMap[json.type]) || {};
     if (this.isConditional) {
-      _filter = conditionalFiltersMap[json.type] || conditionalFiltersMap[json.source];
+      if (this.isMobile) {
+        _filter = mobileConditionalFiltersMap[json.type] || mobileConditionalFiltersMap[json.source];
+      } else {
+        _filter = conditionalFiltersMap[json.type] || conditionalFiltersMap[json.source];
+      }
     }
     if (mainFilterKey) {
       const mainFilter = filtersMap[mainFilterKey];

frontend/app/player/web/MessageLoader.ts

@@ -4,9 +4,9 @@ import MFileReader from './messages/MFileReader';
 import { loadFiles, requestEFSDom, requestEFSDevtools, requestTarball } from './network/loadFiles';
 import logger from 'App/logger';
 import unpack from 'Player/common/unpack';
+import unpackTar from 'Player/common/tarball';
 import MessageManager from 'Player/web/MessageManager';
 import IOSMessageManager from 'Player/mobile/IOSMessageManager';
-import unpackTar from 'Player/common/tarball';
 
 interface State {
   firstFileLoading: boolean;

frontend/app/player/web/TabManager.ts

@@ -169,13 +169,19 @@ export default class TabSessionManager {
       case MType.CanvasNode:
         const managerId = `${msg.timestamp}_${msg.nodeId}`;
         if (!this.canvasManagers[managerId]) {
-          const filename = `${managerId}.mp4`;
+          const fileId = managerId;
           const delta = msg.timestamp - this.sessionStart;
-          const fileUrl = this.session.canvasURL.find((url: string) => url.includes(filename));
+          const canvasNodeLinks = this.session.canvasURL.filter((url: string) => url.includes(fileId)) as string[];
+          const tarball = canvasNodeLinks.find((url: string) => url.includes('.tar.'));
+          const mp4file = canvasNodeLinks.find((url: string) => url.includes('.mp4'));
+          if (!tarball && !mp4file) {
+            console.error('no canvas recording provided')
+            break;
+          }
           const manager = new CanvasManager(
             msg.nodeId,
             delta,
-            fileUrl,
+            [tarball, mp4file],
             this.getNode as (id: number) => VElement | undefined
           );
           this.canvasManagers[managerId] = { manager, start: msg.timestamp, running: false };

frontend/app/player/web/managers/CanvasManager.ts

@@ -1,9 +1,26 @@
-import { VElement } from "Player/web/managers/DOM/VirtualDOM";
+import { TarFile } from 'js-untar';
+import ListWalker from 'Player/common/ListWalker';
+import { VElement } from 'Player/web/managers/DOM/VirtualDOM';
+import unpack from 'Player/common/unpack';
+import unpackTar from 'Player/common/tarball';
 
-export default class CanvasManager {
+const playMode = {
+  video: 'video',
+  snaps: 'snaps',
+} as const;
+
+const TAR_MISSING = 'TAR_404';
+const MP4_MISSING = 'MP4_404';
+
+type Timestamp = { time: number };
+
+export default class CanvasManager extends ListWalker<Timestamp> {
   private fileData: string | undefined;
-  private videoTag = document.createElement('video')
+  private videoTag = document.createElement('video');
+  private snapImage = document.createElement('img');
   private lastTs = 0;
+  private playMode: string = playMode.snaps;
+  private snapshots: Record<number, TarFile> = {};
 
   constructor(
     /**
@@ -14,21 +31,109 @@ export default class CanvasManager {
      * time between node creation and session start
      */
     private readonly delta: number,
-    private readonly filename: string,
-    private readonly getNode: (id: number) => VElement | undefined) {
-      // getting mp4 file composed of canvas snapshot images
-      fetch(this.filename).then((r) => {
-        if (r.status === 200) {
-          r.blob().then((blob) => {
-            this.fileData = URL.createObjectURL(blob);
-          })
+    private readonly links: [tar?: string, mp4?: string],
+    private readonly getNode: (id: number) => VElement | undefined
+  ) {
+    super();
+    console.log(links);
+    // first we try to grab tar, then fallback to mp4
+    this.loadTar()
+      .then((fileArr) => {
+        this.mapToSnapshots(fileArr);
+      })
+      .catch((e) => {
+        if (e === TAR_MISSING && this.links[1]) {
+          this.loadMp4().catch((e2) => {
+            if (e2 === MP4_MISSING) {
+              return console.error(
+                `both tar and mp4 recordings for canvas ${this.nodeId} not found`
+              );
+            } else {
+              return console.error('Failed to load canvas recording');
+            }
+          });
         } else {
-          return Promise.reject(`File ${this.filename} not found`)
+          return console.error('Failed to load canvas recording for node', this.nodeId);
         }
-      }).catch(console.error)
+      });
   }
 
+  public mapToSnapshots(files: TarFile[]) {
+    const tempArr: Timestamp[] = [];
+    const filenameRegexp = /(\d+)_(\d+)_(\d+)\.jpeg$/;
+    const firstPair = files[0].name.match(filenameRegexp);
+    if (!firstPair) {
+      console.error('Invalid file name format', files[0].name);
+      return;
+    }
+    const sessionStart = firstPair ? parseInt(firstPair[1], 10) : 0;
+    files.forEach((file) => {
+      const [_, _1, _2, imageTimestampStr] = file.name.match(filenameRegexp) ?? [0, 0, 0, '0'];
+
+      const imageTimestamp = parseInt(imageTimestampStr, 10);
+
+      const messageTime = imageTimestamp - sessionStart;
+      this.snapshots[messageTime] = file;
+      tempArr.push({ time: messageTime });
+    });
+
+    tempArr
+      .sort((a, b) => a.time - b.time)
+      .forEach((msg) => {
+        this.append(msg);
+      });
+  }
+
+  loadTar = async () => {
+    if (!this.links[0]) {
+      return Promise.reject(TAR_MISSING);
+    }
+    return fetch(this.links[0])
+      .then((r) => {
+        if (r.status === 200) {
+          return r.arrayBuffer();
+        } else {
+          return Promise.reject(TAR_MISSING);
+        }
+      })
+      .then((buf) => {
+        const tar = unpack(new Uint8Array(buf));
+        this.playMode = playMode.snaps;
+        return unpackTar(tar);
+      });
+  };
+
+  loadMp4 = async () => {
+    if (!this.links[1]) {
+      return Promise.reject(MP4_MISSING);
+    }
+    return fetch(this.links[1])
+      .then((r) => {
+        if (r.status === 200) {
+          return r.blob();
+        } else {
+          return Promise.reject(MP4_MISSING);
+        }
+      })
+      .then((blob) => {
+        this.playMode = playMode.video;
+        this.fileData = URL.createObjectURL(blob);
+      });
+  };
+
   startVideo = () => {
+    if (this.playMode === playMode.snaps) {
+      this.snapImage.onload = () => {
+        const node = this.getNode(parseInt(this.nodeId, 10));
+        if (node && node.node) {
+          const canvasCtx = (node.node as HTMLCanvasElement).getContext('2d');
+          const canvasEl = node.node as HTMLVideoElement;
+          canvasCtx?.drawImage(this.snapImage, 0, 0, canvasEl.width, canvasEl.height);
+        } else {
+          console.error(`CanvasManager: Node ${this.nodeId} not found`);
+        }
+      };
+    }
     if (!this.fileData) return;
     this.videoTag.setAttribute('autoplay', 'true');
     this.videoTag.setAttribute('muted', 'true');
@@ -36,25 +141,43 @@ export default class CanvasManager {
     this.videoTag.setAttribute('crossorigin', 'anonymous');
     this.videoTag.src = this.fileData;
     this.videoTag.currentTime = 0;
-  }
+  };
 
   move(t: number) {
+    if (this.playMode === playMode.video) {
+      this.moveReadyVideo(t);
+    } else {
+      this.moveReadySnap(t);
+    }
+  }
+
+  moveReadyVideo = (t: number) => {
     if (Math.abs(t - this.lastTs) < 100) return;
     this.lastTs = t;
-    const playTime = t - this.delta
+    const playTime = t - this.delta;
     if (playTime > 0) {
-      const node = this.getNode(parseInt(this.nodeId, 10))
+      const node = this.getNode(parseInt(this.nodeId, 10));
       if (node && node.node) {
         const canvasCtx = (node.node as HTMLCanvasElement).getContext('2d');
         const canvasEl = node.node as HTMLVideoElement;
         if (!this.videoTag.paused) {
-            void this.videoTag.pause()
+          void this.videoTag.pause();
         }
-        this.videoTag.currentTime = playTime/1000;
+        this.videoTag.currentTime = playTime / 1000;
         canvasCtx?.drawImage(this.videoTag, 0, 0, canvasEl.width, canvasEl.height);
       } else {
-        console.error(`CanvasManager: Node ${this.nodeId} not found`)
+        console.error(`CanvasManager: Node ${this.nodeId} not found`);
       }
     }
-  }
-}
+  };
+
+  moveReadySnap = (t: number) => {
+    const msg = this.moveGetLast(t);
+    if (msg) {
+      const file = this.snapshots[msg.time];
+      if (file) {
+        this.snapImage.src = file.getBlobUrl();
+      }
+    }
+  };
+}