UI patches (28.03) (#3231)

* ui: force getting url for location in tabmanagers

* Assist add turn servers (#3229)

* fixed conflicts

* add offers

* add config to sicket query

* add config to sicket query

* add config init

* removed console logs

* removed wrong updates

* fixed conflicts

* add offers

* add config to sicket query

* add config to sicket query

* add config init

* removed console logs

* removed wrong updates

* ui: fix chat draggable, fix default params

---------

Co-authored-by: nick-delirium <nikita@openreplay.com>

* ui: fix spritemap generation for assist sessions

* ui: fix yarnlock

* fix errors

* updated widget link

* resolved conflicts

* updated widget url

---------

Co-authored-by: Andrey Babushkin <55714097+reyand43@users.noreply.github.com>
Co-authored-by: Андрей Бабушкин <andreybabushkin2000@gmail.com>

assist/utils/socketHandlers.js

@@ -118,7 +118,7 @@ async function onConnect(socket) {
             // Stats
             startAssist(socket, socket.handshake.query.agentID);
         }
-        socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.NEW_AGENT, socket.id, socket.handshake.query.agentInfo);
+        socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.NEW_AGENT, socket.id, { ...socket.handshake.query.agentInfo, config: socket.handshake.query.config });
     }
 
     // Set disconnect handler

frontend/app/components/Assist/ChatWindow/ChatWindow.tsx

@@ -1,7 +1,7 @@
 import React, { useState, useEffect } from 'react';
 import cn from 'classnames';
 import Counter from 'App/components/shared/SessionItem/Counter';
-import Draggable from 'react-draggable';
+import { useDraggable } from '@neodrag/react';
 import type { LocalStream } from 'Player';
 import { PlayerContext } from 'App/components/Session/playerContext';
 import ChatControls from '../ChatControls/ChatControls';
@@ -25,6 +25,8 @@ function ChatWindow({
   isPrestart,
 }: Props) {
   const { t } = useTranslation();
+  const dragRef = React.useRef<HTMLDivElement>(null);
+  useDraggable(dragRef, { bounds: 'body', defaultPosition: { x: 50, y: 200 } })
   const { player } = React.useContext(PlayerContext);
 
   const { toggleVideoLocalStream } = player.assistManager;
@@ -39,11 +41,7 @@ function ChatWindow({
   }, [localVideoEnabled]);
 
   return (
-    <Draggable
+    <div ref={dragRef}>
-      handle=".handle"
-      bounds="body"
-      defaultPosition={{ x: 50, y: 200 }}
-    >
       <div
         className={cn(stl.wrapper, 'fixed radius bg-white shadow-xl mt-16')}
         style={{ width: '280px' }}
@@ -102,7 +100,7 @@ function ChatWindow({
           isPrestart={isPrestart}
         />
       </div>
-    </Draggable>
+    </div>
   );
 }
 

frontend/app/components/Assist/components/AssistActions/AssistActions.tsx

@@ -82,7 +82,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
     { stream: MediaStream; isAgent: boolean }[] | null
   >([]);
   const [localStream, setLocalStream] = useState<LocalStream | null>(null);
-  const [callObject, setCallObject] = useState<{ end: () => void } | null>(
+  const [callObject, setCallObject] = useState<{ end: () => void } | null | undefined>(
     null,
   );
 
@@ -135,6 +135,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
   }, [peerConnectionStatus]);
 
   const addIncomeStream = (stream: MediaStream, isAgent: boolean) => {
+    if (!stream.active) return;
     setIncomeStream((oldState) => {
       if (oldState === null) return [{ stream, isAgent }];
       if (
@@ -149,13 +150,8 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
     });
   };
 
-  const removeIncomeStream = (stream: MediaStream) => {
+  const removeIncomeStream = () => {
-    setIncomeStream((prevState) => {
+    setIncomeStream([]);
-      if (!prevState) return [];
-      return prevState.filter(
-        (existingStream) => existingStream.stream.id !== stream.id,
-      );
-    });
   };
 
   function onReject() {
@@ -181,7 +177,12 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) {
           () => {
             player.assistManager.ping(AssistActionsPing.call.end, agentId);
             lStream.stop.apply(lStream);
-            removeIncomeStream(lStream.stream);
+            removeIncomeStream();
+          },
+          () => {
+            player.assistManager.ping(AssistActionsPing.call.end, agentId);
+            lStream.stop.apply(lStream);
+            removeIncomeStream();
           },
           onReject,
           onError,

frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx

@@ -34,43 +34,40 @@ function VideoContainer({
     }
     const iid = setInterval(() => {
       const track = stream.getVideoTracks()[0];
-      const settings = track?.getSettings();
-      const isDummyVideoTrack = settings
-        ? settings.width === 2 ||
-          settings.frameRate === 0 ||
-          (!settings.frameRate && !settings.width)
-        : true;
-      const shouldBeEnabled = track.enabled && !isDummyVideoTrack;
 
-      if (isEnabled !== shouldBeEnabled) {
+      if (track) {
-        setEnabled(shouldBeEnabled);
+        if (!track.enabled) {
-        setRemoteEnabled?.(shouldBeEnabled);
+          setEnabled(false);
+          setRemoteEnabled?.(false);
+        } else {
+          setEnabled(true);
+          setRemoteEnabled?.(true);
+        }
+      } else {
+        setEnabled(false);
+        setRemoteEnabled?.(false);
       }
     }, 500);
     return () => clearInterval(iid);
-  }, [stream, isEnabled]);
+  }, [stream]);
 
   return (
     <div
       className="flex-1"
       style={{
-        display: isEnabled ? undefined : 'none',
         width: isEnabled ? undefined : '0px!important',
         height: isEnabled ? undefined : '0px !important',
         border: '1px solid grey',
         transform: local ? 'scaleX(-1)' : undefined,
+        display: isEnabled ? 'block' : 'none',
       }}
     >
-      <video autoPlay ref={ref} muted={muted} style={{ height }} />
+      <video
-      {isAgent ? (
+        autoPlay
-        <div
+        ref={ref}
-          style={{
+        muted={muted}
-            position: 'absolute',
+        style={{ height }}
-          }}
+      />
-        >
-          {t('Agent')}
-        </div>
-      ) : null}
     </div>
   );
 }

frontend/app/player/common/ListWalker.ts

@@ -124,13 +124,9 @@ export default class ListWalker<T extends Timed> {
    * Assumed that the current message is already handled so
    * if pointer doesn't change <null> is returned.
    */
-  moveGetLast(t: number, index?: number): T | null {
+  moveGetLast(t: number, index?: number, force?: boolean, debug?: boolean): T | null {
-    let key: string = 'time'; // TODO
+    const key: string = index ? '_index' : 'time';
-    let val = t;
+    const val = index ? index : t;
-    if (index) {
-      key = '_index';
-      val = index;
-    }
 
     let changed = false;
     // @ts-ignore
@@ -143,7 +139,10 @@ export default class ListWalker<T extends Timed> {
       this.movePrev();
       changed = true;
     }
-    return changed ? this.list[this.p - 1] : null;
+    if (debug) {
+      console.log(this.list[this.p - 1])
+    }
+    return changed || force ? this.list[this.p - 1] : null;
   }
 
   prevTs = 0;

frontend/app/player/web/MessageLoader.ts

@@ -43,27 +43,6 @@ export default class MessageLoader {
     this.session = session;
   }
 
-  /**
-   * TODO: has to be moved out of messageLoader logic somehow
-   * */
-  spriteMapSvg: SVGElement | null = null;
-
-  potentialSpriteMap: Record<string, any> = {};
-
-  domParser: DOMParser | null = null;
-
-  createSpriteMap = () => {
-    if (!this.spriteMapSvg) {
-      this.domParser = new DOMParser();
-      this.spriteMapSvg = document.createElementNS(
-        'http://www.w3.org/2000/svg',
-        'svg',
-      );
-      this.spriteMapSvg.setAttribute('style', 'display: none;');
-      this.spriteMapSvg.setAttribute('id', 'reconstructed-sprite');
-    }
-  };
-
   createNewParser(
     shouldDecrypt = true,
     onMessagesDone: (msgs: PlayerMsg[], file?: string) => void,
@@ -101,21 +80,6 @@ export default class MessageLoader {
         let startTimeSet = false;
 
         msgs.forEach((msg, i) => {
-          if (msg.tp === MType.SetNodeAttribute) {
-            if (msg.value.includes('_$OPENREPLAY_SPRITE$_')) {
-              this.createSpriteMap();
-              if (!this.domParser) {
-                return console.error('DOM parser is not initialized?');
-              }
-              handleSprites(
-                this.potentialSpriteMap,
-                this.domParser,
-                msg,
-                this.spriteMapSvg!,
-                i,
-              );
-            }
-          }
           if (msg.tp === MType.Redux || msg.tp === MType.ReduxDeprecated) {
             if ('actionTime' in msg && msg.actionTime) {
               msg.time = msg.actionTime - this.session.startedAt;
@@ -333,10 +297,6 @@ export default class MessageLoader {
 
     await Promise.allSettled([restDomFilesPromise, restDevtoolsFilesPromise]);
     this.messageManager.onFileReadSuccess();
-    // no sprites for mobile
-    if (this.spriteMapSvg && 'injectSpriteMap' in this.messageManager) {
-      this.messageManager.injectSpriteMap(this.spriteMapSvg);
-    }
   };
 
   loadEFSMobs = async () => {
@@ -467,40 +427,6 @@ function findBrokenNodes(nodes: any[]) {
   return result;
 }
 
-function handleSprites(
-  potentialSpriteMap: Record<string, any>,
-  parser: DOMParser,
-  msg: Record<string, any>,
-  spriteMapSvg: SVGElement,
-  i: number,
-) {
-  const [_, svgData] = msg.value.split('_$OPENREPLAY_SPRITE$_');
-  const potentialSprite = potentialSpriteMap[svgData];
-  if (potentialSprite) {
-    msg.value = potentialSprite;
-  } else {
-    const svgDoc = parser.parseFromString(svgData, 'image/svg+xml');
-    const originalSvg = svgDoc.querySelector('svg');
-    if (originalSvg) {
-      const symbol = document.createElementNS(
-        'http://www.w3.org/2000/svg',
-        'symbol',
-      );
-      const symbolId = `symbol-${msg.id || `ind-${i}`}`; // Generate an ID if missing
-      symbol.setAttribute('id', symbolId);
-      symbol.setAttribute(
-        'viewBox',
-        originalSvg.getAttribute('viewBox') || '0 0 24 24',
-      );
-      symbol.innerHTML = originalSvg.innerHTML;
-
-      spriteMapSvg.appendChild(symbol);
-      msg.value = `#${symbolId}`;
-      potentialSpriteMap[svgData] = `#${symbolId}`;
-    }
-  }
-}
-
 // @ts-ignore
 window.searchOrphans = (msgs) =>
   findBrokenNodes(msgs.filter((m) => [8, 9, 10, 70].includes(m.tp)));

frontend/app/player/web/MessageManager.ts

@@ -201,8 +201,16 @@ export default class MessageManager {
     }
 
     Object.values(this.tabs).forEach((tab) => tab.onFileReadSuccess?.());
+
+    this.updateSpriteMap();
   };
 
+  public updateSpriteMap = () => {
+    if (this.spriteMapSvg) {
+      this.injectSpriteMap(this.spriteMapSvg);
+    }
+  }
+
   public onFileReadFailed = (...e: any[]) => {
     logger.error(e);
     this.state.update({ error: true });
@@ -288,15 +296,17 @@ export default class MessageManager {
       }
 
       if (tabId) {
+        const stateUpdate: { currentTab?: string, tabs?: Set<string> } = {}
         if (this.activeTab !== tabId) {
-          this.state.update({ currentTab: tabId });
+          stateUpdate['currentTab'] = tabId;
           this.activeTab = tabId;
           this.tabs[this.activeTab].clean();
         }
         const activeTabs = this.state.get().tabs;
         if (activeTabs.size !== this.activeTabManager.tabInstances.size) {
-          this.state.update({ tabs: this.activeTabManager.tabInstances });
+          stateUpdate['tabs'] = this.activeTabManager.tabInstances;
         }
+        this.state.update(stateUpdate)
       }
 
       if (this.tabs[this.activeTab]) {
@@ -335,9 +345,38 @@ export default class MessageManager {
     this.state.update({ tabChangeEvents: this.tabChangeEvents });
   }
 
+  spriteMapSvg: SVGElement | null = null;
+  potentialSpriteMap: Record<string, any> = {};
+  domParser: DOMParser | null = null;
+  createSpriteMap = () => {
+    if (!this.spriteMapSvg) {
+      this.domParser = new DOMParser();
+      this.spriteMapSvg = document.createElementNS(
+        'http://www.w3.org/2000/svg',
+        'svg',
+      );
+      this.spriteMapSvg.setAttribute('style', 'display: none;');
+      this.spriteMapSvg.setAttribute('id', 'reconstructed-sprite');
+    }
+  };
+
   distributeMessage = (msg: Message & { tabId: string }): void => {
     // @ts-ignore placeholder msg for timestamps
     if (msg.tp === 9999) return;
+    if (msg.tp === MType.SetNodeAttribute) {
+      if (msg.value.includes('_$OPENREPLAY_SPRITE$_')) {
+        this.createSpriteMap();
+        if (!this.domParser) {
+          return console.error('DOM parser is not initialized?');
+        }
+        handleSprites(
+          this.potentialSpriteMap,
+          this.domParser,
+          msg,
+          this.spriteMapSvg!,
+        );
+      }
+    }
     if (!this.tabs[msg.tabId]) {
       this.tabsAmount++;
       this.state.update({
@@ -452,3 +491,36 @@ function mapTabs(tabs: Record<string, TabSessionManager>) {
 
   return tabMap;
 }
+
+function handleSprites(
+  potentialSpriteMap: Record<string, any>,
+  parser: DOMParser,
+  msg: Record<string, any>,
+  spriteMapSvg: SVGElement,
+) {
+  const [_, svgData] = msg.value.split('_$OPENREPLAY_SPRITE$_');
+  const potentialSprite = potentialSpriteMap[svgData];
+  if (potentialSprite) {
+    msg.value = potentialSprite;
+  } else {
+    const svgDoc = parser.parseFromString(svgData, 'image/svg+xml');
+    const originalSvg = svgDoc.querySelector('svg');
+    if (originalSvg) {
+      const symbol = document.createElementNS(
+        'http://www.w3.org/2000/svg',
+        'symbol',
+      );
+      const symbolId = `symbol-${msg.id || `ind-${msg.time}`}`; // Generate an ID if missing
+      symbol.setAttribute('id', symbolId);
+      symbol.setAttribute(
+        'viewBox',
+        originalSvg.getAttribute('viewBox') || '0 0 24 24',
+      );
+      symbol.innerHTML = originalSvg.innerHTML;
+
+      spriteMapSvg.appendChild(symbol);
+      msg.value = `#${symbolId}`;
+      potentialSpriteMap[svgData] = `#${symbolId}`;
+    }
+  }
+}

frontend/app/player/web/TabManager.ts

@@ -98,6 +98,7 @@ export default class TabSessionManager {
     private readonly state: Store<{
       tabStates: { [tabId: string]: TabState };
       tabNames: { [tabId: string]: string };
+      location?: string;
     }>,
     private readonly screen: Screen,
     private readonly id: string,
@@ -415,15 +416,17 @@ export default class TabSessionManager {
       }
     }
     /* === */
-    const lastLocationMsg = this.locationManager.moveGetLast(t, index);
+    const lastLocationMsg = this.locationManager.moveGetLast(t, index, true);
     if (lastLocationMsg) {
-      const { tabNames } = this.state.get();
+      const { tabNames, location } = this.state.get();
+      if (location !== lastLocationMsg.url) {
         if (lastLocationMsg.documentTitle) {
           tabNames[this.id] = lastLocationMsg.documentTitle;
         }
         // @ts-ignore comes from parent state
         this.state.update({ location: lastLocationMsg.url, tabNames });
       }
+    }
 
     const lastPerformanceTrackMessage =
       this.performanceTrackManager.moveGetLast(t, index);

frontend/app/player/web/WebLivePlayer.ts

@@ -43,6 +43,7 @@ export default class WebLivePlayer extends WebPlayer {
       wpState,
       (id) => this.messageManager.getNode(id),
       agentId,
+      this.messageManager.updateSpriteMap,
       uiErrorHandler,
     );
     this.assistManager.connect(session.agentToken!, agentId, projectId);

frontend/app/player/web/WebPlayer.ts

@@ -100,6 +100,8 @@ export default class WebPlayer extends Player {
 
     // @ts-ignore
     window.playerJumpToTime = this.jump.bind(this);
+    // @ts-ignore
+    window.__OPENREPLAY_DEV_TOOLS__.player = this;
   }
 
   preloadFirstFile(data: Uint8Array, fileKey?: string) {

frontend/app/player/web/assist/AssistManager.ts

@@ -3,14 +3,14 @@ import type { Socket } from 'socket.io-client';
 import type { PlayerMsg, Store } from 'App/player';
 import CanvasReceiver from 'Player/web/assist/CanvasReceiver';
 import { gunzipSync } from 'fflate';
-import { Message } from '../messages';
+import { Message, MType } from '../messages';
 import type Screen from '../Screen/Screen';
 import MStreamReader from '../messages/MStreamReader';
 import JSONRawMessageReader from '../messages/JSONRawMessageReader';
 import Call, { CallingState } from './Call';
 import RemoteControl, { RemoteControlStatus } from './RemoteControl';
 import ScreenRecording, { SessionRecordingStatus } from './ScreenRecording';
-
+import { debounceCall } from 'App/utils'
 export { RemoteControlStatus, SessionRecordingStatus, CallingState };
 
 export enum ConnectionStatus {
@@ -82,6 +82,7 @@ export default class AssistManager {
     private store: Store<typeof AssistManager.INITIAL_STATE>,
     private getNode: MessageManager['getNode'],
     public readonly agentId: number,
+    private readonly updateSpriteMap: () => void,
     public readonly uiErrorHandler?: {
       error: (msg: string) => void;
     },
@@ -200,6 +201,7 @@ export default class AssistManager {
             peerId: this.peerID,
             query: document.location.search,
           }),
+          config: JSON.stringify(this.getIceServers()),
         },
       }));
 
@@ -239,6 +241,11 @@ export default class AssistManager {
           msg !== null;
           msg = reader.readNext()
         ) {
+          if (msg.tp === MType.SetNodeAttribute) {
+            if (msg.value.includes('_$OPENREPLAY_SPRITE$_')) {
+              debounceCall(this.updateSpriteMap, 250)()
+            }
+          }
           this.handleMessage(msg, msg._index);
         }
       };
@@ -314,7 +321,7 @@ export default class AssistManager {
       this.callManager = new Call(
         this.store,
         socket,
-        this.config,
+        this.getIceServers(),
         this.peerID,
         this.getAssistVersion,
         {
@@ -354,6 +361,23 @@ export default class AssistManager {
     });
   }
 
+  private getIceServers = () => {
+    if (this.config) {
+      return this.config;
+    }
+    return [
+      {
+        urls: [
+          'stun:stun.l.google.com:19302',
+          'stun:stun1.l.google.com:19302',
+          'stun:stun2.l.google.com:19302',
+          'stun:stun3.l.google.com:19302',
+          'stun:stun4.l.google.com:19302'
+        ],
+      },
+    ] as RTCIceServer[];
+  };
+
   /**
    * Sends event ping to stats service
    * */

frontend/app/player/web/assist/Call.ts

@@ -43,7 +43,7 @@ export default class Call {
   constructor(
     private store: Store<State & { tabs: Set<string> }>,
     private socket: Socket,
-    private config: RTCIceServer[] | null,
+    private config: RTCIceServer[],
     private peerID: string,
     private getAssistVersion: () => number,
     private agent: Record<string, any>,
@@ -146,7 +146,7 @@ export default class Call {
     // create pc with ice config
 
     const pc = new RTCPeerConnection({
-      iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
+      iceServers: this.config,
     });
 
     // If there is a local stream, add its tracks to the connection
@@ -185,8 +185,7 @@ export default class Call {
     pc.ontrack = (event) => {
       const stream = event.streams[0];
       if (stream && !this.videoStreams[remotePeerId]) {
-        const clonnedStream = stream.clone();
+        this.videoStreams[remotePeerId] = stream.getVideoTracks()[0];
-        this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0];
         if (this.store.get().calling !== CallingState.OnCall) {
           this.store.update({ calling: CallingState.OnCall });
         }
@@ -305,7 +304,6 @@ export default class Call {
     }
     try {
       // if the connection is not established yet, then set remoteDescription to peer
-      if (!pc.localDescription) {
       await pc.setRemoteDescription(new RTCSessionDescription(data.offer));
       const answer = await pc.createAnswer();
       await pc.setLocalDescription(answer);
@@ -319,9 +317,6 @@ export default class Call {
       } else {
         this.socket.emit('webrtc_call_answer', { from: fromCallId, answer });
       }
-      } else {
-        logger.warn('Skipping setRemoteDescription: Already in stable state');
-      }
     } catch (e) {
       logger.error('Error setting remote description from answer', e);
       this.callArgs?.onError?.(e);
@@ -388,13 +383,13 @@ export default class Call {
   private handleCallEnd() {
     // If the call is not completed, then call onCallEnd
     if (this.store.get().calling !== CallingState.NoCall) {
-      this.callArgs && this.callArgs.onCallEnd();
+      this.callArgs && this.callArgs.onRemoteCallEnd();
     }
     // change state to NoCall
     this.store.update({ calling: CallingState.NoCall });
     // Close all created RTCPeerConnection
     Object.values(this.connections).forEach((pc) => pc.close());
-    this.callArgs?.onCallEnd();
+    this.callArgs?.onRemoteCallEnd();
     // Clear connections
     this.connections = {};
     this.callArgs = null;
@@ -414,7 +409,7 @@ export default class Call {
       // Close all connections and reset callArgs
       Object.values(this.connections).forEach((pc) => pc.close());
       this.connections = {};
-      this.callArgs?.onCallEnd();
+      this.callArgs?.onRemoteCallEnd();
       this.store.update({ calling: CallingState.NoCall });
       this.callArgs = null;
     } else {
@@ -443,7 +438,8 @@ export default class Call {
   private callArgs: {
     localStream: LocalStream;
     onStream: (s: MediaStream, isAgent: boolean) => void;
-    onCallEnd: () => void;
+    onRemoteCallEnd: () => void;
+    onLocalCallEnd: () => void;
     onReject: () => void;
     onError?: (arg?: any) => void;
   } | null = null;
@@ -451,14 +447,16 @@ export default class Call {
   setCallArgs(
     localStream: LocalStream,
     onStream: (s: MediaStream, isAgent: boolean) => void,
-    onCallEnd: () => void,
+    onRemoteCallEnd: () => void,
+    onLocalCallEnd: () => void,
     onReject: () => void,
     onError?: (e?: any) => void,
   ) {
     this.callArgs = {
       localStream,
       onStream,
-      onCallEnd,
+      onRemoteCallEnd,
+      onLocalCallEnd,
       onReject,
       onError,
     };
@@ -549,7 +547,7 @@ export default class Call {
     void this.initiateCallEnd();
     Object.values(this.connections).forEach((pc) => pc.close());
     this.connections = {};
-    this.callArgs?.onCallEnd();
+    this.callArgs?.onLocalCallEnd();
   }
 }
 

frontend/package.json

@@ -27,6 +27,7 @@
     "@codewonders/html2canvas": "^1.0.2",
     "@eslint/js": "^9.21.0",
     "@medv/finder": "^4.0.2",
+    "@neodrag/react": "^2.3.0",
     "@sentry/browser": "^5.21.1",
     "@svg-maps/world": "^1.0.1",
     "@tanstack/react-query": "^5.56.2",
@@ -72,7 +73,6 @@
     "react-dnd": "^16.0.1",
     "react-dnd-html5-backend": "^15.1.2",
     "react-dom": "^18.2.0",
-    "react-draggable": "^4.4.5",
     "react-google-recaptcha": "^2.1.0",
     "react-i18next": "^15.4.1",
     "react-intersection-observer": "^9.13.1",

frontend/yarn.lock

@@ -2584,6 +2584,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@neodrag/react@npm:^2.3.0":
+  version: 2.3.0
+  resolution: "@neodrag/react@npm:2.3.0"
+  checksum: 10c1/37f549ad0bdab8badb0799bf4e6f1714fe52a2ace6c09447a9dc97a9483a0fb40f75b35b121c78a1cba3a27ec9076a5db9672454af3a216d9efee8dcd34c6736
+  languageName: node
+  linkType: hard
+
 "@nicolo-ribaudo/chokidar-2@npm:2.1.8-no-fsevents.3":
   version: 2.1.8-no-fsevents.3
   resolution: "@nicolo-ribaudo/chokidar-2@npm:2.1.8-no-fsevents.3"
@@ -11989,6 +11996,7 @@ __metadata:
     "@eslint/js": "npm:^9.21.0"
     "@jest/globals": "npm:^29.7.0"
     "@medv/finder": "npm:^4.0.2"
+    "@neodrag/react": "npm:^2.3.0"
     "@openreplay/sourcemap-uploader": "npm:^3.0.10"
     "@sentry/browser": "npm:^5.21.1"
     "@svg-maps/world": "npm:^1.0.1"
@@ -12079,7 +12087,6 @@ __metadata:
     react-dnd: "npm:^16.0.1"
     react-dnd-html5-backend: "npm:^15.1.2"
     react-dom: "npm:^18.2.0"
-    react-draggable: "npm:^4.4.5"
     react-google-recaptcha: "npm:^2.1.0"
     react-i18next: "npm:^15.4.1"
     react-intersection-observer: "npm:^9.13.1"
@@ -14060,19 +14067,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"react-draggable@npm:^4.4.5":
-  version: 4.4.6
-  resolution: "react-draggable@npm:4.4.6"
-  dependencies:
-    clsx: "npm:^1.1.1"
-    prop-types: "npm:^15.8.1"
-  peerDependencies:
-    react: ">= 16.3.0"
-    react-dom: ">= 16.3.0"
-  checksum: 10c1/b8ae807f4556b658ae149b6542af5222d75996da47c549db54be22276579549ca2cd2fd06ca5c0852fbd1b663853cc0568b68215bd6e62433b46a7e154ddf8e2
-  languageName: node
-  linkType: hard
-
 "react-fit@npm:^2.0.0":
   version: 2.0.1
   resolution: "react-fit@npm:2.0.1"

tracker/tracker-assist/src/Assist.ts

@@ -20,6 +20,7 @@ import { gzip } from 'fflate'
 type StartEndCallback = (agentInfo?: Record<string, any>) => ((() => any) | void)
 
 interface AgentInfo {
+  config: string;
   email: string;
   id: number
   name: string
@@ -85,6 +86,7 @@ export default class Assist {
   private remoteControl: RemoteControl | null = null;
   private peerReconnectTimeout: ReturnType<typeof setTimeout> | null = null
   private agents: Record<string, Agent> = {}
+  private config: RTCIceServer[] | undefined
   private readonly options: Options
   private readonly canvasMap: Map<number, Canvas> = new Map()
 
@@ -251,7 +253,7 @@ export default class Assist {
         return
       }
       if (args[0] !== 'webrtc_call_ice_candidate') {
-        app.debug.log('Socket:', ...args)
+        app.debug.log("Socket:", ...args);
       };
       socket.on('close', (e) => {
         app.debug.warn('Socket closed:', e);
@@ -353,6 +355,9 @@ export default class Assist {
         this.app.stop()
         this.app.clearBuffers()
         this.app.waitStatus(0)
+          .then(() => {
+            this.config = JSON.parse(info.config);
+          })
           .then(() => {
             this.app.allowAppStart()
             setTimeout(() => {
@@ -548,6 +553,16 @@ export default class Assist {
       }
     }
 
+    const renegotiateConnection = async ({ pc, from }: { pc: RTCPeerConnection, from: string }) => {
+      try {
+        const offer = await pc.createOffer();
+        await pc.setLocalDescription(offer);
+        this.emit('webrtc_call_offer', { from, offer });
+      } catch (error) {
+        app.debug.error("Error with renegotiation:", error);
+      }
+    };
+
     const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => {
       app.debug.log('handleIncomingCallOffer', from)
       let confirmAnswer: Promise<boolean>
@@ -572,13 +587,19 @@ export default class Assist {
 
       try {
         // waiting for a decision on accepting the challenge
-        const agreed = await confirmAnswer
+        const agreed = await confirmAnswer;
         // if rejected, then terminate the call
         if (!agreed) {
           initiateCallEnd()
           this.options.onCallDeny?.()
           return
         }
+
+         // create a new RTCPeerConnection with ice server config
+        const pc = new RTCPeerConnection({
+          iceServers: this.config,
+        });
+
         if (!callUI) {
           callUI = new CallWindow(app.debug.error, this.options.callUITemplate)
           callUI.setVideoToggleCallback((args: { enabled: boolean }) =>
@@ -598,7 +619,7 @@ export default class Assist {
           if (!lStreams[from]) {
             app.debug.log('starting new stream for', from)
             // request a local stream, and set it to lStreams
-            lStreams[from] = await RequestLocalStream()
+            lStreams[from] = await RequestLocalStream(pc, renegotiateConnection.bind(null, { pc, from }))
           }
          // we pass the received tracks to Call ui
           callUI.setLocalStreams(Object.values(lStreams))
@@ -606,22 +627,50 @@ export default class Assist {
           app.debug.error('Error requesting local stream', e);
           // if something didn't work out, we terminate the call
           initiateCallEnd();
+          this.options.onCallDeny?.();
           return;
         }
-        // create a new RTCPeerConnection with ice server config
+
-        const pc = new RTCPeerConnection({
+        if (!callUI) {
-          iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
+          callUI = new CallWindow(app.debug.error, this.options.callUITemplate);
+          callUI.setVideoToggleCallback((args: { enabled: boolean }) => {
+            this.emit("videofeed", { streamId: from, enabled: args.enabled })
           });
+        }
+        // show buttons in the call window
+        callUI.showControls(initiateCallEnd);
+        if (!annot) {
+          annot = new AnnotationCanvas();
+          annot.mount();
+        }
+
+        
+
+        // callUI.setLocalStreams(Object.values(lStreams))
+        try {
+          // if there are no local streams in lStrems then we set
+          if (!lStreams[from]) {
+            app.debug.log("starting new stream for", from);
+            // request a local stream, and set it to lStreams
+            lStreams[from] = await RequestLocalStream(pc, renegotiateConnection.bind(null, { pc, from }));
+          }
+          // we pass the received tracks to Call ui
+          callUI.setLocalStreams(Object.values(lStreams));
+        } catch (e) {
+          app.debug.error("Error requesting local stream", e);
+          // if something didn't work out, we terminate the call
+          initiateCallEnd();
+          return;
+        }
 
         // get all local tracks and add them to RTCPeerConnection
-        lStreams[from].stream.getTracks().forEach(track => {
-          pc.addTrack(track, lStreams[from].stream);
-        });
-
         // When we receive local ice candidates, we emit them via socket
         pc.onicecandidate = (event) => {
           if (event.candidate) {
-            socket.emit('webrtc_call_ice_candidate', { from, candidate: event.candidate });
+            socket.emit("webrtc_call_ice_candidate", {
+              from,
+              candidate: event.candidate,
+            });
           }
         };
 
@@ -632,9 +681,9 @@ export default class Assist {
             callUI.addRemoteStream(rStream, from);
             const onInteraction = () => {
               callUI?.playRemote();
-              document.removeEventListener('click', onInteraction);
+              document.removeEventListener("click", onInteraction);
             };
-            document.addEventListener('click', onInteraction);
+            document.addEventListener("click", onInteraction);
           }
         };
 
@@ -648,7 +697,7 @@ export default class Assist {
         // set answer as local description
         await pc.setLocalDescription(answer);
         // set the response as local
-        socket.emit('webrtc_call_answer', { from, answer });
+        socket.emit("webrtc_call_answer", { from, answer });
 
         // If the state changes to an error, we terminate the call
         // pc.onconnectionstatechange = () => {
@@ -658,27 +707,35 @@ export default class Assist {
         // };
 
         // Update track when local video changes
-        lStreams[from].onVideoTrack(vTrack => {
+        lStreams[from].onVideoTrack((vTrack) => {
-          const sender = pc.getSenders().find(s => s.track?.kind === 'video');
+          const sender = pc.getSenders().find((s) => s.track?.kind === "video");
           if (!sender) {
-            app.debug.warn('No video sender found')
+            app.debug.warn("No video sender found");
-            return
+            return;
           }
-          sender.replaceTrack(vTrack)
+          sender.replaceTrack(vTrack);
-        })
+        });
 
         // if the user closed the tab or switched, then we end the call
-        document.addEventListener('visibilitychange', () => {
+        document.addEventListener("visibilitychange", () => {
-          initiateCallEnd()
+          initiateCallEnd();
-        })
+        });
 
         // when everything is set, we change the state to true
-        this.setCallingState(CallingState.True)
+        this.setCallingState(CallingState.True);
-        if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() }
+        if (!callEndCallback) {
-        const callingPeerIdsNow = Array.from(this.calls.keys())
+          callEndCallback = this.options.onCallStart?.();
+        }
+        const callingPeerIdsNow = Array.from(this.calls.keys());
         // in session storage we write down everyone with whom the call is established
-        sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIdsNow))
+        sessionStorage.setItem(
-        this.emit('UPDATE_SESSION', { agentIds: callingPeerIdsNow, isCallActive: true })
+          this.options.session_calling_peer_key,
+          JSON.stringify(callingPeerIdsNow)
+        );
+        this.emit("UPDATE_SESSION", {
+          agentIds: callingPeerIdsNow,
+          isCallActive: true,
+        });
       } catch (reason) {
         app.debug.log(reason);
       }
@@ -712,7 +769,7 @@ export default class Assist {
 
         if (!this.canvasPeers[uniqueId]) {
           this.canvasPeers[uniqueId] = new RTCPeerConnection({
-            iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
+            iceServers: this.config,
           });
           this.setupPeerListeners(uniqueId);
 
@@ -726,7 +783,6 @@ export default class Assist {
 
           // Send offer via signaling server
           socket.emit('webrtc_canvas_offer', { offer, id: uniqueId });
-
         }
       }
     }

tracker/tracker-assist/src/CallWindow.ts

@@ -48,7 +48,7 @@ export default class CallWindow {
 		}
 
 		// const baseHref = "https://static.openreplay.com/tracker-assist/test"
-		const baseHref = 'https://static.openreplay.com/tracker-assist/4.0.0'
+		const baseHref = 'https://static.openreplay.com/tracker-assist/widget'
 		// this.load = fetch(this.callUITemplate || baseHref + '/index2.html')
 		this.load = fetch(this.callUITemplate || baseHref + '/index.html')
 			.then((r) => r.text())
@@ -152,15 +152,6 @@ export default class CallWindow {
 					if (this.checkRemoteVideoInterval) {
 						clearInterval(this.checkRemoteVideoInterval)
 					} // just in case
-					let enabled = false
-					this.checkRemoteVideoInterval = setInterval(() => {
-						const settings = this.remoteVideo?.getSettings()
-						const isDummyVideoTrack = !this.remoteVideo.enabled || (!!settings && (settings.width === 2 || settings.frameRate === 0))
-						const shouldBeEnabled = !isDummyVideoTrack
-						if (enabled !== shouldBeEnabled) {
-							this.toggleRemoteVideoUI((enabled = shouldBeEnabled))
-						}
-					}, 1000)
 				}
 
 				// Audio

tracker/tracker-assist/src/LocalStream.ts

@@ -1,88 +1,86 @@
-declare global {
+export default function RequestLocalStream(
-   interface HTMLCanvasElement {
+  pc: RTCPeerConnection,
-     captureStream(frameRate?: number): MediaStream;
+  toggleVideoCb?: () => void
+): Promise<LocalStream> {
+  return navigator.mediaDevices
+    .getUserMedia({ audio: true, video: false })
+    .then((stream) => {
+      const aTrack = stream.getAudioTracks()[0];
+      if (!aTrack) {
+        throw new Error("No audio tracks provided");
       }
-}
+      stream.getTracks().forEach((track) => {
-
+        pc.addTrack(track, stream);
-function dummyTrack(): MediaStreamTrack { 
+      });
-  const canvas = document.createElement('canvas')//, { width: 0, height: 0})
+      return new _LocalStream(stream, pc, toggleVideoCb);
-  canvas.setAttribute('data-openreplay-hidden', '1')
+    });
-  canvas.width=canvas.height=2 // Doesn't work when 1 (?!)
-  const ctx = canvas.getContext('2d')
-  ctx?.fillRect(0, 0, canvas.width, canvas.height)
-  requestAnimationFrame(function draw(){
-    ctx?.fillRect(0,0, canvas.width, canvas.height)
-    requestAnimationFrame(draw)
-  })
-  // Also works. Probably it should be done once connected.
-  //setTimeout(() => { ctx?.fillRect(0,0, canvas.width, canvas.height) }, 4000)
-  return canvas.captureStream(60).getTracks()[0]
-}
-
-export default function RequestLocalStream(): Promise<LocalStream> {
-  return navigator.mediaDevices.getUserMedia({ audio:true, })
-    .then(aStream => {
-      const aTrack = aStream.getAudioTracks()[0]
-
-      if (!aTrack) { throw new Error('No audio tracks provided') }
-      return new _LocalStream(aTrack)
-    })
 }
 
 class _LocalStream {
-  private mediaRequested = false
+  private mediaRequested = false;
-  readonly stream: MediaStream
+  readonly stream: MediaStream;
-  private readonly vdTrack: MediaStreamTrack
+  readonly vTrack: MediaStreamTrack;
-  constructor(aTrack: MediaStreamTrack) {
+  readonly pc: RTCPeerConnection;
-    this.vdTrack = dummyTrack()
+  readonly toggleVideoCb?: () => void;
-    this.stream = new MediaStream([ aTrack, this.vdTrack, ])
+  constructor(stream: MediaStream, pc: RTCPeerConnection, toggleVideoCb?: () => void) {
+    this.stream = stream;
+    this.pc = pc;
+    this.toggleVideoCb = toggleVideoCb;
   }
 
   toggleVideo(): Promise<boolean> {
+    const videoTracks = this.stream.getVideoTracks();
     if (!this.mediaRequested) {
-      return navigator.mediaDevices.getUserMedia({video:true,})
+      return navigator.mediaDevices
-      .then(vStream => {
+        .getUserMedia({ video: true })
-        const vTrack = vStream.getVideoTracks()[0]
+        .then((vStream) => {
+          const vTrack = vStream.getVideoTracks()[0];
           if (!vTrack) {
-          throw new Error('No video track provided')
+            throw new Error("No video track provided");
           }
-        this.stream.addTrack(vTrack)
+
-        this.stream.removeTrack(this.vdTrack)
+          this.pc.addTrack(vTrack, this.stream);
-        this.mediaRequested = true
+          this.stream.addTrack(vTrack);
+
+          if (this.toggleVideoCb) {
+            this.toggleVideoCb();
+          }
+
+          this.mediaRequested = true;
+
           if (this.onVideoTrackCb) {
-          this.onVideoTrackCb(vTrack)
+            this.onVideoTrackCb(vTrack);
           }
-        return true
+          return true;
         })
-      .catch(e => {
+        .catch((e) => {
           // TODO: log
-        console.error(e)
+          return false;
-        return false
+        });
-      })
+    } else {
+      videoTracks.forEach((track) => {
+        track.enabled = !track.enabled;
+      });
     }
-    let enabled = true
+    return Promise.resolve(videoTracks[0].enabled);
-    this.stream.getVideoTracks().forEach(track => {
-      track.enabled = enabled = enabled && !track.enabled
-    })
-    return Promise.resolve(enabled)
   }
 
   toggleAudio(): boolean {
-    let enabled = true
+    let enabled = true;
-    this.stream.getAudioTracks().forEach(track => {
+    this.stream.getAudioTracks().forEach((track) => {
-      track.enabled = enabled = enabled && !track.enabled
+      track.enabled = enabled = enabled && !track.enabled;
-    })
+    });
-    return enabled
+    return enabled;
   }
 
-  private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null
+  private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null;
   onVideoTrack(cb: (t: MediaStreamTrack) => void) {
-    this.onVideoTrackCb = cb
+    this.onVideoTrackCb = cb;
   }
 
   stop() {
-    this.stream.getTracks().forEach(t => t.stop())
+    this.stream.getTracks().forEach((t) => t.stop());
   }
 }
 
-export type LocalStream = InstanceType<typeof _LocalStream>
+export type LocalStream = InstanceType<typeof _LocalStream>;