diff --git a/assist/utils/socketHandlers.js b/assist/utils/socketHandlers.js index 503969c08..7921b4f9e 100644 --- a/assist/utils/socketHandlers.js +++ b/assist/utils/socketHandlers.js @@ -118,7 +118,7 @@ async function onConnect(socket) { // Stats startAssist(socket, socket.handshake.query.agentID); } - socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.NEW_AGENT, socket.id, socket.handshake.query.agentInfo); + socket.to(socket.handshake.query.roomId).emit(EVENTS_DEFINITION.emit.NEW_AGENT, socket.id, { ...socket.handshake.query.agentInfo, config: socket.handshake.query.config }); } // Set disconnect handler diff --git a/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx b/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx index b7fe24127..4ea5723b5 100644 --- a/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx +++ b/frontend/app/components/Assist/ChatWindow/ChatWindow.tsx @@ -1,7 +1,7 @@ import React, { useState, useEffect } from 'react'; import cn from 'classnames'; import Counter from 'App/components/shared/SessionItem/Counter'; -import Draggable from 'react-draggable'; +import { useDraggable } from '@neodrag/react'; import type { LocalStream } from 'Player'; import { PlayerContext } from 'App/components/Session/playerContext'; import ChatControls from '../ChatControls/ChatControls'; @@ -25,6 +25,8 @@ function ChatWindow({ isPrestart, }: Props) { const { t } = useTranslation(); + const dragRef = React.useRef(null); + useDraggable(dragRef, { bounds: 'body', defaultPosition: { x: 50, y: 200 } }) const { player } = React.useContext(PlayerContext); const { toggleVideoLocalStream } = player.assistManager; @@ -39,11 +41,7 @@ function ChatWindow({ }, [localVideoEnabled]); return ( - +
- +
); } diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index b3c1ea884..a5bdc0b6d 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -82,7 +82,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) { { stream: MediaStream; isAgent: boolean }[] | null >([]); const [localStream, setLocalStream] = useState(null); - const [callObject, setCallObject] = useState<{ end: () => void } | null>( + const [callObject, setCallObject] = useState<{ end: () => void } | null | undefined>( null, ); @@ -135,6 +135,7 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) { }, [peerConnectionStatus]); const addIncomeStream = (stream: MediaStream, isAgent: boolean) => { + if (!stream.active) return; setIncomeStream((oldState) => { if (oldState === null) return [{ stream, isAgent }]; if ( @@ -149,13 +150,8 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) { }); }; - const removeIncomeStream = (stream: MediaStream) => { - setIncomeStream((prevState) => { - if (!prevState) return []; - return prevState.filter( - (existingStream) => existingStream.stream.id !== stream.id, - ); - }); + const removeIncomeStream = () => { + setIncomeStream([]); }; function onReject() { @@ -181,7 +177,12 @@ function AssistActions({ userId, isCallActive, agentIds }: Props) { () => { player.assistManager.ping(AssistActionsPing.call.end, agentId); lStream.stop.apply(lStream); - removeIncomeStream(lStream.stream); + removeIncomeStream(); + }, + () => { + player.assistManager.ping(AssistActionsPing.call.end, agentId); + lStream.stop.apply(lStream); + removeIncomeStream(); }, onReject, onError, diff --git a/frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx b/frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx index 3899d1ea0..812d4b9b9 100644 --- a/frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx +++ b/frontend/app/components/Assist/components/VideoContainer/VideoContainer.tsx @@ -34,43 +34,40 @@ function VideoContainer({ } const iid = setInterval(() => { const track = stream.getVideoTracks()[0]; - const settings = track?.getSettings(); - const isDummyVideoTrack = settings - ? settings.width === 2 || - settings.frameRate === 0 || - (!settings.frameRate && !settings.width) - : true; - const shouldBeEnabled = track.enabled && !isDummyVideoTrack; - if (isEnabled !== shouldBeEnabled) { - setEnabled(shouldBeEnabled); - setRemoteEnabled?.(shouldBeEnabled); + if (track) { + if (!track.enabled) { + setEnabled(false); + setRemoteEnabled?.(false); + } else { + setEnabled(true); + setRemoteEnabled?.(true); + } + } else { + setEnabled(false); + setRemoteEnabled?.(false); } }, 500); return () => clearInterval(iid); - }, [stream, isEnabled]); + }, [stream]); return (
-
); } diff --git a/frontend/app/player/common/ListWalker.ts b/frontend/app/player/common/ListWalker.ts index e37083120..a5c80a643 100644 --- a/frontend/app/player/common/ListWalker.ts +++ b/frontend/app/player/common/ListWalker.ts @@ -124,13 +124,9 @@ export default class ListWalker { * Assumed that the current message is already handled so * if pointer doesn't change is returned. */ - moveGetLast(t: number, index?: number): T | null { - let key: string = 'time'; // TODO - let val = t; - if (index) { - key = '_index'; - val = index; - } + moveGetLast(t: number, index?: number, force?: boolean, debug?: boolean): T | null { + const key: string = index ? '_index' : 'time'; + const val = index ? index : t; let changed = false; // @ts-ignore @@ -143,7 +139,10 @@ export default class ListWalker { this.movePrev(); changed = true; } - return changed ? this.list[this.p - 1] : null; + if (debug) { + console.log(this.list[this.p - 1]) + } + return changed || force ? this.list[this.p - 1] : null; } prevTs = 0; diff --git a/frontend/app/player/web/MessageLoader.ts b/frontend/app/player/web/MessageLoader.ts index 790fa2619..160312022 100644 --- a/frontend/app/player/web/MessageLoader.ts +++ b/frontend/app/player/web/MessageLoader.ts @@ -43,27 +43,6 @@ export default class MessageLoader { this.session = session; } - /** - * TODO: has to be moved out of messageLoader logic somehow - * */ - spriteMapSvg: SVGElement | null = null; - - potentialSpriteMap: Record = {}; - - domParser: DOMParser | null = null; - - createSpriteMap = () => { - if (!this.spriteMapSvg) { - this.domParser = new DOMParser(); - this.spriteMapSvg = document.createElementNS( - 'http://www.w3.org/2000/svg', - 'svg', - ); - this.spriteMapSvg.setAttribute('style', 'display: none;'); - this.spriteMapSvg.setAttribute('id', 'reconstructed-sprite'); - } - }; - createNewParser( shouldDecrypt = true, onMessagesDone: (msgs: PlayerMsg[], file?: string) => void, @@ -101,21 +80,6 @@ export default class MessageLoader { let startTimeSet = false; msgs.forEach((msg, i) => { - if (msg.tp === MType.SetNodeAttribute) { - if (msg.value.includes('_$OPENREPLAY_SPRITE$_')) { - this.createSpriteMap(); - if (!this.domParser) { - return console.error('DOM parser is not initialized?'); - } - handleSprites( - this.potentialSpriteMap, - this.domParser, - msg, - this.spriteMapSvg!, - i, - ); - } - } if (msg.tp === MType.Redux || msg.tp === MType.ReduxDeprecated) { if ('actionTime' in msg && msg.actionTime) { msg.time = msg.actionTime - this.session.startedAt; @@ -333,10 +297,6 @@ export default class MessageLoader { await Promise.allSettled([restDomFilesPromise, restDevtoolsFilesPromise]); this.messageManager.onFileReadSuccess(); - // no sprites for mobile - if (this.spriteMapSvg && 'injectSpriteMap' in this.messageManager) { - this.messageManager.injectSpriteMap(this.spriteMapSvg); - } }; loadEFSMobs = async () => { @@ -467,40 +427,6 @@ function findBrokenNodes(nodes: any[]) { return result; } -function handleSprites( - potentialSpriteMap: Record, - parser: DOMParser, - msg: Record, - spriteMapSvg: SVGElement, - i: number, -) { - const [_, svgData] = msg.value.split('_$OPENREPLAY_SPRITE$_'); - const potentialSprite = potentialSpriteMap[svgData]; - if (potentialSprite) { - msg.value = potentialSprite; - } else { - const svgDoc = parser.parseFromString(svgData, 'image/svg+xml'); - const originalSvg = svgDoc.querySelector('svg'); - if (originalSvg) { - const symbol = document.createElementNS( - 'http://www.w3.org/2000/svg', - 'symbol', - ); - const symbolId = `symbol-${msg.id || `ind-${i}`}`; // Generate an ID if missing - symbol.setAttribute('id', symbolId); - symbol.setAttribute( - 'viewBox', - originalSvg.getAttribute('viewBox') || '0 0 24 24', - ); - symbol.innerHTML = originalSvg.innerHTML; - - spriteMapSvg.appendChild(symbol); - msg.value = `#${symbolId}`; - potentialSpriteMap[svgData] = `#${symbolId}`; - } - } -} - // @ts-ignore window.searchOrphans = (msgs) => findBrokenNodes(msgs.filter((m) => [8, 9, 10, 70].includes(m.tp))); diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index 1a532ffcc..785b1d21e 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -201,8 +201,16 @@ export default class MessageManager { } Object.values(this.tabs).forEach((tab) => tab.onFileReadSuccess?.()); + + this.updateSpriteMap(); }; + public updateSpriteMap = () => { + if (this.spriteMapSvg) { + this.injectSpriteMap(this.spriteMapSvg); + } + } + public onFileReadFailed = (...e: any[]) => { logger.error(e); this.state.update({ error: true }); @@ -288,15 +296,17 @@ export default class MessageManager { } if (tabId) { + const stateUpdate: { currentTab?: string, tabs?: Set } = {} if (this.activeTab !== tabId) { - this.state.update({ currentTab: tabId }); + stateUpdate['currentTab'] = tabId; this.activeTab = tabId; this.tabs[this.activeTab].clean(); } const activeTabs = this.state.get().tabs; if (activeTabs.size !== this.activeTabManager.tabInstances.size) { - this.state.update({ tabs: this.activeTabManager.tabInstances }); + stateUpdate['tabs'] = this.activeTabManager.tabInstances; } + this.state.update(stateUpdate) } if (this.tabs[this.activeTab]) { @@ -335,9 +345,38 @@ export default class MessageManager { this.state.update({ tabChangeEvents: this.tabChangeEvents }); } + spriteMapSvg: SVGElement | null = null; + potentialSpriteMap: Record = {}; + domParser: DOMParser | null = null; + createSpriteMap = () => { + if (!this.spriteMapSvg) { + this.domParser = new DOMParser(); + this.spriteMapSvg = document.createElementNS( + 'http://www.w3.org/2000/svg', + 'svg', + ); + this.spriteMapSvg.setAttribute('style', 'display: none;'); + this.spriteMapSvg.setAttribute('id', 'reconstructed-sprite'); + } + }; + distributeMessage = (msg: Message & { tabId: string }): void => { // @ts-ignore placeholder msg for timestamps if (msg.tp === 9999) return; + if (msg.tp === MType.SetNodeAttribute) { + if (msg.value.includes('_$OPENREPLAY_SPRITE$_')) { + this.createSpriteMap(); + if (!this.domParser) { + return console.error('DOM parser is not initialized?'); + } + handleSprites( + this.potentialSpriteMap, + this.domParser, + msg, + this.spriteMapSvg!, + ); + } + } if (!this.tabs[msg.tabId]) { this.tabsAmount++; this.state.update({ @@ -452,3 +491,36 @@ function mapTabs(tabs: Record) { return tabMap; } + +function handleSprites( + potentialSpriteMap: Record, + parser: DOMParser, + msg: Record, + spriteMapSvg: SVGElement, +) { + const [_, svgData] = msg.value.split('_$OPENREPLAY_SPRITE$_'); + const potentialSprite = potentialSpriteMap[svgData]; + if (potentialSprite) { + msg.value = potentialSprite; + } else { + const svgDoc = parser.parseFromString(svgData, 'image/svg+xml'); + const originalSvg = svgDoc.querySelector('svg'); + if (originalSvg) { + const symbol = document.createElementNS( + 'http://www.w3.org/2000/svg', + 'symbol', + ); + const symbolId = `symbol-${msg.id || `ind-${msg.time}`}`; // Generate an ID if missing + symbol.setAttribute('id', symbolId); + symbol.setAttribute( + 'viewBox', + originalSvg.getAttribute('viewBox') || '0 0 24 24', + ); + symbol.innerHTML = originalSvg.innerHTML; + + spriteMapSvg.appendChild(symbol); + msg.value = `#${symbolId}`; + potentialSpriteMap[svgData] = `#${symbolId}`; + } + } +} diff --git a/frontend/app/player/web/TabManager.ts b/frontend/app/player/web/TabManager.ts index 597bc1894..7ab04291f 100644 --- a/frontend/app/player/web/TabManager.ts +++ b/frontend/app/player/web/TabManager.ts @@ -98,6 +98,7 @@ export default class TabSessionManager { private readonly state: Store<{ tabStates: { [tabId: string]: TabState }; tabNames: { [tabId: string]: string }; + location?: string; }>, private readonly screen: Screen, private readonly id: string, @@ -415,14 +416,16 @@ export default class TabSessionManager { } } /* === */ - const lastLocationMsg = this.locationManager.moveGetLast(t, index); + const lastLocationMsg = this.locationManager.moveGetLast(t, index, true); if (lastLocationMsg) { - const { tabNames } = this.state.get(); - if (lastLocationMsg.documentTitle) { - tabNames[this.id] = lastLocationMsg.documentTitle; + const { tabNames, location } = this.state.get(); + if (location !== lastLocationMsg.url) { + if (lastLocationMsg.documentTitle) { + tabNames[this.id] = lastLocationMsg.documentTitle; + } + // @ts-ignore comes from parent state + this.state.update({ location: lastLocationMsg.url, tabNames }); } - // @ts-ignore comes from parent state - this.state.update({ location: lastLocationMsg.url, tabNames }); } const lastPerformanceTrackMessage = diff --git a/frontend/app/player/web/WebLivePlayer.ts b/frontend/app/player/web/WebLivePlayer.ts index bcd217d8d..0baa194ef 100644 --- a/frontend/app/player/web/WebLivePlayer.ts +++ b/frontend/app/player/web/WebLivePlayer.ts @@ -43,6 +43,7 @@ export default class WebLivePlayer extends WebPlayer { wpState, (id) => this.messageManager.getNode(id), agentId, + this.messageManager.updateSpriteMap, uiErrorHandler, ); this.assistManager.connect(session.agentToken!, agentId, projectId); diff --git a/frontend/app/player/web/WebPlayer.ts b/frontend/app/player/web/WebPlayer.ts index 2977f95e4..ee92c7891 100644 --- a/frontend/app/player/web/WebPlayer.ts +++ b/frontend/app/player/web/WebPlayer.ts @@ -100,6 +100,8 @@ export default class WebPlayer extends Player { // @ts-ignore window.playerJumpToTime = this.jump.bind(this); + // @ts-ignore + window.__OPENREPLAY_DEV_TOOLS__.player = this; } preloadFirstFile(data: Uint8Array, fileKey?: string) { diff --git a/frontend/app/player/web/assist/AssistManager.ts b/frontend/app/player/web/assist/AssistManager.ts index 647a8a9e5..1b107cf17 100644 --- a/frontend/app/player/web/assist/AssistManager.ts +++ b/frontend/app/player/web/assist/AssistManager.ts @@ -3,14 +3,14 @@ import type { Socket } from 'socket.io-client'; import type { PlayerMsg, Store } from 'App/player'; import CanvasReceiver from 'Player/web/assist/CanvasReceiver'; import { gunzipSync } from 'fflate'; -import { Message } from '../messages'; +import { Message, MType } from '../messages'; import type Screen from '../Screen/Screen'; import MStreamReader from '../messages/MStreamReader'; import JSONRawMessageReader from '../messages/JSONRawMessageReader'; import Call, { CallingState } from './Call'; import RemoteControl, { RemoteControlStatus } from './RemoteControl'; import ScreenRecording, { SessionRecordingStatus } from './ScreenRecording'; - +import { debounceCall } from 'App/utils' export { RemoteControlStatus, SessionRecordingStatus, CallingState }; export enum ConnectionStatus { @@ -82,6 +82,7 @@ export default class AssistManager { private store: Store, private getNode: MessageManager['getNode'], public readonly agentId: number, + private readonly updateSpriteMap: () => void, public readonly uiErrorHandler?: { error: (msg: string) => void; }, @@ -200,6 +201,7 @@ export default class AssistManager { peerId: this.peerID, query: document.location.search, }), + config: JSON.stringify(this.getIceServers()), }, })); @@ -239,6 +241,11 @@ export default class AssistManager { msg !== null; msg = reader.readNext() ) { + if (msg.tp === MType.SetNodeAttribute) { + if (msg.value.includes('_$OPENREPLAY_SPRITE$_')) { + debounceCall(this.updateSpriteMap, 250)() + } + } this.handleMessage(msg, msg._index); } }; @@ -314,7 +321,7 @@ export default class AssistManager { this.callManager = new Call( this.store, socket, - this.config, + this.getIceServers(), this.peerID, this.getAssistVersion, { @@ -354,6 +361,23 @@ export default class AssistManager { }); } + private getIceServers = () => { + if (this.config) { + return this.config; + } + return [ + { + urls: [ + 'stun:stun.l.google.com:19302', + 'stun:stun1.l.google.com:19302', + 'stun:stun2.l.google.com:19302', + 'stun:stun3.l.google.com:19302', + 'stun:stun4.l.google.com:19302' + ], + }, + ] as RTCIceServer[]; + }; + /** * Sends event ping to stats service * */ diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index 592945e95..972649814 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -43,7 +43,7 @@ export default class Call { constructor( private store: Store }>, private socket: Socket, - private config: RTCIceServer[] | null, + private config: RTCIceServer[], private peerID: string, private getAssistVersion: () => number, private agent: Record, @@ -146,7 +146,7 @@ export default class Call { // create pc with ice config const pc = new RTCPeerConnection({ - iceServers: [{ urls: 'stun:stun.l.google.com:19302' }], + iceServers: this.config, }); // If there is a local stream, add its tracks to the connection @@ -185,8 +185,7 @@ export default class Call { pc.ontrack = (event) => { const stream = event.streams[0]; if (stream && !this.videoStreams[remotePeerId]) { - const clonnedStream = stream.clone(); - this.videoStreams[remotePeerId] = clonnedStream.getVideoTracks()[0]; + this.videoStreams[remotePeerId] = stream.getVideoTracks()[0]; if (this.store.get().calling !== CallingState.OnCall) { this.store.update({ calling: CallingState.OnCall }); } @@ -305,22 +304,18 @@ export default class Call { } try { // if the connection is not established yet, then set remoteDescription to peer - if (!pc.localDescription) { - await pc.setRemoteDescription(new RTCSessionDescription(data.offer)); - const answer = await pc.createAnswer(); - await pc.setLocalDescription(answer); - if (isAgent) { - this.socket.emit('WEBRTC_AGENT_CALL', { - from: this.callID, - answer, - toAgentId: getSocketIdByCallId(fromCallId), - type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER, - }); - } else { - this.socket.emit('webrtc_call_answer', { from: fromCallId, answer }); - } + await pc.setRemoteDescription(new RTCSessionDescription(data.offer)); + const answer = await pc.createAnswer(); + await pc.setLocalDescription(answer); + if (isAgent) { + this.socket.emit('WEBRTC_AGENT_CALL', { + from: this.callID, + answer, + toAgentId: getSocketIdByCallId(fromCallId), + type: WEBRTC_CALL_AGENT_EVENT_TYPES.ANSWER, + }); } else { - logger.warn('Skipping setRemoteDescription: Already in stable state'); + this.socket.emit('webrtc_call_answer', { from: fromCallId, answer }); } } catch (e) { logger.error('Error setting remote description from answer', e); @@ -388,13 +383,13 @@ export default class Call { private handleCallEnd() { // If the call is not completed, then call onCallEnd if (this.store.get().calling !== CallingState.NoCall) { - this.callArgs && this.callArgs.onCallEnd(); + this.callArgs && this.callArgs.onRemoteCallEnd(); } // change state to NoCall this.store.update({ calling: CallingState.NoCall }); // Close all created RTCPeerConnection Object.values(this.connections).forEach((pc) => pc.close()); - this.callArgs?.onCallEnd(); + this.callArgs?.onRemoteCallEnd(); // Clear connections this.connections = {}; this.callArgs = null; @@ -414,7 +409,7 @@ export default class Call { // Close all connections and reset callArgs Object.values(this.connections).forEach((pc) => pc.close()); this.connections = {}; - this.callArgs?.onCallEnd(); + this.callArgs?.onRemoteCallEnd(); this.store.update({ calling: CallingState.NoCall }); this.callArgs = null; } else { @@ -443,7 +438,8 @@ export default class Call { private callArgs: { localStream: LocalStream; onStream: (s: MediaStream, isAgent: boolean) => void; - onCallEnd: () => void; + onRemoteCallEnd: () => void; + onLocalCallEnd: () => void; onReject: () => void; onError?: (arg?: any) => void; } | null = null; @@ -451,14 +447,16 @@ export default class Call { setCallArgs( localStream: LocalStream, onStream: (s: MediaStream, isAgent: boolean) => void, - onCallEnd: () => void, + onRemoteCallEnd: () => void, + onLocalCallEnd: () => void, onReject: () => void, onError?: (e?: any) => void, ) { this.callArgs = { localStream, onStream, - onCallEnd, + onRemoteCallEnd, + onLocalCallEnd, onReject, onError, }; @@ -549,7 +547,7 @@ export default class Call { void this.initiateCallEnd(); Object.values(this.connections).forEach((pc) => pc.close()); this.connections = {}; - this.callArgs?.onCallEnd(); + this.callArgs?.onLocalCallEnd(); } } diff --git a/frontend/package.json b/frontend/package.json index 1c1a1676b..be06f0825 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -27,6 +27,7 @@ "@codewonders/html2canvas": "^1.0.2", "@eslint/js": "^9.21.0", "@medv/finder": "^4.0.2", + "@neodrag/react": "^2.3.0", "@sentry/browser": "^5.21.1", "@svg-maps/world": "^1.0.1", "@tanstack/react-query": "^5.56.2", @@ -72,7 +73,6 @@ "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^15.1.2", "react-dom": "^18.2.0", - "react-draggable": "^4.4.5", "react-google-recaptcha": "^2.1.0", "react-i18next": "^15.4.1", "react-intersection-observer": "^9.13.1", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index d742071cd..0c4c3c15e 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -2584,6 +2584,13 @@ __metadata: languageName: node linkType: hard +"@neodrag/react@npm:^2.3.0": + version: 2.3.0 + resolution: "@neodrag/react@npm:2.3.0" + checksum: 10c1/37f549ad0bdab8badb0799bf4e6f1714fe52a2ace6c09447a9dc97a9483a0fb40f75b35b121c78a1cba3a27ec9076a5db9672454af3a216d9efee8dcd34c6736 + languageName: node + linkType: hard + "@nicolo-ribaudo/chokidar-2@npm:2.1.8-no-fsevents.3": version: 2.1.8-no-fsevents.3 resolution: "@nicolo-ribaudo/chokidar-2@npm:2.1.8-no-fsevents.3" @@ -11989,6 +11996,7 @@ __metadata: "@eslint/js": "npm:^9.21.0" "@jest/globals": "npm:^29.7.0" "@medv/finder": "npm:^4.0.2" + "@neodrag/react": "npm:^2.3.0" "@openreplay/sourcemap-uploader": "npm:^3.0.10" "@sentry/browser": "npm:^5.21.1" "@svg-maps/world": "npm:^1.0.1" @@ -12079,7 +12087,6 @@ __metadata: react-dnd: "npm:^16.0.1" react-dnd-html5-backend: "npm:^15.1.2" react-dom: "npm:^18.2.0" - react-draggable: "npm:^4.4.5" react-google-recaptcha: "npm:^2.1.0" react-i18next: "npm:^15.4.1" react-intersection-observer: "npm:^9.13.1" @@ -14060,19 +14067,6 @@ __metadata: languageName: node linkType: hard -"react-draggable@npm:^4.4.5": - version: 4.4.6 - resolution: "react-draggable@npm:4.4.6" - dependencies: - clsx: "npm:^1.1.1" - prop-types: "npm:^15.8.1" - peerDependencies: - react: ">= 16.3.0" - react-dom: ">= 16.3.0" - checksum: 10c1/b8ae807f4556b658ae149b6542af5222d75996da47c549db54be22276579549ca2cd2fd06ca5c0852fbd1b663853cc0568b68215bd6e62433b46a7e154ddf8e2 - languageName: node - linkType: hard - "react-fit@npm:^2.0.0": version: 2.0.1 resolution: "react-fit@npm:2.0.1" diff --git a/networkProxy/.yarn/install-state.gz b/networkProxy/.yarn/install-state.gz new file mode 100644 index 000000000..361658ae0 Binary files /dev/null and b/networkProxy/.yarn/install-state.gz differ diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 41b3190e0..85ec6fa4f 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -20,6 +20,7 @@ import { gzip } from 'fflate' type StartEndCallback = (agentInfo?: Record) => ((() => any) | void) interface AgentInfo { + config: string; email: string; id: number name: string @@ -85,6 +86,7 @@ export default class Assist { private remoteControl: RemoteControl | null = null; private peerReconnectTimeout: ReturnType | null = null private agents: Record = {} + private config: RTCIceServer[] | undefined private readonly options: Options private readonly canvasMap: Map = new Map() @@ -251,7 +253,7 @@ export default class Assist { return } if (args[0] !== 'webrtc_call_ice_candidate') { - app.debug.log('Socket:', ...args) + app.debug.log("Socket:", ...args); }; socket.on('close', (e) => { app.debug.warn('Socket closed:', e); @@ -353,6 +355,9 @@ export default class Assist { this.app.stop() this.app.clearBuffers() this.app.waitStatus(0) + .then(() => { + this.config = JSON.parse(info.config); + }) .then(() => { this.app.allowAppStart() setTimeout(() => { @@ -401,7 +406,7 @@ export default class Assist { Object.values(this.calls).forEach(pc => pc.close()) this.calls.clear(); - + recordingState.stopAgentRecording(id) endAgentCall({ socketId: id }) }) @@ -548,6 +553,16 @@ export default class Assist { } } + const renegotiateConnection = async ({ pc, from }: { pc: RTCPeerConnection, from: string }) => { + try { + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + this.emit('webrtc_call_offer', { from, offer }); + } catch (error) { + app.debug.error("Error with renegotiation:", error); + } + }; + const handleIncomingCallOffer = async (from: string, offer: RTCSessionDescriptionInit) => { app.debug.log('handleIncomingCallOffer', from) let confirmAnswer: Promise @@ -572,13 +587,19 @@ export default class Assist { try { // waiting for a decision on accepting the challenge - const agreed = await confirmAnswer + const agreed = await confirmAnswer; // if rejected, then terminate the call if (!agreed) { initiateCallEnd() this.options.onCallDeny?.() return } + + // create a new RTCPeerConnection with ice server config + const pc = new RTCPeerConnection({ + iceServers: this.config, + }); + if (!callUI) { callUI = new CallWindow(app.debug.error, this.options.callUITemplate) callUI.setVideoToggleCallback((args: { enabled: boolean }) => @@ -598,7 +619,7 @@ export default class Assist { if (!lStreams[from]) { app.debug.log('starting new stream for', from) // request a local stream, and set it to lStreams - lStreams[from] = await RequestLocalStream() + lStreams[from] = await RequestLocalStream(pc, renegotiateConnection.bind(null, { pc, from })) } // we pass the received tracks to Call ui callUI.setLocalStreams(Object.values(lStreams)) @@ -606,22 +627,50 @@ export default class Assist { app.debug.error('Error requesting local stream', e); // if something didn't work out, we terminate the call initiateCallEnd(); + this.options.onCallDeny?.(); + return; + } + + if (!callUI) { + callUI = new CallWindow(app.debug.error, this.options.callUITemplate); + callUI.setVideoToggleCallback((args: { enabled: boolean }) => { + this.emit("videofeed", { streamId: from, enabled: args.enabled }) + }); + } + // show buttons in the call window + callUI.showControls(initiateCallEnd); + if (!annot) { + annot = new AnnotationCanvas(); + annot.mount(); + } + + + + // callUI.setLocalStreams(Object.values(lStreams)) + try { + // if there are no local streams in lStrems then we set + if (!lStreams[from]) { + app.debug.log("starting new stream for", from); + // request a local stream, and set it to lStreams + lStreams[from] = await RequestLocalStream(pc, renegotiateConnection.bind(null, { pc, from })); + } + // we pass the received tracks to Call ui + callUI.setLocalStreams(Object.values(lStreams)); + } catch (e) { + app.debug.error("Error requesting local stream", e); + // if something didn't work out, we terminate the call + initiateCallEnd(); return; } - // create a new RTCPeerConnection with ice server config - const pc = new RTCPeerConnection({ - iceServers: [{ urls: "stun:stun.l.google.com:19302" }], - }); // get all local tracks and add them to RTCPeerConnection - lStreams[from].stream.getTracks().forEach(track => { - pc.addTrack(track, lStreams[from].stream); - }); - // When we receive local ice candidates, we emit them via socket pc.onicecandidate = (event) => { if (event.candidate) { - socket.emit('webrtc_call_ice_candidate', { from, candidate: event.candidate }); + socket.emit("webrtc_call_ice_candidate", { + from, + candidate: event.candidate, + }); } }; @@ -632,9 +681,9 @@ export default class Assist { callUI.addRemoteStream(rStream, from); const onInteraction = () => { callUI?.playRemote(); - document.removeEventListener('click', onInteraction); + document.removeEventListener("click", onInteraction); }; - document.addEventListener('click', onInteraction); + document.addEventListener("click", onInteraction); } }; @@ -648,7 +697,7 @@ export default class Assist { // set answer as local description await pc.setLocalDescription(answer); // set the response as local - socket.emit('webrtc_call_answer', { from, answer }); + socket.emit("webrtc_call_answer", { from, answer }); // If the state changes to an error, we terminate the call // pc.onconnectionstatechange = () => { @@ -658,27 +707,35 @@ export default class Assist { // }; // Update track when local video changes - lStreams[from].onVideoTrack(vTrack => { - const sender = pc.getSenders().find(s => s.track?.kind === 'video'); + lStreams[from].onVideoTrack((vTrack) => { + const sender = pc.getSenders().find((s) => s.track?.kind === "video"); if (!sender) { - app.debug.warn('No video sender found') - return + app.debug.warn("No video sender found"); + return; } - sender.replaceTrack(vTrack) - }) + sender.replaceTrack(vTrack); + }); // if the user closed the tab or switched, then we end the call - document.addEventListener('visibilitychange', () => { - initiateCallEnd() - }) + document.addEventListener("visibilitychange", () => { + initiateCallEnd(); + }); // when everything is set, we change the state to true - this.setCallingState(CallingState.True) - if (!callEndCallback) { callEndCallback = this.options.onCallStart?.() } - const callingPeerIdsNow = Array.from(this.calls.keys()) + this.setCallingState(CallingState.True); + if (!callEndCallback) { + callEndCallback = this.options.onCallStart?.(); + } + const callingPeerIdsNow = Array.from(this.calls.keys()); // in session storage we write down everyone with whom the call is established - sessionStorage.setItem(this.options.session_calling_peer_key, JSON.stringify(callingPeerIdsNow)) - this.emit('UPDATE_SESSION', { agentIds: callingPeerIdsNow, isCallActive: true }) + sessionStorage.setItem( + this.options.session_calling_peer_key, + JSON.stringify(callingPeerIdsNow) + ); + this.emit("UPDATE_SESSION", { + agentIds: callingPeerIdsNow, + isCallActive: true, + }); } catch (reason) { app.debug.log(reason); } @@ -712,7 +769,7 @@ export default class Assist { if (!this.canvasPeers[uniqueId]) { this.canvasPeers[uniqueId] = new RTCPeerConnection({ - iceServers: [{ urls: "stun:stun.l.google.com:19302" }], + iceServers: this.config, }); this.setupPeerListeners(uniqueId); @@ -726,7 +783,6 @@ export default class Assist { // Send offer via signaling server socket.emit('webrtc_canvas_offer', { offer, id: uniqueId }); - } } } @@ -828,4 +884,4 @@ export default class Assist { * // }) * // slPeer.on('error', console.error) * // this.emit('canvas_stream', { canvasId, }) - * */ \ No newline at end of file + * */ diff --git a/tracker/tracker-assist/src/CallWindow.ts b/tracker/tracker-assist/src/CallWindow.ts index f8bd31796..fd9cfb4ee 100644 --- a/tracker/tracker-assist/src/CallWindow.ts +++ b/tracker/tracker-assist/src/CallWindow.ts @@ -48,7 +48,7 @@ export default class CallWindow { } // const baseHref = "https://static.openreplay.com/tracker-assist/test" - const baseHref = 'https://static.openreplay.com/tracker-assist/4.0.0' + const baseHref = 'https://static.openreplay.com/tracker-assist/widget' // this.load = fetch(this.callUITemplate || baseHref + '/index2.html') this.load = fetch(this.callUITemplate || baseHref + '/index.html') .then((r) => r.text()) @@ -60,7 +60,7 @@ export default class CallWindow { }, 0) //iframe.style.height = doc.body.scrollHeight + 'px'; //iframe.style.width = doc.body.scrollWidth + 'px'; - this.adjustIframeSize() + this.adjustIframeSize() iframe.onload = null } // ? @@ -152,15 +152,6 @@ export default class CallWindow { if (this.checkRemoteVideoInterval) { clearInterval(this.checkRemoteVideoInterval) } // just in case - let enabled = false - this.checkRemoteVideoInterval = setInterval(() => { - const settings = this.remoteVideo?.getSettings() - const isDummyVideoTrack = !this.remoteVideo.enabled || (!!settings && (settings.width === 2 || settings.frameRate === 0)) - const shouldBeEnabled = !isDummyVideoTrack - if (enabled !== shouldBeEnabled) { - this.toggleRemoteVideoUI((enabled = shouldBeEnabled)) - } - }, 1000) } // Audio diff --git a/tracker/tracker-assist/src/LocalStream.ts b/tracker/tracker-assist/src/LocalStream.ts index ef3a135e1..ffda37792 100644 --- a/tracker/tracker-assist/src/LocalStream.ts +++ b/tracker/tracker-assist/src/LocalStream.ts @@ -1,88 +1,86 @@ -declare global { - interface HTMLCanvasElement { - captureStream(frameRate?: number): MediaStream; - } -} - -function dummyTrack(): MediaStreamTrack { - const canvas = document.createElement('canvas')//, { width: 0, height: 0}) - canvas.setAttribute('data-openreplay-hidden', '1') - canvas.width=canvas.height=2 // Doesn't work when 1 (?!) - const ctx = canvas.getContext('2d') - ctx?.fillRect(0, 0, canvas.width, canvas.height) - requestAnimationFrame(function draw(){ - ctx?.fillRect(0,0, canvas.width, canvas.height) - requestAnimationFrame(draw) - }) - // Also works. Probably it should be done once connected. - //setTimeout(() => { ctx?.fillRect(0,0, canvas.width, canvas.height) }, 4000) - return canvas.captureStream(60).getTracks()[0] -} - -export default function RequestLocalStream(): Promise { - return navigator.mediaDevices.getUserMedia({ audio:true, }) - .then(aStream => { - const aTrack = aStream.getAudioTracks()[0] - - if (!aTrack) { throw new Error('No audio tracks provided') } - return new _LocalStream(aTrack) - }) +export default function RequestLocalStream( + pc: RTCPeerConnection, + toggleVideoCb?: () => void +): Promise { + return navigator.mediaDevices + .getUserMedia({ audio: true, video: false }) + .then((stream) => { + const aTrack = stream.getAudioTracks()[0]; + if (!aTrack) { + throw new Error("No audio tracks provided"); + } + stream.getTracks().forEach((track) => { + pc.addTrack(track, stream); + }); + return new _LocalStream(stream, pc, toggleVideoCb); + }); } class _LocalStream { - private mediaRequested = false - readonly stream: MediaStream - private readonly vdTrack: MediaStreamTrack - constructor(aTrack: MediaStreamTrack) { - this.vdTrack = dummyTrack() - this.stream = new MediaStream([ aTrack, this.vdTrack, ]) + private mediaRequested = false; + readonly stream: MediaStream; + readonly vTrack: MediaStreamTrack; + readonly pc: RTCPeerConnection; + readonly toggleVideoCb?: () => void; + constructor(stream: MediaStream, pc: RTCPeerConnection, toggleVideoCb?: () => void) { + this.stream = stream; + this.pc = pc; + this.toggleVideoCb = toggleVideoCb; } toggleVideo(): Promise { + const videoTracks = this.stream.getVideoTracks(); if (!this.mediaRequested) { - return navigator.mediaDevices.getUserMedia({video:true,}) - .then(vStream => { - const vTrack = vStream.getVideoTracks()[0] - if (!vTrack) { - throw new Error('No video track provided') - } - this.stream.addTrack(vTrack) - this.stream.removeTrack(this.vdTrack) - this.mediaRequested = true - if (this.onVideoTrackCb) { - this.onVideoTrackCb(vTrack) - } - return true - }) - .catch(e => { - // TODO: log - console.error(e) - return false - }) + return navigator.mediaDevices + .getUserMedia({ video: true }) + .then((vStream) => { + const vTrack = vStream.getVideoTracks()[0]; + if (!vTrack) { + throw new Error("No video track provided"); + } + + this.pc.addTrack(vTrack, this.stream); + this.stream.addTrack(vTrack); + + if (this.toggleVideoCb) { + this.toggleVideoCb(); + } + + this.mediaRequested = true; + + if (this.onVideoTrackCb) { + this.onVideoTrackCb(vTrack); + } + return true; + }) + .catch((e) => { + // TODO: log + return false; + }); + } else { + videoTracks.forEach((track) => { + track.enabled = !track.enabled; + }); } - let enabled = true - this.stream.getVideoTracks().forEach(track => { - track.enabled = enabled = enabled && !track.enabled - }) - return Promise.resolve(enabled) + return Promise.resolve(videoTracks[0].enabled); } toggleAudio(): boolean { - let enabled = true - this.stream.getAudioTracks().forEach(track => { - track.enabled = enabled = enabled && !track.enabled - }) - return enabled + let enabled = true; + this.stream.getAudioTracks().forEach((track) => { + track.enabled = enabled = enabled && !track.enabled; + }); + return enabled; } - private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null + private onVideoTrackCb: ((t: MediaStreamTrack) => void) | null = null; onVideoTrack(cb: (t: MediaStreamTrack) => void) { - this.onVideoTrackCb = cb + this.onVideoTrackCb = cb; } stop() { - this.stream.getTracks().forEach(t => t.stop()) + this.stream.getTracks().forEach((t) => t.stop()); } } -export type LocalStream = InstanceType +export type LocalStream = InstanceType;