diff --git a/frontend/app/components/Session/Player/LivePlayer/Overlay/LiveOverlay.tsx b/frontend/app/components/Session/Player/LivePlayer/Overlay/LiveOverlay.tsx index 89569d7a4..431fc617f 100644 --- a/frontend/app/components/Session/Player/LivePlayer/Overlay/LiveOverlay.tsx +++ b/frontend/app/components/Session/Player/LivePlayer/Overlay/LiveOverlay.tsx @@ -33,7 +33,8 @@ function Overlay({ tabStates, currentTab } = store.get() - const cssLoading = tabStates[currentTab].cssLoading || false + + const cssLoading = tabStates[currentTab]?.cssLoading || false const loading = messagesLoading || cssLoading const liveStatusText = getStatusText(peerConnectionStatus) const connectionStatus = peerConnectionStatus diff --git a/frontend/app/player/web/assist/AssistManager.ts b/frontend/app/player/web/assist/AssistManager.ts index 0be4efc01..d789b7efc 100644 --- a/frontend/app/player/web/assist/AssistManager.ts +++ b/frontend/app/player/web/assist/AssistManager.ts @@ -169,15 +169,13 @@ export default class AssistManager { let currentTab = '' socket.on('messages', messages => { - jmr.append(messages) // as RawMessage[] - console.log(messages) + jmr.append(messages.data) // as RawMessage[] if (waitingForMessages) { waitingForMessages = false // TODO: more explicit this.setStatus(ConnectionStatus.Connected) } for (let msg = reader.readNext();msg !== null;msg = reader.readNext()) { - console.log(msg) this.handleMessage(msg, msg._index) } }) @@ -188,9 +186,15 @@ export default class AssistManager { this.setStatus(ConnectionStatus.Connected) }) - socket.on('UPDATE_SESSION', ({ active }) => { + socket.on('UPDATE_SESSION', (evData) => { + const { metadata, data } = evData + const { tabId } = metadata + const { active } = data this.clearDisconnectTimeout() !this.inactiveTimeout && this.setStatus(ConnectionStatus.Connected) + if (tabId !== currentTab) { + this.store.update({ currentTab: tabId }) + } if (typeof active === "boolean") { this.clearInactiveTimeout() if (active) { diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index 398776e3f..f58c85ea7 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -18,6 +18,7 @@ export enum CallingState { export interface State { calling: CallingState; + currentTab?: string; } export default class Call { @@ -158,7 +159,7 @@ export default class Call { } initiateCallEnd = async () => { - this.socket?.emit("call_end", appStore.getState().getIn([ 'user', 'account', 'name'])) + this.emitData("call_end", appStore.getState().getIn([ 'user', 'account', 'name'])) this.handleCallEnd() // TODO: We have it separated, right? (check) // const remoteControl = this.store.get().remoteControl @@ -168,6 +169,10 @@ export default class Call { // } } + private emitData = (event: string, data?: any) => { + this.socket?.emit(event, { meta: { tabId: this.store.get().currentTab }, data }) + } + private callArgs: { localStream: LocalStream, @@ -206,7 +211,7 @@ export default class Call { toggleVideoLocalStream(enabled: boolean) { this.getPeer().then((peer) => { - this.socket.emit('videofeed', { streamId: peer.id, enabled }) + this.emitData('videofeed', { streamId: peer.id, enabled }) }) } @@ -223,7 +228,7 @@ export default class Call { if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) { return } this.store.update({ calling: CallingState.Connecting }) this._peerConnection(this.peerID); - this.socket.emit("_agent_name", appStore.getState().getIn([ 'user', 'account', 'name'])) + this.emitData("_agent_name", appStore.getState().getIn([ 'user', 'account', 'name'])) } private async _peerConnection(remotePeerId: string) { diff --git a/frontend/app/player/web/assist/RemoteControl.ts b/frontend/app/player/web/assist/RemoteControl.ts index 4c9021054..92bcd8a0c 100644 --- a/frontend/app/player/web/assist/RemoteControl.ts +++ b/frontend/app/player/web/assist/RemoteControl.ts @@ -12,6 +12,7 @@ export enum RemoteControlStatus { export interface State { annotating: boolean remoteControl: RemoteControlStatus + currentTab?: string } export default class RemoteControl { @@ -28,11 +29,11 @@ export default class RemoteControl { private agentInfo: Object, private onToggle: (active: boolean) => void, ){ - socket.on("control_granted", id => { - this.toggleRemoteControl(id === socket.id) + socket.on("control_granted", ({ meta, data }) => { + this.toggleRemoteControl(data === socket.id) }) - socket.on("control_rejected", id => { - id === socket.id && this.toggleRemoteControl(false) + socket.on("control_rejected", ({ meta, data }) => { + data === socket.id && this.toggleRemoteControl(false) this.onReject() }) socket.on('SESSION_DISCONNECTED', () => { @@ -50,14 +51,19 @@ export default class RemoteControl { private onMouseMove = (e: MouseEvent): void => { const data = this.screen.getInternalCoordinates(e) - this.socket.emit("move", [ data.x, data.y ]) + this.emitData("move", [ data.x, data.y ]) + } + + private emitData = (event: string, data?: any) => { + console.log('emit data', event, data, { meta: { tabId: this.store.get().currentTab }, data }) + this.socket.emit(event, { meta: { tabId: this.store.get().currentTab }, data }) } private onWheel = (e: WheelEvent): void => { e.preventDefault() //throttling makes movements less smooth, so it is omitted //this.onMouseMove(e) - this.socket.emit("scroll", [ e.deltaX, e.deltaY ]) + this.emitData("scroll", [ e.deltaX, e.deltaY ]) } public setCallbacks = ({ onReject }: { onReject: () => void }) => { @@ -76,9 +82,9 @@ export default class RemoteControl { if (el instanceof HTMLTextAreaElement || el instanceof HTMLInputElement ) { - this.socket && this.socket.emit("input", el.value) + this.socket && this.emitData("input", el.value) } else if (el.isContentEditable) { - this.socket && this.socket.emit("input", el.innerText) + this.socket && this.emitData("input", el.innerText) } } // TODO: send "focus" event to assist with the nodeID @@ -92,7 +98,7 @@ export default class RemoteControl { el.onblur = null } } - this.socket.emit("click", [ data.x, data.y ]); + this.emitData("click", [ data.x, data.y ]); } private toggleRemoteControl(enable: boolean){ @@ -116,17 +122,17 @@ export default class RemoteControl { if (remoteControl === RemoteControlStatus.Requesting) { return } if (remoteControl === RemoteControlStatus.Disabled) { this.store.update({ remoteControl: RemoteControlStatus.Requesting }) - this.socket.emit("request_control", JSON.stringify({ - ...this.agentInfo, - query: document.location.search - })) + this.emitData("request_control", JSON.stringify({ + ...this.agentInfo, + query: document.location.search + })) } else { this.releaseRemoteControl() } } releaseRemoteControl = () => { - this.socket.emit("release_control") + this.emitData("release_control",) this.toggleRemoteControl(false) } @@ -140,24 +146,24 @@ export default class RemoteControl { const annot = this.annot = new AnnotationCanvas() annot.mount(this.screen.overlay) annot.canvas.addEventListener("mousedown", e => { - const data = this.screen.getInternalViewportCoordinates(e) + const data = this.screen.getInternalViewportCoordin1ates(e) annot.start([ data.x, data.y ]) - this.socket.emit("startAnnotation", [ data.x, data.y ]) + this.emitData("startAnnotation", [ data.x, data.y ]) }) annot.canvas.addEventListener("mouseleave", () => { annot.stop() - this.socket.emit("stopAnnotation") + this.emitData("stopAnnotation") }) annot.canvas.addEventListener("mouseup", () => { annot.stop() - this.socket.emit("stopAnnotation") + this.emitData("stopAnnotation") }) annot.canvas.addEventListener("mousemove", e => { if (!annot.isPainting()) { return } const data = this.screen.getInternalViewportCoordinates(e) annot.move([ data.x, data.y ]) - this.socket.emit("moveAnnotation", [ data.x, data.y ]) + this.emitData("moveAnnotation", [ data.x, data.y ]) }) this.store.update({ annotating: true }) } else if (!enable && !!this.annot) { diff --git a/frontend/app/player/web/assist/ScreenRecording.ts b/frontend/app/player/web/assist/ScreenRecording.ts index 83b06b497..fdbd850a2 100644 --- a/frontend/app/player/web/assist/ScreenRecording.ts +++ b/frontend/app/player/web/assist/ScreenRecording.ts @@ -10,6 +10,7 @@ export enum SessionRecordingStatus { export interface State { recordingState: SessionRecordingStatus; + currentTab?: string; } export default class ScreenRecording { @@ -46,14 +47,19 @@ export default class ScreenRecording { if (recordingState === SessionRecordingStatus.Requesting) return; this.store.update({ recordingState: SessionRecordingStatus.Requesting }) - this.socket.emit("request_recording", JSON.stringify({ - ...this.agentInfo, - query: document.location.search, - })) + this.emitData("request_recording", JSON.stringify({ + ...this.agentInfo, + query: document.location.search, + }) + ) + } + + private emitData = (event: string, data?: any) => { + this.socket.emit(event, { meta: { tabId: this.store.get().currentTab }, data }) } stopRecording = () => { - this.socket.emit("stop_recording") + this.emitData("stop_recording") this.toggleRecording(false) } diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index b4439135b..e0efa1701 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -127,8 +127,8 @@ export default class Assist { app.session.attachUpdateCallback(sessInfo => this.emit('UPDATE_SESSION', sessInfo)) } - private emit(ev: string, ...args): void { - this.socket && this.socket.emit(ev, ...args) + private emit(ev: string, args?: any): void { + this.socket && this.socket.emit(ev, { meta: { tabId: this.app.getTabId(), }, data: args, }) } private get agentsConnected(): boolean { @@ -164,7 +164,7 @@ export default class Assist { if (!sessionId) { return app.debug.error('No session ID') } - const peerID = `${app.getProjectKey()}-${sessionId}-${app.session.getTabId()}` + const peerID = `${app.getProjectKey()}-${sessionId}-${this.app.getTabId()}` // SocketIO const socket = this.socket = connect(this.getHost(), { @@ -172,6 +172,7 @@ export default class Assist { query: { 'peerId': peerID, 'identity': 'session', + 'tabId': this.app.getTabId(), 'sessionInfo': JSON.stringify({ pageTitle: document.title, active: true, @@ -180,7 +181,12 @@ export default class Assist { }, transports: ['websocket',], }) - socket.onAny((...args) => app.debug.log('Socket:', ...args)) + socket.onAny((...args) => { + if (args[0] === 'messages' || args[0] === 'UPDATE_SESSION') { + return + } + app.debug.log('Socket:', ...args) + }) this.remoteControl = new RemoteControl( this.options, @@ -197,7 +203,11 @@ export default class Assist { annot.mount() return callingAgents.get(id) }, - (id, isDenied) => { + (id, isDenied) => onRelease(id, isDenied), + ) + + const onRelease = (id, isDenied) => { + { if (id) { const cb = this.agents[id].onControlReleased delete this.agents[id].onControlReleased @@ -217,8 +227,8 @@ export default class Assist { const info = id ? this.agents[id]?.agentInfo : {} this.options.onRemoteControlDeny?.(info || {}) } - }, - ) + } + } const onAcceptRecording = () => { socket.emit('recording_accepted') @@ -230,24 +240,37 @@ export default class Assist { } const recordingState = new ScreenRecordingState(this.options.recordingConfirm) - // TODO: check incoming args - socket.on('request_control', this.remoteControl.requestControl) - socket.on('release_control', this.remoteControl.releaseControl) - socket.on('scroll', this.remoteControl.scroll) - socket.on('click', this.remoteControl.click) - socket.on('move', this.remoteControl.move) - socket.on('focus', (clientID, nodeID) => { - const el = app.nodes.getNode(nodeID) - if (el instanceof HTMLElement && this.remoteControl) { - this.remoteControl.focus(clientID, el) + function processEvent(agentId: string, event: { meta: { tabId: string }, data?: any }, callback?: (id: string, data: any) => void) { + if (app.getTabId() === event.meta.tabId) { + return callback?.(agentId, event.data) } - }) - socket.on('input', this.remoteControl.input) + } + if (this.remoteControl !== null) { + socket.on('request_control', (agentId, dataObj) => { + processEvent(agentId, dataObj, this.remoteControl?.requestControl) + }) + socket.on('release_control', (agentId, dataObj) => { + processEvent(agentId, dataObj, (_, data) => + this.remoteControl?.releaseControl(data) + ) + }) + socket.on('scroll', (id, event) => processEvent(id, event, this.remoteControl?.scroll)) + socket.on('click', (id, event) => processEvent(id, event, this.remoteControl?.click)) + socket.on('move', (id, event) => processEvent(id, event, this.remoteControl?.move)) + socket.on('focus', (id, event) => processEvent(id, event, (clientID, nodeID) => { + const el = app.nodes.getNode(nodeID) + if (el instanceof HTMLElement && this.remoteControl) { + this.remoteControl.focus(clientID, el) + } + })) + socket.on('input', (id, event) => processEvent(id, event, this.remoteControl?.input)) + } - socket.on('moveAnnotation', (_, p) => annot && annot.move(p)) // TODO: restrict by id - socket.on('startAnnotation', (_, p) => annot && annot.start(p)) - socket.on('stopAnnotation', () => annot && annot.stop()) + // TODO: restrict by id + socket.on('moveAnnotation', (id, event) => processEvent(id, event, (_, d) => annot && annot.move(d))) + socket.on('startAnnotation', (id, event) => processEvent(id, event, (_, d) => annot?.start(d))) + socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop)) socket.on('NEW_AGENT', (id: string, info) => { this.agents[id] = { @@ -287,7 +310,8 @@ export default class Assist { this.agents = {} if (recordingState.isActive) recordingState.stopRecording() }) - socket.on('call_end', (id) => { + socket.on('call_end', (info) => { + const id = info.data if (!callingAgents.has(id)) { app.debug.warn('Received call_end from unknown agent', id) return @@ -295,14 +319,20 @@ export default class Assist { endAgentCall(id) }) - socket.on('_agent_name', (id, name) => { + socket.on('_agent_name', (id, info) => { + if (app.getTabId() !== info.meta.tabId) return + const name = info.data callingAgents.set(id, name) updateCallerNames() }) - socket.on('videofeed', (_, feedState) => { + socket.on('videofeed', (_, info) => { + if (app.getTabId() !== info.meta.tabId) return + const feedState = info.data callUI?.toggleVideoStream(feedState) }) - socket.on('request_recording', (id, agentData) => { + socket.on('request_recording', (id, info) => { + if (app.getTabId() !== info.meta.tabId) return + const agentData = info.data if (!recordingState.isActive) { this.options.onRecordingRequest?.(JSON.parse(agentData)) recordingState.requestRecording(id, onAcceptRecording, () => onRejectRecording(agentData)) @@ -310,7 +340,8 @@ export default class Assist { this.emit('recording_busy') } }) - socket.on('stop_recording', (id) => { + socket.on('stop_recording', (id, info) => { + if (app.getTabId() !== info.meta.tabId) return if (recordingState.isActive) { recordingState.stopAgentRecording(id) } diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index c56ed3222..b8e556925 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -89,6 +89,9 @@ export default class RemoteControl { } this.mouse = new Mouse(agentName) this.mouse.mount() + document.addEventListener('visibilitychange', () => { + if (document.hidden) this.releaseControl(false) + }) } resetMouse = () => { @@ -97,7 +100,9 @@ export default class RemoteControl { } scroll = (id, d) => { id === this.agentID && this.mouse?.scroll(d) } - move = (id, xy) => { id === this.agentID && this.mouse?.move(xy) } + move = (id, xy) => { + return id === this.agentID && this.mouse?.move(xy) + } private focused: HTMLElement | null = null click = (id, xy) => { if (id !== this.agentID || !this.mouse) { return } diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index fc4a33351..a765d2b08 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -636,6 +636,10 @@ export default class App { }) } } + + getTabId() { + return this.session.getTabId() + } stop(stopWorker = true): void { if (this.activityState !== ActivityState.NotActive) { try { diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index a9201e012..d15fc23fa 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -218,6 +218,13 @@ export default class API { } return this.app.getSessionID() } + + getTabId() { + if (this.app === null) { + return null + } + return this.app.getTabId() + } sessionID(): string | null | undefined { deprecationWarn("'sessionID' method", "'getSessionID' method", '/') return this.getSessionID()