diff --git a/tracker/tracker-assist/layout/index.html b/tracker/tracker-assist/layout/index.html index 6c48602e4..b432a28e2 100644 --- a/tracker/tracker-assist/layout/index.html +++ b/tracker/tracker-assist/layout/index.html @@ -1,26 +1,17 @@ - + - - + + OpenReplay | Assist + + - -
-
The agent is requesting remote control
-
- - -
-
-
-
Answer the call so the agent can assist.
-
- - -
-
+
-
Connecting...
-
-
+
+
Connecting...
+
+
- +

Starting video...

- +

Starting video...

-
+
- - - - + + +
- - -
- - - - -
-
-
- - - - Chat -
-
- - - -
-
-
-
-
Hey, did you get the key?
-
- Username - 00:00 -
-
-
-
Oui, merci!
-
- Username - 00:00 -
-
-
-
- -
- - - -
+
+ End
Remote control active
-
+
diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index a5bc2ce22..a20fadb06 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -158,8 +158,9 @@ export default class Assist { annot = null } callUI?.hideRemoteControl() - if (!CallingState.True) { + if (this.callingState !== CallingState.True) { callUI?.remove() + callUI = null } } @@ -169,13 +170,18 @@ export default class Assist { if (!callUI) { callUI = new CallWindow(app.debug.error) } - callUI.showRemoteControl() - callUI.setRemoteControlEnd(() => releaseControlCb(id)) + setTimeout(() => { + if (this.callingState === CallingState.False) { + callUI?.showRemoteOnly() + } else { + callUI?.showRemoteControl() + } + }, 150) this.agents[id].onControlReleased = this.options.onRemoteControlStart() this.emit('control_granted', id) annot = new AnnotationCanvas() annot.mount() - return callingAgents.get(id) + return { agentName: callingAgents.get(id), callUI, } }, releaseControlCb, ) @@ -317,6 +323,7 @@ export default class Assist { // UI closeCallConfirmWindow() + remoteControl.releaseControl() callUI?.remove() annot?.remove() callUI = null @@ -377,6 +384,8 @@ export default class Assist { callUI = new CallWindow(app.debug.error) } callUI.setCallEndAction(initiateCallEnd) + callUI.showControls() + if (!annot) { annot = new AnnotationCanvas() annot.mount() diff --git a/tracker/tracker-assist/src/CallWindow.ts b/tracker/tracker-assist/src/CallWindow.ts index 0c1b9b043..a85db20e5 100644 --- a/tracker/tracker-assist/src/CallWindow.ts +++ b/tracker/tracker-assist/src/CallWindow.ts @@ -4,274 +4,321 @@ import attachDND from './dnd.js' const SS_START_TS_KEY = '__openreplay_assist_call_start_ts' export default class CallWindow { - private readonly iframe: HTMLIFrameElement - private vRemote: HTMLVideoElement | null = null - private vLocal: HTMLVideoElement | null = null - private audioBtn: HTMLElement | null = null - private videoBtn: HTMLElement | null = null - private endCallBtn: HTMLElement | null = null - private agentNameElem: HTMLElement | null = null - private videoContainer: HTMLElement | null = null - private vPlaceholder: HTMLElement | null = null - private remoteControlContainer: HTMLElement | null = null - private remoteControlEndBtn: HTMLElement | null = null + private readonly iframe: HTMLIFrameElement + private vRemote: HTMLVideoElement | null = null + private vLocal: HTMLVideoElement | null = null + private audioBtn: HTMLElement | null = null + private videoBtn: HTMLElement | null = null + private endCallBtn: HTMLElement | null = null + private agentNameElem: HTMLElement | null = null + private videoContainer: HTMLElement | null = null + private vPlaceholder: HTMLElement | null = null + private remoteControlContainer: HTMLElement | null = null + private remoteControlEndBtn: HTMLElement | null = null + private controlsContainer: HTMLElement | null = null - private tsInterval: ReturnType + private tsInterval: ReturnType - private readonly load: Promise + private readonly load: Promise - constructor(private readonly logError: (...args: any[]) => void) { - const iframe = this.iframe = document.createElement('iframe') - Object.assign(iframe.style, { - position: 'fixed', - zIndex: 2147483647 - 1, - border: 'none', - bottom: '10px', - right: '10px', - height: '200px', - width: '200px', - }) - // TODO: find the best attribute name for the ignoring iframes - iframe.setAttribute('data-openreplay-obscured', '') - iframe.setAttribute('data-openreplay-hidden', '') - iframe.setAttribute('data-openreplay-ignore', '') - document.body.appendChild(iframe) + constructor(private readonly logError: (...args: any[]) => void) { + const iframe = (this.iframe = document.createElement('iframe')) + Object.assign(iframe.style, { + position: 'fixed', + zIndex: 2147483647 - 1, + border: 'none', + bottom: '10px', + right: '10px', + height: '200px', + width: '200px', + }) + // TODO: find the best attribute name for the ignoring iframes + iframe.setAttribute('data-openreplay-obscured', '') + iframe.setAttribute('data-openreplay-hidden', '') + iframe.setAttribute('data-openreplay-ignore', '') + document.body.appendChild(iframe) - const doc = iframe.contentDocument - if (!doc) { - console.error('OpenReplay: CallWindow iframe document is not reachable.') - return - } + const doc = iframe.contentDocument + if (!doc) { + console.error('OpenReplay: CallWindow iframe document is not reachable.') + return + } + //const baseHref = "https://static.openreplay.com/tracker-assist/test" + const baseHref = 'https://static.openreplay.com/tracker-assist/4.0.0' + this.load = fetch(baseHref + '/index.html') + .then((r) => r.text()) + .then((text) => { + iframe.onload = () => { + const assistSection = doc.getElementById('or-assist') + assistSection?.classList.remove('status-connecting') + //iframe.style.height = doc.body.scrollHeight + 'px'; + //iframe.style.width = doc.body.scrollWidth + 'px'; + this.adjustIframeSize() + iframe.onload = null + } - //const baseHref = "https://static.openreplay.com/tracker-assist/test" - const baseHref = 'https://static.openreplay.com/tracker-assist/4.0.0' - this.load = fetch(baseHref + '/index.html') - .then(r => r.text()) - .then((text) => { - iframe.onload = () => { - const assistSection = doc.getElementById('or-assist') - assistSection?.classList.remove('status-connecting') - //iframe.style.height = doc.body.scrollHeight + 'px'; - //iframe.style.width = doc.body.scrollWidth + 'px'; - this.adjustIframeSize() - iframe.onload = null - } + // ? + text = text.replace(/href="css/g, `href="${baseHref}/css`) + doc.open() + doc.write(text) + doc.close() - // ? - text = text.replace(/href="css/g, `href="${baseHref}/css`) - doc.open() - doc.write(text) - doc.close() + this.vLocal = doc.getElementById('video-local') as HTMLVideoElement | null + this.vRemote = doc.getElementById('video-remote') as HTMLVideoElement | null + this.videoContainer = doc.getElementById('video-container') + this.audioBtn = doc.getElementById('audio-btn') + if (this.audioBtn) { + this.audioBtn.onclick = () => this.toggleAudio() + } + this.videoBtn = doc.getElementById('video-btn') + if (this.videoBtn) { + this.videoBtn.onclick = () => this.toggleVideo() + } + this.endCallBtn = doc.getElementById('end-call-btn') - this.vLocal = doc.getElementById('video-local') as (HTMLVideoElement | null) - this.vRemote = doc.getElementById('video-remote') as (HTMLVideoElement | null) - this.videoContainer = doc.getElementById('video-container') + this.agentNameElem = doc.getElementById('agent-name') + this.vPlaceholder = doc.querySelector('#remote-stream p') - this.audioBtn = doc.getElementById('audio-btn') - if (this.audioBtn) { - this.audioBtn.onclick = () => this.toggleAudio() - } - this.videoBtn = doc.getElementById('video-btn') - if (this.videoBtn) { - this.videoBtn.onclick = () => this.toggleVideo() - } - this.endCallBtn = doc.getElementById('end-call-btn') + this.remoteControlContainer = doc.getElementById('remote-control-row') + this.remoteControlEndBtn = doc.getElementById('end-control-btn') + this.controlsContainer = doc.getElementById('controls') - this.agentNameElem = doc.getElementById('agent-name') - this.vPlaceholder = doc.querySelector('#remote-stream p') + const tsElem = doc.getElementById('duration') + if (tsElem) { + const startTs = + Number(sessionStorage.getItem(SS_START_TS_KEY)) || Date.now() + sessionStorage.setItem(SS_START_TS_KEY, startTs.toString()) + this.tsInterval = setInterval(() => { + const ellapsed = Date.now() - startTs + const secsFull = ~~(ellapsed / 1000) + const mins = ~~(secsFull / 60) + const secs = secsFull - mins * 60 + tsElem.innerText = `${mins}:${secs < 10 ? 0 : ''}${secs}` + }, 500) + } - this.remoteControlContainer = doc.getElementById('remote-control-row') - this.remoteControlEndBtn = doc.getElementById('end-control-btn') + const dragArea = doc.querySelector('.drag-area') + if (dragArea) { + // TODO: save coordinates on the new page + attachDND(iframe, dragArea, doc.documentElement) + } + }) - const tsElem = doc.getElementById('duration') - if (tsElem) { - const startTs = Number(sessionStorage.getItem(SS_START_TS_KEY)) || Date.now() - sessionStorage.setItem(SS_START_TS_KEY, startTs.toString()) - this.tsInterval = setInterval(() => { - const ellapsed = Date.now() - startTs - const secsFull = ~~(ellapsed / 1000) - const mins = ~~(secsFull / 60) - const secs = secsFull - mins * 60 - tsElem.innerText = `${mins}:${secs < 10 ? 0 : ''}${secs}` - }, 500) - } + //this.toggleVideoUI(false) + //this.toggleRemoteVideoUI(false) + } - const dragArea = doc.querySelector('.drag-area') - if (dragArea) { - // TODO: save coordinates on the new page - attachDND(iframe, dragArea, doc.documentElement) - } - }) + private adjustIframeSize() { + const doc = this.iframe.contentDocument + if (!doc) { + return + } + this.iframe.style.height = `${doc.body.scrollHeight}px` + this.iframe.style.width = `${doc.body.scrollWidth}px` + } - //this.toggleVideoUI(false) - //this.toggleRemoteVideoUI(false) - } + setCallEndAction(endCall: () => void) { + this.load + .then(() => { + if (this.endCallBtn) { + this.endCallBtn.onclick = endCall + } + }) + .catch((e) => this.logError(e)) + } - private adjustIframeSize() { - const doc = this.iframe.contentDocument - if (!doc) { return } - this.iframe.style.height = `${doc.body.scrollHeight}px` - this.iframe.style.width = `${doc.body.scrollWidth}px` - } + setRemoteControlEnd(endControl: () => void) { + this.load + .then(() => { + if (this.remoteControlEndBtn) { + this.remoteControlEndBtn.onclick = endControl + } + }) + .catch((e) => this.logError(e)) + } - setCallEndAction(endCall: () => void) { - this.load.then(() => { - if (this.endCallBtn) { - this.endCallBtn.onclick = endCall - } - }).catch(e => this.logError(e)) - } + private checkRemoteVideoInterval: ReturnType + private audioContainer: HTMLDivElement | null = null + addRemoteStream(rStream: MediaStream) { + this.load + .then(() => { + // Video + if (this.vRemote && !this.vRemote.srcObject) { + this.vRemote.srcObject = rStream + if (this.vPlaceholder) { + this.vPlaceholder.innerText = + 'Video has been paused. Click anywhere to resume.' + } + // Hack to determine if the remote video is enabled + // TODO: pass this info through socket + if (this.checkRemoteVideoInterval) { + clearInterval(this.checkRemoteVideoInterval) + } // just in case + let enabled = false + this.checkRemoteVideoInterval = setInterval(() => { + const settings = rStream.getVideoTracks()[0]?.getSettings() + const isDummyVideoTrack = + !!settings && (settings.width === 2 || settings.frameRate === 0) + const shouldBeEnabled = !isDummyVideoTrack + if (enabled !== shouldBeEnabled) { + this.toggleRemoteVideoUI((enabled = shouldBeEnabled)) + } + }, 1000) + } - setRemoteControlEnd(endControl: () => void) { - this.load.then(() => { - if (this.remoteControlEndBtn) { - this.remoteControlEndBtn.onclick = endControl - } - }).catch(e => this.logError(e)) - } + // Audio + if (!this.audioContainer) { + this.audioContainer = document.createElement('div') + document.body.appendChild(this.audioContainer) + } + // Hack for audio. Doesen't work inside the iframe + // because of some magical reasons (check if it is connected to autoplay?) + const audioEl = document.createElement('audio') + audioEl.autoplay = true + audioEl.style.display = 'none' + audioEl.srcObject = rStream + this.audioContainer.appendChild(audioEl) + }) + .catch((e) => this.logError(e)) + } - private checkRemoteVideoInterval: ReturnType - private audioContainer: HTMLDivElement | null = null - addRemoteStream(rStream: MediaStream) { - this.load.then(() => { - // Video - if (this.vRemote && !this.vRemote.srcObject) { - this.vRemote.srcObject = rStream - if (this.vPlaceholder) { - this.vPlaceholder.innerText = 'Video has been paused. Click anywhere to resume.' - } - // Hack to determine if the remote video is enabled - // TODO: pass this info through socket - if (this.checkRemoteVideoInterval) { clearInterval(this.checkRemoteVideoInterval) } // just in case - let enabled = false - this.checkRemoteVideoInterval = setInterval(() => { - const settings = rStream.getVideoTracks()[0]?.getSettings() - const isDummyVideoTrack = !!settings && (settings.width === 2 || settings.frameRate === 0) - const shouldBeEnabled = !isDummyVideoTrack - if (enabled !== shouldBeEnabled) { - this.toggleRemoteVideoUI(enabled=shouldBeEnabled) - } - }, 1000) - } + toggleRemoteVideoUI(enable: boolean) { + this.load + .then(() => { + if (this.videoContainer) { + if (enable) { + this.videoContainer.classList.add('remote') + } else { + this.videoContainer.classList.remove('remote') + } + this.adjustIframeSize() + } + }) + .catch((e) => this.logError(e)) + } - // Audio - if (!this.audioContainer) { - this.audioContainer = document.createElement('div') - document.body.appendChild(this.audioContainer) - } - // Hack for audio. Doesen't work inside the iframe - // because of some magical reasons (check if it is connected to autoplay?) - const audioEl = document.createElement('audio') - audioEl.autoplay = true - audioEl.style.display = 'none' - audioEl.srcObject = rStream - this.audioContainer.appendChild(audioEl) - }).catch(e => this.logError(e)) - } + private localStreams: LocalStream[] = [] + // !TODO: separate streams manipulation from ui + setLocalStreams(streams: LocalStream[]) { + this.localStreams = streams + } - toggleRemoteVideoUI(enable: boolean) { - this.load.then(() => { - if (this.videoContainer) { - if (enable) { - this.videoContainer.classList.add('remote') - } else { - this.videoContainer.classList.remove('remote') - } - this.adjustIframeSize() - } - }).catch(e => this.logError(e)) - } + playRemote() { + this.vRemote && this.vRemote.play() + } - private localStreams: LocalStream[] = [] - // !TODO: separate streams manipulation from ui - setLocalStreams(streams: LocalStream[]) { - this.localStreams = streams - } + setAssistentName(callingAgents: Map) { + this.load + .then(() => { + if (this.agentNameElem) { + const nameString = Array.from(callingAgents.values()).join(', ') + const safeNames = + nameString.length > 20 ? nameString.substring(0, 20) + '...' : nameString + this.agentNameElem.innerText = safeNames + } + }) + .catch((e) => this.logError(e)) + } - playRemote() { - this.vRemote && this.vRemote.play() - } + private toggleAudioUI(enabled: boolean) { + if (!this.audioBtn) { + return + } + if (enabled) { + this.audioBtn.classList.remove('muted') + } else { + this.audioBtn.classList.add('muted') + } + } - setAssistentName(callingAgents: Map) { - this.load.then(() => { - if (this.agentNameElem) { - const nameString = Array.from(callingAgents.values()).join(', ') - const safeNames = nameString.length > 20 ? nameString.substring(0, 20) + '...' : nameString - this.agentNameElem.innerText = safeNames - } - }).catch(e => this.logError(e)) - } + private toggleAudio() { + let enabled = false + this.localStreams.forEach((stream) => { + enabled = stream.toggleAudio() || false + }) + this.toggleAudioUI(enabled) + } + private toggleVideoUI(enabled: boolean) { + if (!this.videoBtn || !this.videoContainer) { + return + } + if (enabled) { + this.videoContainer.classList.add('local') + this.videoBtn.classList.remove('off') + } else { + this.videoContainer.classList.remove('local') + this.videoBtn.classList.add('off') + } + this.adjustIframeSize() + } - private toggleAudioUI(enabled: boolean) { - if (!this.audioBtn) { return } - if (enabled) { - this.audioBtn.classList.remove('muted') - } else { - this.audioBtn.classList.add('muted') - } - } + private toggleVideo() { + this.localStreams.forEach((stream) => { + stream + .toggleVideo() + .then((enabled) => { + this.toggleVideoUI(enabled) + this.load + .then(() => { + if (this.vLocal && stream && !this.vLocal.srcObject) { + this.vLocal.srcObject = stream.stream + } + }) + .catch((e) => this.logError(e)) + }) + .catch((e) => this.logError(e)) + }) + } - private toggleAudio() { - let enabled = false - this.localStreams.forEach(stream => { - enabled = stream.toggleAudio() || false - }) - this.toggleAudioUI(enabled) - } + public showRemoteControl() { + if (this.remoteControlContainer) { + this.remoteControlContainer.style.display = 'flex' + } + this.adjustIframeSize() + } - private toggleVideoUI(enabled: boolean) { - if (!this.videoBtn || !this.videoContainer) { return } - if (enabled) { - this.videoContainer.classList.add('local') - this.videoBtn.classList.remove('off') - } else { - this.videoContainer.classList.remove('local') - this.videoBtn.classList.add('off') - } - this.adjustIframeSize() - } + public showRemoteOnly() { + if (this.controlsContainer) { + this.controlsContainer.style.display = 'none' + } + this.showRemoteControl() + } - private toggleVideo() { - this.localStreams.forEach(stream => { - stream.toggleVideo() - .then(enabled => { - this.toggleVideoUI(enabled) - this.load.then(() => { - if (this.vLocal && stream && !this.vLocal.srcObject) { - this.vLocal.srcObject = stream.stream - } - }).catch(e => this.logError(e)) - }).catch(e => this.logError(e)) - }) - } + public showControls() { + if (this.controlsContainer) { + this.controlsContainer.style.display = 'unset' + } + this.adjustIframeSize() + } - public showRemoteControl() { - if (this.remoteControlContainer) { - this.remoteControlContainer.style.display = 'flex' - } - } + public hideControls() { + if (this.controlsContainer) { + this.controlsContainer.style.display = 'none' + } + this.adjustIframeSize() + } - public hideRemoteControl() { - if (this.remoteControlContainer) { - this.remoteControlContainer.style.display = 'none' - } - } - - remove() { - clearInterval(this.tsInterval) - clearInterval(this.checkRemoteVideoInterval) - if (this.audioContainer && this.audioContainer.parentElement) { - this.audioContainer.parentElement.removeChild(this.audioContainer) - this.audioContainer = null - } - if (this.iframe.parentElement) { - this.iframe.parentElement.removeChild(this.iframe) - } - sessionStorage.removeItem(SS_START_TS_KEY) - this.localStreams = [] - } + public hideRemoteControl() { + if (this.remoteControlContainer) { + this.remoteControlContainer.style.display = 'none' + } + this.adjustIframeSize() + } + remove() { + clearInterval(this.tsInterval) + clearInterval(this.checkRemoteVideoInterval) + if (this.audioContainer && this.audioContainer.parentElement) { + this.audioContainer.parentElement.removeChild(this.audioContainer) + this.audioContainer = null + } + if (this.iframe.parentElement) { + this.iframe.parentElement.removeChild(this.iframe) + } + sessionStorage.removeItem(SS_START_TS_KEY) + this.localStreams = [] + } } diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index 5138eeef2..d806fd227 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -24,7 +24,7 @@ export default class RemoteControl { constructor( private readonly options: AssistOptions, - private readonly onGrand: (string?) => string | undefined, + private readonly onGrand: (string?) => { agentName: string | undefined, callUI: any }, private readonly onRelease: (string?) => void) {} reconnect(ids: string[]) { @@ -71,21 +71,31 @@ export default class RemoteControl { this.agentID = id this.status = RCStatus.Enabled sessionStorage.setItem(this.options.session_control_peer_key, id) - const agentName = this.onGrand(id) + const { agentName, callUI, } = this.onGrand(id) + if (callUI) { + callUI?.setRemoteControlEnd(this.releaseControl) + } + if (this.mouse) { + this.resetMouse() + } this.mouse = new Mouse(agentName) this.mouse.mount() } releaseControl = () => { if (!this.agentID) { return } - this.mouse?.remove() - this.mouse = null + this.resetMouse() this.status = RCStatus.Disabled sessionStorage.removeItem(this.options.session_control_peer_key) this.onRelease(this.agentID) this.agentID = null } + resetMouse = () => { + this.mouse?.remove() + this.mouse = null + } + scroll = (id, d) => { id === this.agentID && this.mouse?.scroll(d) } move = (id, xy) => { id === this.agentID && this.mouse?.move(xy) } private focused: HTMLElement | null = null