From b7c66e32143ccf6108317cf3df95817f457aba11 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Wed, 23 Mar 2022 17:19:45 +0100 Subject: [PATCH] feat(tracker-assist):3.5.6:annotation on call; remote typing; RemoteControl logic taken out --- tracker/tracker-assist/package.json | 2 +- .../tracker-assist/src/AnnotationCanvas.ts | 80 +++++++++++ tracker/tracker-assist/src/Assist.ts | 128 +++++++++--------- tracker/tracker-assist/src/ConfirmWindow.ts | 3 +- tracker/tracker-assist/src/Mouse.ts | 2 + tracker/tracker-assist/src/RemoteControl.ts | 88 ++++++++++++ 6 files changed, 239 insertions(+), 64 deletions(-) create mode 100644 tracker/tracker-assist/src/AnnotationCanvas.ts create mode 100644 tracker/tracker-assist/src/RemoteControl.ts diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 30f85875b..0a9fda457 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "3.5.5", + "version": "3.5.6", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/AnnotationCanvas.ts b/tracker/tracker-assist/src/AnnotationCanvas.ts new file mode 100644 index 000000000..afda8e2a5 --- /dev/null +++ b/tracker/tracker-assist/src/AnnotationCanvas.ts @@ -0,0 +1,80 @@ +export default class AnnotationCanvas { + private canvas: HTMLCanvasElement + private ctx: CanvasRenderingContext2D | null = null + private painting: boolean = false + constructor() { + this.canvas = document.createElement('canvas') + Object.assign(this.canvas.style, { + position: "fixed", + left: 0, + top: 0, + pointerEvents: "none", + zIndex: 2147483647 - 2, + }) + } + + private resizeCanvas = () => { + this.canvas.width = window.innerWidth + this.canvas.height = window.innerHeight + } + + private lastPosition: [number, number] = [0,0] + start = (p: [number, number]) => { + this.painting = true + this.clrTmID && clearTimeout(this.clrTmID) + this.lastPosition = p + } + + stop = () => { + this.painting = false + this.fadeOut() + } + + move = (p: [number, number]) =>{ + if (!this.ctx || !this.painting) { return } + this.ctx.globalAlpha = 1.0 + this.ctx.beginPath() + this.ctx.moveTo(this.lastPosition[0], this.lastPosition[1]) + this.ctx.lineTo(p[0], p[1]) + this.ctx.lineWidth = 8 + this.ctx.lineCap = "round" + this.ctx.lineJoin = "round" + this.ctx.strokeStyle = "red" + this.ctx.stroke() + this.lastPosition = p + } + + clrTmID: ReturnType | null = null + private fadeOut() { + let timeoutID: ReturnType + const fadeStep = () => { + if (!this.ctx || this.painting ) { return } + this.ctx.globalCompositeOperation = 'destination-out' + this.ctx.fillStyle = "rgba(255, 255, 255, 0.1)" + this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height) + this.ctx.globalCompositeOperation = 'source-over' + timeoutID = setTimeout(fadeStep,100) + } + this.clrTmID = setTimeout(() => { + clearTimeout(timeoutID) + this.ctx && + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height) + }, 4000) + fadeStep() + } + + + mount() { + document.body.appendChild(this.canvas) + this.ctx = this.canvas.getContext("2d") + window.addEventListener("resize", this.resizeCanvas) + this.resizeCanvas() + } + + remove() { + if (this.canvas.parentNode){ + this.canvas.parentNode.removeChild(this.canvas) + } + window.removeEventListener("resize", this.resizeCanvas) + } +} \ No newline at end of file diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index b5272510b..f7569bf26 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -5,8 +5,9 @@ import type { Properties } from 'csstype'; import { App } from '@openreplay/tracker'; import RequestLocalStream from './LocalStream.js'; -import Mouse from './Mouse.js'; +import RemoteControl from './RemoteControl.js'; import CallWindow from './CallWindow.js'; +import AnnotationCanvas from './AnnotationCanvas.js'; import ConfirmWindow, { callConfirmDefault, controlConfirmDefault } from './ConfirmWindow.js'; import type { Options as ConfirmOptions } from './ConfirmWindow.js'; @@ -14,12 +15,12 @@ import type { Options as ConfirmOptions } from './ConfirmWindow.js'; //@ts-ignore peerjs hack for webpack5 (?!) TODO: ES/node modules; Peer = Peer.default || Peer; -type BehinEndCallback = () => ((()=>{}) | void) +type StartEndCallback = () => ((()=>{}) | void) export interface Options { - onAgentConnect: BehinEndCallback, - onCallStart: BehinEndCallback, - onRemoteControlStart: BehinEndCallback, + onAgentConnect: StartEndCallback, + onCallStart: StartEndCallback, + onRemoteControlStart: StartEndCallback, session_calling_peer_key: string, session_control_peer_key: string, callConfirm: ConfirmOptions, @@ -39,8 +40,11 @@ enum CallingState { }; +// TODO typing???? +type OptionalCallback = (()=>{}) | void type Agent = { - onDisconnect: ((()=>{}) | void), // TODO: better types here + onDisconnect?: OptionalCallback, + onControlReleased?: OptionalCallback, name?: string // } @@ -139,6 +143,34 @@ export default class Assist { }) socket.onAny((...args) => app.debug.log("Socket:", ...args)) + + const remoteControl = new RemoteControl( + this.options, + id => { + this.agents[id].onControlReleased = this.options.onRemoteControlStart() + this.emit("control_granted", id) + }, + id => { + const cb = this.agents[id].onControlReleased + delete this.agents[id].onControlReleased + typeof cb === "function" && cb() + this.emit("control_rejected", id) + }, + ) + + // TODO: check incoming args + socket.on("request_control", remoteControl.requestControl) + socket.on("release_control", remoteControl.releaseControl) + socket.on("scroll", remoteControl.scroll) + socket.on("click", remoteControl.click) + socket.on("move", remoteControl.move) + socket.on("input", remoteControl.input) + + let annot: AnnotationCanvas | null = null + socket.on("moveAnnotation", (_, p) => annot && annot.move(p)) // TODO: restrict by id + socket.on("startAnnotation", (_, p) => annot && annot.start(p)) + socket.on("stopAnnotation", () => annot && annot.stop()) + socket.on("NEW_AGENT", (id: string, info) => { this.agents[id] = { onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(), @@ -148,7 +180,7 @@ export default class Assist { this.app.stop(); this.app.start().then(() => { this.assistDemandedRestart = false }) }) - socket.on("AGENTS_CONNECTED", (ids) => { + socket.on("AGENTS_CONNECTED", (ids: string[]) => { ids.forEach(id =>{ this.agents[id] = { onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(), @@ -157,61 +189,10 @@ export default class Assist { this.assistDemandedRestart = true this.app.stop(); this.app.start().then(() => { this.assistDemandedRestart = false }) - const storedControllingAgent = sessionStorage.getItem(this.options.session_control_peer_key) - if (storedControllingAgent !== null && ids.includes(storedControllingAgent)) { - grantControl(storedControllingAgent) - socket.emit("control_granted", storedControllingAgent) - } else { - sessionStorage.removeItem(this.options.session_control_peer_key) - } + + remoteControl.reconnect(ids) }) - let confirmRC: ConfirmWindow | null = null - const mouse = new Mouse() // TODO: lazy init - let controllingAgent: string | null = null - const requestControl = (id: string) => { - if (controllingAgent !== null) { - socket.emit("control_rejected", id) - return - } - controllingAgent = id // TODO: more explicit pending state - confirmRC = new ConfirmWindow(controlConfirmDefault(this.options.controlConfirm)) - confirmRC.mount().then(allowed => { - if (allowed) { - grantControl(id) - socket.emit("control_granted", id) - } else { - releaseControl() - socket.emit("control_rejected", id) - } - }).catch() - } - let onRemoteControlStop: (()=>void) | null = null - const grantControl = (id: string) => { - controllingAgent = id - mouse.mount() - onRemoteControlStop = this.options.onRemoteControlStart() || null - sessionStorage.setItem(this.options.session_control_peer_key, id) - } - const releaseControl = () => { - typeof onRemoteControlStop === 'function' && onRemoteControlStop() - onRemoteControlStop = null - confirmRC?.remove() - mouse.remove() - controllingAgent = null - sessionStorage.removeItem(this.options.session_control_peer_key) - } - socket.on("request_control", requestControl) - socket.on("release_control", (id: string) => { - if (controllingAgent !== id) { return } - releaseControl() - }) - - - socket.on("scroll", (id, d) => { id === controllingAgent && mouse.scroll(d) }) - socket.on("click", (id, xy) => { id === controllingAgent && mouse.click(xy) }) - socket.on("move", (id, xy) => { id === controllingAgent && mouse.move(xy) }) - let confirmCall:ConfirmWindow | null = null socket.on("AGENT_DISCONNECTED", (id) => { @@ -219,7 +200,7 @@ export default class Assist { this.agents[id] && this.agents[id].onDisconnect != null && this.agents[id].onDisconnect() delete this.agents[id] - controllingAgent === id && releaseControl() + remoteControl.releaseControl(id) // close the call also if (callingAgent === id) { @@ -281,11 +262,20 @@ export default class Assist { style: this.options.confirmStyle, })) confirmAnswer = confirmCall.mount() + this.playNotificationSound() this.onRemoteCallEnd = () => { // if call cancelled by a caller before confirmation app.debug.log("Received call_end during confirm window opened") confirmCall?.remove() setCallingState(CallingState.False) + call.close() } + setTimeout(() => { + if (this.callingState !== CallingState.Requesting) { return } + call.close() + confirmCall?.remove() + this.notifyCallEnd() + setCallingState(CallingState.False) + }, 30000) } confirmAnswer.then(agreed => { @@ -296,13 +286,17 @@ export default class Assist { return } - let callUI = new CallWindow() + const callUI = new CallWindow() + annot = new AnnotationCanvas() + annot.mount() callUI.setAssistentName(agentName) const onCallEnd = this.options.onCallStart() const handleCallEnd = () => { call.close() callUI.remove() + annot && annot.remove() + annot = null setCallingState(CallingState.False) onCallEnd && onCallEnd() } @@ -350,6 +344,16 @@ export default class Assist { }); } + private playNotificationSound() { + if ('Audio' in window) { + new Audio("https://static.openreplay.com/tracker-assist/notification.mp3") + .play() + .catch(e => { + this.app.debug.warn(e) + }) + } + } + private clean() { if (this.peer) { this.peer.destroy() diff --git a/tracker/tracker-assist/src/ConfirmWindow.ts b/tracker/tracker-assist/src/ConfirmWindow.ts index 1b426426d..6cfabbca9 100644 --- a/tracker/tracker-assist/src/ConfirmWindow.ts +++ b/tracker/tracker-assist/src/ConfirmWindow.ts @@ -129,7 +129,7 @@ export default class ConfirmWindow { maxWidth: "fit-content", padding: "20px", background: "#F3F3F3", - opacity: ".75", + //opacity: ".75", color: "black", borderRadius: "3px", boxShadow: "0px 0px 3.99778px 1.99889px rgba(0, 0, 0, 0.1)" @@ -138,6 +138,7 @@ export default class ConfirmWindow { ); Object.assign(wrapper.style, { + position: "fixed", left: 0, top: 0, height: "100%", diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index b02337c7f..911c29236 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -45,7 +45,9 @@ export default class Mouse { if (el instanceof HTMLElement) { el.click() el.focus() + return el } + return null } private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts new file mode 100644 index 000000000..a32f81035 --- /dev/null +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -0,0 +1,88 @@ +import Mouse from './Mouse.js'; +import ConfirmWindow, { controlConfirmDefault } from './ConfirmWindow.js'; +import type { Options as AssistOptions } from './Assist' + +enum RCStatus { + Disabled, + Requesting, + Enabled, +} + +export default class RemoteControl { + private mouse: Mouse | null + private status: RCStatus = RCStatus.Disabled + private agentID: string | null = null + + constructor( + private options: AssistOptions, + private onGrand: (sting?) => void, + private onRelease: (sting?) => void) {} + + reconnect(ids: string[]) { + const storedID = sessionStorage.getItem(this.options.session_control_peer_key) + if (storedID !== null && ids.includes(storedID)) { + this.grantControl(storedID) + } else { + sessionStorage.removeItem(this.options.session_control_peer_key) + } + } + + private confirm: ConfirmWindow | null = null + requestControl = (id: string) => { + if (this.agentID !== null) { + this.releaseControl(id) + return + } + setTimeout(() =>{ + if (this.status === RCStatus.Requesting) { + this.releaseControl(id) + } + }, 30000) + this.agentID = id + this.status = RCStatus.Requesting + this.confirm = new ConfirmWindow(controlConfirmDefault(this.options.controlConfirm)) + this.confirm.mount().then(allowed => { + if (allowed) { + this.grantControl(id) + } else { + this.releaseControl(id) + } + }).catch() + } + grantControl = (id: string) => { + this.agentID = id + this.status = RCStatus.Enabled + this.mouse = new Mouse() + this.mouse.mount() + sessionStorage.setItem(this.options.session_control_peer_key, id) + this.onGrand(id) + } + + releaseControl = (id: string) => { + if (this.agentID !== id) { return } + this.confirm?.remove() + this.mouse?.remove() + this.mouse = null + this.status = RCStatus.Disabled + this.agentID = null + sessionStorage.removeItem(this.options.session_control_peer_key) + this.onRelease(id) + } + + scroll = (id, d) => { id === this.agentID && this.mouse?.scroll(d) } + move = (id, xy) => { id === this.agentID && this.mouse?.move(xy) } + private focused: HTMLElement | null = null + click = (id, xy) => { + if (id !== this.agentID || !this.mouse) { return } + this.focused = this.mouse.click(xy) + } + input = (id, value) => { + if (id !== this.agentID || !this.mouse || !this.focused) { return } + if (this.focused instanceof HTMLTextAreaElement + || this.focused instanceof HTMLInputElement) { + this.focused.value = value + } else if (this.focused.isContentEditable) { + this.focused.innerText = value + } + } +} \ No newline at end of file