feat(tracker-assist):3.5.6:annotation on call; remote typing; RemoteControl logic taken out

tracker/tracker-assist/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openreplay/tracker-assist",
   "description": "Tracker plugin for screen assistance through the WebRTC",
-  "version": "3.5.5",
+  "version": "3.5.6",
   "keywords": [
     "WebRTC",
     "assistance",

tracker/tracker-assist/src/AnnotationCanvas.ts

@@ -0,0 +1,80 @@
+export default class AnnotationCanvas {
+  private canvas: HTMLCanvasElement
+  private ctx: CanvasRenderingContext2D | null = null
+  private painting: boolean = false
+  constructor() {
+    this.canvas = document.createElement('canvas')
+    Object.assign(this.canvas.style, {
+      position: "fixed",
+      left: 0,
+      top: 0,
+      pointerEvents: "none",
+      zIndex: 2147483647 - 2,
+    })
+  }
+
+  private resizeCanvas = () => {
+    this.canvas.width = window.innerWidth
+    this.canvas.height = window.innerHeight
+  }
+
+  private lastPosition: [number, number] = [0,0]
+  start = (p: [number, number]) => {
+    this.painting = true
+    this.clrTmID && clearTimeout(this.clrTmID)
+    this.lastPosition = p
+  }
+
+  stop = () => {
+    this.painting = false
+    this.fadeOut()
+  }
+
+  move = (p: [number, number])  =>{
+    if (!this.ctx || !this.painting) { return }
+    this.ctx.globalAlpha = 1.0
+    this.ctx.beginPath()
+    this.ctx.moveTo(this.lastPosition[0], this.lastPosition[1])
+    this.ctx.lineTo(p[0], p[1])
+    this.ctx.lineWidth = 8
+    this.ctx.lineCap = "round"
+    this.ctx.lineJoin = "round"
+    this.ctx.strokeStyle = "red"
+    this.ctx.stroke()
+    this.lastPosition = p
+  }
+
+  clrTmID: ReturnType<typeof setTimeout> | null = null
+  private fadeOut() {
+    let timeoutID: ReturnType<typeof setTimeout>
+    const fadeStep = () => {
+      if (!this.ctx || this.painting ) { return }
+      this.ctx.globalCompositeOperation = 'destination-out'
+      this.ctx.fillStyle = "rgba(255, 255, 255, 0.1)"
+      this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height)
+      this.ctx.globalCompositeOperation = 'source-over'
+      timeoutID = setTimeout(fadeStep,100)
+    }
+    this.clrTmID = setTimeout(() => {
+      clearTimeout(timeoutID)
+      this.ctx &&
+        this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height)
+    }, 4000)
+    fadeStep()
+  }
+
+
+  mount() {
+    document.body.appendChild(this.canvas)
+    this.ctx = this.canvas.getContext("2d")
+    window.addEventListener("resize", this.resizeCanvas)
+    this.resizeCanvas()
+  }
+
+  remove() {
+    if (this.canvas.parentNode){
+      this.canvas.parentNode.removeChild(this.canvas)
+    }
+    window.removeEventListener("resize", this.resizeCanvas)
+  }
+}

tracker/tracker-assist/src/Assist.ts

@@ -5,8 +5,9 @@ import type { Properties } from 'csstype';
 import { App } from '@openreplay/tracker';
 
 import RequestLocalStream from './LocalStream.js';
-import Mouse from './Mouse.js';
+import RemoteControl from './RemoteControl.js';
 import CallWindow from './CallWindow.js';
+import AnnotationCanvas from './AnnotationCanvas.js';
 import ConfirmWindow, { callConfirmDefault, controlConfirmDefault } from './ConfirmWindow.js';
 import type { Options as ConfirmOptions } from './ConfirmWindow.js';
 
@@ -14,12 +15,12 @@ import type { Options as ConfirmOptions } from './ConfirmWindow.js';
 //@ts-ignore  peerjs hack for webpack5 (?!) TODO: ES/node modules;
 Peer = Peer.default || Peer;
 
-type BehinEndCallback = () => ((()=>{}) | void)
+type StartEndCallback = () => ((()=>{}) | void)
 
 export interface Options {
-  onAgentConnect: BehinEndCallback,
-  onCallStart: BehinEndCallback,
-  onRemoteControlStart: BehinEndCallback,
+  onAgentConnect: StartEndCallback,
+  onCallStart: StartEndCallback,
+  onRemoteControlStart: StartEndCallback,
   session_calling_peer_key: string,
   session_control_peer_key: string,
   callConfirm: ConfirmOptions,
@@ -39,8 +40,11 @@ enum CallingState {
 };
 
 
+// TODO typing????
+type OptionalCallback = (()=>{}) | void
 type Agent = {
-  onDisconnect: ((()=>{}) | void), // TODO: better types here
+  onDisconnect?: OptionalCallback,
+  onControlReleased?: OptionalCallback,
   name?: string
   //
 }
@@ -139,6 +143,34 @@ export default class Assist {
     })
     socket.onAny((...args) => app.debug.log("Socket:", ...args))
 
+
+    const remoteControl = new RemoteControl(
+      this.options,
+      id => {
+        this.agents[id].onControlReleased = this.options.onRemoteControlStart()
+        this.emit("control_granted", id)
+      },
+      id => {
+        const cb = this.agents[id].onControlReleased
+        delete this.agents[id].onControlReleased
+        typeof cb === "function" && cb()
+        this.emit("control_rejected", id)
+      },
+    )
+
+    // TODO: check incoming args
+    socket.on("request_control", remoteControl.requestControl)
+    socket.on("release_control", remoteControl.releaseControl)
+    socket.on("scroll", remoteControl.scroll)
+    socket.on("click", remoteControl.click)
+    socket.on("move", remoteControl.move)
+    socket.on("input", remoteControl.input)
+
+    let annot: AnnotationCanvas | null = null
+    socket.on("moveAnnotation", (_, p) => annot && annot.move(p)) // TODO: restrict by id
+    socket.on("startAnnotation", (_, p) => annot && annot.start(p))
+    socket.on("stopAnnotation", () => annot && annot.stop())
+
     socket.on("NEW_AGENT", (id: string, info) => {
       this.agents[id] = {
         onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(),
@@ -148,7 +180,7 @@ export default class Assist {
       this.app.stop();
       this.app.start().then(() => { this.assistDemandedRestart = false })
     })
-    socket.on("AGENTS_CONNECTED", (ids) => {
+    socket.on("AGENTS_CONNECTED", (ids: string[]) => {
       ids.forEach(id =>{
         this.agents[id] = {
           onDisconnect: this.options.onAgentConnect && this.options.onAgentConnect(),
@@ -157,61 +189,10 @@ export default class Assist {
       this.assistDemandedRestart = true
       this.app.stop();
       this.app.start().then(() => { this.assistDemandedRestart = false })
-      const storedControllingAgent = sessionStorage.getItem(this.options.session_control_peer_key)
-      if (storedControllingAgent !== null && ids.includes(storedControllingAgent)) {
-        grantControl(storedControllingAgent)
-        socket.emit("control_granted", storedControllingAgent)
-      } else {
-        sessionStorage.removeItem(this.options.session_control_peer_key)
-      }
+
+      remoteControl.reconnect(ids)
     })
 
-    let confirmRC: ConfirmWindow | null = null
-    const mouse = new Mouse()     // TODO: lazy init
-    let controllingAgent: string | null = null
-    const requestControl = (id: string) => {
-      if (controllingAgent !== null) { 
-        socket.emit("control_rejected", id)
-        return
-      }
-      controllingAgent = id // TODO: more explicit pending state
-      confirmRC = new ConfirmWindow(controlConfirmDefault(this.options.controlConfirm))
-      confirmRC.mount().then(allowed => {
-        if (allowed) {
-          grantControl(id)
-          socket.emit("control_granted", id)
-        } else {
-          releaseControl()
-          socket.emit("control_rejected", id)
-        }
-      }).catch()
-    }
-    let onRemoteControlStop: (()=>void) | null = null
-    const grantControl = (id: string) => {
-      controllingAgent = id
-      mouse.mount()
-      onRemoteControlStop = this.options.onRemoteControlStart() || null
-      sessionStorage.setItem(this.options.session_control_peer_key, id)
-    }
-    const releaseControl = () => {
-      typeof onRemoteControlStop === 'function' && onRemoteControlStop()
-      onRemoteControlStop = null
-      confirmRC?.remove()
-      mouse.remove()
-      controllingAgent = null
-      sessionStorage.removeItem(this.options.session_control_peer_key)
-    }
-    socket.on("request_control", requestControl)
-    socket.on("release_control", (id: string) => {
-      if (controllingAgent !== id) { return }
-      releaseControl()
-    })
-
-
-    socket.on("scroll", (id, d) => { id === controllingAgent && mouse.scroll(d) })
-    socket.on("click", (id, xy) => { id === controllingAgent && mouse.click(xy) })
-    socket.on("move", (id, xy) => { id === controllingAgent && mouse.move(xy) })
-
     let confirmCall:ConfirmWindow | null = null
 
     socket.on("AGENT_DISCONNECTED", (id) => {
@@ -219,7 +200,7 @@ export default class Assist {
       this.agents[id] && this.agents[id].onDisconnect != null && this.agents[id].onDisconnect()
       delete this.agents[id]
 
-      controllingAgent === id && releaseControl()
+      remoteControl.releaseControl(id)
 
       // close the call also
       if (callingAgent === id) {
@@ -281,11 +262,20 @@ export default class Assist {
           style: this.options.confirmStyle,
         }))
         confirmAnswer = confirmCall.mount()
+        this.playNotificationSound()
         this.onRemoteCallEnd = () => { // if call cancelled by a caller before confirmation
           app.debug.log("Received call_end during confirm window opened")
           confirmCall?.remove()
           setCallingState(CallingState.False)
+          call.close()
         }
+        setTimeout(() => {
+          if (this.callingState !== CallingState.Requesting) { return }
+          call.close()
+          confirmCall?.remove()
+          this.notifyCallEnd()
+          setCallingState(CallingState.False)
+        }, 30000)
       }
 
       confirmAnswer.then(agreed => {
@@ -296,13 +286,17 @@ export default class Assist {
           return
         }
 
-        let callUI = new CallWindow()
+        const callUI = new CallWindow()
+        annot = new AnnotationCanvas()
+        annot.mount()
         callUI.setAssistentName(agentName)
         
         const onCallEnd = this.options.onCallStart()
         const handleCallEnd = () => {
           call.close()
           callUI.remove()
+          annot && annot.remove()
+          annot = null
           setCallingState(CallingState.False)
           onCallEnd && onCallEnd()
         }
@@ -350,6 +344,16 @@ export default class Assist {
     });
   }
 
+  private playNotificationSound() {
+    if ('Audio' in window) {
+      new Audio("https://static.openreplay.com/tracker-assist/notification.mp3")
+      .play()
+      .catch(e => {
+        this.app.debug.warn(e)
+      })
+    }
+  }
+
   private clean() {
     if (this.peer) {
       this.peer.destroy()

tracker/tracker-assist/src/ConfirmWindow.ts

@@ -129,7 +129,7 @@ export default class ConfirmWindow {
         maxWidth: "fit-content",
         padding: "20px",
         background: "#F3F3F3",
-        opacity: ".75",
+        //opacity: ".75",
         color: "black",
         borderRadius: "3px",
         boxShadow: "0px 0px 3.99778px 1.99889px rgba(0, 0, 0, 0.1)"
@@ -138,6 +138,7 @@ export default class ConfirmWindow {
     );
 
     Object.assign(wrapper.style, {
+      position: "fixed",
       left: 0,
       top: 0,
       height: "100%",

tracker/tracker-assist/src/Mouse.ts

@@ -45,7 +45,9 @@ export default class Mouse {
     if (el instanceof HTMLElement) {
       el.click()
       el.focus()
+      return el
     }
+    return null
   }
 
   private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct

tracker/tracker-assist/src/RemoteControl.ts

@@ -0,0 +1,88 @@
+import Mouse from './Mouse.js';
+import ConfirmWindow, { controlConfirmDefault } from './ConfirmWindow.js';
+import type { Options as AssistOptions } from './Assist'
+
+enum RCStatus {
+  Disabled,
+  Requesting,
+  Enabled,
+}
+
+export default class RemoteControl {
+  private mouse: Mouse | null
+  private status: RCStatus = RCStatus.Disabled
+  private agentID: string | null = null
+
+  constructor(
+    private options: AssistOptions,
+    private onGrand: (sting?) => void, 
+    private onRelease: (sting?) => void) {}
+
+  reconnect(ids: string[]) {
+    const storedID = sessionStorage.getItem(this.options.session_control_peer_key)
+    if (storedID !== null && ids.includes(storedID)) {
+      this.grantControl(storedID)
+    } else {
+      sessionStorage.removeItem(this.options.session_control_peer_key)
+    }
+  }
+
+  private confirm: ConfirmWindow | null = null
+  requestControl = (id: string) => {
+    if (this.agentID !== null) {
+      this.releaseControl(id)
+      return 
+    }
+    setTimeout(() =>{
+      if (this.status === RCStatus.Requesting) {
+        this.releaseControl(id)
+      }
+    }, 30000)
+    this.agentID = id
+    this.status = RCStatus.Requesting
+    this.confirm = new ConfirmWindow(controlConfirmDefault(this.options.controlConfirm))
+    this.confirm.mount().then(allowed => {
+      if (allowed) {
+        this.grantControl(id)
+      } else {
+        this.releaseControl(id)
+      }
+    }).catch()
+  }
+  grantControl = (id: string) => {
+    this.agentID = id
+    this.status = RCStatus.Enabled
+    this.mouse = new Mouse()
+    this.mouse.mount()
+    sessionStorage.setItem(this.options.session_control_peer_key, id)
+    this.onGrand(id)
+  }
+
+  releaseControl = (id: string) => {
+    if (this.agentID !== id) { return }
+    this.confirm?.remove()
+    this.mouse?.remove()
+    this.mouse = null
+    this.status = RCStatus.Disabled
+    this.agentID = null
+    sessionStorage.removeItem(this.options.session_control_peer_key)
+    this.onRelease(id)
+  }
+
+  scroll = (id, d) => { id === this.agentID && this.mouse?.scroll(d) }
+  move = (id, xy) => { id === this.agentID && this.mouse?.move(xy) }
+  private focused: HTMLElement | null = null
+  click = (id, xy) => { 
+    if (id !== this.agentID || !this.mouse) { return }
+    this.focused = this.mouse.click(xy) 
+  }
+  input = (id, value) => {
+    if (id !== this.agentID || !this.mouse || !this.focused) { return }
+    if (this.focused instanceof HTMLTextAreaElement 
+      || this.focused instanceof HTMLInputElement) {
+      this.focused.value = value
+    } else if (this.focused.isContentEditable) {
+      this.focused.innerText = value
+    }
+  }
+}