feat(frontend-assist): annotations & iremote typing

frontend/.gitignore

@@ -8,3 +8,4 @@ app/components/ui/SVG.js
 *.DS_Store
 .env
 *css.d.ts
+*.cache

frontend/app/player/MessageDistributor/StatedScreen/Screen/BaseScreen.ts

@@ -70,10 +70,10 @@ export default abstract class BaseScreen {
 
   private boundingRect: DOMRect | null  = null;
   private getBoundingClientRect(): DOMRect {
-    //if (this.boundingRect === null) {
-      return this.boundingRect = this.overlay.getBoundingClientRect(); // expensive operation?
-    //}
-    //return this.boundingRect;
+     if (this.boundingRect === null) {
+      return this.boundingRect = this.overlay.getBoundingClientRect() // expensive operation?
+    }
+    return this.boundingRect
   }
 
   getInternalViewportCoordinates({ x, y }: Point): Point {
@@ -85,17 +85,22 @@ export default abstract class BaseScreen {
     const screenX = (x - overlayX) * scale;
     const screenY = (y - overlayY) * scale;
 
-    return { x: screenX, y: screenY };
+    return { x: Math.round(screenX), y: Math.round(screenY) };
+  }
+
+  getCurrentScroll(): Point {
+    const docEl = this.document?.documentElement
+    const x = docEl ? docEl.scrollLeft : 0
+    const y = docEl ? docEl.scrollTop : 0
+    return { x, y }
   }
 
   getInternalCoordinates(p: Point): Point {
     const { x, y } = this.getInternalViewportCoordinates(p);
 
-    const docEl = this.document?.documentElement
-    const scrollX = docEl ? docEl.scrollLeft : 0
-    const scrollY = docEl ? docEl.scrollTop : 0
+    const sc = this.getCurrentScroll()
 
-    return { x: x+scrollX, y: y+scrollY };
+    return { x: x+sc.x, y: y+sc.y };
   }
 
   getElementFromInternalPoint({ x, y }: Point): Element | null {

frontend/app/player/MessageDistributor/StatedScreen/Screen/screen.css

@@ -12,6 +12,7 @@
   background: white;
 }
 .overlay {
+  user-select: none;
   position: absolute;
   top: 0;
   left: 0;

frontend/app/player/MessageDistributor/managers/AnnotationCanvas.ts

@@ -0,0 +1,84 @@
+export default class AnnotationCanvas {
+  readonly canvas: HTMLCanvasElement
+  private ctx: CanvasRenderingContext2D | null = null
+  private painting: boolean = false
+  constructor() {
+    this.canvas = document.createElement('canvas')
+    Object.assign(this.canvas.style, {
+      position: "fixed",
+      cursor: "url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABMAAAATCAYAAAByUDbMAAAAAXNSR0IArs4c6QAAAWNJREFUOE+l1D1Lw1AUBuD35Catg5NzaCMRMilINnGok7sguLg4OlRcBTd/hqBVB0ed7KDgIPgXhJoaG10Kgk4a83EkhcYYktimd703z31zzuESSqwGIDs1bRvAIiRcWrZ9ETFUwhJ6XTsDsPH7Le1bz08H42JkGMa09+W2CVhKBmHC7jhYlOgUTPdUEa3Q86+SIDN/j4olf43BtJMFjoJl1AgMUJMUcRInZHT+w7KgYakGoDxVafmue0hBsJeLmaapvPffziFhraDjDMKWZdvHRaNRlCi2mUNHYl55dBwrDysFZWGloTQ2EZTEJoZiTFXVmaos34Ixn9e5qNgCaHR6vW7emcFozNVmN1ERbfb9myww3bVCTK9rPsDrpCh37HnXAC3Ek5lqf9ErM0im1zUG8BmGtCqq4mEIjppoeEESA5g/JIkaLMuv7AVHEgfNohqlU/7Fol3mPodiufvS7Yz7cP4ARjbPWyYPZSMAAAAASUVORK5CYII=') 0 20, crosshair",
+      left: 0,
+      top: 0,
+      //zIndex: 2147483647 - 2,
+    })
+  }
+
+  isPainting() {
+    return this.painting
+  }
+
+  private resizeCanvas = () => {
+    if (!this.canvas.parentElement) { return }
+    this.canvas.width = this.canvas.parentElement.offsetWidth
+    this.canvas.height = this.canvas.parentElement.offsetHeight
+  }
+
+  private lastPosition: [number, number] = [0,0]
+  start = (p: [number, number]) => {
+    this.painting = true
+    this.clrTmID && clearTimeout(this.clrTmID)
+    this.lastPosition = p
+  }
+
+  stop = () => {
+    this.painting = false
+    this.fadeOut()
+  }
+
+  move = (p: [number, number])  =>{
+    if (!this.ctx || !this.painting) { return }
+    this.ctx.globalAlpha = 1.0
+    this.ctx.beginPath()
+    this.ctx.moveTo(this.lastPosition[0], this.lastPosition[1])
+    this.ctx.lineTo(p[0], p[1])
+    this.ctx.lineWidth = 8
+    this.ctx.lineCap = "round"
+    this.ctx.lineJoin = "round"
+    this.ctx.strokeStyle = "red"
+    this.ctx.stroke()
+    this.lastPosition = p
+  }
+
+  clrTmID: ReturnType<typeof setTimeout> | null = null
+  private fadeOut() {
+    let timeoutID: ReturnType<typeof setTimeout>
+    const fadeStep = () => {
+      if (!this.ctx || this.painting ) { return }
+      this.ctx.globalCompositeOperation = 'destination-out'
+      this.ctx.fillStyle = "rgba(255, 255, 255, 0.1)"
+      this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height)
+      this.ctx.globalCompositeOperation = 'source-over'
+      timeoutID = setTimeout(fadeStep,100)
+    }
+    this.clrTmID = setTimeout(() => {
+      clearTimeout(timeoutID)
+      this.ctx &&
+        this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height)
+    }, 3700)
+    fadeStep()
+  }
+
+  mount(parent: HTMLElement) {
+    parent.appendChild(this.canvas)
+    this.ctx = this.canvas.getContext("2d")
+    window.addEventListener("resize", this.resizeCanvas)
+    this.resizeCanvas()
+  }
+
+  remove() {
+    if (this.canvas.parentNode){
+      this.canvas.parentNode.removeChild(this.canvas)
+    }
+    window.removeEventListener("resize", this.resizeCanvas)
+  }
+}

frontend/app/player/MessageDistributor/managers/AssistManager.ts

@@ -7,8 +7,8 @@ import store from 'App/store';
 import type { LocalStream } from './LocalStream';
 import { update, getState } from '../../store';
 import { iceServerConfigFromString } from 'App/utils'
-
-import MStreamReader from '../messages/MStreamReader';;
+import AnnotationCanvas from './AnnotationCanvas';
+import MStreamReader from '../messages/MStreamReader';
 import JSONRawMessageReader from '../messages/JSONRawMessageReader'
 
 export enum CallingState {
@@ -136,12 +136,13 @@ export default class AssistManager {
       //socket.onAny((...args) => console.log(...args))
       socket.on("connect", () => {
         waitingForMessages = true
-        this.setStatus(ConnectionStatus.WaitingMessages)
+        this.setStatus(ConnectionStatus.WaitingMessages) // TODO: happens frequently on bad network
       })
       socket.on("disconnect", () => {
         this.toggleRemoteControl(false)
       })
       socket.on('messages', messages => {
+        //console.log(messages.filter(m => m._id === 41 || m._id === 44))
         showDisconnectTimeout && clearTimeout(showDisconnectTimeout);
         jmr.append(messages) // as RawMessage[]
 
@@ -173,9 +174,8 @@ export default class AssistManager {
           this.setStatus(ConnectionStatus.Disconnected)
         }, 30000)
 
-        if (getState().remoteControl === RemoteControlStatus.Requesting ||
-        getState().remoteControl === RemoteControlStatus.Enabled) {
-          this.toggleRemoteControl(false)
+        if (getState().remoteControl === RemoteControlStatus.Requesting) {
+          this.toggleRemoteControl(false) // else its remaining
         }
 
         // Call State
@@ -200,7 +200,7 @@ export default class AssistManager {
   private onMouseMove = (e: MouseEvent): void => {
     if (!this.socket) { return }
     const data = this.md.getInternalCoordinates(e)
-    this.socket.emit("move", [ Math.round(data.x), Math.round(data.y) ])
+    this.socket.emit("move", [ data.x, data.y ])
   }
 
   private onWheel = (e: WheelEvent): void => {
@@ -213,15 +213,23 @@ export default class AssistManager {
 
   private onMouseClick = (e: MouseEvent): void => {
     if (!this.socket) { return; }
-    const data = this.md.getInternalViewportCoordinates(e);
+    const data = this.md.getInternalViewportCoordinates(e)
     // const el = this.md.getElementFromPoint(e); // requires requestiong node_id from domManager
     const el = this.md.getElementFromInternalPoint(data)
     if (el instanceof HTMLElement) {
       el.focus()
-      el.oninput = e => e.preventDefault();
-      el.onkeydown = e => e.preventDefault();
+      el.oninput = e => {
+        if (el instanceof HTMLTextAreaElement 
+          || el instanceof HTMLInputElement
+        ) {
+          this.socket && this.socket.emit("input", el.value)
+        } else if (el.isContentEditable) {
+          this.socket && this.socket.emit("input", el.innerText)
+        }
+      }
+      //el.onkeydown = e => e.preventDefault()
     }
-    this.socket.emit("click",  [ Math.round(data.x), Math.round(data.y) ]);
+    this.socket.emit("click",  [ data.x, data.y ]);
   }
 
   private toggleRemoteControl(newState: boolean){
@@ -310,6 +318,8 @@ export default class AssistManager {
     this.callConnection && this.callConnection.close()
     update({ calling: CallingState.NoCall })
     this.callArgs = null
+    this.annot?.remove()
+    this.annot = null
   }
 
   private initiateCallEnd = () => {
@@ -355,6 +365,8 @@ export default class AssistManager {
     }
   }
 
+  private annot: AnnotationCanvas | null = null
+
   private _call() {
     if (![CallingState.NoCall, CallingState.Reconnecting].includes(getState().calling)) { return }
     update({ calling: CallingState.Connecting })
@@ -379,6 +391,34 @@ export default class AssistManager {
       call.on('stream', stream => {
         update({ calling: CallingState.OnCall })
         this.callArgs && this.callArgs.onStream(stream)
+
+        if (!this.annot) {
+          const annot = this.annot = new AnnotationCanvas()
+          annot.mount(this.md.overlay)
+          annot.canvas.addEventListener("mousedown", e => {
+            if (!this.socket) { return }
+            const data = this.md.getInternalViewportCoordinates(e)
+            annot.start([ data.x, data.y ])
+            this.socket.emit("startAnnotation", [ data.x, data.y ])
+          })
+          annot.canvas.addEventListener("mouseleave", () => {
+            if (!this.socket) { return }
+            annot.stop()
+            this.socket.emit("stopAnnotation")
+          })
+          annot.canvas.addEventListener("mouseup", () => {
+            if (!this.socket) { return }
+            annot.stop()
+            this.socket.emit("stopAnnotation")
+          })
+          annot.canvas.addEventListener("mousemove", e => {
+            if (!this.socket || !annot.isPainting()) { return }
+
+            const data = this.md.getInternalViewportCoordinates(e)
+            annot.move([ data.x, data.y ])
+            this.socket.emit("moveAnnotation", [ data.x, data.y ])
+          })
+        }
       });
       //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track))
 
@@ -409,6 +449,10 @@ export default class AssistManager {
       this.socket.close()
       document.removeEventListener('visibilitychange', this.onVisChange)
     }
+    if (this.annot) {
+      this.annot.remove()
+      this.annot = null
+    }
   }
 }