Patch assist (#3296)

* add global method support

* fix errors

* remove wrong updates

* remove wrong updates

* add onDrag as option

* fix wrong updates

frontend/app/player/web/assist/RemoteControl.ts

@@ -17,6 +17,9 @@ export interface State {
 
 export default class RemoteControl {
   private assistVersion = 1;
+  private isDragging = false;
+  private dragStart: any | null = null;
+  private readonly dragThreshold = 3;
 
   static readonly INITIAL_STATE: Readonly<State> = {
     remoteControl: RemoteControlStatus.Disabled,
@@ -81,6 +84,7 @@ export default class RemoteControl {
   }
 
   private onMouseMove = (e: MouseEvent): void => {
+    if (this.isDragging) return;
     const data = this.screen.getInternalCoordinates(e);
     this.emitData('move', [data.x, data.y]);
   };
@@ -154,16 +158,61 @@ export default class RemoteControl {
     this.emitData('click', [data.x, data.y]);
   };
 
+  private onMouseDown = (e: MouseEvent): void => {
+    if (this.store.get().annotating) return;
+
+    const { x, y } = this.screen.getInternalViewportCoordinates(e);
+    this.dragStart = [x, y];
+    this.isDragging = false;
+
+    const handleMove = (moveEvent: MouseEvent) => {
+      const { x: mx, y: my } =
+        this.screen.getInternalViewportCoordinates(moveEvent);
+      const [sx, sy] = this.dragStart!;
+      const dx = Math.abs(mx - sx);
+      const dy = Math.abs(my - sy);
+
+      if (
+        !this.isDragging &&
+        (dx > this.dragThreshold || dy > this.dragThreshold)
+      ) {
+        this.emitData('startDrag', [sx, sy]);
+        this.isDragging = true;
+      }
+
+      if (this.isDragging) {
+        this.emitData('drag', [mx, my, mx - sx, my - sy]);
+      }
+    };
+
+    const handleUp = () => {
+      if (this.isDragging) {
+        this.emitData('stopDrag');
+      }
+
+      this.dragStart = null;
+      this.isDragging = false;
+
+      window.removeEventListener('mousemove', handleMove);
+      window.removeEventListener('mouseup', handleUp);
+    };
+
+    window.addEventListener('mousemove', handleMove);
+    window.addEventListener('mouseup', handleUp);
+  };
+
   private toggleRemoteControl(enable: boolean) {
     if (enable) {
       this.screen.overlay.addEventListener('mousemove', this.onMouseMove);
       this.screen.overlay.addEventListener('click', this.onMouseClick);
       this.screen.overlay.addEventListener('wheel', this.onWheel);
+      this.screen.overlay.addEventListener('mousedown', this.onMouseDown);
       this.store.update({ remoteControl: RemoteControlStatus.Enabled });
     } else {
       this.screen.overlay.removeEventListener('mousemove', this.onMouseMove);
       this.screen.overlay.removeEventListener('click', this.onMouseClick);
       this.screen.overlay.removeEventListener('wheel', this.onWheel);
+      this.screen.overlay.removeEventListener('mousedown', this.onMouseDown);
       this.store.update({ remoteControl: RemoteControlStatus.Disabled });
       this.toggleAnnotation(false);
     }

tracker/tracker-assist/src/Assist.ts

@@ -37,6 +37,7 @@ export interface Options {
   onCallDeny?: () => any;
   onRemoteControlDeny?: (agentInfo: Record<string, any>) => any;
   onRecordingDeny?: (agentInfo: Record<string, any>) => any;
+  onDragCamera?: (dx: number, dy: number) => void;
   session_calling_peer_key: string;
   session_control_peer_key: string;
   callConfirm: ConfirmOptions;
@@ -106,6 +107,7 @@ export default class Assist {
         onCallStart: () => {},
         onAgentConnect: () => {},
         onRemoteControlStart: () => {},
+        onDragCamera: () => {},
         callConfirm: {},
         controlConfirm: {}, // TODO: clear options passing/merging/overwriting
         recordingConfirm: {},
@@ -376,6 +378,15 @@ export default class Assist {
       socket.on("move", (id, event) =>
         processEvent(id, event, this.remoteControl?.move)
       );
+      socket.on("startDrag", (id, event) =>
+        processEvent(id, event, this.remoteControl?.startDrag)
+      );
+      socket.on("drag", (id, event) =>
+        processEvent(id, event, this.remoteControl?.drag)
+      );
+      socket.on("stopDrag", (id, event) =>
+        processEvent(id, event, this.remoteControl?.stopDrag)
+      );
       socket.on("focus", (id, event) =>
         processEvent(id, event, (clientID, nodeID) => {
           const el = app.nodes.getNode(nodeID);
@@ -752,39 +763,6 @@ export default class Assist {
           return;
         }
 
-        if (!callUI) {
-          callUI = new CallWindow(app.debug.error, this.options.callUITemplate);
-          callUI.setVideoToggleCallback((args: { enabled: boolean }) => {
-            this.emit("videofeed", { streamId: from, enabled: args.enabled });
-          });
-        }
-        // show buttons in the call window
-        callUI.showControls(initiateCallEnd);
-        if (!annot) {
-          annot = new AnnotationCanvas();
-          annot.mount();
-        }
-
-        // callUI.setLocalStreams(Object.values(lStreams))
-        try {
-          // if there are no local streams in lStrems then we set
-          if (!lStreams[from]) {
-            app.debug.log("starting new stream for", from);
-            // request a local stream, and set it to lStreams
-            lStreams[from] = await RequestLocalStream(
-              pc,
-              renegotiateConnection.bind(null, { pc, from })
-            );
-          }
-          // we pass the received tracks to Call ui
-          callUI.setLocalStreams(Object.values(lStreams));
-        } catch (e) {
-          app.debug.error("Error requesting local stream", e);
-          // if something didn't work out, we terminate the call
-          initiateCallEnd();
-          return;
-        }
-
         // get all local tracks and add them to RTCPeerConnection
         // When we receive local ice candidates, we emit them via socket
         pc.onicecandidate = (event) => {

tracker/tracker-assist/src/Mouse.ts

@@ -1,10 +1,15 @@
+import { hasOpenreplayAttribute } from "./utils.js"
+
 type XY = [number, number]
+type XYDXDY = [number, number, number, number]
 
 
 export default class Mouse {
   private readonly mouse: HTMLDivElement
   private position: [number,number] = [0,0,]
-  constructor(private readonly agentName?: string) {
+  private isDragging = false
+
+  constructor(private readonly agentName?: string, private onDragCamera?: (dx: number, dy: number) => void) {
     this.mouse = document.createElement('div')
     const agentBubble = document.createElement('div')
     const svg ='<svg version="1.1" width="20" height="20" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" xml:space="" viewBox="8.2 4.9 11.6 18.2"><polygon fill="#FFFFFF" points="8.2,20.9 8.2,4.9 19.8,16.5 13,16.5 12.6,16.6 "></polygon><polygon fill="#FFFFFF" points="17.3,21.6 13.7,23.1 9,12 12.7,10.5 "></polygon><rect x="12.5" y="13.6" transform="matrix(0.9221 -0.3871 0.3871 0.9221 -5.7605 6.5909)" width="2" height="8"></rect><polygon points="9.2,7.3 9.2,18.5 12.2,15.6 12.6,15.5 17.4,15.5 "></polygon></svg>'
@@ -23,6 +28,8 @@ export default class Mouse {
       whiteSpace: 'nowrap',
     })
 
+    this.onDragCamera = onDragCamera
+
     const agentNameStr = this.agentName ? this.agentName.length > 10 ? this.agentName.slice(0, 9) + '...' : this.agentName : 'Agent'
     agentBubble.innerHTML = `<span>${agentNameStr}</span>`
 
@@ -71,8 +78,63 @@ export default class Mouse {
     return null
   }
 
-  private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct
+  startDrag(pos: XY) {
-  private lastScrEl: Element | 'window' | null = null
+    this.move(pos)
+    const el = document.elementFromPoint(pos[0], pos[1])
+    if (el) {
+      const downEvt = new MouseEvent("mousedown", {
+        bubbles: true,
+        cancelable: true,
+        clientX: pos[0],
+        clientY: pos[1],
+        buttons: 1,
+      });
+      el.dispatchEvent(downEvt);
+      this.isDragging = true;
+    }
+  }
+
+  drag(pos: XYDXDY) {
+    const [x, y, dx, dy] = pos
+    this.move([x, y]);
+
+    if (!this.isDragging) return;
+
+    const el = document.elementFromPoint(x, y);
+    if (el) {
+      const moveEvt = new MouseEvent("mousemove", {
+        bubbles: true,
+        cancelable: true,
+        clientX: x,
+        clientY: y,
+        buttons: 1,
+      });
+      el.dispatchEvent(moveEvt);
+      if (hasOpenreplayAttribute(el, 'draggable') && this.onDragCamera) {
+        this.onDragCamera(dx, dy);
+      }
+    }
+  }
+
+  stopDrag() {
+    if (!this.isDragging) return;
+    const [x, y] = this.position;
+    const el = document.elementFromPoint(x, y);
+    if (el) {
+      const upEvt = new MouseEvent("mouseup", {
+        bubbles: true,
+        cancelable: true,
+        clientX: x,
+        clientY: y,
+        buttons: 0,
+      });
+      el.dispatchEvent(upEvt);
+    }
+    this.isDragging = false;
+  }
+
+  private readonly pScrEl = document.scrollingElement || document.documentElement; // Is it always correct
+  private lastScrEl: Element | "window" | null = null;
   private readonly resetLastScrEl = () => { this.lastScrEl = null }
   private readonly handleWScroll = e => {
     if (e.target !== this.lastScrEl &&

tracker/tracker-assist/src/RemoteControl.ts

@@ -94,7 +94,7 @@ export default class RemoteControl {
     if (this.mouse) {
       this.resetMouse()
     }
-    this.mouse = new Mouse(agentName)
+    this.mouse = new Mouse(agentName, this.options.onDragCamera)
     this.mouse.mount()
     document.addEventListener('visibilitychange', () => {
       if (document.hidden) this.releaseControl(false, true)
@@ -123,6 +123,15 @@ export default class RemoteControl {
   focus = (id, el: HTMLElement) => {
     this.focused = el
   }
+  startDrag = (id, xy) => {
+    this.mouse?.startDrag(xy)
+  }
+  drag = (id, xydxdy) => {
+    this.mouse?.drag(xydxdy);
+  }
+  stopDrag = (id) => {
+    this.mouse?.stopDrag();
+  }
   input = (id, value: string) => {
     if (id !== this.agentID || !this.mouse || !this.focused) { return }
     if (this.focused instanceof HTMLTextAreaElement

tracker/tracker-assist/src/utils.ts

@@ -0,0 +1,33 @@
+export const DOCS_HOST = 'https://docs.openreplay.com'
+
+const warnedFeatures: { [key: string]: boolean } = {}
+
+export function deprecationWarn(nameOfFeature: string, useInstead: string, docsPath = '/'): void {
+    if (warnedFeatures[nameOfFeature]) {
+      return
+    }
+    console.warn(
+      `OpenReplay: ${nameOfFeature} is deprecated. ${
+        useInstead ? `Please, use ${useInstead} instead.` : ''
+      } Visit ${DOCS_HOST}${docsPath} for more information.`,
+    )
+    warnedFeatures[nameOfFeature] = true
+  }
+
+export function hasOpenreplayAttribute(e: Element, attr: string): boolean {
+    const newName = `data-openreplay-${attr}`
+    if (e.hasAttribute(newName)) {
+      // @ts-ignore
+      if (DEPRECATED_ATTRS[attr]) {
+        deprecationWarn(
+          `"${newName}" attribute`,
+          // @ts-ignore
+          `"${DEPRECATED_ATTRS[attr] as string}" attribute`,
+          '/en/sdk/sanitize-data',
+        )
+      }
+      return true
+    }
+  
+    return false
+  }