add global method support

frontend/app/player/web/assist/RemoteControl.ts

@@ -17,6 +17,9 @@ export interface State {
 
 export default class RemoteControl {
   private assistVersion = 1;
+  private isDragging = false;
+  private dragStart: any | null = null;
+  private readonly dragThreshold = 3;
 
   static readonly INITIAL_STATE: Readonly<State> = {
     remoteControl: RemoteControlStatus.Disabled,
@@ -81,6 +84,7 @@ export default class RemoteControl {
   }
 
   private onMouseMove = (e: MouseEvent): void => {
+    if (this.isDragging) return;
     const data = this.screen.getInternalCoordinates(e);
     this.emitData('move', [data.x, data.y]);
   };
@@ -154,16 +158,61 @@ export default class RemoteControl {
     this.emitData('click', [data.x, data.y]);
   };
 
+  private onMouseDown = (e: MouseEvent): void => {
+    if (this.store.get().annotating) return;
+
+    const { x, y } = this.screen.getInternalViewportCoordinates(e);
+    this.dragStart = [x, y];
+    this.isDragging = false;
+
+    const handleMove = (moveEvent: MouseEvent) => {
+      const { x: mx, y: my } =
+        this.screen.getInternalViewportCoordinates(moveEvent);
+      const [sx, sy] = this.dragStart!;
+      const dx = Math.abs(mx - sx);
+      const dy = Math.abs(my - sy);
+
+      if (
+        !this.isDragging &&
+        (dx > this.dragThreshold || dy > this.dragThreshold)
+      ) {
+        this.emitData('startDrag', [sx, sy]);
+        this.isDragging = true;
+      }
+
+      if (this.isDragging) {
+        this.emitData('drag', [mx, my, mx - sx, my - sy]);
+      }
+    };
+
+    const handleUp = () => {
+      if (this.isDragging) {
+        this.emitData('stopDrag');
+      }
+
+      this.dragStart = null;
+      this.isDragging = false;
+
+      window.removeEventListener('mousemove', handleMove);
+      window.removeEventListener('mouseup', handleUp);
+    };
+
+    window.addEventListener('mousemove', handleMove);
+    window.addEventListener('mouseup', handleUp);
+  };
+
   private toggleRemoteControl(enable: boolean) {
     if (enable) {
       this.screen.overlay.addEventListener('mousemove', this.onMouseMove);
       this.screen.overlay.addEventListener('click', this.onMouseClick);
       this.screen.overlay.addEventListener('wheel', this.onWheel);
+      this.screen.overlay.addEventListener('mousedown', this.onMouseDown);
       this.store.update({ remoteControl: RemoteControlStatus.Enabled });
     } else {
       this.screen.overlay.removeEventListener('mousemove', this.onMouseMove);
       this.screen.overlay.removeEventListener('click', this.onMouseClick);
       this.screen.overlay.removeEventListener('wheel', this.onWheel);
+      this.screen.overlay.removeEventListener('mousedown', this.onMouseDown);
       this.store.update({ remoteControl: RemoteControlStatus.Disabled });
       this.toggleAnnotation(false);
     }

tracker/tracker-assist/src/Assist.ts

@@ -376,6 +376,15 @@ export default class Assist {
       socket.on("move", (id, event) =>
         processEvent(id, event, this.remoteControl?.move)
       );
+      socket.on("startDrag", (id, event) =>
+        processEvent(id, event, this.remoteControl?.startDrag)
+      );
+      socket.on("drag", (id, event) =>
+        processEvent(id, event, this.remoteControl?.drag)
+      );
+      socket.on("stopDrag", (id, event) =>
+        processEvent(id, event, this.remoteControl?.stopDrag)
+      );
       socket.on("focus", (id, event) =>
         processEvent(id, event, (clientID, nodeID) => {
           const el = app.nodes.getNode(nodeID);

tracker/tracker-assist/src/Mouse.ts

@@ -1,9 +1,12 @@
 type XY = [number, number]
+type XYDXDY = [number, number, number, number]
 
 
 export default class Mouse {
   private readonly mouse: HTMLDivElement
   private position: [number,number] = [0,0,]
+  private isDragging = false
+
   constructor(private readonly agentName?: string) {
     this.mouse = document.createElement('div')
     const agentBubble = document.createElement('div')
@@ -71,8 +74,63 @@ export default class Mouse {
     return null
   }
 
-  private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct
-  private lastScrEl: Element | 'window' | null = null
+  startDrag(pos: XY) {
+    this.move(pos)
+    const el = document.elementFromPoint(pos[0], pos[1])
+    if (el) {
+      const downEvt = new MouseEvent("mousedown", {
+        bubbles: true,
+        cancelable: true,
+        clientX: pos[0],
+        clientY: pos[1],
+        buttons: 1,
+      });
+      el.dispatchEvent(downEvt);
+      this.isDragging = true;
+    }
+  }
+
+  drag(pos: XYDXDY) {
+    const [x, y, dx, dy] = pos
+    this.move([x, y]);
+
+    if (!this.isDragging) return;
+
+    const el = document.elementFromPoint(x, y);
+    if (el) {
+      const moveEvt = new MouseEvent("mousemove", {
+        bubbles: true,
+        cancelable: true,
+        clientX: x,
+        clientY: y,
+        buttons: 1,
+      });
+      el.dispatchEvent(moveEvt);
+    }
+    if ((window as any).__REMOTE__) {
+      (window as any).__REMOTE__.dragCamera(dx, dy);
+    }
+  }
+
+  stopDrag() {
+    if (!this.isDragging) return;
+    const [x, y] = this.position;
+    const el = document.elementFromPoint(x, y);
+    if (el) {
+      const upEvt = new MouseEvent("mouseup", {
+        bubbles: true,
+        cancelable: true,
+        clientX: x,
+        clientY: y,
+        buttons: 0,
+      });
+      el.dispatchEvent(upEvt);
+    }
+    this.isDragging = false;
+  }
+
+  private readonly pScrEl = document.scrollingElement || document.documentElement; // Is it always correct
+  private lastScrEl: Element | "window" | null = null;
   private readonly resetLastScrEl = () => { this.lastScrEl = null }
   private readonly handleWScroll = e => {
     if (e.target !== this.lastScrEl &&

tracker/tracker-assist/src/RemoteControl.ts

@@ -123,6 +123,15 @@ export default class RemoteControl {
   focus = (id, el: HTMLElement) => {
     this.focused = el
   }
+  startDrag = (id, xy) => {
+    this.mouse?.startDrag(xy)
+  }
+  drag = (id, xydxdy) => {
+    this.mouse?.drag(xydxdy);
+  }
+  stopDrag = (id) => {
+    this.mouse?.stopDrag();
+  }
   input = (id, value: string) => {
     if (id !== this.agentID || !this.mouse || !this.focused) { return }
     if (this.focused instanceof HTMLTextAreaElement