diff --git a/frontend/app/player/web/assist/RemoteControl.ts b/frontend/app/player/web/assist/RemoteControl.ts index 744412a98..ba95055fc 100644 --- a/frontend/app/player/web/assist/RemoteControl.ts +++ b/frontend/app/player/web/assist/RemoteControl.ts @@ -17,6 +17,9 @@ export interface State { export default class RemoteControl { private assistVersion = 1; + private isDragging = false; + private dragStart: any | null = null; + private readonly dragThreshold = 3; static readonly INITIAL_STATE: Readonly = { remoteControl: RemoteControlStatus.Disabled, @@ -81,6 +84,7 @@ export default class RemoteControl { } private onMouseMove = (e: MouseEvent): void => { + if (this.isDragging) return; const data = this.screen.getInternalCoordinates(e); this.emitData('move', [data.x, data.y]); }; @@ -154,16 +158,61 @@ export default class RemoteControl { this.emitData('click', [data.x, data.y]); }; + private onMouseDown = (e: MouseEvent): void => { + if (this.store.get().annotating) return; + + const { x, y } = this.screen.getInternalViewportCoordinates(e); + this.dragStart = [x, y]; + this.isDragging = false; + + const handleMove = (moveEvent: MouseEvent) => { + const { x: mx, y: my } = + this.screen.getInternalViewportCoordinates(moveEvent); + const [sx, sy] = this.dragStart!; + const dx = Math.abs(mx - sx); + const dy = Math.abs(my - sy); + + if ( + !this.isDragging && + (dx > this.dragThreshold || dy > this.dragThreshold) + ) { + this.emitData('startDrag', [sx, sy]); + this.isDragging = true; + } + + if (this.isDragging) { + this.emitData('drag', [mx, my, mx - sx, my - sy]); + } + }; + + const handleUp = () => { + if (this.isDragging) { + this.emitData('stopDrag'); + } + + this.dragStart = null; + this.isDragging = false; + + window.removeEventListener('mousemove', handleMove); + window.removeEventListener('mouseup', handleUp); + }; + + window.addEventListener('mousemove', handleMove); + window.addEventListener('mouseup', handleUp); + }; + private toggleRemoteControl(enable: boolean) { if (enable) { this.screen.overlay.addEventListener('mousemove', this.onMouseMove); this.screen.overlay.addEventListener('click', this.onMouseClick); this.screen.overlay.addEventListener('wheel', this.onWheel); + this.screen.overlay.addEventListener('mousedown', this.onMouseDown); this.store.update({ remoteControl: RemoteControlStatus.Enabled }); } else { this.screen.overlay.removeEventListener('mousemove', this.onMouseMove); this.screen.overlay.removeEventListener('click', this.onMouseClick); this.screen.overlay.removeEventListener('wheel', this.onWheel); + this.screen.overlay.removeEventListener('mousedown', this.onMouseDown); this.store.update({ remoteControl: RemoteControlStatus.Disabled }); this.toggleAnnotation(false); } diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 0e8bb9aba..38ad0d4fe 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -376,6 +376,15 @@ export default class Assist { socket.on("move", (id, event) => processEvent(id, event, this.remoteControl?.move) ); + socket.on("startDrag", (id, event) => + processEvent(id, event, this.remoteControl?.startDrag) + ); + socket.on("drag", (id, event) => + processEvent(id, event, this.remoteControl?.drag) + ); + socket.on("stopDrag", (id, event) => + processEvent(id, event, this.remoteControl?.stopDrag) + ); socket.on("focus", (id, event) => processEvent(id, event, (clientID, nodeID) => { const el = app.nodes.getNode(nodeID); diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index 65ea86312..989c5da21 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -1,9 +1,12 @@ type XY = [number, number] +type XYDXDY = [number, number, number, number] export default class Mouse { private readonly mouse: HTMLDivElement private position: [number,number] = [0,0,] + private isDragging = false + constructor(private readonly agentName?: string) { this.mouse = document.createElement('div') const agentBubble = document.createElement('div') @@ -71,8 +74,63 @@ export default class Mouse { return null } - private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct - private lastScrEl: Element | 'window' | null = null + startDrag(pos: XY) { + this.move(pos) + const el = document.elementFromPoint(pos[0], pos[1]) + if (el) { + const downEvt = new MouseEvent("mousedown", { + bubbles: true, + cancelable: true, + clientX: pos[0], + clientY: pos[1], + buttons: 1, + }); + el.dispatchEvent(downEvt); + this.isDragging = true; + } + } + + drag(pos: XYDXDY) { + const [x, y, dx, dy] = pos + this.move([x, y]); + + if (!this.isDragging) return; + + const el = document.elementFromPoint(x, y); + if (el) { + const moveEvt = new MouseEvent("mousemove", { + bubbles: true, + cancelable: true, + clientX: x, + clientY: y, + buttons: 1, + }); + el.dispatchEvent(moveEvt); + } + if ((window as any).__REMOTE__) { + (window as any).__REMOTE__.dragCamera(dx, dy); + } + } + + stopDrag() { + if (!this.isDragging) return; + const [x, y] = this.position; + const el = document.elementFromPoint(x, y); + if (el) { + const upEvt = new MouseEvent("mouseup", { + bubbles: true, + cancelable: true, + clientX: x, + clientY: y, + buttons: 0, + }); + el.dispatchEvent(upEvt); + } + this.isDragging = false; + } + + private readonly pScrEl = document.scrollingElement || document.documentElement; // Is it always correct + private lastScrEl: Element | "window" | null = null; private readonly resetLastScrEl = () => { this.lastScrEl = null } private readonly handleWScroll = e => { if (e.target !== this.lastScrEl && diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index 26b8e93a0..7c9b426a8 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -123,6 +123,15 @@ export default class RemoteControl { focus = (id, el: HTMLElement) => { this.focused = el } + startDrag = (id, xy) => { + this.mouse?.startDrag(xy) + } + drag = (id, xydxdy) => { + this.mouse?.drag(xydxdy); + } + stopDrag = (id) => { + this.mouse?.stopDrag(); + } input = (id, value: string) => { if (id !== this.agentID || !this.mouse || !this.focused) { return } if (this.focused instanceof HTMLTextAreaElement diff --git a/tracker/tracker/.yarn/install-state.gz b/tracker/tracker/.yarn/install-state.gz index ab4be7265..39932e5b3 100644 Binary files a/tracker/tracker/.yarn/install-state.gz and b/tracker/tracker/.yarn/install-state.gz differ