diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index 666239e09..4b5739141 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -178,17 +178,20 @@ function AssistActions({ }; const requestControl = () => { - if (remoteActive) player.assistManager.ping(AssistActionsPing.control.end, agentId) - setRemoteControlCallbacks({ onReject: onControlReject }); - if (callRequesting || remoteRequesting) return; + const onStart = () => { + player.assistManager.ping(AssistActionsPing.control.start, agentId) + } + const onEnd = () => { + player.assistManager.ping(AssistActionsPing.control.end, agentId) + } + setRemoteControlCallbacks({ + onReject: onControlReject, + onStart: onStart, + onEnd: onEnd, + }); requestReleaseRemoteControl(); }; - React.useEffect(() => { - if (remoteActive) { - player.assistManager.ping(AssistActionsPing.control.start, agentId) - } - }, [remoteActive]) React.useEffect(() => { if (onCall) { player.assistManager.ping(AssistActionsPing.call.start, agentId) diff --git a/frontend/app/player/web/assist/RemoteControl.ts b/frontend/app/player/web/assist/RemoteControl.ts index aa5690e86..a49979885 100644 --- a/frontend/app/player/web/assist/RemoteControl.ts +++ b/frontend/app/player/web/assist/RemoteControl.ts @@ -1,7 +1,7 @@ import AnnotationCanvas from './AnnotationCanvas'; -import type { Socket } from './types' -import type Screen from '../Screen/Screen' -import type { Store } from '../../common/types' +import type { Socket } from './types'; +import type Screen from '../Screen/Screen'; +import type { Store } from '../../common/types'; export enum RemoteControlStatus { Disabled = 0, @@ -10,180 +10,212 @@ export enum RemoteControlStatus { } export interface State { - annotating: boolean - remoteControl: RemoteControlStatus - currentTab?: string + annotating: boolean; + remoteControl: RemoteControlStatus; + currentTab?: string; } export default class RemoteControl { - private assistVersion = 1 - static readonly INITIAL_STATE: Readonly = { - remoteControl: RemoteControlStatus.Disabled, - annotating: false, - } - onReject: () => void = () => {} + private assistVersion = 1; + static readonly INITIAL_STATE: Readonly = { + remoteControl: RemoteControlStatus.Disabled, + annotating: false, + }; + onReject: () => void = () => {}; + onStart: () => void = () => {}; + onEnd: () => void = () => {}; - constructor( - private store: Store, - private socket: Socket, - private screen: Screen, - private agentInfo: Object, - private onToggle: (active: boolean) => void, + constructor( + private store: Store, + private socket: Socket, + private screen: Screen, + private agentInfo: Object, + private onToggle: (active: boolean) => void, private getAssistVersion: () => number - ){ - socket.on("control_granted", ({ meta, data }) => { - this.toggleRemoteControl(data === socket.id) - }) - socket.on("control_rejected", ({ meta, data }) => { - data === socket.id && this.toggleRemoteControl(false) - this.onReject() - }) - socket.on('SESSION_DISCONNECTED', () => { - if (this.store.get().remoteControl === RemoteControlStatus.Requesting) { - this.toggleRemoteControl(false) // else its remaining + ) { + socket.on('control_granted', ({ meta, data }) => { + if (data === socket.id) { + this.toggleRemoteControl(data === socket.id); + this.onStart(); } - }) - socket.on("disconnect", () => { - this.toggleRemoteControl(false) - }) - socket.on("error", () => { - this.toggleRemoteControl(false) - }) - this.assistVersion = getAssistVersion() - } - - private onMouseMove = (e: MouseEvent): void => { - const data = this.screen.getInternalCoordinates(e) - this.emitData("move", [ data.x, data.y ]) + }); + socket.on('control_rejected', ({ meta, data }) => { + if (data === socket.id) { + this.toggleRemoteControl(false); + this.onEnd(); + } + this.onReject(); + if (this.store.get().remoteControl === RemoteControlStatus.Requesting) { + return this.store.update({ remoteControl: RemoteControlStatus.Disabled }); + } + }); + socket.on('SESSION_DISCONNECTED', () => { + if (this.store.get().remoteControl === RemoteControlStatus.Requesting) { + this.toggleRemoteControl(false); // else its remaining + } + }); + socket.on('disconnect', () => { + this.toggleRemoteControl(false); + this.onEnd(); + }); + socket.on('error', () => { + this.toggleRemoteControl(false); + this.onEnd(); + }); + this.assistVersion = getAssistVersion(); } + private onMouseMove = (e: MouseEvent): void => { + const data = this.screen.getInternalCoordinates(e); + this.emitData('move', [data.x, data.y]); + }; + private emitData = (event: string, data?: any) => { if (this.getAssistVersion() === 1) { - this.socket.emit(event, data) + this.socket.emit(event, data); } else { - this.socket.emit(event, { meta: { tabId: this.store.get().currentTab }, data }) + this.socket.emit(event, { meta: { tabId: this.store.get().currentTab }, data }); } - } + }; private onWheel = (e: WheelEvent): void => { - e.preventDefault() + e.preventDefault(); //throttling makes movements less smooth, so it is omitted //this.onMouseMove(e) - this.emitData("scroll", [ e.deltaX, e.deltaY ]) - } + this.emitData('scroll', [e.deltaX, e.deltaY]); + }; - public setCallbacks = ({ onReject }: { onReject: () => void }) => { - this.onReject = onReject - } + public setCallbacks = ({ + onReject, + onStart, + onEnd, + }: { + onReject: () => void; + onStart: () => void; + onEnd: () => void; + }) => { + this.onReject = onReject; + this.onStart = onStart; + this.onEnd = onEnd; + }; private onMouseClick = (e: MouseEvent): void => { - if (this.store.get().annotating) { return; } // ignore clicks while annotating + if (this.store.get().annotating) { + return; + } // ignore clicks while annotating - const data = this.screen.getInternalViewportCoordinates(e) + const data = this.screen.getInternalViewportCoordinates(e); // const el = this.screen.getElementFromPoint(e); // requires requestiong node_id from domManager - const el = this.screen.getElementFromInternalPoint(data) + const el = this.screen.getElementFromInternalPoint(data); if (el instanceof HTMLElement) { - el.focus() - el.oninput = e => { - if (el instanceof HTMLTextAreaElement - || el instanceof HTMLInputElement - ) { - this.socket && this.emitData("input", el.value) + el.focus(); + el.oninput = (e) => { + if (el instanceof HTMLTextAreaElement || el instanceof HTMLInputElement) { + this.socket && this.emitData('input', el.value); } else if (el.isContentEditable) { - this.socket && this.emitData("input", el.innerText) + this.socket && this.emitData('input', el.innerText); } - } + }; // TODO: send "focus" event to assist with the nodeID - el.onkeydown = e => { - if (e.key == "Tab") { - e.preventDefault() + el.onkeydown = (e) => { + if (e.key == 'Tab') { + e.preventDefault(); } - } + }; el.onblur = () => { - el.oninput = null - el.onblur = null - } + el.oninput = null; + el.onblur = null; + }; } - this.emitData("click", [ data.x, data.y ]); - } + this.emitData('click', [data.x, data.y]); + }; - private toggleRemoteControl(enable: boolean){ + private toggleRemoteControl(enable: boolean) { if (enable) { - this.screen.overlay.addEventListener("mousemove", this.onMouseMove) - this.screen.overlay.addEventListener("click", this.onMouseClick) - this.screen.overlay.addEventListener("wheel", this.onWheel) - this.store.update({ remoteControl: RemoteControlStatus.Enabled }) + this.screen.overlay.addEventListener('mousemove', this.onMouseMove); + this.screen.overlay.addEventListener('click', this.onMouseClick); + this.screen.overlay.addEventListener('wheel', this.onWheel); + this.store.update({ remoteControl: RemoteControlStatus.Enabled }); } else { - this.screen.overlay.removeEventListener("mousemove", this.onMouseMove) - this.screen.overlay.removeEventListener("click", this.onMouseClick) - this.screen.overlay.removeEventListener("wheel", this.onWheel) - this.store.update({ remoteControl: RemoteControlStatus.Disabled }) - this.toggleAnnotation(false) + this.screen.overlay.removeEventListener('mousemove', this.onMouseMove); + this.screen.overlay.removeEventListener('click', this.onMouseClick); + this.screen.overlay.removeEventListener('wheel', this.onWheel); + this.store.update({ remoteControl: RemoteControlStatus.Disabled }); + this.toggleAnnotation(false); } - this.onToggle(enable) + this.onToggle(enable); } requestReleaseRemoteControl = () => { - const remoteControl = this.store.get().remoteControl - if (remoteControl === RemoteControlStatus.Requesting) { return } - if (remoteControl === RemoteControlStatus.Disabled) { - this.store.update({ remoteControl: RemoteControlStatus.Requesting }) - this.emitData("request_control", JSON.stringify({ - ...this.agentInfo, - query: document.location.search - })) - } else { - this.releaseRemoteControl() + const remoteControl = this.store.get().remoteControl; + console.log(remoteControl); + if (remoteControl === RemoteControlStatus.Requesting) { + return; } - } + if (remoteControl === RemoteControlStatus.Disabled) { + this.store.update({ remoteControl: RemoteControlStatus.Requesting }); + this.emitData( + 'request_control', + JSON.stringify({ + ...this.agentInfo, + query: document.location.search, + }) + ); + } else { + this.onEnd(); + this.releaseRemoteControl(); + } + }; releaseRemoteControl = () => { - this.emitData("release_control",) - this.toggleRemoteControl(false) - } + this.emitData('release_control'); + this.toggleRemoteControl(false); + }; - private annot: AnnotationCanvas | null = null + private annot: AnnotationCanvas | null = null; toggleAnnotation(enable?: boolean) { - if (typeof enable !== "boolean") { - enable = this.store.get().annotating + if (typeof enable !== 'boolean') { + enable = this.store.get().annotating; } if (enable && !this.annot) { - const annot = this.annot = new AnnotationCanvas() - annot.mount(this.screen.overlay) - annot.canvas.addEventListener("mousedown", e => { - const data = this.screen.getInternalViewportCoordinates(e) - annot.start([ data.x, data.y ]) - this.emitData("startAnnotation", [ data.x, data.y ]) - }) - annot.canvas.addEventListener("mouseleave", () => { - annot.stop() - this.emitData("stopAnnotation") - }) - annot.canvas.addEventListener("mouseup", () => { - annot.stop() - this.emitData("stopAnnotation") - }) - annot.canvas.addEventListener("mousemove", e => { - if (!annot.isPainting()) { return } + const annot = (this.annot = new AnnotationCanvas()); + annot.mount(this.screen.overlay); + annot.canvas.addEventListener('mousedown', (e) => { + const data = this.screen.getInternalViewportCoordinates(e); + annot.start([data.x, data.y]); + this.emitData('startAnnotation', [data.x, data.y]); + }); + annot.canvas.addEventListener('mouseleave', () => { + annot.stop(); + this.emitData('stopAnnotation'); + }); + annot.canvas.addEventListener('mouseup', () => { + annot.stop(); + this.emitData('stopAnnotation'); + }); + annot.canvas.addEventListener('mousemove', (e) => { + if (!annot.isPainting()) { + return; + } - const data = this.screen.getInternalViewportCoordinates(e) - annot.move([ data.x, data.y ]) - this.emitData("moveAnnotation", [ data.x, data.y ]) - }) - this.store.update({ annotating: true }) + const data = this.screen.getInternalViewportCoordinates(e); + annot.move([data.x, data.y]); + this.emitData('moveAnnotation', [data.x, data.y]); + }); + this.store.update({ annotating: true }); } else if (!enable && !!this.annot) { - this.annot.remove() - this.annot = null - this.store.update({ annotating: false }) + this.annot.remove(); + this.annot = null; + this.store.update({ annotating: false }); } } clean() { - this.toggleRemoteControl(false) - if (this.annot) { - this.annot.remove() - this.annot = null + this.toggleRemoteControl(false); + if (this.annot) { + this.annot.remove(); + this.annot = null; } } -} \ No newline at end of file +} diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index 28dccf4c1..926360b58 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -42,7 +42,7 @@ export default class RemoteControl { this.releaseControl() return } - setTimeout(() =>{ + setTimeout(() => { if (this.status === RCStatus.Requesting) { this.releaseControl() }