diff --git a/frontend/app/player/web/assist/RemoteControl.ts b/frontend/app/player/web/assist/RemoteControl.ts index 744412a98..ba95055fc 100644 --- a/frontend/app/player/web/assist/RemoteControl.ts +++ b/frontend/app/player/web/assist/RemoteControl.ts @@ -17,6 +17,9 @@ export interface State { export default class RemoteControl { private assistVersion = 1; + private isDragging = false; + private dragStart: any | null = null; + private readonly dragThreshold = 3; static readonly INITIAL_STATE: Readonly = { remoteControl: RemoteControlStatus.Disabled, @@ -81,6 +84,7 @@ export default class RemoteControl { } private onMouseMove = (e: MouseEvent): void => { + if (this.isDragging) return; const data = this.screen.getInternalCoordinates(e); this.emitData('move', [data.x, data.y]); }; @@ -154,16 +158,61 @@ export default class RemoteControl { this.emitData('click', [data.x, data.y]); }; + private onMouseDown = (e: MouseEvent): void => { + if (this.store.get().annotating) return; + + const { x, y } = this.screen.getInternalViewportCoordinates(e); + this.dragStart = [x, y]; + this.isDragging = false; + + const handleMove = (moveEvent: MouseEvent) => { + const { x: mx, y: my } = + this.screen.getInternalViewportCoordinates(moveEvent); + const [sx, sy] = this.dragStart!; + const dx = Math.abs(mx - sx); + const dy = Math.abs(my - sy); + + if ( + !this.isDragging && + (dx > this.dragThreshold || dy > this.dragThreshold) + ) { + this.emitData('startDrag', [sx, sy]); + this.isDragging = true; + } + + if (this.isDragging) { + this.emitData('drag', [mx, my, mx - sx, my - sy]); + } + }; + + const handleUp = () => { + if (this.isDragging) { + this.emitData('stopDrag'); + } + + this.dragStart = null; + this.isDragging = false; + + window.removeEventListener('mousemove', handleMove); + window.removeEventListener('mouseup', handleUp); + }; + + window.addEventListener('mousemove', handleMove); + window.addEventListener('mouseup', handleUp); + }; + private toggleRemoteControl(enable: boolean) { if (enable) { this.screen.overlay.addEventListener('mousemove', this.onMouseMove); this.screen.overlay.addEventListener('click', this.onMouseClick); this.screen.overlay.addEventListener('wheel', this.onWheel); + this.screen.overlay.addEventListener('mousedown', this.onMouseDown); this.store.update({ remoteControl: RemoteControlStatus.Enabled }); } else { this.screen.overlay.removeEventListener('mousemove', this.onMouseMove); this.screen.overlay.removeEventListener('click', this.onMouseClick); this.screen.overlay.removeEventListener('wheel', this.onWheel); + this.screen.overlay.removeEventListener('mousedown', this.onMouseDown); this.store.update({ remoteControl: RemoteControlStatus.Disabled }); this.toggleAnnotation(false); } diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 0e8bb9aba..8afc5f45f 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -37,6 +37,7 @@ export interface Options { onCallDeny?: () => any; onRemoteControlDeny?: (agentInfo: Record) => any; onRecordingDeny?: (agentInfo: Record) => any; + onDragCamera?: (dx: number, dy: number) => void; session_calling_peer_key: string; session_control_peer_key: string; callConfirm: ConfirmOptions; @@ -106,6 +107,7 @@ export default class Assist { onCallStart: () => {}, onAgentConnect: () => {}, onRemoteControlStart: () => {}, + onDragCamera: () => {}, callConfirm: {}, controlConfirm: {}, // TODO: clear options passing/merging/overwriting recordingConfirm: {}, @@ -376,6 +378,15 @@ export default class Assist { socket.on("move", (id, event) => processEvent(id, event, this.remoteControl?.move) ); + socket.on("startDrag", (id, event) => + processEvent(id, event, this.remoteControl?.startDrag) + ); + socket.on("drag", (id, event) => + processEvent(id, event, this.remoteControl?.drag) + ); + socket.on("stopDrag", (id, event) => + processEvent(id, event, this.remoteControl?.stopDrag) + ); socket.on("focus", (id, event) => processEvent(id, event, (clientID, nodeID) => { const el = app.nodes.getNode(nodeID); @@ -752,39 +763,6 @@ export default class Assist { return; } - if (!callUI) { - callUI = new CallWindow(app.debug.error, this.options.callUITemplate); - callUI.setVideoToggleCallback((args: { enabled: boolean }) => { - this.emit("videofeed", { streamId: from, enabled: args.enabled }); - }); - } - // show buttons in the call window - callUI.showControls(initiateCallEnd); - if (!annot) { - annot = new AnnotationCanvas(); - annot.mount(); - } - - // callUI.setLocalStreams(Object.values(lStreams)) - try { - // if there are no local streams in lStrems then we set - if (!lStreams[from]) { - app.debug.log("starting new stream for", from); - // request a local stream, and set it to lStreams - lStreams[from] = await RequestLocalStream( - pc, - renegotiateConnection.bind(null, { pc, from }) - ); - } - // we pass the received tracks to Call ui - callUI.setLocalStreams(Object.values(lStreams)); - } catch (e) { - app.debug.error("Error requesting local stream", e); - // if something didn't work out, we terminate the call - initiateCallEnd(); - return; - } - // get all local tracks and add them to RTCPeerConnection // When we receive local ice candidates, we emit them via socket pc.onicecandidate = (event) => { diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index 65ea86312..4672bafd0 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -1,10 +1,15 @@ +import { hasOpenreplayAttribute } from "./utils.js" + type XY = [number, number] +type XYDXDY = [number, number, number, number] export default class Mouse { private readonly mouse: HTMLDivElement private position: [number,number] = [0,0,] - constructor(private readonly agentName?: string) { + private isDragging = false + + constructor(private readonly agentName?: string, private onDragCamera?: (dx: number, dy: number) => void) { this.mouse = document.createElement('div') const agentBubble = document.createElement('div') const svg ='' @@ -23,6 +28,8 @@ export default class Mouse { whiteSpace: 'nowrap', }) + this.onDragCamera = onDragCamera + const agentNameStr = this.agentName ? this.agentName.length > 10 ? this.agentName.slice(0, 9) + '...' : this.agentName : 'Agent' agentBubble.innerHTML = `${agentNameStr}` @@ -71,8 +78,63 @@ export default class Mouse { return null } - private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct - private lastScrEl: Element | 'window' | null = null + startDrag(pos: XY) { + this.move(pos) + const el = document.elementFromPoint(pos[0], pos[1]) + if (el) { + const downEvt = new MouseEvent("mousedown", { + bubbles: true, + cancelable: true, + clientX: pos[0], + clientY: pos[1], + buttons: 1, + }); + el.dispatchEvent(downEvt); + this.isDragging = true; + } + } + + drag(pos: XYDXDY) { + const [x, y, dx, dy] = pos + this.move([x, y]); + + if (!this.isDragging) return; + + const el = document.elementFromPoint(x, y); + if (el) { + const moveEvt = new MouseEvent("mousemove", { + bubbles: true, + cancelable: true, + clientX: x, + clientY: y, + buttons: 1, + }); + el.dispatchEvent(moveEvt); + if (hasOpenreplayAttribute(el, 'draggable') && this.onDragCamera) { + this.onDragCamera(dx, dy); + } + } + } + + stopDrag() { + if (!this.isDragging) return; + const [x, y] = this.position; + const el = document.elementFromPoint(x, y); + if (el) { + const upEvt = new MouseEvent("mouseup", { + bubbles: true, + cancelable: true, + clientX: x, + clientY: y, + buttons: 0, + }); + el.dispatchEvent(upEvt); + } + this.isDragging = false; + } + + private readonly pScrEl = document.scrollingElement || document.documentElement; // Is it always correct + private lastScrEl: Element | "window" | null = null; private readonly resetLastScrEl = () => { this.lastScrEl = null } private readonly handleWScroll = e => { if (e.target !== this.lastScrEl && diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index 26b8e93a0..54962cfae 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -94,7 +94,7 @@ export default class RemoteControl { if (this.mouse) { this.resetMouse() } - this.mouse = new Mouse(agentName) + this.mouse = new Mouse(agentName, this.options.onDragCamera) this.mouse.mount() document.addEventListener('visibilitychange', () => { if (document.hidden) this.releaseControl(false, true) @@ -123,6 +123,15 @@ export default class RemoteControl { focus = (id, el: HTMLElement) => { this.focused = el } + startDrag = (id, xy) => { + this.mouse?.startDrag(xy) + } + drag = (id, xydxdy) => { + this.mouse?.drag(xydxdy); + } + stopDrag = (id) => { + this.mouse?.stopDrag(); + } input = (id, value: string) => { if (id !== this.agentID || !this.mouse || !this.focused) { return } if (this.focused instanceof HTMLTextAreaElement diff --git a/tracker/tracker-assist/src/utils.ts b/tracker/tracker-assist/src/utils.ts new file mode 100644 index 000000000..37fb4ed33 --- /dev/null +++ b/tracker/tracker-assist/src/utils.ts @@ -0,0 +1,33 @@ +export const DOCS_HOST = 'https://docs.openreplay.com' + +const warnedFeatures: { [key: string]: boolean } = {} + +export function deprecationWarn(nameOfFeature: string, useInstead: string, docsPath = '/'): void { + if (warnedFeatures[nameOfFeature]) { + return + } + console.warn( + `OpenReplay: ${nameOfFeature} is deprecated. ${ + useInstead ? `Please, use ${useInstead} instead.` : '' + } Visit ${DOCS_HOST}${docsPath} for more information.`, + ) + warnedFeatures[nameOfFeature] = true + } + +export function hasOpenreplayAttribute(e: Element, attr: string): boolean { + const newName = `data-openreplay-${attr}` + if (e.hasAttribute(newName)) { + // @ts-ignore + if (DEPRECATED_ATTRS[attr]) { + deprecationWarn( + `"${newName}" attribute`, + // @ts-ignore + `"${DEPRECATED_ATTRS[attr] as string}" attribute`, + '/en/sdk/sanitize-data', + ) + } + return true + } + + return false + } \ No newline at end of file diff --git a/tracker/tracker/.yarn/install-state.gz b/tracker/tracker/.yarn/install-state.gz index ab4be7265..39932e5b3 100644 Binary files a/tracker/tracker/.yarn/install-state.gz and b/tracker/tracker/.yarn/install-state.gz differ