jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tracker): multitab support for assist sessions

Browse source
This commit is contained in:
nick-delirium 2023-06-02 16:47:59 +02:00
parent 62ae5095b9
commit 0dc3dd2210
9 changed files with 129 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
frontend/app/components/Session/Player/LivePlayer/Overlay/LiveOverlay.tsx
View file

@ -33,7 +33,8 @@ function Overlay({
tabStates,
currentTab
} = store.get()
const cssLoading = tabStates[currentTab].cssLoading || false
const cssLoading = tabStates[currentTab]?.cssLoading || false
const loading = messagesLoading || cssLoading
const liveStatusText = getStatusText(peerConnectionStatus)
const connectionStatus = peerConnectionStatus

12
frontend/app/player/web/assist/AssistManager.ts
View file

@ -169,15 +169,13 @@ export default class AssistManager {
let currentTab = ''
socket.on('messages', messages => {
jmr.append(messages) // as RawMessage[]
console.log(messages)
jmr.append(messages.data) // as RawMessage[]
if (waitingForMessages) {
waitingForMessages = false // TODO: more explicit
this.setStatus(ConnectionStatus.Connected)
}
for (let msg = reader.readNext();msg !== null;msg = reader.readNext()) {
console.log(msg)
this.handleMessage(msg, msg._index)
}
})
@ -188,9 +186,15 @@ export default class AssistManager {
this.setStatus(ConnectionStatus.Connected)
})
socket.on('UPDATE_SESSION', ({ active }) => {
socket.on('UPDATE_SESSION', (evData) => {
const { metadata, data } = evData
const { tabId } = metadata
const { active } = data
this.clearDisconnectTimeout()
!this.inactiveTimeout && this.setStatus(ConnectionStatus.Connected)
if (tabId !== currentTab) {
this.store.update({ currentTab: tabId })
}
if (typeof active === "boolean") {
this.clearInactiveTimeout()
if (active) {

11
frontend/app/player/web/assist/Call.ts
View file

@ -18,6 +18,7 @@ export enum CallingState {
export interface State {
calling: CallingState;
currentTab?: string;
}
export default class Call {
@ -158,7 +159,7 @@ export default class Call {
}
initiateCallEnd = async () => {
this.socket?.emit("call_end", appStore.getState().getIn([ 'user', 'account', 'name']))
this.emitData("call_end", appStore.getState().getIn([ 'user', 'account', 'name']))
this.handleCallEnd()
// TODO: We have it separated, right? (check)
// const remoteControl = this.store.get().remoteControl
@ -168,6 +169,10 @@ export default class Call {
// }
}
private emitData = (event: string, data?: any) => {
this.socket?.emit(event, { meta: { tabId: this.store.get().currentTab }, data })
}
private callArgs: {
localStream: LocalStream,
@ -206,7 +211,7 @@ export default class Call {
toggleVideoLocalStream(enabled: boolean) {
this.getPeer().then((peer) => {
this.socket.emit('videofeed', { streamId: peer.id, enabled })
this.emitData('videofeed', { streamId: peer.id, enabled })
})
}
@ -223,7 +228,7 @@ export default class Call {
if (![CallingState.NoCall, CallingState.Reconnecting].includes(this.store.get().calling)) { return }
this.store.update({ calling: CallingState.Connecting })
this._peerConnection(this.peerID);
this.socket.emit("_agent_name", appStore.getState().getIn([ 'user', 'account', 'name']))
this.emitData("_agent_name", appStore.getState().getIn([ 'user', 'account', 'name']))
}
private async _peerConnection(remotePeerId: string) {

44
frontend/app/player/web/assist/RemoteControl.ts
View file

@ -12,6 +12,7 @@ export enum RemoteControlStatus {
export interface State {
annotating: boolean
remoteControl: RemoteControlStatus
currentTab?: string
}
export default class RemoteControl {
@ -28,11 +29,11 @@ export default class RemoteControl {
private agentInfo: Object,
private onToggle: (active: boolean) => void,
){
socket.on("control_granted", id => {
this.toggleRemoteControl(id === socket.id)
socket.on("control_granted", ({ meta, data }) => {
this.toggleRemoteControl(data === socket.id)
})
socket.on("control_rejected", id => {
id === socket.id && this.toggleRemoteControl(false)
socket.on("control_rejected", ({ meta, data }) => {
data === socket.id && this.toggleRemoteControl(false)
this.onReject()
})
socket.on('SESSION_DISCONNECTED', () => {
@ -50,14 +51,19 @@ export default class RemoteControl {
private onMouseMove = (e: MouseEvent): void => {
const data = this.screen.getInternalCoordinates(e)
this.socket.emit("move", [ data.x, data.y ])
this.emitData("move", [ data.x, data.y ])
}
private emitData = (event: string, data?: any) => {
console.log('emit data', event, data, { meta: { tabId: this.store.get().currentTab }, data })
this.socket.emit(event, { meta: { tabId: this.store.get().currentTab }, data })
}
private onWheel = (e: WheelEvent): void => {
e.preventDefault()
//throttling makes movements less smooth, so it is omitted
//this.onMouseMove(e)
this.socket.emit("scroll", [ e.deltaX, e.deltaY ])
this.emitData("scroll", [ e.deltaX, e.deltaY ])
}
public setCallbacks = ({ onReject }: { onReject: () => void }) => {
@ -76,9 +82,9 @@ export default class RemoteControl {
if (el instanceof HTMLTextAreaElement
|| el instanceof HTMLInputElement
) {
this.socket && this.socket.emit("input", el.value)
this.socket && this.emitData("input", el.value)
} else if (el.isContentEditable) {
this.socket && this.socket.emit("input", el.innerText)
this.socket && this.emitData("input", el.innerText)
}
}
// TODO: send "focus" event to assist with the nodeID
@ -92,7 +98,7 @@ export default class RemoteControl {
el.onblur = null
}
}
this.socket.emit("click", [ data.x, data.y ]);
this.emitData("click", [ data.x, data.y ]);
}
private toggleRemoteControl(enable: boolean){
@ -116,17 +122,17 @@ export default class RemoteControl {
if (remoteControl === RemoteControlStatus.Requesting) { return }
if (remoteControl === RemoteControlStatus.Disabled) {
this.store.update({ remoteControl: RemoteControlStatus.Requesting })
this.socket.emit("request_control", JSON.stringify({
...this.agentInfo,
query: document.location.search
}))
this.emitData("request_control", JSON.stringify({
...this.agentInfo,
query: document.location.search
}))
} else {
this.releaseRemoteControl()
}
}
releaseRemoteControl = () => {
this.socket.emit("release_control")
this.emitData("release_control",)
this.toggleRemoteControl(false)
}
@ -140,24 +146,24 @@ export default class RemoteControl {
const annot = this.annot = new AnnotationCanvas()
annot.mount(this.screen.overlay)
annot.canvas.addEventListener("mousedown", e => {
const data = this.screen.getInternalViewportCoordinates(e)
const data = this.screen.getInternalViewportCoordin1ates(e)
annot.start([ data.x, data.y ])
this.socket.emit("startAnnotation", [ data.x, data.y ])
this.emitData("startAnnotation", [ data.x, data.y ])
})
annot.canvas.addEventListener("mouseleave", () => {
annot.stop()
this.socket.emit("stopAnnotation")
this.emitData("stopAnnotation")
})
annot.canvas.addEventListener("mouseup", () => {
annot.stop()
this.socket.emit("stopAnnotation")
this.emitData("stopAnnotation")
})
annot.canvas.addEventListener("mousemove", e => {
if (!annot.isPainting()) { return }
const data = this.screen.getInternalViewportCoordinates(e)
annot.move([ data.x, data.y ])
this.socket.emit("moveAnnotation", [ data.x, data.y ])
this.emitData("moveAnnotation", [ data.x, data.y ])
})
this.store.update({ annotating: true })
} else if (!enable && !!this.annot) {

16
frontend/app/player/web/assist/ScreenRecording.ts
View file

@ -10,6 +10,7 @@ export enum SessionRecordingStatus {
export interface State {
recordingState: SessionRecordingStatus;
currentTab?: string;
}
export default class ScreenRecording {
@ -46,14 +47,19 @@ export default class ScreenRecording {
if (recordingState === SessionRecordingStatus.Requesting) return;
this.store.update({ recordingState: SessionRecordingStatus.Requesting })
this.socket.emit("request_recording", JSON.stringify({
...this.agentInfo,
query: document.location.search,
}))
this.emitData("request_recording", JSON.stringify({
...this.agentInfo,
query: document.location.search,
})
)
}
private emitData = (event: string, data?: any) => {
this.socket.emit(event, { meta: { tabId: this.store.get().currentTab }, data })
}
stopRecording = () => {
this.socket.emit("stop_recording")
this.emitData("stop_recording")
this.toggleRecording(false)
}

85
tracker/tracker-assist/src/Assist.ts
View file

@ -127,8 +127,8 @@ export default class Assist {
app.session.attachUpdateCallback(sessInfo => this.emit('UPDATE_SESSION', sessInfo))
}
private emit(ev: string, ...args): void {
this.socket && this.socket.emit(ev, ...args)
private emit(ev: string, args?: any): void {
this.socket && this.socket.emit(ev, { meta: { tabId: this.app.getTabId(), }, data: args, })
}
private get agentsConnected(): boolean {
@ -164,7 +164,7 @@ export default class Assist {
if (!sessionId) {
return app.debug.error('No session ID')
}
const peerID = `${app.getProjectKey()}-${sessionId}-${app.session.getTabId()}`
const peerID = `${app.getProjectKey()}-${sessionId}-${this.app.getTabId()}`
// SocketIO
const socket = this.socket = connect(this.getHost(), {
@ -172,6 +172,7 @@ export default class Assist {
query: {
'peerId': peerID,
'identity': 'session',
'tabId': this.app.getTabId(),
'sessionInfo': JSON.stringify({
pageTitle: document.title,
active: true,
@ -180,7 +181,12 @@ export default class Assist {
},
transports: ['websocket',],
})
socket.onAny((...args) => app.debug.log('Socket:', ...args))
socket.onAny((...args) => {
if (args[0] === 'messages' || args[0] === 'UPDATE_SESSION') {
return
}
app.debug.log('Socket:', ...args)
})
this.remoteControl = new RemoteControl(
this.options,
@ -197,7 +203,11 @@ export default class Assist {
annot.mount()
return callingAgents.get(id)
},
(id, isDenied) => {
(id, isDenied) => onRelease(id, isDenied),
)
const onRelease = (id, isDenied) => {
{
if (id) {
const cb = this.agents[id].onControlReleased
delete this.agents[id].onControlReleased
@ -217,8 +227,8 @@ export default class Assist {
const info = id ? this.agents[id]?.agentInfo : {}
this.options.onRemoteControlDeny?.(info || {})
}
},
)
}
}
const onAcceptRecording = () => {
socket.emit('recording_accepted')
@ -230,24 +240,37 @@ export default class Assist {
}
const recordingState = new ScreenRecordingState(this.options.recordingConfirm)
// TODO: check incoming args
socket.on('request_control', this.remoteControl.requestControl)
socket.on('release_control', this.remoteControl.releaseControl)
socket.on('scroll', this.remoteControl.scroll)
socket.on('click', this.remoteControl.click)
socket.on('move', this.remoteControl.move)
socket.on('focus', (clientID, nodeID) => {
const el = app.nodes.getNode(nodeID)
if (el instanceof HTMLElement && this.remoteControl) {
this.remoteControl.focus(clientID, el)
function processEvent(agentId: string, event: { meta: { tabId: string }, data?: any }, callback?: (id: string, data: any) => void) {
if (app.getTabId() === event.meta.tabId) {
return callback?.(agentId, event.data)
}
})
socket.on('input', this.remoteControl.input)
}
if (this.remoteControl !== null) {
socket.on('request_control', (agentId, dataObj) => {
processEvent(agentId, dataObj, this.remoteControl?.requestControl)
})
socket.on('release_control', (agentId, dataObj) => {
processEvent(agentId, dataObj, (_, data) =>
this.remoteControl?.releaseControl(data)
)
})
socket.on('scroll', (id, event) => processEvent(id, event, this.remoteControl?.scroll))
socket.on('click', (id, event) => processEvent(id, event, this.remoteControl?.click))
socket.on('move', (id, event) => processEvent(id, event, this.remoteControl?.move))
socket.on('focus', (id, event) => processEvent(id, event, (clientID, nodeID) => {
const el = app.nodes.getNode(nodeID)
if (el instanceof HTMLElement && this.remoteControl) {
this.remoteControl.focus(clientID, el)
}
}))
socket.on('input', (id, event) => processEvent(id, event, this.remoteControl?.input))
}
socket.on('moveAnnotation', (_, p) => annot && annot.move(p)) // TODO: restrict by id
socket.on('startAnnotation', (_, p) => annot && annot.start(p))
socket.on('stopAnnotation', () => annot && annot.stop())
// TODO: restrict by id
socket.on('moveAnnotation', (id, event) => processEvent(id, event, (_, d) => annot && annot.move(d)))
socket.on('startAnnotation', (id, event) => processEvent(id, event, (_, d) => annot?.start(d)))
socket.on('stopAnnotation', (id, event) => processEvent(id, event, annot?.stop))
socket.on('NEW_AGENT', (id: string, info) => {
this.agents[id] = {
@ -287,7 +310,8 @@ export default class Assist {
this.agents = {}
if (recordingState.isActive) recordingState.stopRecording()
})
socket.on('call_end', (id) => {
socket.on('call_end', (info) => {
const id = info.data
if (!callingAgents.has(id)) {
app.debug.warn('Received call_end from unknown agent', id)
return
@ -295,14 +319,20 @@ export default class Assist {
endAgentCall(id)
})
socket.on('_agent_name', (id, name) => {
socket.on('_agent_name', (id, info) => {
if (app.getTabId() !== info.meta.tabId) return
const name = info.data
callingAgents.set(id, name)
updateCallerNames()
})
socket.on('videofeed', (_, feedState) => {
socket.on('videofeed', (_, info) => {
if (app.getTabId() !== info.meta.tabId) return
const feedState = info.data
callUI?.toggleVideoStream(feedState)
})
socket.on('request_recording', (id, agentData) => {
socket.on('request_recording', (id, info) => {
if (app.getTabId() !== info.meta.tabId) return
const agentData = info.data
if (!recordingState.isActive) {
this.options.onRecordingRequest?.(JSON.parse(agentData))
recordingState.requestRecording(id, onAcceptRecording, () => onRejectRecording(agentData))
@ -310,7 +340,8 @@ export default class Assist {
this.emit('recording_busy')
}
})
socket.on('stop_recording', (id) => {
socket.on('stop_recording', (id, info) => {
if (app.getTabId() !== info.meta.tabId) return
if (recordingState.isActive) {
recordingState.stopAgentRecording(id)
}

7
tracker/tracker-assist/src/RemoteControl.ts
View file

@ -89,6 +89,9 @@ export default class RemoteControl {
}
this.mouse = new Mouse(agentName)
this.mouse.mount()
document.addEventListener('visibilitychange', () => {
if (document.hidden) this.releaseControl(false)
})
}
resetMouse = () => {
@ -97,7 +100,9 @@ export default class RemoteControl {
}
scroll = (id, d) => { id === this.agentID && this.mouse?.scroll(d) }
move = (id, xy) => { id === this.agentID && this.mouse?.move(xy) }
move = (id, xy) => {
return id === this.agentID && this.mouse?.move(xy)
}
private focused: HTMLElement | null = null
click = (id, xy) => {
if (id !== this.agentID || !this.mouse) { return }

4
tracker/tracker/src/main/app/index.ts
View file

@ -636,6 +636,10 @@ export default class App {
})
}
}
getTabId() {
return this.session.getTabId()
}
stop(stopWorker = true): void {
if (this.activityState !== ActivityState.NotActive) {
try {

7
tracker/tracker/src/main/index.ts
View file

@ -218,6 +218,13 @@ export default class API {
}
return this.app.getSessionID()
}
getTabId() {
if (this.app === null) {
return null
}
return this.app.getTabId()
}
sessionID(): string | null | undefined {
deprecationWarn("'sessionID' method", "'getSessionID' method", '/')
return this.getSessionID()