From e42c467d4ff824d22b44a8e216e8b26a2dc3661e Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 7 Feb 2023 17:13:22 +0100 Subject: [PATCH 01/29] feat(tracker): add input hesitation, change input change event handling --- tracker/tracker/src/main/modules/input.ts | 51 +++++++++++++++-------- tracker/tracker/src/main/utils.ts | 11 +++++ 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 15acecaa9..5cd74a9ba 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -1,5 +1,5 @@ import type App from '../app/index.js' -import { normSpaces, IN_BROWSER, getLabelAttribute } from '../utils.js' +import { normSpaces, IN_BROWSER, getLabelAttribute, debounce } from '../utils.js' import { hasTag } from '../app/guards.js' import { SetInputTarget, SetInputValue, SetInputChecked } from '../app/messages.gen.js' @@ -94,12 +94,14 @@ export default function (app: App, opts: Partial): void { }, opts, ) + function sendInputTarget(id: number, node: TextEditableElement): void { const label = getInputLabel(node) if (label !== '') { app.send(SetInputTarget(id, label)) } } + function sendInputValue(id: number, node: TextEditableElement | HTMLSelectElement): void { let value = node.value let inputMode: InputMode = options.defaultInputMode @@ -126,33 +128,20 @@ export default function (app: App, opts: Partial): void { value = '' break } - + // @ts-ignore if hesitationTime > 150 add it ??? + console.log(node.or_inputHesitation) app.send(SetInputValue(id, value, mask)) } const inputValues: Map = new Map() const checkableValues: Map = new Map() - const registeredTargets: Set = new Set() app.attachStopCallback(() => { inputValues.clear() checkableValues.clear() - registeredTargets.clear() }) app.ticker.attach((): void => { - inputValues.forEach((value, id) => { - const node = app.nodes.getNode(id) as HTMLInputElement - if (!node) return inputValues.delete(id) - if (value !== node.value) { - inputValues.set(id, node.value) - if (!registeredTargets.has(id)) { - registeredTargets.add(id) - sendInputTarget(id, node) - } - sendInputValue(id, node) - } - }) checkableValues.forEach((checked, id) => { const node = app.nodes.getNode(id) as HTMLInputElement if (!node) return checkableValues.delete(id) @@ -162,7 +151,11 @@ export default function (app: App, opts: Partial): void { } }) }) - app.ticker.attach(Set.prototype.clear, 100, false, registeredTargets) + + const debouncedUpdate = debounce((id: number, node: TextEditableElement) => { + sendInputTarget(id, node) + sendInputValue(id, node) + }, 125) app.nodes.attachNodeCallback( app.safe((node: Node): void => { @@ -180,6 +173,30 @@ export default function (app: App, opts: Partial): void { if (isTextEditable(node)) { inputValues.set(id, node.value) sendInputValue(id, node) + + node.addEventListener('focus', () => { + // @ts-ignore + Object.assign(node, { or_focusStart: +new Date() }) + }) + node.addEventListener('input', (e) => { + const value = (e.target as HTMLInputElement).value + if (inputValues.get(id) === '' && value !== '') { + const inputTime = +new Date() + // @ts-ignore + const hesitationTime = inputTime - node.or_focusStart + Object.assign(node, { or_inputHesitation: hesitationTime }) + } + inputValues.set(id, value) + debouncedUpdate(id, node) + }) + node.addEventListener('change', (e) => { + const value = (e.target as HTMLInputElement).value + if (inputValues.get(id) !== value) { + inputValues.set(id, value) + debouncedUpdate(id, node) + } + Object.assign(node, { or_inputHesitation: undefined, or_focusStart: undefined }) + }) return } if (isCheckable(node)) { diff --git a/tracker/tracker/src/main/utils.ts b/tracker/tracker/src/main/utils.ts index 739821ea9..daa9f74ce 100644 --- a/tracker/tracker/src/main/utils.ts +++ b/tracker/tracker/src/main/utils.ts @@ -81,3 +81,14 @@ export function hasOpenreplayAttribute(e: Element, attr: string): boolean { return false } + +export function debounce(func: (...args: any[]) => void, timeout = 125) { + let timer: NodeJS.Timeout + return (...args: any[]) => { + clearTimeout(timer) + // @ts-ignore + timer = setTimeout(() => { + func.apply(this, args) + }, timeout) + } +} From 969b192146f79151cbfc942824dbc5db20f1a720 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 7 Feb 2023 17:59:32 +0100 Subject: [PATCH 02/29] change(tracker): fix event attach and types --- tracker/tracker/src/main/app/index.ts | 5 +- tracker/tracker/src/main/modules/input.ts | 60 +++++++++++++---------- tracker/tracker/src/main/utils.ts | 2 +- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index f6d947dd9..10bbadf03 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -278,11 +278,14 @@ export default class App { listener: EventListener, useSafe = true, useCapture = true, + onlyStop = false, ): void { if (useSafe) { listener = this.safe(listener) } - this.attachStartCallback(() => target?.addEventListener(type, listener, useCapture), useSafe) + if (!onlyStop) { + this.attachStartCallback(() => target?.addEventListener(type, listener, useCapture), useSafe) + } this.attachStopCallback(() => target?.removeEventListener(type, listener, useCapture), useSafe) } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 5cd74a9ba..210a1717a 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -7,6 +7,7 @@ const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', ' // TODO: take into consideration "contenteditable" attribute type TextEditableElement = HTMLInputElement | HTMLTextAreaElement + function isTextEditable(node: any): node is TextEditableElement { if (hasTag(node, 'textarea')) { return true @@ -18,7 +19,7 @@ function isTextEditable(node: any): node is TextEditableElement { return INPUT_TYPES.includes(node.type) } -function isCheckable(node: any): node is HTMLInputElement { +function isCheckbox(node: any): node is HTMLInputElement { if (!hasTag(node, 'input')) { return false } @@ -134,22 +135,11 @@ export default function (app: App, opts: Partial): void { } const inputValues: Map = new Map() - const checkableValues: Map = new Map() + const checkboxValues: Map = new Map() app.attachStopCallback(() => { inputValues.clear() - checkableValues.clear() - }) - - app.ticker.attach((): void => { - checkableValues.forEach((checked, id) => { - const node = app.nodes.getNode(id) as HTMLInputElement - if (!node) return checkableValues.delete(id) - if (checked !== node.checked) { - checkableValues.set(id, node.checked) - app.send(SetInputChecked(id, node.checked)) - } - }) + checkboxValues.clear() }) const debouncedUpdate = debounce((id: number, node: TextEditableElement) => { @@ -166,19 +156,20 @@ export default function (app: App, opts: Partial): void { // TODO: support multiple select (?): use selectedOptions; Need send target? if (hasTag(node, 'select')) { sendInputValue(id, node) - app.attachEventListener(node, 'change', () => { + const handler = () => { sendInputValue(id, node) - }) + } + node.addEventListener('change', handler) + app.attachEventListener(node, 'change', handler, false, true, true) } + if (isTextEditable(node)) { inputValues.set(id, node.value) sendInputValue(id, node) - - node.addEventListener('focus', () => { - // @ts-ignore + const setFocus = () => { Object.assign(node, { or_focusStart: +new Date() }) - }) - node.addEventListener('input', (e) => { + } + const inputEvent = (e: InputEvent) => { const value = (e.target as HTMLInputElement).value if (inputValues.get(id) === '' && value !== '') { const inputTime = +new Date() @@ -188,20 +179,37 @@ export default function (app: App, opts: Partial): void { } inputValues.set(id, value) debouncedUpdate(id, node) - }) - node.addEventListener('change', (e) => { + } + const changeEvent = (e: InputEvent) => { const value = (e.target as HTMLInputElement).value if (inputValues.get(id) !== value) { inputValues.set(id, value) debouncedUpdate(id, node) } Object.assign(node, { or_inputHesitation: undefined, or_focusStart: undefined }) - }) + } + node.addEventListener('focus', setFocus) + node.addEventListener('input', inputEvent) + node.addEventListener('change', changeEvent) + app.attachEventListener(node, 'focus', setFocus, false, true, true) + app.attachEventListener(node, 'input', inputEvent, false, true, true) + app.attachEventListener(node, 'change', changeEvent, false, true, true) return } - if (isCheckable(node)) { - checkableValues.set(id, node.checked) + + if (isCheckbox(node)) { + checkboxValues.set(id, node.checked) app.send(SetInputChecked(id, node.checked)) + const checkboxChange = (e: InputEvent) => { + const value = (e.target as HTMLInputElement).checked + if (checkboxValues.get(id) !== value) { + checkboxValues.set(id, value) + app.send(SetInputChecked(id, value)) + } + } + node.addEventListener('change', checkboxChange) + app.attachEventListener(node, 'change', checkboxChange, false, true, true) + return } }), diff --git a/tracker/tracker/src/main/utils.ts b/tracker/tracker/src/main/utils.ts index daa9f74ce..8f99a4c74 100644 --- a/tracker/tracker/src/main/utils.ts +++ b/tracker/tracker/src/main/utils.ts @@ -86,8 +86,8 @@ export function debounce(func: (...args: any[]) => void, timeout = 125) { let timer: NodeJS.Timeout return (...args: any[]) => { clearTimeout(timer) - // @ts-ignore timer = setTimeout(() => { + // @ts-ignore func.apply(this, args) }, timeout) } From 018f618e127519e6e467e401f7291a0941999cfe Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Feb 2023 15:18:00 +0100 Subject: [PATCH 03/29] change(tracker): add mouse shake and update input hesitation message --- backend/pkg/messages/messages.go | 10 +++-- backend/pkg/messages/read-message.go | 3 ++ ee/connectors/msgcodec/messages.py | 3 +- ee/connectors/msgcodec/msgcodec.py | 1 + .../web/messages/RawMessageReader.gen.ts | 2 + frontend/app/player/web/messages/raw.gen.ts | 1 + .../app/player/web/messages/tracker.gen.ts | 4 +- mobs/messages.rb | 1 + tracker/tracker/src/common/messages.gen.ts | 1 + tracker/tracker/src/main/app/messages.gen.ts | 2 + tracker/tracker/src/main/modules/input.ts | 7 ++-- tracker/tracker/src/main/modules/mouse.ts | 41 +++++++++++++++++++ .../src/webworker/MessageEncoder.gen.ts | 2 +- 13 files changed, 68 insertions(+), 10 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index a96f98de8..f3c422af2 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -531,17 +531,19 @@ func (msg *SetInputTarget) TypeID() int { type SetInputValue struct { message - ID uint64 - Value string - Mask int64 + ID uint64 + Value string + HesitationTime int64 + Mask int64 } func (msg *SetInputValue) Encode() []byte { - buf := make([]byte, 31+len(msg.Value)) + buf := make([]byte, 41+len(msg.Value)) buf[0] = 18 p := 1 p = WriteUint(msg.ID, buf, p) p = WriteString(msg.Value, buf, p) + p = WriteInt(msg.HesitationTime, buf, p) p = WriteInt(msg.Mask, buf, p) return buf[:p] } diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index ecc00183f..56d0662aa 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -270,6 +270,9 @@ func DecodeSetInputValue(reader BytesReader) (Message, error) { if msg.Value, err = reader.ReadString(); err != nil { return nil, err } + if msg.HesitationTime, err = reader.ReadInt(); err != nil { + return nil, err + } if msg.Mask, err = reader.ReadInt(); err != nil { return nil, err } diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 54f8df955..53d993bd2 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -163,9 +163,10 @@ class SetInputTarget(Message): class SetInputValue(Message): __id__ = 18 - def __init__(self, id, value, mask): + def __init__(self, id, value, hesitation_time, mask): self.id = id self.value = value + self.hesitation_time = hesitation_time self.mask = mask diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 0ba21ea12..2b1d05a07 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -200,6 +200,7 @@ class MessageCodec(Codec): return SetInputValue( id=self.read_uint(reader), value=self.read_string(reader), + hesitation_time=self.read_int(reader), mask=self.read_int(reader) ) diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 793f609f5..50b5a1a68 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -172,11 +172,13 @@ export default class RawMessageReader extends PrimitiveReader { case 18: { const id = this.readUint(); if (id === null) { return resetPointer() } const value = this.readString(); if (value === null) { return resetPointer() } + const hesitationTime = this.readInt(); if (hesitationTime === null) { return resetPointer() } const mask = this.readInt(); if (mask === null) { return resetPointer() } return { tp: MType.SetInputValue, id, value, + hesitationTime, mask, }; } diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index b51edb40e..8e6d613a4 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -156,6 +156,7 @@ export interface RawSetInputValue { tp: MType.SetInputValue, id: number, value: string, + hesitationTime: number, mask: number, } diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 2c171011c..13f1d82e7 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -98,6 +98,7 @@ type TrSetInputValue = [ type: 18, id: number, value: string, + hesitationTime: number, mask: number, ] @@ -549,7 +550,8 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { tp: MType.SetInputValue, id: tMsg[1], value: tMsg[2], - mask: tMsg[3], + hesitationTime: tMsg[3], + mask: tMsg[4], } } diff --git a/mobs/messages.rb b/mobs/messages.rb index ef36ebfa7..c5ba732b6 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -94,6 +94,7 @@ end message 18, 'SetInputValue' do uint 'ID' string 'Value' + int 'HesitationTime' int 'Mask' end message 19, 'SetInputChecked' do diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 8f61ab917..08461e6b8 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -159,6 +159,7 @@ export type SetInputValue = [ /*type:*/ Type.SetInputValue, /*id:*/ number, /*value:*/ string, + /*hesitationTime:*/ number, /*mask:*/ number, ] diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index bdec57a0f..e8005a430 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -172,12 +172,14 @@ export function SetInputTarget( export function SetInputValue( id: number, value: string, + hesitationTime: number, mask: number, ): Messages.SetInputValue { return [ Messages.Type.SetInputValue, id, value, + hesitationTime, mask, ] } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 210a1717a..83a43313d 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -129,9 +129,10 @@ export default function (app: App, opts: Partial): void { value = '' break } - // @ts-ignore if hesitationTime > 150 add it ??? - console.log(node.or_inputHesitation) - app.send(SetInputValue(id, value, mask)) + + // @ts-ignore maybe if hesitationTime > 150 ? + const hesitationTime = node.or_inputHesitation || 0 + app.send(SetInputValue(id, value, hesitationTime, mask)) } const inputValues: Map = new Map() diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index b00d6d304..7ed64f1c4 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -107,12 +107,45 @@ export default function (app: App): void { let mouseTargetTime = 0 let selectorMap: { [id: number]: string } = {} + let velocity = 0 + let direction = 0 + let directionChangeCount = 0 + let distance = 0 + let checkIntervalId: NodeJS.Timer + const shakeThreshold = 0.005 + const shakeCheckInterval = 225 + + function checkMouseShaking() { + const nextVelocity = distance / shakeCheckInterval + + if (!velocity) { + velocity = nextVelocity + return + } + + const acceleration = (nextVelocity - velocity) / shakeCheckInterval + if (directionChangeCount && acceleration > shakeThreshold) { + console.log('Mouse shake detected!') + } + + distance = 0 + directionChangeCount = 0 + velocity = nextVelocity + } + + app.attachStartCallback(() => { + checkIntervalId = setInterval(() => checkMouseShaking(), shakeCheckInterval) + }) + app.attachStopCallback(() => { mousePositionX = -1 mousePositionY = -1 mousePositionChanged = false mouseTarget = null selectorMap = {} + if (checkIntervalId) { + clearInterval(checkIntervalId) + } }) const sendMouseMove = (): void => { @@ -146,6 +179,14 @@ export default function (app: App): void { mousePositionX = e.clientX + left mousePositionY = e.clientY + top mousePositionChanged = true + + const nextDirection = Math.sign(e.movementX) + distance += Math.abs(e.movementX) + Math.abs(e.movementY) + + if (nextDirection !== direction) { + direction = nextDirection + directionChangeCount++ + } }, false, ) diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index e6e522dd4..60079647f 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -67,7 +67,7 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.SetInputValue: - return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) + return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) && this.int(msg[4]) break case Messages.Type.SetInputChecked: From f5f7f3726065f5cff53bca23a52a794d3bed72e5 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Feb 2023 15:58:14 +0100 Subject: [PATCH 04/29] change(tracker): tune shake check --- tracker/tracker/src/main/modules/mouse.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 7ed64f1c4..c94f117fa 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -112,7 +112,7 @@ export default function (app: App): void { let directionChangeCount = 0 let distance = 0 let checkIntervalId: NodeJS.Timer - const shakeThreshold = 0.005 + const shakeThreshold = 0.008 const shakeCheckInterval = 225 function checkMouseShaking() { @@ -124,7 +124,7 @@ export default function (app: App): void { } const acceleration = (nextVelocity - velocity) / shakeCheckInterval - if (directionChangeCount && acceleration > shakeThreshold) { + if (directionChangeCount > 3 && acceleration > shakeThreshold) { console.log('Mouse shake detected!') } From 3266f464406bfd8a231d08261c894ddffe623d25 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 9 Feb 2023 16:38:13 +0100 Subject: [PATCH 05/29] fix(tracker): some events changes... --- backend/pkg/messages/filters.go | 2 +- backend/pkg/messages/messages.go | 36 ++++++++++--- backend/pkg/messages/read-message.go | 20 +++++-- ee/connectors/msgcodec/messages.py | 12 ++++- ee/connectors/msgcodec/msgcodec.py | 8 ++- .../web/messages/RawMessageReader.gen.ts | 2 - frontend/app/player/web/messages/raw.gen.ts | 1 - .../app/player/web/messages/tracker.gen.ts | 13 +++-- mobs/messages.rb | 6 ++- tracker/tracker/src/common/messages.gen.ts | 11 +++- tracker/tracker/src/main/app/index.ts | 5 +- tracker/tracker/src/main/app/messages.gen.ts | 15 +++++- tracker/tracker/src/main/modules/input.ts | 52 +++++++++---------- .../src/webworker/MessageEncoder.gen.ts | 6 ++- 14 files changed, 133 insertions(+), 56 deletions(-) diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index 30e266194..370dffb99 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,7 +2,7 @@ package messages func IsReplayerType(id int) bool { - return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id + return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 83 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id } func IsIOSType(id int) bool { diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index f3c422af2..5efb81474 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -79,6 +79,7 @@ const ( MsgBatchMeta = 80 MsgBatchMetadata = 81 MsgPartitionedMessage = 82 + MsgInputChange = 83 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 @@ -531,19 +532,17 @@ func (msg *SetInputTarget) TypeID() int { type SetInputValue struct { message - ID uint64 - Value string - HesitationTime int64 - Mask int64 + ID uint64 + Value string + Mask int64 } func (msg *SetInputValue) Encode() []byte { - buf := make([]byte, 41+len(msg.Value)) + buf := make([]byte, 31+len(msg.Value)) buf[0] = 18 p := 1 p = WriteUint(msg.ID, buf, p) p = WriteString(msg.Value, buf, p) - p = WriteInt(msg.HesitationTime, buf, p) p = WriteInt(msg.Mask, buf, p) return buf[:p] } @@ -2119,6 +2118,31 @@ func (msg *PartitionedMessage) TypeID() int { return 82 } +type InputChange struct { + message + ID uint64 + Label string + HesitationTime int64 +} + +func (msg *InputChange) Encode() []byte { + buf := make([]byte, 31+len(msg.Label)) + buf[0] = 83 + p := 1 + p = WriteUint(msg.ID, buf, p) + p = WriteString(msg.Label, buf, p) + p = WriteInt(msg.HesitationTime, buf, p) + return buf[:p] +} + +func (msg *InputChange) Decode() Message { + return msg +} + +func (msg *InputChange) TypeID() int { + return 83 +} + type IssueEvent struct { message MessageID uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 56d0662aa..907c399e6 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -270,9 +270,6 @@ func DecodeSetInputValue(reader BytesReader) (Message, error) { if msg.Value, err = reader.ReadString(); err != nil { return nil, err } - if msg.HesitationTime, err = reader.ReadInt(); err != nil { - return nil, err - } if msg.Mask, err = reader.ReadInt(); err != nil { return nil, err } @@ -1296,6 +1293,21 @@ func DecodePartitionedMessage(reader BytesReader) (Message, error) { return msg, err } +func DecodeInputChange(reader BytesReader) (Message, error) { + var err error = nil + msg := &InputChange{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Label, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.HesitationTime, err = reader.ReadInt(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1905,6 +1917,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeBatchMetadata(reader) case 82: return DecodePartitionedMessage(reader) + case 83: + return DecodeInputChange(reader) case 125: return DecodeIssueEvent(reader) case 126: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 53d993bd2..0c2777c6b 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -163,10 +163,9 @@ class SetInputTarget(Message): class SetInputValue(Message): __id__ = 18 - def __init__(self, id, value, hesitation_time, mask): + def __init__(self, id, value, mask): self.id = id self.value = value - self.hesitation_time = hesitation_time self.mask = mask @@ -745,6 +744,15 @@ class PartitionedMessage(Message): self.part_total = part_total +class InputChange(Message): + __id__ = 83 + + def __init__(self, id, label, hesitation_time): + self.id = id + self.label = label + self.hesitation_time = hesitation_time + + class IssueEvent(Message): __id__ = 125 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 2b1d05a07..861a7f1e6 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -200,7 +200,6 @@ class MessageCodec(Codec): return SetInputValue( id=self.read_uint(reader), value=self.read_string(reader), - hesitation_time=self.read_int(reader), mask=self.read_int(reader) ) @@ -661,6 +660,13 @@ class MessageCodec(Codec): part_total=self.read_uint(reader) ) + if message_id == 83: + return InputChange( + id=self.read_uint(reader), + label=self.read_string(reader), + hesitation_time=self.read_int(reader) + ) + if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 50b5a1a68..793f609f5 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -172,13 +172,11 @@ export default class RawMessageReader extends PrimitiveReader { case 18: { const id = this.readUint(); if (id === null) { return resetPointer() } const value = this.readString(); if (value === null) { return resetPointer() } - const hesitationTime = this.readInt(); if (hesitationTime === null) { return resetPointer() } const mask = this.readInt(); if (mask === null) { return resetPointer() } return { tp: MType.SetInputValue, id, value, - hesitationTime, mask, }; } diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 8e6d613a4..b51edb40e 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -156,7 +156,6 @@ export interface RawSetInputValue { tp: MType.SetInputValue, id: number, value: string, - hesitationTime: number, mask: number, } diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 13f1d82e7..a4b208e44 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -98,7 +98,6 @@ type TrSetInputValue = [ type: 18, id: number, value: string, - hesitationTime: number, mask: number, ] @@ -430,8 +429,15 @@ type TrPartitionedMessage = [ partTotal: number, ] +type TrInputChange = [ + type: 83, + id: number, + label: string, + hesitationTime: number, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -550,8 +556,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { tp: MType.SetInputValue, id: tMsg[1], value: tMsg[2], - hesitationTime: tMsg[3], - mask: tMsg[4], + mask: tMsg[3], } } diff --git a/mobs/messages.rb b/mobs/messages.rb index c5ba732b6..2838029cb 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -94,7 +94,6 @@ end message 18, 'SetInputValue' do uint 'ID' string 'Value' - int 'HesitationTime' int 'Mask' end message 19, 'SetInputChecked' do @@ -476,6 +475,11 @@ message 82, 'PartitionedMessage', :replayer => false do uint 'PartTotal' end +message 83, 'InputChange', :replayer => false do + uint 'ID' + string 'Label' + int 'HesitationTime' +end ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 08461e6b8..b9c5b5f20 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -63,6 +63,7 @@ export declare const enum Type { Zustand = 79, BatchMetadata = 81, PartitionedMessage = 82, + InputChange = 83, } @@ -159,7 +160,6 @@ export type SetInputValue = [ /*type:*/ Type.SetInputValue, /*id:*/ number, /*value:*/ string, - /*hesitationTime:*/ number, /*mask:*/ number, ] @@ -491,6 +491,13 @@ export type PartitionedMessage = [ /*partTotal:*/ number, ] +export type InputChange = [ + /*type:*/ Type.InputChange, + /*id:*/ number, + /*label:*/ string, + /*hesitationTime:*/ number, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange export default Message diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 10bbadf03..f6d947dd9 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -278,14 +278,11 @@ export default class App { listener: EventListener, useSafe = true, useCapture = true, - onlyStop = false, ): void { if (useSafe) { listener = this.safe(listener) } - if (!onlyStop) { - this.attachStartCallback(() => target?.addEventListener(type, listener, useCapture), useSafe) - } + this.attachStartCallback(() => target?.addEventListener(type, listener, useCapture), useSafe) this.attachStopCallback(() => target?.removeEventListener(type, listener, useCapture), useSafe) } diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index e8005a430..d5926d240 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -172,14 +172,12 @@ export function SetInputTarget( export function SetInputValue( id: number, value: string, - hesitationTime: number, mask: number, ): Messages.SetInputValue { return [ Messages.Type.SetInputValue, id, value, - hesitationTime, mask, ] } @@ -794,3 +792,16 @@ export function PartitionedMessage( ] } +export function InputChange( + id: number, + label: string, + hesitationTime: number, +): Messages.InputChange { + return [ + Messages.Type.InputChange, + id, + label, + hesitationTime, + ] +} + diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 83a43313d..f6b915e25 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -1,7 +1,7 @@ import type App from '../app/index.js' import { normSpaces, IN_BROWSER, getLabelAttribute, debounce } from '../utils.js' import { hasTag } from '../app/guards.js' -import { SetInputTarget, SetInputValue, SetInputChecked } from '../app/messages.gen.js' +import { InputChange, SetInputValue, SetInputChecked } from '../app/messages.gen.js' const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', 'date', 'tel'] @@ -86,6 +86,7 @@ export interface Options { } export default function (app: App, opts: Partial): void { + const inputHesitationMap: Map = new Map() const options: Options = Object.assign( { obscureInputNumbers: true, @@ -96,10 +97,12 @@ export default function (app: App, opts: Partial): void { opts, ) - function sendInputTarget(id: number, node: TextEditableElement): void { + function sendInputChange(id: number, node: TextEditableElement): void { const label = getInputLabel(node) + // @ts-ignore maybe if hesitationTime > 150 ? + const { hesitation } = inputHesitationMap.get(id) if (label !== '') { - app.send(SetInputTarget(id, label)) + app.send(InputChange(id, label, hesitation)) } } @@ -130,9 +133,7 @@ export default function (app: App, opts: Partial): void { break } - // @ts-ignore maybe if hesitationTime > 150 ? - const hesitationTime = node.or_inputHesitation || 0 - app.send(SetInputValue(id, value, hesitationTime, mask)) + app.send(SetInputValue(id, value, mask)) } const inputValues: Map = new Map() @@ -144,10 +145,14 @@ export default function (app: App, opts: Partial): void { }) const debouncedUpdate = debounce((id: number, node: TextEditableElement) => { - sendInputTarget(id, node) + sendInputChange(id, node) sendInputValue(id, node) }, 125) + const debouncedTyping = debounce((id: number, node: TextEditableElement) => { + sendInputValue(id, node) + }, 60) + app.nodes.attachNodeCallback( app.safe((node: Node): void => { const id = app.nodes.getID(node) @@ -160,26 +165,27 @@ export default function (app: App, opts: Partial): void { const handler = () => { sendInputValue(id, node) } - node.addEventListener('change', handler) - app.attachEventListener(node, 'change', handler, false, true, true) + app.nodes.attachNodeListener(node, 'change', handler) } if (isTextEditable(node)) { inputValues.set(id, node.value) + inputHesitationMap.set(id, { hesitation: 0, focusEv: 0 }) sendInputValue(id, node) const setFocus = () => { - Object.assign(node, { or_focusStart: +new Date() }) + inputHesitationMap.set(id, { hesitation: 0, focusEv: +new Date() }) } const inputEvent = (e: InputEvent) => { const value = (e.target as HTMLInputElement).value if (inputValues.get(id) === '' && value !== '') { const inputTime = +new Date() + const { focusEv } = inputHesitationMap.get(id)! // @ts-ignore - const hesitationTime = inputTime - node.or_focusStart - Object.assign(node, { or_inputHesitation: hesitationTime }) + const hesitationTime = inputTime - focusEv + inputHesitationMap.set(id, { hesitation: hesitationTime, focusEv }) } inputValues.set(id, value) - debouncedUpdate(id, node) + debouncedTyping(id, node) } const changeEvent = (e: InputEvent) => { const value = (e.target as HTMLInputElement).value @@ -187,14 +193,11 @@ export default function (app: App, opts: Partial): void { inputValues.set(id, value) debouncedUpdate(id, node) } - Object.assign(node, { or_inputHesitation: undefined, or_focusStart: undefined }) + inputHesitationMap.set(id, { hesitation: 0, focusEv: 0 }) } - node.addEventListener('focus', setFocus) - node.addEventListener('input', inputEvent) - node.addEventListener('change', changeEvent) - app.attachEventListener(node, 'focus', setFocus, false, true, true) - app.attachEventListener(node, 'input', inputEvent, false, true, true) - app.attachEventListener(node, 'change', changeEvent, false, true, true) + app.nodes.attachNodeListener(node, 'focus', setFocus) + app.nodes.attachNodeListener(node, 'input', inputEvent) + app.nodes.attachNodeListener(node, 'change', changeEvent) return } @@ -203,13 +206,10 @@ export default function (app: App, opts: Partial): void { app.send(SetInputChecked(id, node.checked)) const checkboxChange = (e: InputEvent) => { const value = (e.target as HTMLInputElement).checked - if (checkboxValues.get(id) !== value) { - checkboxValues.set(id, value) - app.send(SetInputChecked(id, value)) - } + checkboxValues.set(id, value) + app.send(SetInputChecked(id, value)) } - node.addEventListener('change', checkboxChange) - app.attachEventListener(node, 'change', checkboxChange, false, true, true) + app.nodes.attachNodeListener(node, 'change', checkboxChange) return } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 60079647f..048f61001 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -67,7 +67,7 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.SetInputValue: - return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) && this.int(msg[4]) + return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) break case Messages.Type.SetInputChecked: @@ -254,6 +254,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) break + case Messages.Type.InputChange: + return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) + break + } } From 00be3708a16bcf64476b321ff15dede83c70a466 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 10:42:03 +0100 Subject: [PATCH 06/29] change(tracker): return ticker to track custom input changes --- tracker/tracker/src/main/app/ticker.ts | 6 ++++++ tracker/tracker/src/main/modules/input.ts | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/tracker/tracker/src/main/app/ticker.ts b/tracker/tracker/src/main/app/ticker.ts index 70bffdf4c..ee05ed93c 100644 --- a/tracker/tracker/src/main/app/ticker.ts +++ b/tracker/tracker/src/main/app/ticker.ts @@ -18,6 +18,12 @@ export default class Ticker { this.callbacks = [] } + /** + * @param {Callback} callback - repeated cb + * @param {number} n - number of turn skips; ticker have a 30 ms cycle + * @param {boolean} useSafe - using safe wrapper to check if app is active + * @param {object} thisArg - link to + * */ attach(callback: Callback, n = 0, useSafe = true, thisArg?: any) { if (thisArg) { callback = callback.bind(thisArg) diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index f6b915e25..9e929e782 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -153,6 +153,25 @@ export default function (app: App, opts: Partial): void { sendInputValue(id, node) }, 60) + app.ticker.attach(() => { + inputValues.forEach((value, id) => { + const node = app.nodes.getNode(id) as HTMLInputElement + if (!node) return inputValues.delete(id) + if (value !== node.value) { + inputValues.set(id, node.value) + sendInputValue(id, node) + } + }) + checkboxValues.forEach((checked, id) => { + const node = app.nodes.getNode(id) as HTMLInputElement + if (!node) return checkboxValues.delete(id) + if (checked !== node.checked) { + checkboxValues.set(id, node.checked) + app.send(SetInputChecked(id, node.checked)) + } + }) + }, 5) + app.nodes.attachNodeCallback( app.safe((node: Node): void => { const id = app.nodes.getID(node) From c57846eab3e18cadfda29b104aaa4230d4a5bae9 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 11:09:48 +0100 Subject: [PATCH 07/29] fix(tracker): fix typo --- frontend/app/player/web/managers/DOM/DOMManager.ts | 4 ++-- frontend/app/player/web/managers/DOM/VirtualDOM.ts | 2 +- tracker/tracker-assist/src/Assist.ts | 4 ++-- tracker/tracker-fetch/src/index.ts | 4 ++-- tracker/tracker/src/main/app/index.ts | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index bb4c999ae..2bd7a6c0c 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -287,7 +287,7 @@ export default class DOMManager extends ListWalker { } return - // @depricated since 4.0.2 in favor of adopted_ss_insert/delete_rule + add_owner as being common case for StyleSheets + // @deprecated since 4.0.2 in favor of adopted_ss_insert/delete_rule + add_owner as being common case for StyleSheets case MType.CssInsertRule: vn = this.vElements.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } @@ -306,7 +306,7 @@ export default class DOMManager extends ListWalker { } vn.onStyleSheet(sheet => deleteRule(sheet, msg)) return - // end @depricated + // end @deprecated case MType.CreateIFrameDocument: vn = this.vElements.get(msg.frameID) diff --git a/frontend/app/player/web/managers/DOM/VirtualDOM.ts b/frontend/app/player/web/managers/DOM/VirtualDOM.ts index 1be12d68c..91b75eb24 100644 --- a/frontend/app/player/web/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/web/managers/DOM/VirtualDOM.ts @@ -122,7 +122,7 @@ export class VElement extends VParent { type StyleSheetCallback = (s: CSSStyleSheet) => void export type StyleElement = HTMLStyleElement | SVGStyleElement -// @Depricated TODO: remove in favor of PostponedStyleSheet +// @deprecated TODO: remove in favor of PostponedStyleSheet export class VStyleElement extends VElement { private loaded = false private stylesheetCallbacks: StyleSheetCallback[] = [] diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 646360521..8d14740a5 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -31,9 +31,9 @@ export interface Options { controlConfirm: ConfirmOptions; recordingConfirm: ConfirmOptions; - // @depricated + // @deprecated confirmText?: string; - // @depricated + // @deprecated confirmStyle?: Properties; config: RTCConfiguration; diff --git a/tracker/tracker-fetch/src/index.ts b/tracker/tracker-fetch/src/index.ts index 10b75b7ea..cb9d9e4df 100644 --- a/tracker/tracker-fetch/src/index.ts +++ b/tracker/tracker-fetch/src/index.ts @@ -28,7 +28,7 @@ export interface Options { ignoreHeaders: Array | boolean sanitiser?: (RequestResponseData) => RequestResponseData | null - // Depricated + // @deprecated requestSanitizer?: any responseSanitizer?: any } @@ -49,7 +49,7 @@ export default function(opts: Partial = {}): (app: App | null) => Windo opts, ); if (options.requestSanitizer && options.responseSanitizer) { - console.warn("OpenReplay fetch plugin: `requestSanitizer` and `responseSanitizer` options are depricated. Please, use `sanitiser` instead (check out documentation at https://docs.openreplay.com/plugins/fetch).") + console.warn("OpenReplay fetch plugin: `requestSanitizer` and `responseSanitizer` options are deprecated. Please, use `sanitiser` instead (check out documentation at https://docs.openreplay.com/plugins/fetch).") } return (app: App | null) => { diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index f6d947dd9..faca22b9d 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -149,7 +149,7 @@ export default class App { } }) - // @depricated (use sessionHash on start instead) + // @deprecated (use sessionHash on start instead) if (sessionToken != null) { this.session.applySessionHash(sessionToken) } From c2501530a001882f6ef528120c742044ad71fd64 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 12:41:01 +0100 Subject: [PATCH 08/29] fix(tracker): capture selection event --- backend/pkg/messages/filters.go | 2 +- backend/pkg/messages/messages.go | 26 +++++++++++++ backend/pkg/messages/read-message.go | 17 ++++++++ ee/connectors/msgcodec/messages.py | 9 +++++ ee/connectors/msgcodec/msgcodec.py | 7 ++++ .../web/messages/RawMessageReader.gen.ts | 12 ++++++ .../app/player/web/messages/filters.gen.ts | 2 +- .../app/player/web/messages/message.gen.ts | 3 ++ frontend/app/player/web/messages/raw.gen.ts | 10 ++++- .../player/web/messages/tracker-legacy.gen.ts | 1 + .../app/player/web/messages/tracker.gen.ts | 18 ++++++++- mobs/messages.rb | 6 +++ tracker/tracker/src/common/messages.gen.ts | 10 ++++- tracker/tracker/src/main/app/messages.gen.ts | 13 +++++++ tracker/tracker/src/main/index.ts | 2 + tracker/tracker/src/main/modules/selection.ts | 39 +++++++++++++++++++ .../src/webworker/MessageEncoder.gen.ts | 4 ++ 17 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 tracker/tracker/src/main/modules/selection.ts diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index 370dffb99..f586474d0 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -10,5 +10,5 @@ func IsIOSType(id int) bool { } func IsDOMType(id int) bool { - return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id + return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 84 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 5efb81474..2e83ba0cf 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -80,6 +80,7 @@ const ( MsgBatchMetadata = 81 MsgPartitionedMessage = 82 MsgInputChange = 83 + MsgSelectionChange = 84 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 @@ -2143,6 +2144,31 @@ func (msg *InputChange) TypeID() int { return 83 } +type SelectionChange struct { + message + SelectionStart uint64 + SelectionEnd uint64 + Selection string +} + +func (msg *SelectionChange) Encode() []byte { + buf := make([]byte, 31+len(msg.Selection)) + buf[0] = 84 + p := 1 + p = WriteUint(msg.SelectionStart, buf, p) + p = WriteUint(msg.SelectionEnd, buf, p) + p = WriteString(msg.Selection, buf, p) + return buf[:p] +} + +func (msg *SelectionChange) Decode() Message { + return msg +} + +func (msg *SelectionChange) TypeID() int { + return 84 +} + type IssueEvent struct { message MessageID uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 907c399e6..4fe1e98f7 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1308,6 +1308,21 @@ func DecodeInputChange(reader BytesReader) (Message, error) { return msg, err } +func DecodeSelectionChange(reader BytesReader) (Message, error) { + var err error = nil + msg := &SelectionChange{} + if msg.SelectionStart, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.SelectionEnd, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Selection, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1919,6 +1934,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodePartitionedMessage(reader) case 83: return DecodeInputChange(reader) + case 84: + return DecodeSelectionChange(reader) case 125: return DecodeIssueEvent(reader) case 126: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 0c2777c6b..76aee98a2 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -753,6 +753,15 @@ class InputChange(Message): self.hesitation_time = hesitation_time +class SelectionChange(Message): + __id__ = 84 + + def __init__(self, selection_start, selection_end, selection): + self.selection_start = selection_start + self.selection_end = selection_end + self.selection = selection + + class IssueEvent(Message): __id__ = 125 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 861a7f1e6..f5986ceb4 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -667,6 +667,13 @@ class MessageCodec(Codec): hesitation_time=self.read_int(reader) ) + if message_id == 84: + return SelectionChange( + selection_start=self.read_uint(reader), + selection_end=self.read_uint(reader), + selection=self.read_string(reader) + ) + if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 793f609f5..32444011a 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -627,6 +627,18 @@ export default class RawMessageReader extends PrimitiveReader { }; } + case 84: { + const selectionStart = this.readUint(); if (selectionStart === null) { return resetPointer() } + const selectionEnd = this.readUint(); if (selectionEnd === null) { return resetPointer() } + const selection = this.readString(); if (selection === null) { return resetPointer() } + return { + tp: MType.SelectionChange, + selectionStart, + selectionEnd, + selection, + }; + } + case 90: { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const projectID = this.readUint(); if (projectID === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/filters.gen.ts b/frontend/app/player/web/messages/filters.gen.ts index cd664201e..c7a6f60cd 100644 --- a/frontend/app/player/web/messages/filters.gen.ts +++ b/frontend/app/player/web/messages/filters.gen.ts @@ -3,7 +3,7 @@ import { MType } from './raw.gen' -const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,90,93,96,100,102,103,105] +const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,84,90,93,96,100,102,103,105] export function isDOMType(t: MType) { return DOM_TYPES.includes(t) } \ No newline at end of file diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index b39d9ce46..5dd11b8c3 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -55,6 +55,7 @@ import type { RawAdoptedSsAddOwner, RawAdoptedSsRemoveOwner, RawZustand, + RawSelectionChange, RawIosSessionStart, RawIosCustomEvent, RawIosScreenChanges, @@ -169,6 +170,8 @@ export type AdoptedSsRemoveOwner = RawAdoptedSsRemoveOwner & Timed export type Zustand = RawZustand & Timed +export type SelectionChange = RawSelectionChange & Timed + export type IosSessionStart = RawIosSessionStart & Timed export type IosCustomEvent = RawIosCustomEvent & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index b51edb40e..165a6f9b2 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -53,6 +53,7 @@ export const enum MType { AdoptedSsAddOwner = 76, AdoptedSsRemoveOwner = 77, Zustand = 79, + SelectionChange = 84, IosSessionStart = 90, IosCustomEvent = 93, IosScreenChanges = 96, @@ -418,6 +419,13 @@ export interface RawZustand { state: string, } +export interface RawSelectionChange { + tp: MType.SelectionChange, + selectionStart: number, + selectionEnd: number, + selection: string, +} + export interface RawIosSessionStart { tp: MType.IosSessionStart, timestamp: number, @@ -489,4 +497,4 @@ export interface RawIosNetworkCall { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index bd8ba9c85..88f54434a 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -54,6 +54,7 @@ export const TP_MAP = { 76: MType.AdoptedSsAddOwner, 77: MType.AdoptedSsRemoveOwner, 79: MType.Zustand, + 84: MType.SelectionChange, 90: MType.IosSessionStart, 93: MType.IosCustomEvent, 96: MType.IosScreenChanges, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index a4b208e44..8d8a78d2e 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -436,8 +436,15 @@ type TrInputChange = [ hesitationTime: number, ] +type TrSelectionChange = [ + type: 84, + selectionStart: number, + selectionEnd: number, + selection: string, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -874,6 +881,15 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } + case 84: { + return { + tp: MType.SelectionChange, + selectionStart: tMsg[1], + selectionEnd: tMsg[2], + selection: tMsg[3], + } + } + default: return null } diff --git a/mobs/messages.rb b/mobs/messages.rb index 2838029cb..a6b59bd7d 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -481,6 +481,12 @@ message 83, 'InputChange', :replayer => false do int 'HesitationTime' end +message 84, 'SelectionChange' do + uint 'SelectionStart' + uint 'SelectionEnd' + string 'Selection' +end + ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do uint 'MessageID' diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index b9c5b5f20..1cc05b6fa 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -64,6 +64,7 @@ export declare const enum Type { BatchMetadata = 81, PartitionedMessage = 82, InputChange = 83, + SelectionChange = 84, } @@ -498,6 +499,13 @@ export type InputChange = [ /*hesitationTime:*/ number, ] +export type SelectionChange = [ + /*type:*/ Type.SelectionChange, + /*selectionStart:*/ number, + /*selectionEnd:*/ number, + /*selection:*/ string, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index d5926d240..af8dbe357 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -805,3 +805,16 @@ export function InputChange( ] } +export function SelectionChange( + selectionStart: number, + selectionEnd: number, + selection: string, +): Messages.SelectionChange { + return [ + Messages.Type.SelectionChange, + selectionStart, + selectionEnd, + selection, + ] +} + diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index eae1c867c..0688a22bf 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -24,6 +24,7 @@ import Focus from './modules/focus.js' import Fonts from './modules/fonts.js' import Network from './modules/network.js' import ConstructedStyleSheets from './modules/constructedStyleSheets.js' +import Selection from './modules/selection.js' import { IN_BROWSER, deprecationWarn, DOCS_HOST } from './utils.js' import type { Options as AppOptions } from './app/index.js' @@ -131,6 +132,7 @@ export default class API { Focus(app) Fonts(app) Network(app, options.network) + Selection(app) ;(window as any).__OPENREPLAY__ = this if (options.autoResetOnWindowOpen) { diff --git a/tracker/tracker/src/main/modules/selection.ts b/tracker/tracker/src/main/modules/selection.ts new file mode 100644 index 000000000..cd7004c44 --- /dev/null +++ b/tracker/tracker/src/main/modules/selection.ts @@ -0,0 +1,39 @@ +import type App from '../app/index.js' +import { SelectionChange } from '../app/messages.gen.js' + +function selection(app: App) { + app.attachEventListener(document, 'selectionchange', () => { + const selection = document.getSelection() + if (selection !== null && !selection.isCollapsed) { + const selectionStart = app.nodes.getID(selection.anchorNode!) + const selectionEnd = app.nodes.getID(selection.focusNode!) + const selectedText = selection.toString().replace(/\s+/g, ' ') + if (selectionStart && selectionEnd) { + app.send(SelectionChange(selectionStart, selectionEnd, selectedText)) + } + } else { + app.send(SelectionChange(0, 0, '')) + } + }) +} + +export default selection + +/** TODO: research how to get all in-between nodes inside selection range + * including nodes between anchor and focus nodes and their children + * without recursively searching the dom tree + */ + +// if (selection.rangeCount) { +// const nodes = []; +// for (let i = 0; i < selection.rangeCount; i++) { +// const range = selection.getRangeAt(i); +// let node: Node | null = range.startContainer; +// while (node) { +// nodes.push(node); +// if (node === range.endContainer) break; +// node = node.nextSibling; +// } +// } +// // send selected nodes +// } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 048f61001..10b3e205c 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -258,6 +258,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) break + case Messages.Type.SelectionChange: + return this.uint(msg[1]) && this.uint(msg[2]) && this.string(msg[3]) + break + } } From 24cbe068356b321f296616082b86e4a189aca8a6 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 15:28:14 +0100 Subject: [PATCH 09/29] change(player): add focus event to player --- .../app/player/web/managers/DOM/DOMManager.ts | 32 ++++++++------- .../web/managers/DOM/SelectionManager.ts | 39 +++++++++++++++++++ tracker/tracker/src/main/modules/selection.ts | 2 +- 3 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 frontend/app/player/web/managers/DOM/SelectionManager.ts diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index 2bd7a6c0c..efdc29a8e 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -2,25 +2,24 @@ import logger from 'App/logger'; import type Screen from '../../Screen/Screen'; import type { Message, SetNodeScroll } from '../../messages'; - import { MType } from '../../messages'; import ListWalker from '../../../common/ListWalker'; import StylesManager, { rewriteNodeStyleSheet } from './StylesManager'; import FocusManager from './FocusManager'; -import { - VElement, - VText, - VShadowRoot, - VDocument, - VNode, - VStyleElement, - PostponedStyleSheet, -} from './VirtualDOM'; +import SelectionManager from './SelectionManager'; import type { StyleElement } from './VirtualDOM'; -import { insertRule, deleteRule } from './safeCSSRules'; +import { + PostponedStyleSheet, + VDocument, + VElement, + VNode, + VShadowRoot, + VStyleElement, + VText, +} from './VirtualDOM'; +import { deleteRule, insertRule } from './safeCSSRules'; - -type HTMLElementWithValue = HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement +type HTMLElementWithValue = HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement; const IGNORED_ATTRS = [ "autocomplete" ]; const ATTR_NAME_REGEXP = /([^\t\n\f \/>"'=]+)/; // regexp costs ~ @@ -50,7 +49,7 @@ export default class DOMManager extends ListWalker { private nodeScrollManagers: Map> = new Map() private stylesManager: StylesManager private focusManager: FocusManager = new FocusManager(this.vElements) - + private selectionManager: SelectionManager = new SelectionManager(this.vElements) constructor( private readonly screen: Screen, @@ -76,6 +75,10 @@ export default class DOMManager extends ListWalker { this.focusManager.append(m) return } + if (m.tp === MType.SelectionChange) { + this.selectionManager.append(m) + return + } if (m.tp === MType.CreateElementNode) { if(m.tag === "BODY" && this.upperBodyId === -1) { this.upperBodyId = m.id @@ -432,6 +435,7 @@ export default class DOMManager extends ListWalker { return this.stylesManager.moveReady(t).then(() => { // Apply focus this.focusManager.move(t) + this.selectionManager.move(t) // Apply all scrolls after the styles got applied this.nodeScrollManagers.forEach(manager => { const msg = manager.moveGetLast(t) diff --git a/frontend/app/player/web/managers/DOM/SelectionManager.ts b/frontend/app/player/web/managers/DOM/SelectionManager.ts new file mode 100644 index 000000000..f073b7567 --- /dev/null +++ b/frontend/app/player/web/managers/DOM/SelectionManager.ts @@ -0,0 +1,39 @@ +import type { SelectionChange } from '../../messages' +import type { VElement} from "./VirtualDOM"; +import ListWalker from '../../../common/ListWalker'; + +const SELECTION_CLASS = { start: "-openreplay-selection-start", end: "-openreplay-selection-end" } + + +export default class SelectionManager extends ListWalker { + constructor(private readonly vElements: Map) { + super(); + } + + private selected: [Element | null, Element | null] = [null, null] + + move(t: number) { + const msg = this.moveGetLast(t) + if (!msg) { return } + console.log(msg) + if (msg.selectionStart <= 0) { + this.selected[0]?.classList.remove(SELECTION_CLASS.start) + this.selected[0].style.border = 'unset' + this.selected[1]?.classList.remove(SELECTION_CLASS.end) + this.selected = [null, null] + return; + } + const startVNode = this.vElements.get(msg.selectionStart -1) + const endVNode = this.vElements.get(msg.selectionEnd -1) + + console.log(startVNode, endVNode, this.vElements) + + if (startVNode && endVNode) { + this.selected = [startVNode.node, endVNode.node] + + this.selected[0]?.classList.add(SELECTION_CLASS.start) + this.selected[0].style.border = '5px solid red' + this.selected[1]?.classList.add(SELECTION_CLASS.end) + } + } +} \ No newline at end of file diff --git a/tracker/tracker/src/main/modules/selection.ts b/tracker/tracker/src/main/modules/selection.ts index cd7004c44..8a422fedc 100644 --- a/tracker/tracker/src/main/modules/selection.ts +++ b/tracker/tracker/src/main/modules/selection.ts @@ -12,7 +12,7 @@ function selection(app: App) { app.send(SelectionChange(selectionStart, selectionEnd, selectedText)) } } else { - app.send(SelectionChange(0, 0, '')) + app.send(SelectionChange(-1, -1, '')) } }) } From 2cd74dcad100633c859125dc8ea6f0bd1ece254f Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 16:36:29 +0100 Subject: [PATCH 10/29] change(player): add visual styles for selection event --- .../app/player/web/managers/DOM/DOMManager.ts | 3 +- .../player/web/managers/DOM/FocusManager.ts | 1 - .../web/managers/DOM/SelectionManager.ts | 75 +++++++++++++------ .../web/managers/DOM/selection.module.css | 7 ++ 4 files changed, 61 insertions(+), 25 deletions(-) create mode 100644 frontend/app/player/web/managers/DOM/selection.module.css diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index efdc29a8e..7b773860c 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -49,7 +49,7 @@ export default class DOMManager extends ListWalker { private nodeScrollManagers: Map> = new Map() private stylesManager: StylesManager private focusManager: FocusManager = new FocusManager(this.vElements) - private selectionManager: SelectionManager = new SelectionManager(this.vElements) + private selectionManager: SelectionManager constructor( private readonly screen: Screen, @@ -58,6 +58,7 @@ export default class DOMManager extends ListWalker { setCssLoading: ConstructorParameters[1], ) { super() + this.selectionManager = new SelectionManager(this.vElements, screen) this.stylesManager = new StylesManager(screen, setCssLoading) } diff --git a/frontend/app/player/web/managers/DOM/FocusManager.ts b/frontend/app/player/web/managers/DOM/FocusManager.ts index 174335473..c75f3ddc3 100644 --- a/frontend/app/player/web/managers/DOM/FocusManager.ts +++ b/frontend/app/player/web/managers/DOM/FocusManager.ts @@ -3,7 +3,6 @@ import type { SetNodeFocus } from '../../messages'; import type { VElement } from './VirtualDOM'; import ListWalker from '../../../common/ListWalker'; - const FOCUS_CLASS = "-openreplay-focus" export default class FocusManager extends ListWalker { diff --git a/frontend/app/player/web/managers/DOM/SelectionManager.ts b/frontend/app/player/web/managers/DOM/SelectionManager.ts index f073b7567..986fab98b 100644 --- a/frontend/app/player/web/managers/DOM/SelectionManager.ts +++ b/frontend/app/player/web/managers/DOM/SelectionManager.ts @@ -1,39 +1,68 @@ -import type { SelectionChange } from '../../messages' -import type { VElement} from "./VirtualDOM"; +import type { SelectionChange } from '../../messages'; +import type { VElement } from './VirtualDOM'; import ListWalker from '../../../common/ListWalker'; - -const SELECTION_CLASS = { start: "-openreplay-selection-start", end: "-openreplay-selection-end" } - +import Screen from 'Player/web/Screen/Screen'; export default class SelectionManager extends ListWalker { - constructor(private readonly vElements: Map) { + constructor(private readonly vElements: Map, private readonly screen: Screen) { super(); } - private selected: [Element | null, Element | null] = [null, null] + private selected: [{ id: number, node: Element } | null, { id: number, node: Element } | null] = [null, null]; + + clearSelection() { + this.selected[0] && this.screen.overlay.removeChild(this.selected[0].node) && this.selected[0].node.remove(); + this.selected[1] && this.screen.overlay.removeChild(this.selected[1].node) && this.selected[1].node.remove(); + this.selected = [null, null]; + } move(t: number) { - const msg = this.moveGetLast(t) - if (!msg) { return } - console.log(msg) - if (msg.selectionStart <= 0) { - this.selected[0]?.classList.remove(SELECTION_CLASS.start) - this.selected[0].style.border = 'unset' - this.selected[1]?.classList.remove(SELECTION_CLASS.end) - this.selected = [null, null] + const msg = this.moveGetLast(t); + if (!msg) { return; } - const startVNode = this.vElements.get(msg.selectionStart -1) - const endVNode = this.vElements.get(msg.selectionEnd -1) + // in theory: empty selection or selection removed + if (msg.selectionStart <= 0) { + this.clearSelection() + return; + } + // preventing clones + if (this.selected[0] && this.selected[0].id === msg.selectionStart) return; - console.log(startVNode, endVNode, this.vElements) + const startVNode = this.vElements.get(msg.selectionStart - 1); + const endVNode = this.vElements.get(msg.selectionEnd - 1); + + // only one selection present on page at the same time + if (this.selected[0] && this.selected[0]?.id !== msg.selectionStart) this.clearSelection() if (startVNode && endVNode) { - this.selected = [startVNode.node, endVNode.node] + const startCoords = startVNode.node.getBoundingClientRect(); + const endCoords = endVNode.node.getBoundingClientRect(); - this.selected[0]?.classList.add(SELECTION_CLASS.start) - this.selected[0].style.border = '5px solid red' - this.selected[1]?.classList.add(SELECTION_CLASS.end) + const startPointer = document.createElement('div'); + const endPointer = document.createElement('div'); + + Object.assign(endPointer.style, { + top: endCoords.top + 'px', + left: (endCoords.left + endCoords.width + 3) + 'px', + width: '3px', + height: endCoords.height + 'px', + border: '3px solid red', + position: 'absolute', + }); + Object.assign(startPointer.style, { + top: startCoords.top + 'px', + left: (startCoords.left - 3) + 'px', + width: '3px', + height: startCoords.height + 'px', + border: '3px solid red', + position: 'absolute', + }); + + this.screen.overlay.appendChild(startPointer); + this.screen.overlay.appendChild(endPointer); + + this.selected = [{ id: msg.selectionStart, node: startPointer }, { id: msg.selectionEnd, node: endPointer }]; } } -} \ No newline at end of file +} diff --git a/frontend/app/player/web/managers/DOM/selection.module.css b/frontend/app/player/web/managers/DOM/selection.module.css new file mode 100644 index 000000000..c677827bf --- /dev/null +++ b/frontend/app/player/web/managers/DOM/selection.module.css @@ -0,0 +1,7 @@ +.openreplay-selection-start { + border: 2px solid red; +} + +.openreplay-selection-end { + border: 2px solid red; +} \ No newline at end of file From 72785e7ede1716b97bbec01dd1b07d0148fd11c6 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Sun, 12 Feb 2023 18:34:22 +0100 Subject: [PATCH 11/29] fix(tracker): simplify input module logic --- tracker/tracker/src/main/modules/input.ts | 121 +++++++++------------- tracker/tracker/src/main/utils.ts | 11 -- 2 files changed, 50 insertions(+), 82 deletions(-) diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 9e929e782..2b92620e8 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -1,14 +1,14 @@ import type App from '../app/index.js' -import { normSpaces, IN_BROWSER, getLabelAttribute, debounce } from '../utils.js' +import { normSpaces, IN_BROWSER, getLabelAttribute, now } from '../utils.js' import { hasTag } from '../app/guards.js' import { InputChange, SetInputValue, SetInputChecked } from '../app/messages.gen.js' const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', 'date', 'tel'] // TODO: take into consideration "contenteditable" attribute -type TextEditableElement = HTMLInputElement | HTMLTextAreaElement +type TextFeildElement = HTMLInputElement | HTMLTextAreaElement -function isTextEditable(node: any): node is TextEditableElement { +function isTextFeildElement(node: Node): node is TextFeildElement { if (hasTag(node, 'textarea')) { return true } @@ -19,7 +19,7 @@ function isTextEditable(node: any): node is TextEditableElement { return INPUT_TYPES.includes(node.type) } -function isCheckbox(node: any): node is HTMLInputElement { +function isCheckbox(node: Node): node is HTMLInputElement & { type: 'checkbox' | 'radio' } { if (!hasTag(node, 'input')) { return false } @@ -27,7 +27,7 @@ function isCheckbox(node: any): node is HTMLInputElement { return type === 'checkbox' || type === 'radio' } -const labelElementFor: (element: TextEditableElement) => HTMLLabelElement | undefined = +const labelElementFor: (element: TextFeildElement) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype ? (node) => { let p: Node | null = node @@ -57,7 +57,7 @@ const labelElementFor: (element: TextEditableElement) => HTMLLabelElement | unde } } -export function getInputLabel(node: TextEditableElement): string { +export function getInputLabel(node: TextFeildElement): string { let label = getLabelAttribute(node) if (label === null) { const labelElement = labelElementFor(node) @@ -86,7 +86,6 @@ export interface Options { } export default function (app: App, opts: Partial): void { - const inputHesitationMap: Map = new Map() const options: Options = Object.assign( { obscureInputNumbers: true, @@ -97,16 +96,7 @@ export default function (app: App, opts: Partial): void { opts, ) - function sendInputChange(id: number, node: TextEditableElement): void { - const label = getInputLabel(node) - // @ts-ignore maybe if hesitationTime > 150 ? - const { hesitation } = inputHesitationMap.get(id) - if (label !== '') { - app.send(InputChange(id, label, hesitation)) - } - } - - function sendInputValue(id: number, node: TextEditableElement | HTMLSelectElement): void { + function sendInputValue(id: number, node: TextFeildElement | HTMLSelectElement): void { let value = node.value let inputMode: InputMode = options.defaultInputMode @@ -144,92 +134,81 @@ export default function (app: App, opts: Partial): void { checkboxValues.clear() }) - const debouncedUpdate = debounce((id: number, node: TextEditableElement) => { - sendInputChange(id, node) + function trackInputValue(id: number, node: TextFeildElement) { + if (inputValues.get(id) === node.value) { + return + } + inputValues.set(id, node.value) sendInputValue(id, node) - }, 125) + } - const debouncedTyping = debounce((id: number, node: TextEditableElement) => { - sendInputValue(id, node) - }, 60) + function trackCheckboxValue(id: number, value: boolean) { + if (checkboxValues.get(id) === value) { + return + } + checkboxValues.set(id, value) + app.send(SetInputChecked(id, value)) + } + // The only way (to our knowladge) to track all kinds of input changes, including those made by JS app.ticker.attach(() => { inputValues.forEach((value, id) => { const node = app.nodes.getNode(id) as HTMLInputElement if (!node) return inputValues.delete(id) - if (value !== node.value) { - inputValues.set(id, node.value) - sendInputValue(id, node) - } + trackInputValue(id, node) }) checkboxValues.forEach((checked, id) => { const node = app.nodes.getNode(id) as HTMLInputElement if (!node) return checkboxValues.delete(id) - if (checked !== node.checked) { - checkboxValues.set(id, node.checked) - app.send(SetInputChecked(id, node.checked)) - } + trackCheckboxValue(id, node.checked) }) }, 5) + function sendInputChange(id: number, node: TextFeildElement, hesitationTime: number) { + trackInputValue(id, node) + const label = getInputLabel(node) + app.send(InputChange(id, label, hesitationTime)) + } + app.nodes.attachNodeCallback( app.safe((node: Node): void => { const id = app.nodes.getID(node) if (id === undefined) { return } - // TODO: support multiple select (?): use selectedOptions; Need send target? + + // TODO: support multiple select (?): use selectedOptions; if (hasTag(node, 'select')) { sendInputValue(id, node) - const handler = () => { - sendInputValue(id, node) - } - app.nodes.attachNodeListener(node, 'change', handler) + app.nodes.attachNodeListener(node, 'change', () => sendInputValue(id, node)) } - if (isTextEditable(node)) { - inputValues.set(id, node.value) - inputHesitationMap.set(id, { hesitation: 0, focusEv: 0 }) - sendInputValue(id, node) - const setFocus = () => { - inputHesitationMap.set(id, { hesitation: 0, focusEv: +new Date() }) + if (isTextFeildElement(node)) { + trackInputValue(id, node) + let nodeFocusTime = 0 + let nodeHesitationTime = 0 + const onFocus = () => { + nodeFocusTime = now() } - const inputEvent = (e: InputEvent) => { - const value = (e.target as HTMLInputElement).value - if (inputValues.get(id) === '' && value !== '') { - const inputTime = +new Date() - const { focusEv } = inputHesitationMap.get(id)! - // @ts-ignore - const hesitationTime = inputTime - focusEv - inputHesitationMap.set(id, { hesitation: hesitationTime, focusEv }) + const onInput = () => { + const value = node.value + if (nodeHesitationTime === 0) { + nodeHesitationTime = nodeFocusTime - now() } - inputValues.set(id, value) - debouncedTyping(id, node) } - const changeEvent = (e: InputEvent) => { - const value = (e.target as HTMLInputElement).value - if (inputValues.get(id) !== value) { - inputValues.set(id, value) - debouncedUpdate(id, node) - } - inputHesitationMap.set(id, { hesitation: 0, focusEv: 0 }) + const onChange = () => { + sendInputChange(id, node, nodeHesitationTime) + nodeHesitationTime = 0 } - app.nodes.attachNodeListener(node, 'focus', setFocus) - app.nodes.attachNodeListener(node, 'input', inputEvent) - app.nodes.attachNodeListener(node, 'change', changeEvent) + app.nodes.attachNodeListener(node, 'focus', onFocus) + app.nodes.attachNodeListener(node, 'input', onInput) + app.nodes.attachNodeListener(node, 'change', onChange) return } if (isCheckbox(node)) { - checkboxValues.set(id, node.checked) - app.send(SetInputChecked(id, node.checked)) - const checkboxChange = (e: InputEvent) => { - const value = (e.target as HTMLInputElement).checked - checkboxValues.set(id, value) - app.send(SetInputChecked(id, value)) - } - app.nodes.attachNodeListener(node, 'change', checkboxChange) - + trackCheckboxValue(id, node.checked) + app.nodes.attachNodeListener(node, 'change', (e) => trackCheckboxValue(id, node.checked)) return } }), diff --git a/tracker/tracker/src/main/utils.ts b/tracker/tracker/src/main/utils.ts index 8f99a4c74..739821ea9 100644 --- a/tracker/tracker/src/main/utils.ts +++ b/tracker/tracker/src/main/utils.ts @@ -81,14 +81,3 @@ export function hasOpenreplayAttribute(e: Element, attr: string): boolean { return false } - -export function debounce(func: (...args: any[]) => void, timeout = 125) { - let timer: NodeJS.Timeout - return (...args: any[]) => { - clearTimeout(timer) - timer = setTimeout(() => { - // @ts-ignore - func.apply(this, args) - }, timeout) - } -} From e9848bf3351bfc7774cc24aeb699ea0f18c874c8 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 09:57:43 +0100 Subject: [PATCH 12/29] change(player): fix selection borders --- frontend/app/player/web/managers/DOM/SelectionManager.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/app/player/web/managers/DOM/SelectionManager.ts b/frontend/app/player/web/managers/DOM/SelectionManager.ts index 986fab98b..9cf1fd21f 100644 --- a/frontend/app/player/web/managers/DOM/SelectionManager.ts +++ b/frontend/app/player/web/managers/DOM/SelectionManager.ts @@ -9,11 +9,15 @@ export default class SelectionManager extends ListWalker { } private selected: [{ id: number, node: Element } | null, { id: number, node: Element } | null] = [null, null]; + private markers: Element[] = [] clearSelection() { this.selected[0] && this.screen.overlay.removeChild(this.selected[0].node) && this.selected[0].node.remove(); this.selected[1] && this.screen.overlay.removeChild(this.selected[1].node) && this.selected[1].node.remove(); + this.markers.forEach(marker => marker.remove()) + this.selected = [null, null]; + this.markers = []; } move(t: number) { @@ -27,7 +31,7 @@ export default class SelectionManager extends ListWalker { return; } // preventing clones - if (this.selected[0] && this.selected[0].id === msg.selectionStart) return; + if ((this.selected[0] && this.selected[0].id === msg.selectionStart) && (this.selected[1] && this.selected[1].id === msg.selectionEnd)) return; const startVNode = this.vElements.get(msg.selectionStart - 1); const endVNode = this.vElements.get(msg.selectionEnd - 1); @@ -59,6 +63,7 @@ export default class SelectionManager extends ListWalker { position: 'absolute', }); + this.markers.push(startPointer, endPointer); this.screen.overlay.appendChild(startPointer); this.screen.overlay.appendChild(endPointer); From 9628bcdfadbd1751af51c08382fb2c7a7992e3a6 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 11:23:18 +0100 Subject: [PATCH 13/29] change(tracker): add more events to input, fix typo and timestamp calc --- backend/pkg/messages/filters.go | 4 +- backend/pkg/messages/messages.go | 110 +++++++++--------- backend/pkg/messages/read-message.go | 77 ++++++------ ee/connectors/msgcodec/messages.py | 39 ++++--- ee/connectors/msgcodec/msgcodec.py | 31 ++--- .../web/messages/RawMessageReader.gen.ts | 2 +- .../app/player/web/messages/filters.gen.ts | 2 +- frontend/app/player/web/messages/raw.gen.ts | 2 +- .../player/web/messages/tracker-legacy.gen.ts | 2 +- .../app/player/web/messages/tracker.gen.ts | 9 +- mobs/messages.rb | 29 +++-- tracker/tracker/src/common/messages.gen.ts | 7 +- tracker/tracker/src/main/app/messages.gen.ts | 6 + tracker/tracker/src/main/modules/input.ts | 48 +++++--- .../src/webworker/MessageEncoder.gen.ts | 2 +- 15 files changed, 213 insertions(+), 157 deletions(-) diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index f586474d0..a67e8c43f 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,7 +2,7 @@ package messages func IsReplayerType(id int) bool { - return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 83 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id + return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 112 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id } func IsIOSType(id int) bool { @@ -10,5 +10,5 @@ func IsIOSType(id int) bool { } func IsDOMType(id int) bool { - return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 84 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id + return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 2e83ba0cf..6e5bbcfeb 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -79,11 +79,11 @@ const ( MsgBatchMeta = 80 MsgBatchMetadata = 81 MsgPartitionedMessage = 82 - MsgInputChange = 83 - MsgSelectionChange = 84 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 + MsgInputChange = 112 + MsgSelectionChange = 113 MsgIOSBatchMeta = 107 MsgIOSSessionStart = 90 MsgIOSSessionEnd = 91 @@ -2119,56 +2119,6 @@ func (msg *PartitionedMessage) TypeID() int { return 82 } -type InputChange struct { - message - ID uint64 - Label string - HesitationTime int64 -} - -func (msg *InputChange) Encode() []byte { - buf := make([]byte, 31+len(msg.Label)) - buf[0] = 83 - p := 1 - p = WriteUint(msg.ID, buf, p) - p = WriteString(msg.Label, buf, p) - p = WriteInt(msg.HesitationTime, buf, p) - return buf[:p] -} - -func (msg *InputChange) Decode() Message { - return msg -} - -func (msg *InputChange) TypeID() int { - return 83 -} - -type SelectionChange struct { - message - SelectionStart uint64 - SelectionEnd uint64 - Selection string -} - -func (msg *SelectionChange) Encode() []byte { - buf := make([]byte, 31+len(msg.Selection)) - buf[0] = 84 - p := 1 - p = WriteUint(msg.SelectionStart, buf, p) - p = WriteUint(msg.SelectionEnd, buf, p) - p = WriteString(msg.Selection, buf, p) - return buf[:p] -} - -func (msg *SelectionChange) Decode() Message { - return msg -} - -func (msg *SelectionChange) TypeID() int { - return 84 -} - type IssueEvent struct { message MessageID uint64 @@ -2248,6 +2198,62 @@ func (msg *SessionSearch) TypeID() int { return 127 } +type InputChange struct { + message + ID uint64 + Value string + ValueMasked bool + Label string + HesitationTime int64 + InputDuration int64 +} + +func (msg *InputChange) Encode() []byte { + buf := make([]byte, 61+len(msg.Value)+len(msg.Label)) + buf[0] = 112 + p := 1 + p = WriteUint(msg.ID, buf, p) + p = WriteString(msg.Value, buf, p) + p = WriteBoolean(msg.ValueMasked, buf, p) + p = WriteString(msg.Label, buf, p) + p = WriteInt(msg.HesitationTime, buf, p) + p = WriteInt(msg.InputDuration, buf, p) + return buf[:p] +} + +func (msg *InputChange) Decode() Message { + return msg +} + +func (msg *InputChange) TypeID() int { + return 112 +} + +type SelectionChange struct { + message + SelectionStart uint64 + SelectionEnd uint64 + Selection string +} + +func (msg *SelectionChange) Encode() []byte { + buf := make([]byte, 31+len(msg.Selection)) + buf[0] = 113 + p := 1 + p = WriteUint(msg.SelectionStart, buf, p) + p = WriteUint(msg.SelectionEnd, buf, p) + p = WriteString(msg.Selection, buf, p) + return buf[:p] +} + +func (msg *SelectionChange) Decode() Message { + return msg +} + +func (msg *SelectionChange) TypeID() int { + return 113 +} + type IOSBatchMeta struct { message Timestamp uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 4fe1e98f7..80a365168 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1293,36 +1293,6 @@ func DecodePartitionedMessage(reader BytesReader) (Message, error) { return msg, err } -func DecodeInputChange(reader BytesReader) (Message, error) { - var err error = nil - msg := &InputChange{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.HesitationTime, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err -} - -func DecodeSelectionChange(reader BytesReader) (Message, error) { - var err error = nil - msg := &SelectionChange{} - if msg.SelectionStart, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.SelectionEnd, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Selection, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err -} - func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1374,6 +1344,45 @@ func DecodeSessionSearch(reader BytesReader) (Message, error) { return msg, err } +func DecodeInputChange(reader BytesReader) (Message, error) { + var err error = nil + msg := &InputChange{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Value, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.ValueMasked, err = reader.ReadBoolean(); err != nil { + return nil, err + } + if msg.Label, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.HesitationTime, err = reader.ReadInt(); err != nil { + return nil, err + } + if msg.InputDuration, err = reader.ReadInt(); err != nil { + return nil, err + } + return msg, err +} + +func DecodeSelectionChange(reader BytesReader) (Message, error) { + var err error = nil + msg := &SelectionChange{} + if msg.SelectionStart, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.SelectionEnd, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Selection, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIOSBatchMeta(reader BytesReader) (Message, error) { var err error = nil msg := &IOSBatchMeta{} @@ -1932,16 +1941,16 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeBatchMetadata(reader) case 82: return DecodePartitionedMessage(reader) - case 83: - return DecodeInputChange(reader) - case 84: - return DecodeSelectionChange(reader) case 125: return DecodeIssueEvent(reader) case 126: return DecodeSessionEnd(reader) case 127: return DecodeSessionSearch(reader) + case 112: + return DecodeInputChange(reader) + case 113: + return DecodeSelectionChange(reader) case 107: return DecodeIOSBatchMeta(reader) case 90: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 76aee98a2..2ed4cf49d 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -744,24 +744,6 @@ class PartitionedMessage(Message): self.part_total = part_total -class InputChange(Message): - __id__ = 83 - - def __init__(self, id, label, hesitation_time): - self.id = id - self.label = label - self.hesitation_time = hesitation_time - - -class SelectionChange(Message): - __id__ = 84 - - def __init__(self, selection_start, selection_end, selection): - self.selection_start = selection_start - self.selection_end = selection_end - self.selection = selection - - class IssueEvent(Message): __id__ = 125 @@ -791,6 +773,27 @@ class SessionSearch(Message): self.partition = partition +class InputChange(Message): + __id__ = 112 + + def __init__(self, id, value, value_masked, label, hesitation_time, input_duration): + self.id = id + self.value = value + self.value_masked = value_masked + self.label = label + self.hesitation_time = hesitation_time + self.input_duration = input_duration + + +class SelectionChange(Message): + __id__ = 113 + + def __init__(self, selection_start, selection_end, selection): + self.selection_start = selection_start + self.selection_end = selection_end + self.selection = selection + + class IOSBatchMeta(Message): __id__ = 107 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index f5986ceb4..272f2efe2 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -660,20 +660,6 @@ class MessageCodec(Codec): part_total=self.read_uint(reader) ) - if message_id == 83: - return InputChange( - id=self.read_uint(reader), - label=self.read_string(reader), - hesitation_time=self.read_int(reader) - ) - - if message_id == 84: - return SelectionChange( - selection_start=self.read_uint(reader), - selection_end=self.read_uint(reader), - selection=self.read_string(reader) - ) - if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), @@ -697,6 +683,23 @@ class MessageCodec(Codec): partition=self.read_uint(reader) ) + if message_id == 112: + return InputChange( + id=self.read_uint(reader), + value=self.read_string(reader), + value_masked=self.read_boolean(reader), + label=self.read_string(reader), + hesitation_time=self.read_int(reader), + input_duration=self.read_int(reader) + ) + + if message_id == 113: + return SelectionChange( + selection_start=self.read_uint(reader), + selection_end=self.read_uint(reader), + selection=self.read_string(reader) + ) + if message_id == 107: return IOSBatchMeta( timestamp=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 32444011a..aeec66f67 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -627,7 +627,7 @@ export default class RawMessageReader extends PrimitiveReader { }; } - case 84: { + case 113: { const selectionStart = this.readUint(); if (selectionStart === null) { return resetPointer() } const selectionEnd = this.readUint(); if (selectionEnd === null) { return resetPointer() } const selection = this.readString(); if (selection === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/filters.gen.ts b/frontend/app/player/web/messages/filters.gen.ts index c7a6f60cd..3e41afc0d 100644 --- a/frontend/app/player/web/messages/filters.gen.ts +++ b/frontend/app/player/web/messages/filters.gen.ts @@ -3,7 +3,7 @@ import { MType } from './raw.gen' -const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,84,90,93,96,100,102,103,105] +const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,90,93,96,100,102,103,105] export function isDOMType(t: MType) { return DOM_TYPES.includes(t) } \ No newline at end of file diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 165a6f9b2..b399dde79 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -53,7 +53,7 @@ export const enum MType { AdoptedSsAddOwner = 76, AdoptedSsRemoveOwner = 77, Zustand = 79, - SelectionChange = 84, + SelectionChange = 113, IosSessionStart = 90, IosCustomEvent = 93, IosScreenChanges = 96, diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index 88f54434a..8005890ab 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -54,7 +54,7 @@ export const TP_MAP = { 76: MType.AdoptedSsAddOwner, 77: MType.AdoptedSsRemoveOwner, 79: MType.Zustand, - 84: MType.SelectionChange, + 113: MType.SelectionChange, 90: MType.IosSessionStart, 93: MType.IosCustomEvent, 96: MType.IosScreenChanges, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 8d8a78d2e..ef11afff2 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -430,14 +430,17 @@ type TrPartitionedMessage = [ ] type TrInputChange = [ - type: 83, + type: 112, id: number, + value: string, + valueMasked: boolean, label: string, hesitationTime: number, + inputDuration: number, ] type TrSelectionChange = [ - type: 84, + type: 113, selectionStart: number, selectionEnd: number, selection: string, @@ -881,7 +884,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } - case 84: { + case 113: { return { tp: MType.SelectionChange, selectionStart: tMsg[1], diff --git a/mobs/messages.rb b/mobs/messages.rb index a6b59bd7d..cb27a1db2 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -475,17 +475,7 @@ message 82, 'PartitionedMessage', :replayer => false do uint 'PartTotal' end -message 83, 'InputChange', :replayer => false do - uint 'ID' - string 'Label' - int 'HesitationTime' -end - -message 84, 'SelectionChange' do - uint 'SelectionStart' - uint 'SelectionEnd' - string 'Selection' -end +# 90-111 reserved iOS ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do @@ -505,3 +495,20 @@ message 127, 'SessionSearch', :tracker => false, :replayer => false do uint 'Timestamp' uint 'Partition' end + +# since tracker 4.1.10 + +message 112, 'InputChange', :replayer => false do + uint 'ID' + string 'Value' + boolean 'ValueMasked' + string 'Label' + int 'HesitationTime' + int 'InputDuration' +end + +message 113, 'SelectionChange' do + uint 'SelectionStart' + uint 'SelectionEnd' + string 'Selection' +end \ No newline at end of file diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 1cc05b6fa..8dfb0b2de 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -63,8 +63,8 @@ export declare const enum Type { Zustand = 79, BatchMetadata = 81, PartitionedMessage = 82, - InputChange = 83, - SelectionChange = 84, + InputChange = 112, + SelectionChange = 113, } @@ -495,8 +495,11 @@ export type PartitionedMessage = [ export type InputChange = [ /*type:*/ Type.InputChange, /*id:*/ number, + /*value:*/ string, + /*valueMasked:*/ boolean, /*label:*/ string, /*hesitationTime:*/ number, + /*inputDuration:*/ number, ] export type SelectionChange = [ diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index af8dbe357..b6ac90706 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -794,14 +794,20 @@ export function PartitionedMessage( export function InputChange( id: number, + value: string, + valueMasked: boolean, label: string, hesitationTime: number, + inputDuration: number, ): Messages.InputChange { return [ Messages.Type.InputChange, id, + value, + valueMasked, label, hesitationTime, + inputDuration, ] } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 2b92620e8..90463b21e 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -6,9 +6,9 @@ import { InputChange, SetInputValue, SetInputChecked } from '../app/messages.gen const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', 'date', 'tel'] // TODO: take into consideration "contenteditable" attribute -type TextFeildElement = HTMLInputElement | HTMLTextAreaElement +type TextFieldElement = HTMLInputElement | HTMLTextAreaElement -function isTextFeildElement(node: Node): node is TextFeildElement { +function isTextFieldElement(node: Node): node is TextFieldElement { if (hasTag(node, 'textarea')) { return true } @@ -27,7 +27,7 @@ function isCheckbox(node: Node): node is HTMLInputElement & { type: 'checkbox' | return type === 'checkbox' || type === 'radio' } -const labelElementFor: (element: TextFeildElement) => HTMLLabelElement | undefined = +const labelElementFor: (element: TextFieldElement) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype ? (node) => { let p: Node | null = node @@ -57,7 +57,7 @@ const labelElementFor: (element: TextFeildElement) => HTMLLabelElement | undefin } } -export function getInputLabel(node: TextFeildElement): string { +export function getInputLabel(node: TextFieldElement): string { let label = getLabelAttribute(node) if (label === null) { const labelElement = labelElementFor(node) @@ -96,7 +96,7 @@ export default function (app: App, opts: Partial): void { opts, ) - function sendInputValue(id: number, node: TextFeildElement | HTMLSelectElement): void { + function getInputValue(id: number, node: TextFieldElement | HTMLSelectElement) { let value = node.value let inputMode: InputMode = options.defaultInputMode @@ -123,6 +123,11 @@ export default function (app: App, opts: Partial): void { break } + return { value, mask } + } + function sendInputValue(id: number, node: TextFieldElement | HTMLSelectElement): void { + const { value, mask } = getInputValue(id, node) + app.send(SetInputValue(id, value, mask)) } @@ -134,7 +139,7 @@ export default function (app: App, opts: Partial): void { checkboxValues.clear() }) - function trackInputValue(id: number, node: TextFeildElement) { + function trackInputValue(id: number, node: TextFieldElement) { if (inputValues.get(id) === node.value) { return } @@ -150,7 +155,7 @@ export default function (app: App, opts: Partial): void { app.send(SetInputChecked(id, value)) } - // The only way (to our knowladge) to track all kinds of input changes, including those made by JS + // The only way (to our knowledge) to track all kinds of input changes, including those made by JS app.ticker.attach(() => { inputValues.forEach((value, id) => { const node = app.nodes.getNode(id) as HTMLInputElement @@ -162,12 +167,18 @@ export default function (app: App, opts: Partial): void { if (!node) return checkboxValues.delete(id) trackCheckboxValue(id, node.checked) }) - }, 5) + }, 3) - function sendInputChange(id: number, node: TextFeildElement, hesitationTime: number) { - trackInputValue(id, node) + function sendInputChange( + id: number, + node: TextFieldElement, + hesitationTime: number, + inputTime: number, + ) { + const { value, mask } = getInputValue(id, node) const label = getInputLabel(node) - app.send(InputChange(id, label, hesitationTime)) + + app.send(InputChange(id, value, mask !== 0, label, hesitationTime, inputTime)) } app.nodes.attachNodeCallback( @@ -183,22 +194,27 @@ export default function (app: App, opts: Partial): void { app.nodes.attachNodeListener(node, 'change', () => sendInputValue(id, node)) } - if (isTextFeildElement(node)) { + if (isTextFieldElement(node)) { trackInputValue(id, node) let nodeFocusTime = 0 let nodeHesitationTime = 0 + let inputTime = 0 + const onFocus = () => { nodeFocusTime = now() } + const onInput = () => { - const value = node.value if (nodeHesitationTime === 0) { - nodeHesitationTime = nodeFocusTime - now() + nodeHesitationTime = now() - nodeFocusTime } } + const onChange = () => { - sendInputChange(id, node, nodeHesitationTime) + inputTime = now() - nodeFocusTime + sendInputChange(id, node, nodeHesitationTime, inputTime) nodeHesitationTime = 0 + inputTime = 0 } app.nodes.attachNodeListener(node, 'focus', onFocus) app.nodes.attachNodeListener(node, 'input', onInput) @@ -208,7 +224,7 @@ export default function (app: App, opts: Partial): void { if (isCheckbox(node)) { trackCheckboxValue(id, node.checked) - app.nodes.attachNodeListener(node, 'change', (e) => trackCheckboxValue(id, node.checked)) + app.nodes.attachNodeListener(node, 'change', () => trackCheckboxValue(id, node.checked)) return } }), diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 10b3e205c..9e7b6f93f 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -255,7 +255,7 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.InputChange: - return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) + return this.uint(msg[1]) && this.string(msg[2]) && this.boolean(msg[3]) && this.string(msg[4]) && this.int(msg[5]) && this.int(msg[6]) break case Messages.Type.SelectionChange: From 4a96d87cb700ac1706ed07bf5ffaedf3f88882c5 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 11:30:08 +0100 Subject: [PATCH 14/29] change(tracker): add mouse thrashing event message --- backend/pkg/messages/filters.go | 2 +- backend/pkg/messages/messages.go | 22 +++++++++++++++++++ backend/pkg/messages/read-message.go | 11 ++++++++++ ee/connectors/msgcodec/messages.py | 7 ++++++ ee/connectors/msgcodec/msgcodec.py | 5 +++++ .../web/messages/RawMessageReader.gen.ts | 8 +++++++ .../app/player/web/messages/filters.gen.ts | 2 +- .../app/player/web/messages/message.gen.ts | 3 +++ frontend/app/player/web/messages/raw.gen.ts | 8 ++++++- .../player/web/messages/tracker-legacy.gen.ts | 1 + .../app/player/web/messages/tracker.gen.ts | 14 +++++++++++- mobs/messages.rb | 4 ++++ tracker/tracker/src/common/messages.gen.ts | 8 ++++++- tracker/tracker/src/main/app/messages.gen.ts | 9 ++++++++ tracker/tracker/src/main/modules/mouse.ts | 7 +++--- .../src/webworker/MessageEncoder.gen.ts | 4 ++++ 16 files changed, 107 insertions(+), 8 deletions(-) diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index a67e8c43f..b6c688c13 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -10,5 +10,5 @@ func IsIOSType(id int) bool { } func IsDOMType(id int) bool { - return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id + return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 6e5bbcfeb..e0c78bf71 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -84,6 +84,7 @@ const ( MsgSessionSearch = 127 MsgInputChange = 112 MsgSelectionChange = 113 + MsgMouseThrashing = 114 MsgIOSBatchMeta = 107 MsgIOSSessionStart = 90 MsgIOSSessionEnd = 91 @@ -2254,6 +2255,27 @@ func (msg *SelectionChange) TypeID() int { return 113 } +type MouseThrashing struct { + message + Timestamp uint64 +} + +func (msg *MouseThrashing) Encode() []byte { + buf := make([]byte, 11) + buf[0] = 114 + p := 1 + p = WriteUint(msg.Timestamp, buf, p) + return buf[:p] +} + +func (msg *MouseThrashing) Decode() Message { + return msg +} + +func (msg *MouseThrashing) TypeID() int { + return 114 +} + type IOSBatchMeta struct { message Timestamp uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 80a365168..52da274bf 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1383,6 +1383,15 @@ func DecodeSelectionChange(reader BytesReader) (Message, error) { return msg, err } +func DecodeMouseThrashing(reader BytesReader) (Message, error) { + var err error = nil + msg := &MouseThrashing{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIOSBatchMeta(reader BytesReader) (Message, error) { var err error = nil msg := &IOSBatchMeta{} @@ -1951,6 +1960,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeInputChange(reader) case 113: return DecodeSelectionChange(reader) + case 114: + return DecodeMouseThrashing(reader) case 107: return DecodeIOSBatchMeta(reader) case 90: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 2ed4cf49d..102768e60 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -794,6 +794,13 @@ class SelectionChange(Message): self.selection = selection +class MouseThrashing(Message): + __id__ = 114 + + def __init__(self, timestamp): + self.timestamp = timestamp + + class IOSBatchMeta(Message): __id__ = 107 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 272f2efe2..6ebbfff54 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -700,6 +700,11 @@ class MessageCodec(Codec): selection=self.read_string(reader) ) + if message_id == 114: + return MouseThrashing( + timestamp=self.read_uint(reader) + ) + if message_id == 107: return IOSBatchMeta( timestamp=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index aeec66f67..1d0e0e7cd 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -639,6 +639,14 @@ export default class RawMessageReader extends PrimitiveReader { }; } + case 114: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + return { + tp: MType.MouseThrashing, + timestamp, + }; + } + case 90: { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const projectID = this.readUint(); if (projectID === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/filters.gen.ts b/frontend/app/player/web/messages/filters.gen.ts index 3e41afc0d..2cd1b6c25 100644 --- a/frontend/app/player/web/messages/filters.gen.ts +++ b/frontend/app/player/web/messages/filters.gen.ts @@ -3,7 +3,7 @@ import { MType } from './raw.gen' -const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,90,93,96,100,102,103,105] +const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,114,90,93,96,100,102,103,105] export function isDOMType(t: MType) { return DOM_TYPES.includes(t) } \ No newline at end of file diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index 5dd11b8c3..e39d02584 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -56,6 +56,7 @@ import type { RawAdoptedSsRemoveOwner, RawZustand, RawSelectionChange, + RawMouseThrashing, RawIosSessionStart, RawIosCustomEvent, RawIosScreenChanges, @@ -172,6 +173,8 @@ export type Zustand = RawZustand & Timed export type SelectionChange = RawSelectionChange & Timed +export type MouseThrashing = RawMouseThrashing & Timed + export type IosSessionStart = RawIosSessionStart & Timed export type IosCustomEvent = RawIosCustomEvent & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index b399dde79..67f65ab35 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -54,6 +54,7 @@ export const enum MType { AdoptedSsRemoveOwner = 77, Zustand = 79, SelectionChange = 113, + MouseThrashing = 114, IosSessionStart = 90, IosCustomEvent = 93, IosScreenChanges = 96, @@ -426,6 +427,11 @@ export interface RawSelectionChange { selection: string, } +export interface RawMouseThrashing { + tp: MType.MouseThrashing, + timestamp: number, +} + export interface RawIosSessionStart { tp: MType.IosSessionStart, timestamp: number, @@ -497,4 +503,4 @@ export interface RawIosNetworkCall { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawMouseThrashing | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index 8005890ab..ce69f34b3 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -55,6 +55,7 @@ export const TP_MAP = { 77: MType.AdoptedSsRemoveOwner, 79: MType.Zustand, 113: MType.SelectionChange, + 114: MType.MouseThrashing, 90: MType.IosSessionStart, 93: MType.IosCustomEvent, 96: MType.IosScreenChanges, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index ef11afff2..fdd9c6f42 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -446,8 +446,13 @@ type TrSelectionChange = [ selection: string, ] +type TrMouseThrashing = [ + type: 114, + timestamp: number, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -893,6 +898,13 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } + case 114: { + return { + tp: MType.MouseThrashing, + timestamp: tMsg[1], + } + } + default: return null } diff --git a/mobs/messages.rb b/mobs/messages.rb index cb27a1db2..b5f714985 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -511,4 +511,8 @@ message 113, 'SelectionChange' do uint 'SelectionStart' uint 'SelectionEnd' string 'Selection' +end + +message 114, 'MouseThrashing' do + uint 'Timestamp' end \ No newline at end of file diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 8dfb0b2de..ce48b201d 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -65,6 +65,7 @@ export declare const enum Type { PartitionedMessage = 82, InputChange = 112, SelectionChange = 113, + MouseThrashing = 114, } @@ -509,6 +510,11 @@ export type SelectionChange = [ /*selection:*/ string, ] +export type MouseThrashing = [ + /*type:*/ Type.MouseThrashing, + /*timestamp:*/ number, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index b6ac90706..dad81cb6a 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -824,3 +824,12 @@ export function SelectionChange( ] } +export function MouseThrashing( + timestamp: number, +): Messages.MouseThrashing { + return [ + Messages.Type.MouseThrashing, + timestamp, + ] +} + diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index c94f117fa..5e905f205 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -1,7 +1,7 @@ import type App from '../app/index.js' import { hasTag, isSVGElement, isDocument } from '../app/guards.js' -import { normSpaces, hasOpenreplayAttribute, getLabelAttribute } from '../utils.js' -import { MouseMove, MouseClick } from '../app/messages.gen.js' +import { normSpaces, hasOpenreplayAttribute, getLabelAttribute, now } from '../utils.js' +import { MouseMove, MouseClick, MouseThrashing } from '../app/messages.gen.js' import { getInputLabel } from './input.js' function _getSelector(target: Element, document: Document): string { @@ -37,7 +37,7 @@ function isClickable(element: Element): boolean { element.getAttribute('role') === 'button' ) //|| element.className.includes("btn") - // MBTODO: intersept addEventListener + // MBTODO: intercept addEventListener } //TODO: fix (typescript is not sure about target variable after assignation of svg) @@ -126,6 +126,7 @@ export default function (app: App): void { const acceleration = (nextVelocity - velocity) / shakeCheckInterval if (directionChangeCount > 3 && acceleration > shakeThreshold) { console.log('Mouse shake detected!') + app.send(MouseThrashing(now())) } distance = 0 diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 9e7b6f93f..06bd776f0 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -262,6 +262,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) && this.string(msg[3]) break + case Messages.Type.MouseThrashing: + return this.uint(msg[1]) + break + } } From 42847b7c4b2a6ed86e6dbb10002f91171fe21ee9 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 14:13:21 +0100 Subject: [PATCH 15/29] change(player): add mouse thrashing ui event --- frontend/app/player/web/MessageManager.ts | 12 +++++++-- frontend/app/player/web/Screen/Cursor.ts | 25 ++++++++++++++++++- .../app/player/web/Screen/cursor.module.css | 6 ++--- .../player/web/managers/MouseMoveManager.ts | 2 +- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index d0ae18020..8fb928909 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -19,7 +19,7 @@ import WindowNodeCounter from './managers/WindowNodeCounter'; import ActivityManager from './managers/ActivityManager'; import MFileReader from './messages/MFileReader'; -import { MType } from './messages'; +import { MouseThrashing, MType } from "./messages"; import { isDOMType } from './messages/filters.gen'; import type { Message, @@ -100,6 +100,7 @@ export default class MessageManager { private performanceTrackManager: PerformanceTrackManager = new PerformanceTrackManager(); private windowNodeCounter: WindowNodeCounter = new WindowNodeCounter(); private clickManager: ListWalker = new ListWalker(); + private mouseThrashingManager: ListWalker = new ListWalker(); private resizeManager: ListWalker = new ListWalker([]); private pagesManager: PagesManager; @@ -322,9 +323,13 @@ export default class MessageManager { // Moving mouse and setting :hover classes on ready view this.mouseMoveManager.move(t); const lastClick = this.clickManager.moveGetLast(t); - if (!!lastClick && t - lastClick.time < 600) { // happend during last 600ms + if (!!lastClick && t - lastClick.time < 600) { // happened during last 600ms this.screen.cursor.click(); } + const lastThrashing = this.mouseThrashingManager.moveGetLast(t) + if (!!lastThrashing && t - lastThrashing.time < 300) { + this.screen.cursor.shake(); + } }) if (this.waitingForFiles && this.lastMessageTime <= t && t !== this.session.duration.milliseconds) { @@ -365,6 +370,9 @@ export default class MessageManager { case MType.SetViewportSize: this.resizeManager.append(msg); break; + case MType.MouseThrashing: + this.mouseThrashingManager.append(msg); + break; case MType.MouseMove: this.mouseMoveManager.append(msg); break; diff --git a/frontend/app/player/web/Screen/Cursor.ts b/frontend/app/player/web/Screen/Cursor.ts index f2d371062..799075b18 100644 --- a/frontend/app/player/web/Screen/Cursor.ts +++ b/frontend/app/player/web/Screen/Cursor.ts @@ -3,9 +3,11 @@ import styles from './cursor.module.css'; export default class Cursor { + private readonly isMobile: boolean; private readonly cursor: HTMLDivElement; private tagElement: HTMLDivElement; - private isMobile: boolean; + private coords = { x: 0, y: 0 }; + private isMoving = false; constructor(overlay: HTMLDivElement, isMobile: boolean) { this.cursor = document.createElement('div'); @@ -13,6 +15,8 @@ export default class Cursor { if (isMobile) this.cursor.style.backgroundImage = 'unset' overlay.appendChild(this.cursor); this.isMobile = isMobile; + + window.shakeTest = this.shake.bind(this); } toggle(flag: boolean) { @@ -50,8 +54,27 @@ export default class Cursor { } move({ x, y }: Point) { + this.isMoving = true; this.cursor.style.left = x + 'px'; this.cursor.style.top = y + 'px'; + this.coords = { x, y }; + setTimeout(() => this.isMoving = false, 60) + } + + shake(iteration = 1, upwards = true, original: { x: number, y: number } = this.coords) { + if (this.isMoving) return; + if (iteration < 10) { + this.cursor.style.width = 45 + 'px' + this.cursor.style.height = 75 + 'px' + const shift = upwards ? 60 : -60 + this.move({ x: this.coords.x + shift, y: this.coords.y - shift }) + setTimeout(() => this.shake(iteration + 1, !upwards, original), 60) + } else { + this.cursor.style.width = 18 + 'px' + this.cursor.style.height = 30 + 'px' + + this.move(original) + } } click() { diff --git a/frontend/app/player/web/Screen/cursor.module.css b/frontend/app/player/web/Screen/cursor.module.css index 93f3d05ff..67d452953 100644 --- a/frontend/app/player/web/Screen/cursor.module.css +++ b/frontend/app/player/web/Screen/cursor.module.css @@ -1,9 +1,9 @@ .cursor { display: block; position: absolute; - width: 13px; - height: 20px; - background-image: url('data:image/svg+xml;utf8,'); + width: 18px; + height: 30px; + background-image: url('data:image/svg+xml;utf8, '); background-repeat: no-repeat; transition: top .125s linear, left .125s linear; diff --git a/frontend/app/player/web/managers/MouseMoveManager.ts b/frontend/app/player/web/managers/MouseMoveManager.ts index 1b19d7e5b..fa320ab13 100644 --- a/frontend/app/player/web/managers/MouseMoveManager.ts +++ b/frontend/app/player/web/managers/MouseMoveManager.ts @@ -1,5 +1,5 @@ import type Screen from '../Screen/Screen' -import type { MouseMove } from '../messages' +import type { MouseMove } from "../messages"; import ListWalker from '../../common/ListWalker' From b84c1ccc8e8aea650ee5428b59a9cf188b308e51 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 14:14:12 +0100 Subject: [PATCH 16/29] change(player): remove synt event --- frontend/app/player/web/Screen/Cursor.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/app/player/web/Screen/Cursor.ts b/frontend/app/player/web/Screen/Cursor.ts index 799075b18..ee98ab83a 100644 --- a/frontend/app/player/web/Screen/Cursor.ts +++ b/frontend/app/player/web/Screen/Cursor.ts @@ -15,8 +15,6 @@ export default class Cursor { if (isMobile) this.cursor.style.backgroundImage = 'unset' overlay.appendChild(this.cursor); this.isMobile = isMobile; - - window.shakeTest = this.shake.bind(this); } toggle(flag: boolean) { @@ -85,7 +83,7 @@ export default class Cursor { }, 600) } - // TODO (to keep on a different playig speed): + // TODO (to keep on a different playing speed): // transition // setTransitionSpeed() From 5c87872c172ba780832bac6fe56dfa3371d538d7 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 14:39:19 +0100 Subject: [PATCH 17/29] change(player): tune shake animation --- frontend/app/player/web/Screen/Cursor.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/frontend/app/player/web/Screen/Cursor.ts b/frontend/app/player/web/Screen/Cursor.ts index ee98ab83a..b6a475331 100644 --- a/frontend/app/player/web/Screen/Cursor.ts +++ b/frontend/app/player/web/Screen/Cursor.ts @@ -59,19 +59,26 @@ export default class Cursor { setTimeout(() => this.isMoving = false, 60) } + setDefaultStyle() { + this.cursor.style.width = 18 + 'px' + this.cursor.style.height = 30 + 'px' + this.cursor.style.transition = 'top .125s linear, left .125s linear' + } + shake(iteration = 1, upwards = true, original: { x: number, y: number } = this.coords) { - if (this.isMoving) return; + if (this.isMoving) { + return this.setDefaultStyle() + } if (iteration < 10) { + this.cursor.style.transition = 'top .06s linear, left .06s linear' this.cursor.style.width = 45 + 'px' this.cursor.style.height = 75 + 'px' - const shift = upwards ? 60 : -60 + const shift = upwards ? 90 : -90 this.move({ x: this.coords.x + shift, y: this.coords.y - shift }) setTimeout(() => this.shake(iteration + 1, !upwards, original), 60) } else { - this.cursor.style.width = 18 + 'px' - this.cursor.style.height = 30 + 'px' - - this.move(original) + this.setDefaultStyle() + return this.move(original) } } From d5b0444388c4e197e7b4eee670d34732d88d371a Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 14:57:50 +0100 Subject: [PATCH 18/29] change(player): move animation to css --- frontend/app/player/web/Screen/Cursor.ts | 20 ++++---------- .../app/player/web/Screen/cursor.module.css | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/frontend/app/player/web/Screen/Cursor.ts b/frontend/app/player/web/Screen/Cursor.ts index b6a475331..dc3637c1b 100644 --- a/frontend/app/player/web/Screen/Cursor.ts +++ b/frontend/app/player/web/Screen/Cursor.ts @@ -65,21 +65,11 @@ export default class Cursor { this.cursor.style.transition = 'top .125s linear, left .125s linear' } - shake(iteration = 1, upwards = true, original: { x: number, y: number } = this.coords) { - if (this.isMoving) { - return this.setDefaultStyle() - } - if (iteration < 10) { - this.cursor.style.transition = 'top .06s linear, left .06s linear' - this.cursor.style.width = 45 + 'px' - this.cursor.style.height = 75 + 'px' - const shift = upwards ? 90 : -90 - this.move({ x: this.coords.x + shift, y: this.coords.y - shift }) - setTimeout(() => this.shake(iteration + 1, !upwards, original), 60) - } else { - this.setDefaultStyle() - return this.move(original) - } + shake() { + this.cursor.classList.add(styles.shaking) + setTimeout(() => { + this.cursor.classList.remove(styles.shaking) + }, 500) } click() { diff --git a/frontend/app/player/web/Screen/cursor.module.css b/frontend/app/player/web/Screen/cursor.module.css index 67d452953..238a53ca1 100644 --- a/frontend/app/player/web/Screen/cursor.module.css +++ b/frontend/app/player/web/Screen/cursor.module.css @@ -108,3 +108,29 @@ transform: scale3d(1, 1, 1); } } + +.cursor.shaking { + width: 45px; + height: 75px; + -webkit-animation: shaking 0.3s linear; + animation: shaking 0.3s linear; + animation-iteration-count: 2; +} + +@keyframes shaking { + 0% { + transform: translate(60px, -60px); + } + 25% { + transform: translate(-60px, 60px); + } + 50% { + transform: translate(60px, -60px); + } + 75% { + transform: translate(-60px, 60px); + } + 100% { + transform: translate(60px, -60px); + } +} \ No newline at end of file From ded9a88fe97691d753bd66bced9c41e8b2d984fd Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 10 Mar 2023 09:56:31 +0100 Subject: [PATCH 19/29] feat(backend): implemented db inserts for InputDuration and MouseThrashing --- backend/internal/db/datasaver/saver.go | 4 +++ backend/pkg/db/cache/messages-web.go | 18 ++++++++++ backend/pkg/db/postgres/bulks.go | 14 +++++++- backend/pkg/db/postgres/messages-web.go | 44 +++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index 92dbff958..1a017fa6f 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -77,6 +77,10 @@ func (s *saverImpl) handleMessage(msg Message) error { return s.pg.InsertWebJSException(m) case *IntegrationEvent: return s.pg.InsertWebIntegrationEvent(m) + case *InputChange: + return s.pg.InsertWebInputDuration(m) + case *MouseThrashing: + return s.pg.InsertMouseThrashing(m) case *IOSSessionStart: return s.pg.InsertIOSSessionStart(m) case *IOSSessionEnd: diff --git a/backend/pkg/db/cache/messages-web.go b/backend/pkg/db/cache/messages-web.go index 0a870e5a2..58c703318 100644 --- a/backend/pkg/db/cache/messages-web.go +++ b/backend/pkg/db/cache/messages-web.go @@ -180,3 +180,21 @@ func (c *PGCache) InsertWebInputEvent(e *InputEvent) error { } return c.Conn.InsertWebInputEvent(sessionID, session.ProjectID, e) } + +func (c *PGCache) InsertWebInputDuration(e *InputChange) error { + sessionID := e.SessionID() + session, err := c.Cache.GetSession(sessionID) + if err != nil { + return err + } + return c.Conn.InsertWebInputDuration(sessionID, session.ProjectID, e) +} + +func (c *PGCache) InsertMouseThrashing(e *MouseThrashing) error { + sessionID := e.SessionID() + session, err := c.Cache.GetSession(sessionID) + if err != nil { + return err + } + return c.Conn.InsertMouseThrashing(sessionID, session.ProjectID, e) +} diff --git a/backend/pkg/db/postgres/bulks.go b/backend/pkg/db/postgres/bulks.go index 27ab2cafd..0dcfca646 100644 --- a/backend/pkg/db/postgres/bulks.go +++ b/backend/pkg/db/postgres/bulks.go @@ -9,7 +9,7 @@ type bulksTask struct { } func NewBulksTask() *bulksTask { - return &bulksTask{bulks: make([]Bulk, 0, 14)} + return &bulksTask{bulks: make([]Bulk, 0, 15)} } type BulkSet struct { @@ -19,6 +19,7 @@ type BulkSet struct { customEvents Bulk webPageEvents Bulk webInputEvents Bulk + webInputDurations Bulk webGraphQL Bulk webErrors Bulk webErrorEvents Bulk @@ -57,6 +58,8 @@ func (conn *BulkSet) Get(name string) Bulk { return conn.webPageEvents case "webInputEvents": return conn.webInputEvents + case "webInputDurations": + return conn.webInputDurations case "webGraphQL": return conn.webGraphQL case "webErrors": @@ -126,6 +129,14 @@ func (conn *BulkSet) initBulks() { if err != nil { log.Fatalf("can't create webPageEvents bulk: %s", err) } + conn.webInputDurations, err = NewBulk(conn.c, + "events.inputs", + "(session_id, message_id, timestamp, value, label, hesitation, duration)", + "($%d, $%d, $%d, LEFT($%d, 2000), NULLIF(LEFT($%d, 2000),''), $%d, $%d)", + 7, 200) + if err != nil { + log.Fatalf("can't create webPageEvents bulk: %s", err) + } conn.webGraphQL, err = NewBulk(conn.c, "events.graphql", "(session_id, timestamp, message_id, name, request_body, response_body)", @@ -209,6 +220,7 @@ func (conn *BulkSet) Send() { newTask.bulks = append(newTask.bulks, conn.customEvents) newTask.bulks = append(newTask.bulks, conn.webPageEvents) newTask.bulks = append(newTask.bulks, conn.webInputEvents) + newTask.bulks = append(newTask.bulks, conn.webInputDurations) newTask.bulks = append(newTask.bulks, conn.webGraphQL) newTask.bulks = append(newTask.bulks, conn.webErrors) newTask.bulks = append(newTask.bulks, conn.webErrorEvents) diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index 9251a4924..2037612f8 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -1,7 +1,10 @@ package postgres import ( + "encoding/hex" + "hash/fnv" "log" + "strconv" "openreplay/backend/pkg/db/types" . "openreplay/backend/pkg/messages" @@ -89,6 +92,24 @@ func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *Inp return nil } +func (conn *Conn) InsertWebInputDuration(sessionID uint64, projectID uint32, e *InputChange) error { + // Debug log + log.Printf("new InputDuration event: %v", e) + if e.Label == "" { + return nil + } + value := &e.Value + if e.ValueMasked { + value = nil + } + if err := conn.bulks.Get("webInputDurations").Append(sessionID, truncSqIdx(e.ID), e.Timestamp, value, e.Label, e.HesitationTime, e.InputDuration); err != nil { + log.Printf("insert web input event err: %s", err) + } + conn.updateSessionEvents(sessionID, 1, 0) + conn.insertAutocompleteValue(sessionID, projectID, "INPUT", e.Label) + return nil +} + func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *types.ErrorEvent) error { errorID := e.ID(projectID) if err := conn.bulks.Get("webErrors").Append(errorID, projectID, e.Source, e.Name, e.Message, e.Payload); err != nil { @@ -145,3 +166,26 @@ func (conn *Conn) InsertSessionReferrer(sessionID uint64, referrer string) error WHERE session_id = $3 AND referrer IS NULL`, referrer, url.DiscardURLQuery(referrer), sessionID) } + +func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *MouseThrashing) error { + // Debug log + log.Printf("new MouseThrashing event: %v", e) + // + issueID := mouseThrashingID(projectID, sessionID, e.Timestamp) + if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_trashing", e.Url); err != nil { + log.Printf("insert web issue err: %s", err) + } + if err := conn.bulks.Get("webIssueEvents").Append(sessionID, issueID, e.Timestamp, truncSqIdx(e.MsgID()), nil); err != nil { + log.Printf("insert web issue event err: %s", err) + } + conn.updateSessionIssues(sessionID, 0, 50) + return nil +} + +func mouseThrashingID(projectID uint32, sessID, ts uint64) string { + hash := fnv.New128a() + hash.Write([]byte("mouse_trashing")) + hash.Write([]byte(strconv.FormatUint(sessID, 10))) + hash.Write([]byte(strconv.FormatUint(ts, 10))) + return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) +} From 3d7ab8b31d9de399acac06144bf9670e778f0996 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 10 Mar 2023 12:19:32 +0100 Subject: [PATCH 20/29] feat(backend): added new message types to db message filter --- backend/cmd/db/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index ae1228cc3..6086d9491 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -38,7 +38,7 @@ func main() { messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument, messages.MsgMouseClick, messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming, - messages.MsgInputEvent, messages.MsgPageEvent} + messages.MsgInputEvent, messages.MsgPageEvent, messages.MsgMouseThrashing, messages.MsgInputChange} // Init consumer consumer := queue.NewConsumer( From 4480a5d93a2b0f7a57d3ebfa2119f7c20e15a2c8 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 13 Mar 2023 11:11:21 +0100 Subject: [PATCH 21/29] fix(backend): fixed typo in issue type --- backend/pkg/db/postgres/messages-web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index 2037612f8..a15c03335 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -172,7 +172,7 @@ func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *Mo log.Printf("new MouseThrashing event: %v", e) // issueID := mouseThrashingID(projectID, sessionID, e.Timestamp) - if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_trashing", e.Url); err != nil { + if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_thrashing", e.Url); err != nil { log.Printf("insert web issue err: %s", err) } if err := conn.bulks.Get("webIssueEvents").Append(sessionID, issueID, e.Timestamp, truncSqIdx(e.MsgID()), nil); err != nil { From 9ec887b7f8bd28caba7c2eb6e31f7b061c3d94a4 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Mar 2023 12:30:37 +0100 Subject: [PATCH 22/29] change(ui): add new event info to UI --- frontend/app/types/session/event.ts | 102 ++++++++++++++++------------ frontend/app/types/session/issue.ts | 4 +- 2 files changed, 61 insertions(+), 45 deletions(-) diff --git a/frontend/app/types/session/event.ts b/frontend/app/types/session/event.ts index bb901a6b1..99693d756 100644 --- a/frontend/app/types/session/event.ts +++ b/frontend/app/types/session/event.ts @@ -5,12 +5,18 @@ const LOCATION = 'LOCATION'; const CUSTOM = 'CUSTOM'; const CLICKRAGE = 'CLICKRAGE'; const IOS_VIEW = 'VIEW'; -export const TYPES = { CONSOLE, CLICK, INPUT, LOCATION, CUSTOM, CLICKRAGE, IOS_VIEW}; +export const TYPES = { CONSOLE, CLICK, INPUT, LOCATION, CUSTOM, CLICKRAGE, IOS_VIEW }; interface IEvent { time: number; timestamp: number; - type: typeof CONSOLE | typeof CLICK | typeof INPUT | typeof LOCATION | typeof CUSTOM | typeof CLICKRAGE; + type: + | typeof CONSOLE + | typeof CLICK + | typeof INPUT + | typeof LOCATION + | typeof CUSTOM + | typeof CLICKRAGE; name: string; key: number; label: string; @@ -18,18 +24,23 @@ interface IEvent { target: { path: string; label: string; - } + }; } + interface ConsoleEvent extends IEvent { - subtype: string - value: string + subtype: string; + value: string; } + interface ClickEvent extends IEvent { targetContent: string; count: number; } + interface InputEvent extends IEvent { value: string; + hesitation: number; + duration: number; } interface LocationEvent extends IEvent { @@ -51,11 +62,10 @@ interface LocationEvent extends IEvent { export type EventData = ConsoleEvent | ClickEvent | InputEvent | LocationEvent | IEvent; class Event { - key: IEvent["key"] - time: IEvent["time"]; - label: IEvent["label"]; - target: IEvent["target"]; - + key: IEvent['key']; + time: IEvent['time']; + label: IEvent['label']; + target: IEvent['target']; constructor(event: IEvent) { Object.assign(this, { @@ -64,98 +74,102 @@ class Event { key: event.key, target: { path: event.target?.path || event.targetPath, - label: event.target?.label - } - }) + label: event.target?.label, + }, + }); } } class Console extends Event { readonly type = CONSOLE; - readonly name = 'Console' + readonly name = 'Console'; subtype: string; value: string; constructor(evt: ConsoleEvent) { super(evt); - this.subtype = evt.subtype - this.value = evt.value + this.subtype = evt.subtype; + this.value = evt.value; } } export class Click extends Event { readonly type: typeof CLICKRAGE | typeof CLICK = CLICK; - readonly name = 'Click' + readonly name = 'Click'; targetContent = ''; - count: number + count: number; constructor(evt: ClickEvent, isClickRage?: boolean) { + console.log(evt); super(evt); - this.targetContent = evt.targetContent - this.count = evt.count + this.targetContent = evt.targetContent; + this.count = evt.count; if (isClickRage) { - this.type = CLICKRAGE + this.type = CLICKRAGE; } } } class Input extends Event { readonly type = INPUT; - readonly name = 'Input' - value = '' + readonly name = 'Input'; + readonly hesitation: number = 0; + readonly duration: number = 0; + + value = ''; constructor(evt: InputEvent) { super(evt); - this.value = evt.value + this.value = evt.value; + this.hesitation = evt.hesitation; + this.duration = evt.duration; } } - export class Location extends Event { readonly name = 'Location'; readonly type = LOCATION; - url: LocationEvent["url"] - host: LocationEvent["host"]; - fcpTime: LocationEvent["fcpTime"]; - loadTime: LocationEvent["loadTime"]; - domContentLoadedTime: LocationEvent["domContentLoadedTime"]; - domBuildingTime: LocationEvent["domBuildingTime"]; - speedIndex: LocationEvent["speedIndex"]; - visuallyComplete: LocationEvent["visuallyComplete"]; - timeToInteractive: LocationEvent["timeToInteractive"]; - referrer: LocationEvent["referrer"]; + url: LocationEvent['url']; + host: LocationEvent['host']; + fcpTime: LocationEvent['fcpTime']; + loadTime: LocationEvent['loadTime']; + domContentLoadedTime: LocationEvent['domContentLoadedTime']; + domBuildingTime: LocationEvent['domBuildingTime']; + speedIndex: LocationEvent['speedIndex']; + visuallyComplete: LocationEvent['visuallyComplete']; + timeToInteractive: LocationEvent['timeToInteractive']; + referrer: LocationEvent['referrer']; constructor(evt: LocationEvent) { super(evt); Object.assign(this, { ...evt, - fcpTime: evt.firstContentfulPaintTime || evt.firstPaintTime + fcpTime: evt.firstContentfulPaintTime || evt.firstPaintTime, }); } } export type InjectedEvent = Console | Click | Input | Location; -export default function(event: EventData) { +export default function (event: EventData) { if (event.type && event.type === CONSOLE) { - return new Console(event as ConsoleEvent) + return new Console(event as ConsoleEvent); } if (event.type && event.type === CLICK) { - return new Click(event as ClickEvent) + return new Click(event as ClickEvent); } if (event.type && event.type === INPUT) { - return new Input(event as InputEvent) + return new Input(event as InputEvent); } if (event.type && event.type === LOCATION) { - return new Location(event as LocationEvent) + return new Location(event as LocationEvent); } if (event.type && event.type === CLICKRAGE) { - return new Click(event as ClickEvent, true) + return new Click(event as ClickEvent, true); } // not used right now? // if (event.type === CUSTOM || !event.type) { // return new Event(event) // } - console.error(`Unknown event type: ${event.type}`) + console.error(`Unknown event type: ${event.type}`); } - diff --git a/frontend/app/types/session/issue.ts b/frontend/app/types/session/issue.ts index 68ab64001..2abedc5ea 100644 --- a/frontend/app/types/session/issue.ts +++ b/frontend/app/types/session/issue.ts @@ -5,7 +5,8 @@ const types = { JS_EXCEPTION: 'js_exception', BAD_REQUEST: 'bad_request', CRASH: 'crash', - CLICK_RAGE: 'click_rage' + CLICK_RAGE: 'click_rage', + MOUSE_THRASHING: 'mouse_thrashing', } as const type TypeKeys = keyof typeof types @@ -21,6 +22,7 @@ export const issues_types = [ { 'type': types.BAD_REQUEST, 'visible': true, 'order': 2, 'name': 'Bad Requests', 'icon': 'funnel/file-medical-alt' }, { 'type': types.CLICK_RAGE, 'visible': true, 'order': 3, 'name': 'Click Rage', 'icon': 'funnel/emoji-angry' }, { 'type': types.CRASH, 'visible': true, 'order': 4, 'name': 'Crashes', 'icon': 'funnel/file-earmark-break' }, + { 'type': types.MOUSE_THRASHING, 'visible': true, 'order': 5, 'name': 'Mouse Thrashing', 'icon': 'close' }, // { 'type': 'memory', 'visible': true, 'order': 4, 'name': 'High Memory', 'icon': 'funnel/sd-card' }, // { 'type': 'vault', 'visible': true, 'order': 5, 'name': 'Vault', 'icon': 'safe' }, // { 'type': 'bookmark', 'visible': true, 'order': 5, 'name': 'Bookmarks', 'icon': 'safe' }, From 907ae9a1318623ad8e87066cc1767ba5067e34f2 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 13 Mar 2023 12:54:34 +0100 Subject: [PATCH 23/29] feat(backend): added hesitation time for click events --- backend/pkg/db/postgres/bulks.go | 6 +++--- backend/pkg/db/postgres/messages-web.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/pkg/db/postgres/bulks.go b/backend/pkg/db/postgres/bulks.go index 0dcfca646..7eaba41b4 100644 --- a/backend/pkg/db/postgres/bulks.go +++ b/backend/pkg/db/postgres/bulks.go @@ -195,9 +195,9 @@ func (conn *BulkSet) initBulks() { } conn.webClickEvents, err = NewBulk(conn.c, "events.clicks", - "(session_id, message_id, timestamp, label, selector, url, path)", - "($%d, $%d, $%d, NULLIF(LEFT($%d, 2000), ''), LEFT($%d, 8000), LEFT($%d, 2000), LEFT($%d, 2000))", - 7, 200) + "(session_id, message_id, timestamp, label, selector, url, path, hesitation)", + "($%d, $%d, $%d, NULLIF(LEFT($%d, 2000), ''), LEFT($%d, 8000), LEFT($%d, 2000), LEFT($%d, 2000), $%d)", + 8, 200) if err != nil { log.Fatalf("can't create webClickEvents bulk: %s", err) } diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index a15c03335..a062df95a 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -66,7 +66,7 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, projectID uint32, e *Mou } var host, path string host, path, _, _ = url.GetURLParts(e.Url) - if err := conn.bulks.Get("webClickEvents").Append(sessionID, truncSqIdx(e.MsgID()), e.Timestamp, e.Label, e.Selector, host+path, path); err != nil { + if err := conn.bulks.Get("webClickEvents").Append(sessionID, truncSqIdx(e.MsgID()), e.Timestamp, e.Label, e.Selector, host+path, path, e.HesitationTime); err != nil { log.Printf("insert web click err: %s", err) } // Accumulate session updates and exec inside batch with another sql commands From cd2378f7bc8f32672b59eb96a9f52aa6f76c5504 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 15 Mar 2023 11:29:17 +0100 Subject: [PATCH 24/29] change(ui): change selection display --- .../app/player/web/managers/DOM/SelectionManager.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/app/player/web/managers/DOM/SelectionManager.ts b/frontend/app/player/web/managers/DOM/SelectionManager.ts index 9cf1fd21f..c457ec3b8 100644 --- a/frontend/app/player/web/managers/DOM/SelectionManager.ts +++ b/frontend/app/player/web/managers/DOM/SelectionManager.ts @@ -48,19 +48,21 @@ export default class SelectionManager extends ListWalker { Object.assign(endPointer.style, { top: endCoords.top + 'px', - left: (endCoords.left + endCoords.width + 3) + 'px', - width: '3px', + left: (endCoords.left + (endCoords.width / 2) + 3) + 'px', + width: (endCoords.width / 2) + 'px', height: endCoords.height + 'px', - border: '3px solid red', + borderRight: '2px solid blue', position: 'absolute', + boxShadow: '1px 4px 1px -2px blue', }); Object.assign(startPointer.style, { top: startCoords.top + 'px', left: (startCoords.left - 3) + 'px', - width: '3px', + width: (startCoords.width / 2 ) + 'px', height: startCoords.height + 'px', - border: '3px solid red', + borderLeft: '2px solid blue', position: 'absolute', + boxShadow: '1px 4px 1px -2px blue', }); this.markers.push(startPointer, endPointer); From a7ca8ac54f9a49205ce90a7b680c07fb3749ee5e Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 15 Mar 2023 16:21:44 +0100 Subject: [PATCH 25/29] feat(backend): implemented new events for CH --- backend/pkg/db/postgres/messages-web.go | 15 +----- backend/pkg/hashid/hashid.go | 8 +++ ee/backend/internal/db/datasaver/methods.go | 4 ++ ee/backend/pkg/db/clickhouse/connector.go | 56 ++++++++++++++++++++- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index a062df95a..ebba00168 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -1,12 +1,9 @@ package postgres import ( - "encoding/hex" - "hash/fnv" "log" - "strconv" - "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/hashid" . "openreplay/backend/pkg/messages" "openreplay/backend/pkg/url" ) @@ -171,7 +168,7 @@ func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *Mo // Debug log log.Printf("new MouseThrashing event: %v", e) // - issueID := mouseThrashingID(projectID, sessionID, e.Timestamp) + issueID := hashid.MouseThrashingID(projectID, sessionID, e.Timestamp) if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_thrashing", e.Url); err != nil { log.Printf("insert web issue err: %s", err) } @@ -181,11 +178,3 @@ func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *Mo conn.updateSessionIssues(sessionID, 0, 50) return nil } - -func mouseThrashingID(projectID uint32, sessID, ts uint64) string { - hash := fnv.New128a() - hash.Write([]byte("mouse_trashing")) - hash.Write([]byte(strconv.FormatUint(sessID, 10))) - hash.Write([]byte(strconv.FormatUint(ts, 10))) - return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) -} diff --git a/backend/pkg/hashid/hashid.go b/backend/pkg/hashid/hashid.go index 25ce11369..5bcb23578 100644 --- a/backend/pkg/hashid/hashid.go +++ b/backend/pkg/hashid/hashid.go @@ -23,3 +23,11 @@ func IOSCrashID(projectID uint32, crash *messages.IOSCrash) string { hash.Write([]byte(crash.Stacktrace)) return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) } + +func MouseThrashingID(projectID uint32, sessID, ts uint64) string { + hash := fnv.New128a() + hash.Write([]byte("mouse_trashing")) + hash.Write([]byte(strconv.FormatUint(sessID, 10))) + hash.Write([]byte(strconv.FormatUint(ts, 10))) + return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) +} diff --git a/ee/backend/internal/db/datasaver/methods.go b/ee/backend/internal/db/datasaver/methods.go index 277fd8906..ac0a8b88d 100644 --- a/ee/backend/internal/db/datasaver/methods.go +++ b/ee/backend/internal/db/datasaver/methods.go @@ -78,6 +78,10 @@ func (s *saverImpl) handleExtraMessage(msg Message) error { } case *GraphQL: return s.ch.InsertGraphQL(session, m) + case *InputChange: + return s.ch.InsertWebInputDuration(session, m) + case *MouseThrashing: + return s.ch.InsertMouseThrashing(session, m) } return nil } diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 489411550..ad08bd6e4 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -32,6 +32,8 @@ type Connector interface { InsertCustom(session *types.Session, msg *messages.CustomEvent) error InsertGraphQL(session *types.Session, msg *messages.GraphQL) error InsertIssue(session *types.Session, msg *messages.IssueEvent) error + InsertWebInputDuration(session *types.Session, msg *messages.InputChange) error + InsertMouseThrashing(session *types.Session, msg *messages.MouseThrashing) error } type task struct { @@ -97,7 +99,7 @@ var batches = map[string]string{ "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, ?)", "pages": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "clicks": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, hesitation_time, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)", - "inputs": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, event_type) VALUES (?, ?, ?, ?, ?, ?)", + "inputs": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, event_type, duration, hesitation_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "errors": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, source, name, message, error_id, event_type, error_tags_keys, error_tags_values) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "performance": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "requests": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_body, response_body, status, method, duration, success, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?)", @@ -164,11 +166,59 @@ func (c *connectorImpl) checkError(name string, err error) { } } +func (c *connectorImpl) InsertWebInputDuration(session *types.Session, msg *messages.InputChange) error { + if msg.Label == "" { + return nil + } + if err := c.batches["inputs"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MessageID, + datetime(msg.Timestamp), + msg.Label, + "INPUT", + nullableUint16(uint16(msg.Duration)), + nullableUint32(uint32(msg.HesitationTime)), + ); err != nil { + c.checkError("inputs", err) + return fmt.Errorf("can't append to inputs batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMouseThrashing(session *types.Session, msg *messages.MouseThrashing) error { + issueID := hashid.MouseThrashingID(session.projectID, session.sessionID, msg.Timestamp) + // Insert issue event to batches + if err := c.batches["issuesEvents"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MsgID(), + datetime(msg.Timestamp), + issueID, + "mouse_thrashing", + "ISSUE", + msg.Url, + ); err != nil { + c.checkError("issuesEvents", err) + return fmt.Errorf("can't append to issuesEvents batch: %s", err) + } + if err := c.batches["issues"].Append( + uint16(session.ProjectID), + issueID, + "mouse_thrashing", + msg.Url, + ); err != nil { + c.checkError("issues", err) + return fmt.Errorf("can't append to issues batch: %s", err) + } + return nil +} + func (c *connectorImpl) InsertIssue(session *types.Session, msg *messages.IssueEvent) error { issueID := hashid.IssueID(session.ProjectID, msg) // Check issue type before insert to avoid panic from clickhouse lib switch msg.Type { - case "click_rage", "dead_click", "excessive_scrolling", "bad_request", "missing_resource", "memory", "cpu", "slow_resource", "slow_page_load", "crash", "ml_cpu", "ml_memory", "ml_dead_click", "ml_click_rage", "ml_mouse_thrashing", "ml_excessive_scrolling", "ml_slow_resources", "custom", "js_exception": + case "click_rage", "dead_click", "excessive_scrolling", "bad_request", "missing_resource", "memory", "cpu", "slow_resource", "slow_page_load", "crash", "ml_cpu", "ml_memory", "ml_dead_click", "ml_click_rage", "ml_mouse_thrashing", "ml_excessive_scrolling", "ml_slow_resources", "custom", "js_exception", "mouse_thrashing": default: return fmt.Errorf("unknown issueType: %s", msg.Type) } @@ -323,6 +373,8 @@ func (c *connectorImpl) InsertWebInputEvent(session *types.Session, msg *message datetime(msg.Timestamp), msg.Label, "INPUT", + nil, + nil, ); err != nil { c.checkError("inputs", err) return fmt.Errorf("can't append to inputs batch: %s", err) From d6526464c605c5d897f0f761cf0a1ab49e5e73a3 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 15 Mar 2023 16:26:46 +0100 Subject: [PATCH 26/29] fix(backend): fixed field names --- ee/backend/pkg/db/clickhouse/connector.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index ad08bd6e4..0ee0658ef 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -173,11 +173,11 @@ func (c *connectorImpl) InsertWebInputDuration(session *types.Session, msg *mess if err := c.batches["inputs"].Append( session.SessionID, uint16(session.ProjectID), - msg.MessageID, + msg.MsgID(), datetime(msg.Timestamp), msg.Label, "INPUT", - nullableUint16(uint16(msg.Duration)), + nullableUint16(uint16(msg.InputDuration)), nullableUint32(uint32(msg.HesitationTime)), ); err != nil { c.checkError("inputs", err) @@ -187,7 +187,7 @@ func (c *connectorImpl) InsertWebInputDuration(session *types.Session, msg *mess } func (c *connectorImpl) InsertMouseThrashing(session *types.Session, msg *messages.MouseThrashing) error { - issueID := hashid.MouseThrashingID(session.projectID, session.sessionID, msg.Timestamp) + issueID := hashid.MouseThrashingID(session.ProjectID, session.SessionID, msg.Timestamp) // Insert issue event to batches if err := c.batches["issuesEvents"].Append( session.SessionID, From f360ce3d2c350ed4b0235e4fe3715db2dc6d734b Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 16 Mar 2023 17:11:13 +0100 Subject: [PATCH 27/29] change(ui): add visual display for frustrations --- .../components/Session_/EventsBlock/Event.js | 47 +++++++++++----- .../Session_/EventsBlock/event.module.css | 7 +++ .../Session_/OverviewPanel/OverviewPanel.tsx | 4 +- .../FeatureSelection/FeatureSelection.tsx | 8 +-- .../TimelinePointer/TimelinePointer.tsx | 20 ++++--- frontend/app/components/ui/SVG.tsx | 9 +++- .../app/components/ui/Tooltip/Tooltip.tsx | 3 +- frontend/app/player/web/Lists.ts | 2 +- frontend/app/player/web/WebPlayer.ts | 1 + frontend/app/svg/icons/click-hesitation.svg | 15 ++++++ frontend/app/svg/icons/click-rage.svg | 9 ++++ frontend/app/svg/icons/cursor-trash.svg | 15 ++++++ .../app/svg/icons/event/click_hesitation.svg | 15 ++++++ .../app/svg/icons/event/input_hesitation.svg | 17 ++++++ .../app/svg/icons/event/mouse_thrashing.svg | 15 ++++++ frontend/app/svg/icons/input-hesitation.svg | 17 ++++++ frontend/app/types/session/event.ts | 20 ++++--- frontend/app/types/session/issue.ts | 2 +- frontend/app/types/session/session.ts | 53 +++++++++++++++++-- frontend/scripts/icons.ts | 13 ++--- 20 files changed, 245 insertions(+), 47 deletions(-) create mode 100644 frontend/app/svg/icons/click-hesitation.svg create mode 100644 frontend/app/svg/icons/click-rage.svg create mode 100644 frontend/app/svg/icons/cursor-trash.svg create mode 100644 frontend/app/svg/icons/event/click_hesitation.svg create mode 100644 frontend/app/svg/icons/event/input_hesitation.svg create mode 100644 frontend/app/svg/icons/event/mouse_thrashing.svg create mode 100644 frontend/app/svg/icons/input-hesitation.svg diff --git a/frontend/app/components/Session_/EventsBlock/Event.js b/frontend/app/components/Session_/EventsBlock/Event.js index e8f985aa0..a464fcc28 100644 --- a/frontend/app/components/Session_/EventsBlock/Event.js +++ b/frontend/app/components/Session_/EventsBlock/Event.js @@ -1,7 +1,7 @@ import React from 'react'; import copy from 'copy-to-clipboard'; import cn from 'classnames'; -import { Icon, TextEllipsis } from 'UI'; +import { Icon, TextEllipsis, Tooltip } from 'UI'; import { TYPES } from 'Types/session/event'; import { prorata } from 'App/utils'; import withOverlay from 'Components/hocs/withOverlay'; @@ -9,6 +9,16 @@ import LoadInfo from './LoadInfo'; import cls from './event.module.css'; import { numberWithCommas } from 'App/utils'; +function isFrustrationEvent(evt) { + if (evt.type === 'mouse_thrashing' || evt.type === TYPES.CLICKRAGE) { + return true; + } + if (evt.type === TYPES.CLICK || evt.type === TYPES.INPUT) { + return evt.hesitation > 1000 + } + return false +} + @withOverlay() export default class Event extends React.PureComponent { state = { @@ -44,35 +54,50 @@ export default class Event extends React.PureComponent { const { event } = this.props; let title = event.type; let body; + let icon; + const isFrustration = isFrustrationEvent(event); + const tooltip = { disabled: true, text: '' } + switch (event.type) { case TYPES.LOCATION: title = 'Visited'; body = event.url; + icon = 'location'; break; case TYPES.CLICK: title = 'Clicked'; body = event.label; + icon = isFrustration ? 'click_hesitation' : 'click'; + isFrustration ? Object.assign(tooltip, { disabled: false, text: `User hesitated to click for ${Math.round(event.hesitation/1000)}s`, }) : null; break; case TYPES.INPUT: title = 'Input'; body = event.value; + icon = isFrustration ? 'input_hesitation' : 'input'; + isFrustration ? Object.assign(tooltip, { disabled: false, text: `User hesitated to enter a value for ${Math.round(event.hesitation/1000)}s`, }) : null; break; case TYPES.CLICKRAGE: title = `${ event.count } Clicks`; body = event.label; + icon = 'clickrage' break; case TYPES.IOS_VIEW: title = 'View'; body = event.name; + icon = 'ios_view' + break; + case 'mouse_thrashing': + title = 'Mouse Thrashing'; + icon = 'mouse_thrashing' break; } const isLocation = event.type === TYPES.LOCATION; - const isClickrage = event.type === TYPES.CLICKRAGE; return ( +
- { event.type && } + { event.type && }
@@ -100,6 +125,7 @@ export default class Event extends React.PureComponent {
}
+ ); }; @@ -110,17 +136,15 @@ export default class Event extends React.PureComponent { isCurrent, onClick, showSelection, - onCheckboxClick, showLoadInfo, toggleLoadInfo, isRed, - extended, - highlight = false, presentInSearch = false, - isLastInGroup, whiteBg, } = this.props; const { menuOpen } = this.state; + + const isFrustration = isFrustrationEvent(event); return (
{ this.wrapper = ref } } @@ -135,7 +159,7 @@ export default class Event extends React.PureComponent { [ cls.red ]: isRed, [ cls.clickType ]: event.type === TYPES.CLICK, [ cls.inputType ]: event.type === TYPES.INPUT, - [ cls.clickrageType ]: event.type === TYPES.CLICKRAGE, + [ cls.frustration ]: isFrustration, [ cls.highlight ] : presentInSearch, [ cls.lastInGroup ]: whiteBg, }) } @@ -146,13 +170,10 @@ export default class Event extends React.PureComponent { { event.target ? 'Copy CSS' : 'Copy URL' } } -
-
+
+
{ this.renderBody() }
- {/* { event.type === TYPES.LOCATION && -
{event.url}
- } */}
{ event.type === TYPES.LOCATION && (event.fcpTime || event.visuallyComplete || event.timeToInteractive) && [] }) { performanceChartData, stackList: stackEventList, eventList: eventsList, + frustrationsList, exceptionsList, resourceList: resourceListUnmap, fetchList, @@ -46,8 +46,8 @@ function OverviewPanel({ issuesList }: { issuesList: Record[] }) { NETWORK: resourceList, ERRORS: exceptionsList, EVENTS: stackEventList, - CLICKRAGE: eventsList.filter((item: any) => item.type === TYPES.CLICKRAGE), PERFORMANCE: performanceChartData, + FRUSTRATIONS: frustrationsList, }; }, [dataLoaded]); diff --git a/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx b/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx index 8d76a3070..3a841d97c 100644 --- a/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx +++ b/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx @@ -4,15 +4,15 @@ import { Checkbox, Tooltip } from 'UI'; const NETWORK = 'NETWORK'; const ERRORS = 'ERRORS'; const EVENTS = 'EVENTS'; -const CLICKRAGE = 'CLICKRAGE'; +const FRUSTRATIONS = 'FRUSTRATIONS'; const PERFORMANCE = 'PERFORMANCE'; export const HELP_MESSAGE: any = { NETWORK: 'Network requests made in this session', EVENTS: 'Visualizes the events that takes place in the DOM', ERRORS: 'Visualizes native JS errors like Type, URI, Syntax etc.', - CLICKRAGE: 'Indicates user frustration when repeated clicks are recorded', PERFORMANCE: 'Summary of this session’s memory, and CPU consumption on the timeline', + FRUSTRATIONS: 'Indicates user frustrations in the session', }; interface Props { @@ -21,7 +21,7 @@ interface Props { } function FeatureSelection(props: Props) { const { list } = props; - const features = [NETWORK, ERRORS, EVENTS, CLICKRAGE, PERFORMANCE]; + const features = [NETWORK, ERRORS, EVENTS, PERFORMANCE, FRUSTRATIONS]; const disabled = list.length >= 5; return ( @@ -30,7 +30,7 @@ function FeatureSelection(props: Props) { const checked = list.includes(feature); const _disabled = disabled && !checked; return ( - + { ); }; - const renderClickRageElement = (item: any) => { + const renderFrustrationElement = (item: any) => { + const elData = { name: '', icon: ''} + if (item.type === TYPES.CLICK) Object.assign(elData, { name: `User hesitated to click for ${Math.round(item.hesitation/1000)}s`, icon: 'click-hesitation' }) + if (item.type === TYPES.INPUT) Object.assign(elData, { name: `User hesitated to enter a value for ${Math.round(item.hesitation/1000)}s`, icon: 'input-hesitation' }) + if (item.type === TYPES.CLICKRAGE) Object.assign(elData, { name: 'Click Rage', icon: 'click-rage' }) + if (item.type === issueTypes.MOUSE_THRASHING) Object.assign(elData, { name: 'Mouse Thrashing', icon: 'cursor-trash' }) + return ( - {'Click Rage'} + {elData.name}
} delay={0} placement="top" >
- +
); @@ -158,8 +166,8 @@ const TimelinePointer = React.memo((props: Props) => { if (type === 'NETWORK') { return renderNetworkElement(pointer); } - if (type === 'CLICKRAGE') { - return renderClickRageElement(pointer); + if (type === 'FRUSTRATIONS') { + return renderFrustrationElement(pointer); } if (type === 'ERRORS') { return renderExceptionElement(pointer); diff --git a/frontend/app/components/ui/SVG.tsx b/frontend/app/components/ui/SVG.tsx index 66af7be76..eeaeae5dd 100644 --- a/frontend/app/components/ui/SVG.tsx +++ b/frontend/app/components/ui/SVG.tsx @@ -1,7 +1,7 @@ import React from 'react'; -export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope' | 'errors-icon' | 'event/click' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/link' | 'event/location' | 'event/resize' | 'event/view' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; +export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'click-hesitation' | 'click-rage' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'cursor-trash' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope' | 'errors-icon' | 'event/click' | 'event/click_hesitation' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/input_hesitation' | 'event/link' | 'event/location' | 'event/mouse_thrashing' | 'event/resize' | 'event/view' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'input-hesitation' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; interface Props { name: IconNames; @@ -119,6 +119,8 @@ const SVG = (props: Props) => { case 'chevron-up': return ; case 'circle-fill': return ; case 'circle': return ; + case 'click-hesitation': return ; + case 'click-rage': return ; case 'clipboard-list-check': return ; case 'clock': return ; case 'close': return ; @@ -140,6 +142,7 @@ const SVG = (props: Props) => { case 'credit-card-front': return ; case 'cross': return ; case 'cubes': return ; + case 'cursor-trash': return ; case 'dash': return ; case 'dashboard-icn': return ; case 'desktop': return ; @@ -156,12 +159,15 @@ const SVG = (props: Props) => { case 'envelope': return ; case 'errors-icon': return ; case 'event/click': return ; + case 'event/click_hesitation': return ; case 'event/clickrage': return ; case 'event/code': return ; case 'event/i-cursor': return ; case 'event/input': return ; + case 'event/input_hesitation': return ; case 'event/link': return ; case 'event/location': return ; + case 'event/mouse_thrashing': return ; case 'event/resize': return ; case 'event/view': return ; case 'exclamation-circle': return ; @@ -271,6 +277,7 @@ const SVG = (props: Props) => { case 'info-circle': return ; case 'info-square': return ; case 'info': return ; + case 'input-hesitation': return ; case 'inspect': return ; case 'integrations/assist': return ; case 'integrations/bugsnag-text': return ; diff --git a/frontend/app/components/ui/Tooltip/Tooltip.tsx b/frontend/app/components/ui/Tooltip/Tooltip.tsx index fcb5e1687..ee68b71e7 100644 --- a/frontend/app/components/ui/Tooltip/Tooltip.tsx +++ b/frontend/app/components/ui/Tooltip/Tooltip.tsx @@ -23,6 +23,7 @@ function Tooltip(props: Props) { placement, className = '', anchorClassName = '', + containerClassName = '', delay = 500, style = {}, offset = 5, @@ -39,7 +40,7 @@ function Tooltip(props: Props) { }); return ( -
+
{props.children} Log({ level: LogLevel.ERROR, diff --git a/frontend/app/svg/icons/click-hesitation.svg b/frontend/app/svg/icons/click-hesitation.svg new file mode 100644 index 000000000..144b82cd5 --- /dev/null +++ b/frontend/app/svg/icons/click-hesitation.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/click-rage.svg b/frontend/app/svg/icons/click-rage.svg new file mode 100644 index 000000000..54ccb06a6 --- /dev/null +++ b/frontend/app/svg/icons/click-rage.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/frontend/app/svg/icons/cursor-trash.svg b/frontend/app/svg/icons/cursor-trash.svg new file mode 100644 index 000000000..bdf687c91 --- /dev/null +++ b/frontend/app/svg/icons/cursor-trash.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/event/click_hesitation.svg b/frontend/app/svg/icons/event/click_hesitation.svg new file mode 100644 index 000000000..144b82cd5 --- /dev/null +++ b/frontend/app/svg/icons/event/click_hesitation.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/event/input_hesitation.svg b/frontend/app/svg/icons/event/input_hesitation.svg new file mode 100644 index 000000000..a2f79bfb6 --- /dev/null +++ b/frontend/app/svg/icons/event/input_hesitation.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/event/mouse_thrashing.svg b/frontend/app/svg/icons/event/mouse_thrashing.svg new file mode 100644 index 000000000..af00c02cf --- /dev/null +++ b/frontend/app/svg/icons/event/mouse_thrashing.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/input-hesitation.svg b/frontend/app/svg/icons/input-hesitation.svg new file mode 100644 index 000000000..439606a52 --- /dev/null +++ b/frontend/app/svg/icons/input-hesitation.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/frontend/app/types/session/event.ts b/frontend/app/types/session/event.ts index 99693d756..53924ef0c 100644 --- a/frontend/app/types/session/event.ts +++ b/frontend/app/types/session/event.ts @@ -7,16 +7,18 @@ const CLICKRAGE = 'CLICKRAGE'; const IOS_VIEW = 'VIEW'; export const TYPES = { CONSOLE, CLICK, INPUT, LOCATION, CUSTOM, CLICKRAGE, IOS_VIEW }; +export type EventType = + | typeof CONSOLE + | typeof CLICK + | typeof INPUT + | typeof LOCATION + | typeof CUSTOM + | typeof CLICKRAGE; + interface IEvent { time: number; timestamp: number; - type: - | typeof CONSOLE - | typeof CLICK - | typeof INPUT - | typeof LOCATION - | typeof CUSTOM - | typeof CLICKRAGE; + type: EventType name: string; key: number; label: string; @@ -35,6 +37,7 @@ interface ConsoleEvent extends IEvent { interface ClickEvent extends IEvent { targetContent: string; count: number; + hesitation: number; } interface InputEvent extends IEvent { @@ -98,12 +101,13 @@ export class Click extends Event { readonly name = 'Click'; targetContent = ''; count: number; + hesitation: number = 0; constructor(evt: ClickEvent, isClickRage?: boolean) { - console.log(evt); super(evt); this.targetContent = evt.targetContent; this.count = evt.count; + this.hesitation = evt.hesitation; if (isClickRage) { this.type = CLICKRAGE; } diff --git a/frontend/app/types/session/issue.ts b/frontend/app/types/session/issue.ts index 2abedc5ea..8f5f415e3 100644 --- a/frontend/app/types/session/issue.ts +++ b/frontend/app/types/session/issue.ts @@ -1,6 +1,6 @@ import Record from 'Types/Record'; -const types = { +export const types = { ALL: 'all', JS_EXCEPTION: 'js_exception', BAD_REQUEST: 'bad_request', diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index 3b254ae4b..a473ed623 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -2,12 +2,37 @@ import { Duration } from 'luxon'; import SessionEvent, { TYPES, EventData, InjectedEvent } from './event'; import StackEvent from './stackEvent'; import SessionError, { IError } from './error'; -import Issue, { IIssue } from './issue'; +import Issue, { IIssue, types as issueTypes } from './issue'; import { Note } from 'App/services/NotesService' const HASH_MOD = 1610612741; const HASH_P = 53; +function mergeEventLists(arr1: any[], arr2: any[]) { + let merged = []; + let index1 = 0; + let index2 = 0; + let current = 0; + + while (current < (arr1.length + arr2.length)) { + + let isArr1Depleted = index1 >= arr1.length; + let isArr2Depleted = index2 >= arr2.length; + + if (!isArr1Depleted && (isArr2Depleted || (arr1[index1].timestamp < arr2[index2].timestamp))) { + merged[current] = arr1[index1]; + index1++; + } else { + merged[current] = arr2[index2]; + index2++; + } + + current++; + } + + return merged; +} + function hashString(s: string): number { let mul = 1; let hash = 0; @@ -158,6 +183,8 @@ export default class Session { agentToken: ISession["agentToken"] notes: ISession["notes"] notesWithEvents: ISession["notesWithEvents"] + frustrations: Array + fileKey: ISession["fileKey"] constructor(plainSession?: ISession) { @@ -217,8 +244,20 @@ export default class Session { (i, k) => new Issue({ ...i, time: i.timestamp - startedAt, key: k })) || []; const rawNotes = notes; - const notesWithEvents = [...rawEvents, ...rawNotes].sort((a, b) => { - // @ts-ignore just in case + + + const frustrationEvents = events.filter(ev => { + if (ev.type === TYPES.CLICK || ev.type === TYPES.INPUT) { + // @ts-ignore + return ev.hesitation > 1000 + } + return ev.type === TYPES.CLICKRAGE + } + ) + const frustrationIssues = issuesList.filter(i => i.type === issueTypes.MOUSE_THRASHING) + + const frustrationList = [...frustrationEvents, ...frustrationIssues].sort((a, b) => { + // @ts-ignore const aTs = a.timestamp || a.time; // @ts-ignore const bTs = b.timestamp || b.time; @@ -226,6 +265,11 @@ export default class Session { return aTs - bTs; }) || []; + const mixedEventsWithIssues = mergeEventLists( + mergeEventLists(rawEvents, rawNotes), + frustrationIssues + ) + Object.assign(this, { ...session, isIOS: session.platform === 'ios', @@ -255,7 +299,8 @@ export default class Session { domURL, devtoolsURL, notes, - notesWithEvents: notesWithEvents, + notesWithEvents: mixedEventsWithIssues, + frustrations: frustrationList, }) } } \ No newline at end of file diff --git a/frontend/scripts/icons.ts b/frontend/scripts/icons.ts index dab11b224..0d7f67a31 100644 --- a/frontend/scripts/icons.ts +++ b/frontend/scripts/icons.ts @@ -32,6 +32,8 @@ const plugins = (removeFill = true) => { name: 'preset-default', params: { overrides: { + cleanupIds: false, + prefixIds: false, inlineStyles: { onlyMatchedOnce: false, }, @@ -63,12 +65,10 @@ const plugins = (removeFill = true) => { ] } } -// fs.promises.mkdir('/tmp/a/apple', { recursive: true }) -// .then(() => { - fs.writeFileSync(`${UI_DIRNAME}/SVG.tsx`, ` +fs.writeFileSync(`${UI_DIRNAME}/SVG.tsx`, ` import React from 'react'; -export type IconNames = ${icons.map(icon => "'"+ icon.slice(0, -4) + "'").join(' | ')}; +export type IconNames = ${icons.map((icon) => "'"+ icon.slice(0, -4) + "'").join(' | ')}; interface Props { name: IconNames; @@ -88,10 +88,11 @@ ${icons.map(icon => { const { data } = optimize(svg, plugins(canOptimize)); return ` case '${icon.slice(0, -4)}': return ${data.replace(/xlink\:href/g, 'xlinkHref') .replace(/xmlns\:xlink/g, 'xmlnsXlink') - .replace(/clip-path/g, 'clipPath') - .replace(/clip-rule/g, 'clipRule') + .replace(/clip\-path/g, 'clipPath') + .replace(/clip\-rule/g, 'clipRule') // hack to keep fill rule for some icons like stop recording square .replace(/clipRule="evenoddCustomFill"/g, 'clipRule="evenodd" fillRule="evenodd"') + .replace(/fill-rule/g, 'fillRule') .replace(/fill-opacity/g, 'fillOpacity') .replace(/stop-color/g, 'stopColor') From 813a128d4f311437579f101c670a6bc182e3e8c1 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 16 Mar 2023 17:21:01 +0100 Subject: [PATCH 28/29] change(ui): fix conf --- frontend/scripts/icons.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/scripts/icons.ts b/frontend/scripts/icons.ts index 0d7f67a31..3a2f9808c 100644 --- a/frontend/scripts/icons.ts +++ b/frontend/scripts/icons.ts @@ -32,8 +32,6 @@ const plugins = (removeFill = true) => { name: 'preset-default', params: { overrides: { - cleanupIds: false, - prefixIds: false, inlineStyles: { onlyMatchedOnce: false, }, From 71503680d62954e1892b7ccfe678560c17989e7c Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 16 Mar 2023 17:39:39 +0100 Subject: [PATCH 29/29] change(ui): fix types, add frustrations for v2 fetch --- frontend/app/types/session/session.ts | 45 ++++++++++++++++++--------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index 6e6fe5aaa..239871600 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -9,7 +9,7 @@ import { toJS } from 'mobx'; const HASH_MOD = 1610612741; const HASH_P = 53; -function mergeEventLists(arr1: any[], arr2: any[]) { +function mergeEventLists, Y extends Record>(arr1: T[], arr2: Y[]): Array { let merged = []; let index1 = 0; let index2 = 0; @@ -33,6 +33,12 @@ function mergeEventLists(arr1: any[], arr2: any[]) { return merged; } +function sortEvents(a: Record, b: Record) { + const aTs = a.timestamp || a.time; + const bTs = b.timestamp || b.time; + + return aTs - bTs; +} function hashString(s: string): number { let mul = 1; @@ -259,19 +265,12 @@ export default class Session { ) const frustrationIssues = issuesList.filter(i => i.type === issueTypes.MOUSE_THRASHING) - const frustrationList = [...frustrationEvents, ...frustrationIssues].sort((a, b) => { - // @ts-ignore - const aTs = a.timestamp || a.time; - // @ts-ignore - const bTs = b.timestamp || b.time; - - return aTs - bTs; - }) || []; + const frustrationList = [...frustrationEvents, ...frustrationIssues].sort(sortEvents) || []; const mixedEventsWithIssues = mergeEventLists( mergeEventLists(rawEvents, rawNotes), frustrationIssues - ) + ).sort(sortEvents) Object.assign(this, { ...session, @@ -303,11 +302,9 @@ export default class Session { domURL, devtoolsURL, notes, - notesWithEvents: notesWithEvents, - }); notesWithEvents: mixedEventsWithIssues, frustrations: frustrationList, - }) + }); } addEvents( @@ -347,14 +344,32 @@ export default class Session { }); } + const frustrationEvents = events.filter(ev => { + if (ev.type === TYPES.CLICK || ev.type === TYPES.INPUT) { + // @ts-ignore + return ev.hesitation > 1000 + } + return ev.type === TYPES.CLICKRAGE + } + ) + const frustrationIssues = issuesList.filter(i => i.type === issueTypes.MOUSE_THRASHING) + const frustrationList = [...frustrationEvents, ...frustrationIssues].sort(sortEvents) || []; + + const mixedEventsWithIssues = mergeEventLists( + rawEvents, + frustrationIssues + ).sort(sortEvents) + this.events = events; // @ts-ignore - this.notesWithEvents = rawEvents; + this.notesWithEvents = mixedEventsWithIssues; this.errors = exceptions; this.issues = issuesList; // @ts-ignore legacy code? no idea this.resources = resources; this.stackEvents = stackEventsList; + // @ts-ignore + this.frustrations = frustrationList; return this; } @@ -365,7 +380,7 @@ export default class Session { [...this.notesWithEvents, ...sessionNotes].sort((a, b) => { // @ts-ignore just in case const aTs = a.timestamp || a.time; - // @ts-ignore + // @ts-ignore supporting old code... const bTs = b.timestamp || b.time; return aTs - bTs;