From e42c467d4ff824d22b44a8e216e8b26a2dc3661e Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 7 Feb 2023 17:13:22 +0100 Subject: [PATCH 001/253] feat(tracker): add input hesitation, change input change event handling --- tracker/tracker/src/main/modules/input.ts | 51 +++++++++++++++-------- tracker/tracker/src/main/utils.ts | 11 +++++ 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 15acecaa9..5cd74a9ba 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -1,5 +1,5 @@ import type App from '../app/index.js' -import { normSpaces, IN_BROWSER, getLabelAttribute } from '../utils.js' +import { normSpaces, IN_BROWSER, getLabelAttribute, debounce } from '../utils.js' import { hasTag } from '../app/guards.js' import { SetInputTarget, SetInputValue, SetInputChecked } from '../app/messages.gen.js' @@ -94,12 +94,14 @@ export default function (app: App, opts: Partial): void { }, opts, ) + function sendInputTarget(id: number, node: TextEditableElement): void { const label = getInputLabel(node) if (label !== '') { app.send(SetInputTarget(id, label)) } } + function sendInputValue(id: number, node: TextEditableElement | HTMLSelectElement): void { let value = node.value let inputMode: InputMode = options.defaultInputMode @@ -126,33 +128,20 @@ export default function (app: App, opts: Partial): void { value = '' break } - + // @ts-ignore if hesitationTime > 150 add it ??? + console.log(node.or_inputHesitation) app.send(SetInputValue(id, value, mask)) } const inputValues: Map = new Map() const checkableValues: Map = new Map() - const registeredTargets: Set = new Set() app.attachStopCallback(() => { inputValues.clear() checkableValues.clear() - registeredTargets.clear() }) app.ticker.attach((): void => { - inputValues.forEach((value, id) => { - const node = app.nodes.getNode(id) as HTMLInputElement - if (!node) return inputValues.delete(id) - if (value !== node.value) { - inputValues.set(id, node.value) - if (!registeredTargets.has(id)) { - registeredTargets.add(id) - sendInputTarget(id, node) - } - sendInputValue(id, node) - } - }) checkableValues.forEach((checked, id) => { const node = app.nodes.getNode(id) as HTMLInputElement if (!node) return checkableValues.delete(id) @@ -162,7 +151,11 @@ export default function (app: App, opts: Partial): void { } }) }) - app.ticker.attach(Set.prototype.clear, 100, false, registeredTargets) + + const debouncedUpdate = debounce((id: number, node: TextEditableElement) => { + sendInputTarget(id, node) + sendInputValue(id, node) + }, 125) app.nodes.attachNodeCallback( app.safe((node: Node): void => { @@ -180,6 +173,30 @@ export default function (app: App, opts: Partial): void { if (isTextEditable(node)) { inputValues.set(id, node.value) sendInputValue(id, node) + + node.addEventListener('focus', () => { + // @ts-ignore + Object.assign(node, { or_focusStart: +new Date() }) + }) + node.addEventListener('input', (e) => { + const value = (e.target as HTMLInputElement).value + if (inputValues.get(id) === '' && value !== '') { + const inputTime = +new Date() + // @ts-ignore + const hesitationTime = inputTime - node.or_focusStart + Object.assign(node, { or_inputHesitation: hesitationTime }) + } + inputValues.set(id, value) + debouncedUpdate(id, node) + }) + node.addEventListener('change', (e) => { + const value = (e.target as HTMLInputElement).value + if (inputValues.get(id) !== value) { + inputValues.set(id, value) + debouncedUpdate(id, node) + } + Object.assign(node, { or_inputHesitation: undefined, or_focusStart: undefined }) + }) return } if (isCheckable(node)) { diff --git a/tracker/tracker/src/main/utils.ts b/tracker/tracker/src/main/utils.ts index 739821ea9..daa9f74ce 100644 --- a/tracker/tracker/src/main/utils.ts +++ b/tracker/tracker/src/main/utils.ts @@ -81,3 +81,14 @@ export function hasOpenreplayAttribute(e: Element, attr: string): boolean { return false } + +export function debounce(func: (...args: any[]) => void, timeout = 125) { + let timer: NodeJS.Timeout + return (...args: any[]) => { + clearTimeout(timer) + // @ts-ignore + timer = setTimeout(() => { + func.apply(this, args) + }, timeout) + } +} From 969b192146f79151cbfc942824dbc5db20f1a720 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 7 Feb 2023 17:59:32 +0100 Subject: [PATCH 002/253] change(tracker): fix event attach and types --- tracker/tracker/src/main/app/index.ts | 5 +- tracker/tracker/src/main/modules/input.ts | 60 +++++++++++++---------- tracker/tracker/src/main/utils.ts | 2 +- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index f6d947dd9..10bbadf03 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -278,11 +278,14 @@ export default class App { listener: EventListener, useSafe = true, useCapture = true, + onlyStop = false, ): void { if (useSafe) { listener = this.safe(listener) } - this.attachStartCallback(() => target?.addEventListener(type, listener, useCapture), useSafe) + if (!onlyStop) { + this.attachStartCallback(() => target?.addEventListener(type, listener, useCapture), useSafe) + } this.attachStopCallback(() => target?.removeEventListener(type, listener, useCapture), useSafe) } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 5cd74a9ba..210a1717a 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -7,6 +7,7 @@ const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', ' // TODO: take into consideration "contenteditable" attribute type TextEditableElement = HTMLInputElement | HTMLTextAreaElement + function isTextEditable(node: any): node is TextEditableElement { if (hasTag(node, 'textarea')) { return true @@ -18,7 +19,7 @@ function isTextEditable(node: any): node is TextEditableElement { return INPUT_TYPES.includes(node.type) } -function isCheckable(node: any): node is HTMLInputElement { +function isCheckbox(node: any): node is HTMLInputElement { if (!hasTag(node, 'input')) { return false } @@ -134,22 +135,11 @@ export default function (app: App, opts: Partial): void { } const inputValues: Map = new Map() - const checkableValues: Map = new Map() + const checkboxValues: Map = new Map() app.attachStopCallback(() => { inputValues.clear() - checkableValues.clear() - }) - - app.ticker.attach((): void => { - checkableValues.forEach((checked, id) => { - const node = app.nodes.getNode(id) as HTMLInputElement - if (!node) return checkableValues.delete(id) - if (checked !== node.checked) { - checkableValues.set(id, node.checked) - app.send(SetInputChecked(id, node.checked)) - } - }) + checkboxValues.clear() }) const debouncedUpdate = debounce((id: number, node: TextEditableElement) => { @@ -166,19 +156,20 @@ export default function (app: App, opts: Partial): void { // TODO: support multiple select (?): use selectedOptions; Need send target? if (hasTag(node, 'select')) { sendInputValue(id, node) - app.attachEventListener(node, 'change', () => { + const handler = () => { sendInputValue(id, node) - }) + } + node.addEventListener('change', handler) + app.attachEventListener(node, 'change', handler, false, true, true) } + if (isTextEditable(node)) { inputValues.set(id, node.value) sendInputValue(id, node) - - node.addEventListener('focus', () => { - // @ts-ignore + const setFocus = () => { Object.assign(node, { or_focusStart: +new Date() }) - }) - node.addEventListener('input', (e) => { + } + const inputEvent = (e: InputEvent) => { const value = (e.target as HTMLInputElement).value if (inputValues.get(id) === '' && value !== '') { const inputTime = +new Date() @@ -188,20 +179,37 @@ export default function (app: App, opts: Partial): void { } inputValues.set(id, value) debouncedUpdate(id, node) - }) - node.addEventListener('change', (e) => { + } + const changeEvent = (e: InputEvent) => { const value = (e.target as HTMLInputElement).value if (inputValues.get(id) !== value) { inputValues.set(id, value) debouncedUpdate(id, node) } Object.assign(node, { or_inputHesitation: undefined, or_focusStart: undefined }) - }) + } + node.addEventListener('focus', setFocus) + node.addEventListener('input', inputEvent) + node.addEventListener('change', changeEvent) + app.attachEventListener(node, 'focus', setFocus, false, true, true) + app.attachEventListener(node, 'input', inputEvent, false, true, true) + app.attachEventListener(node, 'change', changeEvent, false, true, true) return } - if (isCheckable(node)) { - checkableValues.set(id, node.checked) + + if (isCheckbox(node)) { + checkboxValues.set(id, node.checked) app.send(SetInputChecked(id, node.checked)) + const checkboxChange = (e: InputEvent) => { + const value = (e.target as HTMLInputElement).checked + if (checkboxValues.get(id) !== value) { + checkboxValues.set(id, value) + app.send(SetInputChecked(id, value)) + } + } + node.addEventListener('change', checkboxChange) + app.attachEventListener(node, 'change', checkboxChange, false, true, true) + return } }), diff --git a/tracker/tracker/src/main/utils.ts b/tracker/tracker/src/main/utils.ts index daa9f74ce..8f99a4c74 100644 --- a/tracker/tracker/src/main/utils.ts +++ b/tracker/tracker/src/main/utils.ts @@ -86,8 +86,8 @@ export function debounce(func: (...args: any[]) => void, timeout = 125) { let timer: NodeJS.Timeout return (...args: any[]) => { clearTimeout(timer) - // @ts-ignore timer = setTimeout(() => { + // @ts-ignore func.apply(this, args) }, timeout) } From 018f618e127519e6e467e401f7291a0941999cfe Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Feb 2023 15:18:00 +0100 Subject: [PATCH 003/253] change(tracker): add mouse shake and update input hesitation message --- backend/pkg/messages/messages.go | 10 +++-- backend/pkg/messages/read-message.go | 3 ++ ee/connectors/msgcodec/messages.py | 3 +- ee/connectors/msgcodec/msgcodec.py | 1 + .../web/messages/RawMessageReader.gen.ts | 2 + frontend/app/player/web/messages/raw.gen.ts | 1 + .../app/player/web/messages/tracker.gen.ts | 4 +- mobs/messages.rb | 1 + tracker/tracker/src/common/messages.gen.ts | 1 + tracker/tracker/src/main/app/messages.gen.ts | 2 + tracker/tracker/src/main/modules/input.ts | 7 ++-- tracker/tracker/src/main/modules/mouse.ts | 41 +++++++++++++++++++ .../src/webworker/MessageEncoder.gen.ts | 2 +- 13 files changed, 68 insertions(+), 10 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index a96f98de8..f3c422af2 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -531,17 +531,19 @@ func (msg *SetInputTarget) TypeID() int { type SetInputValue struct { message - ID uint64 - Value string - Mask int64 + ID uint64 + Value string + HesitationTime int64 + Mask int64 } func (msg *SetInputValue) Encode() []byte { - buf := make([]byte, 31+len(msg.Value)) + buf := make([]byte, 41+len(msg.Value)) buf[0] = 18 p := 1 p = WriteUint(msg.ID, buf, p) p = WriteString(msg.Value, buf, p) + p = WriteInt(msg.HesitationTime, buf, p) p = WriteInt(msg.Mask, buf, p) return buf[:p] } diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index ecc00183f..56d0662aa 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -270,6 +270,9 @@ func DecodeSetInputValue(reader BytesReader) (Message, error) { if msg.Value, err = reader.ReadString(); err != nil { return nil, err } + if msg.HesitationTime, err = reader.ReadInt(); err != nil { + return nil, err + } if msg.Mask, err = reader.ReadInt(); err != nil { return nil, err } diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 54f8df955..53d993bd2 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -163,9 +163,10 @@ class SetInputTarget(Message): class SetInputValue(Message): __id__ = 18 - def __init__(self, id, value, mask): + def __init__(self, id, value, hesitation_time, mask): self.id = id self.value = value + self.hesitation_time = hesitation_time self.mask = mask diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 0ba21ea12..2b1d05a07 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -200,6 +200,7 @@ class MessageCodec(Codec): return SetInputValue( id=self.read_uint(reader), value=self.read_string(reader), + hesitation_time=self.read_int(reader), mask=self.read_int(reader) ) diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 793f609f5..50b5a1a68 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -172,11 +172,13 @@ export default class RawMessageReader extends PrimitiveReader { case 18: { const id = this.readUint(); if (id === null) { return resetPointer() } const value = this.readString(); if (value === null) { return resetPointer() } + const hesitationTime = this.readInt(); if (hesitationTime === null) { return resetPointer() } const mask = this.readInt(); if (mask === null) { return resetPointer() } return { tp: MType.SetInputValue, id, value, + hesitationTime, mask, }; } diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index b51edb40e..8e6d613a4 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -156,6 +156,7 @@ export interface RawSetInputValue { tp: MType.SetInputValue, id: number, value: string, + hesitationTime: number, mask: number, } diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 2c171011c..13f1d82e7 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -98,6 +98,7 @@ type TrSetInputValue = [ type: 18, id: number, value: string, + hesitationTime: number, mask: number, ] @@ -549,7 +550,8 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { tp: MType.SetInputValue, id: tMsg[1], value: tMsg[2], - mask: tMsg[3], + hesitationTime: tMsg[3], + mask: tMsg[4], } } diff --git a/mobs/messages.rb b/mobs/messages.rb index ef36ebfa7..c5ba732b6 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -94,6 +94,7 @@ end message 18, 'SetInputValue' do uint 'ID' string 'Value' + int 'HesitationTime' int 'Mask' end message 19, 'SetInputChecked' do diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 8f61ab917..08461e6b8 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -159,6 +159,7 @@ export type SetInputValue = [ /*type:*/ Type.SetInputValue, /*id:*/ number, /*value:*/ string, + /*hesitationTime:*/ number, /*mask:*/ number, ] diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index bdec57a0f..e8005a430 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -172,12 +172,14 @@ export function SetInputTarget( export function SetInputValue( id: number, value: string, + hesitationTime: number, mask: number, ): Messages.SetInputValue { return [ Messages.Type.SetInputValue, id, value, + hesitationTime, mask, ] } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 210a1717a..83a43313d 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -129,9 +129,10 @@ export default function (app: App, opts: Partial): void { value = '' break } - // @ts-ignore if hesitationTime > 150 add it ??? - console.log(node.or_inputHesitation) - app.send(SetInputValue(id, value, mask)) + + // @ts-ignore maybe if hesitationTime > 150 ? + const hesitationTime = node.or_inputHesitation || 0 + app.send(SetInputValue(id, value, hesitationTime, mask)) } const inputValues: Map = new Map() diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index b00d6d304..7ed64f1c4 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -107,12 +107,45 @@ export default function (app: App): void { let mouseTargetTime = 0 let selectorMap: { [id: number]: string } = {} + let velocity = 0 + let direction = 0 + let directionChangeCount = 0 + let distance = 0 + let checkIntervalId: NodeJS.Timer + const shakeThreshold = 0.005 + const shakeCheckInterval = 225 + + function checkMouseShaking() { + const nextVelocity = distance / shakeCheckInterval + + if (!velocity) { + velocity = nextVelocity + return + } + + const acceleration = (nextVelocity - velocity) / shakeCheckInterval + if (directionChangeCount && acceleration > shakeThreshold) { + console.log('Mouse shake detected!') + } + + distance = 0 + directionChangeCount = 0 + velocity = nextVelocity + } + + app.attachStartCallback(() => { + checkIntervalId = setInterval(() => checkMouseShaking(), shakeCheckInterval) + }) + app.attachStopCallback(() => { mousePositionX = -1 mousePositionY = -1 mousePositionChanged = false mouseTarget = null selectorMap = {} + if (checkIntervalId) { + clearInterval(checkIntervalId) + } }) const sendMouseMove = (): void => { @@ -146,6 +179,14 @@ export default function (app: App): void { mousePositionX = e.clientX + left mousePositionY = e.clientY + top mousePositionChanged = true + + const nextDirection = Math.sign(e.movementX) + distance += Math.abs(e.movementX) + Math.abs(e.movementY) + + if (nextDirection !== direction) { + direction = nextDirection + directionChangeCount++ + } }, false, ) diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index e6e522dd4..60079647f 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -67,7 +67,7 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.SetInputValue: - return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) + return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) && this.int(msg[4]) break case Messages.Type.SetInputChecked: From f5f7f3726065f5cff53bca23a52a794d3bed72e5 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Feb 2023 15:58:14 +0100 Subject: [PATCH 004/253] change(tracker): tune shake check --- tracker/tracker/src/main/modules/mouse.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 7ed64f1c4..c94f117fa 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -112,7 +112,7 @@ export default function (app: App): void { let directionChangeCount = 0 let distance = 0 let checkIntervalId: NodeJS.Timer - const shakeThreshold = 0.005 + const shakeThreshold = 0.008 const shakeCheckInterval = 225 function checkMouseShaking() { @@ -124,7 +124,7 @@ export default function (app: App): void { } const acceleration = (nextVelocity - velocity) / shakeCheckInterval - if (directionChangeCount && acceleration > shakeThreshold) { + if (directionChangeCount > 3 && acceleration > shakeThreshold) { console.log('Mouse shake detected!') } From 3266f464406bfd8a231d08261c894ddffe623d25 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 9 Feb 2023 16:38:13 +0100 Subject: [PATCH 005/253] fix(tracker): some events changes... --- backend/pkg/messages/filters.go | 2 +- backend/pkg/messages/messages.go | 36 ++++++++++--- backend/pkg/messages/read-message.go | 20 +++++-- ee/connectors/msgcodec/messages.py | 12 ++++- ee/connectors/msgcodec/msgcodec.py | 8 ++- .../web/messages/RawMessageReader.gen.ts | 2 - frontend/app/player/web/messages/raw.gen.ts | 1 - .../app/player/web/messages/tracker.gen.ts | 13 +++-- mobs/messages.rb | 6 ++- tracker/tracker/src/common/messages.gen.ts | 11 +++- tracker/tracker/src/main/app/index.ts | 5 +- tracker/tracker/src/main/app/messages.gen.ts | 15 +++++- tracker/tracker/src/main/modules/input.ts | 52 +++++++++---------- .../src/webworker/MessageEncoder.gen.ts | 6 ++- 14 files changed, 133 insertions(+), 56 deletions(-) diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index 30e266194..370dffb99 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,7 +2,7 @@ package messages func IsReplayerType(id int) bool { - return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id + return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 83 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id } func IsIOSType(id int) bool { diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index f3c422af2..5efb81474 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -79,6 +79,7 @@ const ( MsgBatchMeta = 80 MsgBatchMetadata = 81 MsgPartitionedMessage = 82 + MsgInputChange = 83 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 @@ -531,19 +532,17 @@ func (msg *SetInputTarget) TypeID() int { type SetInputValue struct { message - ID uint64 - Value string - HesitationTime int64 - Mask int64 + ID uint64 + Value string + Mask int64 } func (msg *SetInputValue) Encode() []byte { - buf := make([]byte, 41+len(msg.Value)) + buf := make([]byte, 31+len(msg.Value)) buf[0] = 18 p := 1 p = WriteUint(msg.ID, buf, p) p = WriteString(msg.Value, buf, p) - p = WriteInt(msg.HesitationTime, buf, p) p = WriteInt(msg.Mask, buf, p) return buf[:p] } @@ -2119,6 +2118,31 @@ func (msg *PartitionedMessage) TypeID() int { return 82 } +type InputChange struct { + message + ID uint64 + Label string + HesitationTime int64 +} + +func (msg *InputChange) Encode() []byte { + buf := make([]byte, 31+len(msg.Label)) + buf[0] = 83 + p := 1 + p = WriteUint(msg.ID, buf, p) + p = WriteString(msg.Label, buf, p) + p = WriteInt(msg.HesitationTime, buf, p) + return buf[:p] +} + +func (msg *InputChange) Decode() Message { + return msg +} + +func (msg *InputChange) TypeID() int { + return 83 +} + type IssueEvent struct { message MessageID uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 56d0662aa..907c399e6 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -270,9 +270,6 @@ func DecodeSetInputValue(reader BytesReader) (Message, error) { if msg.Value, err = reader.ReadString(); err != nil { return nil, err } - if msg.HesitationTime, err = reader.ReadInt(); err != nil { - return nil, err - } if msg.Mask, err = reader.ReadInt(); err != nil { return nil, err } @@ -1296,6 +1293,21 @@ func DecodePartitionedMessage(reader BytesReader) (Message, error) { return msg, err } +func DecodeInputChange(reader BytesReader) (Message, error) { + var err error = nil + msg := &InputChange{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Label, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.HesitationTime, err = reader.ReadInt(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1905,6 +1917,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeBatchMetadata(reader) case 82: return DecodePartitionedMessage(reader) + case 83: + return DecodeInputChange(reader) case 125: return DecodeIssueEvent(reader) case 126: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 53d993bd2..0c2777c6b 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -163,10 +163,9 @@ class SetInputTarget(Message): class SetInputValue(Message): __id__ = 18 - def __init__(self, id, value, hesitation_time, mask): + def __init__(self, id, value, mask): self.id = id self.value = value - self.hesitation_time = hesitation_time self.mask = mask @@ -745,6 +744,15 @@ class PartitionedMessage(Message): self.part_total = part_total +class InputChange(Message): + __id__ = 83 + + def __init__(self, id, label, hesitation_time): + self.id = id + self.label = label + self.hesitation_time = hesitation_time + + class IssueEvent(Message): __id__ = 125 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 2b1d05a07..861a7f1e6 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -200,7 +200,6 @@ class MessageCodec(Codec): return SetInputValue( id=self.read_uint(reader), value=self.read_string(reader), - hesitation_time=self.read_int(reader), mask=self.read_int(reader) ) @@ -661,6 +660,13 @@ class MessageCodec(Codec): part_total=self.read_uint(reader) ) + if message_id == 83: + return InputChange( + id=self.read_uint(reader), + label=self.read_string(reader), + hesitation_time=self.read_int(reader) + ) + if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 50b5a1a68..793f609f5 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -172,13 +172,11 @@ export default class RawMessageReader extends PrimitiveReader { case 18: { const id = this.readUint(); if (id === null) { return resetPointer() } const value = this.readString(); if (value === null) { return resetPointer() } - const hesitationTime = this.readInt(); if (hesitationTime === null) { return resetPointer() } const mask = this.readInt(); if (mask === null) { return resetPointer() } return { tp: MType.SetInputValue, id, value, - hesitationTime, mask, }; } diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 8e6d613a4..b51edb40e 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -156,7 +156,6 @@ export interface RawSetInputValue { tp: MType.SetInputValue, id: number, value: string, - hesitationTime: number, mask: number, } diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 13f1d82e7..a4b208e44 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -98,7 +98,6 @@ type TrSetInputValue = [ type: 18, id: number, value: string, - hesitationTime: number, mask: number, ] @@ -430,8 +429,15 @@ type TrPartitionedMessage = [ partTotal: number, ] +type TrInputChange = [ + type: 83, + id: number, + label: string, + hesitationTime: number, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -550,8 +556,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { tp: MType.SetInputValue, id: tMsg[1], value: tMsg[2], - hesitationTime: tMsg[3], - mask: tMsg[4], + mask: tMsg[3], } } diff --git a/mobs/messages.rb b/mobs/messages.rb index c5ba732b6..2838029cb 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -94,7 +94,6 @@ end message 18, 'SetInputValue' do uint 'ID' string 'Value' - int 'HesitationTime' int 'Mask' end message 19, 'SetInputChecked' do @@ -476,6 +475,11 @@ message 82, 'PartitionedMessage', :replayer => false do uint 'PartTotal' end +message 83, 'InputChange', :replayer => false do + uint 'ID' + string 'Label' + int 'HesitationTime' +end ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 08461e6b8..b9c5b5f20 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -63,6 +63,7 @@ export declare const enum Type { Zustand = 79, BatchMetadata = 81, PartitionedMessage = 82, + InputChange = 83, } @@ -159,7 +160,6 @@ export type SetInputValue = [ /*type:*/ Type.SetInputValue, /*id:*/ number, /*value:*/ string, - /*hesitationTime:*/ number, /*mask:*/ number, ] @@ -491,6 +491,13 @@ export type PartitionedMessage = [ /*partTotal:*/ number, ] +export type InputChange = [ + /*type:*/ Type.InputChange, + /*id:*/ number, + /*label:*/ string, + /*hesitationTime:*/ number, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange export default Message diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 10bbadf03..f6d947dd9 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -278,14 +278,11 @@ export default class App { listener: EventListener, useSafe = true, useCapture = true, - onlyStop = false, ): void { if (useSafe) { listener = this.safe(listener) } - if (!onlyStop) { - this.attachStartCallback(() => target?.addEventListener(type, listener, useCapture), useSafe) - } + this.attachStartCallback(() => target?.addEventListener(type, listener, useCapture), useSafe) this.attachStopCallback(() => target?.removeEventListener(type, listener, useCapture), useSafe) } diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index e8005a430..d5926d240 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -172,14 +172,12 @@ export function SetInputTarget( export function SetInputValue( id: number, value: string, - hesitationTime: number, mask: number, ): Messages.SetInputValue { return [ Messages.Type.SetInputValue, id, value, - hesitationTime, mask, ] } @@ -794,3 +792,16 @@ export function PartitionedMessage( ] } +export function InputChange( + id: number, + label: string, + hesitationTime: number, +): Messages.InputChange { + return [ + Messages.Type.InputChange, + id, + label, + hesitationTime, + ] +} + diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 83a43313d..f6b915e25 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -1,7 +1,7 @@ import type App from '../app/index.js' import { normSpaces, IN_BROWSER, getLabelAttribute, debounce } from '../utils.js' import { hasTag } from '../app/guards.js' -import { SetInputTarget, SetInputValue, SetInputChecked } from '../app/messages.gen.js' +import { InputChange, SetInputValue, SetInputChecked } from '../app/messages.gen.js' const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', 'date', 'tel'] @@ -86,6 +86,7 @@ export interface Options { } export default function (app: App, opts: Partial): void { + const inputHesitationMap: Map = new Map() const options: Options = Object.assign( { obscureInputNumbers: true, @@ -96,10 +97,12 @@ export default function (app: App, opts: Partial): void { opts, ) - function sendInputTarget(id: number, node: TextEditableElement): void { + function sendInputChange(id: number, node: TextEditableElement): void { const label = getInputLabel(node) + // @ts-ignore maybe if hesitationTime > 150 ? + const { hesitation } = inputHesitationMap.get(id) if (label !== '') { - app.send(SetInputTarget(id, label)) + app.send(InputChange(id, label, hesitation)) } } @@ -130,9 +133,7 @@ export default function (app: App, opts: Partial): void { break } - // @ts-ignore maybe if hesitationTime > 150 ? - const hesitationTime = node.or_inputHesitation || 0 - app.send(SetInputValue(id, value, hesitationTime, mask)) + app.send(SetInputValue(id, value, mask)) } const inputValues: Map = new Map() @@ -144,10 +145,14 @@ export default function (app: App, opts: Partial): void { }) const debouncedUpdate = debounce((id: number, node: TextEditableElement) => { - sendInputTarget(id, node) + sendInputChange(id, node) sendInputValue(id, node) }, 125) + const debouncedTyping = debounce((id: number, node: TextEditableElement) => { + sendInputValue(id, node) + }, 60) + app.nodes.attachNodeCallback( app.safe((node: Node): void => { const id = app.nodes.getID(node) @@ -160,26 +165,27 @@ export default function (app: App, opts: Partial): void { const handler = () => { sendInputValue(id, node) } - node.addEventListener('change', handler) - app.attachEventListener(node, 'change', handler, false, true, true) + app.nodes.attachNodeListener(node, 'change', handler) } if (isTextEditable(node)) { inputValues.set(id, node.value) + inputHesitationMap.set(id, { hesitation: 0, focusEv: 0 }) sendInputValue(id, node) const setFocus = () => { - Object.assign(node, { or_focusStart: +new Date() }) + inputHesitationMap.set(id, { hesitation: 0, focusEv: +new Date() }) } const inputEvent = (e: InputEvent) => { const value = (e.target as HTMLInputElement).value if (inputValues.get(id) === '' && value !== '') { const inputTime = +new Date() + const { focusEv } = inputHesitationMap.get(id)! // @ts-ignore - const hesitationTime = inputTime - node.or_focusStart - Object.assign(node, { or_inputHesitation: hesitationTime }) + const hesitationTime = inputTime - focusEv + inputHesitationMap.set(id, { hesitation: hesitationTime, focusEv }) } inputValues.set(id, value) - debouncedUpdate(id, node) + debouncedTyping(id, node) } const changeEvent = (e: InputEvent) => { const value = (e.target as HTMLInputElement).value @@ -187,14 +193,11 @@ export default function (app: App, opts: Partial): void { inputValues.set(id, value) debouncedUpdate(id, node) } - Object.assign(node, { or_inputHesitation: undefined, or_focusStart: undefined }) + inputHesitationMap.set(id, { hesitation: 0, focusEv: 0 }) } - node.addEventListener('focus', setFocus) - node.addEventListener('input', inputEvent) - node.addEventListener('change', changeEvent) - app.attachEventListener(node, 'focus', setFocus, false, true, true) - app.attachEventListener(node, 'input', inputEvent, false, true, true) - app.attachEventListener(node, 'change', changeEvent, false, true, true) + app.nodes.attachNodeListener(node, 'focus', setFocus) + app.nodes.attachNodeListener(node, 'input', inputEvent) + app.nodes.attachNodeListener(node, 'change', changeEvent) return } @@ -203,13 +206,10 @@ export default function (app: App, opts: Partial): void { app.send(SetInputChecked(id, node.checked)) const checkboxChange = (e: InputEvent) => { const value = (e.target as HTMLInputElement).checked - if (checkboxValues.get(id) !== value) { - checkboxValues.set(id, value) - app.send(SetInputChecked(id, value)) - } + checkboxValues.set(id, value) + app.send(SetInputChecked(id, value)) } - node.addEventListener('change', checkboxChange) - app.attachEventListener(node, 'change', checkboxChange, false, true, true) + app.nodes.attachNodeListener(node, 'change', checkboxChange) return } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 60079647f..048f61001 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -67,7 +67,7 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.SetInputValue: - return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) && this.int(msg[4]) + return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) break case Messages.Type.SetInputChecked: @@ -254,6 +254,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) break + case Messages.Type.InputChange: + return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) + break + } } From 00be3708a16bcf64476b321ff15dede83c70a466 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 10:42:03 +0100 Subject: [PATCH 006/253] change(tracker): return ticker to track custom input changes --- tracker/tracker/src/main/app/ticker.ts | 6 ++++++ tracker/tracker/src/main/modules/input.ts | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/tracker/tracker/src/main/app/ticker.ts b/tracker/tracker/src/main/app/ticker.ts index 70bffdf4c..ee05ed93c 100644 --- a/tracker/tracker/src/main/app/ticker.ts +++ b/tracker/tracker/src/main/app/ticker.ts @@ -18,6 +18,12 @@ export default class Ticker { this.callbacks = [] } + /** + * @param {Callback} callback - repeated cb + * @param {number} n - number of turn skips; ticker have a 30 ms cycle + * @param {boolean} useSafe - using safe wrapper to check if app is active + * @param {object} thisArg - link to + * */ attach(callback: Callback, n = 0, useSafe = true, thisArg?: any) { if (thisArg) { callback = callback.bind(thisArg) diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index f6b915e25..9e929e782 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -153,6 +153,25 @@ export default function (app: App, opts: Partial): void { sendInputValue(id, node) }, 60) + app.ticker.attach(() => { + inputValues.forEach((value, id) => { + const node = app.nodes.getNode(id) as HTMLInputElement + if (!node) return inputValues.delete(id) + if (value !== node.value) { + inputValues.set(id, node.value) + sendInputValue(id, node) + } + }) + checkboxValues.forEach((checked, id) => { + const node = app.nodes.getNode(id) as HTMLInputElement + if (!node) return checkboxValues.delete(id) + if (checked !== node.checked) { + checkboxValues.set(id, node.checked) + app.send(SetInputChecked(id, node.checked)) + } + }) + }, 5) + app.nodes.attachNodeCallback( app.safe((node: Node): void => { const id = app.nodes.getID(node) From c57846eab3e18cadfda29b104aaa4230d4a5bae9 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 11:09:48 +0100 Subject: [PATCH 007/253] fix(tracker): fix typo --- frontend/app/player/web/managers/DOM/DOMManager.ts | 4 ++-- frontend/app/player/web/managers/DOM/VirtualDOM.ts | 2 +- tracker/tracker-assist/src/Assist.ts | 4 ++-- tracker/tracker-fetch/src/index.ts | 4 ++-- tracker/tracker/src/main/app/index.ts | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index bb4c999ae..2bd7a6c0c 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -287,7 +287,7 @@ export default class DOMManager extends ListWalker { } return - // @depricated since 4.0.2 in favor of adopted_ss_insert/delete_rule + add_owner as being common case for StyleSheets + // @deprecated since 4.0.2 in favor of adopted_ss_insert/delete_rule + add_owner as being common case for StyleSheets case MType.CssInsertRule: vn = this.vElements.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } @@ -306,7 +306,7 @@ export default class DOMManager extends ListWalker { } vn.onStyleSheet(sheet => deleteRule(sheet, msg)) return - // end @depricated + // end @deprecated case MType.CreateIFrameDocument: vn = this.vElements.get(msg.frameID) diff --git a/frontend/app/player/web/managers/DOM/VirtualDOM.ts b/frontend/app/player/web/managers/DOM/VirtualDOM.ts index 1be12d68c..91b75eb24 100644 --- a/frontend/app/player/web/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/web/managers/DOM/VirtualDOM.ts @@ -122,7 +122,7 @@ export class VElement extends VParent { type StyleSheetCallback = (s: CSSStyleSheet) => void export type StyleElement = HTMLStyleElement | SVGStyleElement -// @Depricated TODO: remove in favor of PostponedStyleSheet +// @deprecated TODO: remove in favor of PostponedStyleSheet export class VStyleElement extends VElement { private loaded = false private stylesheetCallbacks: StyleSheetCallback[] = [] diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 646360521..8d14740a5 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -31,9 +31,9 @@ export interface Options { controlConfirm: ConfirmOptions; recordingConfirm: ConfirmOptions; - // @depricated + // @deprecated confirmText?: string; - // @depricated + // @deprecated confirmStyle?: Properties; config: RTCConfiguration; diff --git a/tracker/tracker-fetch/src/index.ts b/tracker/tracker-fetch/src/index.ts index 10b75b7ea..cb9d9e4df 100644 --- a/tracker/tracker-fetch/src/index.ts +++ b/tracker/tracker-fetch/src/index.ts @@ -28,7 +28,7 @@ export interface Options { ignoreHeaders: Array | boolean sanitiser?: (RequestResponseData) => RequestResponseData | null - // Depricated + // @deprecated requestSanitizer?: any responseSanitizer?: any } @@ -49,7 +49,7 @@ export default function(opts: Partial = {}): (app: App | null) => Windo opts, ); if (options.requestSanitizer && options.responseSanitizer) { - console.warn("OpenReplay fetch plugin: `requestSanitizer` and `responseSanitizer` options are depricated. Please, use `sanitiser` instead (check out documentation at https://docs.openreplay.com/plugins/fetch).") + console.warn("OpenReplay fetch plugin: `requestSanitizer` and `responseSanitizer` options are deprecated. Please, use `sanitiser` instead (check out documentation at https://docs.openreplay.com/plugins/fetch).") } return (app: App | null) => { diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index f6d947dd9..faca22b9d 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -149,7 +149,7 @@ export default class App { } }) - // @depricated (use sessionHash on start instead) + // @deprecated (use sessionHash on start instead) if (sessionToken != null) { this.session.applySessionHash(sessionToken) } From c2501530a001882f6ef528120c742044ad71fd64 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 12:41:01 +0100 Subject: [PATCH 008/253] fix(tracker): capture selection event --- backend/pkg/messages/filters.go | 2 +- backend/pkg/messages/messages.go | 26 +++++++++++++ backend/pkg/messages/read-message.go | 17 ++++++++ ee/connectors/msgcodec/messages.py | 9 +++++ ee/connectors/msgcodec/msgcodec.py | 7 ++++ .../web/messages/RawMessageReader.gen.ts | 12 ++++++ .../app/player/web/messages/filters.gen.ts | 2 +- .../app/player/web/messages/message.gen.ts | 3 ++ frontend/app/player/web/messages/raw.gen.ts | 10 ++++- .../player/web/messages/tracker-legacy.gen.ts | 1 + .../app/player/web/messages/tracker.gen.ts | 18 ++++++++- mobs/messages.rb | 6 +++ tracker/tracker/src/common/messages.gen.ts | 10 ++++- tracker/tracker/src/main/app/messages.gen.ts | 13 +++++++ tracker/tracker/src/main/index.ts | 2 + tracker/tracker/src/main/modules/selection.ts | 39 +++++++++++++++++++ .../src/webworker/MessageEncoder.gen.ts | 4 ++ 17 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 tracker/tracker/src/main/modules/selection.ts diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index 370dffb99..f586474d0 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -10,5 +10,5 @@ func IsIOSType(id int) bool { } func IsDOMType(id int) bool { - return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id + return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 84 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 5efb81474..2e83ba0cf 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -80,6 +80,7 @@ const ( MsgBatchMetadata = 81 MsgPartitionedMessage = 82 MsgInputChange = 83 + MsgSelectionChange = 84 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 @@ -2143,6 +2144,31 @@ func (msg *InputChange) TypeID() int { return 83 } +type SelectionChange struct { + message + SelectionStart uint64 + SelectionEnd uint64 + Selection string +} + +func (msg *SelectionChange) Encode() []byte { + buf := make([]byte, 31+len(msg.Selection)) + buf[0] = 84 + p := 1 + p = WriteUint(msg.SelectionStart, buf, p) + p = WriteUint(msg.SelectionEnd, buf, p) + p = WriteString(msg.Selection, buf, p) + return buf[:p] +} + +func (msg *SelectionChange) Decode() Message { + return msg +} + +func (msg *SelectionChange) TypeID() int { + return 84 +} + type IssueEvent struct { message MessageID uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 907c399e6..4fe1e98f7 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1308,6 +1308,21 @@ func DecodeInputChange(reader BytesReader) (Message, error) { return msg, err } +func DecodeSelectionChange(reader BytesReader) (Message, error) { + var err error = nil + msg := &SelectionChange{} + if msg.SelectionStart, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.SelectionEnd, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Selection, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1919,6 +1934,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodePartitionedMessage(reader) case 83: return DecodeInputChange(reader) + case 84: + return DecodeSelectionChange(reader) case 125: return DecodeIssueEvent(reader) case 126: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 0c2777c6b..76aee98a2 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -753,6 +753,15 @@ class InputChange(Message): self.hesitation_time = hesitation_time +class SelectionChange(Message): + __id__ = 84 + + def __init__(self, selection_start, selection_end, selection): + self.selection_start = selection_start + self.selection_end = selection_end + self.selection = selection + + class IssueEvent(Message): __id__ = 125 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 861a7f1e6..f5986ceb4 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -667,6 +667,13 @@ class MessageCodec(Codec): hesitation_time=self.read_int(reader) ) + if message_id == 84: + return SelectionChange( + selection_start=self.read_uint(reader), + selection_end=self.read_uint(reader), + selection=self.read_string(reader) + ) + if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 793f609f5..32444011a 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -627,6 +627,18 @@ export default class RawMessageReader extends PrimitiveReader { }; } + case 84: { + const selectionStart = this.readUint(); if (selectionStart === null) { return resetPointer() } + const selectionEnd = this.readUint(); if (selectionEnd === null) { return resetPointer() } + const selection = this.readString(); if (selection === null) { return resetPointer() } + return { + tp: MType.SelectionChange, + selectionStart, + selectionEnd, + selection, + }; + } + case 90: { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const projectID = this.readUint(); if (projectID === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/filters.gen.ts b/frontend/app/player/web/messages/filters.gen.ts index cd664201e..c7a6f60cd 100644 --- a/frontend/app/player/web/messages/filters.gen.ts +++ b/frontend/app/player/web/messages/filters.gen.ts @@ -3,7 +3,7 @@ import { MType } from './raw.gen' -const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,90,93,96,100,102,103,105] +const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,84,90,93,96,100,102,103,105] export function isDOMType(t: MType) { return DOM_TYPES.includes(t) } \ No newline at end of file diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index b39d9ce46..5dd11b8c3 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -55,6 +55,7 @@ import type { RawAdoptedSsAddOwner, RawAdoptedSsRemoveOwner, RawZustand, + RawSelectionChange, RawIosSessionStart, RawIosCustomEvent, RawIosScreenChanges, @@ -169,6 +170,8 @@ export type AdoptedSsRemoveOwner = RawAdoptedSsRemoveOwner & Timed export type Zustand = RawZustand & Timed +export type SelectionChange = RawSelectionChange & Timed + export type IosSessionStart = RawIosSessionStart & Timed export type IosCustomEvent = RawIosCustomEvent & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index b51edb40e..165a6f9b2 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -53,6 +53,7 @@ export const enum MType { AdoptedSsAddOwner = 76, AdoptedSsRemoveOwner = 77, Zustand = 79, + SelectionChange = 84, IosSessionStart = 90, IosCustomEvent = 93, IosScreenChanges = 96, @@ -418,6 +419,13 @@ export interface RawZustand { state: string, } +export interface RawSelectionChange { + tp: MType.SelectionChange, + selectionStart: number, + selectionEnd: number, + selection: string, +} + export interface RawIosSessionStart { tp: MType.IosSessionStart, timestamp: number, @@ -489,4 +497,4 @@ export interface RawIosNetworkCall { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index bd8ba9c85..88f54434a 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -54,6 +54,7 @@ export const TP_MAP = { 76: MType.AdoptedSsAddOwner, 77: MType.AdoptedSsRemoveOwner, 79: MType.Zustand, + 84: MType.SelectionChange, 90: MType.IosSessionStart, 93: MType.IosCustomEvent, 96: MType.IosScreenChanges, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index a4b208e44..8d8a78d2e 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -436,8 +436,15 @@ type TrInputChange = [ hesitationTime: number, ] +type TrSelectionChange = [ + type: 84, + selectionStart: number, + selectionEnd: number, + selection: string, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -874,6 +881,15 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } + case 84: { + return { + tp: MType.SelectionChange, + selectionStart: tMsg[1], + selectionEnd: tMsg[2], + selection: tMsg[3], + } + } + default: return null } diff --git a/mobs/messages.rb b/mobs/messages.rb index 2838029cb..a6b59bd7d 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -481,6 +481,12 @@ message 83, 'InputChange', :replayer => false do int 'HesitationTime' end +message 84, 'SelectionChange' do + uint 'SelectionStart' + uint 'SelectionEnd' + string 'Selection' +end + ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do uint 'MessageID' diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index b9c5b5f20..1cc05b6fa 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -64,6 +64,7 @@ export declare const enum Type { BatchMetadata = 81, PartitionedMessage = 82, InputChange = 83, + SelectionChange = 84, } @@ -498,6 +499,13 @@ export type InputChange = [ /*hesitationTime:*/ number, ] +export type SelectionChange = [ + /*type:*/ Type.SelectionChange, + /*selectionStart:*/ number, + /*selectionEnd:*/ number, + /*selection:*/ string, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index d5926d240..af8dbe357 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -805,3 +805,16 @@ export function InputChange( ] } +export function SelectionChange( + selectionStart: number, + selectionEnd: number, + selection: string, +): Messages.SelectionChange { + return [ + Messages.Type.SelectionChange, + selectionStart, + selectionEnd, + selection, + ] +} + diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index eae1c867c..0688a22bf 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -24,6 +24,7 @@ import Focus from './modules/focus.js' import Fonts from './modules/fonts.js' import Network from './modules/network.js' import ConstructedStyleSheets from './modules/constructedStyleSheets.js' +import Selection from './modules/selection.js' import { IN_BROWSER, deprecationWarn, DOCS_HOST } from './utils.js' import type { Options as AppOptions } from './app/index.js' @@ -131,6 +132,7 @@ export default class API { Focus(app) Fonts(app) Network(app, options.network) + Selection(app) ;(window as any).__OPENREPLAY__ = this if (options.autoResetOnWindowOpen) { diff --git a/tracker/tracker/src/main/modules/selection.ts b/tracker/tracker/src/main/modules/selection.ts new file mode 100644 index 000000000..cd7004c44 --- /dev/null +++ b/tracker/tracker/src/main/modules/selection.ts @@ -0,0 +1,39 @@ +import type App from '../app/index.js' +import { SelectionChange } from '../app/messages.gen.js' + +function selection(app: App) { + app.attachEventListener(document, 'selectionchange', () => { + const selection = document.getSelection() + if (selection !== null && !selection.isCollapsed) { + const selectionStart = app.nodes.getID(selection.anchorNode!) + const selectionEnd = app.nodes.getID(selection.focusNode!) + const selectedText = selection.toString().replace(/\s+/g, ' ') + if (selectionStart && selectionEnd) { + app.send(SelectionChange(selectionStart, selectionEnd, selectedText)) + } + } else { + app.send(SelectionChange(0, 0, '')) + } + }) +} + +export default selection + +/** TODO: research how to get all in-between nodes inside selection range + * including nodes between anchor and focus nodes and their children + * without recursively searching the dom tree + */ + +// if (selection.rangeCount) { +// const nodes = []; +// for (let i = 0; i < selection.rangeCount; i++) { +// const range = selection.getRangeAt(i); +// let node: Node | null = range.startContainer; +// while (node) { +// nodes.push(node); +// if (node === range.endContainer) break; +// node = node.nextSibling; +// } +// } +// // send selected nodes +// } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 048f61001..10b3e205c 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -258,6 +258,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) break + case Messages.Type.SelectionChange: + return this.uint(msg[1]) && this.uint(msg[2]) && this.string(msg[3]) + break + } } From 24cbe068356b321f296616082b86e4a189aca8a6 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 15:28:14 +0100 Subject: [PATCH 009/253] change(player): add focus event to player --- .../app/player/web/managers/DOM/DOMManager.ts | 32 ++++++++------- .../web/managers/DOM/SelectionManager.ts | 39 +++++++++++++++++++ tracker/tracker/src/main/modules/selection.ts | 2 +- 3 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 frontend/app/player/web/managers/DOM/SelectionManager.ts diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index 2bd7a6c0c..efdc29a8e 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -2,25 +2,24 @@ import logger from 'App/logger'; import type Screen from '../../Screen/Screen'; import type { Message, SetNodeScroll } from '../../messages'; - import { MType } from '../../messages'; import ListWalker from '../../../common/ListWalker'; import StylesManager, { rewriteNodeStyleSheet } from './StylesManager'; import FocusManager from './FocusManager'; -import { - VElement, - VText, - VShadowRoot, - VDocument, - VNode, - VStyleElement, - PostponedStyleSheet, -} from './VirtualDOM'; +import SelectionManager from './SelectionManager'; import type { StyleElement } from './VirtualDOM'; -import { insertRule, deleteRule } from './safeCSSRules'; +import { + PostponedStyleSheet, + VDocument, + VElement, + VNode, + VShadowRoot, + VStyleElement, + VText, +} from './VirtualDOM'; +import { deleteRule, insertRule } from './safeCSSRules'; - -type HTMLElementWithValue = HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement +type HTMLElementWithValue = HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement; const IGNORED_ATTRS = [ "autocomplete" ]; const ATTR_NAME_REGEXP = /([^\t\n\f \/>"'=]+)/; // regexp costs ~ @@ -50,7 +49,7 @@ export default class DOMManager extends ListWalker { private nodeScrollManagers: Map> = new Map() private stylesManager: StylesManager private focusManager: FocusManager = new FocusManager(this.vElements) - + private selectionManager: SelectionManager = new SelectionManager(this.vElements) constructor( private readonly screen: Screen, @@ -76,6 +75,10 @@ export default class DOMManager extends ListWalker { this.focusManager.append(m) return } + if (m.tp === MType.SelectionChange) { + this.selectionManager.append(m) + return + } if (m.tp === MType.CreateElementNode) { if(m.tag === "BODY" && this.upperBodyId === -1) { this.upperBodyId = m.id @@ -432,6 +435,7 @@ export default class DOMManager extends ListWalker { return this.stylesManager.moveReady(t).then(() => { // Apply focus this.focusManager.move(t) + this.selectionManager.move(t) // Apply all scrolls after the styles got applied this.nodeScrollManagers.forEach(manager => { const msg = manager.moveGetLast(t) diff --git a/frontend/app/player/web/managers/DOM/SelectionManager.ts b/frontend/app/player/web/managers/DOM/SelectionManager.ts new file mode 100644 index 000000000..f073b7567 --- /dev/null +++ b/frontend/app/player/web/managers/DOM/SelectionManager.ts @@ -0,0 +1,39 @@ +import type { SelectionChange } from '../../messages' +import type { VElement} from "./VirtualDOM"; +import ListWalker from '../../../common/ListWalker'; + +const SELECTION_CLASS = { start: "-openreplay-selection-start", end: "-openreplay-selection-end" } + + +export default class SelectionManager extends ListWalker { + constructor(private readonly vElements: Map) { + super(); + } + + private selected: [Element | null, Element | null] = [null, null] + + move(t: number) { + const msg = this.moveGetLast(t) + if (!msg) { return } + console.log(msg) + if (msg.selectionStart <= 0) { + this.selected[0]?.classList.remove(SELECTION_CLASS.start) + this.selected[0].style.border = 'unset' + this.selected[1]?.classList.remove(SELECTION_CLASS.end) + this.selected = [null, null] + return; + } + const startVNode = this.vElements.get(msg.selectionStart -1) + const endVNode = this.vElements.get(msg.selectionEnd -1) + + console.log(startVNode, endVNode, this.vElements) + + if (startVNode && endVNode) { + this.selected = [startVNode.node, endVNode.node] + + this.selected[0]?.classList.add(SELECTION_CLASS.start) + this.selected[0].style.border = '5px solid red' + this.selected[1]?.classList.add(SELECTION_CLASS.end) + } + } +} \ No newline at end of file diff --git a/tracker/tracker/src/main/modules/selection.ts b/tracker/tracker/src/main/modules/selection.ts index cd7004c44..8a422fedc 100644 --- a/tracker/tracker/src/main/modules/selection.ts +++ b/tracker/tracker/src/main/modules/selection.ts @@ -12,7 +12,7 @@ function selection(app: App) { app.send(SelectionChange(selectionStart, selectionEnd, selectedText)) } } else { - app.send(SelectionChange(0, 0, '')) + app.send(SelectionChange(-1, -1, '')) } }) } From 2cd74dcad100633c859125dc8ea6f0bd1ece254f Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Feb 2023 16:36:29 +0100 Subject: [PATCH 010/253] change(player): add visual styles for selection event --- .../app/player/web/managers/DOM/DOMManager.ts | 3 +- .../player/web/managers/DOM/FocusManager.ts | 1 - .../web/managers/DOM/SelectionManager.ts | 75 +++++++++++++------ .../web/managers/DOM/selection.module.css | 7 ++ 4 files changed, 61 insertions(+), 25 deletions(-) create mode 100644 frontend/app/player/web/managers/DOM/selection.module.css diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index efdc29a8e..7b773860c 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -49,7 +49,7 @@ export default class DOMManager extends ListWalker { private nodeScrollManagers: Map> = new Map() private stylesManager: StylesManager private focusManager: FocusManager = new FocusManager(this.vElements) - private selectionManager: SelectionManager = new SelectionManager(this.vElements) + private selectionManager: SelectionManager constructor( private readonly screen: Screen, @@ -58,6 +58,7 @@ export default class DOMManager extends ListWalker { setCssLoading: ConstructorParameters[1], ) { super() + this.selectionManager = new SelectionManager(this.vElements, screen) this.stylesManager = new StylesManager(screen, setCssLoading) } diff --git a/frontend/app/player/web/managers/DOM/FocusManager.ts b/frontend/app/player/web/managers/DOM/FocusManager.ts index 174335473..c75f3ddc3 100644 --- a/frontend/app/player/web/managers/DOM/FocusManager.ts +++ b/frontend/app/player/web/managers/DOM/FocusManager.ts @@ -3,7 +3,6 @@ import type { SetNodeFocus } from '../../messages'; import type { VElement } from './VirtualDOM'; import ListWalker from '../../../common/ListWalker'; - const FOCUS_CLASS = "-openreplay-focus" export default class FocusManager extends ListWalker { diff --git a/frontend/app/player/web/managers/DOM/SelectionManager.ts b/frontend/app/player/web/managers/DOM/SelectionManager.ts index f073b7567..986fab98b 100644 --- a/frontend/app/player/web/managers/DOM/SelectionManager.ts +++ b/frontend/app/player/web/managers/DOM/SelectionManager.ts @@ -1,39 +1,68 @@ -import type { SelectionChange } from '../../messages' -import type { VElement} from "./VirtualDOM"; +import type { SelectionChange } from '../../messages'; +import type { VElement } from './VirtualDOM'; import ListWalker from '../../../common/ListWalker'; - -const SELECTION_CLASS = { start: "-openreplay-selection-start", end: "-openreplay-selection-end" } - +import Screen from 'Player/web/Screen/Screen'; export default class SelectionManager extends ListWalker { - constructor(private readonly vElements: Map) { + constructor(private readonly vElements: Map, private readonly screen: Screen) { super(); } - private selected: [Element | null, Element | null] = [null, null] + private selected: [{ id: number, node: Element } | null, { id: number, node: Element } | null] = [null, null]; + + clearSelection() { + this.selected[0] && this.screen.overlay.removeChild(this.selected[0].node) && this.selected[0].node.remove(); + this.selected[1] && this.screen.overlay.removeChild(this.selected[1].node) && this.selected[1].node.remove(); + this.selected = [null, null]; + } move(t: number) { - const msg = this.moveGetLast(t) - if (!msg) { return } - console.log(msg) - if (msg.selectionStart <= 0) { - this.selected[0]?.classList.remove(SELECTION_CLASS.start) - this.selected[0].style.border = 'unset' - this.selected[1]?.classList.remove(SELECTION_CLASS.end) - this.selected = [null, null] + const msg = this.moveGetLast(t); + if (!msg) { return; } - const startVNode = this.vElements.get(msg.selectionStart -1) - const endVNode = this.vElements.get(msg.selectionEnd -1) + // in theory: empty selection or selection removed + if (msg.selectionStart <= 0) { + this.clearSelection() + return; + } + // preventing clones + if (this.selected[0] && this.selected[0].id === msg.selectionStart) return; - console.log(startVNode, endVNode, this.vElements) + const startVNode = this.vElements.get(msg.selectionStart - 1); + const endVNode = this.vElements.get(msg.selectionEnd - 1); + + // only one selection present on page at the same time + if (this.selected[0] && this.selected[0]?.id !== msg.selectionStart) this.clearSelection() if (startVNode && endVNode) { - this.selected = [startVNode.node, endVNode.node] + const startCoords = startVNode.node.getBoundingClientRect(); + const endCoords = endVNode.node.getBoundingClientRect(); - this.selected[0]?.classList.add(SELECTION_CLASS.start) - this.selected[0].style.border = '5px solid red' - this.selected[1]?.classList.add(SELECTION_CLASS.end) + const startPointer = document.createElement('div'); + const endPointer = document.createElement('div'); + + Object.assign(endPointer.style, { + top: endCoords.top + 'px', + left: (endCoords.left + endCoords.width + 3) + 'px', + width: '3px', + height: endCoords.height + 'px', + border: '3px solid red', + position: 'absolute', + }); + Object.assign(startPointer.style, { + top: startCoords.top + 'px', + left: (startCoords.left - 3) + 'px', + width: '3px', + height: startCoords.height + 'px', + border: '3px solid red', + position: 'absolute', + }); + + this.screen.overlay.appendChild(startPointer); + this.screen.overlay.appendChild(endPointer); + + this.selected = [{ id: msg.selectionStart, node: startPointer }, { id: msg.selectionEnd, node: endPointer }]; } } -} \ No newline at end of file +} diff --git a/frontend/app/player/web/managers/DOM/selection.module.css b/frontend/app/player/web/managers/DOM/selection.module.css new file mode 100644 index 000000000..c677827bf --- /dev/null +++ b/frontend/app/player/web/managers/DOM/selection.module.css @@ -0,0 +1,7 @@ +.openreplay-selection-start { + border: 2px solid red; +} + +.openreplay-selection-end { + border: 2px solid red; +} \ No newline at end of file From 72785e7ede1716b97bbec01dd1b07d0148fd11c6 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Sun, 12 Feb 2023 18:34:22 +0100 Subject: [PATCH 011/253] fix(tracker): simplify input module logic --- tracker/tracker/src/main/modules/input.ts | 121 +++++++++------------- tracker/tracker/src/main/utils.ts | 11 -- 2 files changed, 50 insertions(+), 82 deletions(-) diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 9e929e782..2b92620e8 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -1,14 +1,14 @@ import type App from '../app/index.js' -import { normSpaces, IN_BROWSER, getLabelAttribute, debounce } from '../utils.js' +import { normSpaces, IN_BROWSER, getLabelAttribute, now } from '../utils.js' import { hasTag } from '../app/guards.js' import { InputChange, SetInputValue, SetInputChecked } from '../app/messages.gen.js' const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', 'date', 'tel'] // TODO: take into consideration "contenteditable" attribute -type TextEditableElement = HTMLInputElement | HTMLTextAreaElement +type TextFeildElement = HTMLInputElement | HTMLTextAreaElement -function isTextEditable(node: any): node is TextEditableElement { +function isTextFeildElement(node: Node): node is TextFeildElement { if (hasTag(node, 'textarea')) { return true } @@ -19,7 +19,7 @@ function isTextEditable(node: any): node is TextEditableElement { return INPUT_TYPES.includes(node.type) } -function isCheckbox(node: any): node is HTMLInputElement { +function isCheckbox(node: Node): node is HTMLInputElement & { type: 'checkbox' | 'radio' } { if (!hasTag(node, 'input')) { return false } @@ -27,7 +27,7 @@ function isCheckbox(node: any): node is HTMLInputElement { return type === 'checkbox' || type === 'radio' } -const labelElementFor: (element: TextEditableElement) => HTMLLabelElement | undefined = +const labelElementFor: (element: TextFeildElement) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype ? (node) => { let p: Node | null = node @@ -57,7 +57,7 @@ const labelElementFor: (element: TextEditableElement) => HTMLLabelElement | unde } } -export function getInputLabel(node: TextEditableElement): string { +export function getInputLabel(node: TextFeildElement): string { let label = getLabelAttribute(node) if (label === null) { const labelElement = labelElementFor(node) @@ -86,7 +86,6 @@ export interface Options { } export default function (app: App, opts: Partial): void { - const inputHesitationMap: Map = new Map() const options: Options = Object.assign( { obscureInputNumbers: true, @@ -97,16 +96,7 @@ export default function (app: App, opts: Partial): void { opts, ) - function sendInputChange(id: number, node: TextEditableElement): void { - const label = getInputLabel(node) - // @ts-ignore maybe if hesitationTime > 150 ? - const { hesitation } = inputHesitationMap.get(id) - if (label !== '') { - app.send(InputChange(id, label, hesitation)) - } - } - - function sendInputValue(id: number, node: TextEditableElement | HTMLSelectElement): void { + function sendInputValue(id: number, node: TextFeildElement | HTMLSelectElement): void { let value = node.value let inputMode: InputMode = options.defaultInputMode @@ -144,92 +134,81 @@ export default function (app: App, opts: Partial): void { checkboxValues.clear() }) - const debouncedUpdate = debounce((id: number, node: TextEditableElement) => { - sendInputChange(id, node) + function trackInputValue(id: number, node: TextFeildElement) { + if (inputValues.get(id) === node.value) { + return + } + inputValues.set(id, node.value) sendInputValue(id, node) - }, 125) + } - const debouncedTyping = debounce((id: number, node: TextEditableElement) => { - sendInputValue(id, node) - }, 60) + function trackCheckboxValue(id: number, value: boolean) { + if (checkboxValues.get(id) === value) { + return + } + checkboxValues.set(id, value) + app.send(SetInputChecked(id, value)) + } + // The only way (to our knowladge) to track all kinds of input changes, including those made by JS app.ticker.attach(() => { inputValues.forEach((value, id) => { const node = app.nodes.getNode(id) as HTMLInputElement if (!node) return inputValues.delete(id) - if (value !== node.value) { - inputValues.set(id, node.value) - sendInputValue(id, node) - } + trackInputValue(id, node) }) checkboxValues.forEach((checked, id) => { const node = app.nodes.getNode(id) as HTMLInputElement if (!node) return checkboxValues.delete(id) - if (checked !== node.checked) { - checkboxValues.set(id, node.checked) - app.send(SetInputChecked(id, node.checked)) - } + trackCheckboxValue(id, node.checked) }) }, 5) + function sendInputChange(id: number, node: TextFeildElement, hesitationTime: number) { + trackInputValue(id, node) + const label = getInputLabel(node) + app.send(InputChange(id, label, hesitationTime)) + } + app.nodes.attachNodeCallback( app.safe((node: Node): void => { const id = app.nodes.getID(node) if (id === undefined) { return } - // TODO: support multiple select (?): use selectedOptions; Need send target? + + // TODO: support multiple select (?): use selectedOptions; if (hasTag(node, 'select')) { sendInputValue(id, node) - const handler = () => { - sendInputValue(id, node) - } - app.nodes.attachNodeListener(node, 'change', handler) + app.nodes.attachNodeListener(node, 'change', () => sendInputValue(id, node)) } - if (isTextEditable(node)) { - inputValues.set(id, node.value) - inputHesitationMap.set(id, { hesitation: 0, focusEv: 0 }) - sendInputValue(id, node) - const setFocus = () => { - inputHesitationMap.set(id, { hesitation: 0, focusEv: +new Date() }) + if (isTextFeildElement(node)) { + trackInputValue(id, node) + let nodeFocusTime = 0 + let nodeHesitationTime = 0 + const onFocus = () => { + nodeFocusTime = now() } - const inputEvent = (e: InputEvent) => { - const value = (e.target as HTMLInputElement).value - if (inputValues.get(id) === '' && value !== '') { - const inputTime = +new Date() - const { focusEv } = inputHesitationMap.get(id)! - // @ts-ignore - const hesitationTime = inputTime - focusEv - inputHesitationMap.set(id, { hesitation: hesitationTime, focusEv }) + const onInput = () => { + const value = node.value + if (nodeHesitationTime === 0) { + nodeHesitationTime = nodeFocusTime - now() } - inputValues.set(id, value) - debouncedTyping(id, node) } - const changeEvent = (e: InputEvent) => { - const value = (e.target as HTMLInputElement).value - if (inputValues.get(id) !== value) { - inputValues.set(id, value) - debouncedUpdate(id, node) - } - inputHesitationMap.set(id, { hesitation: 0, focusEv: 0 }) + const onChange = () => { + sendInputChange(id, node, nodeHesitationTime) + nodeHesitationTime = 0 } - app.nodes.attachNodeListener(node, 'focus', setFocus) - app.nodes.attachNodeListener(node, 'input', inputEvent) - app.nodes.attachNodeListener(node, 'change', changeEvent) + app.nodes.attachNodeListener(node, 'focus', onFocus) + app.nodes.attachNodeListener(node, 'input', onInput) + app.nodes.attachNodeListener(node, 'change', onChange) return } if (isCheckbox(node)) { - checkboxValues.set(id, node.checked) - app.send(SetInputChecked(id, node.checked)) - const checkboxChange = (e: InputEvent) => { - const value = (e.target as HTMLInputElement).checked - checkboxValues.set(id, value) - app.send(SetInputChecked(id, value)) - } - app.nodes.attachNodeListener(node, 'change', checkboxChange) - + trackCheckboxValue(id, node.checked) + app.nodes.attachNodeListener(node, 'change', (e) => trackCheckboxValue(id, node.checked)) return } }), diff --git a/tracker/tracker/src/main/utils.ts b/tracker/tracker/src/main/utils.ts index 8f99a4c74..739821ea9 100644 --- a/tracker/tracker/src/main/utils.ts +++ b/tracker/tracker/src/main/utils.ts @@ -81,14 +81,3 @@ export function hasOpenreplayAttribute(e: Element, attr: string): boolean { return false } - -export function debounce(func: (...args: any[]) => void, timeout = 125) { - let timer: NodeJS.Timeout - return (...args: any[]) => { - clearTimeout(timer) - timer = setTimeout(() => { - // @ts-ignore - func.apply(this, args) - }, timeout) - } -} From e9848bf3351bfc7774cc24aeb699ea0f18c874c8 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 09:57:43 +0100 Subject: [PATCH 012/253] change(player): fix selection borders --- frontend/app/player/web/managers/DOM/SelectionManager.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/app/player/web/managers/DOM/SelectionManager.ts b/frontend/app/player/web/managers/DOM/SelectionManager.ts index 986fab98b..9cf1fd21f 100644 --- a/frontend/app/player/web/managers/DOM/SelectionManager.ts +++ b/frontend/app/player/web/managers/DOM/SelectionManager.ts @@ -9,11 +9,15 @@ export default class SelectionManager extends ListWalker { } private selected: [{ id: number, node: Element } | null, { id: number, node: Element } | null] = [null, null]; + private markers: Element[] = [] clearSelection() { this.selected[0] && this.screen.overlay.removeChild(this.selected[0].node) && this.selected[0].node.remove(); this.selected[1] && this.screen.overlay.removeChild(this.selected[1].node) && this.selected[1].node.remove(); + this.markers.forEach(marker => marker.remove()) + this.selected = [null, null]; + this.markers = []; } move(t: number) { @@ -27,7 +31,7 @@ export default class SelectionManager extends ListWalker { return; } // preventing clones - if (this.selected[0] && this.selected[0].id === msg.selectionStart) return; + if ((this.selected[0] && this.selected[0].id === msg.selectionStart) && (this.selected[1] && this.selected[1].id === msg.selectionEnd)) return; const startVNode = this.vElements.get(msg.selectionStart - 1); const endVNode = this.vElements.get(msg.selectionEnd - 1); @@ -59,6 +63,7 @@ export default class SelectionManager extends ListWalker { position: 'absolute', }); + this.markers.push(startPointer, endPointer); this.screen.overlay.appendChild(startPointer); this.screen.overlay.appendChild(endPointer); From 9628bcdfadbd1751af51c08382fb2c7a7992e3a6 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 11:23:18 +0100 Subject: [PATCH 013/253] change(tracker): add more events to input, fix typo and timestamp calc --- backend/pkg/messages/filters.go | 4 +- backend/pkg/messages/messages.go | 110 +++++++++--------- backend/pkg/messages/read-message.go | 77 ++++++------ ee/connectors/msgcodec/messages.py | 39 ++++--- ee/connectors/msgcodec/msgcodec.py | 31 ++--- .../web/messages/RawMessageReader.gen.ts | 2 +- .../app/player/web/messages/filters.gen.ts | 2 +- frontend/app/player/web/messages/raw.gen.ts | 2 +- .../player/web/messages/tracker-legacy.gen.ts | 2 +- .../app/player/web/messages/tracker.gen.ts | 9 +- mobs/messages.rb | 29 +++-- tracker/tracker/src/common/messages.gen.ts | 7 +- tracker/tracker/src/main/app/messages.gen.ts | 6 + tracker/tracker/src/main/modules/input.ts | 48 +++++--- .../src/webworker/MessageEncoder.gen.ts | 2 +- 15 files changed, 213 insertions(+), 157 deletions(-) diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index f586474d0..a67e8c43f 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,7 +2,7 @@ package messages func IsReplayerType(id int) bool { - return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 83 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id + return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 112 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id } func IsIOSType(id int) bool { @@ -10,5 +10,5 @@ func IsIOSType(id int) bool { } func IsDOMType(id int) bool { - return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 84 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id + return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 2e83ba0cf..6e5bbcfeb 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -79,11 +79,11 @@ const ( MsgBatchMeta = 80 MsgBatchMetadata = 81 MsgPartitionedMessage = 82 - MsgInputChange = 83 - MsgSelectionChange = 84 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 + MsgInputChange = 112 + MsgSelectionChange = 113 MsgIOSBatchMeta = 107 MsgIOSSessionStart = 90 MsgIOSSessionEnd = 91 @@ -2119,56 +2119,6 @@ func (msg *PartitionedMessage) TypeID() int { return 82 } -type InputChange struct { - message - ID uint64 - Label string - HesitationTime int64 -} - -func (msg *InputChange) Encode() []byte { - buf := make([]byte, 31+len(msg.Label)) - buf[0] = 83 - p := 1 - p = WriteUint(msg.ID, buf, p) - p = WriteString(msg.Label, buf, p) - p = WriteInt(msg.HesitationTime, buf, p) - return buf[:p] -} - -func (msg *InputChange) Decode() Message { - return msg -} - -func (msg *InputChange) TypeID() int { - return 83 -} - -type SelectionChange struct { - message - SelectionStart uint64 - SelectionEnd uint64 - Selection string -} - -func (msg *SelectionChange) Encode() []byte { - buf := make([]byte, 31+len(msg.Selection)) - buf[0] = 84 - p := 1 - p = WriteUint(msg.SelectionStart, buf, p) - p = WriteUint(msg.SelectionEnd, buf, p) - p = WriteString(msg.Selection, buf, p) - return buf[:p] -} - -func (msg *SelectionChange) Decode() Message { - return msg -} - -func (msg *SelectionChange) TypeID() int { - return 84 -} - type IssueEvent struct { message MessageID uint64 @@ -2248,6 +2198,62 @@ func (msg *SessionSearch) TypeID() int { return 127 } +type InputChange struct { + message + ID uint64 + Value string + ValueMasked bool + Label string + HesitationTime int64 + InputDuration int64 +} + +func (msg *InputChange) Encode() []byte { + buf := make([]byte, 61+len(msg.Value)+len(msg.Label)) + buf[0] = 112 + p := 1 + p = WriteUint(msg.ID, buf, p) + p = WriteString(msg.Value, buf, p) + p = WriteBoolean(msg.ValueMasked, buf, p) + p = WriteString(msg.Label, buf, p) + p = WriteInt(msg.HesitationTime, buf, p) + p = WriteInt(msg.InputDuration, buf, p) + return buf[:p] +} + +func (msg *InputChange) Decode() Message { + return msg +} + +func (msg *InputChange) TypeID() int { + return 112 +} + +type SelectionChange struct { + message + SelectionStart uint64 + SelectionEnd uint64 + Selection string +} + +func (msg *SelectionChange) Encode() []byte { + buf := make([]byte, 31+len(msg.Selection)) + buf[0] = 113 + p := 1 + p = WriteUint(msg.SelectionStart, buf, p) + p = WriteUint(msg.SelectionEnd, buf, p) + p = WriteString(msg.Selection, buf, p) + return buf[:p] +} + +func (msg *SelectionChange) Decode() Message { + return msg +} + +func (msg *SelectionChange) TypeID() int { + return 113 +} + type IOSBatchMeta struct { message Timestamp uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 4fe1e98f7..80a365168 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1293,36 +1293,6 @@ func DecodePartitionedMessage(reader BytesReader) (Message, error) { return msg, err } -func DecodeInputChange(reader BytesReader) (Message, error) { - var err error = nil - msg := &InputChange{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.HesitationTime, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err -} - -func DecodeSelectionChange(reader BytesReader) (Message, error) { - var err error = nil - msg := &SelectionChange{} - if msg.SelectionStart, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.SelectionEnd, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Selection, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err -} - func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1374,6 +1344,45 @@ func DecodeSessionSearch(reader BytesReader) (Message, error) { return msg, err } +func DecodeInputChange(reader BytesReader) (Message, error) { + var err error = nil + msg := &InputChange{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Value, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.ValueMasked, err = reader.ReadBoolean(); err != nil { + return nil, err + } + if msg.Label, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.HesitationTime, err = reader.ReadInt(); err != nil { + return nil, err + } + if msg.InputDuration, err = reader.ReadInt(); err != nil { + return nil, err + } + return msg, err +} + +func DecodeSelectionChange(reader BytesReader) (Message, error) { + var err error = nil + msg := &SelectionChange{} + if msg.SelectionStart, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.SelectionEnd, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Selection, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIOSBatchMeta(reader BytesReader) (Message, error) { var err error = nil msg := &IOSBatchMeta{} @@ -1932,16 +1941,16 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeBatchMetadata(reader) case 82: return DecodePartitionedMessage(reader) - case 83: - return DecodeInputChange(reader) - case 84: - return DecodeSelectionChange(reader) case 125: return DecodeIssueEvent(reader) case 126: return DecodeSessionEnd(reader) case 127: return DecodeSessionSearch(reader) + case 112: + return DecodeInputChange(reader) + case 113: + return DecodeSelectionChange(reader) case 107: return DecodeIOSBatchMeta(reader) case 90: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 76aee98a2..2ed4cf49d 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -744,24 +744,6 @@ class PartitionedMessage(Message): self.part_total = part_total -class InputChange(Message): - __id__ = 83 - - def __init__(self, id, label, hesitation_time): - self.id = id - self.label = label - self.hesitation_time = hesitation_time - - -class SelectionChange(Message): - __id__ = 84 - - def __init__(self, selection_start, selection_end, selection): - self.selection_start = selection_start - self.selection_end = selection_end - self.selection = selection - - class IssueEvent(Message): __id__ = 125 @@ -791,6 +773,27 @@ class SessionSearch(Message): self.partition = partition +class InputChange(Message): + __id__ = 112 + + def __init__(self, id, value, value_masked, label, hesitation_time, input_duration): + self.id = id + self.value = value + self.value_masked = value_masked + self.label = label + self.hesitation_time = hesitation_time + self.input_duration = input_duration + + +class SelectionChange(Message): + __id__ = 113 + + def __init__(self, selection_start, selection_end, selection): + self.selection_start = selection_start + self.selection_end = selection_end + self.selection = selection + + class IOSBatchMeta(Message): __id__ = 107 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index f5986ceb4..272f2efe2 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -660,20 +660,6 @@ class MessageCodec(Codec): part_total=self.read_uint(reader) ) - if message_id == 83: - return InputChange( - id=self.read_uint(reader), - label=self.read_string(reader), - hesitation_time=self.read_int(reader) - ) - - if message_id == 84: - return SelectionChange( - selection_start=self.read_uint(reader), - selection_end=self.read_uint(reader), - selection=self.read_string(reader) - ) - if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), @@ -697,6 +683,23 @@ class MessageCodec(Codec): partition=self.read_uint(reader) ) + if message_id == 112: + return InputChange( + id=self.read_uint(reader), + value=self.read_string(reader), + value_masked=self.read_boolean(reader), + label=self.read_string(reader), + hesitation_time=self.read_int(reader), + input_duration=self.read_int(reader) + ) + + if message_id == 113: + return SelectionChange( + selection_start=self.read_uint(reader), + selection_end=self.read_uint(reader), + selection=self.read_string(reader) + ) + if message_id == 107: return IOSBatchMeta( timestamp=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 32444011a..aeec66f67 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -627,7 +627,7 @@ export default class RawMessageReader extends PrimitiveReader { }; } - case 84: { + case 113: { const selectionStart = this.readUint(); if (selectionStart === null) { return resetPointer() } const selectionEnd = this.readUint(); if (selectionEnd === null) { return resetPointer() } const selection = this.readString(); if (selection === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/filters.gen.ts b/frontend/app/player/web/messages/filters.gen.ts index c7a6f60cd..3e41afc0d 100644 --- a/frontend/app/player/web/messages/filters.gen.ts +++ b/frontend/app/player/web/messages/filters.gen.ts @@ -3,7 +3,7 @@ import { MType } from './raw.gen' -const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,84,90,93,96,100,102,103,105] +const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,90,93,96,100,102,103,105] export function isDOMType(t: MType) { return DOM_TYPES.includes(t) } \ No newline at end of file diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 165a6f9b2..b399dde79 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -53,7 +53,7 @@ export const enum MType { AdoptedSsAddOwner = 76, AdoptedSsRemoveOwner = 77, Zustand = 79, - SelectionChange = 84, + SelectionChange = 113, IosSessionStart = 90, IosCustomEvent = 93, IosScreenChanges = 96, diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index 88f54434a..8005890ab 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -54,7 +54,7 @@ export const TP_MAP = { 76: MType.AdoptedSsAddOwner, 77: MType.AdoptedSsRemoveOwner, 79: MType.Zustand, - 84: MType.SelectionChange, + 113: MType.SelectionChange, 90: MType.IosSessionStart, 93: MType.IosCustomEvent, 96: MType.IosScreenChanges, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 8d8a78d2e..ef11afff2 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -430,14 +430,17 @@ type TrPartitionedMessage = [ ] type TrInputChange = [ - type: 83, + type: 112, id: number, + value: string, + valueMasked: boolean, label: string, hesitationTime: number, + inputDuration: number, ] type TrSelectionChange = [ - type: 84, + type: 113, selectionStart: number, selectionEnd: number, selection: string, @@ -881,7 +884,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } - case 84: { + case 113: { return { tp: MType.SelectionChange, selectionStart: tMsg[1], diff --git a/mobs/messages.rb b/mobs/messages.rb index a6b59bd7d..cb27a1db2 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -475,17 +475,7 @@ message 82, 'PartitionedMessage', :replayer => false do uint 'PartTotal' end -message 83, 'InputChange', :replayer => false do - uint 'ID' - string 'Label' - int 'HesitationTime' -end - -message 84, 'SelectionChange' do - uint 'SelectionStart' - uint 'SelectionEnd' - string 'Selection' -end +# 90-111 reserved iOS ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do @@ -505,3 +495,20 @@ message 127, 'SessionSearch', :tracker => false, :replayer => false do uint 'Timestamp' uint 'Partition' end + +# since tracker 4.1.10 + +message 112, 'InputChange', :replayer => false do + uint 'ID' + string 'Value' + boolean 'ValueMasked' + string 'Label' + int 'HesitationTime' + int 'InputDuration' +end + +message 113, 'SelectionChange' do + uint 'SelectionStart' + uint 'SelectionEnd' + string 'Selection' +end \ No newline at end of file diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 1cc05b6fa..8dfb0b2de 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -63,8 +63,8 @@ export declare const enum Type { Zustand = 79, BatchMetadata = 81, PartitionedMessage = 82, - InputChange = 83, - SelectionChange = 84, + InputChange = 112, + SelectionChange = 113, } @@ -495,8 +495,11 @@ export type PartitionedMessage = [ export type InputChange = [ /*type:*/ Type.InputChange, /*id:*/ number, + /*value:*/ string, + /*valueMasked:*/ boolean, /*label:*/ string, /*hesitationTime:*/ number, + /*inputDuration:*/ number, ] export type SelectionChange = [ diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index af8dbe357..b6ac90706 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -794,14 +794,20 @@ export function PartitionedMessage( export function InputChange( id: number, + value: string, + valueMasked: boolean, label: string, hesitationTime: number, + inputDuration: number, ): Messages.InputChange { return [ Messages.Type.InputChange, id, + value, + valueMasked, label, hesitationTime, + inputDuration, ] } diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 2b92620e8..90463b21e 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -6,9 +6,9 @@ import { InputChange, SetInputValue, SetInputChecked } from '../app/messages.gen const INPUT_TYPES = ['text', 'password', 'email', 'search', 'number', 'range', 'date', 'tel'] // TODO: take into consideration "contenteditable" attribute -type TextFeildElement = HTMLInputElement | HTMLTextAreaElement +type TextFieldElement = HTMLInputElement | HTMLTextAreaElement -function isTextFeildElement(node: Node): node is TextFeildElement { +function isTextFieldElement(node: Node): node is TextFieldElement { if (hasTag(node, 'textarea')) { return true } @@ -27,7 +27,7 @@ function isCheckbox(node: Node): node is HTMLInputElement & { type: 'checkbox' | return type === 'checkbox' || type === 'radio' } -const labelElementFor: (element: TextFeildElement) => HTMLLabelElement | undefined = +const labelElementFor: (element: TextFieldElement) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype ? (node) => { let p: Node | null = node @@ -57,7 +57,7 @@ const labelElementFor: (element: TextFeildElement) => HTMLLabelElement | undefin } } -export function getInputLabel(node: TextFeildElement): string { +export function getInputLabel(node: TextFieldElement): string { let label = getLabelAttribute(node) if (label === null) { const labelElement = labelElementFor(node) @@ -96,7 +96,7 @@ export default function (app: App, opts: Partial): void { opts, ) - function sendInputValue(id: number, node: TextFeildElement | HTMLSelectElement): void { + function getInputValue(id: number, node: TextFieldElement | HTMLSelectElement) { let value = node.value let inputMode: InputMode = options.defaultInputMode @@ -123,6 +123,11 @@ export default function (app: App, opts: Partial): void { break } + return { value, mask } + } + function sendInputValue(id: number, node: TextFieldElement | HTMLSelectElement): void { + const { value, mask } = getInputValue(id, node) + app.send(SetInputValue(id, value, mask)) } @@ -134,7 +139,7 @@ export default function (app: App, opts: Partial): void { checkboxValues.clear() }) - function trackInputValue(id: number, node: TextFeildElement) { + function trackInputValue(id: number, node: TextFieldElement) { if (inputValues.get(id) === node.value) { return } @@ -150,7 +155,7 @@ export default function (app: App, opts: Partial): void { app.send(SetInputChecked(id, value)) } - // The only way (to our knowladge) to track all kinds of input changes, including those made by JS + // The only way (to our knowledge) to track all kinds of input changes, including those made by JS app.ticker.attach(() => { inputValues.forEach((value, id) => { const node = app.nodes.getNode(id) as HTMLInputElement @@ -162,12 +167,18 @@ export default function (app: App, opts: Partial): void { if (!node) return checkboxValues.delete(id) trackCheckboxValue(id, node.checked) }) - }, 5) + }, 3) - function sendInputChange(id: number, node: TextFeildElement, hesitationTime: number) { - trackInputValue(id, node) + function sendInputChange( + id: number, + node: TextFieldElement, + hesitationTime: number, + inputTime: number, + ) { + const { value, mask } = getInputValue(id, node) const label = getInputLabel(node) - app.send(InputChange(id, label, hesitationTime)) + + app.send(InputChange(id, value, mask !== 0, label, hesitationTime, inputTime)) } app.nodes.attachNodeCallback( @@ -183,22 +194,27 @@ export default function (app: App, opts: Partial): void { app.nodes.attachNodeListener(node, 'change', () => sendInputValue(id, node)) } - if (isTextFeildElement(node)) { + if (isTextFieldElement(node)) { trackInputValue(id, node) let nodeFocusTime = 0 let nodeHesitationTime = 0 + let inputTime = 0 + const onFocus = () => { nodeFocusTime = now() } + const onInput = () => { - const value = node.value if (nodeHesitationTime === 0) { - nodeHesitationTime = nodeFocusTime - now() + nodeHesitationTime = now() - nodeFocusTime } } + const onChange = () => { - sendInputChange(id, node, nodeHesitationTime) + inputTime = now() - nodeFocusTime + sendInputChange(id, node, nodeHesitationTime, inputTime) nodeHesitationTime = 0 + inputTime = 0 } app.nodes.attachNodeListener(node, 'focus', onFocus) app.nodes.attachNodeListener(node, 'input', onInput) @@ -208,7 +224,7 @@ export default function (app: App, opts: Partial): void { if (isCheckbox(node)) { trackCheckboxValue(id, node.checked) - app.nodes.attachNodeListener(node, 'change', (e) => trackCheckboxValue(id, node.checked)) + app.nodes.attachNodeListener(node, 'change', () => trackCheckboxValue(id, node.checked)) return } }), diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 10b3e205c..9e7b6f93f 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -255,7 +255,7 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.InputChange: - return this.uint(msg[1]) && this.string(msg[2]) && this.int(msg[3]) + return this.uint(msg[1]) && this.string(msg[2]) && this.boolean(msg[3]) && this.string(msg[4]) && this.int(msg[5]) && this.int(msg[6]) break case Messages.Type.SelectionChange: From 4a96d87cb700ac1706ed07bf5ffaedf3f88882c5 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 11:30:08 +0100 Subject: [PATCH 014/253] change(tracker): add mouse thrashing event message --- backend/pkg/messages/filters.go | 2 +- backend/pkg/messages/messages.go | 22 +++++++++++++++++++ backend/pkg/messages/read-message.go | 11 ++++++++++ ee/connectors/msgcodec/messages.py | 7 ++++++ ee/connectors/msgcodec/msgcodec.py | 5 +++++ .../web/messages/RawMessageReader.gen.ts | 8 +++++++ .../app/player/web/messages/filters.gen.ts | 2 +- .../app/player/web/messages/message.gen.ts | 3 +++ frontend/app/player/web/messages/raw.gen.ts | 8 ++++++- .../player/web/messages/tracker-legacy.gen.ts | 1 + .../app/player/web/messages/tracker.gen.ts | 14 +++++++++++- mobs/messages.rb | 4 ++++ tracker/tracker/src/common/messages.gen.ts | 8 ++++++- tracker/tracker/src/main/app/messages.gen.ts | 9 ++++++++ tracker/tracker/src/main/modules/mouse.ts | 7 +++--- .../src/webworker/MessageEncoder.gen.ts | 4 ++++ 16 files changed, 107 insertions(+), 8 deletions(-) diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index a67e8c43f..b6c688c13 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -10,5 +10,5 @@ func IsIOSType(id int) bool { } func IsDOMType(id int) bool { - return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id + return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 6e5bbcfeb..e0c78bf71 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -84,6 +84,7 @@ const ( MsgSessionSearch = 127 MsgInputChange = 112 MsgSelectionChange = 113 + MsgMouseThrashing = 114 MsgIOSBatchMeta = 107 MsgIOSSessionStart = 90 MsgIOSSessionEnd = 91 @@ -2254,6 +2255,27 @@ func (msg *SelectionChange) TypeID() int { return 113 } +type MouseThrashing struct { + message + Timestamp uint64 +} + +func (msg *MouseThrashing) Encode() []byte { + buf := make([]byte, 11) + buf[0] = 114 + p := 1 + p = WriteUint(msg.Timestamp, buf, p) + return buf[:p] +} + +func (msg *MouseThrashing) Decode() Message { + return msg +} + +func (msg *MouseThrashing) TypeID() int { + return 114 +} + type IOSBatchMeta struct { message Timestamp uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 80a365168..52da274bf 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1383,6 +1383,15 @@ func DecodeSelectionChange(reader BytesReader) (Message, error) { return msg, err } +func DecodeMouseThrashing(reader BytesReader) (Message, error) { + var err error = nil + msg := &MouseThrashing{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIOSBatchMeta(reader BytesReader) (Message, error) { var err error = nil msg := &IOSBatchMeta{} @@ -1951,6 +1960,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeInputChange(reader) case 113: return DecodeSelectionChange(reader) + case 114: + return DecodeMouseThrashing(reader) case 107: return DecodeIOSBatchMeta(reader) case 90: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 2ed4cf49d..102768e60 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -794,6 +794,13 @@ class SelectionChange(Message): self.selection = selection +class MouseThrashing(Message): + __id__ = 114 + + def __init__(self, timestamp): + self.timestamp = timestamp + + class IOSBatchMeta(Message): __id__ = 107 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 272f2efe2..6ebbfff54 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -700,6 +700,11 @@ class MessageCodec(Codec): selection=self.read_string(reader) ) + if message_id == 114: + return MouseThrashing( + timestamp=self.read_uint(reader) + ) + if message_id == 107: return IOSBatchMeta( timestamp=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index aeec66f67..1d0e0e7cd 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -639,6 +639,14 @@ export default class RawMessageReader extends PrimitiveReader { }; } + case 114: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + return { + tp: MType.MouseThrashing, + timestamp, + }; + } + case 90: { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const projectID = this.readUint(); if (projectID === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/filters.gen.ts b/frontend/app/player/web/messages/filters.gen.ts index 3e41afc0d..2cd1b6c25 100644 --- a/frontend/app/player/web/messages/filters.gen.ts +++ b/frontend/app/player/web/messages/filters.gen.ts @@ -3,7 +3,7 @@ import { MType } from './raw.gen' -const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,90,93,96,100,102,103,105] +const DOM_TYPES = [0,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,37,38,49,50,51,54,55,57,58,59,60,61,67,69,70,71,72,73,74,75,76,77,113,114,90,93,96,100,102,103,105] export function isDOMType(t: MType) { return DOM_TYPES.includes(t) } \ No newline at end of file diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index 5dd11b8c3..e39d02584 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -56,6 +56,7 @@ import type { RawAdoptedSsRemoveOwner, RawZustand, RawSelectionChange, + RawMouseThrashing, RawIosSessionStart, RawIosCustomEvent, RawIosScreenChanges, @@ -172,6 +173,8 @@ export type Zustand = RawZustand & Timed export type SelectionChange = RawSelectionChange & Timed +export type MouseThrashing = RawMouseThrashing & Timed + export type IosSessionStart = RawIosSessionStart & Timed export type IosCustomEvent = RawIosCustomEvent & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index b399dde79..67f65ab35 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -54,6 +54,7 @@ export const enum MType { AdoptedSsRemoveOwner = 77, Zustand = 79, SelectionChange = 113, + MouseThrashing = 114, IosSessionStart = 90, IosCustomEvent = 93, IosScreenChanges = 96, @@ -426,6 +427,11 @@ export interface RawSelectionChange { selection: string, } +export interface RawMouseThrashing { + tp: MType.MouseThrashing, + timestamp: number, +} + export interface RawIosSessionStart { tp: MType.IosSessionStart, timestamp: number, @@ -497,4 +503,4 @@ export interface RawIosNetworkCall { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawMouseThrashing | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index 8005890ab..ce69f34b3 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -55,6 +55,7 @@ export const TP_MAP = { 77: MType.AdoptedSsRemoveOwner, 79: MType.Zustand, 113: MType.SelectionChange, + 114: MType.MouseThrashing, 90: MType.IosSessionStart, 93: MType.IosCustomEvent, 96: MType.IosScreenChanges, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index ef11afff2..fdd9c6f42 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -446,8 +446,13 @@ type TrSelectionChange = [ selection: string, ] +type TrMouseThrashing = [ + type: 114, + timestamp: number, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -893,6 +898,13 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } + case 114: { + return { + tp: MType.MouseThrashing, + timestamp: tMsg[1], + } + } + default: return null } diff --git a/mobs/messages.rb b/mobs/messages.rb index cb27a1db2..b5f714985 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -511,4 +511,8 @@ message 113, 'SelectionChange' do uint 'SelectionStart' uint 'SelectionEnd' string 'Selection' +end + +message 114, 'MouseThrashing' do + uint 'Timestamp' end \ No newline at end of file diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 8dfb0b2de..ce48b201d 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -65,6 +65,7 @@ export declare const enum Type { PartitionedMessage = 82, InputChange = 112, SelectionChange = 113, + MouseThrashing = 114, } @@ -509,6 +510,11 @@ export type SelectionChange = [ /*selection:*/ string, ] +export type MouseThrashing = [ + /*type:*/ Type.MouseThrashing, + /*timestamp:*/ number, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index b6ac90706..dad81cb6a 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -824,3 +824,12 @@ export function SelectionChange( ] } +export function MouseThrashing( + timestamp: number, +): Messages.MouseThrashing { + return [ + Messages.Type.MouseThrashing, + timestamp, + ] +} + diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index c94f117fa..5e905f205 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -1,7 +1,7 @@ import type App from '../app/index.js' import { hasTag, isSVGElement, isDocument } from '../app/guards.js' -import { normSpaces, hasOpenreplayAttribute, getLabelAttribute } from '../utils.js' -import { MouseMove, MouseClick } from '../app/messages.gen.js' +import { normSpaces, hasOpenreplayAttribute, getLabelAttribute, now } from '../utils.js' +import { MouseMove, MouseClick, MouseThrashing } from '../app/messages.gen.js' import { getInputLabel } from './input.js' function _getSelector(target: Element, document: Document): string { @@ -37,7 +37,7 @@ function isClickable(element: Element): boolean { element.getAttribute('role') === 'button' ) //|| element.className.includes("btn") - // MBTODO: intersept addEventListener + // MBTODO: intercept addEventListener } //TODO: fix (typescript is not sure about target variable after assignation of svg) @@ -126,6 +126,7 @@ export default function (app: App): void { const acceleration = (nextVelocity - velocity) / shakeCheckInterval if (directionChangeCount > 3 && acceleration > shakeThreshold) { console.log('Mouse shake detected!') + app.send(MouseThrashing(now())) } distance = 0 diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 9e7b6f93f..06bd776f0 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -262,6 +262,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) && this.string(msg[3]) break + case Messages.Type.MouseThrashing: + return this.uint(msg[1]) + break + } } From 42847b7c4b2a6ed86e6dbb10002f91171fe21ee9 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 14:13:21 +0100 Subject: [PATCH 015/253] change(player): add mouse thrashing ui event --- frontend/app/player/web/MessageManager.ts | 12 +++++++-- frontend/app/player/web/Screen/Cursor.ts | 25 ++++++++++++++++++- .../app/player/web/Screen/cursor.module.css | 6 ++--- .../player/web/managers/MouseMoveManager.ts | 2 +- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index d0ae18020..8fb928909 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -19,7 +19,7 @@ import WindowNodeCounter from './managers/WindowNodeCounter'; import ActivityManager from './managers/ActivityManager'; import MFileReader from './messages/MFileReader'; -import { MType } from './messages'; +import { MouseThrashing, MType } from "./messages"; import { isDOMType } from './messages/filters.gen'; import type { Message, @@ -100,6 +100,7 @@ export default class MessageManager { private performanceTrackManager: PerformanceTrackManager = new PerformanceTrackManager(); private windowNodeCounter: WindowNodeCounter = new WindowNodeCounter(); private clickManager: ListWalker = new ListWalker(); + private mouseThrashingManager: ListWalker = new ListWalker(); private resizeManager: ListWalker = new ListWalker([]); private pagesManager: PagesManager; @@ -322,9 +323,13 @@ export default class MessageManager { // Moving mouse and setting :hover classes on ready view this.mouseMoveManager.move(t); const lastClick = this.clickManager.moveGetLast(t); - if (!!lastClick && t - lastClick.time < 600) { // happend during last 600ms + if (!!lastClick && t - lastClick.time < 600) { // happened during last 600ms this.screen.cursor.click(); } + const lastThrashing = this.mouseThrashingManager.moveGetLast(t) + if (!!lastThrashing && t - lastThrashing.time < 300) { + this.screen.cursor.shake(); + } }) if (this.waitingForFiles && this.lastMessageTime <= t && t !== this.session.duration.milliseconds) { @@ -365,6 +370,9 @@ export default class MessageManager { case MType.SetViewportSize: this.resizeManager.append(msg); break; + case MType.MouseThrashing: + this.mouseThrashingManager.append(msg); + break; case MType.MouseMove: this.mouseMoveManager.append(msg); break; diff --git a/frontend/app/player/web/Screen/Cursor.ts b/frontend/app/player/web/Screen/Cursor.ts index f2d371062..799075b18 100644 --- a/frontend/app/player/web/Screen/Cursor.ts +++ b/frontend/app/player/web/Screen/Cursor.ts @@ -3,9 +3,11 @@ import styles from './cursor.module.css'; export default class Cursor { + private readonly isMobile: boolean; private readonly cursor: HTMLDivElement; private tagElement: HTMLDivElement; - private isMobile: boolean; + private coords = { x: 0, y: 0 }; + private isMoving = false; constructor(overlay: HTMLDivElement, isMobile: boolean) { this.cursor = document.createElement('div'); @@ -13,6 +15,8 @@ export default class Cursor { if (isMobile) this.cursor.style.backgroundImage = 'unset' overlay.appendChild(this.cursor); this.isMobile = isMobile; + + window.shakeTest = this.shake.bind(this); } toggle(flag: boolean) { @@ -50,8 +54,27 @@ export default class Cursor { } move({ x, y }: Point) { + this.isMoving = true; this.cursor.style.left = x + 'px'; this.cursor.style.top = y + 'px'; + this.coords = { x, y }; + setTimeout(() => this.isMoving = false, 60) + } + + shake(iteration = 1, upwards = true, original: { x: number, y: number } = this.coords) { + if (this.isMoving) return; + if (iteration < 10) { + this.cursor.style.width = 45 + 'px' + this.cursor.style.height = 75 + 'px' + const shift = upwards ? 60 : -60 + this.move({ x: this.coords.x + shift, y: this.coords.y - shift }) + setTimeout(() => this.shake(iteration + 1, !upwards, original), 60) + } else { + this.cursor.style.width = 18 + 'px' + this.cursor.style.height = 30 + 'px' + + this.move(original) + } } click() { diff --git a/frontend/app/player/web/Screen/cursor.module.css b/frontend/app/player/web/Screen/cursor.module.css index 93f3d05ff..67d452953 100644 --- a/frontend/app/player/web/Screen/cursor.module.css +++ b/frontend/app/player/web/Screen/cursor.module.css @@ -1,9 +1,9 @@ .cursor { display: block; position: absolute; - width: 13px; - height: 20px; - background-image: url('data:image/svg+xml;utf8,'); + width: 18px; + height: 30px; + background-image: url('data:image/svg+xml;utf8, '); background-repeat: no-repeat; transition: top .125s linear, left .125s linear; diff --git a/frontend/app/player/web/managers/MouseMoveManager.ts b/frontend/app/player/web/managers/MouseMoveManager.ts index 1b19d7e5b..fa320ab13 100644 --- a/frontend/app/player/web/managers/MouseMoveManager.ts +++ b/frontend/app/player/web/managers/MouseMoveManager.ts @@ -1,5 +1,5 @@ import type Screen from '../Screen/Screen' -import type { MouseMove } from '../messages' +import type { MouseMove } from "../messages"; import ListWalker from '../../common/ListWalker' From b84c1ccc8e8aea650ee5428b59a9cf188b308e51 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 14:14:12 +0100 Subject: [PATCH 016/253] change(player): remove synt event --- frontend/app/player/web/Screen/Cursor.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/app/player/web/Screen/Cursor.ts b/frontend/app/player/web/Screen/Cursor.ts index 799075b18..ee98ab83a 100644 --- a/frontend/app/player/web/Screen/Cursor.ts +++ b/frontend/app/player/web/Screen/Cursor.ts @@ -15,8 +15,6 @@ export default class Cursor { if (isMobile) this.cursor.style.backgroundImage = 'unset' overlay.appendChild(this.cursor); this.isMobile = isMobile; - - window.shakeTest = this.shake.bind(this); } toggle(flag: boolean) { @@ -85,7 +83,7 @@ export default class Cursor { }, 600) } - // TODO (to keep on a different playig speed): + // TODO (to keep on a different playing speed): // transition // setTransitionSpeed() From 5c87872c172ba780832bac6fe56dfa3371d538d7 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 14:39:19 +0100 Subject: [PATCH 017/253] change(player): tune shake animation --- frontend/app/player/web/Screen/Cursor.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/frontend/app/player/web/Screen/Cursor.ts b/frontend/app/player/web/Screen/Cursor.ts index ee98ab83a..b6a475331 100644 --- a/frontend/app/player/web/Screen/Cursor.ts +++ b/frontend/app/player/web/Screen/Cursor.ts @@ -59,19 +59,26 @@ export default class Cursor { setTimeout(() => this.isMoving = false, 60) } + setDefaultStyle() { + this.cursor.style.width = 18 + 'px' + this.cursor.style.height = 30 + 'px' + this.cursor.style.transition = 'top .125s linear, left .125s linear' + } + shake(iteration = 1, upwards = true, original: { x: number, y: number } = this.coords) { - if (this.isMoving) return; + if (this.isMoving) { + return this.setDefaultStyle() + } if (iteration < 10) { + this.cursor.style.transition = 'top .06s linear, left .06s linear' this.cursor.style.width = 45 + 'px' this.cursor.style.height = 75 + 'px' - const shift = upwards ? 60 : -60 + const shift = upwards ? 90 : -90 this.move({ x: this.coords.x + shift, y: this.coords.y - shift }) setTimeout(() => this.shake(iteration + 1, !upwards, original), 60) } else { - this.cursor.style.width = 18 + 'px' - this.cursor.style.height = 30 + 'px' - - this.move(original) + this.setDefaultStyle() + return this.move(original) } } From d5b0444388c4e197e7b4eee670d34732d88d371a Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 14:57:50 +0100 Subject: [PATCH 018/253] change(player): move animation to css --- frontend/app/player/web/Screen/Cursor.ts | 20 ++++---------- .../app/player/web/Screen/cursor.module.css | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/frontend/app/player/web/Screen/Cursor.ts b/frontend/app/player/web/Screen/Cursor.ts index b6a475331..dc3637c1b 100644 --- a/frontend/app/player/web/Screen/Cursor.ts +++ b/frontend/app/player/web/Screen/Cursor.ts @@ -65,21 +65,11 @@ export default class Cursor { this.cursor.style.transition = 'top .125s linear, left .125s linear' } - shake(iteration = 1, upwards = true, original: { x: number, y: number } = this.coords) { - if (this.isMoving) { - return this.setDefaultStyle() - } - if (iteration < 10) { - this.cursor.style.transition = 'top .06s linear, left .06s linear' - this.cursor.style.width = 45 + 'px' - this.cursor.style.height = 75 + 'px' - const shift = upwards ? 90 : -90 - this.move({ x: this.coords.x + shift, y: this.coords.y - shift }) - setTimeout(() => this.shake(iteration + 1, !upwards, original), 60) - } else { - this.setDefaultStyle() - return this.move(original) - } + shake() { + this.cursor.classList.add(styles.shaking) + setTimeout(() => { + this.cursor.classList.remove(styles.shaking) + }, 500) } click() { diff --git a/frontend/app/player/web/Screen/cursor.module.css b/frontend/app/player/web/Screen/cursor.module.css index 67d452953..238a53ca1 100644 --- a/frontend/app/player/web/Screen/cursor.module.css +++ b/frontend/app/player/web/Screen/cursor.module.css @@ -108,3 +108,29 @@ transform: scale3d(1, 1, 1); } } + +.cursor.shaking { + width: 45px; + height: 75px; + -webkit-animation: shaking 0.3s linear; + animation: shaking 0.3s linear; + animation-iteration-count: 2; +} + +@keyframes shaking { + 0% { + transform: translate(60px, -60px); + } + 25% { + transform: translate(-60px, 60px); + } + 50% { + transform: translate(60px, -60px); + } + 75% { + transform: translate(-60px, 60px); + } + 100% { + transform: translate(60px, -60px); + } +} \ No newline at end of file From d8212351552f3547bbd894feedd5050f163c582b Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 3 Mar 2023 17:03:06 +0100 Subject: [PATCH 019/253] fix(tracker-assist):5.0.1:check presense of window before using it at the top level --- tracker/tracker-assist/package.json | 2 +- tracker/tracker-assist/src/RemoteControl.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index aaa80429d..4277e2e36 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "5.0.0", + "version": "5.0.1", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index fb9850437..017918238 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -11,7 +11,7 @@ export enum RCStatus { let setInputValue = function(this: HTMLInputElement | HTMLTextAreaElement, value: string) { this.value = value } -const nativeInputValueDescriptor = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value') +const nativeInputValueDescriptor = typeof window !== 'undefined' && Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value') if (nativeInputValueDescriptor && nativeInputValueDescriptor.set) { setInputValue = nativeInputValueDescriptor.set } From 740fc4aa2e06972c6b2754811a9324e56e41f871 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Sun, 5 Mar 2023 15:39:00 +0100 Subject: [PATCH 020/253] fix(script): Race condition for migration version Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/files/dbops.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay/files/dbops.sh b/scripts/helmcharts/openreplay/files/dbops.sh index 6402a3c88..8ac336d21 100644 --- a/scripts/helmcharts/openreplay/files/dbops.sh +++ b/scripts/helmcharts/openreplay/files/dbops.sh @@ -39,10 +39,10 @@ function migration() { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }' } all_versions=(`ls -l db/init_dbs/$db | grep -E ^d | grep -v create | awk '{print $NF}'`) - migration_versions=(`for ver in ${all_versions[*]}; do if [[ $(normalise_version $ver) > $(normalise_version "${PREVIOUS_APP_VERSION}") ]]; then echo $ver; fi; done`) + migration_versions=(`for ver in ${all_versions[*]}; do if [[ $(normalise_version $ver) > $(normalise_version "${PREVIOUS_APP_VERSION}") ]]; then echo $ver; fi; done | sort -V`) echo "Migration version: ${migration_versions[*]}" # Can't pass the space seperated array to ansible for migration. So joining them with , - joined_migration_versions=$(IFS=, ; echo "${migration_versions[*]}") + joined_migration_versions=$(IFS=, ; printf '%s\n' "${migration_versions[@]}" | tac | tr '\n' ' '; echo) cd - From 0037b0376fbba20f3a3055704e5b4cc29e905851 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Sun, 5 Mar 2023 15:47:29 +0100 Subject: [PATCH 021/253] fix(migration): regression fix Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/files/dbops.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/files/dbops.sh b/scripts/helmcharts/openreplay/files/dbops.sh index 8ac336d21..0df923178 100644 --- a/scripts/helmcharts/openreplay/files/dbops.sh +++ b/scripts/helmcharts/openreplay/files/dbops.sh @@ -42,7 +42,7 @@ function migration() { migration_versions=(`for ver in ${all_versions[*]}; do if [[ $(normalise_version $ver) > $(normalise_version "${PREVIOUS_APP_VERSION}") ]]; then echo $ver; fi; done | sort -V`) echo "Migration version: ${migration_versions[*]}" # Can't pass the space seperated array to ansible for migration. So joining them with , - joined_migration_versions=$(IFS=, ; printf '%s\n' "${migration_versions[@]}" | tac | tr '\n' ' '; echo) + joined_migration_versions=$(IFS=, ; echo "${migration_versions[*]}") cd - From eb21604af428304eeef35644ef2032572db7867b Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 6 Mar 2023 09:45:39 +0100 Subject: [PATCH 022/253] chore(docker): Adding env variable for max file size Signed-off-by: rjshrjndrn --- backend/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index c7606559e..749900ba5 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -82,7 +82,9 @@ ENV TZ=UTC \ COMPRESSION_TYPE=zstd \ CH_USERNAME="default" \ CH_PASSWORD="" \ - CH_DATABASE="default" + CH_DATABASE="default" \ + # Max file size to process, default to 100MB + MAX_FILE_SIZE=100000000 RUN if [ "$SERVICE_NAME" = "http" ]; then \ From 39a0925abf979d323770e5999d6b188e4473241d Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 6 Mar 2023 10:59:08 +0100 Subject: [PATCH 023/253] fix(ui) - jwt set --- frontend/app/components/Login/Login.js | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/app/components/Login/Login.js b/frontend/app/components/Login/Login.js index da2206bdf..b51970036 100644 --- a/frontend/app/components/Login/Login.js +++ b/frontend/app/components/Login/Login.js @@ -40,7 +40,6 @@ class Login extends React.Component { const jwt = params.get('jwt'); if (jwt) { this.props.setJwt(jwt); - window.location.href = '/'; } } From a74bfa1b61d116c497f17cd32a12a13eb0692e7a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 6 Mar 2023 14:32:34 +0100 Subject: [PATCH 024/253] chore(actions): change actions --- .github/workflows/alerts-ee.yaml | 3 ++- .github/workflows/alerts.yaml | 3 ++- .github/workflows/api-ee.yaml | 3 ++- .github/workflows/api.yaml | 3 ++- .github/workflows/assist-ee.yaml | 1 + .github/workflows/assist.yaml | 1 + .github/workflows/crons-ee.yaml | 3 ++- .github/workflows/peers-ee.yaml | 1 + .github/workflows/peers.yaml | 1 + .github/workflows/sourcemaps-reader.yaml | 1 + 10 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/alerts-ee.yaml b/.github/workflows/alerts-ee.yaml index 10482a7cb..4c1d3b6c4 100644 --- a/.github/workflows/alerts-ee.yaml +++ b/.github/workflows/alerts-ee.yaml @@ -8,7 +8,8 @@ on: default: 'false' push: branches: - - api-v1.10.0 + - dev + - api-* paths: - "ee/api/**" - "api/**" diff --git a/.github/workflows/alerts.yaml b/.github/workflows/alerts.yaml index 539cc5e65..a24f2b855 100644 --- a/.github/workflows/alerts.yaml +++ b/.github/workflows/alerts.yaml @@ -8,7 +8,8 @@ on: default: 'false' push: branches: - - api-v1.10.0 + - dev + - api-* paths: - "api/**" - "!api/.gitignore" diff --git a/.github/workflows/api-ee.yaml b/.github/workflows/api-ee.yaml index b2a31f276..f9a1730f1 100644 --- a/.github/workflows/api-ee.yaml +++ b/.github/workflows/api-ee.yaml @@ -8,7 +8,8 @@ on: default: 'false' push: branches: - - api-v1.10.0 + - dev + - api-* paths: - "ee/api/**" - "api/**" diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 26d59ff87..8e2f7fa7b 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -8,7 +8,8 @@ on: default: 'false' push: branches: - - api-v1.10.0 + - dev + - api-* paths: - "api/**" - "!api/.gitignore" diff --git a/.github/workflows/assist-ee.yaml b/.github/workflows/assist-ee.yaml index 76dcc4a2d..e3f03ef5f 100644 --- a/.github/workflows/assist-ee.yaml +++ b/.github/workflows/assist-ee.yaml @@ -4,6 +4,7 @@ on: push: branches: - dev + - api-* paths: - "ee/utilities/**" - "utilities/**" diff --git a/.github/workflows/assist.yaml b/.github/workflows/assist.yaml index 65ca0348c..03ee1df5f 100644 --- a/.github/workflows/assist.yaml +++ b/.github/workflows/assist.yaml @@ -4,6 +4,7 @@ on: push: branches: - dev + - api-* paths: - "utilities/**" - "!utilities/.gitignore" diff --git a/.github/workflows/crons-ee.yaml b/.github/workflows/crons-ee.yaml index 762dae33e..77c098e4e 100644 --- a/.github/workflows/crons-ee.yaml +++ b/.github/workflows/crons-ee.yaml @@ -8,7 +8,8 @@ on: default: 'false' push: branches: - - api-v1.10.0 + - dev + - api-* paths: - "ee/api/**" - "api/**" diff --git a/.github/workflows/peers-ee.yaml b/.github/workflows/peers-ee.yaml index 5db7436da..dcd003e93 100644 --- a/.github/workflows/peers-ee.yaml +++ b/.github/workflows/peers-ee.yaml @@ -4,6 +4,7 @@ on: push: branches: - dev + - api-* paths: - "ee/peers/**" - "peers/**" diff --git a/.github/workflows/peers.yaml b/.github/workflows/peers.yaml index 7b2a715d8..2de0ae3ed 100644 --- a/.github/workflows/peers.yaml +++ b/.github/workflows/peers.yaml @@ -4,6 +4,7 @@ on: push: branches: - dev + - api-* paths: - "peers/**" - "!peers/.gitignore" diff --git a/.github/workflows/sourcemaps-reader.yaml b/.github/workflows/sourcemaps-reader.yaml index 095a70784..f0059da40 100644 --- a/.github/workflows/sourcemaps-reader.yaml +++ b/.github/workflows/sourcemaps-reader.yaml @@ -4,6 +4,7 @@ on: push: branches: - dev + - api-* paths: - "sourcemap-reader/**" - "!sourcemap-reader/.gitignore" From 850ee04787a827b856f99f776082006767c2492c Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 6 Mar 2023 12:26:04 +0100 Subject: [PATCH 025/253] change(player): decode state messages only on render --- .../components/Session_/Storage/Storage.tsx | 54 +++++++++++++++---- frontend/app/player/web/MessageManager.ts | 39 +++++--------- frontend/app/player/web/WebPlayer.ts | 6 +++ 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/frontend/app/components/Session_/Storage/Storage.tsx b/frontend/app/components/Session_/Storage/Storage.tsx index f955912e7..587d5bc31 100644 --- a/frontend/app/components/Session_/Storage/Storage.tsx +++ b/frontend/app/components/Session_/Storage/Storage.tsx @@ -12,6 +12,8 @@ import BottomBlock from '../BottomBlock/index'; import DiffRow from './DiffRow'; import cn from 'classnames'; import stl from './storage.module.css'; +import logger from "App/logger"; +import { toJS } from 'mobx' function getActionsName(type: string) { switch (type) { @@ -23,10 +25,19 @@ function getActionsName(type: string) { } } +const storageDecodeKeys = { + [STORAGE_TYPES.REDUX]: ['state', 'action'], + [STORAGE_TYPES.NGRX]: ['state', 'action'], + [STORAGE_TYPES.VUEX]: ['state', 'mutation'], + [STORAGE_TYPES.ZUSTAND]: ['state', 'mutation'], + [STORAGE_TYPES.MOBX]: ['payload'], + [STORAGE_TYPES.NONE]: ['state, action', 'payload', 'mutation'], +} interface Props { hideHint: (args: string) => void; hintIsHidden: boolean; } + function Storage(props: Props) { const lastBtnRef = React.useRef(); const [showDiffs, setShowDiffs] = React.useState(false); @@ -37,6 +48,24 @@ function Storage(props: Props) { const list = selectStorageList(state); const type = selectStorageType(state); + const decodeMessage = (msg: any) => { + const decoded = {}; + const pureMSG = toJS(msg) + const keys = storageDecodeKeys[type]; + try { + keys.forEach(key => { + if (pureMSG[key]) { + // @ts-ignore TODO: types for decoder + decoded[key] = player.decodeMessage(pureMSG[key]); + } + }); + } catch (e) { + logger.error("Error on message decoding: ", e, pureMSG); + return null; + } + return { ...pureMSG, ...decoded }; + } + const focusNextButton = () => { if (lastBtnRef.current) { lastBtnRef.current.focus(); @@ -106,27 +135,30 @@ function Storage(props: Props) { player.jump(list[listNow.length].time); }; - const renderItem = (item: Record, i: number, prevItem: Record) => { + const renderItem = (item: Record, i: number, prevItem?: Record) => { let src; let name; + const prevItemD = prevItem ? decodeMessage(prevItem) : undefined + const itemD = decodeMessage(item) + switch (type) { case STORAGE_TYPES.REDUX: case STORAGE_TYPES.NGRX: - src = item.action; + src = itemD.action; name = src && src.type; break; case STORAGE_TYPES.VUEX: - src = item.mutation; + src = itemD.mutation; name = src && src.type; break; case STORAGE_TYPES.MOBX: - src = item.payload; + src = itemD.payload; name = `@${item.type} ${src && src.type}`; break; case STORAGE_TYPES.ZUSTAND: src = null; - name = item.mutation.join(''); + name = itemD.mutation.join(''); } if (src !== null && !showDiffs) { @@ -144,7 +176,7 @@ function Storage(props: Props) { ) : ( <> - {renderDiff(item, prevItem)} + {renderDiff(itemD, prevItemD)}
{typeof item.duration === 'number' && ( -
{formatMs(item.duration)}
+
{formatMs(itemD.duration)}
)}
{i + 1 < listNow.length && ( - )} @@ -281,7 +313,11 @@ function Storage(props: Props) { {'Empty state.'}
) : ( - + )}
)} diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index 330d1c861..dced92641 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -107,7 +107,7 @@ export default class MessageManager { private scrollManager: ListWalker = new ListWalker(); - private readonly decoder = new Decoder(); + public readonly decoder = new Decoder(); private readonly lists: Lists; private activityManager: ActivityManager | null = null; @@ -406,39 +406,24 @@ export default class MessageManager { this.lists.lists.fetch.insert(getResourceFromNetworkRequest(msg, this.sessionStart)) break; case MType.Redux: - decoded = this.decodeStateMessage(msg, ["state", "action"]); - logger.log('redux', decoded) - if (decoded != null) { - this.lists.lists.redux.append(decoded); - } + logger.log('redux', msg) + this.lists.lists.redux.append(msg); break; case MType.NgRx: - decoded = this.decodeStateMessage(msg, ["state", "action"]); - logger.log('ngrx', decoded) - if (decoded != null) { - this.lists.lists.ngrx.append(decoded); - } + logger.log('ngrx', msg) + this.lists.lists.ngrx.append(msg); break; case MType.Vuex: - decoded = this.decodeStateMessage(msg, ["state", "mutation"]); - logger.log('vuex', decoded) - if (decoded != null) { - this.lists.lists.vuex.append(decoded); - } + logger.log('vuex', msg) + this.lists.lists.vuex.append(msg); break; case MType.Zustand: - decoded = this.decodeStateMessage(msg, ["state", "mutation"]) - logger.log('zustand', decoded) - if (decoded != null) { - this.lists.lists.zustand.append(decoded) - } + logger.log('zustand', msg) + this.lists.lists.zustand.append(msg) + break case MType.MobX: - decoded = this.decodeStateMessage(msg, ["payload"]); - logger.log('mobx', decoded) - - if (decoded != null) { - this.lists.lists.mobx.append(decoded); - } + logger.log('mobx', msg) + this.lists.lists.mobx.append(msg); break; case MType.GraphQl: this.lists.lists.graphql.append(msg); diff --git a/frontend/app/player/web/WebPlayer.ts b/frontend/app/player/web/WebPlayer.ts index d1a56f9fd..0889089ee 100644 --- a/frontend/app/player/web/WebPlayer.ts +++ b/frontend/app/player/web/WebPlayer.ts @@ -7,6 +7,7 @@ import MessageManager from './MessageManager' import InspectorController from './addons/InspectorController' import TargetMarker from './addons/TargetMarker' import Screen, { ScaleMode } from './Screen/Screen' +import { Message } from "Player/web/messages"; // export type State = typeof WebPlayer.INITIAL_STATE @@ -83,6 +84,11 @@ export default class WebPlayer extends Player { this.targetMarker.updateMarkedTargets() } + // delayed message decoding for state plugins + decodeMessage = (msg: Message) => { + return this.messageManager.decoder.decode(msg) + } + // Inspector & marker mark(e: Element) { this.inspectorController.marker?.mark(e) From ef83de38a2ed4e3b364ba29b85b0257465ae59f1 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 6 Mar 2023 12:40:45 +0100 Subject: [PATCH 026/253] change(player): use shallow object spread instead of recursive tojs --- frontend/app/components/Session_/Storage/Storage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/Session_/Storage/Storage.tsx b/frontend/app/components/Session_/Storage/Storage.tsx index 587d5bc31..25842840f 100644 --- a/frontend/app/components/Session_/Storage/Storage.tsx +++ b/frontend/app/components/Session_/Storage/Storage.tsx @@ -50,7 +50,7 @@ function Storage(props: Props) { const decodeMessage = (msg: any) => { const decoded = {}; - const pureMSG = toJS(msg) + const pureMSG = { ...msg } const keys = storageDecodeKeys[type]; try { keys.forEach(key => { From 65a66b9c59a8dfcc48fa0cae1e0b3536c15bcde8 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 6 Mar 2023 14:26:59 +0100 Subject: [PATCH 027/253] change(player): small bug fixes --- .../app/components/Session/Player/ReplayPlayer/PlayerInst.tsx | 3 --- frontend/app/components/Session/WebPlayer.tsx | 3 ++- frontend/app/components/Session_/Storage/Storage.tsx | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/Session/Player/ReplayPlayer/PlayerInst.tsx b/frontend/app/components/Session/Player/ReplayPlayer/PlayerInst.tsx index 6a300cce7..097a57694 100644 --- a/frontend/app/components/Session/Player/ReplayPlayer/PlayerInst.tsx +++ b/frontend/app/components/Session/Player/ReplayPlayer/PlayerInst.tsx @@ -68,9 +68,6 @@ function Player(props: IProps) { playerContext.player.attach(parentElement); setAttached(true) } - if (isAttached && isReady) { - playerContext.player.play(); - } }, [isReady]); React.useEffect(() => { diff --git a/frontend/app/components/Session/WebPlayer.tsx b/frontend/app/components/Session/WebPlayer.tsx index 383b70c63..659dbe540 100644 --- a/frontend/app/components/Session/WebPlayer.tsx +++ b/frontend/app/components/Session/WebPlayer.tsx @@ -70,7 +70,8 @@ function WebPlayer(props: any) { if (showNoteModal) { contextValue.player.pause() } - if (activeTab !== 'Click Map' && !showNoteModal && isPlayerReady) { + + if (activeTab === '' && !showNoteModal && isPlayerReady) { contextValue.player && contextValue.player.play() } }, [activeTab, isPlayerReady, showNoteModal]) diff --git a/frontend/app/components/Session_/Storage/Storage.tsx b/frontend/app/components/Session_/Storage/Storage.tsx index 25842840f..7cabca865 100644 --- a/frontend/app/components/Session_/Storage/Storage.tsx +++ b/frontend/app/components/Session_/Storage/Storage.tsx @@ -13,7 +13,6 @@ import DiffRow from './DiffRow'; import cn from 'classnames'; import stl from './storage.module.css'; import logger from "App/logger"; -import { toJS } from 'mobx' function getActionsName(type: string) { switch (type) { From 1cd1f7231049e52681c631ed3c706f635e04d778 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 6 Mar 2023 15:26:02 +0100 Subject: [PATCH 028/253] change(player): only recompute list on change --- .../components/Session_/Storage/Storage.tsx | 58 +++++++++++++++++-- frontend/app/player/web/MessageManager.ts | 4 ++ frontend/app/player/web/WebPlayer.ts | 2 +- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/Session_/Storage/Storage.tsx b/frontend/app/components/Session_/Storage/Storage.tsx index 7cabca865..8f7f3e92e 100644 --- a/frontend/app/components/Session_/Storage/Storage.tsx +++ b/frontend/app/components/Session_/Storage/Storage.tsx @@ -65,6 +65,12 @@ function Storage(props: Props) { return { ...pureMSG, ...decoded }; } + const decodedList = React.useMemo(() => { + return listNow.map(msg => { + return decodeMessage(msg) + }) + }, [listNow.length]) + const focusNextButton = () => { if (lastBtnRef.current) { lastBtnRef.current.focus(); @@ -138,8 +144,8 @@ function Storage(props: Props) { let src; let name; - const prevItemD = prevItem ? decodeMessage(prevItem) : undefined - const itemD = decodeMessage(item) + const itemD = item + const prevItemD = prevItem ? prevItem : undefined switch (type) { case STORAGE_TYPES.REDUX: @@ -322,8 +328,8 @@ function Storage(props: Props) { )}
- {listNow.map((item: Record, i: number) => - renderItem(item, i, i > 0 ? listNow[i - 1] : undefined) + {decodedList.map((item: Record, i: number) => + renderItem(item, i, i > 0 ? decodedList[i - 1] : undefined) )}
@@ -341,3 +347,47 @@ export default connect( hideHint, } )(observer(Storage)); + + +/** + * TODO: compute diff and only decode the required parts + * WIP example + * function useStorageDecryptedList(list: Record[], type: string, player: IWebPlayer) { + * const [decryptedList, setDecryptedList] = React.useState(list); + * const [listLength, setLength] = React.useState(list.length) + * + * const decodeMessage = (msg: any, type: StorageType) => { + * const decoded = {}; + * const pureMSG = { ...msg } + * const keys = storageDecodeKeys[type]; + * try { + * keys.forEach(key => { + * if (pureMSG[key]) { + * // @ts-ignore TODO: types for decoder + * decoded[key] = player.decodeMessage(pureMSG[key]); + * } + * }); + * } catch (e) { + * logger.error("Error on message decoding: ", e, pureMSG); + * return null; + * } + * return { ...pureMSG, ...decoded }; + * } + * + * React.useEffect(() => { + * if (list.length !== listLength) { + * const last = list[list.length - 1]._index; + * let diff; + * if (last < decryptedList[decryptedList.length - 1]._index) { + * + * } + * diff = list.filter(item => !decryptedList.includes(i => i._index === item._index)) + * const decryptedDiff = diff.map(item => { + * return player.decodeMessage(item) + * }) + * const result = + * } + * }, [list.length]) + * } + * + * */ \ No newline at end of file diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index dced92641..0d9aaee3d 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -463,6 +463,10 @@ export default class MessageManager { this.state.update({ messagesLoading, ready: !messagesLoading && !this.state.get().cssLoading }); } + decodeMessage(msg: Message) { + return this.decoder.decode(msg) + } + private setSize({ height, width }: { height: number, width: number }) { this.screen.scale({ height, width }); this.state.update({ width, height }); diff --git a/frontend/app/player/web/WebPlayer.ts b/frontend/app/player/web/WebPlayer.ts index 0889089ee..9ca769598 100644 --- a/frontend/app/player/web/WebPlayer.ts +++ b/frontend/app/player/web/WebPlayer.ts @@ -86,7 +86,7 @@ export default class WebPlayer extends Player { // delayed message decoding for state plugins decodeMessage = (msg: Message) => { - return this.messageManager.decoder.decode(msg) + return this.messageManager.decodeMessage(msg) } // Inspector & marker From 478bf3c69cef40e2d85dea0fe3e8a70cbf4559f9 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 6 Mar 2023 17:05:05 +0100 Subject: [PATCH 029/253] chore(init): Overwrite existing directory cho Signed-off-by: rjshrjndrn --- scripts/helmcharts/init.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/init.sh b/scripts/helmcharts/init.sh index 69a6944b2..525a00a82 100644 --- a/scripts/helmcharts/init.sh +++ b/scripts/helmcharts/init.sh @@ -175,8 +175,14 @@ function main() { install_openreplay sudo mkdir -p /var/lib/openreplay sudo cp -f openreplay-cli /bin/openreplay - sudo cp -rf ../../../openreplay /var/lib/openreplay - sudo cp -f vars.yaml /var/lib/openreplay + [[ ! -d /var/lib/openreplay/openreplay ]] || { + cd /var/lib/openreplay/openreplay + date +%m-%d-%Y-%H%M%S | sudo tee -a /var/lib/openreplay/or_versions.txt + sudo git log -1 2>&1 | sudo tee -a /var/lib/openreplay/or_versions.txt + sudo rm -rf /var/lib/openreplay/openreplay + cd - + } + sudo cp -rf $(cd ../.. && pwd) /var/lib/openreplay/openreplay } } From 40f1eed94a7ee0a0d35fd13cfa4805c1f197f80e Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 6 Mar 2023 18:10:15 +0100 Subject: [PATCH 030/253] change(ui) - tracking options - data recording dropdown option order --- frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx | 2 +- .../TrackingCodeModal/ProjectCodeSnippet/ProjectCodeSnippet.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx b/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx index c9629bf7a..116392534 100644 --- a/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx +++ b/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx @@ -4,8 +4,8 @@ import Highlight from 'react-highlight'; const inputModeOptions = [ { label: 'Record all inputs', value: 'plain' }, - { label: 'Ignore all inputs', value: 'obscured' }, { label: 'Obscure all inputs', value: 'hidden' }, + { label: 'Ignore all inputs', value: 'obscured' }, ]; const inputModeOptionsMap: any = {} diff --git a/frontend/app/components/shared/TrackingCodeModal/ProjectCodeSnippet/ProjectCodeSnippet.js b/frontend/app/components/shared/TrackingCodeModal/ProjectCodeSnippet/ProjectCodeSnippet.js index b8bd87b6a..4b51fc963 100644 --- a/frontend/app/components/shared/TrackingCodeModal/ProjectCodeSnippet/ProjectCodeSnippet.js +++ b/frontend/app/components/shared/TrackingCodeModal/ProjectCodeSnippet/ProjectCodeSnippet.js @@ -9,8 +9,8 @@ import CodeSnippet from '../../CodeSnippet'; const inputModeOptions = [ { label: 'Record all inputs', value: 'plain' }, - { label: 'Ignore all inputs', value: 'obscured' }, { label: 'Obscure all inputs', value: 'hidden' }, + { label: 'Ignore all inputs', value: 'obscured' }, ]; const inputModeOptionsMap = {} From 8d6afb2586cbcf5776c99c48bfeef3465219cc8e Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 7 Mar 2023 06:32:30 +0100 Subject: [PATCH 031/253] chore(helm): Clean cron every 2 days once Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/charts/utilities/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/utilities/values.yaml index 97ee29798..a8a2fddaa 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/values.yaml @@ -80,8 +80,8 @@ nameOverride: "utilities" fullnameOverride: "utilities-openreplay" # 5 3 * * 1 “At 03:05 on Monday.” -# refer: https://crontab.guru/#5_3_*_*_1 -cron: "5 3 * * 1" +# refer: https://crontab.guru/#5_3_*_*_*/2 +cron: "5 3 * * */2" # Pod configurations From 9a0ba57d491ae7d37a26724932f0da001d3bda73 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 7 Mar 2023 09:09:12 +0100 Subject: [PATCH 032/253] fix(ui) - events tab checking for payload --- frontend/app/types/session/stackEvent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/types/session/stackEvent.ts b/frontend/app/types/session/stackEvent.ts index 8ce375fc2..8bbea2778 100644 --- a/frontend/app/types/session/stackEvent.ts +++ b/frontend/app/types/session/stackEvent.ts @@ -58,7 +58,7 @@ export default class StackEvent { level: IStackEvent["level"]; constructor(evt: IStackEvent) { - const event = { ...evt, source: evt.source || OPENREPLAY } + const event = { ...evt, source: evt.source || OPENREPLAY, payload: evt.payload || {} }; Object.assign(this, { ...event, isRed: isRed(event), From cbd8f34ec1628beb882316f1dbab341e09fc16b9 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 7 Mar 2023 10:03:56 +0100 Subject: [PATCH 033/253] change(tracker): change default input mode to obscured --- tracker/tracker/CHANGELOG.md | 4 ++++ tracker/tracker/src/main/modules/input.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index 559e4e865..329f3aa59 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,3 +1,7 @@ +## 5.0.1 + +- Default text input mode is now Obscured + ## 5.0.0 - Added "tel" to supported input types diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 15acecaa9..e2e93bff7 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -89,7 +89,7 @@ export default function (app: App, opts: Partial): void { { obscureInputNumbers: true, obscureInputEmails: true, - defaultInputMode: InputMode.Plain, + defaultInputMode: InputMode.Obscured, obscureInputDates: false, }, opts, From a13363d97386811c7bb6f5ce3646520b6644cda2 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 7 Mar 2023 12:15:58 +0100 Subject: [PATCH 034/253] change(player): sort msgs by timestamp --- frontend/app/player/web/MessageManager.ts | 28 +++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index 0d9aaee3d..e41bf04d5 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -196,7 +196,7 @@ export default class MessageManager { async loadMessages(isClickmap: boolean = false) { this.setMessagesLoading(true) // TODO: reusable decryptor instance - const createNewParser = (shouldDecrypt = true) => { + const createNewParser = (shouldDecrypt = true, file) => { const decrypt = shouldDecrypt && this.session.fileKey ? (b: Uint8Array) => decryptSessionBytes(b, this.session.fileKey) : (b: Uint8Array) => Promise.resolve(b) @@ -206,11 +206,21 @@ export default class MessageManager { fileReader.append(b) const msgs: Array = [] for (let msg = fileReader.readNext();msg !== null;msg = fileReader.readNext()) { - this.distributeMessage(msg, msg._index) msgs.push(msg) } + const sorted = msgs.sort((m1, m2) => m1.time - m2.time) + + let indx = sorted[0]._index + let counter = 0 + sorted.forEach(msg => { + if (indx > msg._index) counter++ + else indx = msg._index + this.distributeMessage(msg, msg._index) + }) + + if (counter > 0) console.warn("Unsorted mob file, error count: ", counter) + logger.info("Messages count: ", msgs.length, sorted, file) - logger.info("Messages count: ", msgs.length, msgs) this._sortMessagesHack(msgs) this.setMessagesLoading(false) }) @@ -219,8 +229,8 @@ export default class MessageManager { this.waitingForFiles = true const loadMethod = this.session.domURL && this.session.domURL.length > 0 - ? { url: this.session.domURL, parser: createNewParser } - : { url: this.session.mobsUrl, parser: () => createNewParser(false)} + ? { url: this.session.domURL, parser: () => createNewParser(true, 'dom') } + : { url: this.session.mobsUrl, parser: () => createNewParser(false, 'dom')} loadFiles(loadMethod.url, loadMethod.parser()) // EFS fallback @@ -235,11 +245,11 @@ export default class MessageManager { // load devtools (TODO: start after the first DOM file download) if (isClickmap) return; this.state.update({ devtoolsLoading: true }) - loadFiles(this.session.devtoolsURL, createNewParser()) + loadFiles(this.session.devtoolsURL, createNewParser(true, 'devtools')) // EFS fallback .catch(() => requestEFSDevtools(this.session.sessionId) - .then(createNewParser(false)) + .then(createNewParser(false, 'devtools')) ) .then(() => { this.state.update(this.lists.getFullListsState()) // TODO: also in case of dynamic update through assist @@ -406,7 +416,7 @@ export default class MessageManager { this.lists.lists.fetch.insert(getResourceFromNetworkRequest(msg, this.sessionStart)) break; case MType.Redux: - logger.log('redux', msg) + // logger.log('redux', msg) this.lists.lists.redux.append(msg); break; case MType.NgRx: @@ -414,7 +424,7 @@ export default class MessageManager { this.lists.lists.ngrx.append(msg); break; case MType.Vuex: - logger.log('vuex', msg) + // logger.log('vuex', msg) this.lists.lists.vuex.append(msg); break; case MType.Zustand: From 6623e902e93ec034387d688adcbc7705b17685cd Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 7 Mar 2023 12:18:43 +0100 Subject: [PATCH 035/253] change(player): small fix --- frontend/app/player/web/MessageManager.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index e41bf04d5..c54b86fb8 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -211,14 +211,14 @@ export default class MessageManager { const sorted = msgs.sort((m1, m2) => m1.time - m2.time) let indx = sorted[0]._index - let counter = 0 + let outOfOrderCounter = 0 sorted.forEach(msg => { - if (indx > msg._index) counter++ + if (indx > msg._index) outOfOrderCounter++ else indx = msg._index this.distributeMessage(msg, msg._index) }) - if (counter > 0) console.warn("Unsorted mob file, error count: ", counter) + if (outOfOrderCounter > 0) console.warn("Unsorted mob file, error count: ", outOfOrderCounter) logger.info("Messages count: ", msgs.length, sorted, file) this._sortMessagesHack(msgs) @@ -236,7 +236,7 @@ export default class MessageManager { // EFS fallback .catch((e) => requestEFSDom(this.session.sessionId) - .then(createNewParser(false)) + .then(createNewParser(false, 'domEFS')) ) .then(this.onFileReadSuccess) .catch(this.onFileReadFailed) @@ -249,7 +249,7 @@ export default class MessageManager { // EFS fallback .catch(() => requestEFSDevtools(this.session.sessionId) - .then(createNewParser(false, 'devtools')) + .then(createNewParser(false, 'devtoolsEFS')) ) .then(() => { this.state.update(this.lists.getFullListsState()) // TODO: also in case of dynamic update through assist @@ -416,23 +416,18 @@ export default class MessageManager { this.lists.lists.fetch.insert(getResourceFromNetworkRequest(msg, this.sessionStart)) break; case MType.Redux: - // logger.log('redux', msg) this.lists.lists.redux.append(msg); break; case MType.NgRx: - logger.log('ngrx', msg) this.lists.lists.ngrx.append(msg); break; case MType.Vuex: - // logger.log('vuex', msg) this.lists.lists.vuex.append(msg); break; case MType.Zustand: - logger.log('zustand', msg) this.lists.lists.zustand.append(msg) break case MType.MobX: - logger.log('mobx', msg) this.lists.lists.mobx.append(msg); break; case MType.GraphQl: From e7791a36a36483614c9d83823f64fa05bf5e9faf Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 14:35:40 +0100 Subject: [PATCH 036/253] feat(assist): refactored --- .github/workflows/assist-ee.yaml | 10 +++++----- .github/workflows/assist.yaml | 8 ++++---- {utilities => assist}/.dockerignore | 0 {utilities => assist}/.gitignore | 0 {utilities => assist}/Dockerfile | 0 {utilities => assist}/build.sh | 10 +++++----- {utilities => assist}/package-lock.json | 0 {utilities => assist}/package.json | 0 {ee/utilities => assist}/run-dev.sh | 0 {utilities => assist}/server.js | 0 {utilities => assist}/servers/websocket.js | 0 {utilities => assist}/utils/HeapSnapshot.js | 0 {utilities => assist}/utils/assistHelper.js | 0 {utilities => assist}/utils/geoIP.js | 0 {utilities => assist}/utils/helper.js | 0 ee/{utilities => assist}/.gitignore | 0 ee/{utilities => assist}/Dockerfile | 0 ee/{utilities => assist}/clean-dev.sh | 0 ee/{utilities => assist}/package-lock.json | 0 ee/{utilities => assist}/package.json | 0 ee/{utilities => assist}/prepare-dev.sh | 2 +- {utilities => ee/assist}/run-dev.sh | 0 ee/{utilities => assist}/server.js | 0 ee/{utilities => assist}/servers/websocket-cluster.js | 0 ee/{utilities => assist}/servers/websocket.js | 0 ee/{utilities => assist}/utils/helper-ee.js | 0 peers/build.sh | 2 +- sourcemap-reader/build.sh | 2 +- sourcemap-reader/prepare-dev.sh | 2 +- 29 files changed, 18 insertions(+), 18 deletions(-) rename {utilities => assist}/.dockerignore (100%) rename {utilities => assist}/.gitignore (100%) rename {utilities => assist}/Dockerfile (100%) rename {utilities => assist}/build.sh (86%) rename {utilities => assist}/package-lock.json (100%) rename {utilities => assist}/package.json (100%) rename {ee/utilities => assist}/run-dev.sh (100%) rename {utilities => assist}/server.js (100%) rename {utilities => assist}/servers/websocket.js (100%) rename {utilities => assist}/utils/HeapSnapshot.js (100%) rename {utilities => assist}/utils/assistHelper.js (100%) rename {utilities => assist}/utils/geoIP.js (100%) rename {utilities => assist}/utils/helper.js (100%) rename ee/{utilities => assist}/.gitignore (100%) rename ee/{utilities => assist}/Dockerfile (100%) rename ee/{utilities => assist}/clean-dev.sh (100%) rename ee/{utilities => assist}/package-lock.json (100%) rename ee/{utilities => assist}/package.json (100%) rename ee/{utilities => assist}/prepare-dev.sh (75%) rename {utilities => ee/assist}/run-dev.sh (100%) rename ee/{utilities => assist}/server.js (100%) rename ee/{utilities => assist}/servers/websocket-cluster.js (100%) rename ee/{utilities => assist}/servers/websocket.js (100%) rename ee/{utilities => assist}/utils/helper-ee.js (100%) diff --git a/.github/workflows/assist-ee.yaml b/.github/workflows/assist-ee.yaml index e3f03ef5f..f2fa32ba2 100644 --- a/.github/workflows/assist-ee.yaml +++ b/.github/workflows/assist-ee.yaml @@ -6,10 +6,10 @@ on: - dev - api-* paths: - - "ee/utilities/**" - - "utilities/**" - - "!utilities/.gitignore" - - "!utilities/*-dev.sh" + - "ee/assist/**" + - "assist/**" + - "!assist/.gitignore" + - "!assist/*-dev.sh" name: Build and Deploy Assist EE @@ -44,7 +44,7 @@ jobs: ENVIRONMENT: staging run: | skip_security_checks=${{ github.event.inputs.skip_security_checks }} - cd utilities + cd assist PUSH_IMAGE=0 bash -x ./build.sh ee [[ "x$skip_security_checks" == "xtrue" ]] || { curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ diff --git a/.github/workflows/assist.yaml b/.github/workflows/assist.yaml index 03ee1df5f..67bfed543 100644 --- a/.github/workflows/assist.yaml +++ b/.github/workflows/assist.yaml @@ -6,9 +6,9 @@ on: - dev - api-* paths: - - "utilities/**" - - "!utilities/.gitignore" - - "!utilities/*-dev.sh" + - "assist/**" + - "!assist/.gitignore" + - "!assist/*-dev.sh" name: Build and Deploy Assist @@ -43,7 +43,7 @@ jobs: ENVIRONMENT: staging run: | skip_security_checks=${{ github.event.inputs.skip_security_checks }} - cd utilities + cd assist PUSH_IMAGE=0 bash -x ./build.sh [[ "x$skip_security_checks" == "xtrue" ]] || { curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ diff --git a/utilities/.dockerignore b/assist/.dockerignore similarity index 100% rename from utilities/.dockerignore rename to assist/.dockerignore diff --git a/utilities/.gitignore b/assist/.gitignore similarity index 100% rename from utilities/.gitignore rename to assist/.gitignore diff --git a/utilities/Dockerfile b/assist/Dockerfile similarity index 100% rename from utilities/Dockerfile rename to assist/Dockerfile diff --git a/utilities/build.sh b/assist/build.sh similarity index 86% rename from utilities/build.sh rename to assist/build.sh index 87ff7f3e6..7a780cc43 100644 --- a/utilities/build.sh +++ b/assist/build.sh @@ -16,20 +16,20 @@ check_prereq() { } function build_api(){ - destination="_utilities" + destination="_assist" [[ $1 == "ee" ]] && { - destination="_utilities_ee" + destination="_assist_ee" } - cp -R ../utilities ../${destination} + cp -R ../assist ../${destination} cd ../${destination} # Copy enterprise code [[ $1 == "ee" ]] && { - cp -rf ../ee/utilities/* ./ + cp -rf ../ee/assist/* ./ } docker build -f ./Dockerfile --build-arg GIT_SHA=$git_sha -t ${DOCKER_REPO:-'local'}/assist:${image_tag} . - cd ../utilities + cd ../assist rm -rf ../${destination} [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/assist:${image_tag} diff --git a/utilities/package-lock.json b/assist/package-lock.json similarity index 100% rename from utilities/package-lock.json rename to assist/package-lock.json diff --git a/utilities/package.json b/assist/package.json similarity index 100% rename from utilities/package.json rename to assist/package.json diff --git a/ee/utilities/run-dev.sh b/assist/run-dev.sh similarity index 100% rename from ee/utilities/run-dev.sh rename to assist/run-dev.sh diff --git a/utilities/server.js b/assist/server.js similarity index 100% rename from utilities/server.js rename to assist/server.js diff --git a/utilities/servers/websocket.js b/assist/servers/websocket.js similarity index 100% rename from utilities/servers/websocket.js rename to assist/servers/websocket.js diff --git a/utilities/utils/HeapSnapshot.js b/assist/utils/HeapSnapshot.js similarity index 100% rename from utilities/utils/HeapSnapshot.js rename to assist/utils/HeapSnapshot.js diff --git a/utilities/utils/assistHelper.js b/assist/utils/assistHelper.js similarity index 100% rename from utilities/utils/assistHelper.js rename to assist/utils/assistHelper.js diff --git a/utilities/utils/geoIP.js b/assist/utils/geoIP.js similarity index 100% rename from utilities/utils/geoIP.js rename to assist/utils/geoIP.js diff --git a/utilities/utils/helper.js b/assist/utils/helper.js similarity index 100% rename from utilities/utils/helper.js rename to assist/utils/helper.js diff --git a/ee/utilities/.gitignore b/ee/assist/.gitignore similarity index 100% rename from ee/utilities/.gitignore rename to ee/assist/.gitignore diff --git a/ee/utilities/Dockerfile b/ee/assist/Dockerfile similarity index 100% rename from ee/utilities/Dockerfile rename to ee/assist/Dockerfile diff --git a/ee/utilities/clean-dev.sh b/ee/assist/clean-dev.sh similarity index 100% rename from ee/utilities/clean-dev.sh rename to ee/assist/clean-dev.sh diff --git a/ee/utilities/package-lock.json b/ee/assist/package-lock.json similarity index 100% rename from ee/utilities/package-lock.json rename to ee/assist/package-lock.json diff --git a/ee/utilities/package.json b/ee/assist/package.json similarity index 100% rename from ee/utilities/package.json rename to ee/assist/package.json diff --git a/ee/utilities/prepare-dev.sh b/ee/assist/prepare-dev.sh similarity index 75% rename from ee/utilities/prepare-dev.sh rename to ee/assist/prepare-dev.sh index 2daecbfc1..8da98eac3 100755 --- a/ee/utilities/prepare-dev.sh +++ b/ee/assist/prepare-dev.sh @@ -1,2 +1,2 @@ #!/bin/bash -rsync -avr --exclude=".*" --exclude="node_modules" --ignore-existing ../../utilities/* ./ \ No newline at end of file +rsync -avr --exclude=".*" --exclude="node_modules" --ignore-existing ../../assist/* ./ \ No newline at end of file diff --git a/utilities/run-dev.sh b/ee/assist/run-dev.sh similarity index 100% rename from utilities/run-dev.sh rename to ee/assist/run-dev.sh diff --git a/ee/utilities/server.js b/ee/assist/server.js similarity index 100% rename from ee/utilities/server.js rename to ee/assist/server.js diff --git a/ee/utilities/servers/websocket-cluster.js b/ee/assist/servers/websocket-cluster.js similarity index 100% rename from ee/utilities/servers/websocket-cluster.js rename to ee/assist/servers/websocket-cluster.js diff --git a/ee/utilities/servers/websocket.js b/ee/assist/servers/websocket.js similarity index 100% rename from ee/utilities/servers/websocket.js rename to ee/assist/servers/websocket.js diff --git a/ee/utilities/utils/helper-ee.js b/ee/assist/utils/helper-ee.js similarity index 100% rename from ee/utilities/utils/helper-ee.js rename to ee/assist/utils/helper-ee.js diff --git a/peers/build.sh b/peers/build.sh index 45cc97892..746a12f9d 100644 --- a/peers/build.sh +++ b/peers/build.sh @@ -22,7 +22,7 @@ function build_api(){ } cp -R ../peers ../${destination} cd ../${destination} - cp -R ../utilities/utils . + cp -R ../assist/utils . # Copy enterprise code [[ $1 == "ee" ]] && { cp -rf ../ee/peers/* ./ diff --git a/sourcemap-reader/build.sh b/sourcemap-reader/build.sh index fbe8762e2..3025e05e4 100644 --- a/sourcemap-reader/build.sh +++ b/sourcemap-reader/build.sh @@ -30,7 +30,7 @@ function build_api(){ } cp -R ../sourcemap-reader ../${destination} cd ../${destination} - cp -R ../utilities/utils . + cp -R ../assist/utils . tag="" # Copy enterprise code [[ $1 == "ee" ]] && { diff --git a/sourcemap-reader/prepare-dev.sh b/sourcemap-reader/prepare-dev.sh index e057555db..78a315946 100755 --- a/sourcemap-reader/prepare-dev.sh +++ b/sourcemap-reader/prepare-dev.sh @@ -1,2 +1,2 @@ #!/bin/bash -rsync -avr --exclude=".*" --ignore-existing ../utilities/utils ./ \ No newline at end of file +rsync -avr --exclude=".*" --ignore-existing ../assist/utils ./ \ No newline at end of file From 2edb88f1b30c9840b47e7a92b33a9bb059c6be8d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 14:37:23 +0100 Subject: [PATCH 037/253] feat(DB): changed GDPR default values feat(DB): heuristics delta --- .../db/init_dbs/postgresql/1.11.0/1.11.0.sql | 24 +++++++++++++++++++ .../db/init_dbs/postgresql/init_schema.sql | 2 +- .../db/init_dbs/postgresql/1.11.0/1.11.0.sql | 20 ++++++++++++++++ .../db/init_dbs/postgresql/init_schema.sql | 2 +- 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql create mode 100644 scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql new file mode 100644 index 000000000..30bb27997 --- /dev/null +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql @@ -0,0 +1,24 @@ +BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.11.0-ee' +$$ LANGUAGE sql IMMUTABLE; + + + +ALTER TABLE events.inputs + ADD COLUMN duration integer NULL, + ADD COLUMN hesitation integer NULL; + + + +ALTER TABLE public.projects + ALTER COLUMN gdpr SET DEFAULT '{ + "maskEmails": true, + "sampleRate": 33, + "maskNumbers": false, + "defaultInputMode": "obscured" + }'::jsonb; + +COMMIT; \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 0b2945b39..308acbda1 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -253,7 +253,7 @@ $$ "maskEmails": true, "sampleRate": 33, "maskNumbers": false, - "defaultInputMode": "plain" + "defaultInputMode": "obscured" }'::jsonb, first_recorded_session_at timestamp without time zone NULL DEFAULT NULL, sessions_last_check_at timestamp without time zone NULL DEFAULT NULL, diff --git a/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql b/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql new file mode 100644 index 000000000..13813f5bc --- /dev/null +++ b/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql @@ -0,0 +1,20 @@ +BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.11.0' +$$ LANGUAGE sql IMMUTABLE; + +ALTER TABLE events.inputs + ADD COLUMN duration integer NULL, + ADD COLUMN hesitation integer NULL; + +ALTER TABLE public.projects + ALTER COLUMN gdpr SET DEFAULT '{ + "maskEmails": true, + "sampleRate": 33, + "maskNumbers": false, + "defaultInputMode": "obscured" + }'::jsonb; + +COMMIT; \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index 57dea2a58..14b6550a2 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -190,7 +190,7 @@ $$ "maskEmails": true, "sampleRate": 33, "maskNumbers": false, - "defaultInputMode": "plain" + "defaultInputMode": "obscured" }'::jsonb, first_recorded_session_at timestamp without time zone NULL DEFAULT NULL, sessions_last_check_at timestamp without time zone NULL DEFAULT NULL, From 4d85adbc0c1b4afc6854a7aa1d51967499650e08 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 14:53:20 +0100 Subject: [PATCH 038/253] chore(actions): changed peers/peersEE actions --- .github/workflows/peers-ee.yaml | 28 ++++++++++++++++++++-------- .github/workflows/peers.yaml | 23 +++++++++++++++++------ 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/.github/workflows/peers-ee.yaml b/.github/workflows/peers-ee.yaml index dcd003e93..b8d66a082 100644 --- a/.github/workflows/peers-ee.yaml +++ b/.github/workflows/peers-ee.yaml @@ -11,7 +11,7 @@ on: - "!peers/.gitignore" - "!peers/*-dev.sh" -name: Build and Deploy Peers +name: Build and Deploy Peers EE jobs: deploy: @@ -48,13 +48,25 @@ jobs: - name: Deploy to kubernetes run: | cd scripts/helmcharts/ - sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.EE_REGISTRY_URL }}\"#g" vars.yaml - sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\" #g" vars.yaml - sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.EE_MINIO_SECRET_KEY }}\" #g" vars.yaml - sed -i "s#domain_name.*#domain_name: \"ee.openreplay.com\" #g" vars.yaml - sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml - sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml - bash kube-install.sh --app peers + + ## Update secerts + sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml + sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.EE_PG_PASSWORD }}\"/g" vars.yaml + sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\"/g" vars.yaml + sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.EE_MINIO_SECRET_KEY }}\"/g" vars.yaml + sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.EE_JWT_SECRET }}\"/g" vars.yaml + sed -i "s/domainName: \"\"/domainName: \"${{ secrets.EE_DOMAIN_NAME }}\"/g" vars.yaml + sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.EE_LICENSE_KEY }}\"/g" vars.yaml + + # Update changed image tag + sed -i "/peers/{n;n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + + cat /tmp/image_override.yaml + # Deploy command + mv openreplay/charts/{ingress-nginx,peers,quickwit} /tmp + rm -rf openreplay/charts/* + mv /tmp/{ingress-nginx,peers,quickwit} openreplay/charts/ + helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f - env: DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} diff --git a/.github/workflows/peers.yaml b/.github/workflows/peers.yaml index 2de0ae3ed..65264d7b5 100644 --- a/.github/workflows/peers.yaml +++ b/.github/workflows/peers.yaml @@ -47,13 +47,24 @@ jobs: - name: Deploy to kubernetes run: | cd scripts/helmcharts/ + + ## Update secerts sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml - sed -i "s#minio_access_key.*#minio_access_key: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\" #g" vars.yaml - sed -i "s#minio_secret_key.*#minio_secret_key: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\" #g" vars.yaml - sed -i "s#domain_name.*#domain_name: \"foss.openreplay.com\" #g" vars.yaml - sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml - sed -i "s/image_tag:.*/image_tag: \"$IMAGE_TAG\"/g" vars.yaml - bash kube-install.sh --app peers + sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.OSS_PG_PASSWORD }}\"/g" vars.yaml + sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.OSS_MINIO_ACCESS_KEY }}\"/g" vars.yaml + sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.OSS_MINIO_SECRET_KEY }}\"/g" vars.yaml + sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.OSS_JWT_SECRET }}\"/g" vars.yaml + sed -i "s/domainName: \"\"/domainName: \"${{ secrets.OSS_DOMAIN_NAME }}\"/g" vars.yaml + + # Update changed image tag + sed -i "/peers/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + + cat /tmp/image_override.yaml + # Deploy command + mv openreplay/charts/{ingress-nginx,peers,quickwit} /tmp + rm -rf openreplay/charts/* + mv /tmp/{ingress-nginx,peers,quickwit} openreplay/charts/ + helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks | kubectl apply -n app -f - env: DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} From 35b938238d786097729f1a7ec5f54ddfa25fd1d7 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 15:01:32 +0100 Subject: [PATCH 039/253] feat(assist): upgraded sub-dependencies feat(peers): upgraded sub-dependencies feat(sourcemaps-reader): upgraded sub-dependencies --- assist/package-lock.json | 12 ++-- ee/assist/package-lock.json | 34 ++++----- peers/package-lock.json | 109 +++++++++++++++++++++++++---- sourcemap-reader/package-lock.json | 6 +- 4 files changed, 123 insertions(+), 38 deletions(-) diff --git a/assist/package-lock.json b/assist/package-lock.json index aba9e43fe..683472320 100644 --- a/assist/package-lock.json +++ b/assist/package-lock.json @@ -45,9 +45,9 @@ } }, "node_modules/@types/node": { - "version": "18.14.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.1.tgz", - "integrity": "sha512-QH+37Qds3E0eDlReeboBxfHbX9omAcBCXEzswCu6jySP642jiM3cYSIkU/REqwhCUqXdonHFuBfJDiAJxMNhaQ==" + "version": "18.14.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.6.tgz", + "integrity": "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA==" }, "node_modules/accepts": { "version": "1.3.8", @@ -987,9 +987,9 @@ } }, "node_modules/ua-parser-js": { - "version": "1.0.33", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.33.tgz", - "integrity": "sha512-RqshF7TPTE0XLYAqmjlu5cLLuGdKrNu9O1KLA/qp39QtbZwuzwv1dT46DZSopoUMsYgXpB3Cv8a03FI8b74oFQ==", + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.34.tgz", + "integrity": "sha512-K9mwJm/DaB6mRLZfw6q8IMXipcrmuT6yfhYmwhAkuh+81sChuYstYA+znlgaflUPaYUa3odxKPKGw6Vw/lANew==", "funding": [ { "type": "opencollective", diff --git a/ee/assist/package-lock.json b/ee/assist/package-lock.json index 1d74677cf..967198a0e 100644 --- a/ee/assist/package-lock.json +++ b/ee/assist/package-lock.json @@ -38,9 +38,9 @@ } }, "node_modules/@redis/client": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.5.tgz", - "integrity": "sha512-fuMnpDYSjT5JXR9rrCW1YWA4L8N/9/uS4ImT3ZEC/hcaQRI1D/9FvwjriRj1UvepIgzZXthFVKMNRzP/LNL7BQ==", + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.6.tgz", + "integrity": "sha512-dFD1S6je+A47Lj22jN/upVU2fj4huR7S9APd7/ziUXsIXDL+11GPYti4Suv5y8FuXaN+0ZG4JF+y1houEJ7ToA==", "dependencies": { "cluster-key-slot": "1.1.2", "generic-pool": "3.9.0", @@ -67,9 +67,9 @@ } }, "node_modules/@redis/search": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.1.tgz", - "integrity": "sha512-pqCXTc5e7wJJgUuJiC3hBgfoFRoPxYzwn0BEfKgejTM7M/9zP3IpUcqcjgfp8hF+LoV8rHZzcNTz7V+pEIY7LQ==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz", + "integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==", "peerDependencies": { "@redis/client": "^1.0.0" } @@ -117,9 +117,9 @@ } }, "node_modules/@types/node": { - "version": "18.14.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.1.tgz", - "integrity": "sha512-QH+37Qds3E0eDlReeboBxfHbX9omAcBCXEzswCu6jySP642jiM3cYSIkU/REqwhCUqXdonHFuBfJDiAJxMNhaQ==" + "version": "18.14.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.6.tgz", + "integrity": "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA==" }, "node_modules/accepts": { "version": "1.3.8", @@ -878,15 +878,15 @@ } }, "node_modules/redis": { - "version": "4.6.4", - "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.4.tgz", - "integrity": "sha512-wi2tgDdQ+Q8q+PR5FLRx4QvDiWaA+PoJbrzsyFqlClN5R4LplHqN3scs/aGjE//mbz++W19SgxiEnQ27jnCRaA==", + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.5.tgz", + "integrity": "sha512-O0OWA36gDQbswOdUuAhRL6mTZpHFN525HlgZgDaVNgCJIAZR3ya06NTESb0R+TUZ+BFaDpz6NnnVvoMx9meUFg==", "dependencies": { "@redis/bloom": "1.2.0", - "@redis/client": "1.5.5", + "@redis/client": "1.5.6", "@redis/graph": "1.1.0", "@redis/json": "1.0.4", - "@redis/search": "1.1.1", + "@redis/search": "1.1.2", "@redis/time-series": "1.0.4" } }, @@ -1085,9 +1085,9 @@ } }, "node_modules/ua-parser-js": { - "version": "1.0.33", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.33.tgz", - "integrity": "sha512-RqshF7TPTE0XLYAqmjlu5cLLuGdKrNu9O1KLA/qp39QtbZwuzwv1dT46DZSopoUMsYgXpB3Cv8a03FI8b74oFQ==", + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.34.tgz", + "integrity": "sha512-K9mwJm/DaB6mRLZfw6q8IMXipcrmuT6yfhYmwhAkuh+81sChuYstYA+znlgaflUPaYUa3odxKPKGw6Vw/lANew==", "funding": [ { "type": "opencollective", diff --git a/peers/package-lock.json b/peers/package-lock.json index da9b72ca1..5811b59e3 100644 --- a/peers/package-lock.json +++ b/peers/package-lock.json @@ -57,9 +57,9 @@ "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" }, "node_modules/@types/node": { - "version": "18.13.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.13.0.tgz", - "integrity": "sha512-gC3TazRzGoOnoKAhUx+Q0t8S9Tzs74z7m0ipwGpSqQrleP14hKxP4/JUeEQcD3W1/aIpnWl8pHowI7WokuZpXg==" + "version": "18.14.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.6.tgz", + "integrity": "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA==" }, "node_modules/@types/qs": { "version": "6.9.7", @@ -72,9 +72,9 @@ "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" }, "node_modules/@types/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==", + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.1.tgz", + "integrity": "sha512-NUo5XNiAdULrJENtJXZZ3fHtfMolzZwczzBbnAeBbqBwG+LaG6YaJtuwzwGSQZ2wsCrxjEhNNjAkKigy3n8teQ==", "dependencies": { "@types/mime": "*", "@types/node": "*" @@ -243,6 +243,14 @@ "node": ">= 0.10" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -348,6 +356,28 @@ "node": ">= 0.10.0" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/finalhandler": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", @@ -365,6 +395,17 @@ "node": ">= 0.8" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -540,6 +581,41 @@ "node": ">= 0.6" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz", + "integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -581,14 +657,15 @@ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "node_modules/peer": { - "version": "1.0.0-rc.9", - "resolved": "https://registry.npmjs.org/peer/-/peer-1.0.0-rc.9.tgz", - "integrity": "sha512-wjt3fWMKxM/lH/1uD5Qs9qinQ1x/aa9br1eZEQuJ2wuBBQrjAcCT85MUuY9PYcyoW5ymyABsDKC3H/q9KmZ3PA==", + "version": "1.0.0-rc.10", + "resolved": "https://registry.npmjs.org/peer/-/peer-1.0.0-rc.10.tgz", + "integrity": "sha512-S7uMqIAd1tTyvnkj4efdpn8EGc6BM1ONQvLg0vZkrnvA1cTisscBRsx+Jbor6DH68NRLnXgZbiY7/6FDER/GXw==", "dependencies": { "@types/express": "^4.17.3", "@types/ws": "^7.2.3 || ^8.0.0", "cors": "^2.8.5", "express": "^4.17.1", + "node-fetch": "^3.3.0", "ws": "^7.2.3 || ^8.0.0", "yargs": "^17.6.2" }, @@ -819,6 +896,14 @@ "node": ">= 0.8" } }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -864,9 +949,9 @@ } }, "node_modules/yargs": { - "version": "17.6.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", - "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", + "version": "17.7.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.1.tgz", + "integrity": "sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw==", "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", diff --git a/sourcemap-reader/package-lock.json b/sourcemap-reader/package-lock.json index cbaebc3c1..e756a0649 100644 --- a/sourcemap-reader/package-lock.json +++ b/sourcemap-reader/package-lock.json @@ -43,9 +43,9 @@ } }, "node_modules/aws-sdk": { - "version": "2.1314.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1314.0.tgz", - "integrity": "sha512-2jsfvgtOQ6kRflaicn50ndME4YoIaBhlus/dZCExtWNXeu8ePh+eAtflsYs6aqIiRPKhCBLaqClzahWm7hC0XA==", + "version": "2.1329.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1329.0.tgz", + "integrity": "sha512-F5M9x/T+PanPiYGiL95atFE6QiwzJWwgPahaEgUdq+qvVAgruiNy5t6nw2B5tBB/yWDPPavHFip3UsXeO0qU3Q==", "dependencies": { "buffer": "4.9.2", "events": "1.1.1", From ec904892e7e74ef23cfd4fc533a634ec0a28c6fe Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 15:09:57 +0100 Subject: [PATCH 040/253] chore(actions): changed peers/peersEE actions --- .github/workflows/peers-ee.yaml | 67 +++++++++++++++++++++++++++++++-- .github/workflows/peers.yaml | 66 ++++++++++++++++++++++++++++++-- 2 files changed, 127 insertions(+), 6 deletions(-) diff --git a/.github/workflows/peers-ee.yaml b/.github/workflows/peers-ee.yaml index b8d66a082..5981b4631 100644 --- a/.github/workflows/peers-ee.yaml +++ b/.github/workflows/peers-ee.yaml @@ -1,6 +1,11 @@ # This action will push the peers changes to aws on: workflow_dispatch: + inputs: + skip_security_checks: + description: 'Skip Security checks if there is a unfixable vuln or error. Value: true/false' + required: false + default: 'false' push: branches: - dev @@ -36,15 +41,60 @@ jobs: kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret. id: setcontext - - name: Building and Pushing api image + # Caching docker images + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + + + - name: Building and Pushing peers image id: build-image env: DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} ENVIRONMENT: staging run: | - cd peers - PUSH_IMAGE=1 bash build.sh ee + skip_security_checks=${{ github.event.inputs.skip_security_checks }} + cd api + PUSH_IMAGE=0 bash -x ./build.sh ee + [[ "x$skip_security_checks" == "xtrue" ]] || { + curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ + images=("peers") + for image in ${images[*]};do + ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG + done + err_code=$? + [[ $err_code -ne 0 ]] && { + exit $err_code + } + } && { + echo "Skipping Security Checks" + } + images=("peers") + for image in ${images[*]};do + docker push $DOCKER_REPO/$image:$IMAGE_TAG + done + - name: Creating old image input + run: | + # + # Create yaml with existing image tags + # + kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\ + tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt + + echo > /tmp/image_override.yaml + + for line in `cat /tmp/image_tag.txt`; + do + image_array=($(echo "$line" | tr ':' '\n')) + cat <> /tmp/image_override.yaml + ${image_array[0]}: + image: + # We've to strip off the -ee, as helm will append it. + tag: `echo ${image_array[1]} | cut -d '-' -f 1` + EOF + done + - name: Deploy to kubernetes run: | cd scripts/helmcharts/ @@ -72,6 +122,17 @@ jobs: IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} ENVIRONMENT: staging + - name: Alert slack + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_CHANNEL: ee + SLACK_TITLE: "Failed ${{ github.workflow }}" + SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff' + SLACK_WEBHOOK: ${{ secrets.SLACK_WEB_HOOK }} + SLACK_USERNAME: "OR Bot" + SLACK_MESSAGE: 'Build failed :bomb:' + # - name: Debug Job # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/peers.yaml b/.github/workflows/peers.yaml index 65264d7b5..ef564ec65 100644 --- a/.github/workflows/peers.yaml +++ b/.github/workflows/peers.yaml @@ -1,6 +1,11 @@ # This action will push the peers changes to aws on: workflow_dispatch: + inputs: + skip_security_checks: + description: 'Skip Security checks if there is a unfixable vuln or error. Value: true/false' + required: false + default: 'false' push: branches: - dev @@ -35,15 +40,59 @@ jobs: kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret. id: setcontext - - name: Building and Pushing api image + # Caching docker images + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + + + - name: Building and Pushing peers image id: build-image env: DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} ENVIRONMENT: staging run: | + skip_security_checks=${{ github.event.inputs.skip_security_checks }} cd peers - PUSH_IMAGE=1 bash build.sh + PUSH_IMAGE=0 bash -x ./build.sh + [[ "x$skip_security_checks" == "xtrue" ]] || { + curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ + images=("peers") + for image in ${images[*]};do + ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG + done + err_code=$? + [[ $err_code -ne 0 ]] && { + exit $err_code + } + } && { + echo "Skipping Security Checks" + } + images=("peers") + for image in ${images[*]};do + docker push $DOCKER_REPO/$image:$IMAGE_TAG + done + - name: Creating old image input + run: | + # + # Create yaml with existing image tags + # + kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\ + tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt + + echo > /tmp/image_override.yaml + + for line in `cat /tmp/image_tag.txt`; + do + image_array=($(echo "$line" | tr ':' '\n')) + cat <> /tmp/image_override.yaml + ${image_array[0]}: + image: + tag: ${image_array[1]} + EOF + done + - name: Deploy to kubernetes run: | cd scripts/helmcharts/ @@ -70,6 +119,17 @@ jobs: IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} ENVIRONMENT: staging + - name: Alert slack + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_CHANNEL: foss + SLACK_TITLE: "Failed ${{ github.workflow }}" + SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff' + SLACK_WEBHOOK: ${{ secrets.SLACK_WEB_HOOK }} + SLACK_USERNAME: "OR Bot" + SLACK_MESSAGE: 'Build failed :bomb:' + # - name: Debug Job # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 @@ -77,4 +137,4 @@ jobs: # DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} # IMAGE_TAG: ${{ github.sha }} # ENVIRONMENT: staging - # + From 071b2e77f48ba611e819bccb1394642fecf03f79 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 15:16:20 +0100 Subject: [PATCH 041/253] chore(actions): changed peers/peersEE actions --- .github/workflows/peers-ee.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/peers-ee.yaml b/.github/workflows/peers-ee.yaml index 5981b4631..564c5cf6d 100644 --- a/.github/workflows/peers-ee.yaml +++ b/.github/workflows/peers-ee.yaml @@ -55,7 +55,7 @@ jobs: ENVIRONMENT: staging run: | skip_security_checks=${{ github.event.inputs.skip_security_checks }} - cd api + cd peers PUSH_IMAGE=0 bash -x ./build.sh ee [[ "x$skip_security_checks" == "xtrue" ]] || { curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ From 1d06e651eab1a4df4f352ff4053bdb69e6720598 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 16:10:08 +0100 Subject: [PATCH 042/253] feat(chalice): health-check test --- api/app.py | 5 +- api/chalicelib/core/health.py | 148 +++++++++++++++ api/routers/subs/health.py | 15 ++ ee/api/.gitignore | 1 + ee/api/app.py | 5 +- ee/api/chalicelib/core/health.py | 173 ++++++++++++++++++ ee/api/clean-dev.sh | 1 + .../db/init_dbs/clickhouse/1.11.0/1.11.0.sql | 1 + .../clickhouse/create/init_schema.sql | 1 + 9 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 api/chalicelib/core/health.py create mode 100644 api/routers/subs/health.py create mode 100644 ee/api/chalicelib/core/health.py create mode 100644 ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql diff --git a/api/app.py b/api/app.py index 43c3b7945..883cf6704 100644 --- a/api/app.py +++ b/api/app.py @@ -12,7 +12,7 @@ from chalicelib.utils import pg_client from routers import core, core_dynamic from routers.crons import core_crons from routers.crons import core_dynamic_crons -from routers.subs import insights, metrics, v1_api +from routers.subs import insights, metrics, v1_api, health app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) app.add_middleware(GZipMiddleware, minimum_size=1000) @@ -51,6 +51,9 @@ app.include_router(core_dynamic.app_apikey) app.include_router(metrics.app) app.include_router(insights.app) app.include_router(v1_api.app_apikey) +app.include_router(health.public_app) +app.include_router(health.app) +app.include_router(health.app_apikey) loglevel = config("LOGLEVEL", default=logging.INFO) print(f">Loglevel set to: {loglevel}") diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py new file mode 100644 index 000000000..5516d7e4e --- /dev/null +++ b/api/chalicelib/core/health.py @@ -0,0 +1,148 @@ +import requests +from decouple import config + +from chalicelib.utils import pg_client + +if config("LOCAL_DEV", cast=bool, default=False): + HEALTH_ENDPOINTS = { + "alerts": "http://127.0.0.1:8888/metrics", + "assets": "http://127.0.0.1:8888/metrics", + "assist": "http://127.0.0.1:8888/metrics", + "chalice": "http://127.0.0.1:8888/metrics", + "db": "http://127.0.0.1:8888/metrics", + "ender": "http://127.0.0.1:8888/metrics", + "frontend": "http://127.0.0.1:8888/metrics", + "heuristics": "http://127.0.0.1:8888/metrics", + "http": "http://127.0.0.1:8888/metrics", + "ingress-nginx": "http://127.0.0.1:8888/metrics", + "integrations": "http://127.0.0.1:8888/metrics", + "peers": "http://127.0.0.1:8888/metrics", + "quickwit": "http://127.0.0.1:8888/metrics", + "sink": "http://127.0.0.1:8888/metrics", + "sourcemapreader": "http://127.0.0.1:8888/metrics", + "storage": "http://127.0.0.1:8888/metrics", + "utilities": "http://127.0.0.1:8888/metrics" + } + +else: + HEALTH_ENDPOINTS = { + "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/metrics", + "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", + "assist": "http://assist-openreplay.app.svc.cluster.local:8888/metrics", + "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", + "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", + "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", + "frontend": "http://frontend-openreplay.app.svc.cluster.local:8888/metrics", + "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics", + "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", + "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", + "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", + "peers": "http://peers-openreplay.app.svc.cluster.local:8888/metrics", + "quickwit": "http://quickwit-openreplay.app.svc.cluster.local:8888/metrics", + "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", + "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/metrics", + "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", + "utilities": "http://utilities-openreplay.app.svc.cluster.local:8888/metrics", + } + + +def __check_database_pg(): + with pg_client.PostgresClient() as cur: + cur.execute("SHOW server_version;") + server_version = cur.fetchone() + cur.execute("SELECT openreplay_version() AS version;") + schema_version = cur.fetchone() + return { + "health": True, + "details": { + "version": server_version["server_version"], + "schema": schema_version["version"] + } + } + + +def __not_supported(): + return {"errors": ["not supported"]} + + +def check_be_service(service_name): + def fn(): + fail_response = { + "health": False, + "details": { + "errors": ["server health-check failed"] + } + } + try: + results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2) + if results.status_code != 200: + print(f"!! issue with the storage-health code:{results.status_code}") + print(results.text) + fail_response["details"]["errors"].append(results.text) + return fail_response + except requests.exceptions.Timeout: + print(f"!! Timeout getting {service_name}-health") + fail_response["details"]["errors"].append("timeout") + return fail_response + except Exception as e: + print("!! Issue getting storage-health response") + print(str(e)) + print("expected JSON, received:") + try: + print(results.text) + fail_response["details"]["errors"].append(results.text) + except: + print("couldn't get response") + fail_response["details"]["errors"].append(str(e)) + return fail_response + return { + "health": True, + "details": {} + } + + return fn + + +def get_health(): + health_map = { + "databases": { + "postgres": __check_database_pg + }, + "ingestionPipeline": { + "redis": __not_supported + }, + "backendServices": { + "alerts": check_be_service("alerts"), + "assets": check_be_service("assets"), + "assist": check_be_service("assist"), + "chalice": check_be_service("chalice"), + "db": check_be_service("db"), + "ender": check_be_service("ender"), + "frontend": check_be_service("frontend"), + "heuristics": check_be_service("heuristics"), + "http": check_be_service("http"), + "ingress-nginx": check_be_service("ingress-nginx"), + "integrations": check_be_service("integrations"), + "peers": check_be_service("peers"), + "quickwit": check_be_service("quickwit"), + "sink": check_be_service("sink"), + "sourcemapreader": check_be_service("sourcemapreader"), + "storage": check_be_service("storage"), + "utilities": check_be_service("utilities") + }, + # "overall": { + # "health": "na", + # "details": { + # "numberOfEventCaptured": "int", + # "numberOfSessionsCaptured": "int" + # }, + # "labels": { + # "parent": "information" + # } + # }, + # "ssl": True + } + for parent_key in health_map.keys(): + for element_key in health_map[parent_key]: + health_map[parent_key][element_key] = health_map[parent_key][element_key]() + return health_map diff --git a/api/routers/subs/health.py b/api/routers/subs/health.py new file mode 100644 index 000000000..6655f2a20 --- /dev/null +++ b/api/routers/subs/health.py @@ -0,0 +1,15 @@ +from typing import Union + +from fastapi import Body, Depends, Request + +import schemas +from chalicelib.core import health +from or_dependencies import OR_context +from routers.base import get_routers + +public_app, app, app_apikey = get_routers() + + +@public_app.get('/health', tags=["dashboard"]) +def get_global_health(): + return {"data": health.get_health()} diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 79aec2ade..9a9636ee1 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -264,5 +264,6 @@ Pipfile.lock /app_alerts.py /build_alerts.sh /build_crons.sh +/routers/subs/health.py /routers/subs/v1_api.py #exp /chalicelib/core/dashboards.py diff --git a/ee/api/app.py b/ee/api/app.py index a1e203005..407e4aa5b 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -18,7 +18,7 @@ from routers.crons import core_crons from routers.crons import core_dynamic_crons from routers.crons import ee_crons from routers.subs import insights, metrics, v1_api_ee -from routers.subs import v1_api +from routers.subs import v1_api, health app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) app.add_middleware(GZipMiddleware, minimum_size=1000) @@ -68,6 +68,9 @@ app.include_router(metrics.app) app.include_router(insights.app) app.include_router(v1_api.app_apikey) app.include_router(v1_api_ee.app_apikey) +app.include_router(health.public_app) +app.include_router(health.app) +app.include_router(health.app_apikey) loglevel = config("LOGLEVEL", default=logging.INFO) print(f">Loglevel set to: {loglevel}") diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py new file mode 100644 index 000000000..4c27ffe95 --- /dev/null +++ b/ee/api/chalicelib/core/health.py @@ -0,0 +1,173 @@ +import requests +from decouple import config + +from chalicelib.utils import pg_client, ch_client + +if config("LOCAL_DEV", cast=bool, default=False): + HEALTH_ENDPOINTS = { + "alerts": "http://127.0.0.1:8888/metrics", + "assets": "http://127.0.0.1:8888/metrics", + "assist": "http://127.0.0.1:8888/metrics", + "chalice": "http://127.0.0.1:8888/metrics", + "db": "http://127.0.0.1:8888/metrics", + "ender": "http://127.0.0.1:8888/metrics", + "frontend": "http://127.0.0.1:8888/metrics", + "heuristics": "http://127.0.0.1:8888/metrics", + "http": "http://127.0.0.1:8888/metrics", + "ingress-nginx": "http://127.0.0.1:8888/metrics", + "integrations": "http://127.0.0.1:8888/metrics", + "peers": "http://127.0.0.1:8888/metrics", + "quickwit": "http://127.0.0.1:8888/metrics", + "sink": "http://127.0.0.1:8888/metrics", + "sourcemapreader": "http://127.0.0.1:8888/metrics", + "storage": "http://127.0.0.1:8888/metrics", + "utilities": "http://127.0.0.1:8888/metrics" + } + +else: + HEALTH_ENDPOINTS = { + "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/metrics", + "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", + "assist": "http://assist-openreplay.app.svc.cluster.local:8888/metrics", + "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", + "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", + "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", + "frontend": "http://frontend-openreplay.app.svc.cluster.local:8888/metrics", + "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics", + "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", + "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", + "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", + "peers": "http://peers-openreplay.app.svc.cluster.local:8888/metrics", + "quickwit": "http://quickwit-openreplay.app.svc.cluster.local:8888/metrics", + "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", + "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/metrics", + "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", + "utilities": "http://utilities-openreplay.app.svc.cluster.local:8888/metrics", + } + + +def __check_database_pg(): + with pg_client.PostgresClient() as cur: + cur.execute("SHOW server_version;") + server_version = cur.fetchone() + cur.execute("SELECT openreplay_version() AS version;") + schema_version = cur.fetchone() + return { + "health": True, + "details": { + "version": server_version["server_version"], + "schema": schema_version["version"] + } + } + + +def __check_database_ch(): + errors = {} + with ch_client.ClickHouseClient() as ch: + server_version = ch.execute("SELECT version() AS server_version;") + schema_version = ch.execute("""SELECT 1 + FROM system.functions + WHERE name = 'openreplay_version';""") + if len(schema_version) > 0: + schema_version = ch.execute("SELECT openreplay_version()() AS version;") + schema_version = schema_version[0]["version"] + else: + schema_version = "unknown" + errors = {"errors": ["clickhouse schema is outdated"]} + return { + "health": True, + "details": { + "version": server_version[0]["server_version"], + "schema": schema_version, + **errors + } + } + + +def __not_supported(): + return {"errors": ["not supported"]} + + +def check_be_service(service_name): + def fn(): + fail_response = { + "health": False, + "details": { + "errors": ["server health-check failed"] + } + } + try: + results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2) + if results.status_code != 200: + print(f"!! issue with the storage-health code:{results.status_code}") + print(results.text) + fail_response["details"]["errors"].append(results.text) + return fail_response + except requests.exceptions.Timeout: + print(f"!! Timeout getting {service_name}-health") + fail_response["details"]["errors"].append("timeout") + return fail_response + except Exception as e: + print("!! Issue getting storage-health response") + print(str(e)) + print("expected JSON, received:") + try: + print(results.text) + fail_response["details"]["errors"].append(results.text) + except: + print("couldn't get response") + fail_response["details"]["errors"].append(str(e)) + return fail_response + return { + "health": True, + "details": {} + } + + return fn + + +def get_health(): + health_map = { + "databases": { + "postgres": __check_database_pg, + "clickhouse": __check_database_ch + }, + "ingestionPipeline": { + "redis": __not_supported, + "kafka": __not_supported + }, + "backendServices": { + "alerts": check_be_service("alerts"), + "assets": check_be_service("assets"), + "assist": check_be_service("assist"), + "chalice": check_be_service("chalice"), + "db": check_be_service("db"), + "ender": check_be_service("ender"), + "frontend": check_be_service("frontend"), + "heuristics": check_be_service("heuristics"), + "http": check_be_service("http"), + "ingress-nginx": check_be_service("ingress-nginx"), + "integrations": check_be_service("integrations"), + "peers": check_be_service("peers"), + "quickwit": check_be_service("quickwit"), + "sink": check_be_service("sink"), + "sourcemapreader": check_be_service("sourcemapreader"), + "storage": check_be_service("storage"), + "utilities": check_be_service("utilities") + }, + # "overall": { + # "health": "na", + # "details": { + # "numberOfEventCaptured": "int", + # "numberOfSessionsCaptured": "int" + # }, + # "labels": { + # "parent": "information" + # } + # }, + # "ssl": True + } + for parent_key in health_map.keys(): + for element_key in health_map[parent_key]: + health_map[parent_key][element_key] = health_map[parent_key][element_key]() + return health_map diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index acc91e7b7..9241b8e48 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -78,6 +78,7 @@ rm -rf ./Dockerfile_bundle rm -rf ./entrypoint.bundle.sh rm -rf ./chalicelib/core/heatmaps.py rm -rf ./schemas.py +rm -rf ./routers/subs/health.py rm -rf ./routers/subs/v1_api.py #exp rm -rf ./chalicelib/core/custom_metrics.py rm -rf ./chalicelib/core/performance_event.py diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql new file mode 100644 index 000000000..5e9c11242 --- /dev/null +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql @@ -0,0 +1 @@ +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.11.0-ee'; \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 9b2cfbbd1..22d2e804e 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -1,3 +1,4 @@ +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.11.0-ee'; CREATE DATABASE IF NOT EXISTS experimental; CREATE TABLE IF NOT EXISTS experimental.autocomplete From 86af6f37c51dff97aefd8225e16c5a72808b623d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 18:12:36 +0100 Subject: [PATCH 043/253] feat(chalice): health-check test --- api/chalicelib/core/health.py | 85 +++++++++--- api/env.default | 2 +- api/requirements.txt | 2 + ee/api/chalicelib/core/health.py | 223 +++++++++++++++++++------------ ee/api/env.default | 5 +- ee/api/requirements.txt | 3 + 6 files changed, 211 insertions(+), 109 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index 5516d7e4e..95b4abdb9 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -1,3 +1,6 @@ +from urllib.parse import urlparse + +import redis import requests from decouple import config @@ -65,7 +68,21 @@ def __not_supported(): return {"errors": ["not supported"]} -def check_be_service(service_name): +def __always_healthy(): + return { + "health": True, + "details": {} + } + + +def __always_healthy_with_version(): + return { + "health": True, + "details": {"version": config("version_number", default="unknown")} + } + + +def __check_be_service(service_name): def fn(): fail_response = { "health": False, @@ -87,7 +104,6 @@ def check_be_service(service_name): except Exception as e: print("!! Issue getting storage-health response") print(str(e)) - print("expected JSON, received:") try: print(results.text) fail_response["details"]["errors"].append(results.text) @@ -103,32 +119,61 @@ def check_be_service(service_name): return fn +def __check_redis(): + fail_response = { + "health": False, + "details": {"errors": ["server health-check failed"]} + } + if config("REDIS_STRING", default=None) is None: + fail_response["details"]["errors"].append("REDIS_STRING not defined in env-vars") + return fail_response + + try: + u = urlparse(config("REDIS_STRING")) + r = redis.Redis(host=u.hostname, port=u.port, socket_timeout=2) + r.ping() + except Exception as e: + print("!! Issue getting assist-health response") + print(str(e)) + fail_response["details"]["errors"].append(str(e)) + return fail_response + + return { + "health": True, + "details": {"version": r.execute_command('INFO')['redis_version']} + } + + +def __check_assist(): + pass + + def get_health(): health_map = { "databases": { "postgres": __check_database_pg }, "ingestionPipeline": { - "redis": __not_supported + "redis": __check_redis }, "backendServices": { - "alerts": check_be_service("alerts"), - "assets": check_be_service("assets"), - "assist": check_be_service("assist"), - "chalice": check_be_service("chalice"), - "db": check_be_service("db"), - "ender": check_be_service("ender"), - "frontend": check_be_service("frontend"), - "heuristics": check_be_service("heuristics"), - "http": check_be_service("http"), - "ingress-nginx": check_be_service("ingress-nginx"), - "integrations": check_be_service("integrations"), - "peers": check_be_service("peers"), - "quickwit": check_be_service("quickwit"), - "sink": check_be_service("sink"), - "sourcemapreader": check_be_service("sourcemapreader"), - "storage": check_be_service("storage"), - "utilities": check_be_service("utilities") + "alerts": __check_be_service("alerts"), + "assets": __check_be_service("assets"), + "assist": __check_assist, + "chalice": __always_healthy_with_version, + "db": __check_be_service("db"), + "ender": __check_be_service("ender"), + "frontend": __check_be_service("frontend"), + "heuristics": __check_be_service("heuristics"), + "http": __check_be_service("http"), + "ingress-nginx": __always_healthy, + "integrations": __check_be_service("integrations"), + "peers": __check_be_service("peers"), + "quickwit": __check_be_service("quickwit"), + "sink": __check_be_service("sink"), + "sourcemapreader": __check_be_service("sourcemapreader"), + "storage": __check_be_service("storage"), + "utilities": __check_be_service("utilities") }, # "overall": { # "health": "na", diff --git a/api/env.default b/api/env.default index 78acd001c..12feccf1f 100644 --- a/api/env.default +++ b/api/env.default @@ -52,4 +52,4 @@ PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= PYTHONUNBUFFERED=1 -THUMBNAILS_BUCKET=thumbnails \ No newline at end of file +REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt index 0a058a94f..4a8d35090 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -13,3 +13,5 @@ uvicorn[standard]==0.20.0 python-decouple==3.7 pydantic[email]==1.10.4 apscheduler==3.10.0 + +redis==4.5.1 \ No newline at end of file diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 4c27ffe95..4de9844f0 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -1,3 +1,6 @@ +from urllib.parse import urlparse + +import redis import requests from decouple import config @@ -61,6 +64,137 @@ def __check_database_pg(): } +def __not_supported(): + return {"errors": ["not supported"]} + + +def __always_healthy(): + return { + "health": True, + "details": {} + } + + +def __always_healthy_with_version(): + return { + "health": True, + "details": {"version": config("version_number", default="unknown")} + } + + +def __check_be_service(service_name): + def fn(): + fail_response = { + "health": False, + "details": { + "errors": ["server health-check failed"] + } + } + try: + results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2) + if results.status_code != 200: + print(f"!! issue with the storage-health code:{results.status_code}") + print(results.text) + fail_response["details"]["errors"].append(results.text) + return fail_response + except requests.exceptions.Timeout: + print(f"!! Timeout getting {service_name}-health") + fail_response["details"]["errors"].append("timeout") + return fail_response + except Exception as e: + print("!! Issue getting storage-health response") + print(str(e)) + try: + print(results.text) + fail_response["details"]["errors"].append(results.text) + except: + print("couldn't get response") + fail_response["details"]["errors"].append(str(e)) + return fail_response + return { + "health": True, + "details": {} + } + + return fn + + +def __check_redis(): + fail_response = { + "health": False, + "details": {"errors": ["server health-check failed"]} + } + if config("REDIS_STRING", default=None) is None: + fail_response["details"]["errors"].append("REDIS_STRING not defined in env-vars") + return fail_response + + try: + u = urlparse(config("REDIS_STRING")) + r = redis.Redis(host=u.hostname, port=u.port, socket_timeout=2) + r.ping() + except Exception as e: + print("!! Issue getting assist-health response") + print(str(e)) + fail_response["details"]["errors"].append(str(e)) + return fail_response + + return { + "health": True, + "details": {"version": r.execute_command('INFO')['redis_version']} + } + + +def __check_assist(): + pass + + +def get_health(): + health_map = { + "databases": { + "postgres": __check_database_pg, + "clickhouse": __check_database_ch + }, + "ingestionPipeline": { + "redis": __check_redis, + "kafka": __not_supported + }, + "backendServices": { + "alerts": __check_be_service("alerts"), + "assets": __check_be_service("assets"), + "assist": __check_assist, + "chalice": __always_healthy_with_version, + "db": __check_be_service("db"), + "ender": __check_be_service("ender"), + "frontend": __check_be_service("frontend"), + "heuristics": __check_be_service("heuristics"), + "http": __check_be_service("http"), + "ingress-nginx": __always_healthy, + "integrations": __check_be_service("integrations"), + "peers": __check_be_service("peers"), + "quickwit": __check_be_service("quickwit"), + "sink": __check_be_service("sink"), + "sourcemapreader": __check_be_service("sourcemapreader"), + "storage": __check_be_service("storage"), + "utilities": __check_be_service("utilities") + }, + # "overall": { + # "health": "na", + # "details": { + # "numberOfEventCaptured": "int", + # "numberOfSessionsCaptured": "int" + # }, + # "labels": { + # "parent": "information" + # } + # }, + # "ssl": True + } + for parent_key in health_map.keys(): + for element_key in health_map[parent_key]: + health_map[parent_key][element_key] = health_map[parent_key][element_key]() + return health_map + + def __check_database_ch(): errors = {} with ch_client.ClickHouseClient() as ch: @@ -84,90 +218,5 @@ def __check_database_ch(): } -def __not_supported(): - return {"errors": ["not supported"]} - - -def check_be_service(service_name): - def fn(): - fail_response = { - "health": False, - "details": { - "errors": ["server health-check failed"] - } - } - try: - results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2) - if results.status_code != 200: - print(f"!! issue with the storage-health code:{results.status_code}") - print(results.text) - fail_response["details"]["errors"].append(results.text) - return fail_response - except requests.exceptions.Timeout: - print(f"!! Timeout getting {service_name}-health") - fail_response["details"]["errors"].append("timeout") - return fail_response - except Exception as e: - print("!! Issue getting storage-health response") - print(str(e)) - print("expected JSON, received:") - try: - print(results.text) - fail_response["details"]["errors"].append(results.text) - except: - print("couldn't get response") - fail_response["details"]["errors"].append(str(e)) - return fail_response - return { - "health": True, - "details": {} - } - - return fn - - -def get_health(): - health_map = { - "databases": { - "postgres": __check_database_pg, - "clickhouse": __check_database_ch - }, - "ingestionPipeline": { - "redis": __not_supported, - "kafka": __not_supported - }, - "backendServices": { - "alerts": check_be_service("alerts"), - "assets": check_be_service("assets"), - "assist": check_be_service("assist"), - "chalice": check_be_service("chalice"), - "db": check_be_service("db"), - "ender": check_be_service("ender"), - "frontend": check_be_service("frontend"), - "heuristics": check_be_service("heuristics"), - "http": check_be_service("http"), - "ingress-nginx": check_be_service("ingress-nginx"), - "integrations": check_be_service("integrations"), - "peers": check_be_service("peers"), - "quickwit": check_be_service("quickwit"), - "sink": check_be_service("sink"), - "sourcemapreader": check_be_service("sourcemapreader"), - "storage": check_be_service("storage"), - "utilities": check_be_service("utilities") - }, - # "overall": { - # "health": "na", - # "details": { - # "numberOfEventCaptured": "int", - # "numberOfSessionsCaptured": "int" - # }, - # "labels": { - # "parent": "information" - # } - # }, - # "ssl": True - } - for parent_key in health_map.keys(): - for element_key in health_map[parent_key]: - health_map[parent_key][element_key] = health_map[parent_key][element_key]() - return health_map +def __check_kafka(): + pass diff --git a/ee/api/env.default b/ee/api/env.default index cdbc3d256..df353d071 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -70,4 +70,7 @@ SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 -ASSIST_JWT_SECRET= \ No newline at end of file +ASSIST_JWT_SECRET= +REDIS_STRING=redis://redis-master.db.svc.cluster.local:6379 +KAFKA_SERVERS=kafka.db.svc.cluster.local:9092 +KAFKA_USE_SSL=false \ No newline at end of file diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index c8b76e700..3d97c63e6 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -17,3 +17,6 @@ apscheduler==3.10.0 clickhouse-driver==0.2.5 python3-saml==1.15.0 python-multipart==0.0.5 + +redis==4.5.1 +kafka-python==2.0.2 \ No newline at end of file From 687dfc1857779d618825a39358568131a1890c5a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 18:16:10 +0100 Subject: [PATCH 044/253] chore(actions): ignore chalice vulnerability check --- .github/workflows/api-ee.yaml | 2 +- .github/workflows/api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/api-ee.yaml b/.github/workflows/api-ee.yaml index f9a1730f1..8feec182c 100644 --- a/.github/workflows/api-ee.yaml +++ b/.github/workflows/api-ee.yaml @@ -5,7 +5,7 @@ on: skip_security_checks: description: 'Skip Security checks if there is a unfixable vuln or error. Value: true/false' required: false - default: 'false' + default: 'true' push: branches: - dev diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 8e2f7fa7b..9537d97b6 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -5,7 +5,7 @@ on: skip_security_checks: description: 'Skip Security checks if there is a unfixable vuln or error. Value: true/false' required: false - default: 'false' + default: 'true' push: branches: - dev From d86b71f66e68f083a25279a2e7ed7f892332838d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 7 Mar 2023 18:36:48 +0100 Subject: [PATCH 045/253] feat(chalice): health-check test --- api/chalicelib/core/health.py | 2 +- ee/api/chalicelib/core/health.py | 32 +++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index 95b4abdb9..071cf7f9c 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -133,7 +133,7 @@ def __check_redis(): r = redis.Redis(host=u.hostname, port=u.port, socket_timeout=2) r.ping() except Exception as e: - print("!! Issue getting assist-health response") + print("!! Issue getting redis-health response") print(str(e)) fail_response["details"]["errors"].append(str(e)) return fail_response diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 4de9844f0..6139992a0 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -3,6 +3,7 @@ from urllib.parse import urlparse import redis import requests from decouple import config +import kafka from chalicelib.utils import pg_client, ch_client @@ -133,7 +134,7 @@ def __check_redis(): r = redis.Redis(host=u.hostname, port=u.port, socket_timeout=2) r.ping() except Exception as e: - print("!! Issue getting assist-health response") + print("!! Issue getting redis-health response") print(str(e)) fail_response["details"]["errors"].append(str(e)) return fail_response @@ -156,7 +157,7 @@ def get_health(): }, "ingestionPipeline": { "redis": __check_redis, - "kafka": __not_supported + "kafka": __check_kafka }, "backendServices": { "alerts": __check_be_service("alerts"), @@ -219,4 +220,29 @@ def __check_database_ch(): def __check_kafka(): - pass + fail_response = { + "health": False, + "details": {"errors": ["server health-check failed"]} + } + if config("KAFKA_SERVERS", default=None) is None: + fail_response["details"]["errors"].append("KAFKA_SERVERS not defined in env-vars") + return fail_response + + try: + # consumer = kafka.KafkaConsumer(group_id='test', bootstrap_servers=[config("KAFKA_SERVERS")]) + # topics = consumer.topics() + # + # if not topics: + # raise RuntimeError() + client =kafka.KafkaClient(bootstrap_servers=[config("KAFKA_SERVERS")]) + except Exception as e: + print("!! Issue getting kafka-health response") + print(str(e)) + fail_response["details"]["errors"].append(str(e)) + return fail_response + + return { + "health": True, + "details": {"version": r.execute_command('INFO')['redis_version']} + } + From dbc031f4a8737971457555d86b47fe9f735b1edb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 8 Mar 2023 11:47:58 +0100 Subject: [PATCH 046/253] chore(actions): changed actions --- .github/workflows/api-ee.yaml | 2 +- .github/workflows/api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/api-ee.yaml b/.github/workflows/api-ee.yaml index 8feec182c..f9a1730f1 100644 --- a/.github/workflows/api-ee.yaml +++ b/.github/workflows/api-ee.yaml @@ -5,7 +5,7 @@ on: skip_security_checks: description: 'Skip Security checks if there is a unfixable vuln or error. Value: true/false' required: false - default: 'true' + default: 'false' push: branches: - dev diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 9537d97b6..8e2f7fa7b 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -5,7 +5,7 @@ on: skip_security_checks: description: 'Skip Security checks if there is a unfixable vuln or error. Value: true/false' required: false - default: 'true' + default: 'false' push: branches: - dev From 7a1979dfd2db6f413e8c4604a1be15c60afd948c Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Mar 2023 12:49:32 +0100 Subject: [PATCH 047/253] fix(player): fix clickmap session size? --- .../app/components/Session/Player/ClickMapRenderer/Renderer.tsx | 1 - frontend/app/player/web/Screen/Screen.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/app/components/Session/Player/ClickMapRenderer/Renderer.tsx b/frontend/app/components/Session/Player/ClickMapRenderer/Renderer.tsx index 8a51717e5..29da5800c 100644 --- a/frontend/app/components/Session/Player/ClickMapRenderer/Renderer.tsx +++ b/frontend/app/components/Session/Player/ClickMapRenderer/Renderer.tsx @@ -16,7 +16,6 @@ function Player() { } }, []); - if (!playerContext.player) return null; return ( diff --git a/frontend/app/player/web/Screen/Screen.ts b/frontend/app/player/web/Screen/Screen.ts index cca56d402..377f704ed 100644 --- a/frontend/app/player/web/Screen/Screen.ts +++ b/frontend/app/player/web/Screen/Screen.ts @@ -218,7 +218,7 @@ export default class Screen { case ScaleMode.AdjustParentHeight: this.scaleRatio = offsetWidth / width translate = "translate(-50%, 0)" - posStyles = { top: 0, height: this.document!.documentElement.getBoundingClientRect().height + 'px', } + posStyles = { top: 0, height: height + 'px', } break; } From c3a4a6012db7995a18f682a17c1ebf67cb11c0ad Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Mar 2023 15:30:22 +0100 Subject: [PATCH 048/253] fix(player): fix clickmap url filtering, fix clickmap highliter --- .../CustomMetricsWidgets/ClickMapCard/ClickMapCard.tsx | 8 +++----- frontend/app/player/web/addons/TargetMarker.ts | 2 +- frontend/app/player/web/addons/clickmapStyles.ts | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/frontend/app/components/Dashboard/Widgets/CustomMetricsWidgets/ClickMapCard/ClickMapCard.tsx b/frontend/app/components/Dashboard/Widgets/CustomMetricsWidgets/ClickMapCard/ClickMapCard.tsx index e8ff709c9..c77dbcd95 100644 --- a/frontend/app/components/Dashboard/Widgets/CustomMetricsWidgets/ClickMapCard/ClickMapCard.tsx +++ b/frontend/app/components/Dashboard/Widgets/CustomMetricsWidgets/ClickMapCard/ClickMapCard.tsx @@ -20,6 +20,7 @@ function ClickMapCard({ const onMarkerClick = (s: string, innerText: string) => { metricStore.changeClickMapSearch(s, innerText) } + const mapUrl = metricStore.instance.series[0].filter.filters[0].value[0] React.useEffect(() => { return () => clearCurrentSession() @@ -32,12 +33,10 @@ function ClickMapCard({ React.useEffect(() => { if (visitedEvents.length) { - const urlOptions = visitedEvents.map(({ url, host }: any) => ({ label: url, value: url, host })) - const url = insightsFilters.url ? insightsFilters.url : host + urlOptions[0].value; const rangeValue = dashboardStore.drillDownPeriod.rangeValue const startDate = dashboardStore.drillDownPeriod.start const endDate = dashboardStore.drillDownPeriod.end - fetchInsights({ ...insightsFilters, url, startDate, endDate, rangeValue, clickRage: metricStore.clickMapFilter }) + fetchInsights({ ...insightsFilters, url: mapUrl || '/', startDate, endDate, rangeValue, clickRage: metricStore.clickMapFilter }) } }, [visitedEvents, metricStore.clickMapFilter]) @@ -62,9 +61,8 @@ function ClickMapCard({ return
Loading session
} - const searchUrl = metricStore.instance.series[0].filter.filters[0].value[0] const jumpToEvent = metricStore.instance.data.events.find((evt: Record) => { - if (searchUrl) return evt.path.includes(searchUrl) + if (mapUrl) return evt.path.includes(mapUrl) return evt }) || { timestamp: metricStore.instance.data.startTs } diff --git a/frontend/app/player/web/addons/TargetMarker.ts b/frontend/app/player/web/addons/TargetMarker.ts index 6629ceaec..452ddd00f 100644 --- a/frontend/app/player/web/addons/TargetMarker.ts +++ b/frontend/app/player/web/addons/TargetMarker.ts @@ -240,7 +240,7 @@ export default class TargetMarker { }) } - Object.assign(smallClicksBubble.style, clickmapStyles.clicks({ top, height, isRage: s.clickRage })) + Object.assign(smallClicksBubble.style, clickmapStyles.clicks({ top, height, isRage: s.clickRage, left })) border.appendChild(smallClicksBubble) overlay.appendChild(bubbleContainer) diff --git a/frontend/app/player/web/addons/clickmapStyles.ts b/frontend/app/player/web/addons/clickmapStyles.ts index 0ab795ea0..f0dc65a9c 100644 --- a/frontend/app/player/web/addons/clickmapStyles.ts +++ b/frontend/app/player/web/addons/clickmapStyles.ts @@ -16,7 +16,7 @@ export const clickmapStyles = { }, bubbleContainer: ({ top, left, height }: { top: number; left: number, height: number }) => ({ position: 'absolute', - top: top > 20 ? top + 'px' : height + 2 + 'px', + top: top > 75 ? top + 'px' : height+75 + 'px', width: '250px', left: `${left}px`, padding: '10px', @@ -51,9 +51,9 @@ export const clickmapStyles = { position: 'absolute', zIndex, }), - clicks: ({ top, height, isRage }: { top: number; height: number, isRage?: boolean }) => ({ + clicks: ({ top, height, isRage, left }: { top: number; height: number, isRage?: boolean, left: number }) => ({ top: top > 20 ? 0 : `${height}px`, - left: 0, + left: left < 5 ? '100%' : 0, position: 'absolute', borderRadius: '999px', padding: '6px', From 986b5a8802ec953f9035b16e87b47425bb6249f9 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Mar 2023 16:13:56 +0100 Subject: [PATCH 049/253] fix(player): fix clickmap selectors --- frontend/app/player/web/Screen/Screen.ts | 3 +-- tracker/tracker/CHANGELOG.md | 1 + tracker/tracker/package.json | 3 ++- tracker/tracker/src/main/modules/mouse.ts | 27 ++++++++--------------- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/frontend/app/player/web/Screen/Screen.ts b/frontend/app/player/web/Screen/Screen.ts index 377f704ed..f27a251f1 100644 --- a/frontend/app/player/web/Screen/Screen.ts +++ b/frontend/app/player/web/Screen/Screen.ts @@ -3,7 +3,6 @@ import Cursor from './Cursor' import type { Point, Dimensions } from './types'; - export type State = Dimensions export const INITIAL_STATE: State = { @@ -182,7 +181,7 @@ export default class Screen { getElementBySelector(selector: string) { if (!selector) return null; try { - const safeSelector = selector.replace(/:/g, '\\\\3A ').replace(/\//g, '\\/'); + const safeSelector = selector.replace(/\//g, '\\/'); return this.document?.querySelector(safeSelector) || null; } catch (e) { console.error("Can not select element. ", e) diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index 329f3aa59..6a8e25690 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,6 +1,7 @@ ## 5.0.1 - Default text input mode is now Obscured +- Use `@medv/finder` instead of our own implementation of `getSelector` for better clickmaps experience ## 5.0.0 diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index c45c15e4a..326b45575 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "5.0.0", + "version": "5.0.1-beta.1", "keywords": [ "logging", "replay" @@ -47,6 +47,7 @@ "typescript": "^4.9.4" }, "dependencies": { + "@medv/finder": "^3.0.0", "error-stack-parser": "^2.0.6" }, "engines": { diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index b00d6d304..2b32a99d7 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -3,26 +3,17 @@ import { hasTag, isSVGElement, isDocument } from '../app/guards.js' import { normSpaces, hasOpenreplayAttribute, getLabelAttribute } from '../utils.js' import { MouseMove, MouseClick } from '../app/messages.gen.js' import { getInputLabel } from './input.js' +import { finder } from '@medv/finder' function _getSelector(target: Element, document: Document): string { - let el: Element | null = target - let selector: string | null = null - do { - if (el.id) { - return `#${el.id}` + (selector ? ` > ${selector}` : '') - } - selector = - el.className - .split(' ') - .map((cn) => cn.trim()) - .filter((cn) => cn !== '') - .reduce((sel, cn) => `${sel}.${cn}`, el.tagName.toLowerCase()) + - (selector ? ` > ${selector}` : '') - if (el === document.body) { - return selector - } - el = el.parentElement - } while (el !== document.body && el !== null) + const selector = finder(target, { + root: document.body, + seedMinLength: 3, + optimizedMinLength: 2, + threshold: 1000, + maxNumberOfTries: 10_000, + }) + return selector } From 0d857822e80f49b9fb92a947dc1fffca113351c3 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 8 Mar 2023 16:40:39 +0100 Subject: [PATCH 050/253] chore(helm): disable service monitor for some services --- scripts/helmcharts/openreplay/charts/alerts/values.yaml | 2 +- scripts/helmcharts/openreplay/charts/chalice/values.yaml | 2 +- scripts/helmcharts/openreplay/charts/peers/values.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/alerts/values.yaml b/scripts/helmcharts/openreplay/charts/alerts/values.yaml index a54418a9f..ca76602c1 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/values.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/values.yaml @@ -51,7 +51,7 @@ service: metrics: 8888 serviceMonitor: - enabled: true + enabled: false additionalLabels: release: observability scrapeConfigs: diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 3269aa503..1a1d496ed 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -51,7 +51,7 @@ service: metrics: 8888 serviceMonitor: - enabled: true + enabled: false additionalLabels: release: observability scrapeConfigs: diff --git a/scripts/helmcharts/openreplay/charts/peers/values.yaml b/scripts/helmcharts/openreplay/charts/peers/values.yaml index 57fc30bde..0bc4b6b14 100644 --- a/scripts/helmcharts/openreplay/charts/peers/values.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/values.yaml @@ -49,7 +49,7 @@ podSecurityContext: # port: 9000 serviceMonitor: - enabled: true + enabled: false additionalLabels: release: observability scrapeConfigs: From a1b3eb57ec34c317ac68d1da585f6605eecca133 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Mar 2023 16:41:49 +0100 Subject: [PATCH 051/253] fix(player): track tr th clicks for map --- third-party.md | 2 +- tracker/tracker/package.json | 2 +- tracker/tracker/src/main/modules/mouse.ts | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/third-party.md b/third-party.md index e0b68d9f6..0cfe2cac2 100644 --- a/third-party.md +++ b/third-party.md @@ -115,4 +115,4 @@ Below is the list of dependencies used in OpenReplay software. Licenses may chan | yq | MIT | Infrastructure | | html2canvas | MIT | JavaScript | | eget | MIT | Infrastructure | - +| @medv/finder | MIT | JavaScript | diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 326b45575..a67073cc9 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "5.0.1-beta.1", + "version": "5.0.1-beta.2", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 2b32a99d7..155a14a8d 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -24,6 +24,8 @@ function isClickable(element: Element): boolean { tag === 'A' || tag === 'LI' || tag === 'SELECT' || + tag === 'TR' || + tag === 'TH' || (element as HTMLElement).onclick != null || element.getAttribute('role') === 'button' ) From c6aac11cbf41a94e30cec2ea3c070c8668890b5e Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 9 Mar 2023 09:54:12 +0100 Subject: [PATCH 052/253] Heuristics refactoring (#987) * feat(backend): refactored heuristics service * feat(backend): refactored db service (moved several events to heuristics) --- backend/cmd/assets/main.go | 4 - backend/cmd/db/main.go | 157 +- backend/cmd/ender/main.go | 10 +- backend/cmd/heuristics/main.go | 79 +- backend/cmd/http/main.go | 4 - backend/cmd/integrations/main.go | 4 - backend/cmd/sink/main.go | 6 +- backend/cmd/storage/main.go | 4 - backend/internal/config/heuristics/config.go | 4 + backend/internal/db/datasaver/messages.go | 74 - backend/internal/db/datasaver/methods.go | 19 + backend/internal/db/datasaver/saver.go | 122 +- backend/internal/db/datasaver/stats.go | 29 - backend/internal/db/service.go | 56 + backend/internal/heuristics/service.go | 64 + backend/internal/service/service.go | 5 + backend/internal/sessionender/ender.go | 4 +- backend/pkg/db/cache/messages-common.go | 6 +- backend/pkg/db/cache/messages-ios.go | 30 +- backend/pkg/db/cache/messages-web.go | 30 +- backend/pkg/db/clickhouse/connector.go | 24 + backend/pkg/db/postgres/batches.go | 2 - backend/pkg/db/postgres/bulks.go | 3 - backend/pkg/db/postgres/connector.go | 2 +- backend/pkg/db/postgres/messages-ios.go | 24 +- backend/pkg/db/postgres/messages-web-stats.go | 22 +- backend/pkg/db/postgres/messages-web.go | 7 +- backend/pkg/db/types/error-event.go | 12 + backend/pkg/handlers/custom/eventMapper.go | 82 - .../pkg/handlers/custom/inputEventBuilder.go | 8 +- .../pkg/handlers/custom/pageEventBuilder.go | 10 +- backend/pkg/handlers/ios/clickRage.go | 2 +- backend/pkg/handlers/messageProcessor.go | 2 +- backend/pkg/handlers/web/clickRage.go | 60 +- backend/pkg/handlers/web/cpuIssue.go | 82 +- backend/pkg/handlers/web/deadClick.go | 63 +- backend/pkg/handlers/web/domDrop.go | 55 - backend/pkg/handlers/web/memoryIssue.go | 11 +- backend/pkg/handlers/web/networkIssue.go | 11 +- .../pkg/handlers/web/performanceAggregator.go | 12 +- backend/pkg/messages/filters.go | 4 +- backend/pkg/messages/iterator-ender.go | 10 +- backend/pkg/messages/iterator-sink.go | 10 +- backend/pkg/messages/iterator.go | 19 +- backend/pkg/messages/message.go | 12 +- backend/pkg/messages/messages.go | 803 +++--- backend/pkg/messages/raw.go | 14 + backend/pkg/messages/read-message.go | 2302 ++++++++--------- backend/pkg/sessions/builder.go | 41 +- backend/pkg/sessions/builderMap.go | 124 +- backend/pkg/terminator/terminator.go | 22 + backend/pkg/url/url.go | 35 +- ee/backend/internal/db/datasaver/fts.go | 106 +- ee/backend/internal/db/datasaver/messages.go | 114 - ee/backend/internal/db/datasaver/methods.go | 83 + ee/backend/internal/db/datasaver/saver.go | 24 - ee/backend/internal/db/datasaver/stats.go | 56 - ee/backend/pkg/db/clickhouse/connector.go | 27 +- ee/connectors/msgcodec/messages.py | 36 - ee/connectors/msgcodec/msgcodec.py | 30 - mobs/messages.rb | 29 - 61 files changed, 2342 insertions(+), 2764 deletions(-) delete mode 100644 backend/internal/db/datasaver/messages.go create mode 100644 backend/internal/db/datasaver/methods.go delete mode 100644 backend/internal/db/datasaver/stats.go create mode 100644 backend/internal/db/service.go create mode 100644 backend/internal/heuristics/service.go create mode 100644 backend/internal/service/service.go create mode 100644 backend/pkg/db/clickhouse/connector.go delete mode 100644 backend/pkg/handlers/custom/eventMapper.go delete mode 100644 backend/pkg/handlers/web/domDrop.go create mode 100644 backend/pkg/terminator/terminator.go delete mode 100644 ee/backend/internal/db/datasaver/messages.go create mode 100644 ee/backend/internal/db/datasaver/methods.go delete mode 100644 ee/backend/internal/db/datasaver/saver.go delete mode 100644 ee/backend/internal/db/datasaver/stats.go diff --git a/backend/cmd/assets/main.go b/backend/cmd/assets/main.go index b05ecbe52..16eac7cb5 100644 --- a/backend/cmd/assets/main.go +++ b/backend/cmd/assets/main.go @@ -13,7 +13,6 @@ import ( "openreplay/backend/pkg/messages" "openreplay/backend/pkg/metrics" assetsMetrics "openreplay/backend/pkg/metrics/assets" - "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue" ) @@ -24,9 +23,6 @@ func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := config.New() - if cfg.UseProfiler { - pprof.StartProfilingServer() - } cacher := cacher.NewCacher(cfg) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 84b0d81ed..ae1228cc3 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -1,174 +1,59 @@ package main import ( - "errors" "log" - "os" - "os/signal" - "syscall" - "time" - "openreplay/backend/internal/config/db" + config "openreplay/backend/internal/config/db" + "openreplay/backend/internal/db" "openreplay/backend/internal/db/datasaver" "openreplay/backend/pkg/db/cache" "openreplay/backend/pkg/db/postgres" - types2 "openreplay/backend/pkg/db/types" - "openreplay/backend/pkg/handlers" - custom2 "openreplay/backend/pkg/handlers/custom" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/metrics" databaseMetrics "openreplay/backend/pkg/metrics/database" - "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue" - "openreplay/backend/pkg/sessions" + "openreplay/backend/pkg/terminator" ) func main() { + log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) + m := metrics.New() m.Register(databaseMetrics.List()) - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - - cfg := db.New() - if cfg.UseProfiler { - pprof.StartProfilingServer() - } + cfg := config.New() // Init database pg := cache.NewPGCache( postgres.NewConn(cfg.Postgres.String(), cfg.BatchQueueLimit, cfg.BatchSizeLimit), cfg.ProjectExpirationTimeoutMs) defer pg.Close() - // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. - handlersFabric := func() []handlers.MessageProcessor { - return []handlers.MessageProcessor{ - &custom2.EventMapper{}, - custom2.NewInputEventBuilder(), - custom2.NewPageEventBuilder(), - } - } - - // Create handler's aggregator - builderMap := sessions.NewBuilderMap(handlersFabric) - - // Init modules - saver := datasaver.New(pg, cfg) - saver.InitStats() + // Init data saver + saver := datasaver.New(cfg, pg) + // Message filter msgFilter := []int{messages.MsgMetadata, messages.MsgIssueEvent, messages.MsgSessionStart, messages.MsgSessionEnd, - messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgClickEvent, - messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr, - messages.MsgJSException, messages.MsgResourceTiming, - messages.MsgCustomEvent, messages.MsgCustomIssue, messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, - messages.MsgStateAction, messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument, - messages.MsgMouseClick, messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming} - - // Handler logic - msgHandler := func(msg messages.Message) { - // Just save session data into db without additional checks - if err := saver.InsertMessage(msg); err != nil { - if !postgres.IsPkeyViolation(err) { - log.Printf("Message Insertion Error %v, SessionID: %v, Message: %v", err, msg.SessionID(), msg) - } - return - } - - var ( - session *types2.Session - err error - ) - if msg.TypeID() == messages.MsgSessionEnd { - session, err = pg.GetSession(msg.SessionID()) - } else { - session, err = pg.Cache.GetSession(msg.SessionID()) - } - if session == nil { - if err != nil && !errors.Is(err, cache.NilSessionInCacheError) { - log.Printf("Error on session retrieving from cache: %v, SessionID: %v, Message: %v", err, msg.SessionID(), msg) - } - return - } - - // Save statistics to db - err = saver.InsertStats(session, msg) - if err != nil { - log.Printf("Stats Insertion Error %v; Session: %v, Message: %v", err, session, msg) - } - - // Handle heuristics and save to temporary queue in memory - builderMap.HandleMessage(msg) - - // Process saved heuristics messages as usual messages above in the code - builderMap.IterateSessionReadyMessages(msg.SessionID(), func(msg messages.Message) { - if err := saver.InsertMessage(msg); err != nil { - if !postgres.IsPkeyViolation(err) { - log.Printf("Message Insertion Error %v; Session: %v, Message %v", err, session, msg) - } - return - } - - if err := saver.InsertStats(session, msg); err != nil { - log.Printf("Stats Insertion Error %v; Session: %v, Message %v", err, session, msg) - } - }) - } + messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr, + messages.MsgJSException, messages.MsgResourceTiming, messages.MsgCustomEvent, messages.MsgCustomIssue, + messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, + messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument, messages.MsgMouseClick, + messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming, + messages.MsgInputEvent, messages.MsgPageEvent} // Init consumer consumer := queue.NewConsumer( cfg.GroupDB, []string{ - cfg.TopicRawWeb, // from tracker - cfg.TopicAnalytics, // from heuristics + cfg.TopicRawWeb, + cfg.TopicAnalytics, }, - messages.NewMessageIterator(msgHandler, msgFilter, true), + messages.NewMessageIterator(saver.Handle, msgFilter, true), false, cfg.MessageSizeLimit, ) + // Run service and wait for TERM signal + service := db.New(cfg, consumer, saver) log.Printf("Db service started\n") - - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - - commitTick := time.Tick(cfg.CommitBatchTimeout) - - // Send collected batches to db - commitDBUpdates := func() { - // Commit collected batches and bulks of information to PG - pg.Commit() - // Commit collected batches of information to CH - if err := saver.CommitStats(); err != nil { - log.Printf("Error on stats commit: %v", err) - } - // Commit current position in queue - if err := consumer.Commit(); err != nil { - log.Printf("Error on consumer commit: %v", err) - } - } - - for { - select { - case sig := <-sigchan: - log.Printf("Caught signal %s: terminating\n", sig.String()) - commitDBUpdates() - if err := pg.Close(); err != nil { - log.Printf("db.Close error: %s", err) - } - if err := saver.Close(); err != nil { - log.Printf("saver.Close error: %s", err) - } - consumer.Close() - os.Exit(0) - case <-commitTick: - commitDBUpdates() - builderMap.ClearOldSessions() - case msg := <-consumer.Rebalanced(): - log.Println(msg) - default: - // Handle new message from queue - if err := consumer.ConsumeNext(); err != nil { - log.Fatalf("Error on consumption: %v", err) - } - } - } + terminator.Wait(service) } diff --git a/backend/cmd/ender/main.go b/backend/cmd/ender/main.go index da7ca9b89..84d816a33 100644 --- a/backend/cmd/ender/main.go +++ b/backend/cmd/ender/main.go @@ -18,7 +18,6 @@ import ( "openreplay/backend/pkg/metrics" databaseMetrics "openreplay/backend/pkg/metrics/database" enderMetrics "openreplay/backend/pkg/metrics/ender" - "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue" ) @@ -30,9 +29,6 @@ func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := ender.New() - if cfg.UseProfiler { - pprof.StartProfilingServer() - } pg := cache.NewPGCache(postgres.NewConn(cfg.Postgres.String(), 0, 0), cfg.ProjectExpirationTimeoutMs) defer pg.Close() @@ -72,12 +68,12 @@ func main() { consumer.Close() os.Exit(0) case <-tick: - failedSessionEnds := make(map[uint64]int64) + failedSessionEnds := make(map[uint64]uint64) duplicatedSessionEnds := make(map[uint64]uint64) // Find ended sessions and send notification to other services - sessions.HandleEndedSessions(func(sessionID uint64, timestamp int64) bool { - msg := &messages.SessionEnd{Timestamp: uint64(timestamp)} + sessions.HandleEndedSessions(func(sessionID uint64, timestamp uint64) bool { + msg := &messages.SessionEnd{Timestamp: timestamp} currDuration, err := pg.GetSessionDuration(sessionID) if err != nil { log.Printf("getSessionDuration failed, sessID: %d, err: %s", sessionID, err) diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index ac55b83bc..073f48611 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -2,90 +2,49 @@ package main import ( "log" - "openreplay/backend/pkg/pprof" - "os" - "os/signal" - "syscall" - "time" - - "openreplay/backend/internal/config/heuristics" + config "openreplay/backend/internal/config/heuristics" + "openreplay/backend/internal/heuristics" "openreplay/backend/pkg/handlers" - web2 "openreplay/backend/pkg/handlers/web" - "openreplay/backend/pkg/intervals" + "openreplay/backend/pkg/handlers/custom" + "openreplay/backend/pkg/handlers/web" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/sessions" + "openreplay/backend/pkg/terminator" ) func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - - cfg := heuristics.New() - if cfg.UseProfiler { - pprof.StartProfilingServer() - } + cfg := config.New() // HandlersFabric returns the list of message handlers we want to be applied to each incoming message. handlersFabric := func() []handlers.MessageProcessor { return []handlers.MessageProcessor{ - // web handlers - &web2.ClickRageDetector{}, - &web2.CpuIssueDetector{}, - &web2.DeadClickDetector{}, - &web2.MemoryIssueDetector{}, - &web2.NetworkIssueDetector{}, - &web2.PerformanceAggregator{}, - // Other handlers (you can add your custom handlers here) - //&custom.CustomHandler{}, + custom.NewInputEventBuilder(), + custom.NewPageEventBuilder(), + web.NewDeadClickDetector(), + &web.ClickRageDetector{}, + &web.CpuIssueDetector{}, + &web.MemoryIssueDetector{}, + &web.NetworkIssueDetector{}, + &web.PerformanceAggregator{}, } } - // Create handler's aggregator - builderMap := sessions.NewBuilderMap(handlersFabric) - - // Init producer and consumer for data bus + eventBuilder := sessions.NewBuilderMap(handlersFabric) producer := queue.NewProducer(cfg.MessageSizeLimit, true) - - msgHandler := func(msg messages.Message) { - builderMap.HandleMessage(msg) - } - consumer := queue.NewConsumer( cfg.GroupHeuristics, []string{ cfg.TopicRawWeb, }, - messages.NewMessageIterator(msgHandler, nil, true), + messages.NewMessageIterator(eventBuilder.HandleMessage, nil, true), false, cfg.MessageSizeLimit, ) + // Run service and wait for TERM signal + service := heuristics.New(cfg, producer, consumer, eventBuilder) log.Printf("Heuristics service started\n") - - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - - tick := time.Tick(intervals.EVENTS_COMMIT_INTERVAL * time.Millisecond) - for { - select { - case sig := <-sigchan: - log.Printf("Caught signal %v: terminating\n", sig) - producer.Close(cfg.ProducerTimeout) - consumer.Commit() - consumer.Close() - os.Exit(0) - case <-tick: - builderMap.IterateReadyMessages(func(sessionID uint64, readyMsg messages.Message) { - producer.Produce(cfg.TopicAnalytics, sessionID, readyMsg.Encode()) - }) - producer.Flush(cfg.ProducerTimeout) - consumer.Commit() - case msg := <-consumer.Rebalanced(): - log.Println(msg) - default: - if err := consumer.ConsumeNext(); err != nil { - log.Fatalf("Error on consuming: %v", err) - } - } - } + terminator.Wait(service) } diff --git a/backend/cmd/http/main.go b/backend/cmd/http/main.go index 83eedaf29..74c58f92b 100644 --- a/backend/cmd/http/main.go +++ b/backend/cmd/http/main.go @@ -15,7 +15,6 @@ import ( "openreplay/backend/pkg/metrics" databaseMetrics "openreplay/backend/pkg/metrics/database" httpMetrics "openreplay/backend/pkg/metrics/http" - "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue" ) @@ -27,9 +26,6 @@ func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := http.New() - if cfg.UseProfiler { - pprof.StartProfilingServer() - } // Connect to queue producer := queue.NewProducer(cfg.MessageSizeLimit, true) diff --git a/backend/cmd/integrations/main.go b/backend/cmd/integrations/main.go index 3fa07ee9c..c179650b9 100644 --- a/backend/cmd/integrations/main.go +++ b/backend/cmd/integrations/main.go @@ -13,7 +13,6 @@ import ( "openreplay/backend/pkg/intervals" "openreplay/backend/pkg/metrics" databaseMetrics "openreplay/backend/pkg/metrics/database" - "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/token" ) @@ -25,9 +24,6 @@ func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := config.New() - if cfg.UseProfiler { - pprof.StartProfilingServer() - } pg := postgres.NewConn(cfg.Postgres.String(), 0, 0) defer pg.Close() diff --git a/backend/cmd/sink/main.go b/backend/cmd/sink/main.go index 4bbaeeee4..e9cf1367a 100644 --- a/backend/cmd/sink/main.go +++ b/backend/cmd/sink/main.go @@ -16,7 +16,6 @@ import ( "openreplay/backend/pkg/messages" "openreplay/backend/pkg/metrics" sinkMetrics "openreplay/backend/pkg/metrics/sink" - "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/url/assets" ) @@ -27,9 +26,6 @@ func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := sink.New() - if cfg.UseProfiler { - pprof.StartProfilingServer() - } if _, err := os.Stat(cfg.FsDir); os.IsNotExist(err) { log.Fatalf("%v doesn't exist. %v", cfg.FsDir, err) @@ -112,7 +108,7 @@ func main() { log.Printf("zero ts; sessID: %d, msgType: %d", msg.SessionID(), msg.TypeID()) } else { // Log ts of last processed message - counter.Update(msg.SessionID(), time.UnixMilli(ts)) + counter.Update(msg.SessionID(), time.UnixMilli(int64(ts))) } // Try to encode message to avoid null data inserts diff --git a/backend/cmd/storage/main.go b/backend/cmd/storage/main.go index 472324b95..2a1f6a402 100644 --- a/backend/cmd/storage/main.go +++ b/backend/cmd/storage/main.go @@ -13,7 +13,6 @@ import ( "openreplay/backend/pkg/messages" "openreplay/backend/pkg/metrics" storageMetrics "openreplay/backend/pkg/metrics/storage" - "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue" cloud "openreplay/backend/pkg/storage" ) @@ -25,9 +24,6 @@ func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := config.New() - if cfg.UseProfiler { - pprof.StartProfilingServer() - } s3 := cloud.NewS3(cfg.S3Region, cfg.S3Bucket) srv, err := storage.New(cfg, s3) diff --git a/backend/internal/config/heuristics/config.go b/backend/internal/config/heuristics/config.go index 6552944a3..d222387c5 100644 --- a/backend/internal/config/heuristics/config.go +++ b/backend/internal/config/heuristics/config.go @@ -3,6 +3,7 @@ package heuristics import ( "openreplay/backend/internal/config/common" "openreplay/backend/internal/config/configurator" + "openreplay/backend/pkg/pprof" ) type Config struct { @@ -19,5 +20,8 @@ type Config struct { func New() *Config { cfg := &Config{} configurator.Process(cfg) + if cfg.UseProfiler { + pprof.StartProfilingServer() + } return cfg } diff --git a/backend/internal/db/datasaver/messages.go b/backend/internal/db/datasaver/messages.go deleted file mode 100644 index 12e7152b4..000000000 --- a/backend/internal/db/datasaver/messages.go +++ /dev/null @@ -1,74 +0,0 @@ -package datasaver - -import ( - "fmt" - . "openreplay/backend/pkg/messages" -) - -func (mi *Saver) InsertMessage(msg Message) error { - sessionID := msg.SessionID() - switch m := msg.(type) { - // Common - case *Metadata: - if err := mi.pg.InsertMetadata(sessionID, m); err != nil { - return fmt.Errorf("insert metadata err: %s", err) - } - return nil - case *IssueEvent: - return mi.pg.InsertIssueEvent(sessionID, m) - //TODO: message adapter (transformer) (at the level of pkg/message) for types: *IOSMetadata, *IOSIssueEvent and others - - // Web - case *SessionStart: - return mi.pg.HandleWebSessionStart(sessionID, m) - case *SessionEnd: - return mi.pg.HandleWebSessionEnd(sessionID, m) - case *UserID: - return mi.pg.InsertWebUserID(sessionID, m) - case *UserAnonymousID: - return mi.pg.InsertWebUserAnonymousID(sessionID, m) - case *CustomEvent: - return mi.pg.InsertWebCustomEvent(sessionID, m) - case *ClickEvent: - return mi.pg.InsertWebClickEvent(sessionID, m) - case *InputEvent: - return mi.pg.InsertWebInputEvent(sessionID, m) - - // Unique Web messages - case *PageEvent: - return mi.pg.InsertWebPageEvent(sessionID, m) - case *NetworkRequest: - return mi.pg.InsertWebNetworkRequest(sessionID, m) - case *GraphQL: - return mi.pg.InsertWebGraphQL(sessionID, m) - case *JSException: - return mi.pg.InsertWebJSException(m) - case *IntegrationEvent: - return mi.pg.InsertWebIntegrationEvent(m) - - // IOS - case *IOSSessionStart: - return mi.pg.InsertIOSSessionStart(sessionID, m) - case *IOSSessionEnd: - return mi.pg.InsertIOSSessionEnd(sessionID, m) - case *IOSUserID: - return mi.pg.InsertIOSUserID(sessionID, m) - case *IOSUserAnonymousID: - return mi.pg.InsertIOSUserAnonymousID(sessionID, m) - case *IOSCustomEvent: - return mi.pg.InsertIOSCustomEvent(sessionID, m) - case *IOSClickEvent: - return mi.pg.InsertIOSClickEvent(sessionID, m) - case *IOSInputEvent: - return mi.pg.InsertIOSInputEvent(sessionID, m) - // Unique IOS messages - case *IOSNetworkCall: - return mi.pg.InsertIOSNetworkCall(sessionID, m) - case *IOSScreenEnter: - return mi.pg.InsertIOSScreenEnter(sessionID, m) - case *IOSCrash: - return mi.pg.InsertIOSCrash(sessionID, m) - - } - return nil // "Not implemented" -} diff --git a/backend/internal/db/datasaver/methods.go b/backend/internal/db/datasaver/methods.go new file mode 100644 index 000000000..c4e83cf09 --- /dev/null +++ b/backend/internal/db/datasaver/methods.go @@ -0,0 +1,19 @@ +package datasaver + +import ( + . "openreplay/backend/pkg/messages" +) + +func (s *saverImpl) init() { + // noop +} + +func (s *saverImpl) handleExtraMessage(msg Message) error { + switch m := msg.(type) { + case *PerformanceTrackAggr: + return s.pg.InsertWebStatsPerformance(m) + case *ResourceTiming: + return s.pg.InsertWebStatsResourceEvent(m) + } + return nil +} diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index 2a356d120..92dbff958 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -1,16 +1,126 @@ package datasaver import ( + "log" + "openreplay/backend/internal/config/db" "openreplay/backend/pkg/db/cache" - "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/db/clickhouse" + "openreplay/backend/pkg/db/postgres" + "openreplay/backend/pkg/db/types" + . "openreplay/backend/pkg/messages" + queue "openreplay/backend/pkg/queue/types" ) -type Saver struct { - pg *cache.PGCache - producer types.Producer +type Saver interface { + Handle(msg Message) + Commit() error + Close() error } -func New(pg *cache.PGCache, _ *db.Config) *Saver { - return &Saver{pg: pg, producer: nil} +type saverImpl struct { + cfg *db.Config + pg *cache.PGCache + ch clickhouse.Connector + producer queue.Producer +} + +func New(cfg *db.Config, pg *cache.PGCache) Saver { + s := &saverImpl{cfg: cfg, pg: pg} + s.init() + return s +} + +func (s *saverImpl) Handle(msg Message) { + if msg.TypeID() == MsgCustomEvent { + defer s.Handle(types.WrapCustomEvent(msg.(*CustomEvent))) + } + if err := s.handleMessage(msg); err != nil { + if !postgres.IsPkeyViolation(err) { + log.Printf("Message Insertion Error %v, SessionID: %v, Message: %v", err, msg.SessionID(), msg) + } + return + } + if err := s.handleExtraMessage(msg); err != nil { + log.Printf("Stats Insertion Error %v; Session: %d, Message: %v", err, msg.SessionID(), msg) + } + return +} + +func (s *saverImpl) handleMessage(msg Message) error { + switch m := msg.(type) { + case *Metadata: + return s.pg.InsertMetadata(m) + case *IssueEvent: + return s.pg.InsertIssueEvent(m) + case *SessionStart: + return s.pg.HandleWebSessionStart(m) + case *SessionEnd: + return s.pg.HandleWebSessionEnd(m) + case *UserID: + return s.pg.InsertWebUserID(m) + case *UserAnonymousID: + return s.pg.InsertWebUserAnonymousID(m) + case *CustomEvent: + return s.pg.InsertWebCustomEvent(m) + case *MouseClick: + return s.pg.InsertWebClickEvent(m) + case *InputEvent: + return s.pg.InsertWebInputEvent(m) + case *PageEvent: + return s.pg.InsertWebPageEvent(m) + case *NetworkRequest: + return s.pg.InsertWebNetworkRequest(m) + case *GraphQL: + return s.pg.InsertWebGraphQL(m) + case *JSException: + return s.pg.InsertWebJSException(m) + case *IntegrationEvent: + return s.pg.InsertWebIntegrationEvent(m) + case *IOSSessionStart: + return s.pg.InsertIOSSessionStart(m) + case *IOSSessionEnd: + return s.pg.InsertIOSSessionEnd(m) + case *IOSUserID: + return s.pg.InsertIOSUserID(m) + case *IOSUserAnonymousID: + return s.pg.InsertIOSUserAnonymousID(m) + case *IOSCustomEvent: + return s.pg.InsertIOSCustomEvent(m) + case *IOSClickEvent: + return s.pg.InsertIOSClickEvent(m) + case *IOSInputEvent: + return s.pg.InsertIOSInputEvent(m) + case *IOSNetworkCall: + return s.pg.InsertIOSNetworkCall(m) + case *IOSScreenEnter: + return s.pg.InsertIOSScreenEnter(m) + case *IOSCrash: + return s.pg.InsertIOSCrash(m) + } + return nil +} + +func (s *saverImpl) Commit() error { + if s.pg != nil { + s.pg.Commit() + } + if s.ch != nil { + s.ch.Commit() + } + return nil +} + +func (s *saverImpl) Close() error { + if s.pg != nil { + if err := s.pg.Close(); err != nil { + log.Printf("pg.Close error: %s", err) + } + } + if s.ch != nil { + if err := s.ch.Stop(); err != nil { + log.Printf("ch.Close error: %s", err) + } + } + return nil } diff --git a/backend/internal/db/datasaver/stats.go b/backend/internal/db/datasaver/stats.go deleted file mode 100644 index c7daeb3dc..000000000 --- a/backend/internal/db/datasaver/stats.go +++ /dev/null @@ -1,29 +0,0 @@ -package datasaver - -import ( - . "openreplay/backend/pkg/db/types" - . "openreplay/backend/pkg/messages" -) - -func (si *Saver) InitStats() { - // noop -} - -func (si *Saver) InsertStats(session *Session, msg Message) error { - switch m := msg.(type) { - // Web - case *PerformanceTrackAggr: - return si.pg.InsertWebStatsPerformance(session.SessionID, m) - case *ResourceEvent: - return si.pg.InsertWebStatsResourceEvent(session.SessionID, m) - } - return nil -} - -func (si *Saver) CommitStats() error { - return nil -} - -func (si *Saver) Close() error { - return nil -} diff --git a/backend/internal/db/service.go b/backend/internal/db/service.go new file mode 100644 index 000000000..69b5cb1cb --- /dev/null +++ b/backend/internal/db/service.go @@ -0,0 +1,56 @@ +package db + +import ( + "log" + "time" + + "openreplay/backend/internal/config/db" + "openreplay/backend/internal/db/datasaver" + "openreplay/backend/internal/service" + "openreplay/backend/pkg/queue/types" +) + +type dbImpl struct { + cfg *db.Config + consumer types.Consumer + saver datasaver.Saver +} + +func New(cfg *db.Config, consumer types.Consumer, saver datasaver.Saver) service.Interface { + s := &dbImpl{ + cfg: cfg, + consumer: consumer, + saver: saver, + } + go s.run() + return s +} + +func (d *dbImpl) run() { + commitTick := time.Tick(d.cfg.CommitBatchTimeout) + for { + select { + case <-commitTick: + d.commit() + case msg := <-d.consumer.Rebalanced(): + log.Println(msg) + default: + if err := d.consumer.ConsumeNext(); err != nil { + log.Fatalf("Error on consumption: %v", err) + } + } + } +} + +func (d *dbImpl) commit() { + d.saver.Commit() + d.consumer.Commit() +} + +func (d *dbImpl) Stop() { + d.commit() + if err := d.saver.Close(); err != nil { + log.Printf("saver.Close error: %s", err) + } + d.consumer.Close() +} diff --git a/backend/internal/heuristics/service.go b/backend/internal/heuristics/service.go new file mode 100644 index 000000000..0063f79f7 --- /dev/null +++ b/backend/internal/heuristics/service.go @@ -0,0 +1,64 @@ +package heuristics + +import ( + "log" + "time" + + "openreplay/backend/internal/config/heuristics" + "openreplay/backend/internal/service" + "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/sessions" +) + +type heuristicsImpl struct { + cfg *heuristics.Config + producer types.Producer + consumer types.Consumer + events sessions.EventBuilder +} + +func New(cfg *heuristics.Config, p types.Producer, c types.Consumer, e sessions.EventBuilder) service.Interface { + s := &heuristicsImpl{ + cfg: cfg, + producer: p, + consumer: c, + events: e, + } + go s.run() + return s +} + +func (h *heuristicsImpl) run() { + tick := time.Tick(10 * time.Second) + for { + select { + case evt := <-h.events.Events(): + if err := h.producer.Produce(h.cfg.TopicAnalytics, evt.SessionID(), evt.Encode()); err != nil { + log.Printf("can't send new event to queue: %s", err) + } + case <-tick: + h.producer.Flush(h.cfg.ProducerTimeout) + h.consumer.Commit() + case msg := <-h.consumer.Rebalanced(): + log.Println(msg) + default: + if err := h.consumer.ConsumeNext(); err != nil { + log.Fatalf("Error on consuming: %v", err) + } + } + } +} + +func (h *heuristicsImpl) Stop() { + // Stop event builder and flush all events + log.Println("stopping heuristics service") + h.events.Stop() + for evt := range h.events.Events() { + if err := h.producer.Produce(h.cfg.TopicAnalytics, evt.SessionID(), evt.Encode()); err != nil { + log.Printf("can't send new event to queue: %s", err) + } + } + h.producer.Close(h.cfg.ProducerTimeout) + h.consumer.Commit() + h.consumer.Close() +} diff --git a/backend/internal/service/service.go b/backend/internal/service/service.go new file mode 100644 index 000000000..a20254093 --- /dev/null +++ b/backend/internal/service/service.go @@ -0,0 +1,5 @@ +package service + +type Interface interface { + Stop() +} diff --git a/backend/internal/sessionender/ender.go b/backend/internal/sessionender/ender.go index e1ddb0ffe..26fcf850e 100644 --- a/backend/internal/sessionender/ender.go +++ b/backend/internal/sessionender/ender.go @@ -9,13 +9,13 @@ import ( ) // EndedSessionHandler handler for ended sessions -type EndedSessionHandler func(sessionID uint64, timestamp int64) bool +type EndedSessionHandler func(sessionID uint64, timestamp uint64) bool // session holds information about user's session live status type session struct { lastTimestamp int64 lastUpdate int64 - lastUserTime int64 + lastUserTime uint64 isEnded bool } diff --git a/backend/pkg/db/cache/messages-common.go b/backend/pkg/db/cache/messages-common.go index 3fc52f395..763f97d90 100644 --- a/backend/pkg/db/cache/messages-common.go +++ b/backend/pkg/db/cache/messages-common.go @@ -21,7 +21,8 @@ func (c *PGCache) HandleSessionEnd(sessionID uint64) error { return nil } -func (c *PGCache) InsertIssueEvent(sessionID uint64, crash *IssueEvent) error { +func (c *PGCache) InsertIssueEvent(crash *IssueEvent) error { + sessionID := crash.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -29,7 +30,8 @@ func (c *PGCache) InsertIssueEvent(sessionID uint64, crash *IssueEvent) error { return c.Conn.InsertIssueEvent(sessionID, session.ProjectID, crash) } -func (c *PGCache) InsertMetadata(sessionID uint64, metadata *Metadata) error { +func (c *PGCache) InsertMetadata(metadata *Metadata) error { + sessionID := metadata.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err diff --git a/backend/pkg/db/cache/messages-ios.go b/backend/pkg/db/cache/messages-ios.go index 961b78dad..93367f925 100644 --- a/backend/pkg/db/cache/messages-ios.go +++ b/backend/pkg/db/cache/messages-ios.go @@ -6,7 +6,8 @@ import ( . "openreplay/backend/pkg/messages" ) -func (c *PGCache) InsertIOSSessionStart(sessionID uint64, s *IOSSessionStart) error { +func (c *PGCache) InsertIOSSessionStart(s *IOSSessionStart) error { + sessionID := s.SessionID() if c.Cache.HasSession(sessionID) { return fmt.Errorf("session %d already in cache", sessionID) } @@ -33,13 +34,15 @@ func (c *PGCache) InsertIOSSessionStart(sessionID uint64, s *IOSSessionStart) er return nil } -func (c *PGCache) InsertIOSSessionEnd(sessionID uint64, e *IOSSessionEnd) error { +func (c *PGCache) InsertIOSSessionEnd(e *IOSSessionEnd) error { + sessionID := e.SessionID() _, err := c.InsertSessionEnd(sessionID, e.Timestamp) return err } -func (c *PGCache) InsertIOSScreenEnter(sessionID uint64, screenEnter *IOSScreenEnter) error { - if err := c.Conn.InsertIOSScreenEnter(sessionID, screenEnter); err != nil { +func (c *PGCache) InsertIOSScreenEnter(screenEnter *IOSScreenEnter) error { + sessionID := screenEnter.SessionID() + if err := c.Conn.InsertIOSScreenEnter(screenEnter); err != nil { return err } session, err := c.Cache.GetSession(sessionID) @@ -50,8 +53,9 @@ func (c *PGCache) InsertIOSScreenEnter(sessionID uint64, screenEnter *IOSScreenE return nil } -func (c *PGCache) InsertIOSClickEvent(sessionID uint64, clickEvent *IOSClickEvent) error { - if err := c.Conn.InsertIOSClickEvent(sessionID, clickEvent); err != nil { +func (c *PGCache) InsertIOSClickEvent(clickEvent *IOSClickEvent) error { + sessionID := clickEvent.SessionID() + if err := c.Conn.InsertIOSClickEvent(clickEvent); err != nil { return err } session, err := c.Cache.GetSession(sessionID) @@ -62,8 +66,9 @@ func (c *PGCache) InsertIOSClickEvent(sessionID uint64, clickEvent *IOSClickEven return nil } -func (c *PGCache) InsertIOSInputEvent(sessionID uint64, inputEvent *IOSInputEvent) error { - if err := c.Conn.InsertIOSInputEvent(sessionID, inputEvent); err != nil { +func (c *PGCache) InsertIOSInputEvent(inputEvent *IOSInputEvent) error { + sessionID := inputEvent.SessionID() + if err := c.Conn.InsertIOSInputEvent(inputEvent); err != nil { return err } session, err := c.Cache.GetSession(sessionID) @@ -74,18 +79,15 @@ func (c *PGCache) InsertIOSInputEvent(sessionID uint64, inputEvent *IOSInputEven return nil } -func (c *PGCache) InsertIOSCrash(sessionID uint64, crash *IOSCrash) error { +func (c *PGCache) InsertIOSCrash(crash *IOSCrash) error { + sessionID := crash.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err } - if err := c.Conn.InsertIOSCrash(sessionID, session.ProjectID, crash); err != nil { + if err := c.Conn.InsertIOSCrash(session.ProjectID, crash); err != nil { return err } session.ErrorsCount += 1 return nil } - -func (c *PGCache) InsertIOSIssueEvent(sessionID uint64, issueEvent *IOSIssueEvent) error { - return nil -} diff --git a/backend/pkg/db/cache/messages-web.go b/backend/pkg/db/cache/messages-web.go index 1df3d1520..0a870e5a2 100644 --- a/backend/pkg/db/cache/messages-web.go +++ b/backend/pkg/db/cache/messages-web.go @@ -30,7 +30,8 @@ func (c *PGCache) InsertWebSessionStart(sessionID uint64, s *SessionStart) error }) } -func (c *PGCache) HandleWebSessionStart(sessionID uint64, s *SessionStart) error { +func (c *PGCache) HandleWebSessionStart(s *SessionStart) error { + sessionID := s.SessionID() if c.Cache.HasSession(sessionID) { return fmt.Errorf("session %d already in cache", sessionID) } @@ -69,7 +70,8 @@ func (c *PGCache) InsertWebSessionEnd(sessionID uint64, e *SessionEnd) error { return err } -func (c *PGCache) HandleWebSessionEnd(sessionID uint64, e *SessionEnd) error { +func (c *PGCache) HandleWebSessionEnd(e *SessionEnd) error { + sessionID := e.SessionID() return c.HandleSessionEnd(sessionID) } @@ -99,7 +101,8 @@ func (c *PGCache) InsertSessionReferrer(sessionID uint64, referrer string) error return c.Conn.InsertSessionReferrer(sessionID, referrer) } -func (c *PGCache) InsertWebNetworkRequest(sessionID uint64, e *NetworkRequest) error { +func (c *PGCache) InsertWebNetworkRequest(e *NetworkRequest) error { + sessionID := e.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -111,7 +114,8 @@ func (c *PGCache) InsertWebNetworkRequest(sessionID uint64, e *NetworkRequest) e return c.Conn.InsertWebNetworkRequest(sessionID, session.ProjectID, project.SaveRequestPayloads, e) } -func (c *PGCache) InsertWebGraphQL(sessionID uint64, e *GraphQL) error { +func (c *PGCache) InsertWebGraphQL(e *GraphQL) error { + sessionID := e.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -123,7 +127,8 @@ func (c *PGCache) InsertWebGraphQL(sessionID uint64, e *GraphQL) error { return c.Conn.InsertWebGraphQL(sessionID, session.ProjectID, project.SaveRequestPayloads, e) } -func (c *PGCache) InsertWebCustomEvent(sessionID uint64, e *CustomEvent) error { +func (c *PGCache) InsertWebCustomEvent(e *CustomEvent) error { + sessionID := e.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -131,7 +136,8 @@ func (c *PGCache) InsertWebCustomEvent(sessionID uint64, e *CustomEvent) error { return c.Conn.InsertWebCustomEvent(sessionID, session.ProjectID, e) } -func (c *PGCache) InsertWebUserID(sessionID uint64, userID *UserID) error { +func (c *PGCache) InsertWebUserID(userID *UserID) error { + sessionID := userID.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -139,7 +145,8 @@ func (c *PGCache) InsertWebUserID(sessionID uint64, userID *UserID) error { return c.Conn.InsertWebUserID(sessionID, session.ProjectID, userID) } -func (c *PGCache) InsertWebUserAnonymousID(sessionID uint64, userAnonymousID *UserAnonymousID) error { +func (c *PGCache) InsertWebUserAnonymousID(userAnonymousID *UserAnonymousID) error { + sessionID := userAnonymousID.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -147,7 +154,8 @@ func (c *PGCache) InsertWebUserAnonymousID(sessionID uint64, userAnonymousID *Us return c.Conn.InsertWebUserAnonymousID(sessionID, session.ProjectID, userAnonymousID) } -func (c *PGCache) InsertWebPageEvent(sessionID uint64, e *PageEvent) error { +func (c *PGCache) InsertWebPageEvent(e *PageEvent) error { + sessionID := e.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -155,7 +163,8 @@ func (c *PGCache) InsertWebPageEvent(sessionID uint64, e *PageEvent) error { return c.Conn.InsertWebPageEvent(sessionID, session.ProjectID, e) } -func (c *PGCache) InsertWebClickEvent(sessionID uint64, e *ClickEvent) error { +func (c *PGCache) InsertWebClickEvent(e *MouseClick) error { + sessionID := e.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err @@ -163,7 +172,8 @@ func (c *PGCache) InsertWebClickEvent(sessionID uint64, e *ClickEvent) error { return c.Conn.InsertWebClickEvent(sessionID, session.ProjectID, e) } -func (c *PGCache) InsertWebInputEvent(sessionID uint64, e *InputEvent) error { +func (c *PGCache) InsertWebInputEvent(e *InputEvent) error { + sessionID := e.SessionID() session, err := c.Cache.GetSession(sessionID) if err != nil { return err diff --git a/backend/pkg/db/clickhouse/connector.go b/backend/pkg/db/clickhouse/connector.go new file mode 100644 index 000000000..1d3a3b4f5 --- /dev/null +++ b/backend/pkg/db/clickhouse/connector.go @@ -0,0 +1,24 @@ +package clickhouse + +import ( + "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/messages" +) + +type Connector interface { + Prepare() error + Commit() error + Stop() error + InsertWebSession(session *types.Session) error + InsertWebResourceEvent(session *types.Session, msg *messages.ResourceTiming) error + InsertWebPageEvent(session *types.Session, msg *messages.PageEvent) error + InsertWebClickEvent(session *types.Session, msg *messages.MouseClick) error + InsertWebInputEvent(session *types.Session, msg *messages.InputEvent) error + InsertWebErrorEvent(session *types.Session, msg *types.ErrorEvent) error + InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error + InsertAutocomplete(session *types.Session, msgType, msgValue string) error + InsertRequest(session *types.Session, msg *messages.NetworkRequest, savePayload bool) error + InsertCustom(session *types.Session, msg *messages.CustomEvent) error + InsertGraphQL(session *types.Session, msg *messages.GraphQL) error + InsertIssue(session *types.Session, msg *messages.IssueEvent) error +} diff --git a/backend/pkg/db/postgres/batches.go b/backend/pkg/db/postgres/batches.go index 8b9f2484d..bf4d1745c 100644 --- a/backend/pkg/db/postgres/batches.go +++ b/backend/pkg/db/postgres/batches.go @@ -193,9 +193,7 @@ func (conn *BatchSet) worker() { for { select { case t := <-conn.workerTask: - start := time.Now() conn.sendBatches(t) - log.Printf("pg batches dur: %d", time.Now().Sub(start).Milliseconds()) case <-conn.done: if len(conn.workerTask) > 0 { for t := range conn.workerTask { diff --git a/backend/pkg/db/postgres/bulks.go b/backend/pkg/db/postgres/bulks.go index f3e9e95c9..27ab2cafd 100644 --- a/backend/pkg/db/postgres/bulks.go +++ b/backend/pkg/db/postgres/bulks.go @@ -2,7 +2,6 @@ package postgres import ( "log" - "time" ) type bulksTask struct { @@ -243,9 +242,7 @@ func (conn *BulkSet) worker() { for { select { case t := <-conn.workerTask: - start := time.Now() conn.sendBulks(t) - log.Printf("pg bulks dur: %d", time.Now().Sub(start).Milliseconds()) case <-conn.done: if len(conn.workerTask) > 0 { for t := range conn.workerTask { diff --git a/backend/pkg/db/postgres/connector.go b/backend/pkg/db/postgres/connector.go index 6904dc135..be748e6a2 100644 --- a/backend/pkg/db/postgres/connector.go +++ b/backend/pkg/db/postgres/connector.go @@ -17,7 +17,7 @@ type Conn struct { c Pool batches *BatchSet bulks *BulkSet - chConn CH + chConn CH // hack for autocomplete inserts, TODO: rewrite } func (conn *Conn) SetClickHouse(ch CH) { diff --git a/backend/pkg/db/postgres/messages-ios.go b/backend/pkg/db/postgres/messages-ios.go index 027cfc968..ace1955f5 100644 --- a/backend/pkg/db/postgres/messages-ios.go +++ b/backend/pkg/db/postgres/messages-ios.go @@ -6,7 +6,8 @@ import ( "openreplay/backend/pkg/url" ) -func (conn *Conn) InsertIOSCustomEvent(sessionID uint64, e *messages.IOSCustomEvent) error { +func (conn *Conn) InsertIOSCustomEvent(e *messages.IOSCustomEvent) error { + sessionID := e.SessionID() err := conn.InsertCustomEvent(sessionID, e.Timestamp, truncSqIdx(e.Index), e.Name, e.Payload) if err == nil { conn.insertAutocompleteValue(sessionID, 0, "CUSTOM_IOS", e.Name) @@ -14,7 +15,8 @@ func (conn *Conn) InsertIOSCustomEvent(sessionID uint64, e *messages.IOSCustomEv return err } -func (conn *Conn) InsertIOSUserID(sessionID uint64, userID *messages.IOSUserID) error { +func (conn *Conn) InsertIOSUserID(userID *messages.IOSUserID) error { + sessionID := userID.SessionID() err := conn.InsertUserID(sessionID, userID.Value) if err == nil { conn.insertAutocompleteValue(sessionID, 0, "USERID_IOS", userID.Value) @@ -22,7 +24,8 @@ func (conn *Conn) InsertIOSUserID(sessionID uint64, userID *messages.IOSUserID) return err } -func (conn *Conn) InsertIOSUserAnonymousID(sessionID uint64, userAnonymousID *messages.IOSUserAnonymousID) error { +func (conn *Conn) InsertIOSUserAnonymousID(userAnonymousID *messages.IOSUserAnonymousID) error { + sessionID := userAnonymousID.SessionID() err := conn.InsertUserAnonymousID(sessionID, userAnonymousID.Value) if err == nil { conn.insertAutocompleteValue(sessionID, 0, "USERANONYMOUSID_IOS", userAnonymousID.Value) @@ -30,7 +33,8 @@ func (conn *Conn) InsertIOSUserAnonymousID(sessionID uint64, userAnonymousID *me return err } -func (conn *Conn) InsertIOSNetworkCall(sessionID uint64, e *messages.IOSNetworkCall) error { +func (conn *Conn) InsertIOSNetworkCall(e *messages.IOSNetworkCall) error { + sessionID := e.SessionID() err := conn.InsertRequest(sessionID, e.Timestamp, truncSqIdx(e.Index), e.URL, e.Duration, e.Success) if err == nil { conn.insertAutocompleteValue(sessionID, 0, "REQUEST_IOS", url.DiscardURLQuery(e.URL)) @@ -38,7 +42,8 @@ func (conn *Conn) InsertIOSNetworkCall(sessionID uint64, e *messages.IOSNetworkC return err } -func (conn *Conn) InsertIOSScreenEnter(sessionID uint64, screenEnter *messages.IOSScreenEnter) error { +func (conn *Conn) InsertIOSScreenEnter(screenEnter *messages.IOSScreenEnter) error { + sessionID := screenEnter.SessionID() tx, err := conn.c.Begin() if err != nil { return err @@ -69,7 +74,8 @@ func (conn *Conn) InsertIOSScreenEnter(sessionID uint64, screenEnter *messages.I return nil } -func (conn *Conn) InsertIOSClickEvent(sessionID uint64, clickEvent *messages.IOSClickEvent) error { +func (conn *Conn) InsertIOSClickEvent(clickEvent *messages.IOSClickEvent) error { + sessionID := clickEvent.SessionID() tx, err := conn.c.Begin() if err != nil { return err @@ -100,7 +106,8 @@ func (conn *Conn) InsertIOSClickEvent(sessionID uint64, clickEvent *messages.IOS return nil } -func (conn *Conn) InsertIOSInputEvent(sessionID uint64, inputEvent *messages.IOSInputEvent) error { +func (conn *Conn) InsertIOSInputEvent(inputEvent *messages.IOSInputEvent) error { + sessionID := inputEvent.SessionID() tx, err := conn.c.Begin() if err != nil { return err @@ -137,7 +144,8 @@ func (conn *Conn) InsertIOSInputEvent(sessionID uint64, inputEvent *messages.IOS return nil } -func (conn *Conn) InsertIOSCrash(sessionID uint64, projectID uint32, crash *messages.IOSCrash) error { +func (conn *Conn) InsertIOSCrash(projectID uint32, crash *messages.IOSCrash) error { + sessionID := crash.SessionID() tx, err := conn.c.Begin() if err != nil { return err diff --git a/backend/pkg/db/postgres/messages-web-stats.go b/backend/pkg/db/postgres/messages-web-stats.go index 42458a497..47bd06974 100644 --- a/backend/pkg/db/postgres/messages-web-stats.go +++ b/backend/pkg/db/postgres/messages-web-stats.go @@ -5,7 +5,8 @@ import ( "openreplay/backend/pkg/url" ) -func (conn *Conn) InsertWebStatsPerformance(sessionID uint64, p *PerformanceTrackAggr) error { +func (conn *Conn) InsertWebStatsPerformance(p *PerformanceTrackAggr) error { + sessionID := p.SessionID() timestamp := (p.TimestampEnd + p.TimestampStart) / 2 sqlRequest := ` @@ -35,40 +36,37 @@ func (conn *Conn) InsertWebStatsPerformance(sessionID uint64, p *PerformanceTrac return nil } -func (conn *Conn) InsertWebStatsResourceEvent(sessionID uint64, e *ResourceEvent) error { +func (conn *Conn) InsertWebStatsResourceEvent(e *ResourceTiming) error { + sessionID := e.SessionID() host, _, _, err := url.GetURLParts(e.URL) if err != nil { return err } - + msgType := url.GetResourceType(e.Initiator, e.URL) sqlRequest := ` INSERT INTO events.resources ( session_id, timestamp, message_id, type, url, url_host, url_hostpath, success, status, - method, duration, ttfb, header_size, encoded_body_size, decoded_body_size ) VALUES ( $1, $2, $3, $4, LEFT($5, 8000), LEFT($6, 300), LEFT($7, 2000), $8, $9, - NULLIF($10, '')::events.resource_method, - NULLIF($11, 0), NULLIF($12, 0), NULLIF($13, 0), NULLIF($14, 0), NULLIF($15, 0) + NULLIF($10, 0), NULLIF($11, 0), NULLIF($12, 0), NULLIF($13, 0), NULLIF($14, 0) )` urlQuery := url.DiscardURLQuery(e.URL) - urlMethod := url.EnsureMethod(e.Method) conn.batchQueue(sessionID, sqlRequest, - sessionID, e.Timestamp, truncSqIdx(e.MessageID), - e.Type, + sessionID, e.Timestamp, truncSqIdx(e.MsgID()), + msgType, e.URL, host, urlQuery, - e.Success, e.Status, - urlMethod, + e.Duration != 0, 0, e.Duration, e.TTFB, e.HeaderSize, e.EncodedBodySize, e.DecodedBodySize, ) // Record approximate message size - conn.updateBatchSize(sessionID, len(sqlRequest)+len(e.Type)+len(e.URL)+len(host)+len(urlQuery)+len(urlMethod)+8*9+1) + conn.updateBatchSize(sessionID, len(sqlRequest)+len(msgType)+len(e.URL)+len(host)+len(urlQuery)+8*9+1) return nil } diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index 08db4491e..9251a4924 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -57,10 +57,13 @@ func (conn *Conn) InsertWebPageEvent(sessionID uint64, projectID uint32, e *Page return nil } -func (conn *Conn) InsertWebClickEvent(sessionID uint64, projectID uint32, e *ClickEvent) error { +func (conn *Conn) InsertWebClickEvent(sessionID uint64, projectID uint32, e *MouseClick) error { + if e.Label == "" { + return nil + } var host, path string host, path, _, _ = url.GetURLParts(e.Url) - if err := conn.bulks.Get("webClickEvents").Append(sessionID, truncSqIdx(e.MessageID), e.Timestamp, e.Label, e.Selector, host+path, path); err != nil { + if err := conn.bulks.Get("webClickEvents").Append(sessionID, truncSqIdx(e.MsgID()), e.Timestamp, e.Label, e.Selector, host+path, path); err != nil { log.Printf("insert web click err: %s", err) } // Accumulate session updates and exec inside batch with another sql commands diff --git a/backend/pkg/db/types/error-event.go b/backend/pkg/db/types/error-event.go index bef9abd99..9f2f1a886 100644 --- a/backend/pkg/db/types/error-event.go +++ b/backend/pkg/db/types/error-event.go @@ -120,3 +120,15 @@ func (e *ErrorEvent) ID(projectID uint32) string { } return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) } + +func WrapCustomEvent(m *CustomEvent) *IssueEvent { + msg := &IssueEvent{ + Type: "custom", + Timestamp: m.Time(), + MessageID: m.MsgID(), + ContextString: m.Name, + Payload: m.Payload, + } + msg.Meta().SetMeta(m.Meta()) + return msg +} diff --git a/backend/pkg/handlers/custom/eventMapper.go b/backend/pkg/handlers/custom/eventMapper.go deleted file mode 100644 index a85ebbdf0..000000000 --- a/backend/pkg/handlers/custom/eventMapper.go +++ /dev/null @@ -1,82 +0,0 @@ -package custom - -import ( - "net/url" - "strings" - - . "openreplay/backend/pkg/messages" -) - -func getURLExtention(URL string) string { - u, err := url.Parse(URL) - if err != nil { - return "" - } - i := strings.LastIndex(u.Path, ".") - return u.Path[i+1:] -} - -func getResourceType(initiator string, URL string) string { - switch initiator { - case "xmlhttprequest", "fetch": - return "fetch" - case "img": - return "img" - default: - switch getURLExtention(URL) { - case "css": - return "stylesheet" - case "js": - return "script" - case "png", "gif", "jpg", "jpeg", "svg": - return "img" - case "mp4", "mkv", "ogg", "webm", "avi", "mp3": - return "media" - default: - return "other" - } - } -} - -type EventMapper struct{} - -func (b *EventMapper) Build() Message { - return nil -} - -func (b *EventMapper) Handle(message Message, messageID uint64, timestamp uint64) Message { - switch msg := message.(type) { - case *MouseClick: - if msg.Label != "" { - return &ClickEvent{ - MessageID: messageID, - Label: msg.Label, - HesitationTime: msg.HesitationTime, - Timestamp: timestamp, - Selector: msg.Selector, - } - } - case *ResourceTiming: - return &ResourceEvent{ - MessageID: messageID, - Timestamp: msg.Timestamp, - Duration: msg.Duration, - TTFB: msg.TTFB, - HeaderSize: msg.HeaderSize, - EncodedBodySize: msg.EncodedBodySize, - DecodedBodySize: msg.DecodedBodySize, - URL: msg.URL, - Type: getResourceType(msg.Initiator, msg.URL), - Success: msg.Duration != 0, - } - case *CustomIssue: - return &IssueEvent{ - Type: "custom", - Timestamp: timestamp, - MessageID: messageID, - ContextString: msg.Name, - Payload: msg.Payload, - } - } - return nil -} diff --git a/backend/pkg/handlers/custom/inputEventBuilder.go b/backend/pkg/handlers/custom/inputEventBuilder.go index e07470f37..d057db3e3 100644 --- a/backend/pkg/handlers/custom/inputEventBuilder.go +++ b/backend/pkg/handlers/custom/inputEventBuilder.go @@ -4,7 +4,7 @@ import ( . "openreplay/backend/pkg/messages" ) -const INPUT_EVENT_TIMEOUT = 1 * 60 * 1000 +const InputEventTimeout = 1 * 60 * 1000 type inputLabels map[uint64]string @@ -24,7 +24,7 @@ func (b *inputEventBuilder) clearLabels() { b.inputLabels = make(inputLabels) } -func (b *inputEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (b *inputEventBuilder) Handle(message Message, timestamp uint64) Message { var inputEvent Message = nil switch msg := message.(type) { case *SetInputTarget: @@ -41,7 +41,7 @@ func (b *inputEventBuilder) Handle(message Message, messageID uint64, timestamp } if b.inputEvent == nil { b.inputEvent = &InputEvent{ - MessageID: messageID, + MessageID: message.MsgID(), Timestamp: timestamp, Value: msg.Value, ValueMasked: msg.Mask > 0, @@ -59,7 +59,7 @@ func (b *inputEventBuilder) Handle(message Message, messageID uint64, timestamp return b.Build() } - if b.inputEvent != nil && b.inputEvent.Timestamp+INPUT_EVENT_TIMEOUT < timestamp { + if b.inputEvent != nil && b.inputEvent.Timestamp+InputEventTimeout < timestamp { return b.Build() } return nil diff --git a/backend/pkg/handlers/custom/pageEventBuilder.go b/backend/pkg/handlers/custom/pageEventBuilder.go index d95768983..5bab7d4cc 100644 --- a/backend/pkg/handlers/custom/pageEventBuilder.go +++ b/backend/pkg/handlers/custom/pageEventBuilder.go @@ -4,7 +4,7 @@ import ( . "openreplay/backend/pkg/messages" ) -const PAGE_EVENT_TIMEOUT = 1 * 60 * 1000 +const PageEventTimeout = 1 * 60 * 1000 type pageEventBuilder struct { pageEvent *PageEvent @@ -16,7 +16,7 @@ func NewPageEventBuilder() *pageEventBuilder { return ieBuilder } -func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (b *pageEventBuilder) Handle(message Message, timestamp uint64) Message { switch msg := message.(type) { case *SetPageLocation: if msg.NavigationStart == 0 { // routing without new page loading @@ -24,7 +24,7 @@ func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp u URL: msg.URL, Referrer: msg.Referrer, Loaded: false, - MessageID: messageID, + MessageID: message.MsgID(), Timestamp: timestamp, } } else { @@ -33,7 +33,7 @@ func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp u URL: msg.URL, Referrer: msg.Referrer, Loaded: true, - MessageID: messageID, + MessageID: message.MsgID(), Timestamp: timestamp, } return pageEvent @@ -81,7 +81,7 @@ func (b *pageEventBuilder) Handle(message Message, messageID uint64, timestamp u } - if b.pageEvent != nil && b.pageEvent.Timestamp+PAGE_EVENT_TIMEOUT < timestamp { + if b.pageEvent != nil && b.pageEvent.Timestamp+PageEventTimeout < timestamp { return b.Build() } return nil diff --git a/backend/pkg/handlers/ios/clickRage.go b/backend/pkg/handlers/ios/clickRage.go index 84c130dae..91283de90 100644 --- a/backend/pkg/handlers/ios/clickRage.go +++ b/backend/pkg/handlers/ios/clickRage.go @@ -48,7 +48,7 @@ func (h *ClickRageDetector) Handle(message Message, messageID uint64, timestamp } func (h *ClickRageDetector) Build() Message { - if h.countsInARow >= web.MIN_CLICKS_IN_A_ROW { + if h.countsInARow >= web.MinClicksInARow { event := &IOSIssueEvent{ Type: "click_rage", ContextString: h.lastLabel, diff --git a/backend/pkg/handlers/messageProcessor.go b/backend/pkg/handlers/messageProcessor.go index c4235c18b..38d02ab1a 100644 --- a/backend/pkg/handlers/messageProcessor.go +++ b/backend/pkg/handlers/messageProcessor.go @@ -6,6 +6,6 @@ import . "openreplay/backend/pkg/messages" // U can create your own message handler and easily connect to heuristics service type MessageProcessor interface { - Handle(message Message, messageID uint64, timestamp uint64) Message + Handle(message Message, timestamp uint64) Message Build() Message } diff --git a/backend/pkg/handlers/web/clickRage.go b/backend/pkg/handlers/web/clickRage.go index 6974ee1b0..56692765e 100644 --- a/backend/pkg/handlers/web/clickRage.go +++ b/backend/pkg/handlers/web/clickRage.go @@ -7,14 +7,8 @@ import ( . "openreplay/backend/pkg/messages" ) -/* - Handler name: ClickRage - Input event: MouseClick - Output event: IssueEvent -*/ - -const MAX_TIME_DIFF = 300 -const MIN_CLICKS_IN_A_ROW = 3 +const MaxTimeDiff = 300 +const MinClicksInARow = 3 type ClickRageDetector struct { lastTimestamp uint64 @@ -34,46 +28,54 @@ func (crd *ClickRageDetector) reset() { crd.url = "" } -func (crd *ClickRageDetector) Build() Message { - defer crd.reset() - if crd.countsInARow >= MIN_CLICKS_IN_A_ROW { - payload, err := json.Marshal(struct{ Count int }{crd.countsInARow}) - if err != nil { - log.Printf("can't marshal ClickRage payload to json: %s", err) - } - event := &IssueEvent{ - Type: "click_rage", - ContextString: crd.lastLabel, - Payload: string(payload), - Timestamp: crd.firstInARawTimestamp, - MessageID: crd.firstInARawMessageId, - URL: crd.url, - } - return event +func (crd *ClickRageDetector) createPayload() string { + p, err := json.Marshal(struct{ Count int }{crd.countsInARow}) + if err != nil { + log.Printf("can't marshal ClickRage payload to json: %s", err) + return "" } - return nil + return string(p) } -func (crd *ClickRageDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (crd *ClickRageDetector) Build() Message { + defer crd.reset() + if crd.countsInARow < MinClicksInARow { + return nil + } + return &IssueEvent{ + Type: "click_rage", + ContextString: crd.lastLabel, + Payload: crd.createPayload(), + Timestamp: crd.firstInARawTimestamp, + MessageID: crd.firstInARawMessageId, + URL: crd.url, + } +} + +func (crd *ClickRageDetector) Handle(message Message, timestamp uint64) Message { switch msg := message.(type) { case *MouseClick: + // Set click url if crd.url == "" && msg.Url != "" { crd.url = msg.Url } - // TODO: check if we it is ok to capture clickRage event without the connected ClickEvent in db. + // Click on different object -> build if we can and reset the builder if msg.Label == "" { return crd.Build() } - if crd.lastLabel == msg.Label && timestamp-crd.lastTimestamp < MAX_TIME_DIFF { + // Update builder with last information + if crd.lastLabel == msg.Label && timestamp-crd.lastTimestamp < MaxTimeDiff { crd.lastTimestamp = timestamp crd.countsInARow += 1 return nil } + // Try to build event event := crd.Build() + // Use current message as init values for new event crd.lastTimestamp = timestamp crd.lastLabel = msg.Label crd.firstInARawTimestamp = timestamp - crd.firstInARawMessageId = messageID + crd.firstInARawMessageId = message.MsgID() crd.countsInARow = 1 if crd.url == "" && msg.Url != "" { crd.url = msg.Url diff --git a/backend/pkg/handlers/web/cpuIssue.go b/backend/pkg/handlers/web/cpuIssue.go index 56f483e8b..74117fc1f 100644 --- a/backend/pkg/handlers/web/cpuIssue.go +++ b/backend/pkg/handlers/web/cpuIssue.go @@ -15,8 +15,8 @@ import ( Output event: IssueEvent */ -const CPU_THRESHOLD = 70 // % out of 100 -const CPU_MIN_DURATION_TRIGGER = 6 * 1000 +const CpuThreshold = 70 // % out of 100 +const CpuMinDurationTrigger = 6 * 1000 type CpuIssueDetector struct { startTimestamp uint64 @@ -26,65 +26,61 @@ type CpuIssueDetector struct { contextString string } -func (f *CpuIssueDetector) Build() Message { - if f.startTimestamp == 0 { - return nil - } - duration := f.lastTimestamp - f.startTimestamp - timestamp := f.startTimestamp - messageID := f.startMessageID - maxRate := f.maxRate - - f.startTimestamp = 0 - f.startMessageID = 0 - f.maxRate = 0 - if duration < CPU_MIN_DURATION_TRIGGER { - return nil - } - - payload, err := json.Marshal(struct { +func (f *CpuIssueDetector) createPayload() string { + p, err := json.Marshal(struct { Duration uint64 Rate uint64 - }{duration, maxRate}) + }{f.duration(), f.maxRate}) if err != nil { log.Printf("can't marshal CpuIssue payload to json: %s", err) } + return string(p) +} +func (f *CpuIssueDetector) duration() uint64 { + return f.lastTimestamp - f.startTimestamp +} + +func (f *CpuIssueDetector) reset() { + f.startTimestamp = 0 + f.startMessageID = 0 + f.maxRate = 0 +} + +func (f *CpuIssueDetector) Build() Message { + defer f.reset() + if f.startTimestamp == 0 || f.duration() < CpuMinDurationTrigger { + return nil + } return &IssueEvent{ Type: "cpu", - Timestamp: timestamp, - MessageID: messageID, + Timestamp: f.startTimestamp, + MessageID: f.startMessageID, ContextString: f.contextString, - Payload: string(payload), + Payload: f.createPayload(), } } -func (f *CpuIssueDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (f *CpuIssueDetector) Handle(message Message, timestamp uint64) Message { switch msg := message.(type) { case *PerformanceTrack: - dt := performance.TimeDiff(timestamp, f.lastTimestamp) - if dt == 0 { - return nil // TODO: handle error + // Ignore if it's a wrong message order + if timestamp < f.lastTimestamp { + return nil } - f.lastTimestamp = timestamp - - if msg.Frames == -1 || msg.Ticks == -1 { + cpuRate := performance.CPURate(msg.Ticks, performance.TimeDiff(timestamp, f.lastTimestamp)) + // Build event if cpu issue have gone + if msg.Frames == -1 || msg.Ticks == -1 || cpuRate < CpuThreshold { return f.Build() } - - cpuRate := performance.CPURate(msg.Ticks, dt) - - if cpuRate >= CPU_THRESHOLD { - if f.startTimestamp == 0 { - f.startTimestamp = timestamp - f.startMessageID = messageID - } - if f.maxRate < cpuRate { - f.maxRate = cpuRate - } - } else { - return f.Build() + // Update values + if f.startTimestamp == 0 { + f.startTimestamp = timestamp + f.startMessageID = message.MsgID() + } + if f.maxRate < cpuRate { + f.maxRate = cpuRate } case *SetPageLocation: f.contextString = msg.URL diff --git a/backend/pkg/handlers/web/deadClick.go b/backend/pkg/handlers/web/deadClick.go index 434e6c1ce..5b02dc498 100644 --- a/backend/pkg/handlers/web/deadClick.go +++ b/backend/pkg/handlers/web/deadClick.go @@ -4,43 +4,39 @@ import ( . "openreplay/backend/pkg/messages" ) -/* - Handler name: DeadClick - Input events: SetInputTarget, - CreateDocument, - MouseClick, - SetNodeAttribute, - RemoveNodeAttribute, - CreateElementNode, - CreateTextNode, - MoveNode, - RemoveNode, - SetCSSData, - CSSInsertRule, - CSSDeleteRule - Output event: IssueEvent -*/ - -const CLICK_RELATION_TIME = 1234 +const ClickRelationTime = 1234 type DeadClickDetector struct { - lastTimestamp uint64 lastMouseClick *MouseClick + lastTimestamp uint64 lastClickTimestamp uint64 lastMessageID uint64 inputIDSet map[uint64]bool } +func NewDeadClickDetector() *DeadClickDetector { + return &DeadClickDetector{inputIDSet: make(map[uint64]bool)} +} + +func (d *DeadClickDetector) addInputID(id uint64) { + d.inputIDSet[id] = true +} + +func (d *DeadClickDetector) clearInputIDs() { + d.inputIDSet = make(map[uint64]bool) +} + func (d *DeadClickDetector) reset() { - d.inputIDSet = nil d.lastMouseClick = nil d.lastClickTimestamp = 0 d.lastMessageID = 0 + d.clearInputIDs() } -func (d *DeadClickDetector) build(timestamp uint64) Message { +func (d *DeadClickDetector) Build() Message { + // remove reset from external Build call defer d.reset() - if d.lastMouseClick == nil || d.lastClickTimestamp+CLICK_RELATION_TIME > timestamp { // reaction is instant + if d.lastMouseClick == nil || d.lastClickTimestamp+ClickRelationTime > d.lastTimestamp { // reaction is instant return nil } event := &IssueEvent{ @@ -52,42 +48,37 @@ func (d *DeadClickDetector) build(timestamp uint64) Message { return event } -func (d *DeadClickDetector) Build() Message { - return d.build(d.lastTimestamp) -} - -func (d *DeadClickDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (d *DeadClickDetector) Handle(message Message, timestamp uint64) Message { d.lastTimestamp = timestamp switch msg := message.(type) { case *SetInputTarget: - if d.inputIDSet == nil { - d.inputIDSet = make(map[uint64]bool) - } - d.inputIDSet[msg.ID] = true + d.addInputID(msg.ID) case *CreateDocument: - d.inputIDSet = nil + d.clearInputIDs() case *MouseClick: if msg.Label == "" { return nil } - event := d.build(timestamp) - if d.inputIDSet[msg.ID] { // ignore if input + isInputEvent := d.inputIDSet[msg.ID] + event := d.Build() + if isInputEvent { return event } d.lastMouseClick = msg d.lastClickTimestamp = timestamp - d.lastMessageID = messageID + d.lastMessageID = message.MsgID() return event case *SetNodeAttribute, *RemoveNodeAttribute, *CreateElementNode, *CreateTextNode, + *SetNodeFocus, *MoveNode, *RemoveNode, *SetCSSData, *CSSInsertRule, *CSSDeleteRule: - return d.build(timestamp) + return d.Build() } return nil } diff --git a/backend/pkg/handlers/web/domDrop.go b/backend/pkg/handlers/web/domDrop.go deleted file mode 100644 index 4a3ec2065..000000000 --- a/backend/pkg/handlers/web/domDrop.go +++ /dev/null @@ -1,55 +0,0 @@ -package web - -import ( - . "openreplay/backend/pkg/messages" -) - -/* - Handler name: DomDrop - Input events: CreateElementNode, - CreateTextNode, - RemoveNode - Output event: DOMDrop -*/ - -const DROP_WINDOW = 200 //ms -const CRITICAL_COUNT = 1 // Our login page contains 20. But on crush it removes only roots (1-3 nodes). -// TODO: smart detection (making whole DOM tree would eat all memory) - -type domDropDetector struct { - removedCount int - lastDropTimestamp uint64 -} - -func (dd *domDropDetector) reset() { - dd.removedCount = 0 - dd.lastDropTimestamp = 0 -} - -func (dd *domDropDetector) Handle(message Message, _ uint64, timestamp uint64) Message { - switch message.(type) { - case *CreateElementNode, - *CreateTextNode: - dd.removedCount = 0 - dd.lastDropTimestamp = 0 - case *RemoveNode: - if dd.lastDropTimestamp+DROP_WINDOW > timestamp { - dd.removedCount += 1 - } else { - dd.removedCount = 1 - } - dd.lastDropTimestamp = timestamp - } - return nil -} - -func (dd *domDropDetector) Build() Message { - defer dd.reset() - if dd.removedCount >= CRITICAL_COUNT { - domDrop := &DOMDrop{ - Timestamp: dd.lastDropTimestamp, - } - return domDrop - } - return nil -} diff --git a/backend/pkg/handlers/web/memoryIssue.go b/backend/pkg/handlers/web/memoryIssue.go index 487c396a9..4b3022d74 100644 --- a/backend/pkg/handlers/web/memoryIssue.go +++ b/backend/pkg/handlers/web/memoryIssue.go @@ -8,13 +8,6 @@ import ( . "openreplay/backend/pkg/messages" ) -/* - Handler name: MemoryIssue - Input events: PerformanceTrack, - SetPageLocation - Output event: IssueEvent -*/ - const MIN_COUNT = 3 const MEM_RATE_THRESHOLD = 300 // % to average @@ -52,7 +45,7 @@ func (f *MemoryIssueDetector) Build() Message { return event } -func (f *MemoryIssueDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (f *MemoryIssueDetector) Handle(message Message, timestamp uint64) Message { switch msg := message.(type) { case *PerformanceTrack: if f.count < MIN_COUNT { @@ -70,7 +63,7 @@ func (f *MemoryIssueDetector) Handle(message Message, messageID uint64, timestam if rate >= MEM_RATE_THRESHOLD { if f.startTimestamp == 0 { f.startTimestamp = timestamp - f.startMessageID = messageID + f.startMessageID = message.MsgID() } if f.rate < rate { f.rate = rate diff --git a/backend/pkg/handlers/web/networkIssue.go b/backend/pkg/handlers/web/networkIssue.go index 20ef412dd..9dd1f7a04 100644 --- a/backend/pkg/handlers/web/networkIssue.go +++ b/backend/pkg/handlers/web/networkIssue.go @@ -4,26 +4,19 @@ import ( . "openreplay/backend/pkg/messages" ) -/* - Handler name: NetworkIssue - Input events: ResourceTiming, - NetworkRequest - Output event: IssueEvent -*/ - type NetworkIssueDetector struct{} func (f *NetworkIssueDetector) Build() Message { return nil } -func (f *NetworkIssueDetector) Handle(message Message, messageID uint64, timestamp uint64) Message { +func (f *NetworkIssueDetector) Handle(message Message, timestamp uint64) Message { switch msg := message.(type) { case *NetworkRequest: if msg.Status >= 400 { return &IssueEvent{ Type: "bad_request", - MessageID: messageID, + MessageID: message.MsgID(), Timestamp: msg.Timestamp, ContextString: msg.URL, } diff --git a/backend/pkg/handlers/web/performanceAggregator.go b/backend/pkg/handlers/web/performanceAggregator.go index ba23978b2..babe136a5 100644 --- a/backend/pkg/handlers/web/performanceAggregator.go +++ b/backend/pkg/handlers/web/performanceAggregator.go @@ -7,13 +7,7 @@ import ( "openreplay/backend/pkg/messages/performance" ) -/* - Handler name: PerformanceAggregator - Input event: PerformanceTrack - Output event: PerformanceTrackAggr -*/ - -const AGGREGATION_WINDOW = 2 * 60 * 1000 +const AggregationWindow = 2 * 60 * 1000 type PerformanceAggregator struct { *PerformanceTrackAggr @@ -42,7 +36,7 @@ func (b *PerformanceAggregator) reset() { b.lastTimestamp = 0 } -func (b *PerformanceAggregator) Handle(message Message, _ uint64, timestamp uint64) Message { +func (b *PerformanceAggregator) Handle(message Message, timestamp uint64) Message { switch msg := message.(type) { case *PerformanceTrack: if b.PerformanceTrackAggr == nil || msg.Frames == -1 || msg.Ticks == -1 { @@ -93,7 +87,7 @@ func (b *PerformanceAggregator) Handle(message Message, _ uint64, timestamp uint b.lastTimestamp = timestamp } if b.PerformanceTrackAggr != nil && - timestamp-b.PerformanceTrackAggr.TimestampStart >= AGGREGATION_WINDOW { + timestamp-b.PerformanceTrackAggr.TimestampStart >= AggregationWindow { return b.Build() } return nil diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index 30e266194..300e38883 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,7 +2,7 @@ package messages func IsReplayerType(id int) bool { - return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id + return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 42 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id } func IsIOSType(id int) bool { @@ -11,4 +11,4 @@ func IsIOSType(id int) bool { func IsDOMType(id int) bool { return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id -} +} \ No newline at end of file diff --git a/backend/pkg/messages/iterator-ender.go b/backend/pkg/messages/iterator-ender.go index 6ca7db034..1d9063749 100644 --- a/backend/pkg/messages/iterator-ender.go +++ b/backend/pkg/messages/iterator-ender.go @@ -126,7 +126,7 @@ func (i *enderMessageIteratorImpl) preprocessing(msg Message) error { return fmt.Errorf("incorrect batch version: %d, skip current batch, info: %s", i.version, i.batchInfo.Info()) } i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = m.Timestamp + i.messageInfo.Timestamp = uint64(m.Timestamp) if m.Timestamp == 0 { i.zeroTsLog("BatchMetadata") } @@ -139,7 +139,7 @@ func (i *enderMessageIteratorImpl) preprocessing(msg Message) error { return fmt.Errorf("batchMeta found at the end of the batch, info: %s", i.batchInfo.Info()) } i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = m.Timestamp + i.messageInfo.Timestamp = uint64(m.Timestamp) if m.Timestamp == 0 { i.zeroTsLog("BatchMeta") } @@ -149,13 +149,13 @@ func (i *enderMessageIteratorImpl) preprocessing(msg Message) error { } case *Timestamp: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("Timestamp") } case *SessionStart: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("SessionStart") log.Printf("zero session start, project: %d, UA: %s, tracker: %s, info: %s", @@ -163,7 +163,7 @@ func (i *enderMessageIteratorImpl) preprocessing(msg Message) error { } case *SessionEnd: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("SessionEnd") } diff --git a/backend/pkg/messages/iterator-sink.go b/backend/pkg/messages/iterator-sink.go index be12b63eb..15ab5c077 100644 --- a/backend/pkg/messages/iterator-sink.go +++ b/backend/pkg/messages/iterator-sink.go @@ -128,7 +128,7 @@ func (i *sinkMessageIteratorImpl) preprocessing(msg Message) error { return fmt.Errorf("incorrect batch version: %d, skip current batch, info: %s", i.version, i.batchInfo.Info()) } i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = m.Timestamp + i.messageInfo.Timestamp = uint64(m.Timestamp) if m.Timestamp == 0 { i.zeroTsLog("BatchMetadata") } @@ -141,7 +141,7 @@ func (i *sinkMessageIteratorImpl) preprocessing(msg Message) error { return fmt.Errorf("batchMeta found at the end of the batch, info: %s", i.batchInfo.Info()) } i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = m.Timestamp + i.messageInfo.Timestamp = uint64(m.Timestamp) if m.Timestamp == 0 { i.zeroTsLog("BatchMeta") } @@ -151,13 +151,13 @@ func (i *sinkMessageIteratorImpl) preprocessing(msg Message) error { } case *Timestamp: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("Timestamp") } case *SessionStart: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("SessionStart") log.Printf("zero session start, project: %d, UA: %s, tracker: %s, info: %s", @@ -165,7 +165,7 @@ func (i *sinkMessageIteratorImpl) preprocessing(msg Message) error { } case *SessionEnd: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("SessionEnd") } diff --git a/backend/pkg/messages/iterator.go b/backend/pkg/messages/iterator.go index f7b014d30..4a39a7fce 100644 --- a/backend/pkg/messages/iterator.go +++ b/backend/pkg/messages/iterator.go @@ -108,11 +108,20 @@ func (i *messageIteratorImpl) Iterate(batchData []byte, batchInfo *BatchInfo) { // Set meta information for message msg.Meta().SetMeta(i.messageInfo) + // Update timestamp value for iOS message types + if IsIOSType(msgType) { + msg.Meta().Timestamp = i.getIOSTimestamp(msg) + } + // Process message i.handler(msg) } } +func (i *messageIteratorImpl) getIOSTimestamp(msg Message) uint64 { + return GetTimestamp(msg) +} + func (i *messageIteratorImpl) zeroTsLog(msgType string) { log.Printf("zero timestamp in %s, info: %s", msgType, i.batchInfo.Info()) } @@ -127,7 +136,7 @@ func (i *messageIteratorImpl) preprocessing(msg Message) error { return fmt.Errorf("incorrect batch version: %d, skip current batch, info: %s", i.version, i.batchInfo.Info()) } i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = m.Timestamp + i.messageInfo.Timestamp = uint64(m.Timestamp) if m.Timestamp == 0 { i.zeroTsLog("BatchMetadata") } @@ -140,7 +149,7 @@ func (i *messageIteratorImpl) preprocessing(msg Message) error { return fmt.Errorf("batchMeta found at the end of the batch, info: %s", i.batchInfo.Info()) } i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = m.Timestamp + i.messageInfo.Timestamp = uint64(m.Timestamp) if m.Timestamp == 0 { i.zeroTsLog("BatchMeta") } @@ -150,13 +159,13 @@ func (i *messageIteratorImpl) preprocessing(msg Message) error { } case *Timestamp: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("Timestamp") } case *SessionStart: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("SessionStart") log.Printf("zero session start, project: %d, UA: %s, tracker: %s, info: %s", @@ -164,7 +173,7 @@ func (i *messageIteratorImpl) preprocessing(msg Message) error { } case *SessionEnd: - i.messageInfo.Timestamp = int64(m.Timestamp) + i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { i.zeroTsLog("SessionEnd") } diff --git a/backend/pkg/messages/message.go b/backend/pkg/messages/message.go index 3a8e029d5..6ae02e6c5 100644 --- a/backend/pkg/messages/message.go +++ b/backend/pkg/messages/message.go @@ -8,6 +8,8 @@ type Message interface { TypeID() int Meta() *message SessionID() uint64 + MsgID() uint64 + Time() uint64 } // BatchInfo represents common information for all messages inside data batch @@ -47,7 +49,7 @@ func (b *BatchInfo) Info() string { } type message struct { - Timestamp int64 + Timestamp uint64 Index uint64 Url string batch *BatchInfo @@ -72,6 +74,14 @@ func (m *message) SessionID() uint64 { return m.batch.sessionID } +func (m *message) MsgID() uint64 { + return m.Meta().Index +} + +func (m *message) Time() uint64 { + return m.Meta().Timestamp +} + func (m *message) SetSessionID(sessID uint64) { if m.batch == nil { m.batch = &BatchInfo{} diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index a96f98de8..52fbb7a64 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -2,107 +2,105 @@ package messages const ( - MsgTimestamp = 0 - MsgSessionStart = 1 - MsgSessionEndDeprecated = 3 - MsgSetPageLocation = 4 - MsgSetViewportSize = 5 - MsgSetViewportScroll = 6 - MsgCreateDocument = 7 - MsgCreateElementNode = 8 - MsgCreateTextNode = 9 - MsgMoveNode = 10 - MsgRemoveNode = 11 - MsgSetNodeAttribute = 12 - MsgRemoveNodeAttribute = 13 - MsgSetNodeData = 14 - MsgSetCSSData = 15 - MsgSetNodeScroll = 16 - MsgSetInputTarget = 17 - MsgSetInputValue = 18 - MsgSetInputChecked = 19 - MsgMouseMove = 20 - MsgNetworkRequest = 21 - MsgConsoleLog = 22 - MsgPageLoadTiming = 23 - MsgPageRenderTiming = 24 - MsgJSExceptionDeprecated = 25 - MsgIntegrationEvent = 26 - MsgCustomEvent = 27 - MsgUserID = 28 - MsgUserAnonymousID = 29 - MsgMetadata = 30 - MsgPageEvent = 31 - MsgInputEvent = 32 - MsgClickEvent = 33 - MsgResourceEvent = 35 - MsgCSSInsertRule = 37 - MsgCSSDeleteRule = 38 - MsgFetch = 39 - MsgProfiler = 40 - MsgOTable = 41 - MsgStateAction = 42 - MsgRedux = 44 - MsgVuex = 45 - MsgMobX = 46 - MsgNgRx = 47 - MsgGraphQL = 48 - MsgPerformanceTrack = 49 - MsgStringDict = 50 - MsgSetNodeAttributeDict = 51 - MsgDOMDrop = 52 - MsgResourceTiming = 53 - MsgConnectionInformation = 54 - MsgSetPageVisibility = 55 - MsgPerformanceTrackAggr = 56 - MsgLoadFontFace = 57 - MsgSetNodeFocus = 58 - MsgLongTask = 59 - MsgSetNodeAttributeURLBased = 60 - MsgSetCSSDataURLBased = 61 - MsgIssueEventDeprecated = 62 - MsgTechnicalInfo = 63 - MsgCustomIssue = 64 - MsgAssetCache = 66 - MsgCSSInsertRuleURLBased = 67 - MsgMouseClick = 69 - MsgCreateIFrameDocument = 70 - MsgAdoptedSSReplaceURLBased = 71 - MsgAdoptedSSReplace = 72 - MsgAdoptedSSInsertRuleURLBased = 73 - MsgAdoptedSSInsertRule = 74 - MsgAdoptedSSDeleteRule = 75 - MsgAdoptedSSAddOwner = 76 - MsgAdoptedSSRemoveOwner = 77 - MsgJSException = 78 - MsgZustand = 79 - MsgBatchMeta = 80 - MsgBatchMetadata = 81 - MsgPartitionedMessage = 82 - MsgIssueEvent = 125 - MsgSessionEnd = 126 - MsgSessionSearch = 127 - MsgIOSBatchMeta = 107 - MsgIOSSessionStart = 90 - MsgIOSSessionEnd = 91 - MsgIOSMetadata = 92 - MsgIOSCustomEvent = 93 - MsgIOSUserID = 94 - MsgIOSUserAnonymousID = 95 - MsgIOSScreenChanges = 96 - MsgIOSCrash = 97 - MsgIOSScreenEnter = 98 - MsgIOSScreenLeave = 99 - MsgIOSClickEvent = 100 - MsgIOSInputEvent = 101 - MsgIOSPerformanceEvent = 102 - MsgIOSLog = 103 - MsgIOSInternalError = 104 - MsgIOSNetworkCall = 105 - MsgIOSPerformanceAggregated = 110 - MsgIOSIssueEvent = 111 + MsgTimestamp = 0 + MsgSessionStart = 1 + MsgSessionEndDeprecated = 3 + MsgSetPageLocation = 4 + MsgSetViewportSize = 5 + MsgSetViewportScroll = 6 + MsgCreateDocument = 7 + MsgCreateElementNode = 8 + MsgCreateTextNode = 9 + MsgMoveNode = 10 + MsgRemoveNode = 11 + MsgSetNodeAttribute = 12 + MsgRemoveNodeAttribute = 13 + MsgSetNodeData = 14 + MsgSetCSSData = 15 + MsgSetNodeScroll = 16 + MsgSetInputTarget = 17 + MsgSetInputValue = 18 + MsgSetInputChecked = 19 + MsgMouseMove = 20 + MsgNetworkRequest = 21 + MsgConsoleLog = 22 + MsgPageLoadTiming = 23 + MsgPageRenderTiming = 24 + MsgJSExceptionDeprecated = 25 + MsgIntegrationEvent = 26 + MsgCustomEvent = 27 + MsgUserID = 28 + MsgUserAnonymousID = 29 + MsgMetadata = 30 + MsgPageEvent = 31 + MsgInputEvent = 32 + MsgCSSInsertRule = 37 + MsgCSSDeleteRule = 38 + MsgFetch = 39 + MsgProfiler = 40 + MsgOTable = 41 + MsgStateAction = 42 + MsgRedux = 44 + MsgVuex = 45 + MsgMobX = 46 + MsgNgRx = 47 + MsgGraphQL = 48 + MsgPerformanceTrack = 49 + MsgStringDict = 50 + MsgSetNodeAttributeDict = 51 + MsgResourceTiming = 53 + MsgConnectionInformation = 54 + MsgSetPageVisibility = 55 + MsgPerformanceTrackAggr = 56 + MsgLoadFontFace = 57 + MsgSetNodeFocus = 58 + MsgLongTask = 59 + MsgSetNodeAttributeURLBased = 60 + MsgSetCSSDataURLBased = 61 + MsgIssueEventDeprecated = 62 + MsgTechnicalInfo = 63 + MsgCustomIssue = 64 + MsgAssetCache = 66 + MsgCSSInsertRuleURLBased = 67 + MsgMouseClick = 69 + MsgCreateIFrameDocument = 70 + MsgAdoptedSSReplaceURLBased = 71 + MsgAdoptedSSReplace = 72 + MsgAdoptedSSInsertRuleURLBased = 73 + MsgAdoptedSSInsertRule = 74 + MsgAdoptedSSDeleteRule = 75 + MsgAdoptedSSAddOwner = 76 + MsgAdoptedSSRemoveOwner = 77 + MsgJSException = 78 + MsgZustand = 79 + MsgBatchMeta = 80 + MsgBatchMetadata = 81 + MsgPartitionedMessage = 82 + MsgIssueEvent = 125 + MsgSessionEnd = 126 + MsgSessionSearch = 127 + MsgIOSBatchMeta = 107 + MsgIOSSessionStart = 90 + MsgIOSSessionEnd = 91 + MsgIOSMetadata = 92 + MsgIOSCustomEvent = 93 + MsgIOSUserID = 94 + MsgIOSUserAnonymousID = 95 + MsgIOSScreenChanges = 96 + MsgIOSCrash = 97 + MsgIOSScreenEnter = 98 + MsgIOSScreenLeave = 99 + MsgIOSClickEvent = 100 + MsgIOSInputEvent = 101 + MsgIOSPerformanceEvent = 102 + MsgIOSLog = 103 + MsgIOSInternalError = 104 + MsgIOSNetworkCall = 105 + MsgIOSPerformanceAggregated = 110 + MsgIOSIssueEvent = 111 ) + type Timestamp struct { message Timestamp uint64 @@ -126,22 +124,22 @@ func (msg *Timestamp) TypeID() int { type SessionStart struct { message - Timestamp uint64 - ProjectID uint64 - TrackerVersion string - RevID string - UserUUID string - UserAgent string - UserOS string - UserOSVersion string - UserBrowser string - UserBrowserVersion string - UserDevice string - UserDeviceType string + Timestamp uint64 + ProjectID uint64 + TrackerVersion string + RevID string + UserUUID string + UserAgent string + UserOS string + UserOSVersion string + UserBrowser string + UserBrowserVersion string + UserDevice string + UserDeviceType string UserDeviceMemorySize uint64 - UserDeviceHeapSize uint64 - UserCountry string - UserID string + UserDeviceHeapSize uint64 + UserCountry string + UserID string } func (msg *SessionStart) Encode() []byte { @@ -198,8 +196,8 @@ func (msg *SessionEndDeprecated) TypeID() int { type SetPageLocation struct { message - URL string - Referrer string + URL string + Referrer string NavigationStart uint64 } @@ -223,7 +221,7 @@ func (msg *SetPageLocation) TypeID() int { type SetViewportSize struct { message - Width uint64 + Width uint64 Height uint64 } @@ -269,6 +267,7 @@ func (msg *SetViewportScroll) TypeID() int { type CreateDocument struct { message + } func (msg *CreateDocument) Encode() []byte { @@ -289,11 +288,11 @@ func (msg *CreateDocument) TypeID() int { type CreateElementNode struct { message - ID uint64 + ID uint64 ParentID uint64 - index uint64 - Tag string - SVG bool + index uint64 + Tag string + SVG bool } func (msg *CreateElementNode) Encode() []byte { @@ -318,9 +317,9 @@ func (msg *CreateElementNode) TypeID() int { type CreateTextNode struct { message - ID uint64 + ID uint64 ParentID uint64 - Index uint64 + Index uint64 } func (msg *CreateTextNode) Encode() []byte { @@ -343,9 +342,9 @@ func (msg *CreateTextNode) TypeID() int { type MoveNode struct { message - ID uint64 + ID uint64 ParentID uint64 - Index uint64 + Index uint64 } func (msg *MoveNode) Encode() []byte { @@ -389,8 +388,8 @@ func (msg *RemoveNode) TypeID() int { type SetNodeAttribute struct { message - ID uint64 - Name string + ID uint64 + Name string Value string } @@ -414,7 +413,7 @@ func (msg *SetNodeAttribute) TypeID() int { type RemoveNodeAttribute struct { message - ID uint64 + ID uint64 Name string } @@ -437,7 +436,7 @@ func (msg *RemoveNodeAttribute) TypeID() int { type SetNodeData struct { message - ID uint64 + ID uint64 Data string } @@ -460,7 +459,7 @@ func (msg *SetNodeData) TypeID() int { type SetCSSData struct { message - ID uint64 + ID uint64 Data string } @@ -484,8 +483,8 @@ func (msg *SetCSSData) TypeID() int { type SetNodeScroll struct { message ID uint64 - X int64 - Y int64 + X int64 + Y int64 } func (msg *SetNodeScroll) Encode() []byte { @@ -508,7 +507,7 @@ func (msg *SetNodeScroll) TypeID() int { type SetInputTarget struct { message - ID uint64 + ID uint64 Label string } @@ -531,9 +530,9 @@ func (msg *SetInputTarget) TypeID() int { type SetInputValue struct { message - ID uint64 + ID uint64 Value string - Mask int64 + Mask int64 } func (msg *SetInputValue) Encode() []byte { @@ -556,7 +555,7 @@ func (msg *SetInputValue) TypeID() int { type SetInputChecked struct { message - ID uint64 + ID uint64 Checked bool } @@ -602,14 +601,14 @@ func (msg *MouseMove) TypeID() int { type NetworkRequest struct { message - Type string - Method string - URL string - Request string - Response string - Status uint64 + Type string + Method string + URL string + Request string + Response string + Status uint64 Timestamp uint64 - Duration uint64 + Duration uint64 } func (msg *NetworkRequest) Encode() []byte { @@ -660,15 +659,15 @@ func (msg *ConsoleLog) TypeID() int { type PageLoadTiming struct { message - RequestStart uint64 - ResponseStart uint64 - ResponseEnd uint64 + RequestStart uint64 + ResponseStart uint64 + ResponseEnd uint64 DomContentLoadedEventStart uint64 - DomContentLoadedEventEnd uint64 - LoadEventStart uint64 - LoadEventEnd uint64 - FirstPaint uint64 - FirstContentfulPaint uint64 + DomContentLoadedEventEnd uint64 + LoadEventStart uint64 + LoadEventEnd uint64 + FirstPaint uint64 + FirstContentfulPaint uint64 } func (msg *PageLoadTiming) Encode() []byte { @@ -697,8 +696,8 @@ func (msg *PageLoadTiming) TypeID() int { type PageRenderTiming struct { message - SpeedIndex uint64 - VisuallyComplete uint64 + SpeedIndex uint64 + VisuallyComplete uint64 TimeToInteractive uint64 } @@ -722,7 +721,7 @@ func (msg *PageRenderTiming) TypeID() int { type JSExceptionDeprecated struct { message - Name string + Name string Message string Payload string } @@ -748,10 +747,10 @@ func (msg *JSExceptionDeprecated) TypeID() int { type IntegrationEvent struct { message Timestamp uint64 - Source string - Name string - Message string - Payload string + Source string + Name string + Message string + Payload string } func (msg *IntegrationEvent) Encode() []byte { @@ -776,7 +775,7 @@ func (msg *IntegrationEvent) TypeID() int { type CustomEvent struct { message - Name string + Name string Payload string } @@ -841,7 +840,7 @@ func (msg *UserAnonymousID) TypeID() int { type Metadata struct { message - Key string + Key string Value string } @@ -864,23 +863,23 @@ func (msg *Metadata) TypeID() int { type PageEvent struct { message - MessageID uint64 - Timestamp uint64 - URL string - Referrer string - Loaded bool - RequestStart uint64 - ResponseStart uint64 - ResponseEnd uint64 + MessageID uint64 + Timestamp uint64 + URL string + Referrer string + Loaded bool + RequestStart uint64 + ResponseStart uint64 + ResponseEnd uint64 DomContentLoadedEventStart uint64 - DomContentLoadedEventEnd uint64 - LoadEventStart uint64 - LoadEventEnd uint64 - FirstPaint uint64 - FirstContentfulPaint uint64 - SpeedIndex uint64 - VisuallyComplete uint64 - TimeToInteractive uint64 + DomContentLoadedEventEnd uint64 + LoadEventStart uint64 + LoadEventEnd uint64 + FirstPaint uint64 + FirstContentfulPaint uint64 + SpeedIndex uint64 + VisuallyComplete uint64 + TimeToInteractive uint64 } func (msg *PageEvent) Encode() []byte { @@ -917,11 +916,11 @@ func (msg *PageEvent) TypeID() int { type InputEvent struct { message - MessageID uint64 - Timestamp uint64 - Value string + MessageID uint64 + Timestamp uint64 + Value string ValueMasked bool - Label string + Label string } func (msg *InputEvent) Encode() []byte { @@ -944,82 +943,10 @@ func (msg *InputEvent) TypeID() int { return 32 } -type ClickEvent struct { - message - MessageID uint64 - Timestamp uint64 - HesitationTime uint64 - Label string - Selector string -} - -func (msg *ClickEvent) Encode() []byte { - buf := make([]byte, 51+len(msg.Label)+len(msg.Selector)) - buf[0] = 33 - p := 1 - p = WriteUint(msg.MessageID, buf, p) - p = WriteUint(msg.Timestamp, buf, p) - p = WriteUint(msg.HesitationTime, buf, p) - p = WriteString(msg.Label, buf, p) - p = WriteString(msg.Selector, buf, p) - return buf[:p] -} - -func (msg *ClickEvent) Decode() Message { - return msg -} - -func (msg *ClickEvent) TypeID() int { - return 33 -} - -type ResourceEvent struct { - message - MessageID uint64 - Timestamp uint64 - Duration uint64 - TTFB uint64 - HeaderSize uint64 - EncodedBodySize uint64 - DecodedBodySize uint64 - URL string - Type string - Success bool - Method string - Status uint64 -} - -func (msg *ResourceEvent) Encode() []byte { - buf := make([]byte, 121+len(msg.URL)+len(msg.Type)+len(msg.Method)) - buf[0] = 35 - p := 1 - p = WriteUint(msg.MessageID, buf, p) - p = WriteUint(msg.Timestamp, buf, p) - p = WriteUint(msg.Duration, buf, p) - p = WriteUint(msg.TTFB, buf, p) - p = WriteUint(msg.HeaderSize, buf, p) - p = WriteUint(msg.EncodedBodySize, buf, p) - p = WriteUint(msg.DecodedBodySize, buf, p) - p = WriteString(msg.URL, buf, p) - p = WriteString(msg.Type, buf, p) - p = WriteBoolean(msg.Success, buf, p) - p = WriteString(msg.Method, buf, p) - p = WriteUint(msg.Status, buf, p) - return buf[:p] -} - -func (msg *ResourceEvent) Decode() Message { - return msg -} - -func (msg *ResourceEvent) TypeID() int { - return 35 -} - type CSSInsertRule struct { message - ID uint64 - Rule string + ID uint64 + Rule string Index uint64 } @@ -1043,7 +970,7 @@ func (msg *CSSInsertRule) TypeID() int { type CSSDeleteRule struct { message - ID uint64 + ID uint64 Index uint64 } @@ -1066,13 +993,13 @@ func (msg *CSSDeleteRule) TypeID() int { type Fetch struct { message - Method string - URL string - Request string - Response string - Status uint64 + Method string + URL string + Request string + Response string + Status uint64 Timestamp uint64 - Duration uint64 + Duration uint64 } func (msg *Fetch) Encode() []byte { @@ -1099,10 +1026,10 @@ func (msg *Fetch) TypeID() int { type Profiler struct { message - Name string + Name string Duration uint64 - Args string - Result string + Args string + Result string } func (msg *Profiler) Encode() []byte { @@ -1126,7 +1053,7 @@ func (msg *Profiler) TypeID() int { type OTable struct { message - Key string + Key string Value string } @@ -1170,8 +1097,8 @@ func (msg *StateAction) TypeID() int { type Redux struct { message - Action string - State string + Action string + State string Duration uint64 } @@ -1196,7 +1123,7 @@ func (msg *Redux) TypeID() int { type Vuex struct { message Mutation string - State string + State string } func (msg *Vuex) Encode() []byte { @@ -1218,7 +1145,7 @@ func (msg *Vuex) TypeID() int { type MobX struct { message - Type string + Type string Payload string } @@ -1241,8 +1168,8 @@ func (msg *MobX) TypeID() int { type NgRx struct { message - Action string - State string + Action string + State string Duration uint64 } @@ -1268,8 +1195,8 @@ type GraphQL struct { message OperationKind string OperationName string - Variables string - Response string + Variables string + Response string } func (msg *GraphQL) Encode() []byte { @@ -1293,10 +1220,10 @@ func (msg *GraphQL) TypeID() int { type PerformanceTrack struct { message - Frames int64 - Ticks int64 + Frames int64 + Ticks int64 TotalJSHeapSize uint64 - UsedJSHeapSize uint64 + UsedJSHeapSize uint64 } func (msg *PerformanceTrack) Encode() []byte { @@ -1320,7 +1247,7 @@ func (msg *PerformanceTrack) TypeID() int { type StringDict struct { message - Key uint64 + Key uint64 Value string } @@ -1343,8 +1270,8 @@ func (msg *StringDict) TypeID() int { type SetNodeAttributeDict struct { message - ID uint64 - NameKey uint64 + ID uint64 + NameKey uint64 ValueKey uint64 } @@ -1366,37 +1293,16 @@ func (msg *SetNodeAttributeDict) TypeID() int { return 51 } -type DOMDrop struct { - message - Timestamp uint64 -} - -func (msg *DOMDrop) Encode() []byte { - buf := make([]byte, 11) - buf[0] = 52 - p := 1 - p = WriteUint(msg.Timestamp, buf, p) - return buf[:p] -} - -func (msg *DOMDrop) Decode() Message { - return msg -} - -func (msg *DOMDrop) TypeID() int { - return 52 -} - type ResourceTiming struct { message - Timestamp uint64 - Duration uint64 - TTFB uint64 - HeaderSize uint64 + Timestamp uint64 + Duration uint64 + TTFB uint64 + HeaderSize uint64 EncodedBodySize uint64 DecodedBodySize uint64 - URL string - Initiator string + URL string + Initiator string } func (msg *ResourceTiming) Encode() []byte { @@ -1425,7 +1331,7 @@ func (msg *ResourceTiming) TypeID() int { type ConnectionInformation struct { message Downlink uint64 - Type string + Type string } func (msg *ConnectionInformation) Encode() []byte { @@ -1468,20 +1374,20 @@ func (msg *SetPageVisibility) TypeID() int { type PerformanceTrackAggr struct { message - TimestampStart uint64 - TimestampEnd uint64 - MinFPS uint64 - AvgFPS uint64 - MaxFPS uint64 - MinCPU uint64 - AvgCPU uint64 - MaxCPU uint64 + TimestampStart uint64 + TimestampEnd uint64 + MinFPS uint64 + AvgFPS uint64 + MaxFPS uint64 + MinCPU uint64 + AvgCPU uint64 + MaxCPU uint64 MinTotalJSHeapSize uint64 AvgTotalJSHeapSize uint64 MaxTotalJSHeapSize uint64 - MinUsedJSHeapSize uint64 - AvgUsedJSHeapSize uint64 - MaxUsedJSHeapSize uint64 + MinUsedJSHeapSize uint64 + AvgUsedJSHeapSize uint64 + MaxUsedJSHeapSize uint64 } func (msg *PerformanceTrackAggr) Encode() []byte { @@ -1515,9 +1421,9 @@ func (msg *PerformanceTrackAggr) TypeID() int { type LoadFontFace struct { message - ParentID uint64 - Family string - Source string + ParentID uint64 + Family string + Source string Descriptors string } @@ -1563,12 +1469,12 @@ func (msg *SetNodeFocus) TypeID() int { type LongTask struct { message - Timestamp uint64 - Duration uint64 - Context uint64 + Timestamp uint64 + Duration uint64 + Context uint64 ContainerType uint64 - ContainerSrc string - ContainerId string + ContainerSrc string + ContainerId string ContainerName string } @@ -1596,9 +1502,9 @@ func (msg *LongTask) TypeID() int { type SetNodeAttributeURLBased struct { message - ID uint64 - Name string - Value string + ID uint64 + Name string + Value string BaseURL string } @@ -1623,8 +1529,8 @@ func (msg *SetNodeAttributeURLBased) TypeID() int { type SetCSSDataURLBased struct { message - ID uint64 - Data string + ID uint64 + Data string BaseURL string } @@ -1648,12 +1554,12 @@ func (msg *SetCSSDataURLBased) TypeID() int { type IssueEventDeprecated struct { message - MessageID uint64 - Timestamp uint64 - Type string + MessageID uint64 + Timestamp uint64 + Type string ContextString string - Context string - Payload string + Context string + Payload string } func (msg *IssueEventDeprecated) Encode() []byte { @@ -1679,7 +1585,7 @@ func (msg *IssueEventDeprecated) TypeID() int { type TechnicalInfo struct { message - Type string + Type string Value string } @@ -1702,7 +1608,7 @@ func (msg *TechnicalInfo) TypeID() int { type CustomIssue struct { message - Name string + Name string Payload string } @@ -1746,9 +1652,9 @@ func (msg *AssetCache) TypeID() int { type CSSInsertRuleURLBased struct { message - ID uint64 - Rule string - Index uint64 + ID uint64 + Rule string + Index uint64 BaseURL string } @@ -1773,10 +1679,10 @@ func (msg *CSSInsertRuleURLBased) TypeID() int { type MouseClick struct { message - ID uint64 + ID uint64 HesitationTime uint64 - Label string - Selector string + Label string + Selector string } func (msg *MouseClick) Encode() []byte { @@ -1801,7 +1707,7 @@ func (msg *MouseClick) TypeID() int { type CreateIFrameDocument struct { message FrameID uint64 - ID uint64 + ID uint64 } func (msg *CreateIFrameDocument) Encode() []byte { @@ -1824,7 +1730,7 @@ func (msg *CreateIFrameDocument) TypeID() int { type AdoptedSSReplaceURLBased struct { message SheetID uint64 - Text string + Text string BaseURL string } @@ -1849,7 +1755,7 @@ func (msg *AdoptedSSReplaceURLBased) TypeID() int { type AdoptedSSReplace struct { message SheetID uint64 - Text string + Text string } func (msg *AdoptedSSReplace) Encode() []byte { @@ -1872,8 +1778,8 @@ func (msg *AdoptedSSReplace) TypeID() int { type AdoptedSSInsertRuleURLBased struct { message SheetID uint64 - Rule string - Index uint64 + Rule string + Index uint64 BaseURL string } @@ -1899,8 +1805,8 @@ func (msg *AdoptedSSInsertRuleURLBased) TypeID() int { type AdoptedSSInsertRule struct { message SheetID uint64 - Rule string - Index uint64 + Rule string + Index uint64 } func (msg *AdoptedSSInsertRule) Encode() []byte { @@ -1924,7 +1830,7 @@ func (msg *AdoptedSSInsertRule) TypeID() int { type AdoptedSSDeleteRule struct { message SheetID uint64 - Index uint64 + Index uint64 } func (msg *AdoptedSSDeleteRule) Encode() []byte { @@ -1947,7 +1853,7 @@ func (msg *AdoptedSSDeleteRule) TypeID() int { type AdoptedSSAddOwner struct { message SheetID uint64 - ID uint64 + ID uint64 } func (msg *AdoptedSSAddOwner) Encode() []byte { @@ -1970,7 +1876,7 @@ func (msg *AdoptedSSAddOwner) TypeID() int { type AdoptedSSRemoveOwner struct { message SheetID uint64 - ID uint64 + ID uint64 } func (msg *AdoptedSSRemoveOwner) Encode() []byte { @@ -1992,9 +1898,9 @@ func (msg *AdoptedSSRemoveOwner) TypeID() int { type JSException struct { message - Name string - Message string - Payload string + Name string + Message string + Payload string Metadata string } @@ -2020,7 +1926,7 @@ func (msg *JSException) TypeID() int { type Zustand struct { message Mutation string - State string + State string } func (msg *Zustand) Encode() []byte { @@ -2042,9 +1948,9 @@ func (msg *Zustand) TypeID() int { type BatchMeta struct { message - PageNo uint64 + PageNo uint64 FirstIndex uint64 - Timestamp int64 + Timestamp int64 } func (msg *BatchMeta) Encode() []byte { @@ -2067,11 +1973,11 @@ func (msg *BatchMeta) TypeID() int { type BatchMetadata struct { message - Version uint64 - PageNo uint64 + Version uint64 + PageNo uint64 FirstIndex uint64 - Timestamp int64 - Location string + Timestamp int64 + Location string } func (msg *BatchMetadata) Encode() []byte { @@ -2096,7 +2002,7 @@ func (msg *BatchMetadata) TypeID() int { type PartitionedMessage struct { message - PartNo uint64 + PartNo uint64 PartTotal uint64 } @@ -2119,13 +2025,13 @@ func (msg *PartitionedMessage) TypeID() int { type IssueEvent struct { message - MessageID uint64 - Timestamp uint64 - Type string + MessageID uint64 + Timestamp uint64 + Type string ContextString string - Context string - Payload string - URL string + Context string + Payload string + URL string } func (msg *IssueEvent) Encode() []byte { @@ -2152,7 +2058,7 @@ func (msg *IssueEvent) TypeID() int { type SessionEnd struct { message - Timestamp uint64 + Timestamp uint64 EncryptionKey string } @@ -2198,8 +2104,8 @@ func (msg *SessionSearch) TypeID() int { type IOSBatchMeta struct { message - Timestamp uint64 - Length uint64 + Timestamp uint64 + Length uint64 FirstIndex uint64 } @@ -2223,16 +2129,16 @@ func (msg *IOSBatchMeta) TypeID() int { type IOSSessionStart struct { message - Timestamp uint64 - ProjectID uint64 + Timestamp uint64 + ProjectID uint64 TrackerVersion string - RevID string - UserUUID string - UserOS string - UserOSVersion string - UserDevice string + RevID string + UserUUID string + UserOS string + UserOSVersion string + UserDevice string UserDeviceType string - UserCountry string + UserCountry string } func (msg *IOSSessionStart) Encode() []byte { @@ -2284,9 +2190,9 @@ func (msg *IOSSessionEnd) TypeID() int { type IOSMetadata struct { message Timestamp uint64 - Length uint64 - Key string - Value string + Length uint64 + Key string + Value string } func (msg *IOSMetadata) Encode() []byte { @@ -2311,9 +2217,9 @@ func (msg *IOSMetadata) TypeID() int { type IOSCustomEvent struct { message Timestamp uint64 - Length uint64 - Name string - Payload string + Length uint64 + Name string + Payload string } func (msg *IOSCustomEvent) Encode() []byte { @@ -2338,8 +2244,8 @@ func (msg *IOSCustomEvent) TypeID() int { type IOSUserID struct { message Timestamp uint64 - Length uint64 - Value string + Length uint64 + Value string } func (msg *IOSUserID) Encode() []byte { @@ -2363,8 +2269,8 @@ func (msg *IOSUserID) TypeID() int { type IOSUserAnonymousID struct { message Timestamp uint64 - Length uint64 - Value string + Length uint64 + Value string } func (msg *IOSUserAnonymousID) Encode() []byte { @@ -2388,11 +2294,11 @@ func (msg *IOSUserAnonymousID) TypeID() int { type IOSScreenChanges struct { message Timestamp uint64 - Length uint64 - X uint64 - Y uint64 - Width uint64 - Height uint64 + Length uint64 + X uint64 + Y uint64 + Width uint64 + Height uint64 } func (msg *IOSScreenChanges) Encode() []byte { @@ -2418,10 +2324,10 @@ func (msg *IOSScreenChanges) TypeID() int { type IOSCrash struct { message - Timestamp uint64 - Length uint64 - Name string - Reason string + Timestamp uint64 + Length uint64 + Name string + Reason string Stacktrace string } @@ -2448,9 +2354,9 @@ func (msg *IOSCrash) TypeID() int { type IOSScreenEnter struct { message Timestamp uint64 - Length uint64 - Title string - ViewName string + Length uint64 + Title string + ViewName string } func (msg *IOSScreenEnter) Encode() []byte { @@ -2475,9 +2381,9 @@ func (msg *IOSScreenEnter) TypeID() int { type IOSScreenLeave struct { message Timestamp uint64 - Length uint64 - Title string - ViewName string + Length uint64 + Title string + ViewName string } func (msg *IOSScreenLeave) Encode() []byte { @@ -2502,10 +2408,10 @@ func (msg *IOSScreenLeave) TypeID() int { type IOSClickEvent struct { message Timestamp uint64 - Length uint64 - Label string - X uint64 - Y uint64 + Length uint64 + Label string + X uint64 + Y uint64 } func (msg *IOSClickEvent) Encode() []byte { @@ -2530,11 +2436,11 @@ func (msg *IOSClickEvent) TypeID() int { type IOSInputEvent struct { message - Timestamp uint64 - Length uint64 - Value string + Timestamp uint64 + Length uint64 + Value string ValueMasked bool - Label string + Label string } func (msg *IOSInputEvent) Encode() []byte { @@ -2560,9 +2466,9 @@ func (msg *IOSInputEvent) TypeID() int { type IOSPerformanceEvent struct { message Timestamp uint64 - Length uint64 - Name string - Value uint64 + Length uint64 + Name string + Value uint64 } func (msg *IOSPerformanceEvent) Encode() []byte { @@ -2587,9 +2493,9 @@ func (msg *IOSPerformanceEvent) TypeID() int { type IOSLog struct { message Timestamp uint64 - Length uint64 - Severity string - Content string + Length uint64 + Severity string + Content string } func (msg *IOSLog) Encode() []byte { @@ -2614,8 +2520,8 @@ func (msg *IOSLog) TypeID() int { type IOSInternalError struct { message Timestamp uint64 - Length uint64 - Content string + Length uint64 + Content string } func (msg *IOSInternalError) Encode() []byte { @@ -2639,14 +2545,14 @@ func (msg *IOSInternalError) TypeID() int { type IOSNetworkCall struct { message Timestamp uint64 - Length uint64 - Duration uint64 - Headers string - Body string - URL string - Success bool - Method string - Status uint64 + Length uint64 + Duration uint64 + Headers string + Body string + URL string + Success bool + Method string + Status uint64 } func (msg *IOSNetworkCall) Encode() []byte { @@ -2676,19 +2582,19 @@ func (msg *IOSNetworkCall) TypeID() int { type IOSPerformanceAggregated struct { message TimestampStart uint64 - TimestampEnd uint64 - MinFPS uint64 - AvgFPS uint64 - MaxFPS uint64 - MinCPU uint64 - AvgCPU uint64 - MaxCPU uint64 - MinMemory uint64 - AvgMemory uint64 - MaxMemory uint64 - MinBattery uint64 - AvgBattery uint64 - MaxBattery uint64 + TimestampEnd uint64 + MinFPS uint64 + AvgFPS uint64 + MaxFPS uint64 + MinCPU uint64 + AvgCPU uint64 + MaxCPU uint64 + MinMemory uint64 + AvgMemory uint64 + MaxMemory uint64 + MinBattery uint64 + AvgBattery uint64 + MaxBattery uint64 } func (msg *IOSPerformanceAggregated) Encode() []byte { @@ -2722,11 +2628,11 @@ func (msg *IOSPerformanceAggregated) TypeID() int { type IOSIssueEvent struct { message - Timestamp uint64 - Type string + Timestamp uint64 + Type string ContextString string - Context string - Payload string + Context string + Payload string } func (msg *IOSIssueEvent) Encode() []byte { @@ -2748,3 +2654,4 @@ func (msg *IOSIssueEvent) Decode() Message { func (msg *IOSIssueEvent) TypeID() int { return 111 } + diff --git a/backend/pkg/messages/raw.go b/backend/pkg/messages/raw.go index 44f666c69..ae8b97365 100644 --- a/backend/pkg/messages/raw.go +++ b/backend/pkg/messages/raw.go @@ -42,3 +42,17 @@ func (m *RawMessage) SessionID() uint64 { } return 0 } + +func (m *RawMessage) MsgID() uint64 { + if m.meta != nil { + return m.meta.Index + } + return 0 +} + +func (m *RawMessage) Time() uint64 { + if m.meta != nil { + return m.meta.Timestamp + } + return 0 +} diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index ecc00183f..7b0fc37ea 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -6,1744 +6,1672 @@ import ( ) func DecodeTimestamp(reader BytesReader) (Message, error) { - var err error = nil - msg := &Timestamp{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &Timestamp{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err } func DecodeSessionStart(reader BytesReader) (Message, error) { - var err error = nil - msg := &SessionStart{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SessionStart{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ProjectID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TrackerVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.RevID, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserUUID, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserAgent, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserOS, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserOSVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserBrowser, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserBrowserVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDevice, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDeviceType, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDeviceMemorySize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDeviceHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.UserCountry, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserID, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSessionEndDeprecated(reader BytesReader) (Message, error) { - var err error = nil - msg := &SessionEndDeprecated{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &SessionEndDeprecated{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err } func DecodeSetPageLocation(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetPageLocation{} - if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &SetPageLocation{} + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Referrer, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.NavigationStart, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetViewportSize(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetViewportSize{} - if msg.Width, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetViewportSize{} + if msg.Width, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Height, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetViewportScroll(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetViewportScroll{} - if msg.X, err = reader.ReadInt(); err != nil { - return nil, err - } + var err error = nil + msg := &SetViewportScroll{} + if msg.X, err = reader.ReadInt(); err != nil { + return nil, err + } if msg.Y, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCreateDocument(reader BytesReader) (Message, error) { - var err error = nil - msg := &CreateDocument{} - - return msg, err + var err error = nil + msg := &CreateDocument{} + + return msg, err } func DecodeCreateElementNode(reader BytesReader) (Message, error) { - var err error = nil - msg := &CreateElementNode{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CreateElementNode{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ParentID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.index, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Tag, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.SVG, err = reader.ReadBoolean(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCreateTextNode(reader BytesReader) (Message, error) { - var err error = nil - msg := &CreateTextNode{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CreateTextNode{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ParentID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeMoveNode(reader BytesReader) (Message, error) { - var err error = nil - msg := &MoveNode{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &MoveNode{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ParentID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeRemoveNode(reader BytesReader) (Message, error) { - var err error = nil - msg := &RemoveNode{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &RemoveNode{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err } func DecodeSetNodeAttribute(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeAttribute{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeAttribute{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeRemoveNodeAttribute(reader BytesReader) (Message, error) { - var err error = nil - msg := &RemoveNodeAttribute{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &RemoveNodeAttribute{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeData(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeData{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeData{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Data, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetCSSData(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetCSSData{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetCSSData{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Data, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeScroll(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeScroll{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeScroll{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.X, err = reader.ReadInt(); err != nil { - return nil, err - } + return nil, err + } if msg.Y, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetInputTarget(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetInputTarget{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetInputTarget{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetInputValue(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetInputValue{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetInputValue{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Mask, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetInputChecked(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetInputChecked{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetInputChecked{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Checked, err = reader.ReadBoolean(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeMouseMove(reader BytesReader) (Message, error) { - var err error = nil - msg := &MouseMove{} - if msg.X, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &MouseMove{} + if msg.X, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Y, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeNetworkRequest(reader BytesReader) (Message, error) { - var err error = nil - msg := &NetworkRequest{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &NetworkRequest{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Request, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Response, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeConsoleLog(reader BytesReader) (Message, error) { - var err error = nil - msg := &ConsoleLog{} - if msg.Level, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &ConsoleLog{} + if msg.Level, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePageLoadTiming(reader BytesReader) (Message, error) { - var err error = nil - msg := &PageLoadTiming{} - if msg.RequestStart, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PageLoadTiming{} + if msg.RequestStart, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ResponseStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ResponseEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DomContentLoadedEventStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DomContentLoadedEventEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.LoadEventStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.LoadEventEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstPaint, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstContentfulPaint, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePageRenderTiming(reader BytesReader) (Message, error) { - var err error = nil - msg := &PageRenderTiming{} - if msg.SpeedIndex, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PageRenderTiming{} + if msg.SpeedIndex, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.VisuallyComplete, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TimeToInteractive, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeJSExceptionDeprecated(reader BytesReader) (Message, error) { - var err error = nil - msg := &JSExceptionDeprecated{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &JSExceptionDeprecated{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Message, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIntegrationEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IntegrationEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IntegrationEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Source, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Message, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCustomEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &CustomEvent{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &CustomEvent{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeUserID(reader BytesReader) (Message, error) { - var err error = nil - msg := &UserID{} - if msg.ID, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &UserID{} + if msg.ID, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err } func DecodeUserAnonymousID(reader BytesReader) (Message, error) { - var err error = nil - msg := &UserAnonymousID{} - if msg.ID, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &UserAnonymousID{} + if msg.ID, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err } func DecodeMetadata(reader BytesReader) (Message, error) { - var err error = nil - msg := &Metadata{} - if msg.Key, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Metadata{} + if msg.Key, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePageEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &PageEvent{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PageEvent{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Referrer, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Loaded, err = reader.ReadBoolean(); err != nil { - return nil, err - } + return nil, err + } if msg.RequestStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ResponseStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ResponseEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DomContentLoadedEventStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DomContentLoadedEventEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.LoadEventStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.LoadEventEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstPaint, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstContentfulPaint, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.SpeedIndex, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.VisuallyComplete, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TimeToInteractive, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeInputEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &InputEvent{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &InputEvent{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ValueMasked, err = reader.ReadBoolean(); err != nil { - return nil, err - } + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err -} - -func DecodeClickEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &ClickEvent{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.HesitationTime, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Selector, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err -} - -func DecodeResourceEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &ResourceEvent{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.TTFB, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.HeaderSize, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.EncodedBodySize, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.DecodedBodySize, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Success, err = reader.ReadBoolean(); err != nil { - return nil, err - } - if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCSSInsertRule(reader BytesReader) (Message, error) { - var err error = nil - msg := &CSSInsertRule{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CSSInsertRule{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Rule, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCSSDeleteRule(reader BytesReader) (Message, error) { - var err error = nil - msg := &CSSDeleteRule{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CSSDeleteRule{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeFetch(reader BytesReader) (Message, error) { - var err error = nil - msg := &Fetch{} - if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Fetch{} + if msg.Method, err = reader.ReadString(); err != nil { + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Request, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Response, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeProfiler(reader BytesReader) (Message, error) { - var err error = nil - msg := &Profiler{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Profiler{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Args, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Result, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeOTable(reader BytesReader) (Message, error) { - var err error = nil - msg := &OTable{} - if msg.Key, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &OTable{} + if msg.Key, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeStateAction(reader BytesReader) (Message, error) { - var err error = nil - msg := &StateAction{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &StateAction{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err } func DecodeRedux(reader BytesReader) (Message, error) { - var err error = nil - msg := &Redux{} - if msg.Action, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Redux{} + if msg.Action, err = reader.ReadString(); err != nil { + return nil, err + } if msg.State, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeVuex(reader BytesReader) (Message, error) { - var err error = nil - msg := &Vuex{} - if msg.Mutation, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Vuex{} + if msg.Mutation, err = reader.ReadString(); err != nil { + return nil, err + } if msg.State, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeMobX(reader BytesReader) (Message, error) { - var err error = nil - msg := &MobX{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &MobX{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeNgRx(reader BytesReader) (Message, error) { - var err error = nil - msg := &NgRx{} - if msg.Action, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &NgRx{} + if msg.Action, err = reader.ReadString(); err != nil { + return nil, err + } if msg.State, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeGraphQL(reader BytesReader) (Message, error) { - var err error = nil - msg := &GraphQL{} - if msg.OperationKind, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &GraphQL{} + if msg.OperationKind, err = reader.ReadString(); err != nil { + return nil, err + } if msg.OperationName, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Variables, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Response, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePerformanceTrack(reader BytesReader) (Message, error) { - var err error = nil - msg := &PerformanceTrack{} - if msg.Frames, err = reader.ReadInt(); err != nil { - return nil, err - } + var err error = nil + msg := &PerformanceTrack{} + if msg.Frames, err = reader.ReadInt(); err != nil { + return nil, err + } if msg.Ticks, err = reader.ReadInt(); err != nil { - return nil, err - } + return nil, err + } if msg.TotalJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.UsedJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeStringDict(reader BytesReader) (Message, error) { - var err error = nil - msg := &StringDict{} - if msg.Key, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &StringDict{} + if msg.Key, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeAttributeDict(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeAttributeDict{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeAttributeDict{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.NameKey, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ValueKey, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err -} - -func DecodeDOMDrop(reader BytesReader) (Message, error) { - var err error = nil - msg := &DOMDrop{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeResourceTiming(reader BytesReader) (Message, error) { - var err error = nil - msg := &ResourceTiming{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &ResourceTiming{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TTFB, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.HeaderSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.EncodedBodySize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DecodedBodySize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Initiator, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeConnectionInformation(reader BytesReader) (Message, error) { - var err error = nil - msg := &ConnectionInformation{} - if msg.Downlink, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &ConnectionInformation{} + if msg.Downlink, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetPageVisibility(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetPageVisibility{} - if msg.hidden, err = reader.ReadBoolean(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &SetPageVisibility{} + if msg.hidden, err = reader.ReadBoolean(); err != nil { + return nil, err + } + return msg, err } func DecodePerformanceTrackAggr(reader BytesReader) (Message, error) { - var err error = nil - msg := &PerformanceTrackAggr{} - if msg.TimestampStart, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PerformanceTrackAggr{} + if msg.TimestampStart, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.TimestampEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinTotalJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgTotalJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxTotalJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinUsedJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgUsedJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxUsedJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeLoadFontFace(reader BytesReader) (Message, error) { - var err error = nil - msg := &LoadFontFace{} - if msg.ParentID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &LoadFontFace{} + if msg.ParentID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Family, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Source, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Descriptors, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeFocus(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeFocus{} - if msg.ID, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &SetNodeFocus{} + if msg.ID, err = reader.ReadInt(); err != nil { + return nil, err + } + return msg, err } func DecodeLongTask(reader BytesReader) (Message, error) { - var err error = nil - msg := &LongTask{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &LongTask{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Context, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ContainerType, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ContainerSrc, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContainerId, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContainerName, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeAttributeURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeAttributeURLBased{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeAttributeURLBased{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetCSSDataURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetCSSDataURLBased{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetCSSDataURLBased{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Data, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIssueEventDeprecated(reader BytesReader) (Message, error) { - var err error = nil - msg := &IssueEventDeprecated{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IssueEventDeprecated{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContextString, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Context, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeTechnicalInfo(reader BytesReader) (Message, error) { - var err error = nil - msg := &TechnicalInfo{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &TechnicalInfo{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCustomIssue(reader BytesReader) (Message, error) { - var err error = nil - msg := &CustomIssue{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &CustomIssue{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAssetCache(reader BytesReader) (Message, error) { - var err error = nil - msg := &AssetCache{} - if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &AssetCache{} + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err } func DecodeCSSInsertRuleURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &CSSInsertRuleURLBased{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CSSInsertRuleURLBased{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Rule, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeMouseClick(reader BytesReader) (Message, error) { - var err error = nil - msg := &MouseClick{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &MouseClick{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.HesitationTime, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Selector, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCreateIFrameDocument(reader BytesReader) (Message, error) { - var err error = nil - msg := &CreateIFrameDocument{} - if msg.FrameID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CreateIFrameDocument{} + if msg.FrameID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSReplaceURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSReplaceURLBased{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSReplaceURLBased{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Text, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSReplace(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSReplace{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSReplace{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Text, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSInsertRuleURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSInsertRuleURLBased{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSInsertRuleURLBased{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Rule, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSInsertRule(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSInsertRule{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSInsertRule{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Rule, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSDeleteRule(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSDeleteRule{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSDeleteRule{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSAddOwner(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSAddOwner{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSAddOwner{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSRemoveOwner(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSRemoveOwner{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSRemoveOwner{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeJSException(reader BytesReader) (Message, error) { - var err error = nil - msg := &JSException{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &JSException{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Message, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Metadata, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeZustand(reader BytesReader) (Message, error) { - var err error = nil - msg := &Zustand{} - if msg.Mutation, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Zustand{} + if msg.Mutation, err = reader.ReadString(); err != nil { + return nil, err + } if msg.State, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeBatchMeta(reader BytesReader) (Message, error) { - var err error = nil - msg := &BatchMeta{} - if msg.PageNo, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &BatchMeta{} + if msg.PageNo, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.FirstIndex, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Timestamp, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeBatchMetadata(reader BytesReader) (Message, error) { - var err error = nil - msg := &BatchMetadata{} - if msg.Version, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &BatchMetadata{} + if msg.Version, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.PageNo, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstIndex, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Timestamp, err = reader.ReadInt(); err != nil { - return nil, err - } + return nil, err + } if msg.Location, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePartitionedMessage(reader BytesReader) (Message, error) { - var err error = nil - msg := &PartitionedMessage{} - if msg.PartNo, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PartitionedMessage{} + if msg.PartNo, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.PartTotal, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIssueEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IssueEvent{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IssueEvent{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContextString, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Context, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSessionEnd(reader BytesReader) (Message, error) { - var err error = nil - msg := &SessionEnd{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SessionEnd{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.EncryptionKey, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSessionSearch(reader BytesReader) (Message, error) { - var err error = nil - msg := &SessionSearch{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SessionSearch{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Partition, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSBatchMeta(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSBatchMeta{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSBatchMeta{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstIndex, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSSessionStart(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSSessionStart{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSSessionStart{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ProjectID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TrackerVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.RevID, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserUUID, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserOS, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserOSVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDevice, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDeviceType, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserCountry, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSSessionEnd(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSSessionEnd{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &IOSSessionEnd{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err } func DecodeIOSMetadata(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSMetadata{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSMetadata{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Key, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSCustomEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSCustomEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSCustomEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSUserID(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSUserID{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSUserID{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSUserAnonymousID(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSUserAnonymousID{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSUserAnonymousID{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSScreenChanges(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSScreenChanges{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSScreenChanges{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.X, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Y, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Width, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Height, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSCrash(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSCrash{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSCrash{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Reason, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Stacktrace, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSScreenEnter(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSScreenEnter{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSScreenEnter{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Title, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ViewName, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSScreenLeave(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSScreenLeave{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSScreenLeave{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Title, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ViewName, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSClickEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSClickEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSClickEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.X, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Y, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSInputEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSInputEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSInputEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ValueMasked, err = reader.ReadBoolean(); err != nil { - return nil, err - } + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSPerformanceEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSPerformanceEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSPerformanceEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSLog(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSLog{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSLog{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Severity, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Content, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSInternalError(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSInternalError{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSInternalError{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Content, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSNetworkCall(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSNetworkCall{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSNetworkCall{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Headers, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Body, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Success, err = reader.ReadBoolean(); err != nil { - return nil, err - } + return nil, err + } if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSPerformanceAggregated(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSPerformanceAggregated{} - if msg.TimestampStart, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSPerformanceAggregated{} + if msg.TimestampStart, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.TimestampEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinMemory, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgMemory, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxMemory, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinBattery, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgBattery, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxBattery, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSIssueEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSIssueEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSIssueEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContextString, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Context, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func ReadMessage(t uint64, reader BytesReader) (Message, error) { @@ -1812,10 +1740,6 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodePageEvent(reader) case 32: return DecodeInputEvent(reader) - case 33: - return DecodeClickEvent(reader) - case 35: - return DecodeResourceEvent(reader) case 37: return DecodeCSSInsertRule(reader) case 38: @@ -1844,8 +1768,6 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeStringDict(reader) case 51: return DecodeSetNodeAttributeDict(reader) - case 52: - return DecodeDOMDrop(reader) case 53: return DecodeResourceTiming(reader) case 54: diff --git a/backend/pkg/sessions/builder.go b/backend/pkg/sessions/builder.go index d21fd890a..683e4ccb5 100644 --- a/backend/pkg/sessions/builder.go +++ b/backend/pkg/sessions/builder.go @@ -10,7 +10,7 @@ import ( type builder struct { sessionID uint64 - readyMsgs []Message + readyMsgs chan Message timestamp uint64 lastMessageID uint64 lastSystemTime time.Time @@ -18,20 +18,14 @@ type builder struct { ended bool } -func NewBuilder(sessionID uint64, handlers ...handlers.MessageProcessor) *builder { +func NewBuilder(sessionID uint64, events chan Message, handlers ...handlers.MessageProcessor) *builder { return &builder{ sessionID: sessionID, processors: handlers, + readyMsgs: events, } } -func (b *builder) iterateReadyMessages(iter func(msg Message)) { - for _, readyMsg := range b.readyMsgs { - iter(readyMsg) - } - b.readyMsgs = nil -} - func (b *builder) checkSessionEnd(message Message) { if _, isEnd := message.(*IOSSessionEnd); isEnd { b.ended = true @@ -41,34 +35,31 @@ func (b *builder) checkSessionEnd(message Message) { } } -func (b *builder) handleMessage(message Message, messageID uint64) { - if messageID < b.lastMessageID { +func (b *builder) handleMessage(m Message) { + if m.MsgID() < b.lastMessageID { // May happen in case of duplicated messages in kafka (if `idempotence: false`) - log.Printf("skip message with wrong msgID, sessID: %d, msgID: %d, lastID: %d", b.sessionID, messageID, b.lastMessageID) + log.Printf("skip message with wrong msgID, sessID: %d, msgID: %d, lastID: %d", b.sessionID, m.MsgID(), b.lastMessageID) return } - timestamp := GetTimestamp(message) - if timestamp == 0 { - switch message.(type) { + if m.Time() <= 0 { + switch m.(type) { case *IssueEvent, *PerformanceTrackAggr: break default: - log.Printf("skip message with empty timestamp, sessID: %d, msgID: %d, msgType: %d", b.sessionID, messageID, message.TypeID()) + log.Printf("skip message with incorrect timestamp, sessID: %d, msgID: %d, msgType: %d", b.sessionID, m.MsgID(), m.TypeID()) } return } - if timestamp < b.timestamp { - //log.Printf("skip message with wrong timestamp, sessID: %d, msgID: %d, type: %d, msgTS: %d, lastTS: %d", b.sessionID, messageID, message.TypeID(), timestamp, b.timestamp) - } else { - b.timestamp = timestamp + if m.Time() > b.timestamp { + b.timestamp = m.Time() } - b.lastSystemTime = time.Now() + // Process current message for _, p := range b.processors { - if rm := p.Handle(message, messageID, b.timestamp); rm != nil { - rm.Meta().SetMeta(message.Meta()) - b.readyMsgs = append(b.readyMsgs, rm) + if rm := p.Handle(m, b.timestamp); rm != nil { + rm.Meta().SetMeta(m.Meta()) + b.readyMsgs <- rm } } - b.checkSessionEnd(message) + b.checkSessionEnd(m) } diff --git a/backend/pkg/sessions/builderMap.go b/backend/pkg/sessions/builderMap.go index 85e787929..9e66ce260 100644 --- a/backend/pkg/sessions/builderMap.go +++ b/backend/pkg/sessions/builderMap.go @@ -2,92 +2,98 @@ package sessions import ( "log" - "openreplay/backend/pkg/handlers" + "sync" "time" + "openreplay/backend/pkg/handlers" . "openreplay/backend/pkg/messages" ) -const FORCE_DELETE_TIMEOUT = 4 * time.Hour +const ForceDeleteTimeout = 30 * time.Minute type builderMap struct { handlersFabric func() []handlers.MessageProcessor sessions map[uint64]*builder + mutex *sync.Mutex + events chan Message + done chan struct{} } -func NewBuilderMap(handlersFabric func() []handlers.MessageProcessor) *builderMap { - return &builderMap{ +type EventBuilder interface { + Events() chan Message + HandleMessage(msg Message) + Stop() +} + +func NewBuilderMap(handlersFabric func() []handlers.MessageProcessor) EventBuilder { + b := &builderMap{ handlersFabric: handlersFabric, sessions: make(map[uint64]*builder), + mutex: &sync.Mutex{}, + events: make(chan Message, 1024*10), + done: make(chan struct{}), } -} - -func (m *builderMap) GetBuilder(sessionID uint64) *builder { - b := m.sessions[sessionID] - if b == nil { - b = NewBuilder(sessionID, m.handlersFabric()...) // Should create new instances - m.sessions[sessionID] = b - } + go b.worker() return b } -func (m *builderMap) HandleMessage(msg Message) { - sessionID := msg.SessionID() - messageID := msg.Meta().Index - b := m.GetBuilder(sessionID) - b.handleMessage(msg, messageID) +func (m *builderMap) getBuilder(sessionID uint64) *builder { + m.mutex.Lock() + b := m.sessions[sessionID] + if b == nil { + b = NewBuilder(sessionID, m.events, m.handlersFabric()...) + m.sessions[sessionID] = b + } + m.mutex.Unlock() + return b } -func (m *builderMap) ClearOldSessions() { +func (m *builderMap) Events() chan Message { + return m.events +} + +func (m *builderMap) HandleMessage(msg Message) { + m.getBuilder(msg.SessionID()).handleMessage(msg) +} + +func (m *builderMap) worker() { + tick := time.Tick(10 * time.Second) + for { + select { + case <-tick: + m.checkSessions() + case <-m.done: + return + } + } +} + +func (m *builderMap) checkSessions() { + m.mutex.Lock() deleted := 0 now := time.Now() - for id, sess := range m.sessions { - if sess.lastSystemTime.Add(FORCE_DELETE_TIMEOUT).Before(now) { - // Should delete zombie session - delete(m.sessions, id) + for sessID, b := range m.sessions { + // Check session's events + if b.ended || b.lastSystemTime.Add(ForceDeleteTimeout).Before(now) { + // Build rest of messages + for _, p := range b.processors { + if rm := p.Build(); rm != nil { + rm.Meta().SetSessionID(sessID) + m.events <- rm + } + } + delete(m.sessions, sessID) deleted++ } } + m.mutex.Unlock() if deleted > 0 { log.Printf("deleted %d sessions from message builder", deleted) } } -func (m *builderMap) iterateSessionReadyMessages(sessionID uint64, b *builder, iter func(msg Message)) { - if b.ended || b.lastSystemTime.Add(FORCE_DELETE_TIMEOUT).Before(time.Now()) { - for _, p := range b.processors { - if rm := p.Build(); rm != nil { - rm.Meta().SetSessionID(sessionID) - b.readyMsgs = append(b.readyMsgs, rm) - } - } - } - b.iterateReadyMessages(iter) - if b.ended { - delete(m.sessions, sessionID) - } -} - -func (m *builderMap) IterateReadyMessages(iter func(sessionID uint64, msg Message)) { - for sessionID, session := range m.sessions { - m.iterateSessionReadyMessages( - sessionID, - session, - func(msg Message) { - iter(sessionID, msg) - }, - ) - } -} - -func (m *builderMap) IterateSessionReadyMessages(sessionID uint64, iter func(msg Message)) { - session, ok := m.sessions[sessionID] - if !ok { - return - } - m.iterateSessionReadyMessages( - sessionID, - session, - iter, - ) +func (m *builderMap) Stop() { + m.done <- struct{}{} + m.checkSessions() + close(m.events) } diff --git a/backend/pkg/terminator/terminator.go b/backend/pkg/terminator/terminator.go new file mode 100644 index 000000000..29e106aa1 --- /dev/null +++ b/backend/pkg/terminator/terminator.go @@ -0,0 +1,22 @@ +package terminator + +import ( + "log" + "os" + "os/signal" + "syscall" +) + +// ServiceStopper is a common interface for all services +type ServiceStopper interface { + Stop() +} + +func Wait(s ServiceStopper) { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + sig := <-sigChan + log.Printf("Caught signal %v: terminating\n", sig) + s.Stop() + os.Exit(0) +} diff --git a/backend/pkg/url/url.go b/backend/pkg/url/url.go index 654e803eb..30b3f2a7e 100644 --- a/backend/pkg/url/url.go +++ b/backend/pkg/url/url.go @@ -1,7 +1,7 @@ package url import ( - _url "net/url" + "net/url" "strings" ) @@ -11,7 +11,7 @@ func DiscardURLQuery(url string) string { func GetURLParts(rawURL string) (string, string, string, error) { rawURL = strings.Replace(rawURL, "\t", "", -1) // Other chars? - u, err := _url.Parse(rawURL) + u, err := url.Parse(rawURL) if err != nil { return "", "", "", err } @@ -22,3 +22,34 @@ func GetURLParts(rawURL string) (string, string, string, error) { } return u.Host, path, u.RawQuery, nil } + +func getURLExtension(URL string) string { + u, err := url.Parse(URL) + if err != nil { + return "" + } + i := strings.LastIndex(u.Path, ".") + return u.Path[i+1:] +} + +func GetResourceType(initiator string, URL string) string { + switch initiator { + case "xmlhttprequest", "fetch": + return "fetch" + case "img": + return "img" + default: + switch getURLExtension(URL) { + case "css": + return "stylesheet" + case "js": + return "script" + case "png", "gif", "jpg", "jpeg", "svg": + return "img" + case "mp4", "mkv", "ogg", "webm", "avi", "mp3": + return "media" + default: + return "other" + } + } +} diff --git a/ee/backend/internal/db/datasaver/fts.go b/ee/backend/internal/db/datasaver/fts.go index 1ff546d27..3c049acae 100644 --- a/ee/backend/internal/db/datasaver/fts.go +++ b/ee/backend/internal/db/datasaver/fts.go @@ -18,6 +18,20 @@ type NetworkRequestFTS struct { Duration uint64 `json:"duration"` } +func WrapNetworkRequest(m *messages.NetworkRequest, projID uint32) *NetworkRequestFTS { + return &NetworkRequestFTS{ + SessionID: m.SessionID(), + ProjectID: projID, + Method: m.Method, + URL: m.URL, + Request: m.Request, + Response: m.Response, + Status: m.Status, + Timestamp: m.Timestamp, + Duration: m.Duration, + } +} + type PageEventFTS struct { SessionID uint64 `json:"session_id"` ProjectID uint32 `json:"project_id"` @@ -40,6 +54,30 @@ type PageEventFTS struct { TimeToInteractive uint64 `json:"time_to_interactive"` } +func WrapPageEvent(m *messages.PageEvent, projID uint32) *PageEventFTS { + return &PageEventFTS{ + SessionID: m.SessionID(), + ProjectID: projID, + MessageID: m.MessageID, + Timestamp: m.Timestamp, + URL: m.URL, + Referrer: m.Referrer, + Loaded: m.Loaded, + RequestStart: m.RequestStart, + ResponseStart: m.ResponseStart, + ResponseEnd: m.ResponseEnd, + DomContentLoadedEventStart: m.DomContentLoadedEventStart, + DomContentLoadedEventEnd: m.DomContentLoadedEventEnd, + LoadEventStart: m.LoadEventStart, + LoadEventEnd: m.LoadEventEnd, + FirstPaint: m.FirstPaint, + FirstContentfulPaint: m.FirstContentfulPaint, + SpeedIndex: m.SpeedIndex, + VisuallyComplete: m.VisuallyComplete, + TimeToInteractive: m.TimeToInteractive, + } +} + type GraphQLFTS struct { SessionID uint64 `json:"session_id"` ProjectID uint32 `json:"project_id"` @@ -49,68 +87,46 @@ type GraphQLFTS struct { Response string `json:"response"` } -func (s *Saver) SendToFTS(msg messages.Message, projID uint32) { +func WrapGraphQL(m *messages.GraphQL, projID uint32) *GraphQLFTS { + return &GraphQLFTS{ + SessionID: m.SessionID(), + ProjectID: projID, + OperationKind: m.OperationKind, + OperationName: m.OperationName, + Variables: m.Variables, + Response: m.Response, + } +} + +func (s *saverImpl) sendToFTS(msg messages.Message) { // Skip, if FTS is disabled if s.producer == nil { return } - var ( - event []byte - err error + projID uint32 + event []byte + err error ) + if sess, err := s.pg.Cache.GetSession(msg.SessionID()); err == nil { + projID = sess.ProjectID + } + switch m := msg.(type) { // Common case *messages.NetworkRequest: - event, err = json.Marshal(NetworkRequestFTS{ - SessionID: msg.SessionID(), - ProjectID: projID, - Method: m.Method, - URL: m.URL, - Request: m.Request, - Response: m.Response, - Status: m.Status, - Timestamp: m.Timestamp, - Duration: m.Duration, - }) + event, err = json.Marshal(WrapNetworkRequest(m, projID)) case *messages.PageEvent: - event, err = json.Marshal(PageEventFTS{ - SessionID: msg.SessionID(), - ProjectID: projID, - MessageID: m.MessageID, - Timestamp: m.Timestamp, - URL: m.URL, - Referrer: m.Referrer, - Loaded: m.Loaded, - RequestStart: m.RequestStart, - ResponseStart: m.ResponseStart, - ResponseEnd: m.ResponseEnd, - DomContentLoadedEventStart: m.DomContentLoadedEventStart, - DomContentLoadedEventEnd: m.DomContentLoadedEventEnd, - LoadEventStart: m.LoadEventStart, - LoadEventEnd: m.LoadEventEnd, - FirstPaint: m.FirstPaint, - FirstContentfulPaint: m.FirstContentfulPaint, - SpeedIndex: m.SpeedIndex, - VisuallyComplete: m.VisuallyComplete, - TimeToInteractive: m.TimeToInteractive, - }) + event, err = json.Marshal(WrapPageEvent(m, projID)) case *messages.GraphQL: - event, err = json.Marshal(GraphQLFTS{ - SessionID: msg.SessionID(), - ProjectID: projID, - OperationKind: m.OperationKind, - OperationName: m.OperationName, - Variables: m.Variables, - Response: m.Response, - }) + event, err = json.Marshal(WrapGraphQL(m, projID)) } if err != nil { log.Printf("can't marshal json for quickwit: %s", err) } else { if len(event) > 0 { - if err := s.producer.Produce(s.topic, msg.SessionID(), event); err != nil { + if err := s.producer.Produce(s.cfg.QuickwitTopic, msg.SessionID(), event); err != nil { log.Printf("can't send event to quickwit: %s", err) } } diff --git a/ee/backend/internal/db/datasaver/messages.go b/ee/backend/internal/db/datasaver/messages.go deleted file mode 100644 index 0a729ee63..000000000 --- a/ee/backend/internal/db/datasaver/messages.go +++ /dev/null @@ -1,114 +0,0 @@ -package datasaver - -import ( - "fmt" - "log" - . "openreplay/backend/pkg/messages" -) - -func (mi *Saver) InsertMessage(msg Message) error { - sessionID := msg.SessionID() - switch m := msg.(type) { - // Common - case *Metadata: - if err := mi.pg.InsertMetadata(sessionID, m); err != nil { - return fmt.Errorf("insert metadata err: %s", err) - } - return nil - case *IssueEvent: - session, err := mi.pg.Cache.GetSession(sessionID) - if err != nil { - log.Printf("can't get session info for CH: %s", err) - } else { - if err := mi.ch.InsertIssue(session, m); err != nil { - log.Printf("can't insert issue event into clickhouse: %s", err) - } - } - return mi.pg.InsertIssueEvent(sessionID, m) - //TODO: message adapter (transformer) (at the level of pkg/message) for types: *IOSMetadata, *IOSIssueEvent and others - - // Web - case *SessionStart: - return mi.pg.HandleWebSessionStart(sessionID, m) - case *SessionEnd: - return mi.pg.HandleWebSessionEnd(sessionID, m) - case *UserID: - return mi.pg.InsertWebUserID(sessionID, m) - case *UserAnonymousID: - return mi.pg.InsertWebUserAnonymousID(sessionID, m) - case *CustomEvent: - session, err := mi.pg.Cache.GetSession(sessionID) - if err != nil { - log.Printf("can't get session info for CH: %s", err) - } else { - if err := mi.ch.InsertCustom(session, m); err != nil { - log.Printf("can't insert graphQL event into clickhouse: %s", err) - } - } - return mi.pg.InsertWebCustomEvent(sessionID, m) - case *ClickEvent: - return mi.pg.InsertWebClickEvent(sessionID, m) - case *InputEvent: - return mi.pg.InsertWebInputEvent(sessionID, m) - - // Unique Web messages - case *PageEvent: - return mi.pg.InsertWebPageEvent(sessionID, m) - case *JSException: - return mi.pg.InsertWebJSException(m) - case *IntegrationEvent: - return mi.pg.InsertWebIntegrationEvent(m) - case *NetworkRequest: - session, err := mi.pg.Cache.GetSession(sessionID) - if err != nil { - log.Printf("can't get session info for CH: %s", err) - } else { - project, err := mi.pg.GetProject(session.ProjectID) - if err != nil { - log.Printf("can't get project: %s", err) - } else { - if err := mi.ch.InsertRequest(session, m, project.SaveRequestPayloads); err != nil { - log.Printf("can't insert request event into clickhouse: %s", err) - } - } - } - return mi.pg.InsertWebNetworkRequest(sessionID, m) - case *GraphQL: - session, err := mi.pg.Cache.GetSession(sessionID) - if err != nil { - log.Printf("can't get session info for CH: %s", err) - } else { - if err := mi.ch.InsertGraphQL(session, m); err != nil { - log.Printf("can't insert graphQL event into clickhouse: %s", err) - } - } - return mi.pg.InsertWebGraphQL(sessionID, m) - case *SetPageLocation: - return mi.pg.InsertSessionReferrer(sessionID, m.Referrer) - - // IOS - case *IOSSessionStart: - return mi.pg.InsertIOSSessionStart(sessionID, m) - case *IOSSessionEnd: - return mi.pg.InsertIOSSessionEnd(sessionID, m) - case *IOSUserID: - return mi.pg.InsertIOSUserID(sessionID, m) - case *IOSUserAnonymousID: - return mi.pg.InsertIOSUserAnonymousID(sessionID, m) - case *IOSCustomEvent: - return mi.pg.InsertIOSCustomEvent(sessionID, m) - case *IOSClickEvent: - return mi.pg.InsertIOSClickEvent(sessionID, m) - case *IOSInputEvent: - return mi.pg.InsertIOSInputEvent(sessionID, m) - // Unique IOS messages - case *IOSNetworkCall: - return mi.pg.InsertIOSNetworkCall(sessionID, m) - case *IOSScreenEnter: - return mi.pg.InsertIOSScreenEnter(sessionID, m) - case *IOSCrash: - return mi.pg.InsertIOSCrash(sessionID, m) - - } - return nil // "Not implemented" -} diff --git a/ee/backend/internal/db/datasaver/methods.go b/ee/backend/internal/db/datasaver/methods.go new file mode 100644 index 000000000..277fd8906 --- /dev/null +++ b/ee/backend/internal/db/datasaver/methods.go @@ -0,0 +1,83 @@ +package datasaver + +import ( + "errors" + "log" + + "openreplay/backend/pkg/db/cache" + "openreplay/backend/pkg/db/clickhouse" + "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/env" + . "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/queue" +) + +func (s *saverImpl) init() { + s.ch = clickhouse.NewConnector(env.String("CLICKHOUSE_STRING")) + if err := s.ch.Prepare(); err != nil { + log.Fatalf("can't prepare clickhouse: %s", err) + } + s.pg.Conn.SetClickHouse(s.ch) + if s.cfg.UseQuickwit { + s.producer = queue.NewProducer(s.cfg.MessageSizeLimit, true) + } +} + +func (s *saverImpl) handleExtraMessage(msg Message) error { + // Send data to quickwit + s.sendToFTS(msg) + + // Get session data + var ( + session *types.Session + err error + ) + if msg.TypeID() == MsgSessionEnd { + session, err = s.pg.GetSession(msg.SessionID()) + } else { + session, err = s.pg.Cache.GetSession(msg.SessionID()) + } + if session == nil { + if err != nil && !errors.Is(err, cache.NilSessionInCacheError) { + log.Printf("Error on session retrieving from cache: %v, SessionID: %v, Message: %v", err, msg.SessionID(), msg) + } + return err + } + + // Handle message + switch m := msg.(type) { + case *SessionEnd: + return s.ch.InsertWebSession(session) + case *PerformanceTrackAggr: + return s.ch.InsertWebPerformanceTrackAggr(session, m) + case *MouseClick: + return s.ch.InsertWebClickEvent(session, m) + case *InputEvent: + return s.ch.InsertWebInputEvent(session, m) + // Unique for Web + case *PageEvent: + return s.ch.InsertWebPageEvent(session, m) + case *ResourceTiming: + return s.ch.InsertWebResourceEvent(session, m) + case *JSException: + return s.ch.InsertWebErrorEvent(session, types.WrapJSException(m)) + case *IntegrationEvent: + return s.ch.InsertWebErrorEvent(session, types.WrapIntegrationEvent(m)) + case *IssueEvent: + return s.ch.InsertIssue(session, m) + case *CustomEvent: + return s.ch.InsertCustom(session, m) + case *NetworkRequest: + project, err := s.pg.GetProject(session.ProjectID) + if err != nil { + log.Printf("can't get project: %s", err) + } else { + if err := s.ch.InsertRequest(session, m, project.SaveRequestPayloads); err != nil { + log.Printf("can't insert request event into clickhouse: %s", err) + } + } + case *GraphQL: + return s.ch.InsertGraphQL(session, m) + } + return nil +} diff --git a/ee/backend/internal/db/datasaver/saver.go b/ee/backend/internal/db/datasaver/saver.go deleted file mode 100644 index e05e502f1..000000000 --- a/ee/backend/internal/db/datasaver/saver.go +++ /dev/null @@ -1,24 +0,0 @@ -package datasaver - -import ( - "openreplay/backend/internal/config/db" - "openreplay/backend/pkg/db/cache" - "openreplay/backend/pkg/db/clickhouse" - "openreplay/backend/pkg/queue" - "openreplay/backend/pkg/queue/types" -) - -type Saver struct { - pg *cache.PGCache - ch clickhouse.Connector - producer types.Producer - topic string -} - -func New(pg *cache.PGCache, cfg *db.Config) *Saver { - var producer types.Producer = nil - if cfg.UseQuickwit { - producer = queue.NewProducer(cfg.MessageSizeLimit, true) - } - return &Saver{pg: pg, producer: producer, topic: cfg.QuickwitTopic} -} diff --git a/ee/backend/internal/db/datasaver/stats.go b/ee/backend/internal/db/datasaver/stats.go deleted file mode 100644 index 049c319bd..000000000 --- a/ee/backend/internal/db/datasaver/stats.go +++ /dev/null @@ -1,56 +0,0 @@ -package datasaver - -import ( - "log" - "openreplay/backend/pkg/db/clickhouse" - "openreplay/backend/pkg/db/types" - "openreplay/backend/pkg/env" - "openreplay/backend/pkg/messages" -) - -func (si *Saver) InitStats() { - si.ch = clickhouse.NewConnector(env.String("CLICKHOUSE_STRING")) - if err := si.ch.Prepare(); err != nil { - log.Fatalf("Clickhouse prepare error: %v\n", err) - } - si.pg.Conn.SetClickHouse(si.ch) -} - -func (si *Saver) InsertStats(session *types.Session, msg messages.Message) error { - // Send data to quickwit - if sess, err := si.pg.Cache.GetSession(msg.SessionID()); err != nil { - si.SendToFTS(msg, 0) - } else { - si.SendToFTS(msg, sess.ProjectID) - } - - switch m := msg.(type) { - // Web - case *messages.SessionEnd: - return si.ch.InsertWebSession(session) - case *messages.PerformanceTrackAggr: - return si.ch.InsertWebPerformanceTrackAggr(session, m) - case *messages.ClickEvent: - return si.ch.InsertWebClickEvent(session, m) - case *messages.InputEvent: - return si.ch.InsertWebInputEvent(session, m) - // Unique for Web - case *messages.PageEvent: - return si.ch.InsertWebPageEvent(session, m) - case *messages.ResourceEvent: - return si.ch.InsertWebResourceEvent(session, m) - case *messages.JSException: - return si.ch.InsertWebErrorEvent(session, types.WrapJSException(m)) - case *messages.IntegrationEvent: - return si.ch.InsertWebErrorEvent(session, types.WrapIntegrationEvent(m)) - } - return nil -} - -func (si *Saver) CommitStats() error { - return si.ch.Commit() -} - -func (si *Saver) Close() error { - return si.ch.Stop() -} diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index b872adcc2..489411550 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -21,9 +21,9 @@ type Connector interface { Commit() error Stop() error InsertWebSession(session *types.Session) error - InsertWebResourceEvent(session *types.Session, msg *messages.ResourceEvent) error + InsertWebResourceEvent(session *types.Session, msg *messages.ResourceTiming) error InsertWebPageEvent(session *types.Session, msg *messages.PageEvent) error - InsertWebClickEvent(session *types.Session, msg *messages.ClickEvent) error + InsertWebClickEvent(session *types.Session, msg *messages.MouseClick) error InsertWebInputEvent(session *types.Session, msg *messages.InputEvent) error InsertWebErrorEvent(session *types.Session, msg *types.ErrorEvent) error InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error @@ -147,9 +147,7 @@ func (c *connectorImpl) worker() { for { select { case t := <-c.workerTask: - start := time.Now() c.sendBulks(t) - log.Printf("ch bulks dur: %d", time.Now().Sub(start).Milliseconds()) case <-c.done: for t := range c.workerTask { c.sendBulks(t) @@ -242,28 +240,25 @@ func (c *connectorImpl) InsertWebSession(session *types.Session) error { return nil } -func (c *connectorImpl) InsertWebResourceEvent(session *types.Session, msg *messages.ResourceEvent) error { - var method interface{} = url.EnsureMethod(msg.Method) - if method == "" { - method = nil - } - resourceType := url.EnsureType(msg.Type) +func (c *connectorImpl) InsertWebResourceEvent(session *types.Session, msg *messages.ResourceTiming) error { + msgType := url.GetResourceType(msg.Initiator, msg.URL) + resourceType := url.EnsureType(msgType) if resourceType == "" { - return fmt.Errorf("can't parse resource type, sess: %s, type: %s", session.SessionID, msg.Type) + return fmt.Errorf("can't parse resource type, sess: %d, type: %s", session.SessionID, msgType) } if err := c.batches["resources"].Append( session.SessionID, uint16(session.ProjectID), - msg.MessageID, + msg.MsgID(), datetime(msg.Timestamp), url.DiscardURLQuery(msg.URL), - msg.Type, + msgType, nullableUint16(uint16(msg.Duration)), nullableUint16(uint16(msg.TTFB)), nullableUint16(uint16(msg.HeaderSize)), nullableUint32(uint32(msg.EncodedBodySize)), nullableUint32(uint32(msg.DecodedBodySize)), - msg.Success, + msg.Duration != 0, ); err != nil { c.checkError("resources", err) return fmt.Errorf("can't append to resources batch: %s", err) @@ -298,14 +293,14 @@ func (c *connectorImpl) InsertWebPageEvent(session *types.Session, msg *messages return nil } -func (c *connectorImpl) InsertWebClickEvent(session *types.Session, msg *messages.ClickEvent) error { +func (c *connectorImpl) InsertWebClickEvent(session *types.Session, msg *messages.MouseClick) error { if msg.Label == "" { return nil } if err := c.batches["clicks"].Append( session.SessionID, uint16(session.ProjectID), - msg.MessageID, + msg.MsgID(), datetime(msg.Timestamp), msg.Label, nullableUint32(uint32(msg.HesitationTime)), diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 54f8df955..d2f684148 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -315,35 +315,6 @@ class InputEvent(Message): self.label = label -class ClickEvent(Message): - __id__ = 33 - - def __init__(self, message_id, timestamp, hesitation_time, label, selector): - self.message_id = message_id - self.timestamp = timestamp - self.hesitation_time = hesitation_time - self.label = label - self.selector = selector - - -class ResourceEvent(Message): - __id__ = 35 - - def __init__(self, message_id, timestamp, duration, ttfb, header_size, encoded_body_size, decoded_body_size, url, type, success, method, status): - self.message_id = message_id - self.timestamp = timestamp - self.duration = duration - self.ttfb = ttfb - self.header_size = header_size - self.encoded_body_size = encoded_body_size - self.decoded_body_size = decoded_body_size - self.url = url - self.type = type - self.success = success - self.method = method - self.status = status - - class CSSInsertRule(Message): __id__ = 37 @@ -470,13 +441,6 @@ class SetNodeAttributeDict(Message): self.value_key = value_key -class DOMDrop(Message): - __id__ = 52 - - def __init__(self, timestamp): - self.timestamp = timestamp - - class ResourceTiming(Message): __id__ = 53 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 0ba21ea12..cd23833da 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -321,31 +321,6 @@ class MessageCodec(Codec): label=self.read_string(reader) ) - if message_id == 33: - return ClickEvent( - message_id=self.read_uint(reader), - timestamp=self.read_uint(reader), - hesitation_time=self.read_uint(reader), - label=self.read_string(reader), - selector=self.read_string(reader) - ) - - if message_id == 35: - return ResourceEvent( - message_id=self.read_uint(reader), - timestamp=self.read_uint(reader), - duration=self.read_uint(reader), - ttfb=self.read_uint(reader), - header_size=self.read_uint(reader), - encoded_body_size=self.read_uint(reader), - decoded_body_size=self.read_uint(reader), - url=self.read_string(reader), - type=self.read_string(reader), - success=self.read_boolean(reader), - method=self.read_string(reader), - status=self.read_uint(reader) - ) - if message_id == 37: return CSSInsertRule( id=self.read_uint(reader), @@ -444,11 +419,6 @@ class MessageCodec(Codec): value_key=self.read_uint(reader) ) - if message_id == 52: - return DOMDrop( - timestamp=self.read_uint(reader) - ) - if message_id == 53: return ResourceTiming( timestamp=self.read_uint(reader), diff --git a/mobs/messages.rb b/mobs/messages.rb index ef36ebfa7..c4124226e 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -187,29 +187,6 @@ message 32, 'InputEvent', :tracker => false, :replayer => false do boolean 'ValueMasked' string 'Label' end -message 33, 'ClickEvent', :tracker => false, :replayer => false do - uint 'MessageID' - uint 'Timestamp' - uint 'HesitationTime' - string 'Label' - string 'Selector' -end -## 34 -message 35, 'ResourceEvent', :tracker => false, :replayer => false do - uint 'MessageID' - uint 'Timestamp' - uint 'Duration' - uint 'TTFB' - uint 'HeaderSize' - uint 'EncodedBodySize' - uint 'DecodedBodySize' - string 'URL' - string 'Type' - boolean 'Success' - string 'Method' - uint 'Status' -end -#36 # DEPRECATED since 4.0.2 in favor of AdoptedSSInsertRule + AdoptedSSAddOwner message 37, 'CSSInsertRule' do @@ -288,12 +265,6 @@ message 51, "SetNodeAttributeDict" do uint 'NameKey' uint 'ValueKey' end - -## 50,51 -# Doesn't work properly. TODO: Make proper detections in tracker -message 52, 'DOMDrop', :tracker => false, :replayer => false do - uint 'Timestamp' -end message 53, 'ResourceTiming', :replayer => :devtools do uint 'Timestamp' uint 'Duration' From 73d6fc9dab79207a80230dd4e06146e3fac14f60 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 9 Mar 2023 10:36:46 +0100 Subject: [PATCH 053/253] chore(helm): calice updating liveness probe Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/charts/chalice/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 1a1d496ed..383f00e06 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -121,11 +121,11 @@ affinity: {} healthCheck: livenessProbe: httpGet: - path: / + path: /signup port: 8000 - initialDelaySeconds: 100 - periodSeconds: 15 - timeoutSeconds: 10 + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 15 pvc: From 64cea03ce05bb9ab26a642e19b3b797d23e6709f Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 9 Mar 2023 10:50:17 +0100 Subject: [PATCH 054/253] fix(player): fix abs time tooltip tracking --- .../Session/Player/ClickMapRenderer/ThinPlayer.tsx | 6 +++++- .../app/components/Session_/Player/Controls/Timeline.tsx | 9 +++++++-- .../Session_/Player/Controls/components/TimeTooltip.tsx | 2 +- tracker/tracker-assist/package.json | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/Session/Player/ClickMapRenderer/ThinPlayer.tsx b/frontend/app/components/Session/Player/ClickMapRenderer/ThinPlayer.tsx index 886bb848c..1855a5046 100644 --- a/frontend/app/components/Session/Player/ClickMapRenderer/ThinPlayer.tsx +++ b/frontend/app/components/Session/Player/ClickMapRenderer/ThinPlayer.tsx @@ -25,7 +25,11 @@ function WebPlayer(props: any) { ); setContextValue({ player: WebPlayerInst, store: PlayerStore }); - return () => WebPlayerInst.clean(); + return () => { + WebPlayerInst.clean(); + // @ts-ignore + setContextValue(defaultContextValue); + } }, [session.sessionId]); const isPlayerReady = contextValue.store?.get().ready diff --git a/frontend/app/components/Session_/Player/Controls/Timeline.tsx b/frontend/app/components/Session_/Player/Controls/Timeline.tsx index 7cde52a96..678982aa9 100644 --- a/frontend/app/components/Session_/Player/Controls/Timeline.tsx +++ b/frontend/app/components/Session_/Player/Controls/Timeline.tsx @@ -81,7 +81,12 @@ function Timeline(props: IProps) { }; const showTimeTooltip = (e: React.MouseEvent) => { - if (e.target !== progressRef.current && e.target !== timelineRef.current) { + if ( + e.target !== progressRef.current + && e.target !== timelineRef.current + // @ts-ignore black magic + && !progressRef.current.contains(e.target) + ) { return props.tooltipVisible && hideTimeTooltip(); } @@ -91,7 +96,7 @@ function Timeline(props: IProps) { const timeLineTooltip = { time: Duration.fromMillis(time).toFormat(`mm:ss`), timeStr, - offset: e.nativeEvent.offsetX, + offset: e.nativeEvent.pageX, isVisible: true, }; diff --git a/frontend/app/components/Session_/Player/Controls/components/TimeTooltip.tsx b/frontend/app/components/Session_/Player/Controls/components/TimeTooltip.tsx index e47593b97..5f746f336 100644 --- a/frontend/app/components/Session_/Player/Controls/components/TimeTooltip.tsx +++ b/frontend/app/components/Session_/Player/Controls/components/TimeTooltip.tsx @@ -22,7 +22,7 @@ function TimeTooltip({ className={stl.timeTooltip} style={{ top: 0, - left: offset, + left: `calc(${offset}px - 0.5rem)`, display: isVisible ? 'block' : 'none', transform: 'translate(-50%, -110%)', whiteSpace: 'nowrap', diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 4277e2e36..4c8c98d53 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -31,7 +31,7 @@ "socket.io-client": "^4.4.1" }, "peerDependencies": { - "@openreplay/tracker": ">=3.6.0" + "@openreplay/tracker": ">=5.0.0" }, "devDependencies": { "@openreplay/tracker": "file:../tracker", From fc8604db92c654f2a2819a0c8fd4ffe374d826dc Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 9 Mar 2023 13:01:39 +0100 Subject: [PATCH 055/253] chore(build): Skip confirmation for signing Signed-off-by: rjshrjndrn --- scripts/helmcharts/build_deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/helmcharts/build_deploy.sh b/scripts/helmcharts/build_deploy.sh index c922878d4..f37f26c94 100644 --- a/scripts/helmcharts/build_deploy.sh +++ b/scripts/helmcharts/build_deploy.sh @@ -10,6 +10,7 @@ docker rmi alpine || true # Signing image # cosign sign --key awskms:///alias/openreplay-container-sign image_url:tag +export COSIGN_YES=true # Skip confirmation export SIGN_IMAGE=1 export PUSH_IMAGE=1 export AWS_DEFAULT_REGION="eu-central-1" From d95e7c62118264fa41733b09879543986cabc682 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 9 Mar 2023 14:58:32 +0100 Subject: [PATCH 056/253] feat(assist): allow maxHttpBufferSize in bytes --- assist/servers/websocket.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assist/servers/websocket.js b/assist/servers/websocket.js index f5d029bc2..0258960d0 100644 --- a/assist/servers/websocket.js +++ b/assist/servers/websocket.js @@ -26,7 +26,8 @@ const debug = process.env.debug === "1"; const createSocketIOServer = function (server, prefix) { io = _io(server, { - maxHttpBufferSize: (parseInt(process.env.maxHttpBufferSize) || 5) * 1e6, + // maxHttpBufferSize: (parseInt(process.env.maxHttpBufferSize) || 5) * 1e6, + maxHttpBufferSize: parseInt(process.env.maxHttpBufferSizeBytes), cors: { origin: "*", methods: ["GET", "POST", "PUT"] From f8311b8b381dc0dee9a7ccb6d82ff96b601d298c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 9 Mar 2023 15:09:31 +0100 Subject: [PATCH 057/253] chore(actions): changed actions --- .github/workflows/assist-ee.yaml | 4 ++-- .github/workflows/assist.yaml | 4 ++-- .github/workflows/sourcemaps-reader.yaml | 8 ++++---- assist/Dockerfile | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/assist-ee.yaml b/.github/workflows/assist-ee.yaml index f2fa32ba2..44fcb5650 100644 --- a/.github/workflows/assist-ee.yaml +++ b/.github/workflows/assist-ee.yaml @@ -101,9 +101,9 @@ jobs: cat /tmp/image_override.yaml # Deploy command - mv openreplay/charts/{ingress-nginx,chalice,quickwit} /tmp + mv openreplay/charts/{ingress-nginx,assist,quickwit} /tmp rm -rf openreplay/charts/* - mv /tmp/{ingress-nginx,chalice,quickwit} openreplay/charts/ + mv /tmp/{ingress-nginx,assist,quickwit} openreplay/charts/ helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f - env: DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} diff --git a/.github/workflows/assist.yaml b/.github/workflows/assist.yaml index 67bfed543..37582d7d0 100644 --- a/.github/workflows/assist.yaml +++ b/.github/workflows/assist.yaml @@ -100,9 +100,9 @@ jobs: cat /tmp/image_override.yaml # Deploy command - mv openreplay/charts/{ingress-nginx,chalice,quickwit} /tmp + mv openreplay/charts/{ingress-nginx,assist,quickwit} /tmp rm -rf openreplay/charts/* - mv /tmp/{ingress-nginx,chalice,quickwit} openreplay/charts/ + mv /tmp/{ingress-nginx,assist,quickwit} openreplay/charts/ helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks --kube-version=$k_version | kubectl apply -f - env: DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} diff --git a/.github/workflows/sourcemaps-reader.yaml b/.github/workflows/sourcemaps-reader.yaml index f0059da40..5b7c11d01 100644 --- a/.github/workflows/sourcemaps-reader.yaml +++ b/.github/workflows/sourcemaps-reader.yaml @@ -1,4 +1,4 @@ -# This action will push the chalice changes to aws +# This action will push the sourcemapreader changes to aws on: workflow_dispatch: push: @@ -83,13 +83,13 @@ jobs: sed -i "s/domainName: \"\"/domainName: \"${{ secrets.OSS_DOMAIN_NAME }}\"/g" vars.yaml # Update changed image tag - sed -i "/chalice/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + sed -i "/sourcemapreader/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml cat /tmp/image_override.yaml # Deploy command - mv openreplay/charts/{ingress-nginx,chalice,quickwit} /tmp + mv openreplay/charts/{ingress-nginx,sourcemapreader,quickwit} /tmp rm -rf openreplay/charts/* - mv /tmp/{ingress-nginx,chalice,quickwit} openreplay/charts/ + mv /tmp/{ingress-nginx,sourcemapreader,quickwit} openreplay/charts/ helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks | kubectl apply -n app -f - env: DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} diff --git a/assist/Dockerfile b/assist/Dockerfile index edbaae03c..84b54c906 100644 --- a/assist/Dockerfile +++ b/assist/Dockerfile @@ -18,4 +18,4 @@ USER 1001 ADD --chown=1001 https://static.openreplay.com/geoip/GeoLite2-Country.mmdb $MAXMINDDB_FILE ENTRYPOINT ["/sbin/tini", "--"] -CMD npm start +CMD npm start \ No newline at end of file From 1d7a5446cc7d042c08b0234d05132e643c906242 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 9 Mar 2023 15:43:29 +0100 Subject: [PATCH 058/253] feat(assist): accept maxHttpBufferSize as a float --- assist/servers/websocket.js | 3 +-- ee/assist/servers/websocket-cluster.js | 4 ++-- ee/assist/servers/websocket.js | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/assist/servers/websocket.js b/assist/servers/websocket.js index 0258960d0..4c4a657bb 100644 --- a/assist/servers/websocket.js +++ b/assist/servers/websocket.js @@ -26,8 +26,7 @@ const debug = process.env.debug === "1"; const createSocketIOServer = function (server, prefix) { io = _io(server, { - // maxHttpBufferSize: (parseInt(process.env.maxHttpBufferSize) || 5) * 1e6, - maxHttpBufferSize: parseInt(process.env.maxHttpBufferSizeBytes), + maxHttpBufferSize: (parseFloat(process.env.maxHttpBufferSize) || 5) * 1e6, cors: { origin: "*", methods: ["GET", "POST", "PUT"] diff --git a/ee/assist/servers/websocket-cluster.js b/ee/assist/servers/websocket-cluster.js index e129bfcb6..a1f389685 100644 --- a/ee/assist/servers/websocket-cluster.js +++ b/ee/assist/servers/websocket-cluster.js @@ -34,7 +34,7 @@ const debug = process.env.debug === "1"; const createSocketIOServer = function (server, prefix) { if (process.env.uws !== "true") { io = _io(server, { - maxHttpBufferSize: (parseInt(process.env.maxHttpBufferSize) || 5) * 1e6, + maxHttpBufferSize: (parseFloat(process.env.maxHttpBufferSize) || 5) * 1e6, cors: { origin: "*", methods: ["GET", "POST", "PUT"] @@ -43,7 +43,7 @@ const createSocketIOServer = function (server, prefix) { }); } else { io = new _io.Server({ - maxHttpBufferSize: (parseInt(process.env.maxHttpBufferSize) || 5) * 1e6, + maxHttpBufferSize: (parseFloat(process.env.maxHttpBufferSize) || 5) * 1e6, cors: { origin: "*", methods: ["GET", "POST", "PUT"] diff --git a/ee/assist/servers/websocket.js b/ee/assist/servers/websocket.js index c906b5987..330361df3 100644 --- a/ee/assist/servers/websocket.js +++ b/ee/assist/servers/websocket.js @@ -29,7 +29,7 @@ const debug = process.env.debug === "1"; const createSocketIOServer = function (server, prefix) { if (process.env.uws !== "true") { io = _io(server, { - maxHttpBufferSize: (parseInt(process.env.maxHttpBufferSize) || 5) * 1e6, + maxHttpBufferSize: (parseFloat(process.env.maxHttpBufferSize) || 5) * 1e6, cors: { origin: "*", methods: ["GET", "POST", "PUT"] @@ -38,7 +38,7 @@ const createSocketIOServer = function (server, prefix) { }); } else { io = new _io.Server({ - maxHttpBufferSize: (parseInt(process.env.maxHttpBufferSize) || 5) * 1e6, + maxHttpBufferSize: (parseFloat(process.env.maxHttpBufferSize) || 5) * 1e6, cors: { origin: "*", methods: ["GET", "POST", "PUT"] From 8beae3188915a525e7fc2a23971deb42517961b1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 9 Mar 2023 18:02:46 +0100 Subject: [PATCH 059/253] feat(chalice): pg execute wrapper to handle all query failures --- api/chalicelib/utils/pg_client.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index 69a5b5a8b..4cfd8b0e3 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -111,6 +111,8 @@ class PostgresClient: def __enter__(self): if self.cursor is None: self.cursor = self.connection.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + self.cursor.cursor_execute = self.cursor.execute + self.cursor.execute = self.__execute self.cursor.recreate = self.recreate_cursor return self.cursor @@ -136,6 +138,17 @@ class PostgresClient: and not self.unlimited_query: postgreSQL_pool.putconn(self.connection) + def __execute(self, query, vars=None): + try: + result = self.cursor.cursor_execute(query=query, vars=vars) + except psycopg2.Error as error: + logging.error(f"!!! Error of type:{type(error)} while executing query:") + logging.error(query) + logging.info("starting rollback to allow future execution") + self.connection.rollback() + raise error + return result + def recreate_cursor(self, rollback=False): if rollback: try: From ded9a88fe97691d753bd66bced9c41e8b2d984fd Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 10 Mar 2023 09:56:31 +0100 Subject: [PATCH 060/253] feat(backend): implemented db inserts for InputDuration and MouseThrashing --- backend/internal/db/datasaver/saver.go | 4 +++ backend/pkg/db/cache/messages-web.go | 18 ++++++++++ backend/pkg/db/postgres/bulks.go | 14 +++++++- backend/pkg/db/postgres/messages-web.go | 44 +++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index 92dbff958..1a017fa6f 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -77,6 +77,10 @@ func (s *saverImpl) handleMessage(msg Message) error { return s.pg.InsertWebJSException(m) case *IntegrationEvent: return s.pg.InsertWebIntegrationEvent(m) + case *InputChange: + return s.pg.InsertWebInputDuration(m) + case *MouseThrashing: + return s.pg.InsertMouseThrashing(m) case *IOSSessionStart: return s.pg.InsertIOSSessionStart(m) case *IOSSessionEnd: diff --git a/backend/pkg/db/cache/messages-web.go b/backend/pkg/db/cache/messages-web.go index 0a870e5a2..58c703318 100644 --- a/backend/pkg/db/cache/messages-web.go +++ b/backend/pkg/db/cache/messages-web.go @@ -180,3 +180,21 @@ func (c *PGCache) InsertWebInputEvent(e *InputEvent) error { } return c.Conn.InsertWebInputEvent(sessionID, session.ProjectID, e) } + +func (c *PGCache) InsertWebInputDuration(e *InputChange) error { + sessionID := e.SessionID() + session, err := c.Cache.GetSession(sessionID) + if err != nil { + return err + } + return c.Conn.InsertWebInputDuration(sessionID, session.ProjectID, e) +} + +func (c *PGCache) InsertMouseThrashing(e *MouseThrashing) error { + sessionID := e.SessionID() + session, err := c.Cache.GetSession(sessionID) + if err != nil { + return err + } + return c.Conn.InsertMouseThrashing(sessionID, session.ProjectID, e) +} diff --git a/backend/pkg/db/postgres/bulks.go b/backend/pkg/db/postgres/bulks.go index 27ab2cafd..0dcfca646 100644 --- a/backend/pkg/db/postgres/bulks.go +++ b/backend/pkg/db/postgres/bulks.go @@ -9,7 +9,7 @@ type bulksTask struct { } func NewBulksTask() *bulksTask { - return &bulksTask{bulks: make([]Bulk, 0, 14)} + return &bulksTask{bulks: make([]Bulk, 0, 15)} } type BulkSet struct { @@ -19,6 +19,7 @@ type BulkSet struct { customEvents Bulk webPageEvents Bulk webInputEvents Bulk + webInputDurations Bulk webGraphQL Bulk webErrors Bulk webErrorEvents Bulk @@ -57,6 +58,8 @@ func (conn *BulkSet) Get(name string) Bulk { return conn.webPageEvents case "webInputEvents": return conn.webInputEvents + case "webInputDurations": + return conn.webInputDurations case "webGraphQL": return conn.webGraphQL case "webErrors": @@ -126,6 +129,14 @@ func (conn *BulkSet) initBulks() { if err != nil { log.Fatalf("can't create webPageEvents bulk: %s", err) } + conn.webInputDurations, err = NewBulk(conn.c, + "events.inputs", + "(session_id, message_id, timestamp, value, label, hesitation, duration)", + "($%d, $%d, $%d, LEFT($%d, 2000), NULLIF(LEFT($%d, 2000),''), $%d, $%d)", + 7, 200) + if err != nil { + log.Fatalf("can't create webPageEvents bulk: %s", err) + } conn.webGraphQL, err = NewBulk(conn.c, "events.graphql", "(session_id, timestamp, message_id, name, request_body, response_body)", @@ -209,6 +220,7 @@ func (conn *BulkSet) Send() { newTask.bulks = append(newTask.bulks, conn.customEvents) newTask.bulks = append(newTask.bulks, conn.webPageEvents) newTask.bulks = append(newTask.bulks, conn.webInputEvents) + newTask.bulks = append(newTask.bulks, conn.webInputDurations) newTask.bulks = append(newTask.bulks, conn.webGraphQL) newTask.bulks = append(newTask.bulks, conn.webErrors) newTask.bulks = append(newTask.bulks, conn.webErrorEvents) diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index 9251a4924..2037612f8 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -1,7 +1,10 @@ package postgres import ( + "encoding/hex" + "hash/fnv" "log" + "strconv" "openreplay/backend/pkg/db/types" . "openreplay/backend/pkg/messages" @@ -89,6 +92,24 @@ func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *Inp return nil } +func (conn *Conn) InsertWebInputDuration(sessionID uint64, projectID uint32, e *InputChange) error { + // Debug log + log.Printf("new InputDuration event: %v", e) + if e.Label == "" { + return nil + } + value := &e.Value + if e.ValueMasked { + value = nil + } + if err := conn.bulks.Get("webInputDurations").Append(sessionID, truncSqIdx(e.ID), e.Timestamp, value, e.Label, e.HesitationTime, e.InputDuration); err != nil { + log.Printf("insert web input event err: %s", err) + } + conn.updateSessionEvents(sessionID, 1, 0) + conn.insertAutocompleteValue(sessionID, projectID, "INPUT", e.Label) + return nil +} + func (conn *Conn) InsertWebErrorEvent(sessionID uint64, projectID uint32, e *types.ErrorEvent) error { errorID := e.ID(projectID) if err := conn.bulks.Get("webErrors").Append(errorID, projectID, e.Source, e.Name, e.Message, e.Payload); err != nil { @@ -145,3 +166,26 @@ func (conn *Conn) InsertSessionReferrer(sessionID uint64, referrer string) error WHERE session_id = $3 AND referrer IS NULL`, referrer, url.DiscardURLQuery(referrer), sessionID) } + +func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *MouseThrashing) error { + // Debug log + log.Printf("new MouseThrashing event: %v", e) + // + issueID := mouseThrashingID(projectID, sessionID, e.Timestamp) + if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_trashing", e.Url); err != nil { + log.Printf("insert web issue err: %s", err) + } + if err := conn.bulks.Get("webIssueEvents").Append(sessionID, issueID, e.Timestamp, truncSqIdx(e.MsgID()), nil); err != nil { + log.Printf("insert web issue event err: %s", err) + } + conn.updateSessionIssues(sessionID, 0, 50) + return nil +} + +func mouseThrashingID(projectID uint32, sessID, ts uint64) string { + hash := fnv.New128a() + hash.Write([]byte("mouse_trashing")) + hash.Write([]byte(strconv.FormatUint(sessID, 10))) + hash.Write([]byte(strconv.FormatUint(ts, 10))) + return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) +} From 3d7ab8b31d9de399acac06144bf9670e778f0996 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 10 Mar 2023 12:19:32 +0100 Subject: [PATCH 061/253] feat(backend): added new message types to db message filter --- backend/cmd/db/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index ae1228cc3..6086d9491 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -38,7 +38,7 @@ func main() { messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument, messages.MsgMouseClick, messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming, - messages.MsgInputEvent, messages.MsgPageEvent} + messages.MsgInputEvent, messages.MsgPageEvent, messages.MsgMouseThrashing, messages.MsgInputChange} // Init consumer consumer := queue.NewConsumer( From c1ec53c39eb82b7e5fac547bbd9ea184e0b06fce Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 10 Mar 2023 14:48:06 +0100 Subject: [PATCH 062/253] feat(chalice): health-check for kafka --- ee/api/chalicelib/core/health.py | 16 +++++++--------- ee/api/requirements.txt | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 6139992a0..784dfceab 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -2,8 +2,8 @@ from urllib.parse import urlparse import redis import requests +from confluent_kafka.admin import AdminClient from decouple import config -import kafka from chalicelib.utils import pg_client, ch_client @@ -229,12 +229,11 @@ def __check_kafka(): return fail_response try: - # consumer = kafka.KafkaConsumer(group_id='test', bootstrap_servers=[config("KAFKA_SERVERS")]) - # topics = consumer.topics() - # - # if not topics: - # raise RuntimeError() - client =kafka.KafkaClient(bootstrap_servers=[config("KAFKA_SERVERS")]) + a = AdminClient({'bootstrap.servers': config("KAFKA_SERVERS"), "socket.connection.setup.timeout.ms": 3000}) + topics = a.list_topics().topics + if not topics: + raise Exception('topics not found') + except Exception as e: print("!! Issue getting kafka-health response") print(str(e)) @@ -243,6 +242,5 @@ def __check_kafka(): return { "health": True, - "details": {"version": r.execute_command('INFO')['redis_version']} + "details": {} } - diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 3d97c63e6..9ce06fe06 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -19,4 +19,4 @@ python3-saml==1.15.0 python-multipart==0.0.5 redis==4.5.1 -kafka-python==2.0.2 \ No newline at end of file +confluent-kafka==2.0.2 \ No newline at end of file From d58b3181dce2795f606adfeec112352b90013145 Mon Sep 17 00:00:00 2001 From: Dayan Graham Date: Thu, 9 Mar 2023 15:36:38 +0000 Subject: [PATCH 063/253] change(tracker): webworker has a bug where after being foregrounded (on mobile especially), if the writer or sender is not present, it will throw an error which will bubble up and crash the entire app. Instead, log a debug message and allow the writer / sender to reinit --- tracker/tracker/src/webworker/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tracker/tracker/src/webworker/index.ts b/tracker/tracker/src/webworker/index.ts index 656796b98..f192730b0 100644 --- a/tracker/tracker/src/webworker/index.ts +++ b/tracker/tracker/src/webworker/index.ts @@ -136,13 +136,15 @@ self.onmessage = ({ data }: any): any => { if (data.type === 'auth') { if (!sender) { - throw new Error('WebWorker: sender not initialised. Received auth.') + console.debug('WebWorker: sender not initialised. Received auth.') } if (!writer) { - throw new Error('WebWorker: writer not initialised. Received auth.') + console.debug('WebWorker: writer not initialised. Received auth.') + } + if (sender && writer) { + sender.authorise(data.token) + data.beaconSizeLimit && writer.setBeaconSizeLimit(data.beaconSizeLimit) } - sender.authorise(data.token) - data.beaconSizeLimit && writer.setBeaconSizeLimit(data.beaconSizeLimit) return } } From ace7b3ad389dcad347a4553073415a8f5d4f31e0 Mon Sep 17 00:00:00 2001 From: Dayan Graham Date: Thu, 9 Mar 2023 16:41:32 +0000 Subject: [PATCH 064/253] Return upon uninitialised sender or writer --- tracker/tracker/src/webworker/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tracker/tracker/src/webworker/index.ts b/tracker/tracker/src/webworker/index.ts index f192730b0..4f0f75ad8 100644 --- a/tracker/tracker/src/webworker/index.ts +++ b/tracker/tracker/src/webworker/index.ts @@ -137,14 +137,16 @@ self.onmessage = ({ data }: any): any => { if (data.type === 'auth') { if (!sender) { console.debug('WebWorker: sender not initialised. Received auth.') + return } + if (!writer) { console.debug('WebWorker: writer not initialised. Received auth.') + return } - if (sender && writer) { - sender.authorise(data.token) - data.beaconSizeLimit && writer.setBeaconSizeLimit(data.beaconSizeLimit) - } + + sender.authorise(data.token) + data.beaconSizeLimit && writer.setBeaconSizeLimit(data.beaconSizeLimit) return } } From 7379b5b9ebdbb6596b50a23a83858ba65dc38392 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Mar 2023 09:53:13 +0100 Subject: [PATCH 065/253] change(tracker): ignore comment nodes --- tracker/tracker/src/main/app/guards.ts | 4 ++++ tracker/tracker/src/main/app/observer/observer.ts | 12 +++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tracker/tracker/src/main/app/guards.ts b/tracker/tracker/src/main/app/guards.ts index c58d03a74..5379f9387 100644 --- a/tracker/tracker/src/main/app/guards.ts +++ b/tracker/tracker/src/main/app/guards.ts @@ -11,6 +11,10 @@ export function isElementNode(node: Node): node is Element { return node.nodeType === Node.ELEMENT_NODE } +export function isCommentNode(node: Node): node is Comment { + return node.nodeType === Node.COMMENT_NODE +} + export function isTextNode(node: Node): node is Text { return node.nodeType === Node.TEXT_NODE } diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts index c13739622..9e93dde2d 100644 --- a/tracker/tracker/src/main/app/observer/observer.ts +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -10,9 +10,19 @@ import { RemoveNode, } from '../messages.gen.js' import App from '../index.js' -import { isRootNode, isTextNode, isElementNode, isSVGElement, hasTag } from '../guards.js' +import { + isRootNode, + isTextNode, + isElementNode, + isSVGElement, + hasTag, + isCommentNode, +} from '../guards.js' function isIgnored(node: Node): boolean { + if (isCommentNode(node)) { + return true + } if (isTextNode(node)) { return false } From 0dddaecd67cda4272858617d6075b90fe8d94253 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Mar 2023 10:36:39 +0100 Subject: [PATCH 066/253] change(tracker): restart worker if its dead; fix zustand peer d version --- tracker/tracker-zustand/package.json | 4 ++-- tracker/tracker/CHANGELOG.md | 1 + tracker/tracker/src/webworker/index.ts | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tracker/tracker-zustand/package.json b/tracker/tracker-zustand/package.json index cae316ad6..67a4a812a 100644 --- a/tracker/tracker-zustand/package.json +++ b/tracker/tracker-zustand/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-zustand", "description": "Tracker plugin for Zustand state recording", - "version": "1.0.2", + "version": "1.0.3", "keywords": [ "zustand", "state", @@ -24,7 +24,7 @@ }, "dependencies": {}, "peerDependencies": { - "@openreplay/tracker": "^4.0.1" + "@openreplay/tracker": ">=4.0.1" }, "devDependencies": { "@openreplay/tracker": "^4.0.1", diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index 6a8e25690..6908f5bd4 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,5 +1,6 @@ ## 5.0.1 +- Re-init worker after device sleep/hybernation - Default text input mode is now Obscured - Use `@medv/finder` instead of our own implementation of `getSelector` for better clickmaps experience diff --git a/tracker/tracker/src/webworker/index.ts b/tracker/tracker/src/webworker/index.ts index 4f0f75ad8..5afa42cee 100644 --- a/tracker/tracker/src/webworker/index.ts +++ b/tracker/tracker/src/webworker/index.ts @@ -137,11 +137,13 @@ self.onmessage = ({ data }: any): any => { if (data.type === 'auth') { if (!sender) { console.debug('WebWorker: sender not initialised. Received auth.') + initiateRestart() return } - + if (!writer) { console.debug('WebWorker: writer not initialised. Received auth.') + initiateRestart() return } From 4480a5d93a2b0f7a57d3ebfa2119f7c20e15a2c8 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 13 Mar 2023 11:11:21 +0100 Subject: [PATCH 067/253] fix(backend): fixed typo in issue type --- backend/pkg/db/postgres/messages-web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index 2037612f8..a15c03335 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -172,7 +172,7 @@ func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *Mo log.Printf("new MouseThrashing event: %v", e) // issueID := mouseThrashingID(projectID, sessionID, e.Timestamp) - if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_trashing", e.Url); err != nil { + if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_thrashing", e.Url); err != nil { log.Printf("insert web issue err: %s", err) } if err := conn.bulks.Get("webIssueEvents").Append(sessionID, issueID, e.Timestamp, truncSqIdx(e.MsgID()), nil); err != nil { From c945d47a7c0cf46684cf9f81ca9180f9aa2d7c68 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Mar 2023 11:43:02 +0100 Subject: [PATCH 068/253] change(tracker): tracker 5.0.1 --- tracker/tracker/package.json | 2 +- tracker/tracker/src/main/modules/mouse.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index a67073cc9..c22de968a 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "5.0.1-beta.2", + "version": "5.0.1", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 155a14a8d..fef7b7754 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -155,7 +155,7 @@ export default function (app: App): void { id, mouseTarget === target ? Math.round(performance.now() - mouseTargetTime) : 0, getTargetLabel(target), - getSelector(id, target), + isClickable(target) ? getSelector(id, target) : '', ), true, ) From 9ec887b7f8bd28caba7c2eb6e31f7b061c3d94a4 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Mar 2023 12:30:37 +0100 Subject: [PATCH 069/253] change(ui): add new event info to UI --- frontend/app/types/session/event.ts | 102 ++++++++++++++++------------ frontend/app/types/session/issue.ts | 4 +- 2 files changed, 61 insertions(+), 45 deletions(-) diff --git a/frontend/app/types/session/event.ts b/frontend/app/types/session/event.ts index bb901a6b1..99693d756 100644 --- a/frontend/app/types/session/event.ts +++ b/frontend/app/types/session/event.ts @@ -5,12 +5,18 @@ const LOCATION = 'LOCATION'; const CUSTOM = 'CUSTOM'; const CLICKRAGE = 'CLICKRAGE'; const IOS_VIEW = 'VIEW'; -export const TYPES = { CONSOLE, CLICK, INPUT, LOCATION, CUSTOM, CLICKRAGE, IOS_VIEW}; +export const TYPES = { CONSOLE, CLICK, INPUT, LOCATION, CUSTOM, CLICKRAGE, IOS_VIEW }; interface IEvent { time: number; timestamp: number; - type: typeof CONSOLE | typeof CLICK | typeof INPUT | typeof LOCATION | typeof CUSTOM | typeof CLICKRAGE; + type: + | typeof CONSOLE + | typeof CLICK + | typeof INPUT + | typeof LOCATION + | typeof CUSTOM + | typeof CLICKRAGE; name: string; key: number; label: string; @@ -18,18 +24,23 @@ interface IEvent { target: { path: string; label: string; - } + }; } + interface ConsoleEvent extends IEvent { - subtype: string - value: string + subtype: string; + value: string; } + interface ClickEvent extends IEvent { targetContent: string; count: number; } + interface InputEvent extends IEvent { value: string; + hesitation: number; + duration: number; } interface LocationEvent extends IEvent { @@ -51,11 +62,10 @@ interface LocationEvent extends IEvent { export type EventData = ConsoleEvent | ClickEvent | InputEvent | LocationEvent | IEvent; class Event { - key: IEvent["key"] - time: IEvent["time"]; - label: IEvent["label"]; - target: IEvent["target"]; - + key: IEvent['key']; + time: IEvent['time']; + label: IEvent['label']; + target: IEvent['target']; constructor(event: IEvent) { Object.assign(this, { @@ -64,98 +74,102 @@ class Event { key: event.key, target: { path: event.target?.path || event.targetPath, - label: event.target?.label - } - }) + label: event.target?.label, + }, + }); } } class Console extends Event { readonly type = CONSOLE; - readonly name = 'Console' + readonly name = 'Console'; subtype: string; value: string; constructor(evt: ConsoleEvent) { super(evt); - this.subtype = evt.subtype - this.value = evt.value + this.subtype = evt.subtype; + this.value = evt.value; } } export class Click extends Event { readonly type: typeof CLICKRAGE | typeof CLICK = CLICK; - readonly name = 'Click' + readonly name = 'Click'; targetContent = ''; - count: number + count: number; constructor(evt: ClickEvent, isClickRage?: boolean) { + console.log(evt); super(evt); - this.targetContent = evt.targetContent - this.count = evt.count + this.targetContent = evt.targetContent; + this.count = evt.count; if (isClickRage) { - this.type = CLICKRAGE + this.type = CLICKRAGE; } } } class Input extends Event { readonly type = INPUT; - readonly name = 'Input' - value = '' + readonly name = 'Input'; + readonly hesitation: number = 0; + readonly duration: number = 0; + + value = ''; constructor(evt: InputEvent) { super(evt); - this.value = evt.value + this.value = evt.value; + this.hesitation = evt.hesitation; + this.duration = evt.duration; } } - export class Location extends Event { readonly name = 'Location'; readonly type = LOCATION; - url: LocationEvent["url"] - host: LocationEvent["host"]; - fcpTime: LocationEvent["fcpTime"]; - loadTime: LocationEvent["loadTime"]; - domContentLoadedTime: LocationEvent["domContentLoadedTime"]; - domBuildingTime: LocationEvent["domBuildingTime"]; - speedIndex: LocationEvent["speedIndex"]; - visuallyComplete: LocationEvent["visuallyComplete"]; - timeToInteractive: LocationEvent["timeToInteractive"]; - referrer: LocationEvent["referrer"]; + url: LocationEvent['url']; + host: LocationEvent['host']; + fcpTime: LocationEvent['fcpTime']; + loadTime: LocationEvent['loadTime']; + domContentLoadedTime: LocationEvent['domContentLoadedTime']; + domBuildingTime: LocationEvent['domBuildingTime']; + speedIndex: LocationEvent['speedIndex']; + visuallyComplete: LocationEvent['visuallyComplete']; + timeToInteractive: LocationEvent['timeToInteractive']; + referrer: LocationEvent['referrer']; constructor(evt: LocationEvent) { super(evt); Object.assign(this, { ...evt, - fcpTime: evt.firstContentfulPaintTime || evt.firstPaintTime + fcpTime: evt.firstContentfulPaintTime || evt.firstPaintTime, }); } } export type InjectedEvent = Console | Click | Input | Location; -export default function(event: EventData) { +export default function (event: EventData) { if (event.type && event.type === CONSOLE) { - return new Console(event as ConsoleEvent) + return new Console(event as ConsoleEvent); } if (event.type && event.type === CLICK) { - return new Click(event as ClickEvent) + return new Click(event as ClickEvent); } if (event.type && event.type === INPUT) { - return new Input(event as InputEvent) + return new Input(event as InputEvent); } if (event.type && event.type === LOCATION) { - return new Location(event as LocationEvent) + return new Location(event as LocationEvent); } if (event.type && event.type === CLICKRAGE) { - return new Click(event as ClickEvent, true) + return new Click(event as ClickEvent, true); } // not used right now? // if (event.type === CUSTOM || !event.type) { // return new Event(event) // } - console.error(`Unknown event type: ${event.type}`) + console.error(`Unknown event type: ${event.type}`); } - diff --git a/frontend/app/types/session/issue.ts b/frontend/app/types/session/issue.ts index 68ab64001..2abedc5ea 100644 --- a/frontend/app/types/session/issue.ts +++ b/frontend/app/types/session/issue.ts @@ -5,7 +5,8 @@ const types = { JS_EXCEPTION: 'js_exception', BAD_REQUEST: 'bad_request', CRASH: 'crash', - CLICK_RAGE: 'click_rage' + CLICK_RAGE: 'click_rage', + MOUSE_THRASHING: 'mouse_thrashing', } as const type TypeKeys = keyof typeof types @@ -21,6 +22,7 @@ export const issues_types = [ { 'type': types.BAD_REQUEST, 'visible': true, 'order': 2, 'name': 'Bad Requests', 'icon': 'funnel/file-medical-alt' }, { 'type': types.CLICK_RAGE, 'visible': true, 'order': 3, 'name': 'Click Rage', 'icon': 'funnel/emoji-angry' }, { 'type': types.CRASH, 'visible': true, 'order': 4, 'name': 'Crashes', 'icon': 'funnel/file-earmark-break' }, + { 'type': types.MOUSE_THRASHING, 'visible': true, 'order': 5, 'name': 'Mouse Thrashing', 'icon': 'close' }, // { 'type': 'memory', 'visible': true, 'order': 4, 'name': 'High Memory', 'icon': 'funnel/sd-card' }, // { 'type': 'vault', 'visible': true, 'order': 5, 'name': 'Vault', 'icon': 'safe' }, // { 'type': 'bookmark', 'visible': true, 'order': 5, 'name': 'Bookmarks', 'icon': 'safe' }, From 6e76074fe9ee07e9c9242c7faa04787801ca957d Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Mar 2023 12:39:09 +0100 Subject: [PATCH 070/253] change(ui): split events and issues adding into session model --- frontend/app/types/session/session.ts | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index 3b254ae4b..c843206fe 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -159,6 +159,7 @@ export default class Session { notes: ISession["notes"] notesWithEvents: ISession["notesWithEvents"] fileKey: ISession["fileKey"] + durationSeconds: number constructor(plainSession?: ISession) { const sessionData = plainSession || (emptyValues as unknown as ISession) @@ -238,6 +239,7 @@ export default class Session { isMobile, startedAt, duration, + durationSeconds, userNumericHash: hashString( session.userId || session.userAnonymousId || @@ -258,4 +260,45 @@ export default class Session { notesWithEvents: notesWithEvents, }) } + + addIssues(issues: IIssue[]) { + const issuesList = issues.map( + (i, k) => new Issue({ ...i, time: i.timestamp - this.startedAt, key: k })) || []; + + // @ts-ignore + this.issues = issuesList; + } + + addEvents(sessionEvents: EventData[], sessionNotes: Note[]) { + const events: InjectedEvent[] = [] + const rawEvents: (EventData & { key: number })[] = [] + + if (sessionEvents.length) { + sessionEvents.forEach((event, k) => { + const time = event.timestamp - this.startedAt + if (event.type !== TYPES.CONSOLE && time <= this.durationSeconds) { + const EventClass = SessionEvent({ ...event, time, key: k }) + if (EventClass) { + events.push(EventClass); + } + rawEvents.push({ ...event, time, key: k }); + } + }) + } + + const rawNotes = sessionNotes; + const notesWithEvents = [...rawEvents, ...rawNotes].sort((a, b) => { + // @ts-ignore just in case + const aTs = a.timestamp || a.time; + // @ts-ignore + const bTs = b.timestamp || b.time; + + return aTs - bTs; + }) || []; + + // @ts-ignore + this.notesWithEvents = notesWithEvents; + this.notes = sessionNotes + this.events = events + } } \ No newline at end of file From 907ae9a1318623ad8e87066cc1767ba5067e34f2 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Mon, 13 Mar 2023 12:54:34 +0100 Subject: [PATCH 071/253] feat(backend): added hesitation time for click events --- backend/pkg/db/postgres/bulks.go | 6 +++--- backend/pkg/db/postgres/messages-web.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/pkg/db/postgres/bulks.go b/backend/pkg/db/postgres/bulks.go index 0dcfca646..7eaba41b4 100644 --- a/backend/pkg/db/postgres/bulks.go +++ b/backend/pkg/db/postgres/bulks.go @@ -195,9 +195,9 @@ func (conn *BulkSet) initBulks() { } conn.webClickEvents, err = NewBulk(conn.c, "events.clicks", - "(session_id, message_id, timestamp, label, selector, url, path)", - "($%d, $%d, $%d, NULLIF(LEFT($%d, 2000), ''), LEFT($%d, 8000), LEFT($%d, 2000), LEFT($%d, 2000))", - 7, 200) + "(session_id, message_id, timestamp, label, selector, url, path, hesitation)", + "($%d, $%d, $%d, NULLIF(LEFT($%d, 2000), ''), LEFT($%d, 8000), LEFT($%d, 2000), LEFT($%d, 2000), $%d)", + 8, 200) if err != nil { log.Fatalf("can't create webClickEvents bulk: %s", err) } diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index a15c03335..a062df95a 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -66,7 +66,7 @@ func (conn *Conn) InsertWebClickEvent(sessionID uint64, projectID uint32, e *Mou } var host, path string host, path, _, _ = url.GetURLParts(e.Url) - if err := conn.bulks.Get("webClickEvents").Append(sessionID, truncSqIdx(e.MsgID()), e.Timestamp, e.Label, e.Selector, host+path, path); err != nil { + if err := conn.bulks.Get("webClickEvents").Append(sessionID, truncSqIdx(e.MsgID()), e.Timestamp, e.Label, e.Selector, host+path, path, e.HesitationTime); err != nil { log.Printf("insert web click err: %s", err) } // Accumulate session updates and exec inside batch with another sql commands From c07474bdfafe5c35baa01d3dc446ae1ad32d8c7b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 13 Mar 2023 17:02:11 +0100 Subject: [PATCH 072/253] feat(chalice): health-check feat(assist): health-check feat(peers): health-check feat(sourcemaps-reader): health-check --- api/chalicelib/core/health.py | 10 ++- assist/package.json | 2 +- assist/server.js | 16 ++--- assist/servers/websocket.js | 18 +++++- assist/utils/health.js | 54 ++++++++++++++++ ee/api/chalicelib/core/health.py | 12 ++-- ee/assist/package-lock.json | 10 +-- ee/assist/package.json | 2 +- ee/assist/server.js | 21 ++----- ee/assist/servers/websocket-cluster.js | 17 ++++++ ee/assist/servers/websocket.js | 18 ++++++ ee/assist/utils/health.js | 61 +++++++++++++++++++ peers/clean-dev.sh | 3 + peers/package-lock.json | 28 ++++----- peers/package.json | 2 +- peers/prepare-dev.sh | 3 + peers/run-dev.sh | 6 ++ peers/server.js | 9 +-- .../db/init_dbs/postgresql/1.11.0/1.11.0.sql | 5 ++ sourcemap-reader/.gitignore | 5 +- sourcemap-reader/clean-dev.sh | 5 +- sourcemap-reader/package-lock.json | 10 +-- sourcemap-reader/package.json | 2 +- sourcemap-reader/server.js | 14 ++--- sourcemap-reader/utils/health.js | 52 ++++++++++++++++ 25 files changed, 296 insertions(+), 89 deletions(-) create mode 100644 assist/utils/health.js create mode 100644 ee/assist/utils/health.js create mode 100755 peers/clean-dev.sh create mode 100755 peers/prepare-dev.sh create mode 100755 peers/run-dev.sh create mode 100644 sourcemap-reader/utils/health.js diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index 071cf7f9c..0a73661b7 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -31,7 +31,7 @@ else: HEALTH_ENDPOINTS = { "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/metrics", "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", - "assist": "http://assist-openreplay.app.svc.cluster.local:8888/metrics", + "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", @@ -40,12 +40,11 @@ else: "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", - "peers": "http://peers-openreplay.app.svc.cluster.local:8888/metrics", + "peers": "http://peers-openreplay.app.svc.cluster.local:8888/health", "quickwit": "http://quickwit-openreplay.app.svc.cluster.local:8888/metrics", "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", - "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/metrics", + "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health", "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", - "utilities": "http://utilities-openreplay.app.svc.cluster.local:8888/metrics", } @@ -172,8 +171,7 @@ def get_health(): "quickwit": __check_be_service("quickwit"), "sink": __check_be_service("sink"), "sourcemapreader": __check_be_service("sourcemapreader"), - "storage": __check_be_service("storage"), - "utilities": __check_be_service("utilities") + "storage": __check_be_service("storage") }, # "overall": { # "health": "na", diff --git a/assist/package.json b/assist/package.json index b06c8cae5..ad9794fea 100644 --- a/assist/package.json +++ b/assist/package.json @@ -1,6 +1,6 @@ { "name": "assist-server", - "version": "1.0.0", + "version": "v1.11.0", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", "scripts": { diff --git a/assist/server.js b/assist/server.js index d71aca65d..5eb6c2e16 100644 --- a/assist/server.js +++ b/assist/server.js @@ -2,6 +2,7 @@ const dumps = require('./utils/HeapSnapshot'); const express = require('express'); const socket = require("./servers/websocket"); const {request_logger} = require("./utils/helper"); +const health = require("./utils/health"); const assert = require('assert').strict; const debug = process.env.debug === "1"; @@ -10,7 +11,7 @@ const HOST = process.env.LISTEN_HOST || '0.0.0.0'; const PORT = process.env.LISTEN_PORT || 9001; assert.ok(process.env.ASSIST_KEY, 'The "ASSIST_KEY" environment variable is required'); const P_KEY = process.env.ASSIST_KEY; -const PREFIX = process.env.PREFIX || process.env.prefix || `/assist` +const PREFIX = process.env.PREFIX || process.env.prefix || `/assist`; const wsapp = express(); wsapp.use(express.json()); @@ -27,16 +28,9 @@ heapdump && wsapp.use(`${PREFIX}/${P_KEY}/heapdump`, dumps.router); const wsserver = wsapp.listen(PORT, HOST, () => { console.log(`WS App listening on http://${HOST}:${PORT}`); - console.log('Press Ctrl+C to quit.'); + health.healthApp.listen(health.PORT, HOST, health.listen_cb); }); + wsapp.enable('trust proxy'); socket.start(wsserver); -module.exports = {wsserver}; - -wsapp.get('/private/shutdown', (req, res) => { - console.log("Requested shutdown"); - res.statusCode = 200; - res.end("ok!"); - process.kill(1, "SIGTERM"); - } -); \ No newline at end of file +module.exports = {wsserver}; \ No newline at end of file diff --git a/assist/servers/websocket.js b/assist/servers/websocket.js index 4c4a657bb..0fdda85f2 100644 --- a/assist/servers/websocket.js +++ b/assist/servers/websocket.js @@ -45,7 +45,22 @@ const respond = function (res, data) { res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify({"data": data})); } - +const countSessions = async function () { + let count = 0; + try { + const arr = Array.from(io.sockets.adapter.rooms); + const filtered = arr.filter(room => !room[1].has(room[0])); + for (let i of filtered) { + let {projectKey, sessionId} = extractPeerId(i[0]); + if (projectKey !== null && sessionId !== null) { + count++; + } + } + } catch (e) { + console.error(e); + } + return count; +} const socketsList = async function (req, res) { debug && console.log("[WS]looking for all available sessions"); let filters = extractPayloadFromRequest(req); @@ -360,6 +375,7 @@ module.exports = { socketConnexionTimeout(io); }, + countSessions, handlers: { socketsList, socketsListByProject, diff --git a/assist/utils/health.js b/assist/utils/health.js new file mode 100644 index 000000000..d71864e71 --- /dev/null +++ b/assist/utils/health.js @@ -0,0 +1,54 @@ +const express = require('express'); +const socket = require("../servers/websocket"); +const HOST = process.env.LISTEN_HOST || '0.0.0.0'; +const PORT = process.env.HEALTH_PORT || 8888; + + +const {request_logger} = require("./helper"); +const debug = process.env.debug === "1"; +const respond = function (res, data) { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({"data": data})); +} + +const check_health = async function (req, res) { + debug && console.log("[WS]looking for all available sessions"); + respond(res, { + "health": true, + "details": { + "version": process.env.npm_package_version, + "connectedSessions": await socket.countSessions() + } + }); +} + + +const healthApp = express(); +healthApp.use(express.json()); +healthApp.use(express.urlencoded({extended: true})); +healthApp.use(request_logger("[healthApp]")); +healthApp.get(['/'], (req, res) => { + res.statusCode = 200; + res.end("healthApp ok!"); + } +); +healthApp.get('/health', check_health); +healthApp.get('/shutdown', (req, res) => { + console.log("Requested shutdown"); + res.statusCode = 200; + res.end("ok!"); + process.kill(1, "SIGTERM"); + } +); + +const listen_cb = async function () { + console.log(`Health App listening on http://${HOST}:${PORT}`); + console.log('Press Ctrl+C to quit.'); +} + +module.exports = { + healthApp, + PORT, + listen_cb +}; diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 784dfceab..514425ddf 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -32,7 +32,7 @@ else: HEALTH_ENDPOINTS = { "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/metrics", "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", - "assist": "http://assist-openreplay.app.svc.cluster.local:8888/metrics", + "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", @@ -41,12 +41,11 @@ else: "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", - "peers": "http://peers-openreplay.app.svc.cluster.local:8888/metrics", + "peers": "http://peers-openreplay.app.svc.cluster.local:8888/health", "quickwit": "http://quickwit-openreplay.app.svc.cluster.local:8888/metrics", "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", - "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/metrics", - "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", - "utilities": "http://utilities-openreplay.app.svc.cluster.local:8888/metrics", + "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health", + "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics" } @@ -175,8 +174,7 @@ def get_health(): "quickwit": __check_be_service("quickwit"), "sink": __check_be_service("sink"), "sourcemapreader": __check_be_service("sourcemapreader"), - "storage": __check_be_service("storage"), - "utilities": __check_be_service("utilities") + "storage": __check_be_service("storage") }, # "overall": { # "health": "na", diff --git a/ee/assist/package-lock.json b/ee/assist/package-lock.json index 967198a0e..a94f1d5bb 100644 --- a/ee/assist/package-lock.json +++ b/ee/assist/package-lock.json @@ -1,12 +1,12 @@ { "name": "assist-server", - "version": "1.0.0", + "version": "v1.11.0-ee", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "assist-server", - "version": "1.0.0", + "version": "v1.11.0-ee", "license": "Elastic License 2.0 (ELv2)", "dependencies": { "@maxmind/geoip2-node": "^3.5.0", @@ -117,9 +117,9 @@ } }, "node_modules/@types/node": { - "version": "18.14.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.6.tgz", - "integrity": "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA==" + "version": "18.15.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.1.tgz", + "integrity": "sha512-U2TWca8AeHSmbpi314QBESRk7oPjSZjDsR+c+H4ECC1l+kFgpZf8Ydhv3SJpPy51VyZHHqxlb6mTTqYNNRVAIw==" }, "node_modules/accepts": { "version": "1.3.8", diff --git a/ee/assist/package.json b/ee/assist/package.json index 3fcedf03b..4ef88774a 100644 --- a/ee/assist/package.json +++ b/ee/assist/package.json @@ -1,6 +1,6 @@ { "name": "assist-server", - "version": "1.0.0", + "version": "v1.11.0-ee", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", "scripts": { diff --git a/ee/assist/server.js b/ee/assist/server.js index 482ddcd17..414d7ac2c 100644 --- a/ee/assist/server.js +++ b/ee/assist/server.js @@ -1,6 +1,7 @@ const dumps = require('./utils/HeapSnapshot'); const {request_logger} = require('./utils/helper'); const express = require('express'); +const health = require("./utils/health"); const assert = require('assert').strict; let socket; @@ -14,7 +15,7 @@ const HOST = process.env.LISTEN_HOST || '0.0.0.0'; const PORT = process.env.LISTEN_PORT || 9001; assert.ok(process.env.ASSIST_KEY, 'The "ASSIST_KEY" environment variable is required'); const P_KEY = process.env.ASSIST_KEY; -const PREFIX = process.env.PREFIX || process.env.prefix || `/assist` +const PREFIX = process.env.PREFIX || process.env.prefix || `/assist`; let debug = process.env.debug === "1"; const heapdump = process.env.heapdump === "1"; @@ -31,18 +32,11 @@ if (process.env.uws !== "true") { ); heapdump && wsapp.use(`${PREFIX}/${P_KEY}/heapdump`, dumps.router); wsapp.use(`${PREFIX}/${P_KEY}`, socket.wsRouter); - wsapp.get('/private/shutdown', (req, res) => { - console.log("Requested shutdown"); - res.statusCode = 200; - res.end("ok!"); - process.kill(1, "SIGTERM"); - } - ); wsapp.enable('trust proxy'); const wsserver = wsapp.listen(PORT, HOST, () => { console.log(`WS App listening on http://${HOST}:${PORT}`); - console.log('Press Ctrl+C to quit.'); + health.healthApp.listen(health.PORT, HOST, health.listen_cb); }); socket.start(wsserver); @@ -102,13 +96,6 @@ if (process.env.uws !== "true") { uapp.post(`${PREFIX}/${P_KEY}/sockets-live/:projectKey`, uWrapper(socket.handlers.socketsLiveByProject)); uapp.get(`${PREFIX}/${P_KEY}/sockets-live/:projectKey/:sessionId`, uWrapper(socket.handlers.socketsLiveByProject)); - uapp.get('/private/shutdown', (res, req) => { - console.log("Requested shutdown"); - res.writeStatus('200 OK').end("ok!"); - process.kill(1, "SIGTERM"); - } - ); - socket.start(uapp); uapp.listen(HOST, PORT, (token) => { @@ -116,7 +103,7 @@ if (process.env.uws !== "true") { console.warn("port already in use"); } console.log(`WS App listening on http://${HOST}:${PORT}`); - console.log('Press Ctrl+C to quit.'); + health.healthApp.listen(health.PORT, HOST, health.listen_cb); }); diff --git a/ee/assist/servers/websocket-cluster.js b/ee/assist/servers/websocket-cluster.js index a1f389685..4618a6184 100644 --- a/ee/assist/servers/websocket-cluster.js +++ b/ee/assist/servers/websocket-cluster.js @@ -83,6 +83,22 @@ const respond = function (res, data) { } } +const countSessions = async function () { + let count = 0; + try { + let rooms = await io.of('/').adapter.allRooms(); + for (let i of rooms) { + let {projectKey, sessionId} = extractPeerId(i); + if (projectKey !== undefined && sessionId !== undefined) { + count++; + } + } + } catch (e) { + console.error(e); + } + return count; +} + const socketsList = async function (req, res) { debug && console.log("[WS]looking for all available sessions"); let filters = await extractPayloadFromRequest(req, res); @@ -417,6 +433,7 @@ module.exports = { process.exit(2); }); }, + countSessions, handlers: { socketsList, socketsListByProject, diff --git a/ee/assist/servers/websocket.js b/ee/assist/servers/websocket.js index 330361df3..7fb1c9684 100644 --- a/ee/assist/servers/websocket.js +++ b/ee/assist/servers/websocket.js @@ -66,6 +66,23 @@ const respond = function (res, data) { } } +const countSessions = async function () { + let count = 0; + try { + const arr = Array.from(io.sockets.adapter.rooms); + const filtered = arr.filter(room => !room[1].has(room[0])); + for (let i of filtered) { + let {projectKey, sessionId} = extractPeerId(i[0]); + if (projectKey !== null && sessionId !== null) { + count++; + } + } + } catch (e) { + console.error(e); + } + return count; +} + const socketsList = async function (req, res) { debug && console.log("[WS]looking for all available sessions"); let filters = await extractPayloadFromRequest(req, res); @@ -379,6 +396,7 @@ module.exports = { socketConnexionTimeout(io); }, + countSessions, handlers: { socketsList, socketsListByProject, diff --git a/ee/assist/utils/health.js b/ee/assist/utils/health.js new file mode 100644 index 000000000..bcb64f61c --- /dev/null +++ b/ee/assist/utils/health.js @@ -0,0 +1,61 @@ +const express = require('express'); +let socket; +if (process.env.redis === "true") { + socket = require("../servers/websocket-cluster"); +} else { + socket = require("../servers/websocket"); +} +const HOST = process.env.LISTEN_HOST || '0.0.0.0'; +const PORT = process.env.HEALTH_PORT || 8888; + + +const {request_logger} = require("./helper"); +const debug = process.env.debug === "1"; +const respond = function (res, data) { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({"data": data})); +} + +const check_health = async function (req, res) { + debug && console.log("[WS]looking for all available sessions"); + respond(res, { + "health": true, + "details": { + "version": process.env.npm_package_version, + "connectedSessions": await socket.countSessions(), + "uWebSocket": process.env.uws === "true", + "redis": process.env.redis === "true" + } + }); +} + + +const healthApp = express(); +healthApp.use(express.json()); +healthApp.use(express.urlencoded({extended: true})); +healthApp.use(request_logger("[healthApp]")); +healthApp.get(['/'], (req, res) => { + res.statusCode = 200; + res.end("healthApp ok!"); + } +); +healthApp.get('/health', check_health); +healthApp.get('/shutdown', (req, res) => { + console.log("Requested shutdown"); + res.statusCode = 200; + res.end("ok!"); + process.kill(1, "SIGTERM"); + } +); + +const listen_cb = async function () { + console.log(`Health App listening on http://${HOST}:${PORT}`); + console.log('Press Ctrl+C to quit.'); +} + +module.exports = { + healthApp, + PORT, + listen_cb +}; diff --git a/peers/clean-dev.sh b/peers/clean-dev.sh new file mode 100755 index 000000000..a0cb5c9ed --- /dev/null +++ b/peers/clean-dev.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf ./utils \ No newline at end of file diff --git a/peers/package-lock.json b/peers/package-lock.json index 5811b59e3..b18dca820 100644 --- a/peers/package-lock.json +++ b/peers/package-lock.json @@ -1,12 +1,12 @@ { "name": "peers-server", - "version": "1.0.0", + "version": "v1.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "peers-server", - "version": "1.0.0", + "version": "v1.11.0", "license": "Elastic License 2.0 (ELv2)", "dependencies": { "express": "^4.18.2", @@ -57,9 +57,9 @@ "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" }, "node_modules/@types/node": { - "version": "18.14.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.6.tgz", - "integrity": "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA==" + "version": "18.15.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.1.tgz", + "integrity": "sha512-U2TWca8AeHSmbpi314QBESRk7oPjSZjDsR+c+H4ECC1l+kFgpZf8Ydhv3SJpPy51VyZHHqxlb6mTTqYNNRVAIw==" }, "node_modules/@types/qs": { "version": "6.9.7", @@ -600,9 +600,9 @@ } }, "node_modules/node-fetch": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz", - "integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz", + "integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==", "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -657,9 +657,9 @@ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "node_modules/peer": { - "version": "1.0.0-rc.10", - "resolved": "https://registry.npmjs.org/peer/-/peer-1.0.0-rc.10.tgz", - "integrity": "sha512-S7uMqIAd1tTyvnkj4efdpn8EGc6BM1ONQvLg0vZkrnvA1cTisscBRsx+Jbor6DH68NRLnXgZbiY7/6FDER/GXw==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/peer/-/peer-1.0.0.tgz", + "integrity": "sha512-fPVtyCKZWVfjbf7XnY7MskhTlu+pBpMvQV81sngT8aXIuT5YF9y9bwIw8y5BlI98DV0NsDpLjow/oemFNvcKkg==", "dependencies": { "@types/express": "^4.17.3", "@types/ws": "^7.2.3 || ^8.0.0", @@ -921,9 +921,9 @@ } }, "node_modules/ws": { - "version": "8.12.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz", - "integrity": "sha512-1qo+M9Ba+xNhPB+YTWUlK6M17brTut5EXbcBaMRN5pH5dFrXz7lzz1ChFSUq3bOUl8yEvSenhHmYUNJxFzdJew==", + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", "engines": { "node": ">=10.0.0" }, diff --git a/peers/package.json b/peers/package.json index 400274ffc..82fd0ddf6 100644 --- a/peers/package.json +++ b/peers/package.json @@ -1,6 +1,6 @@ { "name": "peers-server", - "version": "1.0.0", + "version": "v1.11.0", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", "scripts": { diff --git a/peers/prepare-dev.sh b/peers/prepare-dev.sh new file mode 100755 index 000000000..d4825a3d0 --- /dev/null +++ b/peers/prepare-dev.sh @@ -0,0 +1,3 @@ +#!/bin/bash +rsync -avr --exclude=".*" --ignore-existing ../assist/utils ./ +cp ../sourcemap-reader/utils/health.js ./utils/. \ No newline at end of file diff --git a/peers/run-dev.sh b/peers/run-dev.sh new file mode 100755 index 000000000..00e8d5a4b --- /dev/null +++ b/peers/run-dev.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -a +source .env +set +a + +npm start \ No newline at end of file diff --git a/peers/server.js b/peers/server.js index 4e25a0827..8cf12d546 100644 --- a/peers/server.js +++ b/peers/server.js @@ -1,5 +1,6 @@ const dumps = require('./utils/HeapSnapshot'); const {request_logger} = require('./utils/helper'); +const health = require("./utils/health"); const assert = require('assert').strict; const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server'); const express = require('express'); @@ -44,10 +45,4 @@ process.on('uncaughtException', err => { // process.exit(1); }); -app.get('/private/shutdown', (req, res) => { - console.log("Requested shutdown"); - res.statusCode = 200; - res.end("ok!"); - process.kill(1, "SIGTERM"); - } -); \ No newline at end of file +health.healthApp.listen(health.PORT, HOST, health.listen_cb); \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql b/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql index 13813f5bc..41521a886 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql @@ -17,4 +17,9 @@ ALTER TABLE public.projects "defaultInputMode": "obscured" }'::jsonb; +ALTER TYPE issue_type ADD VALUE IF NOT EXISTS 'mouse_thrashing'; + +ALTER TABLE events.clicks + ADD COLUMN hesitation integer NULL; + COMMIT; \ No newline at end of file diff --git a/sourcemap-reader/.gitignore b/sourcemap-reader/.gitignore index 09c49b304..f2686decf 100644 --- a/sourcemap-reader/.gitignore +++ b/sourcemap-reader/.gitignore @@ -3,5 +3,8 @@ node_modules npm-debug.log .cache test.html -/utils/ +/utils/assistHelper.js +/utils/geoIP.js +/utils/HeapSnapshot.js +/utils/helper.js mappings.wasm diff --git a/sourcemap-reader/clean-dev.sh b/sourcemap-reader/clean-dev.sh index a0cb5c9ed..ebc1c36c6 100755 --- a/sourcemap-reader/clean-dev.sh +++ b/sourcemap-reader/clean-dev.sh @@ -1,3 +1,6 @@ #!/bin/bash -rm -rf ./utils \ No newline at end of file +rm -rf ./utils/assistHelper.js +rm -rf ./utils/geoIP.js +rm -rf ./utils/HeapSnapshot.js +rm -rf ./utils/helper.js \ No newline at end of file diff --git a/sourcemap-reader/package-lock.json b/sourcemap-reader/package-lock.json index e756a0649..1b3f5ec82 100644 --- a/sourcemap-reader/package-lock.json +++ b/sourcemap-reader/package-lock.json @@ -1,12 +1,12 @@ { "name": "sourcemaps-reader", - "version": "1.0.0", + "version": "v1.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sourcemaps-reader", - "version": "1.0.0", + "version": "v1.11.0", "license": "Elastic License 2.0 (ELv2)", "dependencies": { "aws-sdk": "^2.1314.0", @@ -43,9 +43,9 @@ } }, "node_modules/aws-sdk": { - "version": "2.1329.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1329.0.tgz", - "integrity": "sha512-F5M9x/T+PanPiYGiL95atFE6QiwzJWwgPahaEgUdq+qvVAgruiNy5t6nw2B5tBB/yWDPPavHFip3UsXeO0qU3Q==", + "version": "2.1333.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1333.0.tgz", + "integrity": "sha512-MvOuleNeRryJtkCGXGEWDHPqqgxuqdi4/hGzJEpn9tnjsW9LNK8UgFPpYzUZ24ZO/3S+jiUh8DMMrL5nVGnagg==", "dependencies": { "buffer": "4.9.2", "events": "1.1.1", diff --git a/sourcemap-reader/package.json b/sourcemap-reader/package.json index 9d5a2806b..5a9b28ef8 100644 --- a/sourcemap-reader/package.json +++ b/sourcemap-reader/package.json @@ -1,6 +1,6 @@ { "name": "sourcemaps-reader", - "version": "1.0.0", + "version": "v1.11.0", "description": "assist server to get live sessions & sourcemaps reader to get stack trace", "main": "peerjs-server.js", "scripts": { diff --git a/sourcemap-reader/server.js b/sourcemap-reader/server.js index 02f63475b..08e3f926f 100644 --- a/sourcemap-reader/server.js +++ b/sourcemap-reader/server.js @@ -1,11 +1,12 @@ const dumps = require('./utils/HeapSnapshot'); const sourcemapsReaderServer = require('./servers/sourcemaps-server'); const express = require('express'); +const health = require("./utils/health"); const {request_logger} = require("./utils/helper"); const HOST = process.env.SMR_HOST || '127.0.0.1'; const PORT = process.env.SMR_PORT || 9000; -const PREFIX = process.env.PREFIX || process.env.prefix || '' +const PREFIX = process.env.PREFIX || process.env.prefix || ''; const P_KEY = process.env.SMR_KEY || 'smr'; const heapdump = process.env.heapdump === "1"; @@ -21,14 +22,7 @@ heapdump && app.use(`${PREFIX}/${P_KEY}/heapdump`, dumps.router); const server = app.listen(PORT, HOST, () => { console.log(`SR App listening on http://${HOST}:${PORT}`); - console.log('Press Ctrl+C to quit.'); + health.healthApp.listen(health.PORT, HOST, health.listen_cb); }); -module.exports = {server}; -app.get('/private/shutdown', (req, res) => { - console.log("Requested shutdown"); - res.statusCode = 200; - res.end("ok!"); - process.kill(1, "SIGTERM"); - } -); \ No newline at end of file +module.exports = {server}; \ No newline at end of file diff --git a/sourcemap-reader/utils/health.js b/sourcemap-reader/utils/health.js new file mode 100644 index 000000000..0b89dd1d8 --- /dev/null +++ b/sourcemap-reader/utils/health.js @@ -0,0 +1,52 @@ +const express = require('express'); +const HOST = process.env.LISTEN_HOST || '0.0.0.0'; +const PORT = process.env.HEALTH_PORT || 8888; + + +const {request_logger} = require("./helper"); +const debug = process.env.debug === "1"; +const respond = function (res, data) { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({"data": data})); +} + +const check_health = async function (req, res) { + debug && console.log("[WS]looking for all available sessions"); + respond(res, { + "health": true, + "details": { + "version": process.env.npm_package_version + } + }); +} + + +const healthApp = express(); +healthApp.use(express.json()); +healthApp.use(express.urlencoded({extended: true})); +healthApp.use(request_logger("[healthApp]")); +healthApp.get(['/'], (req, res) => { + res.statusCode = 200; + res.end("healthApp ok!"); + } +); +healthApp.get('/health', check_health); +healthApp.get('/shutdown', (req, res) => { + console.log("Requested shutdown"); + res.statusCode = 200; + res.end("ok!"); + process.kill(1, "SIGTERM"); + } +); + +const listen_cb = async function () { + console.log(`Health App listening on http://${HOST}:${PORT}`); + console.log('Press Ctrl+C to quit.'); +} + +module.exports = { + healthApp, + PORT, + listen_cb +}; From 0fe47eee487765f116054eea5b131e4d83e0c66e Mon Sep 17 00:00:00 2001 From: Dayan Graham Date: Mon, 13 Mar 2023 16:58:39 +0000 Subject: [PATCH 073/253] feat(assets): Add support for mutual TLS to allow the assets service to fetch files behind authentication walls (#1034) --- backend/internal/assets/cacher/cacher.go | 35 ++++++++++++++++++++++-- backend/internal/config/assets/config.go | 3 ++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/backend/internal/assets/cacher/cacher.go b/backend/internal/assets/cacher/cacher.go index 4b0353a9a..1df32ca26 100644 --- a/backend/internal/assets/cacher/cacher.go +++ b/backend/internal/assets/cacher/cacher.go @@ -2,9 +2,11 @@ package cacher import ( "crypto/tls" + "crypto/x509" "fmt" "io" "io/ioutil" + "log" "mime" "net/http" metrics "openreplay/backend/pkg/metrics/assets" @@ -38,14 +40,43 @@ func (c *cacher) CanCache() bool { func NewCacher(cfg *config.Config) *cacher { rewriter := assets.NewRewriter(cfg.AssetsOrigin) + + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + + if cfg.ClientCertFilePath != "" && cfg.ClientKeyFilePath != "" && cfg.CaCertFilePath != "" { + + var cert tls.Certificate + var err error + + cert, err = tls.LoadX509KeyPair(cfg.ClientCertFilePath, cfg.ClientKeyFilePath) + if err != nil { + log.Fatalf("Error creating x509 keypair from the client cert file %s and client key file %s , Error: %s", err, cfg.ClientCertFilePath, cfg.ClientKeyFilePath) + } + + caCert, err := ioutil.ReadFile(cfg.CaCertFilePath) + if err != nil { + log.Fatalf("Error opening cert file %s, Error: %s", cfg.CaCertFilePath, err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + } + + } + c := &cacher{ timeoutMap: newTimeoutMap(), s3: storage.NewS3(cfg.AWSRegion, cfg.S3BucketAssets), httpClient: &http.Client{ Timeout: time.Duration(6) * time.Second, Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: tlsConfig, }, }, rewriter: rewriter, diff --git a/backend/internal/config/assets/config.go b/backend/internal/config/assets/config.go index 399ee84f4..19c747e71 100644 --- a/backend/internal/config/assets/config.go +++ b/backend/internal/config/assets/config.go @@ -15,6 +15,9 @@ type Config struct { AssetsSizeLimit int `env:"ASSETS_SIZE_LIMIT,required"` AssetsRequestHeaders map[string]string `env:"ASSETS_REQUEST_HEADERS"` UseProfiler bool `env:"PROFILER_ENABLED,default=false"` + ClientKeyFilePath string `env:"CLIENT_KEY_FILE_PATH"` + CaCertFilePath string `env:"CA_CERT_FILE_PATH"` + ClientCertFilePath string `env:"CLIENT_CERT_FILE_PATH"` } func New() *Config { From b8ddbee0cf67b85b629453514b1e5cd0d752b0f7 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 13 Mar 2023 18:12:39 +0100 Subject: [PATCH 074/253] feat(chalice): health-check changes chore(deployment): changed service ports --- api/chalicelib/core/health.py | 2 -- scripts/helmcharts/openreplay/charts/assist/values.yaml | 1 + scripts/helmcharts/openreplay/charts/peers/values.yaml | 1 + .../helmcharts/openreplay/charts/sourcemapreader/values.yaml | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index 0a73661b7..980c7a41b 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -41,7 +41,6 @@ else: "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", "peers": "http://peers-openreplay.app.svc.cluster.local:8888/health", - "quickwit": "http://quickwit-openreplay.app.svc.cluster.local:8888/metrics", "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health", "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", @@ -168,7 +167,6 @@ def get_health(): "ingress-nginx": __always_healthy, "integrations": __check_be_service("integrations"), "peers": __check_be_service("peers"), - "quickwit": __check_be_service("quickwit"), "sink": __check_be_service("sink"), "sourcemapreader": __check_be_service("sourcemapreader"), "storage": __check_be_service("storage") diff --git a/scripts/helmcharts/openreplay/charts/assist/values.yaml b/scripts/helmcharts/openreplay/charts/assist/values.yaml index 8ff07d2d0..5e84e8c60 100644 --- a/scripts/helmcharts/openreplay/charts/assist/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/values.yaml @@ -64,6 +64,7 @@ service: type: ClusterIP ports: socketio: 9001 + metrics: 8888 ingress: enabled: true diff --git a/scripts/helmcharts/openreplay/charts/peers/values.yaml b/scripts/helmcharts/openreplay/charts/peers/values.yaml index 0bc4b6b14..0bf7fc27e 100644 --- a/scripts/helmcharts/openreplay/charts/peers/values.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/values.yaml @@ -64,6 +64,7 @@ service: type: ClusterIP ports: peerjs: 9000 + metrics: 8888 ingress: enabled: true diff --git a/scripts/helmcharts/openreplay/charts/sourcemapreader/values.yaml b/scripts/helmcharts/openreplay/charts/sourcemapreader/values.yaml index ec9fe9655..d14069fca 100644 --- a/scripts/helmcharts/openreplay/charts/sourcemapreader/values.yaml +++ b/scripts/helmcharts/openreplay/charts/sourcemapreader/values.yaml @@ -48,6 +48,7 @@ service: type: ClusterIP ports: sourcemapreader: 9000 + metrics: 8888 serviceMonitor: enabled: false From 9e59d5e1abd7320c11c00d29eba433888d04253c Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 14 Mar 2023 11:12:46 +0100 Subject: [PATCH 075/253] feat(backend): added heuristics metric --- backend/cmd/heuristics/main.go | 5 +++++ backend/internal/heuristics/service.go | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/backend/cmd/heuristics/main.go b/backend/cmd/heuristics/main.go index 073f48611..3a7abb7a0 100644 --- a/backend/cmd/heuristics/main.go +++ b/backend/cmd/heuristics/main.go @@ -8,12 +8,17 @@ import ( "openreplay/backend/pkg/handlers/custom" "openreplay/backend/pkg/handlers/web" "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/metrics" + heuristicsMetrics "openreplay/backend/pkg/metrics/heuristics" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/sessions" "openreplay/backend/pkg/terminator" ) func main() { + m := metrics.New() + m.Register(heuristicsMetrics.List()) + log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) cfg := config.New() diff --git a/backend/internal/heuristics/service.go b/backend/internal/heuristics/service.go index 0063f79f7..44b4034e2 100644 --- a/backend/internal/heuristics/service.go +++ b/backend/internal/heuristics/service.go @@ -1,7 +1,10 @@ package heuristics import ( + "fmt" "log" + "openreplay/backend/pkg/messages" + metrics "openreplay/backend/pkg/metrics/heuristics" "time" "openreplay/backend/internal/config/heuristics" @@ -35,6 +38,8 @@ func (h *heuristicsImpl) run() { case evt := <-h.events.Events(): if err := h.producer.Produce(h.cfg.TopicAnalytics, evt.SessionID(), evt.Encode()); err != nil { log.Printf("can't send new event to queue: %s", err) + } else { + metrics.IncreaseTotalEvents(messageTypeName(evt)) } case <-tick: h.producer.Flush(h.cfg.ProducerTimeout) @@ -62,3 +67,21 @@ func (h *heuristicsImpl) Stop() { h.consumer.Commit() h.consumer.Close() } + +func messageTypeName(msg messages.Message) string { + switch msg.TypeID() { + case 31: + return "PageEvent" + case 32: + return "InputEvent" + case 56: + return "PerformanceTrackAggr" + case 69: + return "MouseClick" + case 125: + m := msg.(*messages.IssueEvent) + return fmt.Sprintf("IssueEvent(%s)", m.Type) + default: + return "unknown" + } +} From 321c07d914fa49f7e4e9cbfa505fd15b84a51e7f Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 14 Mar 2023 11:23:15 +0100 Subject: [PATCH 076/253] feat(backend): added heuristics metrics builder --- backend/pkg/metrics/heuristics/metrics.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 backend/pkg/metrics/heuristics/metrics.go diff --git a/backend/pkg/metrics/heuristics/metrics.go b/backend/pkg/metrics/heuristics/metrics.go new file mode 100644 index 000000000..61a84dc49 --- /dev/null +++ b/backend/pkg/metrics/heuristics/metrics.go @@ -0,0 +1,22 @@ +package heuristics + +import "github.com/prometheus/client_golang/prometheus" + +var heuristicsTotalEvents = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "heuristics", + Name: "events_total", + Help: "A counter displaying the number of all processed events", + }, + []string{"type"}, +) + +func IncreaseTotalEvents(eventType string) { + heuristicsTotalEvents.WithLabelValues(eventType).Inc() +} + +func List() []prometheus.Collector { + return []prometheus.Collector{ + heuristicsTotalEvents, + } +} From e890ff8a14db31a5b612e7e73e7b5b6f7c3b5186 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 11:51:02 +0100 Subject: [PATCH 077/253] feat(chalice): split replay --- api/chalicelib/core/sessions_replay.py | 191 +++++++++++++++++++++++++ api/routers/core_dynamic.py | 40 +++++- 2 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 api/chalicelib/core/sessions_replay.py diff --git a/api/chalicelib/core/sessions_replay.py b/api/chalicelib/core/sessions_replay.py new file mode 100644 index 000000000..84c6703eb --- /dev/null +++ b/api/chalicelib/core/sessions_replay.py @@ -0,0 +1,191 @@ +from typing import List + +import schemas +from chalicelib.core import events, metadata, events_ios, \ + sessions_mobs, issues, projects, resources, assist, performance_event, sessions_favorite, \ + sessions_devtool, sessions_notes +from chalicelib.utils import errors_helper +from chalicelib.utils import pg_client, helper, metrics_helper +from chalicelib.utils import sql_helper as sh + + +def __group_metadata(session, project_metadata): + meta = {} + for m in project_metadata.keys(): + if project_metadata[m] is not None and session.get(m) is not None: + meta[project_metadata[m]] = session[m] + session.pop(m) + return meta + + +# for backward compatibility +def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_data=False, include_fav_viewed=False, + group_metadata=False, live=True): + with pg_client.PostgresClient() as cur: + extra_query = [] + if include_fav_viewed: + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite""") + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS viewed""") + query = cur.mogrify( + f"""\ + SELECT + s.*, + s.session_id::text AS session_id, + (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key + {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} + {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''} + FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id, "userId": context.user_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + data = cur.fetchone() + if data is not None: + data = helper.dict_to_camel_case(data) + if full_data: + if data["platform"] == 'ios': + data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) + for e in data['events']: + if e["type"].endswith("_IOS"): + e["type"] = e["type"][:-len("_IOS")] + data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) + data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, + session_id=session_id) + data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) + else: + data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, + group_clickrage=True) + all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) + data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] + # to keep only the first stack + # limit the number of errors to reduce the response-body size + data['errors'] = [errors_helper.format_first_stack_frame(e) for e in all_errors + if e['source'] == "js_exception"][:500] + data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, + session_id=session_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) + data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, + start_ts=data["startTs"], duration=data["duration"]) + + data['notes'] = sessions_notes.get_session_notes(tenant_id=context.tenant_id, project_id=project_id, + session_id=session_id, user_id=context.user_id) + data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) + data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) + data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, + project_key=data["projectKey"]) + data["inDB"] = True + return data + elif live: + return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) + else: + return None + + +def get_replay(project_id, session_id, context: schemas.CurrentContext, full_data=False, include_fav_viewed=False, + group_metadata=False, live=True): + with pg_client.PostgresClient() as cur: + extra_query = [] + if include_fav_viewed: + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite""") + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS viewed""") + query = cur.mogrify( + f"""\ + SELECT + s.*, + s.session_id::text AS session_id, + (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key + {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} + {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''} + FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id, "userId": context.user_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + data = cur.fetchone() + if data is not None: + data = helper.dict_to_camel_case(data) + if full_data: + if data["platform"] == 'ios': + data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) + else: + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) + + data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) + data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, + project_key=data["projectKey"]) + data["inDB"] = True + return data + elif live: + return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) + else: + return None + + +def get_events(project_id, session_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + f"""SELECT s.*, + s.session_id::text AS session_id + FROM public.sessions AS s + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + data = cur.fetchone() + if data is not None: + data = helper.dict_to_camel_case(data) + + if data["platform"] == 'ios': + data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) + for e in data['events']: + if e["type"].endswith("_IOS"): + e["type"] = e["type"][:-len("_IOS")] + data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) + data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, + session_id=session_id) + else: + data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, + group_clickrage=True) + all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) + data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] + # to keep only the first stack + # limit the number of errors to reduce the response-body size + data['errors'] = [errors_helper.format_first_stack_frame(e) for e in all_errors + if e['source'] == "js_exception"][:500] + data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, + session_id=session_id) + data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, + start_ts=data["startTs"], duration=data["duration"]) + + data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) + return data + else: + return None diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index 3389074bf..cdd57a327 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -6,7 +6,7 @@ from starlette.responses import RedirectResponse, FileResponse import schemas from chalicelib.core import sessions, errors, errors_viewed, errors_favorite, sessions_assignments, heatmaps, \ - sessions_favorite, assist, sessions_notes, click_maps + sessions_favorite, assist, sessions_notes, click_maps, sessions_replay from chalicelib.core import sessions_viewed from chalicelib.core import tenants, users, projects, license from chalicelib.core import webhook @@ -145,13 +145,14 @@ async def get_projects(context: schemas.CurrentContext = Depends(OR_context)): stack_integrations=True)} -@app.get('/{projectId}/sessions/{sessionId}', tags=["sessions"]) +# for backward compatibility +@app.get('/{projectId}/sessions/{sessionId}', tags=["sessions", "replay"]) async def get_session(projectId: int, sessionId: Union[int, str], background_tasks: BackgroundTasks, context: schemas.CurrentContext = Depends(OR_context)): if isinstance(sessionId, str): return {"errors": ["session not found"]} - data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, - include_fav_viewed=True, group_metadata=True, context=context) + data = sessions_replay.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, + include_fav_viewed=True, group_metadata=True, context=context) if data is None: return {"errors": ["session not found"]} if data.get("inDB"): @@ -162,6 +163,37 @@ async def get_session(projectId: int, sessionId: Union[int, str], background_tas } +@app.get('/{projectId}/sessions/{sessionId}/replay', tags=["sessions", "replay"]) +async def get_session_events(projectId: int, sessionId: Union[int, str], background_tasks: BackgroundTasks, + context: schemas.CurrentContext = Depends(OR_context)): + if isinstance(sessionId, str): + return {"errors": ["session not found"]} + data = sessions_replay.get_replay(project_id=projectId, session_id=sessionId, full_data=True, + include_fav_viewed=True, group_metadata=True, context=context) + if data is None: + return {"errors": ["session not found"]} + if data.get("inDB"): + background_tasks.add_task(sessions_viewed.view_session, project_id=projectId, user_id=context.user_id, + session_id=sessionId) + return { + 'data': data + } + + +@app.get('/{projectId}/sessions/{sessionId}/events', tags=["sessions", "replay"]) +async def get_session_events(projectId: int, sessionId: Union[int, str], + context: schemas.CurrentContext = Depends(OR_context)): + if isinstance(sessionId, str): + return {"errors": ["session not found"]} + data = sessions_replay.get_events(project_id=projectId, session_id=sessionId) + if data is None: + return {"errors": ["session not found"]} + + return { + 'data': data + } + + @app.get('/{projectId}/sessions/{sessionId}/errors/{errorId}/sourcemaps', tags=["sessions", "sourcemaps"]) async def get_error_trace(projectId: int, sessionId: int, errorId: str, context: schemas.CurrentContext = Depends(OR_context)): From 89d45d2247dea087687023abfd136b06f676a7f7 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 14 Mar 2023 11:53:20 +0100 Subject: [PATCH 078/253] feat(backend): added skipped session metric for storage service --- backend/internal/storage/storage.go | 2 ++ backend/pkg/metrics/storage/metrics.go | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go index 1e2507163..b1e6b21fb 100644 --- a/backend/internal/storage/storage.go +++ b/backend/internal/storage/storage.go @@ -95,6 +95,7 @@ func (s *Storage) Upload(msg *messages.SessionEnd) (err error) { if err != nil { if strings.Contains(err.Error(), "big file") { log.Printf("%s, sess: %d", err, msg.SessionID()) + metrics.IncreaseStorageTotalSkippedSessions() return nil } return err @@ -110,6 +111,7 @@ func (s *Storage) openSession(filePath string, tp FileType) ([]byte, error) { // Check file size before download into memory info, err := os.Stat(filePath) if err == nil && info.Size() > s.cfg.MaxFileSize { + metrics.RecordSkippedSessionSize(float64(info.Size()), tp.String()) return nil, fmt.Errorf("big file, size: %d", info.Size()) } // Read file into memory diff --git a/backend/pkg/metrics/storage/metrics.go b/backend/pkg/metrics/storage/metrics.go index 26459c90d..2579d7e7c 100644 --- a/backend/pkg/metrics/storage/metrics.go +++ b/backend/pkg/metrics/storage/metrics.go @@ -31,6 +31,32 @@ func IncreaseStorageTotalSessions() { storageTotalSessions.Inc() } +var storageSkippedSessionSize = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "storage", + Name: "session_size_bytes", + Help: "A histogram displaying the size of each skipped session file in bytes.", + Buckets: common.DefaultSizeBuckets, + }, + []string{"file_type"}, +) + +func RecordSkippedSessionSize(fileSize float64, fileType string) { + storageSkippedSessionSize.WithLabelValues(fileType).Observe(fileSize) +} + +var storageTotalSkippedSessions = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "storage", + Name: "sessions_skipped_total", + Help: "A counter displaying the total number of all skipped sessions because of the size limits.", + }, +) + +func IncreaseStorageTotalSkippedSessions() { + storageTotalSkippedSessions.Inc() +} + var storageSessionReadDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "storage", From 68dd0a7f146eb85f665dc8b8b7a4fb9755ae13a3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 13:25:14 +0100 Subject: [PATCH 079/253] feat(chalice): split replay --- api/chalicelib/core/sessions.py | 88 +-------- api/chalicelib/core/sessions_favorite.py | 19 +- api/chalicelib/core/sessions_replay.py | 23 +-- api/routers/core_dynamic.py | 4 +- ee/api/chalicelib/core/sessions.py | 91 +--------- ee/api/chalicelib/core/sessions_favorite.py | 27 +-- ee/api/chalicelib/core/sessions_replay.py | 189 ++++++++++++++++++++ ee/api/routers/core_dynamic.py | 50 +++++- 8 files changed, 273 insertions(+), 218 deletions(-) create mode 100644 ee/api/chalicelib/core/sessions_replay.py diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index c95bed903..8f98aac83 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1,10 +1,7 @@ from typing import List import schemas -from chalicelib.core import events, metadata, events_ios, \ - sessions_mobs, issues, projects, resources, assist, performance_event, sessions_favorite, \ - sessions_devtool, sessions_notes -from chalicelib.utils import errors_helper +from chalicelib.core import events, metadata, projects, performance_event, sessions_favorite from chalicelib.utils import pg_client, helper, metrics_helper from chalicelib.utils import sql_helper as sh @@ -33,89 +30,6 @@ COALESCE((SELECT TRUE AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """ -def __group_metadata(session, project_metadata): - meta = {} - for m in project_metadata.keys(): - if project_metadata[m] is not None and session.get(m) is not None: - meta[project_metadata[m]] = session[m] - session.pop(m) - return meta - - -def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_data=False, include_fav_viewed=False, - group_metadata=False, live=True): - with pg_client.PostgresClient() as cur: - extra_query = [] - if include_fav_viewed: - extra_query.append("""COALESCE((SELECT TRUE - FROM public.user_favorite_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s), FALSE) AS favorite""") - extra_query.append("""COALESCE((SELECT TRUE - FROM public.user_viewed_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s), FALSE) AS viewed""") - query = cur.mogrify( - f"""\ - SELECT - s.*, - s.session_id::text AS session_id, - (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key - {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} - {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''} - FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} - WHERE s.project_id = %(project_id)s - AND s.session_id = %(session_id)s;""", - {"project_id": project_id, "session_id": session_id, "userId": context.user_id} - ) - # print("===============") - # print(query) - cur.execute(query=query) - - data = cur.fetchone() - if data is not None: - data = helper.dict_to_camel_case(data) - if full_data: - if data["platform"] == 'ios': - data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) - for e in data['events']: - if e["type"].endswith("_IOS"): - e["type"] = e["type"][:-len("_IOS")] - data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) - data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, - session_id=session_id) - data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) - else: - data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, - group_clickrage=True) - all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) - data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] - # to keep only the first stack - # limit the number of errors to reduce the response-body size - data['errors'] = [errors_helper.format_first_stack_frame(e) for e in all_errors - if e['source'] == "js_exception"][:500] - data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, - session_id=session_id) - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) - data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) - data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, - start_ts=data["startTs"], duration=data["duration"]) - - data['notes'] = sessions_notes.get_session_notes(tenant_id=context.tenant_id, project_id=project_id, - session_id=session_id, user_id=context.user_id) - data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) - data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) - data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, - project_key=data["projectKey"]) - data["inDB"] = True - return data - elif live: - return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) - else: - return None - - # This function executes the query and return result def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False): diff --git a/api/chalicelib/core/sessions_favorite.py b/api/chalicelib/core/sessions_favorite.py index 00228b31f..d3bf5e9b4 100644 --- a/api/chalicelib/core/sessions_favorite.py +++ b/api/chalicelib/core/sessions_favorite.py @@ -1,5 +1,4 @@ import schemas -from chalicelib.core import sessions from chalicelib.utils import pg_client @@ -8,11 +7,14 @@ def add_favorite_session(context: schemas.CurrentContext, project_id, session_id cur.execute( cur.mogrify(f"""\ INSERT INTO public.user_favorite_sessions(user_id, session_id) - VALUES (%(userId)s,%(session_id)s);""", + VALUES (%(userId)s,%(session_id)s) + RETURNING session_id;""", {"userId": context.user_id, "session_id": session_id}) ) - return sessions.get_by_id2_pg(context=context, project_id=project_id, session_id=session_id, - full_data=False, include_fav_viewed=True) + row = cur.fetchone() + if row: + return {"data": {"sessionId": session_id}} + return {"errors": ["something went wrong"]} def remove_favorite_session(context: schemas.CurrentContext, project_id, session_id): @@ -21,11 +23,14 @@ def remove_favorite_session(context: schemas.CurrentContext, project_id, session cur.mogrify(f"""\ DELETE FROM public.user_favorite_sessions WHERE user_id = %(userId)s - AND session_id = %(session_id)s;""", + AND session_id = %(session_id)s + RETURNING session_id;""", {"userId": context.user_id, "session_id": session_id}) ) - return sessions.get_by_id2_pg(context=context, project_id=project_id, session_id=session_id, - full_data=False, include_fav_viewed=True) + row = cur.fetchone() + if row: + return {"data": {"sessionId": session_id}} + return {"errors": ["something went wrong"]} def favorite_session(context: schemas.CurrentContext, project_id, session_id): diff --git a/api/chalicelib/core/sessions_replay.py b/api/chalicelib/core/sessions_replay.py index 84c6703eb..94e3cc504 100644 --- a/api/chalicelib/core/sessions_replay.py +++ b/api/chalicelib/core/sessions_replay.py @@ -1,12 +1,8 @@ -from typing import List - import schemas from chalicelib.core import events, metadata, events_ios, \ - sessions_mobs, issues, projects, resources, assist, performance_event, sessions_favorite, \ - sessions_devtool, sessions_notes + sessions_mobs, issues, resources, assist, sessions_devtool, sessions_notes from chalicelib.utils import errors_helper -from chalicelib.utils import pg_client, helper, metrics_helper -from chalicelib.utils import sql_helper as sh +from chalicelib.utils import pg_client, helper def __group_metadata(session, project_metadata): @@ -148,8 +144,7 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat def get_events(project_id, session_id): with pg_client.PostgresClient() as cur: query = cur.mogrify( - f"""SELECT s.*, - s.session_id::text AS session_id + f"""SELECT session_id, platform, start_ts, duration FROM public.sessions AS s WHERE s.project_id = %(project_id)s AND s.session_id = %(session_id)s;""", @@ -159,11 +154,11 @@ def get_events(project_id, session_id): # print(query) cur.execute(query=query) - data = cur.fetchone() - if data is not None: - data = helper.dict_to_camel_case(data) - - if data["platform"] == 'ios': + s_data = cur.fetchone() + if s_data is not None: + s_data = helper.dict_to_camel_case(s_data) + data = {} + if s_data["platform"] == 'ios': data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) for e in data['events']: if e["type"].endswith("_IOS"): @@ -183,7 +178,7 @@ def get_events(project_id, session_id): data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, session_id=session_id) data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, - start_ts=data["startTs"], duration=data["duration"]) + start_ts=s_data["startTs"], duration=s_data["duration"]) data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) return data diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index cdd57a327..c6f71e88f 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -271,8 +271,8 @@ async def get_live_session(projectId: int, sessionId: str, background_tasks: Bac context: schemas.CurrentContext = Depends(OR_context)): data = assist.get_live_session_by_id(project_id=projectId, session_id=sessionId) if data is None: - data = sessions.get_by_id2_pg(context=context, project_id=projectId, session_id=sessionId, - full_data=True, include_fav_viewed=True, group_metadata=True, live=False) + data = sessions_replay.get_replay(context=context, project_id=projectId, session_id=sessionId, + full_data=True, include_fav_viewed=True, group_metadata=True, live=False) if data is None: return {"errors": ["session not found"]} if data.get("inDB"): diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py index 6d92c3954..8f98aac83 100644 --- a/ee/api/chalicelib/core/sessions.py +++ b/ee/api/chalicelib/core/sessions.py @@ -1,11 +1,7 @@ from typing import List import schemas -import schemas_ee -from chalicelib.core import events, metadata, events_ios, \ - sessions_mobs, issues, projects, resources, assist, performance_event, sessions_favorite, \ - sessions_devtool, sessions_notes -from chalicelib.utils import errors_helper +from chalicelib.core import events, metadata, projects, performance_event, sessions_favorite from chalicelib.utils import pg_client, helper, metrics_helper from chalicelib.utils import sql_helper as sh @@ -34,91 +30,6 @@ COALESCE((SELECT TRUE AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """ -def __group_metadata(session, project_metadata): - meta = {} - for m in project_metadata.keys(): - if project_metadata[m] is not None and session.get(m) is not None: - meta[project_metadata[m]] = session[m] - session.pop(m) - return meta - - -def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, full_data=False, - include_fav_viewed=False, group_metadata=False, live=True): - with pg_client.PostgresClient() as cur: - extra_query = [] - if include_fav_viewed: - extra_query.append("""COALESCE((SELECT TRUE - FROM public.user_favorite_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s), FALSE) AS favorite""") - extra_query.append("""COALESCE((SELECT TRUE - FROM public.user_viewed_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s), FALSE) AS viewed""") - query = cur.mogrify( - f"""\ - SELECT - s.*, - s.session_id::text AS session_id, - (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key, - encode(file_key,'hex') AS file_key - {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} - {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''} - FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} - WHERE s.project_id = %(project_id)s - AND s.session_id = %(session_id)s;""", - {"project_id": project_id, "session_id": session_id, "userId": context.user_id} - ) - # print("===============") - # print(query) - cur.execute(query=query) - - data = cur.fetchone() - if data is not None: - data = helper.dict_to_camel_case(data) - if full_data: - if data["platform"] == 'ios': - data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) - for e in data['events']: - if e["type"].endswith("_IOS"): - e["type"] = e["type"][:-len("_IOS")] - data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) - data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, - session_id=session_id) - data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) - else: - data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, - group_clickrage=True) - all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) - data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] - # to keep only the first stack - # limit the number of errors to reduce the response-body size - data['errors'] = [errors_helper.format_first_stack_frame(e) for e in all_errors - if e['source'] == "js_exception"][:500] - data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, - session_id=session_id) - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) - data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, - context=context) - data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, - start_ts=data["startTs"], duration=data["duration"]) - - data['notes'] = sessions_notes.get_session_notes(tenant_id=context.tenant_id, project_id=project_id, - session_id=session_id, user_id=context.user_id) - data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) - data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) - data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, - project_key=data["projectKey"]) - data["inDB"] = True - return data - elif live: - return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) - else: - return None - - # This function executes the query and return result def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False): diff --git a/ee/api/chalicelib/core/sessions_favorite.py b/ee/api/chalicelib/core/sessions_favorite.py index d8ae4e1f7..85e308756 100644 --- a/ee/api/chalicelib/core/sessions_favorite.py +++ b/ee/api/chalicelib/core/sessions_favorite.py @@ -10,13 +10,15 @@ def add_favorite_session(context: schemas_ee.CurrentContext, project_id, session cur.execute( cur.mogrify(f"""\ INSERT INTO public.user_favorite_sessions(user_id, session_id) - VALUES (%(userId)s,%(sessionId)s);""", - {"userId": context.user_id, "sessionId": session_id}) + VALUES (%(userId)s,%(session_id)s) + RETURNING session_id;""", + {"userId": context.user_id, "session_id": session_id}) ) - - sessions_favorite_exp.add_favorite_session(project_id=project_id, user_id=context.user_id, session_id=session_id) - return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, - full_data=False, include_fav_viewed=True, context=context) + row = cur.fetchone() + if row: + sessions_favorite_exp.add_favorite_session(project_id=project_id, user_id=context.user_id, session_id=session_id) + return {"data": {"sessionId": session_id}} + return {"errors": ["something went wrong"]} def remove_favorite_session(context: schemas_ee.CurrentContext, project_id, session_id): @@ -25,12 +27,15 @@ def remove_favorite_session(context: schemas_ee.CurrentContext, project_id, sess cur.mogrify(f"""\ DELETE FROM public.user_favorite_sessions WHERE user_id = %(userId)s - AND session_id = %(sessionId)s;""", - {"userId": context.user_id, "sessionId": session_id}) + AND session_id = %(session_id)s + RETURNING session_id;""", + {"userId": context.user_id, "session_id": session_id}) ) - sessions_favorite_exp.remove_favorite_session(project_id=project_id, user_id=context.user_id, session_id=session_id) - return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, - full_data=False, include_fav_viewed=True, context=context) + row = cur.fetchone() + if row: + sessions_favorite_exp.remove_favorite_session(project_id=project_id, user_id=context.user_id, session_id=session_id) + return {"data": {"sessionId": session_id}} + return {"errors": ["something went wrong"]} def favorite_session(context: schemas_ee.CurrentContext, project_id, session_id): diff --git a/ee/api/chalicelib/core/sessions_replay.py b/ee/api/chalicelib/core/sessions_replay.py new file mode 100644 index 000000000..798029aee --- /dev/null +++ b/ee/api/chalicelib/core/sessions_replay.py @@ -0,0 +1,189 @@ +import schemas +import schemas_ee +from chalicelib.core import events, metadata, events_ios, \ + sessions_mobs, issues, resources, assist, sessions_devtool, sessions_notes +from chalicelib.utils import errors_helper +from chalicelib.utils import pg_client, helper + + +def __group_metadata(session, project_metadata): + meta = {} + for m in project_metadata.keys(): + if project_metadata[m] is not None and session.get(m) is not None: + meta[project_metadata[m]] = session[m] + session.pop(m) + return meta + + +# for backward compatibility +def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, full_data=False, + include_fav_viewed=False, group_metadata=False, live=True): + with pg_client.PostgresClient() as cur: + extra_query = [] + if include_fav_viewed: + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite""") + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS viewed""") + query = cur.mogrify( + f"""\ + SELECT + s.*, + s.session_id::text AS session_id, + (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key, + encode(file_key,'hex') AS file_key + {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} + {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''} + FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id, "userId": context.user_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + data = cur.fetchone() + if data is not None: + data = helper.dict_to_camel_case(data) + if full_data: + if data["platform"] == 'ios': + data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) + for e in data['events']: + if e["type"].endswith("_IOS"): + e["type"] = e["type"][:-len("_IOS")] + data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) + data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, + session_id=session_id) + data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) + else: + data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, + group_clickrage=True) + all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) + data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] + # to keep only the first stack + # limit the number of errors to reduce the response-body size + data['errors'] = [errors_helper.format_first_stack_frame(e) for e in all_errors + if e['source'] == "js_exception"][:500] + data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, + session_id=session_id) + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, + context=context) + data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, + start_ts=data["startTs"], duration=data["duration"]) + + data['notes'] = sessions_notes.get_session_notes(tenant_id=context.tenant_id, project_id=project_id, + session_id=session_id, user_id=context.user_id) + data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) + data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) + data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, + project_key=data["projectKey"]) + data["inDB"] = True + return data + elif live: + return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) + else: + return None + + +def get_replay(project_id, session_id, context: schemas.CurrentContext, full_data=False, include_fav_viewed=False, + group_metadata=False, live=True): + with pg_client.PostgresClient() as cur: + extra_query = [] + if include_fav_viewed: + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_favorite_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS favorite""") + extra_query.append("""COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s), FALSE) AS viewed""") + query = cur.mogrify( + f"""\ + SELECT + s.*, + s.session_id::text AS session_id, + (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key + {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} + {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''} + FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id, "userId": context.user_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + data = cur.fetchone() + if data is not None: + data = helper.dict_to_camel_case(data) + if full_data: + if data["platform"] == 'ios': + data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) + else: + data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) + data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) + + data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) + data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, + project_key=data["projectKey"]) + data["inDB"] = True + return data + elif live: + return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) + else: + return None + + +def get_events(project_id, session_id): + with pg_client.PostgresClient() as cur: + query = cur.mogrify( + f"""SELECT session_id, platform, start_ts, duration + FROM public.sessions AS s + WHERE s.project_id = %(project_id)s + AND s.session_id = %(session_id)s;""", + {"project_id": project_id, "session_id": session_id} + ) + # print("===============") + # print(query) + cur.execute(query=query) + + s_data = cur.fetchone() + if s_data is not None: + s_data = helper.dict_to_camel_case(s_data) + data = {} + if s_data["platform"] == 'ios': + data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) + for e in data['events']: + if e["type"].endswith("_IOS"): + e["type"] = e["type"][:-len("_IOS")] + data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) + data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, + session_id=session_id) + else: + data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, + group_clickrage=True) + all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) + data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] + # to keep only the first stack + # limit the number of errors to reduce the response-body size + data['errors'] = [errors_helper.format_first_stack_frame(e) for e in all_errors + if e['source'] == "js_exception"][:500] + data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, + session_id=session_id) + data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, + start_ts=s_data["startTs"], duration=s_data["duration"]) + + data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) + return data + else: + return None diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index 8c8aa55b6..209fdbd6d 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -7,7 +7,7 @@ from starlette.responses import RedirectResponse, FileResponse import schemas import schemas_ee from chalicelib.core import sessions, assist, heatmaps, sessions_favorite, sessions_assignments, errors, errors_viewed, \ - errors_favorite, sessions_notes, click_maps + errors_favorite, sessions_notes, click_maps, sessions_replay from chalicelib.core import sessions_viewed from chalicelib.core import tenants, users, projects, license from chalicelib.core import webhook @@ -59,7 +59,8 @@ async def edit_account(data: schemas_ee.EditUserSchema = Body(...), @app.post('/integrations/slack', tags=['integrations']) @app.put('/integrations/slack', tags=['integrations']) -async def add_slack_client(data: schemas.AddCollaborationSchema, context: schemas.CurrentContext = Depends(OR_context)): +async def add_slack_integration(data: schemas.AddCollaborationSchema, + context: schemas.CurrentContext = Depends(OR_context)): n = Slack.add(tenant_id=context.tenant_id, data=data) if n is None: return { @@ -155,13 +156,15 @@ async def get_projects(context: schemas.CurrentContext = Depends(OR_context)): stack_integrations=True, user_id=context.user_id)} -@app.get('/{projectId}/sessions/{sessionId}', tags=["sessions"], dependencies=[OR_scope(Permissions.session_replay)]) +# for backward compatibility +@app.get('/{projectId}/sessions/{sessionId}', tags=["sessions", "replay"], + dependencies=[OR_scope(Permissions.session_replay)]) async def get_session(projectId: int, sessionId: Union[int, str], background_tasks: BackgroundTasks, context: schemas.CurrentContext = Depends(OR_context)): if isinstance(sessionId, str): return {"errors": ["session not found"]} - data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, - include_fav_viewed=True, group_metadata=True, context=context) + data = sessions_replay.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, + include_fav_viewed=True, group_metadata=True, context=context) if data is None: return {"errors": ["session not found"]} if data.get("inDB"): @@ -172,6 +175,39 @@ async def get_session(projectId: int, sessionId: Union[int, str], background_tas } +@app.get('/{projectId}/sessions/{sessionId}/replay', tags=["sessions", "replay"], + dependencies=[OR_scope(Permissions.session_replay)]) +async def get_session_events(projectId: int, sessionId: Union[int, str], background_tasks: BackgroundTasks, + context: schemas.CurrentContext = Depends(OR_context)): + if isinstance(sessionId, str): + return {"errors": ["session not found"]} + data = sessions_replay.get_replay(project_id=projectId, session_id=sessionId, full_data=True, + include_fav_viewed=True, group_metadata=True, context=context) + if data is None: + return {"errors": ["session not found"]} + if data.get("inDB"): + background_tasks.add_task(sessions_viewed.view_session, project_id=projectId, user_id=context.user_id, + session_id=sessionId) + return { + 'data': data + } + + +@app.get('/{projectId}/sessions/{sessionId}/events', tags=["sessions", "replay"], + dependencies=[OR_scope(Permissions.session_replay)]) +async def get_session_events(projectId: int, sessionId: Union[int, str], + context: schemas.CurrentContext = Depends(OR_context)): + if isinstance(sessionId, str): + return {"errors": ["session not found"]} + data = sessions_replay.get_events(project_id=projectId, session_id=sessionId) + if data is None: + return {"errors": ["session not found"]} + + return { + 'data': data + } + + @app.get('/{projectId}/sessions/{sessionId}/errors/{errorId}/sourcemaps', tags=["sessions", "sourcemaps"], dependencies=[OR_scope(Permissions.dev_tools)]) async def get_error_trace(projectId: int, sessionId: int, errorId: str, @@ -250,8 +286,8 @@ async def get_live_session(projectId: int, sessionId: str, background_tasks: Bac context: schemas_ee.CurrentContext = Depends(OR_context)): data = assist.get_live_session_by_id(project_id=projectId, session_id=sessionId) if data is None: - data = sessions.get_by_id2_pg(context=context, project_id=projectId, session_id=sessionId, - full_data=True, include_fav_viewed=True, group_metadata=True, live=False) + data = sessions_replay.get_replay(context=context, project_id=projectId, session_id=sessionId, + full_data=True, include_fav_viewed=True, group_metadata=True, live=False) if data is None: return {"errors": ["session not found"]} if data.get("inDB"): From 0654808f68f87d5d5f4cc9d3d69c5a2c31ef99de Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 14:03:45 +0100 Subject: [PATCH 080/253] feat(chalice): code cleaning --- ee/api/.gitignore | 1 + ee/api/chalicelib/core/sessions.py | 1126 ---------------------------- ee/api/clean-dev.sh | 1 + 3 files changed, 2 insertions(+), 1126 deletions(-) delete mode 100644 ee/api/chalicelib/core/sessions.py diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 9a9636ee1..27ac41f5c 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -215,6 +215,7 @@ Pipfile.lock /chalicelib/core/log_tool_sumologic.py /chalicelib/core/metadata.py /chalicelib/core/mobile.py +/chalicelib/core/sessions.py /chalicelib/core/sessions_assignments.py #exp /chalicelib/core/sessions_metas.py /chalicelib/core/sessions_mobs.py diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py deleted file mode 100644 index 8f98aac83..000000000 --- a/ee/api/chalicelib/core/sessions.py +++ /dev/null @@ -1,1126 +0,0 @@ -from typing import List - -import schemas -from chalicelib.core import events, metadata, projects, performance_event, sessions_favorite -from chalicelib.utils import pg_client, helper, metrics_helper -from chalicelib.utils import sql_helper as sh - -SESSION_PROJECTION_COLS = """s.project_id, -s.session_id::text AS session_id, -s.user_uuid, -s.user_id, -s.user_os, -s.user_browser, -s.user_device, -s.user_device_type, -s.user_country, -s.start_ts, -s.duration, -s.events_count, -s.pages_count, -s.errors_count, -s.user_anonymous_id, -s.platform, -s.issue_score, -to_jsonb(s.issue_types) AS issue_types, -favorite_sessions.session_id NOTNULL AS favorite, -COALESCE((SELECT TRUE - FROM public.user_viewed_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """ - - -# This function executes the query and return result -def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, - error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False): - if data.bookmarked: - data.startDate, data.endDate = sessions_favorite.get_start_end_timestamp(project_id, user_id) - - full_args, query_part = search_query_parts(data=data, error_status=error_status, errors_only=errors_only, - favorite_only=data.bookmarked, issue=issue, project_id=project_id, - user_id=user_id) - if data.limit is not None and data.page is not None: - full_args["sessions_limit"] = data.limit - full_args["sessions_limit_s"] = (data.page - 1) * data.limit - full_args["sessions_limit_e"] = data.page * data.limit - else: - full_args["sessions_limit"] = 200 - full_args["sessions_limit_s"] = 1 - full_args["sessions_limit_e"] = 200 - - meta_keys = [] - with pg_client.PostgresClient() as cur: - if errors_only: - main_query = cur.mogrify(f"""SELECT DISTINCT er.error_id, - COALESCE((SELECT TRUE - FROM public.user_viewed_errors AS ve - WHERE er.error_id = ve.error_id - AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed - {query_part};""", full_args) - - elif count_only: - main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, - COUNT(DISTINCT s.user_uuid) AS count_users - {query_part};""", full_args) - elif data.group_by_user: - g_sort = "count(full_sessions)" - if data.order is None: - data.order = schemas.SortOrderType.desc.value - else: - data.order = data.order.value - if data.sort is not None and data.sort != 'sessionsCount': - sort = helper.key_to_snake_case(data.sort) - g_sort = f"{'MIN' if data.order == schemas.SortOrderType.desc else 'MAX'}({sort})" - else: - sort = 'start_ts' - - meta_keys = metadata.get(project_id=project_id) - main_query = cur.mogrify(f"""SELECT COUNT(*) AS count, - COALESCE(JSONB_AGG(users_sessions) - FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions - FROM (SELECT user_id, - count(full_sessions) AS user_sessions_count, - jsonb_agg(full_sessions) FILTER (WHERE rn <= 1) AS last_session, - MIN(full_sessions.start_ts) AS first_session_ts, - ROW_NUMBER() OVER (ORDER BY {g_sort} {data.order}) AS rn - FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY {sort} {data.order}) AS rn - FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} - {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} - {query_part} - ) AS filtred_sessions - ) AS full_sessions - GROUP BY user_id - ) AS users_sessions;""", - full_args) - elif ids_only: - main_query = cur.mogrify(f"""SELECT DISTINCT ON(s.session_id) s.session_id - {query_part} - ORDER BY s.session_id desc - LIMIT %(sessions_limit)s OFFSET %(sessions_limit_s)s;""", - full_args) - else: - if data.order is None: - data.order = schemas.SortOrderType.desc.value - else: - data.order = data.order.value - sort = 'session_id' - if data.sort is not None and data.sort != "session_id": - # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) - sort = helper.key_to_snake_case(data.sort) - meta_keys = metadata.get(project_id=project_id) - main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, - COALESCE(JSONB_AGG(full_sessions) - FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions - FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY {sort} {data.order}, issue_score DESC) AS rn - FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} - {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} - {query_part} - ORDER BY s.session_id desc) AS filtred_sessions - ORDER BY {sort} {data.order}, issue_score DESC) AS full_sessions;""", - full_args) - # print("--------------------") - # print(main_query) - # print("--------------------") - try: - cur.execute(main_query) - except Exception as err: - print("--------- SESSIONS SEARCH QUERY EXCEPTION -----------") - print(main_query.decode('UTF-8')) - print("--------- PAYLOAD -----------") - print(data.json()) - print("--------------------") - raise err - if errors_only or ids_only: - return helper.list_to_camel_case(cur.fetchall()) - - sessions = cur.fetchone() - if count_only: - return helper.dict_to_camel_case(sessions) - - total = sessions["count"] - sessions = sessions["sessions"] - - if data.group_by_user: - for i, s in enumerate(sessions): - sessions[i] = {**s.pop("last_session")[0], **s} - sessions[i].pop("rn") - sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ - if sessions[i][f'metadata_{k["index"]}'] is not None} - else: - for i, s in enumerate(sessions): - sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ - if sessions[i][f'metadata_{k["index"]}'] is not None} - # if not data.group_by_user and data.sort is not None and data.sort != "session_id": - # sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], - # reverse=data.order.upper() == "DESC") - return { - 'total': total, - 'sessions': helper.list_to_camel_case(sessions) - } - - -def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, density: int, - view_type: schemas.MetricTimeseriesViewType, metric_type: schemas.MetricType, - metric_of: schemas.MetricOfTable, metric_value: List): - step_size = int(metrics_helper.__get_step_size(endTimestamp=data.endDate, startTimestamp=data.startDate, - density=density, factor=1, decimal=True)) - extra_event = None - if metric_of == schemas.MetricOfTable.visited_url: - extra_event = "events.pages" - elif metric_of == schemas.MetricOfTable.issues and len(metric_value) > 0: - data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.issue, - operator=schemas.SearchEventOperator._is)) - full_args, query_part = search_query_parts(data=data, error_status=None, errors_only=False, - favorite_only=False, issue=None, project_id=project_id, - user_id=None, extra_event=extra_event) - full_args["step_size"] = step_size - sessions = [] - with pg_client.PostgresClient() as cur: - if metric_type == schemas.MetricType.timeseries: - if view_type == schemas.MetricTimeseriesViewType.line_chart: - main_query = cur.mogrify(f"""WITH full_sessions AS (SELECT DISTINCT ON(s.session_id) s.session_id, s.start_ts - {query_part}) - SELECT generated_timestamp AS timestamp, - COUNT(s) AS count - FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT 1 AS s - FROM full_sessions - WHERE start_ts >= generated_timestamp - AND start_ts <= generated_timestamp + %(step_size)s) AS sessions ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""", full_args) - else: - main_query = cur.mogrify(f"""SELECT count(DISTINCT s.session_id) AS count - {query_part};""", full_args) - - # print("--------------------") - # print(main_query) - # print("--------------------") - try: - cur.execute(main_query) - except Exception as err: - print("--------- SESSIONS-SERIES QUERY EXCEPTION -----------") - print(main_query.decode('UTF-8')) - print("--------- PAYLOAD -----------") - print(data.json()) - print("--------------------") - raise err - if view_type == schemas.MetricTimeseriesViewType.line_chart: - sessions = cur.fetchall() - else: - sessions = cur.fetchone()["count"] - elif metric_type == schemas.MetricType.table: - if isinstance(metric_of, schemas.MetricOfTable): - main_col = "user_id" - extra_col = "" - extra_where = "" - pre_query = "" - distinct_on = "s.session_id" - if metric_of == schemas.MetricOfTable.user_country: - main_col = "user_country" - elif metric_of == schemas.MetricOfTable.user_device: - main_col = "user_device" - elif metric_of == schemas.MetricOfTable.user_browser: - main_col = "user_browser" - elif metric_of == schemas.MetricOfTable.issues: - main_col = "issue" - extra_col = f", UNNEST(s.issue_types) AS {main_col}" - if len(metric_value) > 0: - extra_where = [] - for i in range(len(metric_value)): - arg_name = f"selected_issue_{i}" - extra_where.append(f"{main_col} = %({arg_name})s") - full_args[arg_name] = metric_value[i] - extra_where = f"WHERE ({' OR '.join(extra_where)})" - elif metric_of == schemas.MetricOfTable.visited_url: - main_col = "path" - extra_col = ", path" - distinct_on += ",path" - main_query = cur.mogrify(f"""{pre_query} - SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values - FROM (SELECT {main_col} AS name, - count(DISTINCT session_id) AS session_count, - ROW_NUMBER() OVER (ORDER BY count(full_sessions) DESC) AS rn - FROM (SELECT * - FROM (SELECT DISTINCT ON({distinct_on}) s.session_id, s.user_uuid, - s.user_id, s.user_os, - s.user_browser, s.user_device, - s.user_device_type, s.user_country, s.issue_types{extra_col} - {query_part} - ORDER BY s.session_id desc) AS filtred_sessions - ) AS full_sessions - {extra_where} - GROUP BY {main_col} - ORDER BY session_count DESC) AS users_sessions;""", - full_args) - # print("--------------------") - # print(main_query) - # print("--------------------") - cur.execute(main_query) - sessions = cur.fetchone() - for s in sessions["values"]: - s.pop("rn") - sessions["values"] = helper.list_to_camel_case(sessions["values"]) - - return sessions - - -def __is_valid_event(is_any: bool, event: schemas._SessionSearchEventSchema): - return not (not is_any and len(event.value) == 0 and event.type not in [schemas.EventType.request_details, - schemas.EventType.graphql] \ - or event.type in [schemas.PerformanceEventType.location_dom_complete, - schemas.PerformanceEventType.location_largest_contentful_paint_time, - schemas.PerformanceEventType.location_ttfb, - schemas.PerformanceEventType.location_avg_cpu_load, - schemas.PerformanceEventType.location_avg_memory_usage - ] and (event.source is None or len(event.source) == 0) \ - or event.type in [schemas.EventType.request_details, schemas.EventType.graphql] and ( - event.filters is None or len(event.filters) == 0)) - - -# this function generates the query and return the generated-query with the dict of query arguments -def search_query_parts(data: schemas.SessionsSearchPayloadSchema, error_status, errors_only, favorite_only, issue, - project_id, user_id, extra_event=None): - ss_constraints = [] - full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, - "projectId": project_id, "userId": user_id} - extra_constraints = [ - "s.project_id = %(project_id)s", - "s.duration IS NOT NULL" - ] - extra_from = "" - events_query_part = "" - if len(data.filters) > 0: - meta_keys = None - for i, f in enumerate(data.filters): - if not isinstance(f.value, list): - f.value = [f.value] - filter_type = f.type - f.value = helper.values_for_operator(value=f.value, op=f.operator) - f_k = f"f_value{i}" - full_args = {**full_args, **sh.multi_values(f.value, value_key=f_k)} - op = sh.get_sql_operator(f.operator) \ - if filter_type not in [schemas.FilterType.events_count] else f.operator - is_any = sh.isAny_opreator(f.operator) - is_undefined = sh.isUndefined_operator(f.operator) - if not is_any and not is_undefined and len(f.value) == 0: - continue - is_not = False - if sh.is_negation_operator(f.operator): - is_not = True - if filter_type == schemas.FilterType.user_browser: - if is_any: - extra_constraints.append('s.user_browser IS NOT NULL') - ss_constraints.append('ms.user_browser IS NOT NULL') - else: - extra_constraints.append( - sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: - if is_any: - extra_constraints.append('s.user_os IS NOT NULL') - ss_constraints.append('ms.user_os IS NOT NULL') - else: - extra_constraints.append( - sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: - if is_any: - extra_constraints.append('s.user_device IS NOT NULL') - ss_constraints.append('ms.user_device IS NOT NULL') - else: - extra_constraints.append( - sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: - if is_any: - extra_constraints.append('s.user_country IS NOT NULL') - ss_constraints.append('ms.user_country IS NOT NULL') - else: - extra_constraints.append( - sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - - elif filter_type in [schemas.FilterType.utm_source]: - if is_any: - extra_constraints.append('s.utm_source IS NOT NULL') - ss_constraints.append('ms.utm_source IS NOT NULL') - elif is_undefined: - extra_constraints.append('s.utm_source IS NULL') - ss_constraints.append('ms.utm_source IS NULL') - else: - extra_constraints.append( - sh.multi_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.utm_medium]: - if is_any: - extra_constraints.append('s.utm_medium IS NOT NULL') - ss_constraints.append('ms.utm_medium IS NOT NULL') - elif is_undefined: - extra_constraints.append('s.utm_medium IS NULL') - ss_constraints.append('ms.utm_medium IS NULL') - else: - extra_constraints.append( - sh.multi_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.utm_campaign]: - if is_any: - extra_constraints.append('s.utm_campaign IS NOT NULL') - ss_constraints.append('ms.utm_campaign IS NOT NULL') - elif is_undefined: - extra_constraints.append('s.utm_campaign IS NULL') - ss_constraints.append('ms.utm_campaign IS NULL') - else: - extra_constraints.append( - sh.multi_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not, - value_key=f_k)) - - elif filter_type == schemas.FilterType.duration: - if len(f.value) > 0 and f.value[0] is not None: - extra_constraints.append("s.duration >= %(minDuration)s") - ss_constraints.append("ms.duration >= %(minDuration)s") - full_args["minDuration"] = f.value[0] - if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: - extra_constraints.append("s.duration <= %(maxDuration)s") - ss_constraints.append("ms.duration <= %(maxDuration)s") - full_args["maxDuration"] = f.value[1] - elif filter_type == schemas.FilterType.referrer: - # extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" - if is_any: - extra_constraints.append('s.base_referrer IS NOT NULL') - else: - extra_constraints.append( - sh.multi_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type == events.EventType.METADATA.ui_type: - # get metadata list only if you need it - if meta_keys is None: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} - if f.source in meta_keys.keys(): - if is_any: - extra_constraints.append(f"s.{metadata.index_to_colname(meta_keys[f.source])} IS NOT NULL") - ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NOT NULL") - elif is_undefined: - extra_constraints.append(f"s.{metadata.index_to_colname(meta_keys[f.source])} IS NULL") - ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NULL") - else: - extra_constraints.append( - sh.multi_conditions( - f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", - f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - sh.multi_conditions( - f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text", - f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: - if is_any: - extra_constraints.append('s.user_id IS NOT NULL') - ss_constraints.append('ms.user_id IS NOT NULL') - elif is_undefined: - extra_constraints.append('s.user_id IS NULL') - ss_constraints.append('ms.user_id IS NULL') - else: - extra_constraints.append( - sh.multi_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.user_anonymous_id, - schemas.FilterType.user_anonymous_id_ios]: - if is_any: - extra_constraints.append('s.user_anonymous_id IS NOT NULL') - ss_constraints.append('ms.user_anonymous_id IS NOT NULL') - elif is_undefined: - extra_constraints.append('s.user_anonymous_id IS NULL') - ss_constraints.append('ms.user_anonymous_id IS NULL') - else: - extra_constraints.append( - sh.multi_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: - if is_any: - extra_constraints.append('s.rev_id IS NOT NULL') - ss_constraints.append('ms.rev_id IS NOT NULL') - elif is_undefined: - extra_constraints.append('s.rev_id IS NULL') - ss_constraints.append('ms.rev_id IS NULL') - else: - extra_constraints.append( - sh.multi_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type == schemas.FilterType.platform: - # op = __ sh.get_sql_operator(f.operator) - extra_constraints.append( - sh.multi_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type == schemas.FilterType.issue: - if is_any: - extra_constraints.append("array_length(s.issue_types, 1) > 0") - ss_constraints.append("array_length(ms.issue_types, 1) > 0") - else: - extra_constraints.append( - sh.multi_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, - value_key=f_k)) - # search sessions with click_rage on a specific selector - if len(f.filters) > 0 and schemas.IssueType.click_rage in f.value: - for j, sf in enumerate(f.filters): - if sf.operator == schemas.IssueFilterOperator._on_selector: - f_k = f"f_value{i}_{j}" - full_args = {**full_args, **sh.multi_values(sf.value, value_key=f_k)} - extra_constraints += ["mc.timestamp>=%(startDate)s", - "mc.timestamp<=%(endDate)s", - "mis.type='click_rage'", - sh.multi_conditions(f"mc.selector=%({f_k})s", - sf.value, is_not=is_not, - value_key=f_k)] - - extra_from += """INNER JOIN events.clicks AS mc USING(session_id) - INNER JOIN events_common.issues USING (session_id,timestamp) - INNER JOIN public.issues AS mis USING (issue_id)\n""" - - elif filter_type == schemas.FilterType.events_count: - extra_constraints.append( - sh.multi_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - sh.multi_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - # --------------------------------------------------------------------------- - if len(data.events) > 0: - valid_events_count = 0 - for event in data.events: - is_any = sh.isAny_opreator(event.operator) - if not isinstance(event.value, list): - event.value = [event.value] - if __is_valid_event(is_any=is_any, event=event): - valid_events_count += 1 - events_query_from = [] - event_index = 0 - or_events = data.events_order == schemas.SearchEventOrder._or - # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " - events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " - for i, event in enumerate(data.events): - event_type = event.type - is_any = sh.isAny_opreator(event.operator) - if not isinstance(event.value, list): - event.value = [event.value] - if not __is_valid_event(is_any=is_any, event=event): - continue - op = sh.get_sql_operator(event.operator) - is_not = False - if sh.is_negation_operator(event.operator): - is_not = True - op = sh.reverse_sql_operator(op) - if event_index == 0 or or_events: - event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" - event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", - "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", - "ms.start_ts <= %(endDate)s", "ms.duration IS NOT NULL"] - if favorite_only and not errors_only: - event_from += "INNER JOIN public.user_favorite_sessions AS fs USING(session_id)" - event_where.append("fs.user_id = %(userId)s") - else: - event_from = "%s" - event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", - "main.session_id=event_0.session_id"] - if data.events_order == schemas.SearchEventOrder._then: - event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") - e_k = f"e_value{i}" - s_k = e_k + "_source" - if event.type != schemas.PerformanceEventType.time_between_events: - event.value = helper.values_for_operator(value=event.value, op=event.operator) - full_args = {**full_args, - **sh.multi_values(event.value, value_key=e_k), - **sh.multi_values(event.source, value_key=s_k)} - - if event_type == events.EventType.CLICK.ui_type: - event_from = event_from % f"{events.EventType.CLICK.table} AS main " - if not is_any: - if event.operator == schemas.ClickEventExtraOperator._on_selector: - event_where.append( - sh.multi_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k)) - else: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.CLICK.column} {op} %({e_k})s", event.value, - value_key=e_k)) - - elif event_type == events.EventType.INPUT.ui_type: - event_from = event_from % f"{events.EventType.INPUT.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.INPUT.column} {op} %({e_k})s", event.value, - value_key=e_k)) - if event.source is not None and len(event.source) > 0: - event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source, - value_key=f"custom{i}")) - full_args = {**full_args, **sh.multi_values(event.source, value_key=f"custom{i}")} - - elif event_type == events.EventType.LOCATION.ui_type: - event_from = event_from % f"{events.EventType.LOCATION.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.EventType.CUSTOM.ui_type: - event_from = event_from % f"{events.EventType.CUSTOM.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.CUSTOM.column} {op} %({e_k})s", event.value, - value_key=e_k)) - elif event_type == events.EventType.REQUEST.ui_type: - event_from = event_from % f"{events.EventType.REQUEST.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.REQUEST.column} {op} %({e_k})s", event.value, - value_key=e_k)) - # elif event_type == events.event_type.GRAPHQL.ui_type: - # event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " - # if not is_any: - # event_where.append( - # _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, - # value_key=e_k)) - elif event_type == events.EventType.STATEACTION.ui_type: - event_from = event_from % f"{events.EventType.STATEACTION.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.STATEACTION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.EventType.ERROR.ui_type: - event_from = event_from % f"{events.EventType.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" - event.source = list(set(event.source)) - if not is_any and event.value not in [None, "*", ""]: - event_where.append( - sh.multi_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)", - event.value, value_key=e_k)) - if len(event.source) > 0 and event.source[0] not in [None, "*", ""]: - event_where.append(sh.multi_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k)) - - - # ----- IOS - elif event_type == events.EventType.CLICK_IOS.ui_type: - event_from = event_from % f"{events.EventType.CLICK_IOS.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.CLICK_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - - elif event_type == events.EventType.INPUT_IOS.ui_type: - event_from = event_from % f"{events.EventType.INPUT_IOS.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.INPUT_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - if event.source is not None and len(event.source) > 0: - event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source, - value_key="custom{i}")) - full_args = {**full_args, **sh.multi_values(event.source, f"custom{i}")} - elif event_type == events.EventType.VIEW_IOS.ui_type: - event_from = event_from % f"{events.EventType.VIEW_IOS.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.VIEW_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.EventType.CUSTOM_IOS.ui_type: - event_from = event_from % f"{events.EventType.CUSTOM_IOS.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.CUSTOM_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.EventType.REQUEST_IOS.ui_type: - event_from = event_from % f"{events.EventType.REQUEST_IOS.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.REQUEST_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.EventType.ERROR_IOS.ui_type: - event_from = event_from % f"{events.EventType.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" - if not is_any and event.value not in [None, "*", ""]: - event_where.append( - sh.multi_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", - event.value, value_key=e_k)) - elif event_type == schemas.PerformanceEventType.fetch_failed: - event_from = event_from % f"{events.EventType.REQUEST.table} AS main " - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.REQUEST.column} {op} %({e_k})s", - event.value, value_key=e_k)) - col = performance_event.get_col(event_type) - colname = col["column"] - event_where.append(f"main.{colname} = FALSE") - # elif event_type == schemas.PerformanceEventType.fetch_duration: - # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - # if not is_any: - # event_where.append( - # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", - # event.value, value_key=e_k)) - # col = performance_event.get_col(event_type) - # colname = col["column"] - # tname = "main" - # e_k += "_custom" - # full_args = {**full_args, **_ sh.multiple_values(event.source, value_key=e_k)} - # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - # _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", - # event.source, value_key=e_k)) - elif event_type in [schemas.PerformanceEventType.location_dom_complete, - schemas.PerformanceEventType.location_largest_contentful_paint_time, - schemas.PerformanceEventType.location_ttfb, - schemas.PerformanceEventType.location_avg_cpu_load, - schemas.PerformanceEventType.location_avg_memory_usage - ]: - event_from = event_from % f"{events.EventType.LOCATION.table} AS main " - col = performance_event.get_col(event_type) - colname = col["column"] - tname = "main" - if col.get("extraJoin") is not None: - tname = "ej" - event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" - event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", - f"{tname}.timestamp <= %(endDate)s"] - if not is_any: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - e_k += "_custom" - full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)} - - event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - sh.multi_conditions(f"{tname}.{colname} {event.sourceOperator.value} %({e_k})s", - event.source, value_key=e_k)) - elif event_type == schemas.PerformanceEventType.time_between_events: - event_from = event_from % f"{getattr(events.EventType, event.value[0].type).table} AS main INNER JOIN {getattr(events.EventType, event.value[1].type).table} AS main2 USING(session_id) " - if not isinstance(event.value[0].value, list): - event.value[0].value = [event.value[0].value] - if not isinstance(event.value[1].value, list): - event.value[1].value = [event.value[1].value] - event.value[0].value = helper.values_for_operator(value=event.value[0].value, - op=event.value[0].operator) - event.value[1].value = helper.values_for_operator(value=event.value[1].value, - op=event.value[0].operator) - e_k1 = e_k + "_e1" - e_k2 = e_k + "_e2" - full_args = {**full_args, - **sh.multi_values(event.value[0].value, value_key=e_k1), - **sh.multi_values(event.value[1].value, value_key=e_k2)} - s_op = sh.get_sql_operator(event.value[0].operator) - event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] - if event_index > 0 and not or_events: - event_where.append("main2.session_id=event_0.session_id") - is_any = sh.isAny_opreator(event.value[0].operator) - if not is_any: - event_where.append( - sh.multi_conditions( - f"main.{getattr(events.EventType, event.value[0].type).column} {s_op} %({e_k1})s", - event.value[0].value, value_key=e_k1)) - s_op = sh.get_sql_operator(event.value[1].operator) - is_any = sh.isAny_opreator(event.value[1].operator) - if not is_any: - event_where.append( - sh.multi_conditions( - f"main2.{getattr(events.EventType, event.value[1].type).column} {s_op} %({e_k2})s", - event.value[1].value, value_key=e_k2)) - - e_k += "_custom" - full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)} - event_where.append( - sh.multi_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator.value} %({e_k})s", - event.source, value_key=e_k)) - - elif event_type == schemas.EventType.request_details: - event_from = event_from % f"{events.EventType.REQUEST.table} AS main " - apply = False - for j, f in enumerate(event.filters): - is_any = sh.isAny_opreator(f.operator) - if is_any or len(f.value) == 0: - continue - f.value = helper.values_for_operator(value=f.value, op=f.operator) - op = sh.get_sql_operator(f.operator) - e_k_f = e_k + f"_fetch{j}" - full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)} - if f.type == schemas.FetchFilterType._url: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.REQUEST.column} {op} %({e_k_f})s::text", - f.value, value_key=e_k_f)) - apply = True - elif f.type == schemas.FetchFilterType._status_code: - event_where.append( - sh.multi_conditions(f"main.status_code {f.operator.value} %({e_k_f})s::integer", f.value, - value_key=e_k_f)) - apply = True - elif f.type == schemas.FetchFilterType._method: - event_where.append( - sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) - apply = True - elif f.type == schemas.FetchFilterType._duration: - event_where.append( - sh.multi_conditions(f"main.duration {f.operator.value} %({e_k_f})s::integer", f.value, - value_key=e_k_f)) - apply = True - elif f.type == schemas.FetchFilterType._request_body: - event_where.append( - sh.multi_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, - value_key=e_k_f)) - apply = True - elif f.type == schemas.FetchFilterType._response_body: - event_where.append( - sh.multi_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value, - value_key=e_k_f)) - apply = True - else: - print(f"undefined FETCH filter: {f.type}") - if not apply: - continue - elif event_type == schemas.EventType.graphql: - event_from = event_from % f"{events.EventType.GRAPHQL.table} AS main " - for j, f in enumerate(event.filters): - is_any = sh.isAny_opreator(f.operator) - if is_any or len(f.value) == 0: - continue - f.value = helper.values_for_operator(value=f.value, op=f.operator) - op = sh.get_sql_operator(f.operator) - e_k_f = e_k + f"_graphql{j}" - full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)} - if f.type == schemas.GraphqlFilterType._name: - event_where.append( - sh.multi_conditions(f"main.{events.EventType.GRAPHQL.column} {op} %({e_k_f})s", f.value, - value_key=e_k_f)) - elif f.type == schemas.GraphqlFilterType._method: - event_where.append( - sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) - elif f.type == schemas.GraphqlFilterType._request_body: - event_where.append( - sh.multi_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) - elif f.type == schemas.GraphqlFilterType._response_body: - event_where.append( - sh.multi_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) - else: - print(f"undefined GRAPHQL filter: {f.type}") - else: - continue - if event_index == 0 or or_events: - event_where += ss_constraints - if is_not: - if event_index == 0 or or_events: - events_query_from.append(f"""\ - (SELECT - session_id, - 0 AS timestamp - FROM sessions - WHERE EXISTS(SELECT session_id - FROM {event_from} - WHERE {" AND ".join(event_where)} - AND sessions.session_id=ms.session_id) IS FALSE - AND project_id = %(projectId)s - AND start_ts >= %(startDate)s - AND start_ts <= %(endDate)s - AND duration IS NOT NULL - ) {"" if or_events else (f"AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ - """) - else: - events_query_from.append(f"""\ - (SELECT - event_0.session_id, - event_{event_index - 1}.timestamp AS timestamp - WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE - ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ - """) - else: - events_query_from.append(f"""\ - (SELECT main.session_id, {"MIN" if event_index < (valid_events_count - 1) else "MAX"}(main.timestamp) AS timestamp - FROM {event_from} - WHERE {" AND ".join(event_where)} - GROUP BY 1 - ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ - """) - event_index += 1 - if event_index > 0: - if or_events: - events_query_part = f"""SELECT - session_id, - MIN(timestamp) AS first_event_ts, - MAX(timestamp) AS last_event_ts - FROM ({events_joiner.join(events_query_from)}) AS u - GROUP BY 1""" - else: - events_query_part = f"""SELECT - event_0.session_id, - MIN(event_0.timestamp) AS first_event_ts, - MAX(event_{event_index - 1}.timestamp) AS last_event_ts - FROM {events_joiner.join(events_query_from)} - GROUP BY 1""" - else: - data.events = [] - # --------------------------------------------------------------------------- - if data.startDate is not None: - extra_constraints.append("s.start_ts >= %(startDate)s") - if data.endDate is not None: - extra_constraints.append("s.start_ts <= %(endDate)s") - # if data.platform is not None: - # if data.platform == schemas.PlatformType.mobile: - # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") - # elif data.platform == schemas.PlatformType.desktop: - # extra_constraints.append( - # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") - - if errors_only: - extra_from += f" INNER JOIN {events.EventType.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" - extra_constraints.append("ser.source = 'js_exception'") - extra_constraints.append("ser.project_id = %(project_id)s") - # if error_status != schemas.ErrorStatus.all: - # extra_constraints.append("ser.status = %(error_status)s") - # full_args["error_status"] = error_status - # if favorite_only: - # extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" - # extra_constraints.append("ufe.user_id = %(userId)s") - - if favorite_only and not errors_only and user_id is not None: - extra_from += """INNER JOIN (SELECT user_id, session_id - FROM public.user_favorite_sessions - WHERE user_id = %(userId)s) AS favorite_sessions - USING (session_id)""" - elif not favorite_only and not errors_only and user_id is not None: - extra_from += """LEFT JOIN (SELECT user_id, session_id - FROM public.user_favorite_sessions - WHERE user_id = %(userId)s) AS favorite_sessions - USING (session_id)""" - extra_join = "" - if issue is not None: - extra_join = """ - INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) - WHERE issues.session_id=f.session_id - AND p_issues.type=%(issue_type)s - AND p_issues.context_string=%(issue_contextString)s - AND timestamp >= f.first_event_ts - AND timestamp <= f.last_event_ts) AS issues ON(TRUE) - """ - full_args["issue_contextString"] = issue["contextString"] - full_args["issue_type"] = issue["type"] - if extra_event: - extra_join += f"""INNER JOIN {extra_event} AS ev USING(session_id)""" - extra_constraints.append("ev.timestamp>=%(startDate)s") - extra_constraints.append("ev.timestamp<=%(endDate)s") - query_part = f"""\ - FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} - {extra_join} - {"INNER JOIN public.sessions AS s USING(session_id)" if len(events_query_part) > 0 else ""} - {extra_from} - WHERE - {" AND ".join(extra_constraints)}""" - return full_args, query_part - - -def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): - if project_id is None: - all_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) - else: - all_projects = [ - projects.get_project(tenant_id=tenant_id, project_id=int(project_id), include_last_session=False, - include_gdpr=False)] - - all_projects = {int(p["projectId"]): p["name"] for p in all_projects} - project_ids = list(all_projects.keys()) - - available_keys = metadata.get_keys_by_projects(project_ids) - for i in available_keys: - available_keys[i]["user_id"] = schemas.FilterType.user_id - available_keys[i]["user_anonymous_id"] = schemas.FilterType.user_anonymous_id - results = {} - for i in project_ids: - if m_key not in available_keys[i].values(): - available_keys.pop(i) - results[i] = {"total": 0, "sessions": [], "missingMetadata": True} - project_ids = list(available_keys.keys()) - if len(project_ids) > 0: - with pg_client.PostgresClient() as cur: - sub_queries = [] - for i in project_ids: - col_name = list(available_keys[i].keys())[list(available_keys[i].values()).index(m_key)] - sub_queries.append(cur.mogrify( - f"(SELECT COALESCE(COUNT(s.*)) AS count FROM public.sessions AS s WHERE s.project_id = %(id)s AND s.{col_name} = %(value)s) AS \"{i}\"", - {"id": i, "value": m_value}).decode('UTF-8')) - query = f"""SELECT {", ".join(sub_queries)};""" - cur.execute(query=query) - - rows = cur.fetchone() - - sub_queries = [] - for i in rows.keys(): - results[i] = {"total": rows[i], "sessions": [], "missingMetadata": False, "name": all_projects[int(i)]} - if rows[i] > 0: - col_name = list(available_keys[int(i)].keys())[list(available_keys[int(i)].values()).index(m_key)] - sub_queries.append( - cur.mogrify( - f"""( - SELECT * - FROM ( - SELECT DISTINCT ON(favorite_sessions.session_id, s.session_id) {SESSION_PROJECTION_COLS} - FROM public.sessions AS s LEFT JOIN (SELECT session_id - FROM public.user_favorite_sessions - WHERE user_favorite_sessions.user_id = %(userId)s - ) AS favorite_sessions USING (session_id) - WHERE s.project_id = %(id)s AND s.duration IS NOT NULL AND s.{col_name} = %(value)s - ) AS full_sessions - ORDER BY favorite DESC, issue_score DESC - LIMIT 10 - )""", - {"id": i, "value": m_value, "userId": user_id}).decode('UTF-8')) - if len(sub_queries) > 0: - cur.execute("\nUNION\n".join(sub_queries)) - rows = cur.fetchall() - for i in rows: - results[str(i["project_id"])]["sessions"].append(helper.dict_to_camel_case(i)) - return results - - -def get_user_sessions(project_id, user_id, start_date, end_date): - with pg_client.PostgresClient() as cur: - constraints = ["s.project_id = %(projectId)s", "s.user_id = %(userId)s"] - if start_date is not None: - constraints.append("s.start_ts >= %(startDate)s") - if end_date is not None: - constraints.append("s.start_ts <= %(endDate)s") - - query_part = f"""\ - FROM public.sessions AS s - WHERE {" AND ".join(constraints)}""" - - cur.execute(cur.mogrify(f"""\ - SELECT s.project_id, - s.session_id::text AS session_id, - s.user_uuid, - s.user_id, - s.user_os, - s.user_browser, - s.user_device, - s.user_country, - s.start_ts, - s.duration, - s.events_count, - s.pages_count, - s.errors_count - {query_part} - ORDER BY s.session_id - LIMIT 50;""", { - "projectId": project_id, - "userId": user_id, - "startDate": start_date, - "endDate": end_date - })) - - sessions = cur.fetchall() - return helper.list_to_camel_case(sessions) - - -def get_session_user(project_id, user_id): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """\ - SELECT - user_id, - count(*) as session_count, - max(start_ts) as last_seen, - min(start_ts) as first_seen - FROM - "public".sessions - WHERE - project_id = %(project_id)s - AND user_id = %(userId)s - AND duration is not null - GROUP BY user_id; - """, - {"project_id": project_id, "userId": user_id} - ) - cur.execute(query=query) - data = cur.fetchone() - return helper.dict_to_camel_case(data) - - -def get_session_ids_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient() as cur: - query = cur.mogrify( - """\ - SELECT session_id FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - ids = cur.execute(query=query) - return ids - - -def delete_sessions_by_session_ids(session_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - session_id IN %(session_ids)s;""", - {"session_ids": tuple(session_ids)} - ) - cur.execute(query=query) - - return True - - -def delete_sessions_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient(unlimited_query=True) as cur: - query = cur.mogrify( - """\ - DELETE FROM public.sessions - WHERE - project_id = %(project_id)s AND user_id IN %(userId)s;""", - {"project_id": project_id, "userId": tuple(user_ids)} - ) - cur.execute(query=query) - - return True - - -def count_all(): - with pg_client.PostgresClient(unlimited_query=True) as cur: - cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") - row = cur.fetchone() - return row.get("count", 0) if row else 0 - - -def session_exists(project_id, session_id): - with pg_client.PostgresClient() as cur: - query = cur.mogrify("""SELECT 1 - FROM public.sessions - WHERE session_id=%(session_id)s - AND project_id=%(project_id)s - LIMIT 1;""", - {"project_id": project_id, "session_id": session_id}) - cur.execute(query) - row = cur.fetchone() - return row is not None diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index 9241b8e48..a160cf9c2 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -35,6 +35,7 @@ rm -rf ./chalicelib/core/log_tool_stackdriver.py rm -rf ./chalicelib/core/log_tool_sumologic.py rm -rf ./chalicelib/core/metadata.py rm -rf ./chalicelib/core/mobile.py +rm -rf ./chalicelib/core/sessions.py rm -rf ./chalicelib/core/sessions_assignments.py #exp rm -rf ./chalicelib/core/sessions_metas.py rm -rf ./chalicelib/core/sessions_mobs.py From 0fe1adf522c24980d1272100815f75d2f99e180d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 14:06:59 +0100 Subject: [PATCH 081/253] feat(chalice): code cleaning --- ee/api/chalicelib/core/sessions_exp.py | 95 +---------------------- ee/api/chalicelib/core/sessions_replay.py | 2 + 2 files changed, 4 insertions(+), 93 deletions(-) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index f60090ed4..888800681 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -2,11 +2,8 @@ from typing import List, Union import schemas import schemas_ee -from chalicelib.core import events, metadata, events_ios, \ - sessions_mobs, issues, projects, resources, assist, performance_event, metrics, sessions_devtool, \ - sessions_notes -from chalicelib.utils import pg_client, helper, metrics_helper, ch_client, exp_ch_helper, errors_helper -from chalicelib.utils import sql_helper as sh +from chalicelib.core import events, metadata, projects, performance_event, metrics +from chalicelib.utils import pg_client, helper, metrics_helper, ch_client, exp_ch_helper SESSION_PROJECTION_COLS_CH = """\ s.project_id, @@ -51,94 +48,6 @@ SESSION_PROJECTION_COLS_CH_MAP = """\ """ -def __group_metadata(session, project_metadata): - meta = {} - for m in project_metadata.keys(): - if project_metadata[m] is not None and session.get(m) is not None: - meta[project_metadata[m]] = session[m] - session.pop(m) - return meta - - -# This function should not use Clickhouse because it doesn't have `file_key` -def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, full_data=False, include_fav_viewed=False, - group_metadata=False, live=True): - with pg_client.PostgresClient() as cur: - extra_query = [] - if include_fav_viewed: - extra_query.append("""COALESCE((SELECT TRUE - FROM public.user_favorite_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s), FALSE) AS favorite""") - extra_query.append("""COALESCE((SELECT TRUE - FROM public.user_viewed_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s), FALSE) AS viewed""") - query = cur.mogrify( - f"""\ - SELECT - s.*, - s.session_id::text AS session_id, - (SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key, - encode(file_key,'hex') AS file_key - {"," if len(extra_query) > 0 else ""}{",".join(extra_query)} - {(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''} - FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""} - WHERE s.project_id = %(project_id)s - AND s.session_id = %(session_id)s;""", - {"project_id": project_id, "session_id": session_id, "userId": context.user_id} - ) - # print("===============") - # print(query) - cur.execute(query=query) - - data = cur.fetchone() - if data is not None: - data = helper.dict_to_camel_case(data) - if full_data: - if data["platform"] == 'ios': - data['events'] = events_ios.get_by_sessionId(project_id=project_id, session_id=session_id) - for e in data['events']: - if e["type"].endswith("_IOS"): - e["type"] = e["type"][:-len("_IOS")] - data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id) - data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id, - session_id=session_id) - data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id) - else: - data['events'] = events.get_by_session_id(project_id=project_id, session_id=session_id, - group_clickrage=True) - all_errors = events.get_errors_by_session_id(session_id=session_id, project_id=project_id) - data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"] - # to keep only the first stack - # limit the number of errors to reduce the response-body size - data['errors'] = [errors_helper.format_first_stack_frame(e) for e in all_errors - if e['source'] == "js_exception"][:500] - data['userEvents'] = events.get_customs_by_session_id(project_id=project_id, - session_id=session_id) - data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) - data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) - data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, - context=context) - data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id, - start_ts=data["startTs"], - duration=data["duration"]) - - data['notes'] = sessions_notes.get_session_notes(tenant_id=context.tenant_id, project_id=project_id, - session_id=session_id, user_id=context.user_id) - data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) - data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) - data['live'] = live and assist.is_live(project_id=project_id, - session_id=session_id, - project_key=data["projectKey"]) - data["inDB"] = True - return data - elif live: - return assist.get_live_session_by_id(project_id=project_id, session_id=session_id) - else: - return None - - def __get_sql_operator(op: schemas.SearchEventOperator): return { schemas.SearchEventOperator._is: "=", diff --git a/ee/api/chalicelib/core/sessions_replay.py b/ee/api/chalicelib/core/sessions_replay.py index 798029aee..993855637 100644 --- a/ee/api/chalicelib/core/sessions_replay.py +++ b/ee/api/chalicelib/core/sessions_replay.py @@ -16,6 +16,7 @@ def __group_metadata(session, project_metadata): # for backward compatibility +# This function should not use Clickhouse because it doesn't have `file_key` def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, full_data=False, include_fav_viewed=False, group_metadata=False, live=True): with pg_client.PostgresClient() as cur: @@ -92,6 +93,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu return None +# This function should not use Clickhouse because it doesn't have `file_key` def get_replay(project_id, session_id, context: schemas.CurrentContext, full_data=False, include_fav_viewed=False, group_metadata=False, live=True): with pg_client.PostgresClient() as cur: From 11af70852008fa452e458fa23606db5a453481f1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 15:00:18 +0100 Subject: [PATCH 082/253] feat(chalice): dynamic health-check endpoint --- api/chalicelib/core/tenants.py | 4 ++-- api/chalicelib/utils/pg_client.py | 9 ++++++--- api/routers/subs/health.py | 17 ++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/api/chalicelib/core/tenants.py b/api/chalicelib/core/tenants.py index 5479178d8..4d95ae491 100644 --- a/api/chalicelib/core/tenants.py +++ b/api/chalicelib/core/tenants.py @@ -68,7 +68,7 @@ def update(tenant_id, user_id, data: schemas.UpdateTenantSchema): return edit_client(tenant_id=tenant_id, changes=changes) -def tenants_exists(): - with pg_client.PostgresClient() as cur: +def tenants_exists(use_pool=True): + with pg_client.PostgresClient(use_pool=use_pool) as cur: cur.execute(f"SELECT EXISTS(SELECT 1 FROM public.tenants)") return cur.fetchone()["exists"] diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index 4cfd8b0e3..64ca1719f 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -87,9 +87,10 @@ class PostgresClient: long_query = False unlimited_query = False - def __init__(self, long_query=False, unlimited_query=False): + def __init__(self, long_query=False, unlimited_query=False, use_pool=True): self.long_query = long_query self.unlimited_query = unlimited_query + self.use_pool = use_pool if unlimited_query: long_config = dict(_PG_CONFIG) long_config["application_name"] += "-UNLIMITED" @@ -100,7 +101,7 @@ class PostgresClient: long_config["options"] = f"-c statement_timeout=" \ f"{config('pg_long_timeout', cast=int, default=5 * 60) * 1000}" self.connection = psycopg2.connect(**long_config) - elif not config('PG_POOL', cast=bool, default=True): + elif not use_pool or not config('PG_POOL', cast=bool, default=True): single_config = dict(_PG_CONFIG) single_config["application_name"] += "-NOPOOL" single_config["options"] = f"-c statement_timeout={config('PG_TIMEOUT', cast=int, default=30) * 1000}" @@ -120,11 +121,12 @@ class PostgresClient: try: self.connection.commit() self.cursor.close() - if self.long_query or self.unlimited_query: + if not self.use_pool or self.long_query or self.unlimited_query: self.connection.close() except Exception as error: logging.error("Error while committing/closing PG-connection", error) if str(error) == "connection already closed" \ + and self.use_pool \ and not self.long_query \ and not self.unlimited_query \ and config('PG_POOL', cast=bool, default=True): @@ -134,6 +136,7 @@ class PostgresClient: raise error finally: if config('PG_POOL', cast=bool, default=True) \ + and self.use_pool \ and not self.long_query \ and not self.unlimited_query: postgreSQL_pool.putconn(self.connection) diff --git a/api/routers/subs/health.py b/api/routers/subs/health.py index 6655f2a20..5e3c10f07 100644 --- a/api/routers/subs/health.py +++ b/api/routers/subs/health.py @@ -1,15 +1,14 @@ -from typing import Union - -from fastapi import Body, Depends, Request - -import schemas -from chalicelib.core import health -from or_dependencies import OR_context +from chalicelib.core import health, tenants from routers.base import get_routers public_app, app, app_apikey = get_routers() +health_router = public_app -@public_app.get('/health', tags=["dashboard"]) -def get_global_health(): +if tenants.tenants_exists(use_pool=False): + health_router = app + + +@health_router.get('/health', tags=["health-check"]) +def get_global_health_status(): return {"data": health.get_health()} From 8e5ae800d5071fd6fa9178a9d4ba0e14e3567337 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 15:15:48 +0100 Subject: [PATCH 083/253] feat(chalice): upgraded startup/shutdown logic --- api/app.py | 77 ++++++++++++++++--------------- api/requirements-alerts.txt | 2 +- api/requirements.txt | 2 +- ee/api/chalicelib/core/tenants.py | 6 +-- ee/api/requirements-alerts.txt | 2 +- ee/api/requirements.txt | 2 +- 6 files changed, 48 insertions(+), 43 deletions(-) diff --git a/api/app.py b/api/app.py index 883cf6704..50cd7342f 100644 --- a/api/app.py +++ b/api/app.py @@ -1,4 +1,5 @@ import logging +from contextlib import asynccontextmanager from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config @@ -14,7 +15,40 @@ from routers.crons import core_crons from routers.crons import core_dynamic_crons from routers.subs import insights, metrics, v1_api, health -app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) +loglevel = config("LOGLEVEL", default=logging.INFO) +print(f">Loglevel set to: {loglevel}") +logging.basicConfig(level=loglevel) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Startup + logging.info(">>>>> starting up <<<<<") + ap_logger = logging.getLogger('apscheduler') + ap_logger.setLevel(loglevel) + + app.schedule = AsyncIOScheduler() + await pg_client.init() + app.schedule.start() + + for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs: + app.schedule.add_job(id=job["func"].__name__, **job) + + ap_logger.info(">Scheduled jobs:") + for job in app.schedule.get_jobs(): + ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) + + # App listening + yield + + # Shutdown + logging.info(">>>>> shutting down <<<<<") + app.schedule.shutdown(wait=False) + await pg_client.terminate() + + +app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""), + lifespan=lifespan) app.add_middleware(GZipMiddleware, minimum_size=1000) @@ -55,38 +89,9 @@ app.include_router(health.public_app) app.include_router(health.app) app.include_router(health.app_apikey) -loglevel = config("LOGLEVEL", default=logging.INFO) -print(f">Loglevel set to: {loglevel}") -logging.basicConfig(level=loglevel) -ap_logger = logging.getLogger('apscheduler') -ap_logger.setLevel(loglevel) -app.schedule = AsyncIOScheduler() - - -@app.on_event("startup") -async def startup(): - logging.info(">>>>> starting up <<<<<") - await pg_client.init() - app.schedule.start() - - for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs: - app.schedule.add_job(id=job["func"].__name__, **job) - - ap_logger.info(">Scheduled jobs:") - for job in app.schedule.get_jobs(): - ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) - - -@app.on_event("shutdown") -async def shutdown(): - logging.info(">>>>> shutting down <<<<<") - app.schedule.shutdown(wait=False) - await pg_client.terminate() - - -@app.get('/private/shutdown', tags=["private"]) -async def stop_server(): - logging.info("Requested shutdown") - await shutdown() - import os, signal - os.kill(1, signal.SIGTERM) +# @app.get('/private/shutdown', tags=["private"]) +# async def stop_server(): +# logging.info("Requested shutdown") +# await shutdown() +# import os, signal +# os.kill(1, signal.SIGTERM) diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index b208d28c2..edb644c87 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -8,7 +8,7 @@ jira==3.4.1 -fastapi==0.92.0 +fastapi==0.94.1 uvicorn[standard]==0.20.0 python-decouple==3.7 pydantic[email]==1.10.4 diff --git a/api/requirements.txt b/api/requirements.txt index 4a8d35090..27b95f17e 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -8,7 +8,7 @@ jira==3.4.1 -fastapi==0.92.0 +fastapi==0.94.1 uvicorn[standard]==0.20.0 python-decouple==3.7 pydantic[email]==1.10.4 diff --git a/ee/api/chalicelib/core/tenants.py b/ee/api/chalicelib/core/tenants.py index 30a87bd29..7ea621007 100644 --- a/ee/api/chalicelib/core/tenants.py +++ b/ee/api/chalicelib/core/tenants.py @@ -51,7 +51,7 @@ def get_by_api_key(api_key): WHERE tenants.api_key = %(api_key)s AND tenants.deleted_at ISNULL LIMIT 1;""", - {"api_key": api_key}) + {"api_key": api_key}) cur.execute(query=query) return helper.dict_to_camel_case(cur.fetchone()) @@ -94,7 +94,7 @@ def update(tenant_id, user_id, data: schemas.UpdateTenantSchema): return edit_client(tenant_id=tenant_id, changes=changes) -def tenants_exists(): - with pg_client.PostgresClient() as cur: +def tenants_exists(use_pool=True): + with pg_client.PostgresClient(use_pool=use_pool) as cur: cur.execute(f"SELECT EXISTS(SELECT 1 FROM public.tenants)") return cur.fetchone()["exists"] diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index 250882623..6b6901ca5 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -8,7 +8,7 @@ jira==3.4.1 -fastapi==0.92.0 +fastapi==0.94.1 uvicorn[standard]==0.20.0 python-decouple==3.7 pydantic[email]==1.10.4 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 9ce06fe06..cad05e873 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -8,7 +8,7 @@ jira==3.4.1 -fastapi==0.92.0 +fastapi==0.94.1 uvicorn[standard]==0.20.0 python-decouple==3.7 pydantic[email]==1.10.4 From d7dc9b684f3e03fe9728a7ac3d9ba3552792b142 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 14 Mar 2023 15:00:55 +0100 Subject: [PATCH 084/253] change(ui): split session info into separate calls for faster replay time --- frontend/app/components/Session/Session.js | 8 +- frontend/app/components/Session/WebPlayer.tsx | 6 + .../Session_/Player/Controls/Timeline.tsx | 1 + frontend/app/duck/sessions.ts | 141 ++++++- frontend/app/player/web/MessageManager.ts | 14 +- frontend/app/player/web/WebPlayer.ts | 15 + frontend/app/types/session/event.ts | 2 +- frontend/app/types/session/session.ts | 367 ++++++++++-------- 8 files changed, 375 insertions(+), 179 deletions(-) diff --git a/frontend/app/components/Session/Session.js b/frontend/app/components/Session/Session.js index 2a7cf81d8..a218b4ca9 100644 --- a/frontend/app/components/Session/Session.js +++ b/frontend/app/components/Session/Session.js @@ -2,7 +2,7 @@ import React from 'react'; import { useEffect, useState } from 'react'; import { connect } from 'react-redux'; import usePageTitle from 'App/hooks/usePageTitle'; -import { fetch as fetchSession } from 'Duck/sessions'; +import { fetchV2 } from "Duck/sessions"; import { fetchList as fetchSlackList } from 'Duck/integrations/slack'; import { Link, NoContent, Loader } from 'UI'; import { sessions as sessionsRoute } from 'App/routes'; @@ -17,14 +17,14 @@ function Session({ sessionId, loading, hasErrors, - fetchSession, + fetchV2, }) { usePageTitle("OpenReplay Session Player"); const [ initializing, setInitializing ] = useState(true) const { sessionStore } = useStore(); useEffect(() => { if (sessionId != null) { - fetchSession(sessionId) + fetchV2(sessionId) } else { console.error("No sessionID in route.") } @@ -63,6 +63,6 @@ export default withPermissions(['SESSION_REPLAY'], '', true)(connect((state, pro session: state.getIn([ 'sessions', 'current' ]), }; }, { - fetchSession, fetchSlackList, + fetchV2, })(Session)); diff --git a/frontend/app/components/Session/WebPlayer.tsx b/frontend/app/components/Session/WebPlayer.tsx index 659dbe540..16b3d54ee 100644 --- a/frontend/app/components/Session/WebPlayer.tsx +++ b/frontend/app/components/Session/WebPlayer.tsx @@ -64,6 +64,12 @@ function WebPlayer(props: any) { return () => WebPlayerInst.clean(); }, [session.sessionId]); + React.useEffect(() => { + if (session.events.length > 0 || session.errors.length > 0) { + contextValue.player.updateLists(session) + } + }, [session.events, session.errors]) + const isPlayerReady = contextValue.store?.get().ready React.useEffect(() => { diff --git a/frontend/app/components/Session_/Player/Controls/Timeline.tsx b/frontend/app/components/Session_/Player/Controls/Timeline.tsx index 678982aa9..a69189b49 100644 --- a/frontend/app/components/Session_/Player/Controls/Timeline.tsx +++ b/frontend/app/components/Session_/Player/Controls/Timeline.tsx @@ -91,6 +91,7 @@ function Timeline(props: IProps) { } const time = getTime(e); + if (!time) return; const tz = settingsStore.sessionSettings.timezone.value const timeStr = DateTime.fromMillis(props.startedAt + time).setZone(tz).toFormat(`hh:mm:ss a`) const timeLineTooltip = { diff --git a/frontend/app/duck/sessions.ts b/frontend/app/duck/sessions.ts index b87afaaa4..113dd4e25 100644 --- a/frontend/app/duck/sessions.ts +++ b/frontend/app/duck/sessions.ts @@ -1,18 +1,25 @@ import { List, Map } from 'immutable'; import Session from 'Types/session'; import ErrorStack from 'Types/session/errorStack'; -import { Location } from 'Types/session/event' +import { EventData, Location } from "Types/session/event"; import Watchdog from 'Types/watchdog'; import { clean as cleanParams } from 'App/api_client'; import withRequestState, { RequestTypes } from './requestStateCreator'; import { getRE, setSessionFilter, getSessionFilter, compareJsonObjects, cleanSessionFilters } from 'App/utils'; import { LAST_7_DAYS } from 'Types/app/period'; import { getDateRangeFromValue } from 'App/dateRange'; +import APIClient from 'App/api_client'; +import { FETCH_ACCOUNT, UPDATE_JWT } from "Duck/user"; +import logger from "App/logger"; const name = 'sessions'; const FETCH_LIST = new RequestTypes('sessions/FETCH_LIST'); const FETCH_AUTOPLAY_LIST = new RequestTypes('sessions/FETCH_AUTOPLAY_LIST'); -const FETCH = new RequestTypes('sessions/FETCH'); +const FETCH = new RequestTypes('sessions/FETCH') +const FETCHV2 = new RequestTypes('sessions/FETCHV2') +const FETCH_EVENTS = new RequestTypes('sessions/FETCH_EVENTS'); +const FETCH_NOTES = new RequestTypes('sessions/FETCH_NOTES'); + const FETCH_FAVORITE_LIST = new RequestTypes('sessions/FETCH_FAVORITE_LIST'); const FETCH_LIVE_LIST = new RequestTypes('sessions/FETCH_LIVE_LIST'); const TOGGLE_FAVORITE = new RequestTypes('sessions/TOGGLE_FAVORITE'); @@ -160,11 +167,82 @@ const reducer = (state = initialState, action: IAction) => { } }); }); + return state + .set('current', session) + .set('eventsIndex', matching) + .set('visitedEvents', visitedEvents) + .set('host', visitedEvents[0] && visitedEvents[0].host); + } + case FETCHV2.SUCCESS: { + const session = new Session(action.data); + return state .set('current', session) - .set('eventsIndex', matching) - .set('visitedEvents', visitedEvents) - .set('host', visitedEvents[0] && visitedEvents[0].host); + } + case FETCH_EVENTS.SUCCESS: { + const { + errors, + events, + issues, + resources, + stackEvents, + userEvents + } = action.data as { errors: any[], events: any[], issues: any[], resources: any[], stackEvents: any[], userEvents: EventData[] }; + const filterEvents = action.filter.events as Record[]; + const session = state.get('current') as Session; + const matching: number[] = []; + + const visitedEvents: Location[] = []; + const tmpMap = new Set(); + events.forEach((event) => { + // @ts-ignore assume that event is LocationEvent + if (event.type === 'LOCATION' && !tmpMap.has(event.url)) { + // @ts-ignore assume that event is LocationEvent + tmpMap.add(event.url); + // @ts-ignore assume that event is LocationEvent + visitedEvents.push(event); + } + }); + + filterEvents.forEach(({ key, operator, value }) => { + events.forEach((e, index) => { + if (key === e.type) { + // @ts-ignore assume that event is LocationEvent + const val = e.type === 'LOCATION' ? e.url : e.value; + if (operator === 'is' && value === val) { + matching.push(index); + } + if (operator === 'contains' && val.includes(value)) { + matching.push(index); + } + } + }); + }); + + const newSession = session.addEvents( + events, + errors, + issues, + resources, + stackEvents, + userEvents + ); + + const forceUpdate = state.set('current', {}) + return forceUpdate + .set('current', newSession) + .set('eventsIndex', matching) + .set('visitedEvents', visitedEvents) + .set('host', visitedEvents[0] && visitedEvents[0].host); + } + case FETCH_NOTES.SUCCESS: { + const notes = action.data; + if (notes.length > 0) { + const session = state.get('current') as Session; + const newSession = session.addNotes(notes); + return state.set('current', newSession); + } + return state } case FETCH_FAVORITE_LIST.SUCCESS: return state.set('favoriteList', action.data.map(s => new Session(s))); @@ -321,6 +399,59 @@ export const fetch = }); }; +function parseError(e: any) { + try { + return [...JSON.parse(e).errors] || []; + } catch { + return Array.isArray(e) ? e : [e]; + } +} + +// implementing custom middleware-like request to keep the behavior +// TODO: move all to mobx +export const fetchV2 = (sessionId: string) => + (dispatch, getState) => { + const apiClient = new APIClient() + const apiGet = (url: string, dispatch: any, FAILURE: string) => apiClient.get(url) + .then(async (response) => { + if (response.status === 403) { + dispatch({ type: FETCH_ACCOUNT.FAILURE }); + } + if (!response.ok) { + const text = await response.text(); + return Promise.reject(text); + } + return response.json(); + }) + .then((json) => json || {}) + .catch(async (e) => { + const data = await e.response?.json(); + logger.error('Error during API request. ', e); + return dispatch({ type: FAILURE, errors: data ? parseError(data.errors) : [] }); + }); + + const filter = getState().getIn(['filters', 'appliedFilter']) + apiGet(`/sessions/${sessionId}/replay`, dispatch, FETCH.FAILURE) + .then(async ({ jwt, errors, data }) => { + if (errors) { + dispatch({ type: FETCH.FAILURE, errors, data }); + } else { + dispatch({ type: FETCHV2.SUCCESS, data, ...filter }); + + let [events, notes] = await Promise.all([ + apiGet(`/sessions/${sessionId}/events`, dispatch, FETCH_EVENTS.FAILURE), + apiGet(`/sessions/${sessionId}/notes`, dispatch, FETCH_NOTES.FAILURE), + ]); + dispatch({ type: FETCH_EVENTS.SUCCESS, data: events.data, filter }); + dispatch({ type: FETCH_NOTES.SUCCESS, data: notes.data }); + } + if (jwt) { + dispatch({ type: UPDATE_JWT, data: jwt }); + } + }); + + } + export function clearCurrentSession() { return { type: CLEAR_CURRENT_SESSION diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index c54b86fb8..705ad08d3 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -108,7 +108,7 @@ export default class MessageManager { private scrollManager: ListWalker = new ListWalker(); public readonly decoder = new Decoder(); - private readonly lists: Lists; + private lists: Lists; private activityManager: ActivityManager | null = null; @@ -137,6 +137,18 @@ export default class MessageManager { this.activityManager = new ActivityManager(this.session.duration.milliseconds) // only if not-live } + public updateLists(lists: Partial) { + this.lists = new Lists(lists) + + lists?.event?.forEach((e: Record) => { + if (e.type === EVENT_TYPES.LOCATION) { + this.locationEventManager.append(e); + } + }) + + this.state.update({ ...this.lists.getFullListsState() }); + } + private setCSSLoading = (cssLoading: boolean) => { this.screen.displayFrame(!cssLoading) this.state.update({ cssLoading, ready: !this.state.get().messagesLoading && !cssLoading }) diff --git a/frontend/app/player/web/WebPlayer.ts b/frontend/app/player/web/WebPlayer.ts index 9ca769598..5cd255e10 100644 --- a/frontend/app/player/web/WebPlayer.ts +++ b/frontend/app/player/web/WebPlayer.ts @@ -68,6 +68,21 @@ export default class WebPlayer extends Player { } + updateLists = (session: any) => { + let lists = { + event: session.events || [], + stack: session.stackEvents || [], + exceptions: session.errors?.map(({ name, ...rest }: any) => + Log({ + level: LogLevel.ERROR, + value: name, + ...rest, + }) + ) || [], + } + this.messageManager.updateLists(lists) + } + attach = (parent: HTMLElement, isClickmap?: boolean) => { this.screen.attach(parent) if (!isClickmap) { diff --git a/frontend/app/types/session/event.ts b/frontend/app/types/session/event.ts index bb901a6b1..b08f07140 100644 --- a/frontend/app/types/session/event.ts +++ b/frontend/app/types/session/event.ts @@ -32,7 +32,7 @@ interface InputEvent extends IEvent { value: string; } -interface LocationEvent extends IEvent { +export interface LocationEvent extends IEvent { url: string; host: string; pageLoad: boolean; diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index c843206fe..7b1f95061 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -3,7 +3,8 @@ import SessionEvent, { TYPES, EventData, InjectedEvent } from './event'; import StackEvent from './stackEvent'; import SessionError, { IError } from './error'; import Issue, { IIssue } from './issue'; -import { Note } from 'App/services/NotesService' +import { Note } from 'App/services/NotesService'; +import { toJS } from 'mobx'; const HASH_MOD = 1610612741; const HASH_P = 53; @@ -19,70 +20,70 @@ function hashString(s: string): number { } export interface ISession { - sessionId: string, - pageTitle: string, - active: boolean, - siteId: string, - projectKey: string, - peerId: string, - live: boolean, - startedAt: number, - duration: number, - events: InjectedEvent[], - stackEvents: StackEvent[], - metadata: [], - favorite: boolean, - filterId?: string, - domURL: string[], - devtoolsURL: string[], + sessionId: string; + pageTitle: string; + active: boolean; + siteId: string; + projectKey: string; + peerId: string; + live: boolean; + startedAt: number; + duration: number; + events: InjectedEvent[]; + stackEvents: StackEvent[]; + metadata: []; + favorite: boolean; + filterId?: string; + domURL: string[]; + devtoolsURL: string[]; /** * @deprecated */ - mobsUrl: string[], - userBrowser: string, - userBrowserVersion: string, - userCountry: string, - userDevice: string, - userDeviceType: string, - isMobile: boolean, - userOs: string, - userOsVersion: string, - userId: string, - userAnonymousId: string, - userUuid: string, - userDisplayName: string, - userNumericHash: number, - viewed: boolean, - consoleLogCount: number, - eventsCount: number, - pagesCount: number, - errorsCount: number, - issueTypes: string[], - issues: [], - referrer: string | null, - userDeviceHeapSize: number, - userDeviceMemorySize: number, - errors: SessionError[], - crashes?: [], - socket: string, - isIOS: boolean, - revId: string | null, - agentIds?: string[], - isCallActive?: boolean, - agentToken: string, - notes: Note[], - notesWithEvents: Array, - fileKey: string, - platform: string, - projectId: string, - startTs: number, - timestamp: number, - backendErrors: number, - consoleErrors: number, - sessionID?: string, - userID: string, - userUUID: string, - userEvents: any[], + mobsUrl: string[]; + userBrowser: string; + userBrowserVersion: string; + userCountry: string; + userDevice: string; + userDeviceType: string; + isMobile: boolean; + userOs: string; + userOsVersion: string; + userId: string; + userAnonymousId: string; + userUuid: string; + userDisplayName: string; + userNumericHash: number; + viewed: boolean; + consoleLogCount: number; + eventsCount: number; + pagesCount: number; + errorsCount: number; + issueTypes: string[]; + issues: IIssue[]; + referrer: string | null; + userDeviceHeapSize: number; + userDeviceMemorySize: number; + errors: SessionError[]; + crashes?: []; + socket: string; + isIOS: boolean; + revId: string | null; + agentIds?: string[]; + isCallActive?: boolean; + agentToken: string; + notes: Note[]; + notesWithEvents: Array; + fileKey: string; + platform: string; + projectId: string; + startTs: number; + timestamp: number; + backendErrors: number; + consoleErrors: number; + sessionID?: string; + userID: string; + userUUID: string; + userEvents: any[]; } const emptyValues = { @@ -102,67 +103,67 @@ const emptyValues = { notes: [], metadata: {}, startedAt: 0, -} +}; export default class Session { - sessionId: ISession["sessionId"] - pageTitle: ISession["pageTitle"] - active: ISession["active"] - siteId: ISession["siteId"] - projectKey: ISession["projectKey"] - peerId: ISession["peerId"] - live: ISession["live"] - startedAt: ISession["startedAt"] - duration: ISession["duration"] - events: ISession["events"] - stackEvents: ISession["stackEvents"] - metadata: ISession["metadata"] - favorite: ISession["favorite"] - filterId?: ISession["filterId"] - domURL: ISession["domURL"] - devtoolsURL: ISession["devtoolsURL"] + sessionId: ISession['sessionId']; + pageTitle: ISession['pageTitle']; + active: ISession['active']; + siteId: ISession['siteId']; + projectKey: ISession['projectKey']; + peerId: ISession['peerId']; + live: ISession['live']; + startedAt: ISession['startedAt']; + duration: ISession['duration']; + events: ISession['events']; + stackEvents: ISession['stackEvents']; + metadata: ISession['metadata']; + favorite: ISession['favorite']; + filterId?: ISession['filterId']; + domURL: ISession['domURL']; + devtoolsURL: ISession['devtoolsURL']; /** * @deprecated */ - mobsUrl: ISession["mobsUrl"] - userBrowser: ISession["userBrowser"] - userBrowserVersion: ISession["userBrowserVersion"] - userCountry: ISession["userCountry"] - userDevice: ISession["userDevice"] - userDeviceType: ISession["userDeviceType"] - isMobile: ISession["isMobile"] - userOs: ISession["userOs"] - userOsVersion: ISession["userOsVersion"] - userId: ISession["userId"] - userAnonymousId: ISession["userAnonymousId"] - userUuid: ISession["userUuid"] - userDisplayName: ISession["userDisplayName"] - userNumericHash: ISession["userNumericHash"] - viewed: ISession["viewed"] - consoleLogCount: ISession["consoleLogCount"] - eventsCount: ISession["eventsCount"] - pagesCount: ISession["pagesCount"] - errorsCount: ISession["errorsCount"] - issueTypes: ISession["issueTypes"] - issues: ISession["issues"] - referrer: ISession["referrer"] - userDeviceHeapSize: ISession["userDeviceHeapSize"] - userDeviceMemorySize: ISession["userDeviceMemorySize"] - errors: ISession["errors"] - crashes?: ISession["crashes"] - socket: ISession["socket"] - isIOS: ISession["isIOS"] - revId: ISession["revId"] - agentIds?: ISession["agentIds"] - isCallActive?: ISession["isCallActive"] - agentToken: ISession["agentToken"] - notes: ISession["notes"] - notesWithEvents: ISession["notesWithEvents"] - fileKey: ISession["fileKey"] - durationSeconds: number + mobsUrl: ISession['mobsUrl']; + userBrowser: ISession['userBrowser']; + userBrowserVersion: ISession['userBrowserVersion']; + userCountry: ISession['userCountry']; + userDevice: ISession['userDevice']; + userDeviceType: ISession['userDeviceType']; + isMobile: ISession['isMobile']; + userOs: ISession['userOs']; + userOsVersion: ISession['userOsVersion']; + userId: ISession['userId']; + userAnonymousId: ISession['userAnonymousId']; + userUuid: ISession['userUuid']; + userDisplayName: ISession['userDisplayName']; + userNumericHash: ISession['userNumericHash']; + viewed: ISession['viewed']; + consoleLogCount: ISession['consoleLogCount']; + eventsCount: ISession['eventsCount']; + pagesCount: ISession['pagesCount']; + errorsCount: ISession['errorsCount']; + issueTypes: ISession['issueTypes']; + issues: Issue[]; + referrer: ISession['referrer']; + userDeviceHeapSize: ISession['userDeviceHeapSize']; + userDeviceMemorySize: ISession['userDeviceMemorySize']; + errors: ISession['errors']; + crashes?: ISession['crashes']; + socket: ISession['socket']; + isIOS: ISession['isIOS']; + revId: ISession['revId']; + agentIds?: ISession['agentIds']; + isCallActive?: ISession['isCallActive']; + agentToken: ISession['agentToken']; + notes: ISession['notes']; + notesWithEvents: ISession['notesWithEvents']; + fileKey: ISession['fileKey']; + durationSeconds: number; constructor(plainSession?: ISession) { - const sessionData = plainSession || (emptyValues as unknown as ISession) + const sessionData = plainSession || (emptyValues as unknown as ISession); const { startTs = 0, timestamp = 0, @@ -179,7 +180,7 @@ export default class Session { mobsUrl = [], notes = [], ...session - } = sessionData + } = sessionData; const duration = Duration.fromMillis(session.duration < 1000 ? 1000 : session.duration); const durationSeconds = duration.valueOf(); const startedAt = +startTs || +timestamp; @@ -188,44 +189,46 @@ export default class Session { const userDeviceType = session.userDeviceType || 'other'; const isMobile = ['console', 'mobile', 'tablet'].includes(userDeviceType); - const events: InjectedEvent[] = [] - const rawEvents: (EventData & { key: number })[] = [] + const events: InjectedEvent[] = []; + const rawEvents: (EventData & { key: number })[] = []; if (session.events?.length) { (session.events as EventData[]).forEach((event: EventData, k) => { - const time = event.timestamp - startedAt + const time = event.timestamp - startedAt; if (event.type !== TYPES.CONSOLE && time <= durationSeconds) { - const EventClass = SessionEvent({ ...event, time, key: k }) + const EventClass = SessionEvent({ ...event, time, key: k }); if (EventClass) { events.push(EventClass); } rawEvents.push({ ...event, time, key: k }); } - }) + }); } - const stackEventsList: StackEvent[] = [] + const stackEventsList: StackEvent[] = []; if (stackEvents?.length || session.userEvents?.length) { const mergedArrays = [...stackEvents, ...session.userEvents] .sort((a, b) => a.timestamp - b.timestamp) - .map((se) => new StackEvent({ ...se, time: se.timestamp - startedAt })) + .map((se) => new StackEvent({ ...se, time: se.timestamp - startedAt })); stackEventsList.push(...mergedArrays); } - const exceptions = (errors as IError[]).map(e => new SessionError(e)) || []; + const exceptions = (errors as IError[]).map((e) => new SessionError(e)) || []; - const issuesList = (issues as IIssue[]).map( - (i, k) => new Issue({ ...i, time: i.timestamp - startedAt, key: k })) || []; + const issuesList = + (issues as IIssue[]).map( + (i, k) => new Issue({ ...i, time: i.timestamp - startedAt, key: k }) + ) || []; - const rawNotes = notes; - const notesWithEvents = [...rawEvents, ...rawNotes].sort((a, b) => { - // @ts-ignore just in case - const aTs = a.timestamp || a.time; - // @ts-ignore - const bTs = b.timestamp || b.time; + const notesWithEvents = + [...rawEvents, ...notes].sort((a, b) => { + // @ts-ignore just in case + const aTs = a.timestamp || a.time; + // @ts-ignore + const bTs = b.timestamp || b.time; - return aTs - bTs; - }) || []; + return aTs - bTs; + }) || []; Object.assign(this, { ...session, @@ -242,11 +245,11 @@ export default class Session { durationSeconds, userNumericHash: hashString( session.userId || - session.userAnonymousId || - session.userUuid || - session.userID || - session.userUUID || - '' + session.userAnonymousId || + session.userUuid || + session.userID || + session.userUUID || + '' ), userDisplayName: session.userId || session.userAnonymousId || session.userID || 'Anonymous User', @@ -258,47 +261,75 @@ export default class Session { devtoolsURL, notes, notesWithEvents: notesWithEvents, - }) + }); } - addIssues(issues: IIssue[]) { - const issuesList = issues.map( - (i, k) => new Issue({ ...i, time: i.timestamp - this.startedAt, key: k })) || []; + addEvents( + sessionEvents: EventData[], + errors: any[], + issues: any[], + resources: any[], + userEvents: any[], + stackEvents: any[] + ) { + const exceptions = (errors as IError[]).map((e) => new SessionError(e)) || []; + const issuesList = + (issues as IIssue[]).map( + (i, k) => new Issue({ ...i, time: i.timestamp - this.startedAt, key: k }) + ) || []; + const stackEventsList: StackEvent[] = []; + if (stackEvents?.length || userEvents?.length) { + const mergedArrays = [...stackEvents, ...userEvents] + .sort((a, b) => a.timestamp - b.timestamp) + .map((se) => new StackEvent({ ...se, time: se.timestamp - this.startedAt })); + stackEventsList.push(...mergedArrays); + } - // @ts-ignore - this.issues = issuesList; - } - - addEvents(sessionEvents: EventData[], sessionNotes: Note[]) { - const events: InjectedEvent[] = [] - const rawEvents: (EventData & { key: number })[] = [] + const events: InjectedEvent[] = []; + const rawEvents: (EventData & { key: number })[] = []; if (sessionEvents.length) { sessionEvents.forEach((event, k) => { - const time = event.timestamp - this.startedAt + const time = event.timestamp - this.startedAt; if (event.type !== TYPES.CONSOLE && time <= this.durationSeconds) { - const EventClass = SessionEvent({ ...event, time, key: k }) + const EventClass = SessionEvent({ ...event, time, key: k }); if (EventClass) { events.push(EventClass); } rawEvents.push({ ...event, time, key: k }); } - }) + }); } - const rawNotes = sessionNotes; - const notesWithEvents = [...rawEvents, ...rawNotes].sort((a, b) => { - // @ts-ignore just in case - const aTs = a.timestamp || a.time; - // @ts-ignore - const bTs = b.timestamp || b.time; - - return aTs - bTs; - }) || []; - + this.events = events; // @ts-ignore - this.notesWithEvents = notesWithEvents; - this.notes = sessionNotes - this.events = events + this.notesWithEvents = rawEvents; + this.errors = exceptions; + this.issues = issuesList; + // @ts-ignore legacy code? no idea + this.resources = resources; + this.stackEvents = stackEventsList; + + return this; } -} \ No newline at end of file + + addNotes(sessionNotes: Note[]) { + // @ts-ignore + this.notesWithEvents = + [...this.notesWithEvents, ...sessionNotes].sort((a, b) => { + // @ts-ignore just in case + const aTs = a.timestamp || a.time; + // @ts-ignore + const bTs = b.timestamp || b.time; + + return aTs - bTs; + }) || []; + this.notes = sessionNotes; + + return this; + } + + toJS() { + return { ...toJS(this) }; + } +} From d6818e0d886fafeb798803069e1e03885216b964 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 14 Mar 2023 15:09:16 +0100 Subject: [PATCH 085/253] change(ui): remove random log --- .../app/components/Session/Player/LivePlayer/LivePlayerInst.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/app/components/Session/Player/LivePlayer/LivePlayerInst.tsx b/frontend/app/components/Session/Player/LivePlayer/LivePlayerInst.tsx index c17007648..148b203c8 100644 --- a/frontend/app/components/Session/Player/LivePlayer/LivePlayerInst.tsx +++ b/frontend/app/components/Session/Player/LivePlayer/LivePlayerInst.tsx @@ -29,7 +29,6 @@ function Player(props: IProps) { const screenWrapper = React.useRef(null); const ready = playerContext.store.get().ready - console.log(ready) React.useEffect(() => { if (!props.closedLive || isMultiview) { const parentElement = findDOMNode(screenWrapper.current) as HTMLDivElement | null; //TODO: good architecture From 0c9b2cb6d89de49c7637a0c4f66aba5cab5ef135 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 14 Mar 2023 15:17:15 +0100 Subject: [PATCH 086/253] change(ui): remove random log and duplicates --- frontend/app/api_middleware.js | 2 +- frontend/app/duck/sessions.ts | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/frontend/app/api_middleware.js b/frontend/app/api_middleware.js index d00074945..9bfca6593 100644 --- a/frontend/app/api_middleware.js +++ b/frontend/app/api_middleware.js @@ -40,7 +40,7 @@ export default () => (next) => (action) => { }); }; -function parseError(e) { +export function parseError(e) { try { return [...JSON.parse(e).errors] || []; } catch { diff --git a/frontend/app/duck/sessions.ts b/frontend/app/duck/sessions.ts index 113dd4e25..0da12dd1c 100644 --- a/frontend/app/duck/sessions.ts +++ b/frontend/app/duck/sessions.ts @@ -11,6 +11,7 @@ import { getDateRangeFromValue } from 'App/dateRange'; import APIClient from 'App/api_client'; import { FETCH_ACCOUNT, UPDATE_JWT } from "Duck/user"; import logger from "App/logger"; +import { parseError } from 'App/api_middleware' const name = 'sessions'; const FETCH_LIST = new RequestTypes('sessions/FETCH_LIST'); @@ -399,14 +400,6 @@ export const fetch = }); }; -function parseError(e: any) { - try { - return [...JSON.parse(e).errors] || []; - } catch { - return Array.isArray(e) ? e : [e]; - } -} - // implementing custom middleware-like request to keep the behavior // TODO: move all to mobx export const fetchV2 = (sessionId: string) => From 8b6ebbe81511c8331111b97b1281b5bcdcff8594 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 16:27:40 +0100 Subject: [PATCH 087/253] feat(chalice): upgraded startup/shutdown logic feat(alerts): health-check endpoint --- api/app_alerts.py | 64 ++++++++++++++------------- api/chalicelib/core/health.py | 2 +- api/entrypoint.sh | 2 +- api/entrypoint_alerts.sh | 2 +- api/run-alerts-dev.sh | 2 +- ee/api/.gitignore | 2 + ee/api/app.py | 81 +++++++++++++++++------------------ ee/api/clean-dev.sh | 2 + ee/api/entrypoint.sh | 2 +- ee/api/entrypoint_alerts.sh | 2 +- ee/api/run-dev.sh | 3 -- 11 files changed, 83 insertions(+), 81 deletions(-) delete mode 100755 ee/api/run-dev.sh diff --git a/api/app_alerts.py b/api/app_alerts.py index 111bad2a1..02147ef23 100644 --- a/api/app_alerts.py +++ b/api/app_alerts.py @@ -1,33 +1,17 @@ import logging +from contextlib import asynccontextmanager from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config from fastapi import FastAPI -from chalicelib.utils import pg_client from chalicelib.core import alerts_processor - -app = FastAPI(root_path="/alerts", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) -logging.info("============= ALERTS =============") +from chalicelib.utils import pg_client -@app.get("/") -async def root(): - return {"status": "Running"} - - -app.schedule = AsyncIOScheduler() - -loglevel = config("LOGLEVEL", default=logging.INFO) -print(f">Loglevel set to: {loglevel}") -logging.basicConfig(level=loglevel) -ap_logger = logging.getLogger('apscheduler') -ap_logger.setLevel(loglevel) -app.schedule = AsyncIOScheduler() - - -@app.on_event("startup") -async def startup(): +@asynccontextmanager +async def lifespan(app: FastAPI): + # Startup logging.info(">>>>> starting up <<<<<") await pg_client.init() app.schedule.start() @@ -39,24 +23,44 @@ async def startup(): for job in app.schedule.get_jobs(): ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) + # App listening + yield -@app.on_event("shutdown") -async def shutdown(): + # Shutdown logging.info(">>>>> shutting down <<<<<") app.schedule.shutdown(wait=False) await pg_client.terminate() -@app.get('/private/shutdown', tags=["private"]) -async def stop_server(): - logging.info("Requested shutdown") - await shutdown() - import os, signal - os.kill(1, signal.SIGTERM) +app = FastAPI(root_path="/alerts", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""), + lifespan=lifespan) +logging.info("============= ALERTS =============") +@app.get("/") +async def root(): + return {"status": "Running"} + + +@app.get("/health") +async def get_health_status(): + return {"data": { + "health": True, + "details": {"version": config("version_number", default="unknown")} + }} + + +app.schedule = AsyncIOScheduler() + +loglevel = config("LOGLEVEL", default=logging.INFO) +print(f">Loglevel set to: {loglevel}") +logging.basicConfig(level=loglevel) +ap_logger = logging.getLogger('apscheduler') +ap_logger.setLevel(loglevel) +app.schedule = AsyncIOScheduler() + if config("LOCAL_DEV", default=False, cast=bool): - @app.get('/private/trigger', tags=["private"]) + @app.get('/trigger', tags=["private"]) async def trigger_main_cron(): logging.info("Triggering main cron") alerts_processor.process() diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index 980c7a41b..6f236a0f5 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -29,7 +29,7 @@ if config("LOCAL_DEV", cast=bool, default=False): else: HEALTH_ENDPOINTS = { - "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/metrics", + "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health", "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", diff --git a/api/entrypoint.sh b/api/entrypoint.sh index e140268ef..401046526 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/sh -uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload --proxy-headers +uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --proxy-headers diff --git a/api/entrypoint_alerts.sh b/api/entrypoint_alerts.sh index dedfa102b..9ac93dd60 100755 --- a/api/entrypoint_alerts.sh +++ b/api/entrypoint_alerts.sh @@ -1,3 +1,3 @@ #!/bin/sh export ASSIST_KEY=ignore -uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload +uvicorn app:app --host 0.0.0.0 --port 8888 diff --git a/api/run-alerts-dev.sh b/api/run-alerts-dev.sh index 54db30171..309356133 100755 --- a/api/run-alerts-dev.sh +++ b/api/run-alerts-dev.sh @@ -1,3 +1,3 @@ #!/bin/zsh -uvicorn app_alerts:app --reload \ No newline at end of file +uvicorn app_alerts:app --reload --port 8888 \ No newline at end of file diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 27ac41f5c..1e342e8bc 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -265,6 +265,8 @@ Pipfile.lock /app_alerts.py /build_alerts.sh /build_crons.sh +/run-dev.sh +/run-alerts-dev.sh /routers/subs/health.py /routers/subs/v1_api.py #exp /chalicelib/core/dashboards.py diff --git a/ee/api/app.py b/ee/api/app.py index 407e4aa5b..9104a2db4 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -1,5 +1,6 @@ import logging import queue +from contextlib import asynccontextmanager from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config @@ -10,9 +11,9 @@ from starlette import status from starlette.responses import StreamingResponse, JSONResponse from chalicelib.core import traces +from chalicelib.utils import events_queue from chalicelib.utils import helper from chalicelib.utils import pg_client -from chalicelib.utils import events_queue from routers import core, core_dynamic, ee, saml from routers.crons import core_crons from routers.crons import core_dynamic_crons @@ -20,7 +21,43 @@ from routers.crons import ee_crons from routers.subs import insights, metrics, v1_api_ee from routers.subs import v1_api, health -app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) +loglevel = config("LOGLEVEL", default=logging.INFO) +print(f">Loglevel set to: {loglevel}") +logging.basicConfig(level=loglevel) +ap_logger = logging.getLogger('apscheduler') +ap_logger.setLevel(loglevel) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Startup + logging.info(">>>>> starting up <<<<<") + app.schedule = AsyncIOScheduler() + app.queue_system = queue.Queue() + await pg_client.init() + await events_queue.init() + app.schedule.start() + + for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs + traces.cron_jobs + ee_crons.ee_cron_jobs: + app.schedule.add_job(id=job["func"].__name__, **job) + + ap_logger.info(">Scheduled jobs:") + for job in app.schedule.get_jobs(): + ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) + + # App listening + yield + + # Shutdown + logging.info(">>>>> shutting down <<<<<") + app.schedule.shutdown(wait=True) + await traces.process_traces_queue() + await events_queue.terminate() + await pg_client.terminate() + + +app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""), + lifespan=lifespan) app.add_middleware(GZipMiddleware, minimum_size=1000) @@ -71,43 +108,3 @@ app.include_router(v1_api_ee.app_apikey) app.include_router(health.public_app) app.include_router(health.app) app.include_router(health.app_apikey) - -loglevel = config("LOGLEVEL", default=logging.INFO) -print(f">Loglevel set to: {loglevel}") -logging.basicConfig(level=loglevel) -ap_logger = logging.getLogger('apscheduler') -ap_logger.setLevel(loglevel) -app.schedule = AsyncIOScheduler() -app.queue_system = queue.Queue() - - -@app.on_event("startup") -async def startup(): - logging.info(">>>>> starting up <<<<<") - await pg_client.init() - await events_queue.init() - app.schedule.start() - - for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs + traces.cron_jobs + ee_crons.ee_cron_jobs: - app.schedule.add_job(id=job["func"].__name__, **job) - - ap_logger.info(">Scheduled jobs:") - for job in app.schedule.get_jobs(): - ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) - - -@app.on_event("shutdown") -async def shutdown(): - logging.info(">>>>> shutting down <<<<<") - app.schedule.shutdown(wait=True) - await traces.process_traces_queue() - await events_queue.terminate() - await pg_client.terminate() - - -@app.get('/private/shutdown', tags=["private"]) -async def stop_server(): - logging.info("Requested shutdown") - await shutdown() - import os, signal - os.kill(1, signal.SIGTERM) diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index a160cf9c2..c47a80ee8 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -86,3 +86,5 @@ rm -rf ./chalicelib/core/performance_event.py rm -rf ./chalicelib/core/saved_search.py rm -rf ./app_alerts.py rm -rf ./build_alerts.sh +rm -rf ./run-dev.sh +rm -rf ./run-alerts-dev.sh diff --git a/ee/api/entrypoint.sh b/ee/api/entrypoint.sh index ebd646a7d..e63d4e2af 100755 --- a/ee/api/entrypoint.sh +++ b/ee/api/entrypoint.sh @@ -2,4 +2,4 @@ sh env_vars.sh source /tmp/.env.override -uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload --proxy-headers +uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --proxy-headers diff --git a/ee/api/entrypoint_alerts.sh b/ee/api/entrypoint_alerts.sh index acf8b390a..410015142 100755 --- a/ee/api/entrypoint_alerts.sh +++ b/ee/api/entrypoint_alerts.sh @@ -2,4 +2,4 @@ export ASSIST_KEY=ignore sh env_vars.sh source /tmp/.env.override -uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload +uvicorn app:app --host 0.0.0.0 --port 8888 diff --git a/ee/api/run-dev.sh b/ee/api/run-dev.sh deleted file mode 100755 index 76682286d..000000000 --- a/ee/api/run-dev.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/zsh - -uvicorn app:app --reload \ No newline at end of file From f7fbefba70220f4ec9a7634535591b2f9b3f3ca4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 16:33:01 +0100 Subject: [PATCH 088/253] feat(chalice): cleaned code --- ee/api/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ee/api/app.py b/ee/api/app.py index 9104a2db4..034d93565 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -24,14 +24,15 @@ from routers.subs import v1_api, health loglevel = config("LOGLEVEL", default=logging.INFO) print(f">Loglevel set to: {loglevel}") logging.basicConfig(level=loglevel) -ap_logger = logging.getLogger('apscheduler') -ap_logger.setLevel(loglevel) @asynccontextmanager async def lifespan(app: FastAPI): # Startup logging.info(">>>>> starting up <<<<<") + ap_logger = logging.getLogger('apscheduler') + ap_logger.setLevel(loglevel) + app.schedule = AsyncIOScheduler() app.queue_system = queue.Queue() await pg_client.init() From a0ba2feea2abfa4de2d780e8831f59c56f1ea1c4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 18:11:02 +0100 Subject: [PATCH 089/253] feat(chalice): fixed health-check --- api/chalicelib/core/health.py | 6 +----- ee/api/chalicelib/core/health.py | 10 +++------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index 6f236a0f5..f4e2abc6c 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -142,10 +142,6 @@ def __check_redis(): } -def __check_assist(): - pass - - def get_health(): health_map = { "databases": { @@ -157,7 +153,7 @@ def get_health(): "backendServices": { "alerts": __check_be_service("alerts"), "assets": __check_be_service("assets"), - "assist": __check_assist, + "assist": __check_be_service("assist"), "chalice": __always_healthy_with_version, "db": __check_be_service("db"), "ender": __check_be_service("ender"), diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 514425ddf..80abdacd7 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -30,7 +30,7 @@ if config("LOCAL_DEV", cast=bool, default=False): else: HEALTH_ENDPOINTS = { - "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/metrics", + "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health", "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", @@ -45,7 +45,7 @@ else: "quickwit": "http://quickwit-openreplay.app.svc.cluster.local:8888/metrics", "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health", - "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics" + "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", } @@ -144,10 +144,6 @@ def __check_redis(): } -def __check_assist(): - pass - - def get_health(): health_map = { "databases": { @@ -161,7 +157,7 @@ def get_health(): "backendServices": { "alerts": __check_be_service("alerts"), "assets": __check_be_service("assets"), - "assist": __check_assist, + "assist": __check_be_service("assist"), "chalice": __always_healthy_with_version, "db": __check_be_service("db"), "ender": __check_be_service("ender"), From cd2378f7bc8f32672b59eb96a9f52aa6f76c5504 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 15 Mar 2023 11:29:17 +0100 Subject: [PATCH 090/253] change(ui): change selection display --- .../app/player/web/managers/DOM/SelectionManager.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/app/player/web/managers/DOM/SelectionManager.ts b/frontend/app/player/web/managers/DOM/SelectionManager.ts index 9cf1fd21f..c457ec3b8 100644 --- a/frontend/app/player/web/managers/DOM/SelectionManager.ts +++ b/frontend/app/player/web/managers/DOM/SelectionManager.ts @@ -48,19 +48,21 @@ export default class SelectionManager extends ListWalker { Object.assign(endPointer.style, { top: endCoords.top + 'px', - left: (endCoords.left + endCoords.width + 3) + 'px', - width: '3px', + left: (endCoords.left + (endCoords.width / 2) + 3) + 'px', + width: (endCoords.width / 2) + 'px', height: endCoords.height + 'px', - border: '3px solid red', + borderRight: '2px solid blue', position: 'absolute', + boxShadow: '1px 4px 1px -2px blue', }); Object.assign(startPointer.style, { top: startCoords.top + 'px', left: (startCoords.left - 3) + 'px', - width: '3px', + width: (startCoords.width / 2 ) + 'px', height: startCoords.height + 'px', - border: '3px solid red', + borderLeft: '2px solid blue', position: 'absolute', + boxShadow: '1px 4px 1px -2px blue', }); this.markers.push(startPointer, endPointer); From ac617f36eec24590ac2fd22109205d2d6f4d4b2b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 15 Mar 2023 12:29:54 +0100 Subject: [PATCH 091/253] feat(DB): support new issue type --- ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 22d2e804e..cb40061d5 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -79,7 +79,7 @@ CREATE TABLE IF NOT EXISTS experimental.events success Nullable(UInt8), request_body Nullable(String), response_body Nullable(String), - issue_type Nullable(Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19)), + issue_type Nullable(Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20)), issue_id Nullable(String), error_tags_keys Array(String), error_tags_values Array(Nullable(String)), From 2fd52f46282a9f8eb3e324688c40bc28fe14cf6d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 15 Mar 2023 12:33:59 +0100 Subject: [PATCH 092/253] feat(DB): support new issue type --- ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql index 5e9c11242..2ae951a50 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql @@ -1 +1,4 @@ -CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.11.0-ee'; \ No newline at end of file +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.11.0-ee'; + +ALTER TABLE experimental.events + MODIFY COLUMN issue_type Nullable(Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20)); From a7ca8ac54f9a49205ce90a7b680c07fb3749ee5e Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 15 Mar 2023 16:21:44 +0100 Subject: [PATCH 093/253] feat(backend): implemented new events for CH --- backend/pkg/db/postgres/messages-web.go | 15 +----- backend/pkg/hashid/hashid.go | 8 +++ ee/backend/internal/db/datasaver/methods.go | 4 ++ ee/backend/pkg/db/clickhouse/connector.go | 56 ++++++++++++++++++++- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index a062df95a..ebba00168 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -1,12 +1,9 @@ package postgres import ( - "encoding/hex" - "hash/fnv" "log" - "strconv" - "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/hashid" . "openreplay/backend/pkg/messages" "openreplay/backend/pkg/url" ) @@ -171,7 +168,7 @@ func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *Mo // Debug log log.Printf("new MouseThrashing event: %v", e) // - issueID := mouseThrashingID(projectID, sessionID, e.Timestamp) + issueID := hashid.MouseThrashingID(projectID, sessionID, e.Timestamp) if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_thrashing", e.Url); err != nil { log.Printf("insert web issue err: %s", err) } @@ -181,11 +178,3 @@ func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *Mo conn.updateSessionIssues(sessionID, 0, 50) return nil } - -func mouseThrashingID(projectID uint32, sessID, ts uint64) string { - hash := fnv.New128a() - hash.Write([]byte("mouse_trashing")) - hash.Write([]byte(strconv.FormatUint(sessID, 10))) - hash.Write([]byte(strconv.FormatUint(ts, 10))) - return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) -} diff --git a/backend/pkg/hashid/hashid.go b/backend/pkg/hashid/hashid.go index 25ce11369..5bcb23578 100644 --- a/backend/pkg/hashid/hashid.go +++ b/backend/pkg/hashid/hashid.go @@ -23,3 +23,11 @@ func IOSCrashID(projectID uint32, crash *messages.IOSCrash) string { hash.Write([]byte(crash.Stacktrace)) return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) } + +func MouseThrashingID(projectID uint32, sessID, ts uint64) string { + hash := fnv.New128a() + hash.Write([]byte("mouse_trashing")) + hash.Write([]byte(strconv.FormatUint(sessID, 10))) + hash.Write([]byte(strconv.FormatUint(ts, 10))) + return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) +} diff --git a/ee/backend/internal/db/datasaver/methods.go b/ee/backend/internal/db/datasaver/methods.go index 277fd8906..ac0a8b88d 100644 --- a/ee/backend/internal/db/datasaver/methods.go +++ b/ee/backend/internal/db/datasaver/methods.go @@ -78,6 +78,10 @@ func (s *saverImpl) handleExtraMessage(msg Message) error { } case *GraphQL: return s.ch.InsertGraphQL(session, m) + case *InputChange: + return s.ch.InsertWebInputDuration(session, m) + case *MouseThrashing: + return s.ch.InsertMouseThrashing(session, m) } return nil } diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 489411550..ad08bd6e4 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -32,6 +32,8 @@ type Connector interface { InsertCustom(session *types.Session, msg *messages.CustomEvent) error InsertGraphQL(session *types.Session, msg *messages.GraphQL) error InsertIssue(session *types.Session, msg *messages.IssueEvent) error + InsertWebInputDuration(session *types.Session, msg *messages.InputChange) error + InsertMouseThrashing(session *types.Session, msg *messages.MouseThrashing) error } type task struct { @@ -97,7 +99,7 @@ var batches = map[string]string{ "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, ?)", "pages": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "clicks": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, hesitation_time, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)", - "inputs": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, event_type) VALUES (?, ?, ?, ?, ?, ?)", + "inputs": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, event_type, duration, hesitation_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "errors": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, source, name, message, error_id, event_type, error_tags_keys, error_tags_values) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "performance": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "requests": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_body, response_body, status, method, duration, success, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?)", @@ -164,11 +166,59 @@ func (c *connectorImpl) checkError(name string, err error) { } } +func (c *connectorImpl) InsertWebInputDuration(session *types.Session, msg *messages.InputChange) error { + if msg.Label == "" { + return nil + } + if err := c.batches["inputs"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MessageID, + datetime(msg.Timestamp), + msg.Label, + "INPUT", + nullableUint16(uint16(msg.Duration)), + nullableUint32(uint32(msg.HesitationTime)), + ); err != nil { + c.checkError("inputs", err) + return fmt.Errorf("can't append to inputs batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertMouseThrashing(session *types.Session, msg *messages.MouseThrashing) error { + issueID := hashid.MouseThrashingID(session.projectID, session.sessionID, msg.Timestamp) + // Insert issue event to batches + if err := c.batches["issuesEvents"].Append( + session.SessionID, + uint16(session.ProjectID), + msg.MsgID(), + datetime(msg.Timestamp), + issueID, + "mouse_thrashing", + "ISSUE", + msg.Url, + ); err != nil { + c.checkError("issuesEvents", err) + return fmt.Errorf("can't append to issuesEvents batch: %s", err) + } + if err := c.batches["issues"].Append( + uint16(session.ProjectID), + issueID, + "mouse_thrashing", + msg.Url, + ); err != nil { + c.checkError("issues", err) + return fmt.Errorf("can't append to issues batch: %s", err) + } + return nil +} + func (c *connectorImpl) InsertIssue(session *types.Session, msg *messages.IssueEvent) error { issueID := hashid.IssueID(session.ProjectID, msg) // Check issue type before insert to avoid panic from clickhouse lib switch msg.Type { - case "click_rage", "dead_click", "excessive_scrolling", "bad_request", "missing_resource", "memory", "cpu", "slow_resource", "slow_page_load", "crash", "ml_cpu", "ml_memory", "ml_dead_click", "ml_click_rage", "ml_mouse_thrashing", "ml_excessive_scrolling", "ml_slow_resources", "custom", "js_exception": + case "click_rage", "dead_click", "excessive_scrolling", "bad_request", "missing_resource", "memory", "cpu", "slow_resource", "slow_page_load", "crash", "ml_cpu", "ml_memory", "ml_dead_click", "ml_click_rage", "ml_mouse_thrashing", "ml_excessive_scrolling", "ml_slow_resources", "custom", "js_exception", "mouse_thrashing": default: return fmt.Errorf("unknown issueType: %s", msg.Type) } @@ -323,6 +373,8 @@ func (c *connectorImpl) InsertWebInputEvent(session *types.Session, msg *message datetime(msg.Timestamp), msg.Label, "INPUT", + nil, + nil, ); err != nil { c.checkError("inputs", err) return fmt.Errorf("can't append to inputs batch: %s", err) From d6526464c605c5d897f0f761cf0a1ab49e5e73a3 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 15 Mar 2023 16:26:46 +0100 Subject: [PATCH 094/253] fix(backend): fixed field names --- ee/backend/pkg/db/clickhouse/connector.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index ad08bd6e4..0ee0658ef 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -173,11 +173,11 @@ func (c *connectorImpl) InsertWebInputDuration(session *types.Session, msg *mess if err := c.batches["inputs"].Append( session.SessionID, uint16(session.ProjectID), - msg.MessageID, + msg.MsgID(), datetime(msg.Timestamp), msg.Label, "INPUT", - nullableUint16(uint16(msg.Duration)), + nullableUint16(uint16(msg.InputDuration)), nullableUint32(uint32(msg.HesitationTime)), ); err != nil { c.checkError("inputs", err) @@ -187,7 +187,7 @@ func (c *connectorImpl) InsertWebInputDuration(session *types.Session, msg *mess } func (c *connectorImpl) InsertMouseThrashing(session *types.Session, msg *messages.MouseThrashing) error { - issueID := hashid.MouseThrashingID(session.projectID, session.sessionID, msg.Timestamp) + issueID := hashid.MouseThrashingID(session.ProjectID, session.SessionID, msg.Timestamp) // Insert issue event to batches if err := c.batches["issuesEvents"].Append( session.SessionID, From c9c50d6650b68f16747f1b4351e1af4ed973b600 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 15 Mar 2023 18:06:30 +0100 Subject: [PATCH 095/253] feat(DB): support new issue type --- .../schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql | 4 ++++ .../schema/db/init_dbs/clickhouse/create/init_schema.sql | 2 +- .../schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql | 9 +++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql index 2ae951a50..1962fde10 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql @@ -2,3 +2,7 @@ CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.11.0-ee'; ALTER TABLE experimental.events MODIFY COLUMN issue_type Nullable(Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20)); + +ALTER TABLE experimental.issues + MODIFY COLUMN type Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20); + diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index cb40061d5..9536307d8 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -201,7 +201,7 @@ CREATE TABLE IF NOT EXISTS experimental.issues ( project_id UInt16, issue_id String, - type Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19), + type Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20), context_string String, context_keys Array(String), context_values Array(Nullable(String)), diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql index 30bb27997..cf80dcad1 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql @@ -5,14 +5,10 @@ $$ SELECT 'v1.11.0-ee' $$ LANGUAGE sql IMMUTABLE; - - ALTER TABLE events.inputs ADD COLUMN duration integer NULL, ADD COLUMN hesitation integer NULL; - - ALTER TABLE public.projects ALTER COLUMN gdpr SET DEFAULT '{ "maskEmails": true, @@ -21,4 +17,9 @@ ALTER TABLE public.projects "defaultInputMode": "obscured" }'::jsonb; +ALTER TYPE issue_type ADD VALUE IF NOT EXISTS 'mouse_thrashing'; + +ALTER TABLE events.clicks + ADD COLUMN hesitation integer NULL; + COMMIT; \ No newline at end of file From 4608911f5b5191a1163d2a7e1b1e91a761fa7d24 Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 16 Mar 2023 10:53:05 +0100 Subject: [PATCH 096/253] Sink and Ender message iterators refactoring (#1038) * feat(backend): removed duplicate code in ender message iterator * feat(backend): removed duplicate code in sink message iterator --- backend/pkg/messages/iterator-ender.go | 185 +++---------------------- backend/pkg/messages/iterator-sink.go | 176 ++--------------------- 2 files changed, 31 insertions(+), 330 deletions(-) diff --git a/backend/pkg/messages/iterator-ender.go b/backend/pkg/messages/iterator-ender.go index 1d9063749..b822d166e 100644 --- a/backend/pkg/messages/iterator-ender.go +++ b/backend/pkg/messages/iterator-ender.go @@ -1,179 +1,30 @@ package messages -import ( - "fmt" - "log" -) - -type enderMessageIteratorImpl struct { - filter map[int]struct{} - preFilter map[int]struct{} - handler MessageHandler - autoDecode bool - version uint64 - size uint64 - canSkip bool - broken bool - messageInfo *message - batchInfo *BatchInfo - urls *pageLocations +type enderIteratorImpl struct { + coreIterator MessageIterator + handler MessageHandler + lastMessage Message } func NewEnderMessageIterator(messageHandler MessageHandler, messageFilter []int, autoDecode bool) MessageIterator { - iter := &enderMessageIteratorImpl{ - handler: messageHandler, - autoDecode: autoDecode, - urls: NewPageLocations(), + enderIter := &enderIteratorImpl{ + handler: messageHandler, } - if len(messageFilter) != 0 { - filter := make(map[int]struct{}, len(messageFilter)) - for _, msgType := range messageFilter { - filter[msgType] = struct{}{} - } - iter.filter = filter - } - iter.preFilter = map[int]struct{}{ - MsgBatchMetadata: {}, MsgBatchMeta: {}, MsgTimestamp: {}, - MsgSessionStart: {}, MsgSessionEnd: {}, MsgSetPageLocation: {}, - } - return iter + enderIter.coreIterator = NewMessageIterator(enderIter.handle, messageFilter, autoDecode) + return enderIter } -func (i *enderMessageIteratorImpl) prepareVars(batchInfo *BatchInfo) { - i.batchInfo = batchInfo - i.messageInfo = &message{batch: batchInfo} - i.version = 0 - i.canSkip = false - i.broken = false - i.size = 0 +func (e *enderIteratorImpl) handle(message Message) { + e.lastMessage = message } -func (i *enderMessageIteratorImpl) Iterate(batchData []byte, batchInfo *BatchInfo) { - // Create new message reader - reader := NewMessageReader(batchData) - - // Pre-decode batch data - if err := reader.Parse(); err != nil { - log.Printf("pre-decode batch err: %s, info: %s", err, batchInfo.Info()) - return +func (e *enderIteratorImpl) Iterate(batchData []byte, batchInfo *BatchInfo) { + // Reset last message + e.lastMessage = nil + // Call core iterator + e.coreIterator.Iterate(batchData, batchInfo) + // Call handler if last message is not nil + if e.lastMessage != nil { + e.handler(e.lastMessage) } - - // Prepare iterator before processing messages in batch - i.prepareVars(batchInfo) - - // Store last timestamp message here - var lastMessage Message - - for reader.Next() { - // Increase message index (can be overwritten by batch info message) - i.messageInfo.Index++ - - msg := reader.Message() - - // Preprocess "system" messages - if _, ok := i.preFilter[msg.TypeID()]; ok { - msg = msg.Decode() - if msg == nil { - log.Printf("decode error, type: %d, info: %s", msg.TypeID(), i.batchInfo.Info()) - return - } - msg = transformDeprecated(msg) - if err := i.preprocessing(msg); err != nil { - log.Printf("message preprocessing err: %s", err) - return - } - } - - // Skip messages we don't have in filter - if i.filter != nil { - if _, ok := i.filter[msg.TypeID()]; !ok { - continue - } - } - - if i.autoDecode { - msg = msg.Decode() - if msg == nil { - log.Printf("decode error, type: %d, info: %s", msg.TypeID(), i.batchInfo.Info()) - return - } - } - - // Set meta information for message - msg.Meta().SetMeta(i.messageInfo) - - // Update last timestamp message - lastMessage = msg - } - - if lastMessage != nil { - i.handler(lastMessage) - } - -} - -func (i *enderMessageIteratorImpl) zeroTsLog(msgType string) { - log.Printf("zero timestamp in %s, info: %s", msgType, i.batchInfo.Info()) -} - -func (i *enderMessageIteratorImpl) preprocessing(msg Message) error { - switch m := msg.(type) { - case *BatchMetadata: - if i.messageInfo.Index > 1 { // Might be several 0-0 BatchMeta in a row without an error though - return fmt.Errorf("batchMetadata found at the end of the batch, info: %s", i.batchInfo.Info()) - } - if m.Version > 1 { - return fmt.Errorf("incorrect batch version: %d, skip current batch, info: %s", i.version, i.batchInfo.Info()) - } - i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = uint64(m.Timestamp) - if m.Timestamp == 0 { - i.zeroTsLog("BatchMetadata") - } - i.messageInfo.Url = m.Location - i.version = m.Version - i.batchInfo.version = m.Version - - case *BatchMeta: // Is not required to be present in batch since IOS doesn't have it (though we might change it) - if i.messageInfo.Index > 1 { // Might be several 0-0 BatchMeta in a row without an error though - return fmt.Errorf("batchMeta found at the end of the batch, info: %s", i.batchInfo.Info()) - } - i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = uint64(m.Timestamp) - if m.Timestamp == 0 { - i.zeroTsLog("BatchMeta") - } - // Try to get saved session's page url - if savedURL := i.urls.Get(i.messageInfo.batch.sessionID); savedURL != "" { - i.messageInfo.Url = savedURL - } - - case *Timestamp: - i.messageInfo.Timestamp = m.Timestamp - if m.Timestamp == 0 { - i.zeroTsLog("Timestamp") - } - - case *SessionStart: - i.messageInfo.Timestamp = m.Timestamp - if m.Timestamp == 0 { - i.zeroTsLog("SessionStart") - log.Printf("zero session start, project: %d, UA: %s, tracker: %s, info: %s", - m.ProjectID, m.UserAgent, m.TrackerVersion, i.batchInfo.Info()) - } - - case *SessionEnd: - i.messageInfo.Timestamp = m.Timestamp - if m.Timestamp == 0 { - i.zeroTsLog("SessionEnd") - } - // Delete session from urls cache layer - i.urls.Delete(i.messageInfo.batch.sessionID) - - case *SetPageLocation: - i.messageInfo.Url = m.URL - // Save session page url in cache for using in next batches - i.urls.Set(i.messageInfo.batch.sessionID, m.URL) - } - return nil } diff --git a/backend/pkg/messages/iterator-sink.go b/backend/pkg/messages/iterator-sink.go index 15ab5c077..de33f5438 100644 --- a/backend/pkg/messages/iterator-sink.go +++ b/backend/pkg/messages/iterator-sink.go @@ -1,181 +1,31 @@ package messages import ( - "fmt" - "log" "openreplay/backend/pkg/metrics/sink" ) -type sinkMessageIteratorImpl struct { - filter map[int]struct{} - preFilter map[int]struct{} - handler MessageHandler - autoDecode bool - version uint64 - size uint64 - canSkip bool - broken bool - messageInfo *message - batchInfo *BatchInfo - urls *pageLocations +type sinkIteratorImpl struct { + coreIterator MessageIterator + handler MessageHandler } func NewSinkMessageIterator(messageHandler MessageHandler, messageFilter []int, autoDecode bool) MessageIterator { - iter := &sinkMessageIteratorImpl{ - handler: messageHandler, - autoDecode: autoDecode, - urls: NewPageLocations(), - } - if len(messageFilter) != 0 { - filter := make(map[int]struct{}, len(messageFilter)) - for _, msgType := range messageFilter { - filter[msgType] = struct{}{} - } - iter.filter = filter - } - iter.preFilter = map[int]struct{}{ - MsgBatchMetadata: {}, MsgBatchMeta: {}, MsgTimestamp: {}, - MsgSessionStart: {}, MsgSessionEnd: {}, MsgSetPageLocation: {}, + iter := &sinkIteratorImpl{ + handler: messageHandler, } + iter.coreIterator = NewMessageIterator(iter.handle, messageFilter, autoDecode) return iter } -func (i *sinkMessageIteratorImpl) prepareVars(batchInfo *BatchInfo) { - i.batchInfo = batchInfo - i.messageInfo = &message{batch: batchInfo} - i.version = 0 - i.canSkip = false - i.broken = false - i.size = 0 +func (i *sinkIteratorImpl) handle(message Message) { + i.handler(message) } -func (i *sinkMessageIteratorImpl) sendBatchEnd() { - i.handler(nil) -} - -func (i *sinkMessageIteratorImpl) Iterate(batchData []byte, batchInfo *BatchInfo) { +func (i *sinkIteratorImpl) Iterate(batchData []byte, batchInfo *BatchInfo) { sink.RecordBatchSize(float64(len(batchData))) sink.IncreaseTotalBatches() - // Create new message reader - reader := NewMessageReader(batchData) - - // Pre-decode batch data - if err := reader.Parse(); err != nil { - log.Printf("pre-decode batch err: %s, info: %s", err, batchInfo.Info()) - return - } - - // Prepare iterator before processing messages in batch - i.prepareVars(batchInfo) - - for reader.Next() { - // Increase message index (can be overwritten by batch info message) - i.messageInfo.Index++ - - msg := reader.Message() - - // Preprocess "system" messages - if _, ok := i.preFilter[msg.TypeID()]; ok { - msg = msg.Decode() - if msg == nil { - log.Printf("decode error, type: %d, info: %s", msg.TypeID(), i.batchInfo.Info()) - break - } - msg = transformDeprecated(msg) - if err := i.preprocessing(msg); err != nil { - log.Printf("message preprocessing err: %s", err) - break - } - } - - // Skip messages we don't have in filter - if i.filter != nil { - if _, ok := i.filter[msg.TypeID()]; !ok { - continue - } - } - - if i.autoDecode { - msg = msg.Decode() - if msg == nil { - log.Printf("decode error, type: %d, info: %s", msg.TypeID(), i.batchInfo.Info()) - break - } - } - - // Set meta information for message - msg.Meta().SetMeta(i.messageInfo) - - // Process message - i.handler(msg) - } - - // Inform sink about end of batch - i.sendBatchEnd() -} - -func (i *sinkMessageIteratorImpl) zeroTsLog(msgType string) { - log.Printf("zero timestamp in %s, info: %s", msgType, i.batchInfo.Info()) -} - -func (i *sinkMessageIteratorImpl) preprocessing(msg Message) error { - switch m := msg.(type) { - case *BatchMetadata: - if i.messageInfo.Index > 1 { // Might be several 0-0 BatchMeta in a row without an error though - return fmt.Errorf("batchMetadata found at the end of the batch, info: %s", i.batchInfo.Info()) - } - if m.Version > 1 { - return fmt.Errorf("incorrect batch version: %d, skip current batch, info: %s", i.version, i.batchInfo.Info()) - } - i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = uint64(m.Timestamp) - if m.Timestamp == 0 { - i.zeroTsLog("BatchMetadata") - } - i.messageInfo.Url = m.Location - i.version = m.Version - i.batchInfo.version = m.Version - - case *BatchMeta: // Is not required to be present in batch since IOS doesn't have it (though we might change it) - if i.messageInfo.Index > 1 { // Might be several 0-0 BatchMeta in a row without an error though - return fmt.Errorf("batchMeta found at the end of the batch, info: %s", i.batchInfo.Info()) - } - i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) - i.messageInfo.Timestamp = uint64(m.Timestamp) - if m.Timestamp == 0 { - i.zeroTsLog("BatchMeta") - } - // Try to get saved session's page url - if savedURL := i.urls.Get(i.messageInfo.batch.sessionID); savedURL != "" { - i.messageInfo.Url = savedURL - } - - case *Timestamp: - i.messageInfo.Timestamp = m.Timestamp - if m.Timestamp == 0 { - i.zeroTsLog("Timestamp") - } - - case *SessionStart: - i.messageInfo.Timestamp = m.Timestamp - if m.Timestamp == 0 { - i.zeroTsLog("SessionStart") - log.Printf("zero session start, project: %d, UA: %s, tracker: %s, info: %s", - m.ProjectID, m.UserAgent, m.TrackerVersion, i.batchInfo.Info()) - } - - case *SessionEnd: - i.messageInfo.Timestamp = m.Timestamp - if m.Timestamp == 0 { - i.zeroTsLog("SessionEnd") - } - // Delete session from urls cache layer - i.urls.Delete(i.messageInfo.batch.sessionID) - - case *SetPageLocation: - i.messageInfo.Url = m.URL - // Save session page url in cache for using in next batches - i.urls.Set(i.messageInfo.batch.sessionID, m.URL) - } - return nil + // Call core iterator + i.coreIterator.Iterate(batchData, batchInfo) + // Send batch end signal + i.handler(nil) } From f360ce3d2c350ed4b0235e4fe3715db2dc6d734b Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 16 Mar 2023 17:11:13 +0100 Subject: [PATCH 097/253] change(ui): add visual display for frustrations --- .../components/Session_/EventsBlock/Event.js | 47 +++++++++++----- .../Session_/EventsBlock/event.module.css | 7 +++ .../Session_/OverviewPanel/OverviewPanel.tsx | 4 +- .../FeatureSelection/FeatureSelection.tsx | 8 +-- .../TimelinePointer/TimelinePointer.tsx | 20 ++++--- frontend/app/components/ui/SVG.tsx | 9 +++- .../app/components/ui/Tooltip/Tooltip.tsx | 3 +- frontend/app/player/web/Lists.ts | 2 +- frontend/app/player/web/WebPlayer.ts | 1 + frontend/app/svg/icons/click-hesitation.svg | 15 ++++++ frontend/app/svg/icons/click-rage.svg | 9 ++++ frontend/app/svg/icons/cursor-trash.svg | 15 ++++++ .../app/svg/icons/event/click_hesitation.svg | 15 ++++++ .../app/svg/icons/event/input_hesitation.svg | 17 ++++++ .../app/svg/icons/event/mouse_thrashing.svg | 15 ++++++ frontend/app/svg/icons/input-hesitation.svg | 17 ++++++ frontend/app/types/session/event.ts | 20 ++++--- frontend/app/types/session/issue.ts | 2 +- frontend/app/types/session/session.ts | 53 +++++++++++++++++-- frontend/scripts/icons.ts | 13 ++--- 20 files changed, 245 insertions(+), 47 deletions(-) create mode 100644 frontend/app/svg/icons/click-hesitation.svg create mode 100644 frontend/app/svg/icons/click-rage.svg create mode 100644 frontend/app/svg/icons/cursor-trash.svg create mode 100644 frontend/app/svg/icons/event/click_hesitation.svg create mode 100644 frontend/app/svg/icons/event/input_hesitation.svg create mode 100644 frontend/app/svg/icons/event/mouse_thrashing.svg create mode 100644 frontend/app/svg/icons/input-hesitation.svg diff --git a/frontend/app/components/Session_/EventsBlock/Event.js b/frontend/app/components/Session_/EventsBlock/Event.js index e8f985aa0..a464fcc28 100644 --- a/frontend/app/components/Session_/EventsBlock/Event.js +++ b/frontend/app/components/Session_/EventsBlock/Event.js @@ -1,7 +1,7 @@ import React from 'react'; import copy from 'copy-to-clipboard'; import cn from 'classnames'; -import { Icon, TextEllipsis } from 'UI'; +import { Icon, TextEllipsis, Tooltip } from 'UI'; import { TYPES } from 'Types/session/event'; import { prorata } from 'App/utils'; import withOverlay from 'Components/hocs/withOverlay'; @@ -9,6 +9,16 @@ import LoadInfo from './LoadInfo'; import cls from './event.module.css'; import { numberWithCommas } from 'App/utils'; +function isFrustrationEvent(evt) { + if (evt.type === 'mouse_thrashing' || evt.type === TYPES.CLICKRAGE) { + return true; + } + if (evt.type === TYPES.CLICK || evt.type === TYPES.INPUT) { + return evt.hesitation > 1000 + } + return false +} + @withOverlay() export default class Event extends React.PureComponent { state = { @@ -44,35 +54,50 @@ export default class Event extends React.PureComponent { const { event } = this.props; let title = event.type; let body; + let icon; + const isFrustration = isFrustrationEvent(event); + const tooltip = { disabled: true, text: '' } + switch (event.type) { case TYPES.LOCATION: title = 'Visited'; body = event.url; + icon = 'location'; break; case TYPES.CLICK: title = 'Clicked'; body = event.label; + icon = isFrustration ? 'click_hesitation' : 'click'; + isFrustration ? Object.assign(tooltip, { disabled: false, text: `User hesitated to click for ${Math.round(event.hesitation/1000)}s`, }) : null; break; case TYPES.INPUT: title = 'Input'; body = event.value; + icon = isFrustration ? 'input_hesitation' : 'input'; + isFrustration ? Object.assign(tooltip, { disabled: false, text: `User hesitated to enter a value for ${Math.round(event.hesitation/1000)}s`, }) : null; break; case TYPES.CLICKRAGE: title = `${ event.count } Clicks`; body = event.label; + icon = 'clickrage' break; case TYPES.IOS_VIEW: title = 'View'; body = event.name; + icon = 'ios_view' + break; + case 'mouse_thrashing': + title = 'Mouse Thrashing'; + icon = 'mouse_thrashing' break; } const isLocation = event.type === TYPES.LOCATION; - const isClickrage = event.type === TYPES.CLICKRAGE; return ( +
- { event.type && } + { event.type && }
@@ -100,6 +125,7 @@ export default class Event extends React.PureComponent {
}
+ ); }; @@ -110,17 +136,15 @@ export default class Event extends React.PureComponent { isCurrent, onClick, showSelection, - onCheckboxClick, showLoadInfo, toggleLoadInfo, isRed, - extended, - highlight = false, presentInSearch = false, - isLastInGroup, whiteBg, } = this.props; const { menuOpen } = this.state; + + const isFrustration = isFrustrationEvent(event); return (
{ this.wrapper = ref } } @@ -135,7 +159,7 @@ export default class Event extends React.PureComponent { [ cls.red ]: isRed, [ cls.clickType ]: event.type === TYPES.CLICK, [ cls.inputType ]: event.type === TYPES.INPUT, - [ cls.clickrageType ]: event.type === TYPES.CLICKRAGE, + [ cls.frustration ]: isFrustration, [ cls.highlight ] : presentInSearch, [ cls.lastInGroup ]: whiteBg, }) } @@ -146,13 +170,10 @@ export default class Event extends React.PureComponent { { event.target ? 'Copy CSS' : 'Copy URL' } } -
-
+
+
{ this.renderBody() }
- {/* { event.type === TYPES.LOCATION && -
{event.url}
- } */}
{ event.type === TYPES.LOCATION && (event.fcpTime || event.visuallyComplete || event.timeToInteractive) && [] }) { performanceChartData, stackList: stackEventList, eventList: eventsList, + frustrationsList, exceptionsList, resourceList: resourceListUnmap, fetchList, @@ -46,8 +46,8 @@ function OverviewPanel({ issuesList }: { issuesList: Record[] }) { NETWORK: resourceList, ERRORS: exceptionsList, EVENTS: stackEventList, - CLICKRAGE: eventsList.filter((item: any) => item.type === TYPES.CLICKRAGE), PERFORMANCE: performanceChartData, + FRUSTRATIONS: frustrationsList, }; }, [dataLoaded]); diff --git a/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx b/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx index 8d76a3070..3a841d97c 100644 --- a/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx +++ b/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx @@ -4,15 +4,15 @@ import { Checkbox, Tooltip } from 'UI'; const NETWORK = 'NETWORK'; const ERRORS = 'ERRORS'; const EVENTS = 'EVENTS'; -const CLICKRAGE = 'CLICKRAGE'; +const FRUSTRATIONS = 'FRUSTRATIONS'; const PERFORMANCE = 'PERFORMANCE'; export const HELP_MESSAGE: any = { NETWORK: 'Network requests made in this session', EVENTS: 'Visualizes the events that takes place in the DOM', ERRORS: 'Visualizes native JS errors like Type, URI, Syntax etc.', - CLICKRAGE: 'Indicates user frustration when repeated clicks are recorded', PERFORMANCE: 'Summary of this session’s memory, and CPU consumption on the timeline', + FRUSTRATIONS: 'Indicates user frustrations in the session', }; interface Props { @@ -21,7 +21,7 @@ interface Props { } function FeatureSelection(props: Props) { const { list } = props; - const features = [NETWORK, ERRORS, EVENTS, CLICKRAGE, PERFORMANCE]; + const features = [NETWORK, ERRORS, EVENTS, PERFORMANCE, FRUSTRATIONS]; const disabled = list.length >= 5; return ( @@ -30,7 +30,7 @@ function FeatureSelection(props: Props) { const checked = list.includes(feature); const _disabled = disabled && !checked; return ( - + { ); }; - const renderClickRageElement = (item: any) => { + const renderFrustrationElement = (item: any) => { + const elData = { name: '', icon: ''} + if (item.type === TYPES.CLICK) Object.assign(elData, { name: `User hesitated to click for ${Math.round(item.hesitation/1000)}s`, icon: 'click-hesitation' }) + if (item.type === TYPES.INPUT) Object.assign(elData, { name: `User hesitated to enter a value for ${Math.round(item.hesitation/1000)}s`, icon: 'input-hesitation' }) + if (item.type === TYPES.CLICKRAGE) Object.assign(elData, { name: 'Click Rage', icon: 'click-rage' }) + if (item.type === issueTypes.MOUSE_THRASHING) Object.assign(elData, { name: 'Mouse Thrashing', icon: 'cursor-trash' }) + return ( - {'Click Rage'} + {elData.name}
} delay={0} placement="top" >
- +
); @@ -158,8 +166,8 @@ const TimelinePointer = React.memo((props: Props) => { if (type === 'NETWORK') { return renderNetworkElement(pointer); } - if (type === 'CLICKRAGE') { - return renderClickRageElement(pointer); + if (type === 'FRUSTRATIONS') { + return renderFrustrationElement(pointer); } if (type === 'ERRORS') { return renderExceptionElement(pointer); diff --git a/frontend/app/components/ui/SVG.tsx b/frontend/app/components/ui/SVG.tsx index 66af7be76..eeaeae5dd 100644 --- a/frontend/app/components/ui/SVG.tsx +++ b/frontend/app/components/ui/SVG.tsx @@ -1,7 +1,7 @@ import React from 'react'; -export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope' | 'errors-icon' | 'event/click' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/link' | 'event/location' | 'event/resize' | 'event/view' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; +export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'click-hesitation' | 'click-rage' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'cursor-trash' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope' | 'errors-icon' | 'event/click' | 'event/click_hesitation' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/input_hesitation' | 'event/link' | 'event/location' | 'event/mouse_thrashing' | 'event/resize' | 'event/view' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'input-hesitation' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; interface Props { name: IconNames; @@ -119,6 +119,8 @@ const SVG = (props: Props) => { case 'chevron-up': return ; case 'circle-fill': return ; case 'circle': return ; + case 'click-hesitation': return ; + case 'click-rage': return ; case 'clipboard-list-check': return ; case 'clock': return ; case 'close': return ; @@ -140,6 +142,7 @@ const SVG = (props: Props) => { case 'credit-card-front': return ; case 'cross': return ; case 'cubes': return ; + case 'cursor-trash': return ; case 'dash': return ; case 'dashboard-icn': return ; case 'desktop': return ; @@ -156,12 +159,15 @@ const SVG = (props: Props) => { case 'envelope': return ; case 'errors-icon': return ; case 'event/click': return ; + case 'event/click_hesitation': return ; case 'event/clickrage': return ; case 'event/code': return ; case 'event/i-cursor': return ; case 'event/input': return ; + case 'event/input_hesitation': return ; case 'event/link': return ; case 'event/location': return ; + case 'event/mouse_thrashing': return ; case 'event/resize': return ; case 'event/view': return ; case 'exclamation-circle': return ; @@ -271,6 +277,7 @@ const SVG = (props: Props) => { case 'info-circle': return ; case 'info-square': return ; case 'info': return ; + case 'input-hesitation': return ; case 'inspect': return ; case 'integrations/assist': return ; case 'integrations/bugsnag-text': return ; diff --git a/frontend/app/components/ui/Tooltip/Tooltip.tsx b/frontend/app/components/ui/Tooltip/Tooltip.tsx index fcb5e1687..ee68b71e7 100644 --- a/frontend/app/components/ui/Tooltip/Tooltip.tsx +++ b/frontend/app/components/ui/Tooltip/Tooltip.tsx @@ -23,6 +23,7 @@ function Tooltip(props: Props) { placement, className = '', anchorClassName = '', + containerClassName = '', delay = 500, style = {}, offset = 5, @@ -39,7 +40,7 @@ function Tooltip(props: Props) { }); return ( -
+
{props.children} Log({ level: LogLevel.ERROR, diff --git a/frontend/app/svg/icons/click-hesitation.svg b/frontend/app/svg/icons/click-hesitation.svg new file mode 100644 index 000000000..144b82cd5 --- /dev/null +++ b/frontend/app/svg/icons/click-hesitation.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/click-rage.svg b/frontend/app/svg/icons/click-rage.svg new file mode 100644 index 000000000..54ccb06a6 --- /dev/null +++ b/frontend/app/svg/icons/click-rage.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/frontend/app/svg/icons/cursor-trash.svg b/frontend/app/svg/icons/cursor-trash.svg new file mode 100644 index 000000000..bdf687c91 --- /dev/null +++ b/frontend/app/svg/icons/cursor-trash.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/event/click_hesitation.svg b/frontend/app/svg/icons/event/click_hesitation.svg new file mode 100644 index 000000000..144b82cd5 --- /dev/null +++ b/frontend/app/svg/icons/event/click_hesitation.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/event/input_hesitation.svg b/frontend/app/svg/icons/event/input_hesitation.svg new file mode 100644 index 000000000..a2f79bfb6 --- /dev/null +++ b/frontend/app/svg/icons/event/input_hesitation.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/event/mouse_thrashing.svg b/frontend/app/svg/icons/event/mouse_thrashing.svg new file mode 100644 index 000000000..af00c02cf --- /dev/null +++ b/frontend/app/svg/icons/event/mouse_thrashing.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/input-hesitation.svg b/frontend/app/svg/icons/input-hesitation.svg new file mode 100644 index 000000000..439606a52 --- /dev/null +++ b/frontend/app/svg/icons/input-hesitation.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/frontend/app/types/session/event.ts b/frontend/app/types/session/event.ts index 99693d756..53924ef0c 100644 --- a/frontend/app/types/session/event.ts +++ b/frontend/app/types/session/event.ts @@ -7,16 +7,18 @@ const CLICKRAGE = 'CLICKRAGE'; const IOS_VIEW = 'VIEW'; export const TYPES = { CONSOLE, CLICK, INPUT, LOCATION, CUSTOM, CLICKRAGE, IOS_VIEW }; +export type EventType = + | typeof CONSOLE + | typeof CLICK + | typeof INPUT + | typeof LOCATION + | typeof CUSTOM + | typeof CLICKRAGE; + interface IEvent { time: number; timestamp: number; - type: - | typeof CONSOLE - | typeof CLICK - | typeof INPUT - | typeof LOCATION - | typeof CUSTOM - | typeof CLICKRAGE; + type: EventType name: string; key: number; label: string; @@ -35,6 +37,7 @@ interface ConsoleEvent extends IEvent { interface ClickEvent extends IEvent { targetContent: string; count: number; + hesitation: number; } interface InputEvent extends IEvent { @@ -98,12 +101,13 @@ export class Click extends Event { readonly name = 'Click'; targetContent = ''; count: number; + hesitation: number = 0; constructor(evt: ClickEvent, isClickRage?: boolean) { - console.log(evt); super(evt); this.targetContent = evt.targetContent; this.count = evt.count; + this.hesitation = evt.hesitation; if (isClickRage) { this.type = CLICKRAGE; } diff --git a/frontend/app/types/session/issue.ts b/frontend/app/types/session/issue.ts index 2abedc5ea..8f5f415e3 100644 --- a/frontend/app/types/session/issue.ts +++ b/frontend/app/types/session/issue.ts @@ -1,6 +1,6 @@ import Record from 'Types/Record'; -const types = { +export const types = { ALL: 'all', JS_EXCEPTION: 'js_exception', BAD_REQUEST: 'bad_request', diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index 3b254ae4b..a473ed623 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -2,12 +2,37 @@ import { Duration } from 'luxon'; import SessionEvent, { TYPES, EventData, InjectedEvent } from './event'; import StackEvent from './stackEvent'; import SessionError, { IError } from './error'; -import Issue, { IIssue } from './issue'; +import Issue, { IIssue, types as issueTypes } from './issue'; import { Note } from 'App/services/NotesService' const HASH_MOD = 1610612741; const HASH_P = 53; +function mergeEventLists(arr1: any[], arr2: any[]) { + let merged = []; + let index1 = 0; + let index2 = 0; + let current = 0; + + while (current < (arr1.length + arr2.length)) { + + let isArr1Depleted = index1 >= arr1.length; + let isArr2Depleted = index2 >= arr2.length; + + if (!isArr1Depleted && (isArr2Depleted || (arr1[index1].timestamp < arr2[index2].timestamp))) { + merged[current] = arr1[index1]; + index1++; + } else { + merged[current] = arr2[index2]; + index2++; + } + + current++; + } + + return merged; +} + function hashString(s: string): number { let mul = 1; let hash = 0; @@ -158,6 +183,8 @@ export default class Session { agentToken: ISession["agentToken"] notes: ISession["notes"] notesWithEvents: ISession["notesWithEvents"] + frustrations: Array + fileKey: ISession["fileKey"] constructor(plainSession?: ISession) { @@ -217,8 +244,20 @@ export default class Session { (i, k) => new Issue({ ...i, time: i.timestamp - startedAt, key: k })) || []; const rawNotes = notes; - const notesWithEvents = [...rawEvents, ...rawNotes].sort((a, b) => { - // @ts-ignore just in case + + + const frustrationEvents = events.filter(ev => { + if (ev.type === TYPES.CLICK || ev.type === TYPES.INPUT) { + // @ts-ignore + return ev.hesitation > 1000 + } + return ev.type === TYPES.CLICKRAGE + } + ) + const frustrationIssues = issuesList.filter(i => i.type === issueTypes.MOUSE_THRASHING) + + const frustrationList = [...frustrationEvents, ...frustrationIssues].sort((a, b) => { + // @ts-ignore const aTs = a.timestamp || a.time; // @ts-ignore const bTs = b.timestamp || b.time; @@ -226,6 +265,11 @@ export default class Session { return aTs - bTs; }) || []; + const mixedEventsWithIssues = mergeEventLists( + mergeEventLists(rawEvents, rawNotes), + frustrationIssues + ) + Object.assign(this, { ...session, isIOS: session.platform === 'ios', @@ -255,7 +299,8 @@ export default class Session { domURL, devtoolsURL, notes, - notesWithEvents: notesWithEvents, + notesWithEvents: mixedEventsWithIssues, + frustrations: frustrationList, }) } } \ No newline at end of file diff --git a/frontend/scripts/icons.ts b/frontend/scripts/icons.ts index dab11b224..0d7f67a31 100644 --- a/frontend/scripts/icons.ts +++ b/frontend/scripts/icons.ts @@ -32,6 +32,8 @@ const plugins = (removeFill = true) => { name: 'preset-default', params: { overrides: { + cleanupIds: false, + prefixIds: false, inlineStyles: { onlyMatchedOnce: false, }, @@ -63,12 +65,10 @@ const plugins = (removeFill = true) => { ] } } -// fs.promises.mkdir('/tmp/a/apple', { recursive: true }) -// .then(() => { - fs.writeFileSync(`${UI_DIRNAME}/SVG.tsx`, ` +fs.writeFileSync(`${UI_DIRNAME}/SVG.tsx`, ` import React from 'react'; -export type IconNames = ${icons.map(icon => "'"+ icon.slice(0, -4) + "'").join(' | ')}; +export type IconNames = ${icons.map((icon) => "'"+ icon.slice(0, -4) + "'").join(' | ')}; interface Props { name: IconNames; @@ -88,10 +88,11 @@ ${icons.map(icon => { const { data } = optimize(svg, plugins(canOptimize)); return ` case '${icon.slice(0, -4)}': return ${data.replace(/xlink\:href/g, 'xlinkHref') .replace(/xmlns\:xlink/g, 'xmlnsXlink') - .replace(/clip-path/g, 'clipPath') - .replace(/clip-rule/g, 'clipRule') + .replace(/clip\-path/g, 'clipPath') + .replace(/clip\-rule/g, 'clipRule') // hack to keep fill rule for some icons like stop recording square .replace(/clipRule="evenoddCustomFill"/g, 'clipRule="evenodd" fillRule="evenodd"') + .replace(/fill-rule/g, 'fillRule') .replace(/fill-opacity/g, 'fillOpacity') .replace(/stop-color/g, 'stopColor') From 813a128d4f311437579f101c670a6bc182e3e8c1 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 16 Mar 2023 17:21:01 +0100 Subject: [PATCH 098/253] change(ui): fix conf --- frontend/scripts/icons.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/scripts/icons.ts b/frontend/scripts/icons.ts index 0d7f67a31..3a2f9808c 100644 --- a/frontend/scripts/icons.ts +++ b/frontend/scripts/icons.ts @@ -32,8 +32,6 @@ const plugins = (removeFill = true) => { name: 'preset-default', params: { overrides: { - cleanupIds: false, - prefixIds: false, inlineStyles: { onlyMatchedOnce: false, }, From 71503680d62954e1892b7ccfe678560c17989e7c Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 16 Mar 2023 17:39:39 +0100 Subject: [PATCH 099/253] change(ui): fix types, add frustrations for v2 fetch --- frontend/app/types/session/session.ts | 45 ++++++++++++++++++--------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/frontend/app/types/session/session.ts b/frontend/app/types/session/session.ts index 6e6fe5aaa..239871600 100644 --- a/frontend/app/types/session/session.ts +++ b/frontend/app/types/session/session.ts @@ -9,7 +9,7 @@ import { toJS } from 'mobx'; const HASH_MOD = 1610612741; const HASH_P = 53; -function mergeEventLists(arr1: any[], arr2: any[]) { +function mergeEventLists, Y extends Record>(arr1: T[], arr2: Y[]): Array { let merged = []; let index1 = 0; let index2 = 0; @@ -33,6 +33,12 @@ function mergeEventLists(arr1: any[], arr2: any[]) { return merged; } +function sortEvents(a: Record, b: Record) { + const aTs = a.timestamp || a.time; + const bTs = b.timestamp || b.time; + + return aTs - bTs; +} function hashString(s: string): number { let mul = 1; @@ -259,19 +265,12 @@ export default class Session { ) const frustrationIssues = issuesList.filter(i => i.type === issueTypes.MOUSE_THRASHING) - const frustrationList = [...frustrationEvents, ...frustrationIssues].sort((a, b) => { - // @ts-ignore - const aTs = a.timestamp || a.time; - // @ts-ignore - const bTs = b.timestamp || b.time; - - return aTs - bTs; - }) || []; + const frustrationList = [...frustrationEvents, ...frustrationIssues].sort(sortEvents) || []; const mixedEventsWithIssues = mergeEventLists( mergeEventLists(rawEvents, rawNotes), frustrationIssues - ) + ).sort(sortEvents) Object.assign(this, { ...session, @@ -303,11 +302,9 @@ export default class Session { domURL, devtoolsURL, notes, - notesWithEvents: notesWithEvents, - }); notesWithEvents: mixedEventsWithIssues, frustrations: frustrationList, - }) + }); } addEvents( @@ -347,14 +344,32 @@ export default class Session { }); } + const frustrationEvents = events.filter(ev => { + if (ev.type === TYPES.CLICK || ev.type === TYPES.INPUT) { + // @ts-ignore + return ev.hesitation > 1000 + } + return ev.type === TYPES.CLICKRAGE + } + ) + const frustrationIssues = issuesList.filter(i => i.type === issueTypes.MOUSE_THRASHING) + const frustrationList = [...frustrationEvents, ...frustrationIssues].sort(sortEvents) || []; + + const mixedEventsWithIssues = mergeEventLists( + rawEvents, + frustrationIssues + ).sort(sortEvents) + this.events = events; // @ts-ignore - this.notesWithEvents = rawEvents; + this.notesWithEvents = mixedEventsWithIssues; this.errors = exceptions; this.issues = issuesList; // @ts-ignore legacy code? no idea this.resources = resources; this.stackEvents = stackEventsList; + // @ts-ignore + this.frustrations = frustrationList; return this; } @@ -365,7 +380,7 @@ export default class Session { [...this.notesWithEvents, ...sessionNotes].sort((a, b) => { // @ts-ignore just in case const aTs = a.timestamp || a.time; - // @ts-ignore + // @ts-ignore supporting old code... const bTs = b.timestamp || b.time; return aTs - bTs; From d9ab7b6853bf260c307101f78408ffd8bbaf1bfd Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Mar 2023 11:13:27 +0100 Subject: [PATCH 100/253] change(tracker): add nodes drop message --- backend/pkg/messages/filters.go | 4 ++ backend/pkg/messages/messages.go | 24 ++++++++++ ee/connectors/msgcodec/messages.py | 8 ++++ ee/connectors/msgcodec/msgcodec.py | 6 +++ mobs/messages.rb | 44 +++++++++++-------- tracker/tracker/src/main/app/nodes.ts | 4 ++ .../tracker/src/main/app/observer/observer.ts | 6 ++- 7 files changed, 76 insertions(+), 20 deletions(-) diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index c1a28eb55..5682b2463 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,7 +2,11 @@ package messages func IsReplayerType(id int) bool { +<<<<<<< HEAD return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 112 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id +======= + return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 115 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id +>>>>>>> 3e1b8c39f (change(tracker): add nodes drop message) } func IsIOSType(id int) bool { diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index bff0daea0..9890c5fff 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -79,6 +79,7 @@ const ( MsgBatchMeta = 80 MsgBatchMetadata = 81 MsgPartitionedMessage = 82 + MsgRemovedNodesCount = 115 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 @@ -2029,6 +2030,29 @@ func (msg *PartitionedMessage) TypeID() int { return 82 } +type RemovedNodesCount struct { + message + NodesCount uint64 + DOMDropped bool +} + +func (msg *RemovedNodesCount) Encode() []byte { + buf := make([]byte, 21) + buf[0] = 115 + p := 1 + p = WriteUint(msg.NodesCount, buf, p) + p = WriteBoolean(msg.DOMDropped, buf, p) + return buf[:p] +} + +func (msg *RemovedNodesCount) Decode() Message { + return msg +} + +func (msg *RemovedNodesCount) TypeID() int { + return 115 +} + type IssueEvent struct { message MessageID uint64 diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 91fb36a63..16763fe59 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -708,6 +708,14 @@ class PartitionedMessage(Message): self.part_total = part_total +class RemovedNodesCount(Message): + __id__ = 115 + + def __init__(self, nodes_count, dom_dropped): + self.nodes_count = nodes_count + self.dom_dropped = dom_dropped + + class IssueEvent(Message): __id__ = 125 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 02b5b5557..40e2ef648 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -630,6 +630,12 @@ class MessageCodec(Codec): part_total=self.read_uint(reader) ) + if message_id == 115: + return RemovedNodesCount( + nodes_count=self.read_uint(reader), + dom_dropped=self.read_boolean(reader) + ) + if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), diff --git a/mobs/messages.rb b/mobs/messages.rb index 99e917778..88ae76f20 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -448,6 +448,31 @@ end # 90-111 reserved iOS + +message 112, 'InputChange', :replayer => false do + uint 'ID' + string 'Value' + boolean 'ValueMasked' + string 'Label' + int 'HesitationTime' + int 'InputDuration' +end + +message 113, 'SelectionChange' do + uint 'SelectionStart' + uint 'SelectionEnd' + string 'Selection' +end + +message 114, 'MouseThrashing' do + uint 'Timestamp' +end + +message 115, 'RemovedNodesCount', :replayer => false do + uint 'NodesCount' + boolean 'DOMDropped' +end + ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do uint 'MessageID' @@ -468,22 +493,3 @@ message 127, 'SessionSearch', :tracker => false, :replayer => false do end # since tracker 4.1.10 - -message 112, 'InputChange', :replayer => false do - uint 'ID' - string 'Value' - boolean 'ValueMasked' - string 'Label' - int 'HesitationTime' - int 'InputDuration' -end - -message 113, 'SelectionChange' do - uint 'SelectionStart' - uint 'SelectionEnd' - string 'Selection' -end - -message 114, 'MouseThrashing' do - uint 'Timestamp' -end \ No newline at end of file diff --git a/tracker/tracker/src/main/app/nodes.ts b/tracker/tracker/src/main/app/nodes.ts index 387679380..b94823cce 100644 --- a/tracker/tracker/src/main/app/nodes.ts +++ b/tracker/tracker/src/main/app/nodes.ts @@ -26,6 +26,10 @@ export default class Nodes { listeners.push([type, listener, useCapture]) } + getNodesCount() { + return this.nodes.length + } + registerNode(node: Node): [/*id:*/ number, /*isNew:*/ boolean] { let id: number = (node as any)[this.node_id] const isNew = id === undefined diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts index 9e93dde2d..5972e9850 100644 --- a/tracker/tracker/src/main/app/observer/observer.ts +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -8,6 +8,7 @@ import { CreateElementNode, MoveNode, RemoveNode, + RemovedNodesCount, } from '../messages.gen.js' import App from '../index.js' import { @@ -216,6 +217,8 @@ export default abstract class Observer { } private unbindTree(node: Node) { + let removed = 0 + const nodesCount = this.app.nodes.getNodesCount() const id = this.app.nodes.unregisterNode(node) if (id !== undefined && this.recents.get(id) === RecentsType.Removed) { // Sending RemoveNode only for parent to maintain @@ -235,9 +238,10 @@ export default abstract class Observer { false, ) while (walker.nextNode()) { + removed += 1 this.app.nodes.unregisterNode(walker.currentNode) } - // MBTODO: count and send RemovedNodesCount (for the page crash detection in heuristics) + this.app.send(RemovedNodesCount(removed, removed / nodesCount > 0.5)) } } From 01e44128157d47a346aef0809a57a2e0c45b66c7 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Feb 2023 15:47:39 +0100 Subject: [PATCH 101/253] change(tracker): add nodes drop message --- backend/pkg/messages/filters.go | 7 +- backend/pkg/messages/messages.go | 714 +++--- backend/pkg/messages/read-message.go | 2284 +++++++++-------- ee/connectors/msgcodec/messages.py | 56 +- ee/connectors/msgcodec/msgcodec.py | 44 +- .../app/player/web/messages/tracker.gen.ts | 8 +- tracker/tracker/src/common/messages.gen.ts | 9 +- tracker/tracker/src/main/app/messages.gen.ts | 11 + .../src/webworker/MessageEncoder.gen.ts | 4 + 9 files changed, 1584 insertions(+), 1553 deletions(-) diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index 5682b2463..f8997f418 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,11 +2,7 @@ package messages func IsReplayerType(id int) bool { -<<<<<<< HEAD - return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 125 != id && 126 != id && 127 != id && 112 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id -======= - return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 35 != id && 42 != id && 52 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 115 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id ->>>>>>> 3e1b8c39f (change(tracker): add nodes drop message) + return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 42 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 112 != id && 115 != id && 125 != id && 126 != id && 127 != id && 107 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 99 != id && 101 != id && 104 != id && 110 != id && 111 != id } func IsIOSType(id int) bool { @@ -16,4 +12,3 @@ func IsIOSType(id int) bool { func IsDOMType(id int) bool { return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 90 == id || 93 == id || 96 == id || 100 == id || 102 == id || 103 == id || 105 == id } - diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 9890c5fff..9ee0e572b 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -34,8 +34,6 @@ const ( MsgMetadata = 30 MsgPageEvent = 31 MsgInputEvent = 32 - MsgClickEvent = 33 - MsgResourceEvent = 35 MsgCSSInsertRule = 37 MsgCSSDeleteRule = 38 MsgFetch = 39 @@ -50,7 +48,6 @@ const ( MsgPerformanceTrack = 49 MsgStringDict = 50 MsgSetNodeAttributeDict = 51 - MsgDOMDrop = 52 MsgResourceTiming = 53 MsgConnectionInformation = 54 MsgSetPageVisibility = 55 @@ -79,13 +76,13 @@ const ( MsgBatchMeta = 80 MsgBatchMetadata = 81 MsgPartitionedMessage = 82 + MsgInputChange = 112 + MsgSelectionChange = 113 + MsgMouseThrashing = 114 MsgRemovedNodesCount = 115 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 - MsgInputChange = 112 - MsgSelectionChange = 113 - MsgMouseThrashing = 114 MsgIOSBatchMeta = 107 MsgIOSSessionStart = 90 MsgIOSSessionEnd = 91 @@ -107,7 +104,6 @@ const ( MsgIOSIssueEvent = 111 ) - type Timestamp struct { message Timestamp uint64 @@ -131,22 +127,22 @@ func (msg *Timestamp) TypeID() int { type SessionStart struct { message - Timestamp uint64 - ProjectID uint64 - TrackerVersion string - RevID string - UserUUID string - UserAgent string - UserOS string - UserOSVersion string - UserBrowser string - UserBrowserVersion string - UserDevice string - UserDeviceType string + Timestamp uint64 + ProjectID uint64 + TrackerVersion string + RevID string + UserUUID string + UserAgent string + UserOS string + UserOSVersion string + UserBrowser string + UserBrowserVersion string + UserDevice string + UserDeviceType string UserDeviceMemorySize uint64 - UserDeviceHeapSize uint64 - UserCountry string - UserID string + UserDeviceHeapSize uint64 + UserCountry string + UserID string } func (msg *SessionStart) Encode() []byte { @@ -203,8 +199,8 @@ func (msg *SessionEndDeprecated) TypeID() int { type SetPageLocation struct { message - URL string - Referrer string + URL string + Referrer string NavigationStart uint64 } @@ -228,7 +224,7 @@ func (msg *SetPageLocation) TypeID() int { type SetViewportSize struct { message - Width uint64 + Width uint64 Height uint64 } @@ -274,7 +270,6 @@ func (msg *SetViewportScroll) TypeID() int { type CreateDocument struct { message - } func (msg *CreateDocument) Encode() []byte { @@ -295,11 +290,11 @@ func (msg *CreateDocument) TypeID() int { type CreateElementNode struct { message - ID uint64 + ID uint64 ParentID uint64 - index uint64 - Tag string - SVG bool + index uint64 + Tag string + SVG bool } func (msg *CreateElementNode) Encode() []byte { @@ -324,9 +319,9 @@ func (msg *CreateElementNode) TypeID() int { type CreateTextNode struct { message - ID uint64 + ID uint64 ParentID uint64 - Index uint64 + Index uint64 } func (msg *CreateTextNode) Encode() []byte { @@ -349,9 +344,9 @@ func (msg *CreateTextNode) TypeID() int { type MoveNode struct { message - ID uint64 + ID uint64 ParentID uint64 - Index uint64 + Index uint64 } func (msg *MoveNode) Encode() []byte { @@ -395,8 +390,8 @@ func (msg *RemoveNode) TypeID() int { type SetNodeAttribute struct { message - ID uint64 - Name string + ID uint64 + Name string Value string } @@ -420,7 +415,7 @@ func (msg *SetNodeAttribute) TypeID() int { type RemoveNodeAttribute struct { message - ID uint64 + ID uint64 Name string } @@ -443,7 +438,7 @@ func (msg *RemoveNodeAttribute) TypeID() int { type SetNodeData struct { message - ID uint64 + ID uint64 Data string } @@ -466,7 +461,7 @@ func (msg *SetNodeData) TypeID() int { type SetCSSData struct { message - ID uint64 + ID uint64 Data string } @@ -490,8 +485,8 @@ func (msg *SetCSSData) TypeID() int { type SetNodeScroll struct { message ID uint64 - X int64 - Y int64 + X int64 + Y int64 } func (msg *SetNodeScroll) Encode() []byte { @@ -514,7 +509,7 @@ func (msg *SetNodeScroll) TypeID() int { type SetInputTarget struct { message - ID uint64 + ID uint64 Label string } @@ -537,9 +532,9 @@ func (msg *SetInputTarget) TypeID() int { type SetInputValue struct { message - ID uint64 + ID uint64 Value string - Mask int64 + Mask int64 } func (msg *SetInputValue) Encode() []byte { @@ -562,7 +557,7 @@ func (msg *SetInputValue) TypeID() int { type SetInputChecked struct { message - ID uint64 + ID uint64 Checked bool } @@ -608,14 +603,14 @@ func (msg *MouseMove) TypeID() int { type NetworkRequest struct { message - Type string - Method string - URL string - Request string - Response string - Status uint64 + Type string + Method string + URL string + Request string + Response string + Status uint64 Timestamp uint64 - Duration uint64 + Duration uint64 } func (msg *NetworkRequest) Encode() []byte { @@ -666,15 +661,15 @@ func (msg *ConsoleLog) TypeID() int { type PageLoadTiming struct { message - RequestStart uint64 - ResponseStart uint64 - ResponseEnd uint64 + RequestStart uint64 + ResponseStart uint64 + ResponseEnd uint64 DomContentLoadedEventStart uint64 - DomContentLoadedEventEnd uint64 - LoadEventStart uint64 - LoadEventEnd uint64 - FirstPaint uint64 - FirstContentfulPaint uint64 + DomContentLoadedEventEnd uint64 + LoadEventStart uint64 + LoadEventEnd uint64 + FirstPaint uint64 + FirstContentfulPaint uint64 } func (msg *PageLoadTiming) Encode() []byte { @@ -703,8 +698,8 @@ func (msg *PageLoadTiming) TypeID() int { type PageRenderTiming struct { message - SpeedIndex uint64 - VisuallyComplete uint64 + SpeedIndex uint64 + VisuallyComplete uint64 TimeToInteractive uint64 } @@ -728,7 +723,7 @@ func (msg *PageRenderTiming) TypeID() int { type JSExceptionDeprecated struct { message - Name string + Name string Message string Payload string } @@ -754,10 +749,10 @@ func (msg *JSExceptionDeprecated) TypeID() int { type IntegrationEvent struct { message Timestamp uint64 - Source string - Name string - Message string - Payload string + Source string + Name string + Message string + Payload string } func (msg *IntegrationEvent) Encode() []byte { @@ -782,7 +777,7 @@ func (msg *IntegrationEvent) TypeID() int { type CustomEvent struct { message - Name string + Name string Payload string } @@ -847,7 +842,7 @@ func (msg *UserAnonymousID) TypeID() int { type Metadata struct { message - Key string + Key string Value string } @@ -870,23 +865,23 @@ func (msg *Metadata) TypeID() int { type PageEvent struct { message - MessageID uint64 - Timestamp uint64 - URL string - Referrer string - Loaded bool - RequestStart uint64 - ResponseStart uint64 - ResponseEnd uint64 + MessageID uint64 + Timestamp uint64 + URL string + Referrer string + Loaded bool + RequestStart uint64 + ResponseStart uint64 + ResponseEnd uint64 DomContentLoadedEventStart uint64 - DomContentLoadedEventEnd uint64 - LoadEventStart uint64 - LoadEventEnd uint64 - FirstPaint uint64 - FirstContentfulPaint uint64 - SpeedIndex uint64 - VisuallyComplete uint64 - TimeToInteractive uint64 + DomContentLoadedEventEnd uint64 + LoadEventStart uint64 + LoadEventEnd uint64 + FirstPaint uint64 + FirstContentfulPaint uint64 + SpeedIndex uint64 + VisuallyComplete uint64 + TimeToInteractive uint64 } func (msg *PageEvent) Encode() []byte { @@ -923,11 +918,11 @@ func (msg *PageEvent) TypeID() int { type InputEvent struct { message - MessageID uint64 - Timestamp uint64 - Value string + MessageID uint64 + Timestamp uint64 + Value string ValueMasked bool - Label string + Label string } func (msg *InputEvent) Encode() []byte { @@ -952,8 +947,8 @@ func (msg *InputEvent) TypeID() int { type CSSInsertRule struct { message - ID uint64 - Rule string + ID uint64 + Rule string Index uint64 } @@ -977,7 +972,7 @@ func (msg *CSSInsertRule) TypeID() int { type CSSDeleteRule struct { message - ID uint64 + ID uint64 Index uint64 } @@ -1000,13 +995,13 @@ func (msg *CSSDeleteRule) TypeID() int { type Fetch struct { message - Method string - URL string - Request string - Response string - Status uint64 + Method string + URL string + Request string + Response string + Status uint64 Timestamp uint64 - Duration uint64 + Duration uint64 } func (msg *Fetch) Encode() []byte { @@ -1033,10 +1028,10 @@ func (msg *Fetch) TypeID() int { type Profiler struct { message - Name string + Name string Duration uint64 - Args string - Result string + Args string + Result string } func (msg *Profiler) Encode() []byte { @@ -1060,7 +1055,7 @@ func (msg *Profiler) TypeID() int { type OTable struct { message - Key string + Key string Value string } @@ -1104,8 +1099,8 @@ func (msg *StateAction) TypeID() int { type Redux struct { message - Action string - State string + Action string + State string Duration uint64 } @@ -1130,7 +1125,7 @@ func (msg *Redux) TypeID() int { type Vuex struct { message Mutation string - State string + State string } func (msg *Vuex) Encode() []byte { @@ -1152,7 +1147,7 @@ func (msg *Vuex) TypeID() int { type MobX struct { message - Type string + Type string Payload string } @@ -1175,8 +1170,8 @@ func (msg *MobX) TypeID() int { type NgRx struct { message - Action string - State string + Action string + State string Duration uint64 } @@ -1202,8 +1197,8 @@ type GraphQL struct { message OperationKind string OperationName string - Variables string - Response string + Variables string + Response string } func (msg *GraphQL) Encode() []byte { @@ -1227,10 +1222,10 @@ func (msg *GraphQL) TypeID() int { type PerformanceTrack struct { message - Frames int64 - Ticks int64 + Frames int64 + Ticks int64 TotalJSHeapSize uint64 - UsedJSHeapSize uint64 + UsedJSHeapSize uint64 } func (msg *PerformanceTrack) Encode() []byte { @@ -1254,7 +1249,7 @@ func (msg *PerformanceTrack) TypeID() int { type StringDict struct { message - Key uint64 + Key uint64 Value string } @@ -1277,8 +1272,8 @@ func (msg *StringDict) TypeID() int { type SetNodeAttributeDict struct { message - ID uint64 - NameKey uint64 + ID uint64 + NameKey uint64 ValueKey uint64 } @@ -1302,14 +1297,14 @@ func (msg *SetNodeAttributeDict) TypeID() int { type ResourceTiming struct { message - Timestamp uint64 - Duration uint64 - TTFB uint64 - HeaderSize uint64 + Timestamp uint64 + Duration uint64 + TTFB uint64 + HeaderSize uint64 EncodedBodySize uint64 DecodedBodySize uint64 - URL string - Initiator string + URL string + Initiator string } func (msg *ResourceTiming) Encode() []byte { @@ -1338,7 +1333,7 @@ func (msg *ResourceTiming) TypeID() int { type ConnectionInformation struct { message Downlink uint64 - Type string + Type string } func (msg *ConnectionInformation) Encode() []byte { @@ -1381,20 +1376,20 @@ func (msg *SetPageVisibility) TypeID() int { type PerformanceTrackAggr struct { message - TimestampStart uint64 - TimestampEnd uint64 - MinFPS uint64 - AvgFPS uint64 - MaxFPS uint64 - MinCPU uint64 - AvgCPU uint64 - MaxCPU uint64 + TimestampStart uint64 + TimestampEnd uint64 + MinFPS uint64 + AvgFPS uint64 + MaxFPS uint64 + MinCPU uint64 + AvgCPU uint64 + MaxCPU uint64 MinTotalJSHeapSize uint64 AvgTotalJSHeapSize uint64 MaxTotalJSHeapSize uint64 - MinUsedJSHeapSize uint64 - AvgUsedJSHeapSize uint64 - MaxUsedJSHeapSize uint64 + MinUsedJSHeapSize uint64 + AvgUsedJSHeapSize uint64 + MaxUsedJSHeapSize uint64 } func (msg *PerformanceTrackAggr) Encode() []byte { @@ -1428,9 +1423,9 @@ func (msg *PerformanceTrackAggr) TypeID() int { type LoadFontFace struct { message - ParentID uint64 - Family string - Source string + ParentID uint64 + Family string + Source string Descriptors string } @@ -1476,12 +1471,12 @@ func (msg *SetNodeFocus) TypeID() int { type LongTask struct { message - Timestamp uint64 - Duration uint64 - Context uint64 + Timestamp uint64 + Duration uint64 + Context uint64 ContainerType uint64 - ContainerSrc string - ContainerId string + ContainerSrc string + ContainerId string ContainerName string } @@ -1509,9 +1504,9 @@ func (msg *LongTask) TypeID() int { type SetNodeAttributeURLBased struct { message - ID uint64 - Name string - Value string + ID uint64 + Name string + Value string BaseURL string } @@ -1536,8 +1531,8 @@ func (msg *SetNodeAttributeURLBased) TypeID() int { type SetCSSDataURLBased struct { message - ID uint64 - Data string + ID uint64 + Data string BaseURL string } @@ -1561,12 +1556,12 @@ func (msg *SetCSSDataURLBased) TypeID() int { type IssueEventDeprecated struct { message - MessageID uint64 - Timestamp uint64 - Type string + MessageID uint64 + Timestamp uint64 + Type string ContextString string - Context string - Payload string + Context string + Payload string } func (msg *IssueEventDeprecated) Encode() []byte { @@ -1592,7 +1587,7 @@ func (msg *IssueEventDeprecated) TypeID() int { type TechnicalInfo struct { message - Type string + Type string Value string } @@ -1615,7 +1610,7 @@ func (msg *TechnicalInfo) TypeID() int { type CustomIssue struct { message - Name string + Name string Payload string } @@ -1659,9 +1654,9 @@ func (msg *AssetCache) TypeID() int { type CSSInsertRuleURLBased struct { message - ID uint64 - Rule string - Index uint64 + ID uint64 + Rule string + Index uint64 BaseURL string } @@ -1686,10 +1681,10 @@ func (msg *CSSInsertRuleURLBased) TypeID() int { type MouseClick struct { message - ID uint64 + ID uint64 HesitationTime uint64 - Label string - Selector string + Label string + Selector string } func (msg *MouseClick) Encode() []byte { @@ -1714,7 +1709,7 @@ func (msg *MouseClick) TypeID() int { type CreateIFrameDocument struct { message FrameID uint64 - ID uint64 + ID uint64 } func (msg *CreateIFrameDocument) Encode() []byte { @@ -1737,7 +1732,7 @@ func (msg *CreateIFrameDocument) TypeID() int { type AdoptedSSReplaceURLBased struct { message SheetID uint64 - Text string + Text string BaseURL string } @@ -1762,7 +1757,7 @@ func (msg *AdoptedSSReplaceURLBased) TypeID() int { type AdoptedSSReplace struct { message SheetID uint64 - Text string + Text string } func (msg *AdoptedSSReplace) Encode() []byte { @@ -1785,8 +1780,8 @@ func (msg *AdoptedSSReplace) TypeID() int { type AdoptedSSInsertRuleURLBased struct { message SheetID uint64 - Rule string - Index uint64 + Rule string + Index uint64 BaseURL string } @@ -1812,8 +1807,8 @@ func (msg *AdoptedSSInsertRuleURLBased) TypeID() int { type AdoptedSSInsertRule struct { message SheetID uint64 - Rule string - Index uint64 + Rule string + Index uint64 } func (msg *AdoptedSSInsertRule) Encode() []byte { @@ -1837,7 +1832,7 @@ func (msg *AdoptedSSInsertRule) TypeID() int { type AdoptedSSDeleteRule struct { message SheetID uint64 - Index uint64 + Index uint64 } func (msg *AdoptedSSDeleteRule) Encode() []byte { @@ -1860,7 +1855,7 @@ func (msg *AdoptedSSDeleteRule) TypeID() int { type AdoptedSSAddOwner struct { message SheetID uint64 - ID uint64 + ID uint64 } func (msg *AdoptedSSAddOwner) Encode() []byte { @@ -1883,7 +1878,7 @@ func (msg *AdoptedSSAddOwner) TypeID() int { type AdoptedSSRemoveOwner struct { message SheetID uint64 - ID uint64 + ID uint64 } func (msg *AdoptedSSRemoveOwner) Encode() []byte { @@ -1905,9 +1900,9 @@ func (msg *AdoptedSSRemoveOwner) TypeID() int { type JSException struct { message - Name string - Message string - Payload string + Name string + Message string + Payload string Metadata string } @@ -1933,7 +1928,7 @@ func (msg *JSException) TypeID() int { type Zustand struct { message Mutation string - State string + State string } func (msg *Zustand) Encode() []byte { @@ -1955,9 +1950,9 @@ func (msg *Zustand) TypeID() int { type BatchMeta struct { message - PageNo uint64 + PageNo uint64 FirstIndex uint64 - Timestamp int64 + Timestamp int64 } func (msg *BatchMeta) Encode() []byte { @@ -1980,11 +1975,11 @@ func (msg *BatchMeta) TypeID() int { type BatchMetadata struct { message - Version uint64 - PageNo uint64 + Version uint64 + PageNo uint64 FirstIndex uint64 - Timestamp int64 - Location string + Timestamp int64 + Location string } func (msg *BatchMetadata) Encode() []byte { @@ -2009,7 +2004,7 @@ func (msg *BatchMetadata) TypeID() int { type PartitionedMessage struct { message - PartNo uint64 + PartNo uint64 PartTotal uint64 } @@ -2030,108 +2025,6 @@ func (msg *PartitionedMessage) TypeID() int { return 82 } -type RemovedNodesCount struct { - message - NodesCount uint64 - DOMDropped bool -} - -func (msg *RemovedNodesCount) Encode() []byte { - buf := make([]byte, 21) - buf[0] = 115 - p := 1 - p = WriteUint(msg.NodesCount, buf, p) - p = WriteBoolean(msg.DOMDropped, buf, p) - return buf[:p] -} - -func (msg *RemovedNodesCount) Decode() Message { - return msg -} - -func (msg *RemovedNodesCount) TypeID() int { - return 115 -} - -type IssueEvent struct { - message - MessageID uint64 - Timestamp uint64 - Type string - ContextString string - Context string - Payload string - URL string -} - -func (msg *IssueEvent) Encode() []byte { - buf := make([]byte, 71+len(msg.Type)+len(msg.ContextString)+len(msg.Context)+len(msg.Payload)+len(msg.URL)) - buf[0] = 125 - p := 1 - p = WriteUint(msg.MessageID, buf, p) - p = WriteUint(msg.Timestamp, buf, p) - p = WriteString(msg.Type, buf, p) - p = WriteString(msg.ContextString, buf, p) - p = WriteString(msg.Context, buf, p) - p = WriteString(msg.Payload, buf, p) - p = WriteString(msg.URL, buf, p) - return buf[:p] -} - -func (msg *IssueEvent) Decode() Message { - return msg -} - -func (msg *IssueEvent) TypeID() int { - return 125 -} - -type SessionEnd struct { - message - Timestamp uint64 - EncryptionKey string -} - -func (msg *SessionEnd) Encode() []byte { - buf := make([]byte, 21+len(msg.EncryptionKey)) - buf[0] = 126 - p := 1 - p = WriteUint(msg.Timestamp, buf, p) - p = WriteString(msg.EncryptionKey, buf, p) - return buf[:p] -} - -func (msg *SessionEnd) Decode() Message { - return msg -} - -func (msg *SessionEnd) TypeID() int { - return 126 -} - -type SessionSearch struct { - message - Timestamp uint64 - Partition uint64 -} - -func (msg *SessionSearch) Encode() []byte { - buf := make([]byte, 21) - buf[0] = 127 - p := 1 - p = WriteUint(msg.Timestamp, buf, p) - p = WriteUint(msg.Partition, buf, p) - return buf[:p] -} - -func (msg *SessionSearch) Decode() Message { - return msg -} - -func (msg *SessionSearch) TypeID() int { - return 127 -} - type InputChange struct { message ID uint64 @@ -2209,10 +2102,112 @@ func (msg *MouseThrashing) TypeID() int { return 114 } -type IOSBatchMeta struct { +type RemovedNodesCount struct { + message + NodesCount uint64 + DOMDropped bool +} + +func (msg *RemovedNodesCount) Encode() []byte { + buf := make([]byte, 21) + buf[0] = 115 + p := 1 + p = WriteUint(msg.NodesCount, buf, p) + p = WriteBoolean(msg.DOMDropped, buf, p) + return buf[:p] +} + +func (msg *RemovedNodesCount) Decode() Message { + return msg +} + +func (msg *RemovedNodesCount) TypeID() int { + return 115 +} + +type IssueEvent struct { + message + MessageID uint64 + Timestamp uint64 + Type string + ContextString string + Context string + Payload string + URL string +} + +func (msg *IssueEvent) Encode() []byte { + buf := make([]byte, 71+len(msg.Type)+len(msg.ContextString)+len(msg.Context)+len(msg.Payload)+len(msg.URL)) + buf[0] = 125 + p := 1 + p = WriteUint(msg.MessageID, buf, p) + p = WriteUint(msg.Timestamp, buf, p) + p = WriteString(msg.Type, buf, p) + p = WriteString(msg.ContextString, buf, p) + p = WriteString(msg.Context, buf, p) + p = WriteString(msg.Payload, buf, p) + p = WriteString(msg.URL, buf, p) + return buf[:p] +} + +func (msg *IssueEvent) Decode() Message { + return msg +} + +func (msg *IssueEvent) TypeID() int { + return 125 +} + +type SessionEnd struct { + message + Timestamp uint64 + EncryptionKey string +} + +func (msg *SessionEnd) Encode() []byte { + buf := make([]byte, 21+len(msg.EncryptionKey)) + buf[0] = 126 + p := 1 + p = WriteUint(msg.Timestamp, buf, p) + p = WriteString(msg.EncryptionKey, buf, p) + return buf[:p] +} + +func (msg *SessionEnd) Decode() Message { + return msg +} + +func (msg *SessionEnd) TypeID() int { + return 126 +} + +type SessionSearch struct { message Timestamp uint64 - Length uint64 + Partition uint64 +} + +func (msg *SessionSearch) Encode() []byte { + buf := make([]byte, 21) + buf[0] = 127 + p := 1 + p = WriteUint(msg.Timestamp, buf, p) + p = WriteUint(msg.Partition, buf, p) + return buf[:p] +} + +func (msg *SessionSearch) Decode() Message { + return msg +} + +func (msg *SessionSearch) TypeID() int { + return 127 +} + +type IOSBatchMeta struct { + message + Timestamp uint64 + Length uint64 FirstIndex uint64 } @@ -2236,16 +2231,16 @@ func (msg *IOSBatchMeta) TypeID() int { type IOSSessionStart struct { message - Timestamp uint64 - ProjectID uint64 + Timestamp uint64 + ProjectID uint64 TrackerVersion string - RevID string - UserUUID string - UserOS string - UserOSVersion string - UserDevice string + RevID string + UserUUID string + UserOS string + UserOSVersion string + UserDevice string UserDeviceType string - UserCountry string + UserCountry string } func (msg *IOSSessionStart) Encode() []byte { @@ -2297,9 +2292,9 @@ func (msg *IOSSessionEnd) TypeID() int { type IOSMetadata struct { message Timestamp uint64 - Length uint64 - Key string - Value string + Length uint64 + Key string + Value string } func (msg *IOSMetadata) Encode() []byte { @@ -2324,9 +2319,9 @@ func (msg *IOSMetadata) TypeID() int { type IOSCustomEvent struct { message Timestamp uint64 - Length uint64 - Name string - Payload string + Length uint64 + Name string + Payload string } func (msg *IOSCustomEvent) Encode() []byte { @@ -2351,8 +2346,8 @@ func (msg *IOSCustomEvent) TypeID() int { type IOSUserID struct { message Timestamp uint64 - Length uint64 - Value string + Length uint64 + Value string } func (msg *IOSUserID) Encode() []byte { @@ -2376,8 +2371,8 @@ func (msg *IOSUserID) TypeID() int { type IOSUserAnonymousID struct { message Timestamp uint64 - Length uint64 - Value string + Length uint64 + Value string } func (msg *IOSUserAnonymousID) Encode() []byte { @@ -2401,11 +2396,11 @@ func (msg *IOSUserAnonymousID) TypeID() int { type IOSScreenChanges struct { message Timestamp uint64 - Length uint64 - X uint64 - Y uint64 - Width uint64 - Height uint64 + Length uint64 + X uint64 + Y uint64 + Width uint64 + Height uint64 } func (msg *IOSScreenChanges) Encode() []byte { @@ -2431,10 +2426,10 @@ func (msg *IOSScreenChanges) TypeID() int { type IOSCrash struct { message - Timestamp uint64 - Length uint64 - Name string - Reason string + Timestamp uint64 + Length uint64 + Name string + Reason string Stacktrace string } @@ -2461,9 +2456,9 @@ func (msg *IOSCrash) TypeID() int { type IOSScreenEnter struct { message Timestamp uint64 - Length uint64 - Title string - ViewName string + Length uint64 + Title string + ViewName string } func (msg *IOSScreenEnter) Encode() []byte { @@ -2488,9 +2483,9 @@ func (msg *IOSScreenEnter) TypeID() int { type IOSScreenLeave struct { message Timestamp uint64 - Length uint64 - Title string - ViewName string + Length uint64 + Title string + ViewName string } func (msg *IOSScreenLeave) Encode() []byte { @@ -2515,10 +2510,10 @@ func (msg *IOSScreenLeave) TypeID() int { type IOSClickEvent struct { message Timestamp uint64 - Length uint64 - Label string - X uint64 - Y uint64 + Length uint64 + Label string + X uint64 + Y uint64 } func (msg *IOSClickEvent) Encode() []byte { @@ -2543,11 +2538,11 @@ func (msg *IOSClickEvent) TypeID() int { type IOSInputEvent struct { message - Timestamp uint64 - Length uint64 - Value string + Timestamp uint64 + Length uint64 + Value string ValueMasked bool - Label string + Label string } func (msg *IOSInputEvent) Encode() []byte { @@ -2573,9 +2568,9 @@ func (msg *IOSInputEvent) TypeID() int { type IOSPerformanceEvent struct { message Timestamp uint64 - Length uint64 - Name string - Value uint64 + Length uint64 + Name string + Value uint64 } func (msg *IOSPerformanceEvent) Encode() []byte { @@ -2600,9 +2595,9 @@ func (msg *IOSPerformanceEvent) TypeID() int { type IOSLog struct { message Timestamp uint64 - Length uint64 - Severity string - Content string + Length uint64 + Severity string + Content string } func (msg *IOSLog) Encode() []byte { @@ -2627,8 +2622,8 @@ func (msg *IOSLog) TypeID() int { type IOSInternalError struct { message Timestamp uint64 - Length uint64 - Content string + Length uint64 + Content string } func (msg *IOSInternalError) Encode() []byte { @@ -2652,14 +2647,14 @@ func (msg *IOSInternalError) TypeID() int { type IOSNetworkCall struct { message Timestamp uint64 - Length uint64 - Duration uint64 - Headers string - Body string - URL string - Success bool - Method string - Status uint64 + Length uint64 + Duration uint64 + Headers string + Body string + URL string + Success bool + Method string + Status uint64 } func (msg *IOSNetworkCall) Encode() []byte { @@ -2689,19 +2684,19 @@ func (msg *IOSNetworkCall) TypeID() int { type IOSPerformanceAggregated struct { message TimestampStart uint64 - TimestampEnd uint64 - MinFPS uint64 - AvgFPS uint64 - MaxFPS uint64 - MinCPU uint64 - AvgCPU uint64 - MaxCPU uint64 - MinMemory uint64 - AvgMemory uint64 - MaxMemory uint64 - MinBattery uint64 - AvgBattery uint64 - MaxBattery uint64 + TimestampEnd uint64 + MinFPS uint64 + AvgFPS uint64 + MaxFPS uint64 + MinCPU uint64 + AvgCPU uint64 + MaxCPU uint64 + MinMemory uint64 + AvgMemory uint64 + MaxMemory uint64 + MinBattery uint64 + AvgBattery uint64 + MaxBattery uint64 } func (msg *IOSPerformanceAggregated) Encode() []byte { @@ -2735,11 +2730,11 @@ func (msg *IOSPerformanceAggregated) TypeID() int { type IOSIssueEvent struct { message - Timestamp uint64 - Type string + Timestamp uint64 + Type string ContextString string - Context string - Payload string + Context string + Payload string } func (msg *IOSIssueEvent) Encode() []byte { @@ -2761,4 +2756,3 @@ func (msg *IOSIssueEvent) Decode() Message { func (msg *IOSIssueEvent) TypeID() int { return 111 } - diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index d227b3295..bd2176a30 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -6,1270 +6,1219 @@ import ( ) func DecodeTimestamp(reader BytesReader) (Message, error) { - var err error = nil - msg := &Timestamp{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &Timestamp{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err } func DecodeSessionStart(reader BytesReader) (Message, error) { - var err error = nil - msg := &SessionStart{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SessionStart{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ProjectID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TrackerVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.RevID, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserUUID, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserAgent, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserOS, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserOSVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserBrowser, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserBrowserVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDevice, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDeviceType, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDeviceMemorySize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDeviceHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.UserCountry, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserID, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSessionEndDeprecated(reader BytesReader) (Message, error) { - var err error = nil - msg := &SessionEndDeprecated{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &SessionEndDeprecated{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err } func DecodeSetPageLocation(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetPageLocation{} - if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &SetPageLocation{} + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Referrer, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.NavigationStart, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetViewportSize(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetViewportSize{} - if msg.Width, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetViewportSize{} + if msg.Width, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Height, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetViewportScroll(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetViewportScroll{} - if msg.X, err = reader.ReadInt(); err != nil { - return nil, err - } + var err error = nil + msg := &SetViewportScroll{} + if msg.X, err = reader.ReadInt(); err != nil { + return nil, err + } if msg.Y, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCreateDocument(reader BytesReader) (Message, error) { - var err error = nil - msg := &CreateDocument{} - - return msg, err + var err error = nil + msg := &CreateDocument{} + + return msg, err } func DecodeCreateElementNode(reader BytesReader) (Message, error) { - var err error = nil - msg := &CreateElementNode{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CreateElementNode{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ParentID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.index, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Tag, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.SVG, err = reader.ReadBoolean(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCreateTextNode(reader BytesReader) (Message, error) { - var err error = nil - msg := &CreateTextNode{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CreateTextNode{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ParentID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeMoveNode(reader BytesReader) (Message, error) { - var err error = nil - msg := &MoveNode{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &MoveNode{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ParentID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeRemoveNode(reader BytesReader) (Message, error) { - var err error = nil - msg := &RemoveNode{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &RemoveNode{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err } func DecodeSetNodeAttribute(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeAttribute{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeAttribute{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeRemoveNodeAttribute(reader BytesReader) (Message, error) { - var err error = nil - msg := &RemoveNodeAttribute{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &RemoveNodeAttribute{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeData(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeData{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeData{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Data, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetCSSData(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetCSSData{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetCSSData{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Data, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeScroll(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeScroll{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeScroll{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.X, err = reader.ReadInt(); err != nil { - return nil, err - } + return nil, err + } if msg.Y, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetInputTarget(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetInputTarget{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetInputTarget{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetInputValue(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetInputValue{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetInputValue{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Mask, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetInputChecked(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetInputChecked{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetInputChecked{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Checked, err = reader.ReadBoolean(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeMouseMove(reader BytesReader) (Message, error) { - var err error = nil - msg := &MouseMove{} - if msg.X, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &MouseMove{} + if msg.X, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Y, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeNetworkRequest(reader BytesReader) (Message, error) { - var err error = nil - msg := &NetworkRequest{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &NetworkRequest{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Request, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Response, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeConsoleLog(reader BytesReader) (Message, error) { - var err error = nil - msg := &ConsoleLog{} - if msg.Level, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &ConsoleLog{} + if msg.Level, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePageLoadTiming(reader BytesReader) (Message, error) { - var err error = nil - msg := &PageLoadTiming{} - if msg.RequestStart, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PageLoadTiming{} + if msg.RequestStart, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ResponseStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ResponseEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DomContentLoadedEventStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DomContentLoadedEventEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.LoadEventStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.LoadEventEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstPaint, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstContentfulPaint, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePageRenderTiming(reader BytesReader) (Message, error) { - var err error = nil - msg := &PageRenderTiming{} - if msg.SpeedIndex, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PageRenderTiming{} + if msg.SpeedIndex, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.VisuallyComplete, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TimeToInteractive, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeJSExceptionDeprecated(reader BytesReader) (Message, error) { - var err error = nil - msg := &JSExceptionDeprecated{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &JSExceptionDeprecated{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Message, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIntegrationEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IntegrationEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IntegrationEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Source, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Message, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCustomEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &CustomEvent{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &CustomEvent{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeUserID(reader BytesReader) (Message, error) { - var err error = nil - msg := &UserID{} - if msg.ID, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &UserID{} + if msg.ID, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err } func DecodeUserAnonymousID(reader BytesReader) (Message, error) { - var err error = nil - msg := &UserAnonymousID{} - if msg.ID, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &UserAnonymousID{} + if msg.ID, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err } func DecodeMetadata(reader BytesReader) (Message, error) { - var err error = nil - msg := &Metadata{} - if msg.Key, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Metadata{} + if msg.Key, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePageEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &PageEvent{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PageEvent{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Referrer, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Loaded, err = reader.ReadBoolean(); err != nil { - return nil, err - } + return nil, err + } if msg.RequestStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ResponseStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ResponseEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DomContentLoadedEventStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DomContentLoadedEventEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.LoadEventStart, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.LoadEventEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstPaint, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstContentfulPaint, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.SpeedIndex, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.VisuallyComplete, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TimeToInteractive, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeInputEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &InputEvent{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &InputEvent{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ValueMasked, err = reader.ReadBoolean(); err != nil { - return nil, err - } + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCSSInsertRule(reader BytesReader) (Message, error) { - var err error = nil - msg := &CSSInsertRule{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CSSInsertRule{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Rule, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCSSDeleteRule(reader BytesReader) (Message, error) { - var err error = nil - msg := &CSSDeleteRule{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CSSDeleteRule{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeFetch(reader BytesReader) (Message, error) { - var err error = nil - msg := &Fetch{} - if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Fetch{} + if msg.Method, err = reader.ReadString(); err != nil { + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Request, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Response, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeProfiler(reader BytesReader) (Message, error) { - var err error = nil - msg := &Profiler{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Profiler{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Args, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Result, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeOTable(reader BytesReader) (Message, error) { - var err error = nil - msg := &OTable{} - if msg.Key, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &OTable{} + if msg.Key, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeStateAction(reader BytesReader) (Message, error) { - var err error = nil - msg := &StateAction{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &StateAction{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err } func DecodeRedux(reader BytesReader) (Message, error) { - var err error = nil - msg := &Redux{} - if msg.Action, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Redux{} + if msg.Action, err = reader.ReadString(); err != nil { + return nil, err + } if msg.State, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeVuex(reader BytesReader) (Message, error) { - var err error = nil - msg := &Vuex{} - if msg.Mutation, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Vuex{} + if msg.Mutation, err = reader.ReadString(); err != nil { + return nil, err + } if msg.State, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeMobX(reader BytesReader) (Message, error) { - var err error = nil - msg := &MobX{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &MobX{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeNgRx(reader BytesReader) (Message, error) { - var err error = nil - msg := &NgRx{} - if msg.Action, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &NgRx{} + if msg.Action, err = reader.ReadString(); err != nil { + return nil, err + } if msg.State, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeGraphQL(reader BytesReader) (Message, error) { - var err error = nil - msg := &GraphQL{} - if msg.OperationKind, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &GraphQL{} + if msg.OperationKind, err = reader.ReadString(); err != nil { + return nil, err + } if msg.OperationName, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Variables, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Response, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePerformanceTrack(reader BytesReader) (Message, error) { - var err error = nil - msg := &PerformanceTrack{} - if msg.Frames, err = reader.ReadInt(); err != nil { - return nil, err - } + var err error = nil + msg := &PerformanceTrack{} + if msg.Frames, err = reader.ReadInt(); err != nil { + return nil, err + } if msg.Ticks, err = reader.ReadInt(); err != nil { - return nil, err - } + return nil, err + } if msg.TotalJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.UsedJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeStringDict(reader BytesReader) (Message, error) { - var err error = nil - msg := &StringDict{} - if msg.Key, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &StringDict{} + if msg.Key, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeAttributeDict(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeAttributeDict{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeAttributeDict{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.NameKey, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ValueKey, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeResourceTiming(reader BytesReader) (Message, error) { - var err error = nil - msg := &ResourceTiming{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &ResourceTiming{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TTFB, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.HeaderSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.EncodedBodySize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.DecodedBodySize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Initiator, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeConnectionInformation(reader BytesReader) (Message, error) { - var err error = nil - msg := &ConnectionInformation{} - if msg.Downlink, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &ConnectionInformation{} + if msg.Downlink, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetPageVisibility(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetPageVisibility{} - if msg.hidden, err = reader.ReadBoolean(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &SetPageVisibility{} + if msg.hidden, err = reader.ReadBoolean(); err != nil { + return nil, err + } + return msg, err } func DecodePerformanceTrackAggr(reader BytesReader) (Message, error) { - var err error = nil - msg := &PerformanceTrackAggr{} - if msg.TimestampStart, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PerformanceTrackAggr{} + if msg.TimestampStart, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.TimestampEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinTotalJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgTotalJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxTotalJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinUsedJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgUsedJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxUsedJSHeapSize, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeLoadFontFace(reader BytesReader) (Message, error) { - var err error = nil - msg := &LoadFontFace{} - if msg.ParentID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &LoadFontFace{} + if msg.ParentID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Family, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Source, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Descriptors, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeFocus(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeFocus{} - if msg.ID, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &SetNodeFocus{} + if msg.ID, err = reader.ReadInt(); err != nil { + return nil, err + } + return msg, err } func DecodeLongTask(reader BytesReader) (Message, error) { - var err error = nil - msg := &LongTask{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &LongTask{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Context, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ContainerType, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.ContainerSrc, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContainerId, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContainerName, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetNodeAttributeURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetNodeAttributeURLBased{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetNodeAttributeURLBased{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeSetCSSDataURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &SetCSSDataURLBased{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &SetCSSDataURLBased{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Data, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIssueEventDeprecated(reader BytesReader) (Message, error) { - var err error = nil - msg := &IssueEventDeprecated{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IssueEventDeprecated{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContextString, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Context, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeTechnicalInfo(reader BytesReader) (Message, error) { - var err error = nil - msg := &TechnicalInfo{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &TechnicalInfo{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCustomIssue(reader BytesReader) (Message, error) { - var err error = nil - msg := &CustomIssue{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &CustomIssue{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAssetCache(reader BytesReader) (Message, error) { - var err error = nil - msg := &AssetCache{} - if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &AssetCache{} + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err } func DecodeCSSInsertRuleURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &CSSInsertRuleURLBased{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CSSInsertRuleURLBased{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Rule, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeMouseClick(reader BytesReader) (Message, error) { - var err error = nil - msg := &MouseClick{} - if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &MouseClick{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.HesitationTime, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Selector, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeCreateIFrameDocument(reader BytesReader) (Message, error) { - var err error = nil - msg := &CreateIFrameDocument{} - if msg.FrameID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &CreateIFrameDocument{} + if msg.FrameID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSReplaceURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSReplaceURLBased{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSReplaceURLBased{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Text, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSReplace(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSReplace{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSReplace{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Text, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSInsertRuleURLBased(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSInsertRuleURLBased{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSInsertRuleURLBased{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Rule, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.BaseURL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSInsertRule(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSInsertRule{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSInsertRule{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Rule, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSDeleteRule(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSDeleteRule{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSDeleteRule{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Index, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSAddOwner(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSAddOwner{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSAddOwner{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeAdoptedSSRemoveOwner(reader BytesReader) (Message, error) { - var err error = nil - msg := &AdoptedSSRemoveOwner{} - if msg.SheetID, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &AdoptedSSRemoveOwner{} + if msg.SheetID, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ID, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeJSException(reader BytesReader) (Message, error) { - var err error = nil - msg := &JSException{} - if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &JSException{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } if msg.Message, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Metadata, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeZustand(reader BytesReader) (Message, error) { - var err error = nil - msg := &Zustand{} - if msg.Mutation, err = reader.ReadString(); err != nil { - return nil, err - } + var err error = nil + msg := &Zustand{} + if msg.Mutation, err = reader.ReadString(); err != nil { + return nil, err + } if msg.State, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeBatchMeta(reader BytesReader) (Message, error) { - var err error = nil - msg := &BatchMeta{} - if msg.PageNo, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &BatchMeta{} + if msg.PageNo, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.FirstIndex, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Timestamp, err = reader.ReadInt(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeBatchMetadata(reader BytesReader) (Message, error) { - var err error = nil - msg := &BatchMetadata{} - if msg.Version, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &BatchMetadata{} + if msg.Version, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.PageNo, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstIndex, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Timestamp, err = reader.ReadInt(); err != nil { - return nil, err - } + return nil, err + } if msg.Location, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodePartitionedMessage(reader BytesReader) (Message, error) { - var err error = nil - msg := &PartitionedMessage{} - if msg.PartNo, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &PartitionedMessage{} + if msg.PartNo, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.PartTotal, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err -} - -func DecodeIssueEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IssueEvent{} - if msg.MessageID, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.ContextString, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Context, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err -} - -func DecodeSessionEnd(reader BytesReader) (Message, error) { - var err error = nil - msg := &SessionEnd{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.EncryptionKey, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err -} - -func DecodeSessionSearch(reader BytesReader) (Message, error) { - var err error = nil - msg := &SessionSearch{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Partition, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeInputChange(reader BytesReader) (Message, error) { @@ -1320,406 +1269,469 @@ func DecodeMouseThrashing(reader BytesReader) (Message, error) { return msg, err } +func DecodeRemovedNodesCount(reader BytesReader) (Message, error) { + var err error = nil + msg := &RemovedNodesCount{} + if msg.NodesCount, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.DOMDropped, err = reader.ReadBoolean(); err != nil { + return nil, err + } + return msg, err +} + +func DecodeIssueEvent(reader BytesReader) (Message, error) { + var err error = nil + msg := &IssueEvent{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.ContextString, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Context, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Payload, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + +func DecodeSessionEnd(reader BytesReader) (Message, error) { + var err error = nil + msg := &SessionEnd{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.EncryptionKey, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + +func DecodeSessionSearch(reader BytesReader) (Message, error) { + var err error = nil + msg := &SessionSearch{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Partition, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIOSBatchMeta(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSBatchMeta{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSBatchMeta{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.FirstIndex, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSSessionStart(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSSessionStart{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSSessionStart{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.ProjectID, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.TrackerVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.RevID, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserUUID, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserOS, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserOSVersion, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDevice, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserDeviceType, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.UserCountry, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSSessionEnd(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSSessionEnd{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + var err error = nil + msg := &IOSSessionEnd{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err } func DecodeIOSMetadata(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSMetadata{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSMetadata{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Key, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSCustomEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSCustomEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSCustomEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSUserID(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSUserID{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSUserID{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSUserAnonymousID(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSUserAnonymousID{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSUserAnonymousID{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSScreenChanges(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSScreenChanges{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSScreenChanges{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.X, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Y, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Width, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Height, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSCrash(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSCrash{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSCrash{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Reason, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Stacktrace, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSScreenEnter(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSScreenEnter{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSScreenEnter{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Title, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ViewName, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSScreenLeave(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSScreenLeave{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSScreenLeave{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Title, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ViewName, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSClickEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSClickEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSClickEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.X, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Y, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSInputEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSInputEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSInputEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ValueMasked, err = reader.ReadBoolean(); err != nil { - return nil, err - } + return nil, err + } if msg.Label, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSPerformanceEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSPerformanceEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSPerformanceEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Name, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Value, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSLog(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSLog{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSLog{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Severity, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Content, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSInternalError(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSInternalError{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSInternalError{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Content, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSNetworkCall(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSNetworkCall{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSNetworkCall{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Length, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.Headers, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Body, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Success, err = reader.ReadBoolean(); err != nil { - return nil, err - } + return nil, err + } if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSPerformanceAggregated(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSPerformanceAggregated{} - if msg.TimestampStart, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSPerformanceAggregated{} + if msg.TimestampStart, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.TimestampEnd, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxFPS, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxCPU, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinMemory, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgMemory, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxMemory, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MinBattery, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.AvgBattery, err = reader.ReadUint(); err != nil { - return nil, err - } + return nil, err + } if msg.MaxBattery, err = reader.ReadUint(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func DecodeIOSIssueEvent(reader BytesReader) (Message, error) { - var err error = nil - msg := &IOSIssueEvent{} - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } + var err error = nil + msg := &IOSIssueEvent{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.ContextString, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Context, err = reader.ReadString(); err != nil { - return nil, err - } + return nil, err + } if msg.Payload, err = reader.ReadString(); err != nil { - return nil, err - } - return msg, err + return nil, err + } + return msg, err } func ReadMessage(t uint64, reader BytesReader) (Message, error) { @@ -1872,18 +1884,20 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeBatchMetadata(reader) case 82: return DecodePartitionedMessage(reader) - case 125: - return DecodeIssueEvent(reader) - case 126: - return DecodeSessionEnd(reader) - case 127: - return DecodeSessionSearch(reader) case 112: return DecodeInputChange(reader) case 113: return DecodeSelectionChange(reader) case 114: return DecodeMouseThrashing(reader) + case 115: + return DecodeRemovedNodesCount(reader) + case 125: + return DecodeIssueEvent(reader) + case 126: + return DecodeSessionEnd(reader) + case 127: + return DecodeSessionSearch(reader) case 107: return DecodeIOSBatchMeta(reader) case 90: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 16763fe59..2060eff81 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -708,6 +708,34 @@ class PartitionedMessage(Message): self.part_total = part_total +class InputChange(Message): + __id__ = 112 + + def __init__(self, id, value, value_masked, label, hesitation_time, input_duration): + self.id = id + self.value = value + self.value_masked = value_masked + self.label = label + self.hesitation_time = hesitation_time + self.input_duration = input_duration + + +class SelectionChange(Message): + __id__ = 113 + + def __init__(self, selection_start, selection_end, selection): + self.selection_start = selection_start + self.selection_end = selection_end + self.selection = selection + + +class MouseThrashing(Message): + __id__ = 114 + + def __init__(self, timestamp): + self.timestamp = timestamp + + class RemovedNodesCount(Message): __id__ = 115 @@ -745,34 +773,6 @@ class SessionSearch(Message): self.partition = partition -class InputChange(Message): - __id__ = 112 - - def __init__(self, id, value, value_masked, label, hesitation_time, input_duration): - self.id = id - self.value = value - self.value_masked = value_masked - self.label = label - self.hesitation_time = hesitation_time - self.input_duration = input_duration - - -class SelectionChange(Message): - __id__ = 113 - - def __init__(self, selection_start, selection_end, selection): - self.selection_start = selection_start - self.selection_end = selection_end - self.selection = selection - - -class MouseThrashing(Message): - __id__ = 114 - - def __init__(self, timestamp): - self.timestamp = timestamp - - class IOSBatchMeta(Message): __id__ = 107 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 40e2ef648..984fd153a 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -630,6 +630,28 @@ class MessageCodec(Codec): part_total=self.read_uint(reader) ) + if message_id == 112: + return InputChange( + id=self.read_uint(reader), + value=self.read_string(reader), + value_masked=self.read_boolean(reader), + label=self.read_string(reader), + hesitation_time=self.read_int(reader), + input_duration=self.read_int(reader) + ) + + if message_id == 113: + return SelectionChange( + selection_start=self.read_uint(reader), + selection_end=self.read_uint(reader), + selection=self.read_string(reader) + ) + + if message_id == 114: + return MouseThrashing( + timestamp=self.read_uint(reader) + ) + if message_id == 115: return RemovedNodesCount( nodes_count=self.read_uint(reader), @@ -659,28 +681,6 @@ class MessageCodec(Codec): partition=self.read_uint(reader) ) - if message_id == 112: - return InputChange( - id=self.read_uint(reader), - value=self.read_string(reader), - value_masked=self.read_boolean(reader), - label=self.read_string(reader), - hesitation_time=self.read_int(reader), - input_duration=self.read_int(reader) - ) - - if message_id == 113: - return SelectionChange( - selection_start=self.read_uint(reader), - selection_end=self.read_uint(reader), - selection=self.read_string(reader) - ) - - if message_id == 114: - return MouseThrashing( - timestamp=self.read_uint(reader) - ) - if message_id == 107: return IOSBatchMeta( timestamp=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index fdd9c6f42..55afdd58a 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -451,8 +451,14 @@ type TrMouseThrashing = [ timestamp: number, ] +type TrRemovedNodesCount = [ + type: 115, + nodesCount: number, + domdropped: boolean, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrRemovedNodesCount export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index ce48b201d..a4c0e4f2a 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -66,6 +66,7 @@ export declare const enum Type { InputChange = 112, SelectionChange = 113, MouseThrashing = 114, + RemovedNodesCount = 115, } @@ -515,6 +516,12 @@ export type MouseThrashing = [ /*timestamp:*/ number, ] +export type RemovedNodesCount = [ + /*type:*/ Type.RemovedNodesCount, + /*nodesCount:*/ number, + /*domdropped:*/ boolean, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | RemovedNodesCount export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index dad81cb6a..052a1caea 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -833,3 +833,14 @@ export function MouseThrashing( ] } +export function RemovedNodesCount( + nodesCount: number, + domdropped: boolean, +): Messages.RemovedNodesCount { + return [ + Messages.Type.RemovedNodesCount, + nodesCount, + domdropped, + ] +} + diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 06bd776f0..370c18d50 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -266,6 +266,10 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) break + case Messages.Type.RemovedNodesCount: + return this.uint(msg[1]) && this.boolean(msg[2]) + break + } } From 532d9b82f82a4261a86f6d8138ba385e4a38b33f Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 14 Feb 2023 11:20:24 +0100 Subject: [PATCH 102/253] change(tracker): remove dom drop detection on tracker side --- backend/pkg/messages/messages.go | 4 +--- backend/pkg/messages/read-message.go | 3 --- ee/connectors/msgcodec/messages.py | 3 +-- ee/connectors/msgcodec/msgcodec.py | 3 +-- frontend/app/player/web/messages/tracker.gen.ts | 1 - mobs/messages.rb | 1 - tracker/tracker/src/common/messages.gen.ts | 1 - tracker/tracker/src/main/app/messages.gen.ts | 2 -- tracker/tracker/src/main/app/nodes.ts | 4 ---- tracker/tracker/src/main/app/observer/observer.ts | 3 +-- tracker/tracker/src/webworker/MessageEncoder.gen.ts | 2 +- 11 files changed, 5 insertions(+), 22 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 9ee0e572b..232f116c0 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -2105,15 +2105,13 @@ func (msg *MouseThrashing) TypeID() int { type RemovedNodesCount struct { message NodesCount uint64 - DOMDropped bool } func (msg *RemovedNodesCount) Encode() []byte { - buf := make([]byte, 21) + buf := make([]byte, 11) buf[0] = 115 p := 1 p = WriteUint(msg.NodesCount, buf, p) - p = WriteBoolean(msg.DOMDropped, buf, p) return buf[:p] } diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index bd2176a30..715f81d98 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1275,9 +1275,6 @@ func DecodeRemovedNodesCount(reader BytesReader) (Message, error) { if msg.NodesCount, err = reader.ReadUint(); err != nil { return nil, err } - if msg.DOMDropped, err = reader.ReadBoolean(); err != nil { - return nil, err - } return msg, err } diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 2060eff81..75cc5309f 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -739,9 +739,8 @@ class MouseThrashing(Message): class RemovedNodesCount(Message): __id__ = 115 - def __init__(self, nodes_count, dom_dropped): + def __init__(self, nodes_count): self.nodes_count = nodes_count - self.dom_dropped = dom_dropped class IssueEvent(Message): diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 984fd153a..75ccca196 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -654,8 +654,7 @@ class MessageCodec(Codec): if message_id == 115: return RemovedNodesCount( - nodes_count=self.read_uint(reader), - dom_dropped=self.read_boolean(reader) + nodes_count=self.read_uint(reader) ) if message_id == 125: diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 55afdd58a..fbfe3c6c2 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -454,7 +454,6 @@ type TrMouseThrashing = [ type TrRemovedNodesCount = [ type: 115, nodesCount: number, - domdropped: boolean, ] diff --git a/mobs/messages.rb b/mobs/messages.rb index 88ae76f20..018effc30 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -470,7 +470,6 @@ end message 115, 'RemovedNodesCount', :replayer => false do uint 'NodesCount' - boolean 'DOMDropped' end ## Backend-only diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index a4c0e4f2a..2578c00a3 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -519,7 +519,6 @@ export type MouseThrashing = [ export type RemovedNodesCount = [ /*type:*/ Type.RemovedNodesCount, /*nodesCount:*/ number, - /*domdropped:*/ boolean, ] diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index 052a1caea..589442c2e 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -835,12 +835,10 @@ export function MouseThrashing( export function RemovedNodesCount( nodesCount: number, - domdropped: boolean, ): Messages.RemovedNodesCount { return [ Messages.Type.RemovedNodesCount, nodesCount, - domdropped, ] } diff --git a/tracker/tracker/src/main/app/nodes.ts b/tracker/tracker/src/main/app/nodes.ts index b94823cce..387679380 100644 --- a/tracker/tracker/src/main/app/nodes.ts +++ b/tracker/tracker/src/main/app/nodes.ts @@ -26,10 +26,6 @@ export default class Nodes { listeners.push([type, listener, useCapture]) } - getNodesCount() { - return this.nodes.length - } - registerNode(node: Node): [/*id:*/ number, /*isNew:*/ boolean] { let id: number = (node as any)[this.node_id] const isNew = id === undefined diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts index 5972e9850..fc41c4885 100644 --- a/tracker/tracker/src/main/app/observer/observer.ts +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -218,7 +218,6 @@ export default abstract class Observer { private unbindTree(node: Node) { let removed = 0 - const nodesCount = this.app.nodes.getNodesCount() const id = this.app.nodes.unregisterNode(node) if (id !== undefined && this.recents.get(id) === RecentsType.Removed) { // Sending RemoveNode only for parent to maintain @@ -241,7 +240,7 @@ export default abstract class Observer { removed += 1 this.app.nodes.unregisterNode(walker.currentNode) } - this.app.send(RemovedNodesCount(removed, removed / nodesCount > 0.5)) + this.app.send(RemovedNodesCount(removed)) } } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 370c18d50..97fca51e0 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -267,7 +267,7 @@ export default class MessageEncoder extends PrimitiveEncoder { break case Messages.Type.RemovedNodesCount: - return this.uint(msg[1]) && this.boolean(msg[2]) + return this.uint(msg[1]) break } From f9872f596569dfbbb957b28f300c36863b2b6230 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 20 Feb 2023 10:40:45 +0100 Subject: [PATCH 103/253] change(tracker): move var --- tracker/tracker/src/main/app/observer/observer.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts index fc41c4885..1fd84c904 100644 --- a/tracker/tracker/src/main/app/observer/observer.ts +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -217,7 +217,6 @@ export default abstract class Observer { } private unbindTree(node: Node) { - let removed = 0 const id = this.app.nodes.unregisterNode(node) if (id !== undefined && this.recents.get(id) === RecentsType.Removed) { // Sending RemoveNode only for parent to maintain @@ -236,6 +235,8 @@ export default abstract class Observer { // @ts-ignore false, ) + + let removed = 0 while (walker.nextNode()) { removed += 1 this.app.nodes.unregisterNode(walker.currentNode) From a34d557f7f3a7331acec94c6974cf5ecda73650d Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 10 Mar 2023 11:49:53 +0100 Subject: [PATCH 104/253] change(tracker): send percent of removed nodes instead of int --- ee/connectors/msgcodec/msgcodec.py | 4 ++-- mobs/messages.rb | 5 ++--- tracker/tracker/src/main/app/nodes.ts | 4 ++++ tracker/tracker/src/main/app/observer/observer.ts | 5 +++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 75ccca196..df6133cbb 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -653,8 +653,8 @@ class MessageCodec(Codec): ) if message_id == 115: - return RemovedNodesCount( - nodes_count=self.read_uint(reader) + return UnbindNodes( + total_removed_percent=self.read_uint(reader) ) if message_id == 125: diff --git a/mobs/messages.rb b/mobs/messages.rb index 018effc30..a107d5dcc 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -448,7 +448,6 @@ end # 90-111 reserved iOS - message 112, 'InputChange', :replayer => false do uint 'ID' string 'Value' @@ -468,8 +467,8 @@ message 114, 'MouseThrashing' do uint 'Timestamp' end -message 115, 'RemovedNodesCount', :replayer => false do - uint 'NodesCount' +message 115, 'UnbindNodes', :replayer => false do + uint 'TotalRemovedPercent' end ## Backend-only diff --git a/tracker/tracker/src/main/app/nodes.ts b/tracker/tracker/src/main/app/nodes.ts index 387679380..097cce2c1 100644 --- a/tracker/tracker/src/main/app/nodes.ts +++ b/tracker/tracker/src/main/app/nodes.ts @@ -73,6 +73,10 @@ export default class Nodes { return this.nodes[id] } + getNodeCount() { + return this.nodes.filter(Boolean).length + } + clear(): void { for (let id = 0; id < this.nodes.length; id++) { const node = this.nodes[id] diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts index 1fd84c904..0d7030855 100644 --- a/tracker/tracker/src/main/app/observer/observer.ts +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -8,7 +8,7 @@ import { CreateElementNode, MoveNode, RemoveNode, - RemovedNodesCount, + UnbindNodes, } from '../messages.gen.js' import App from '../index.js' import { @@ -237,11 +237,12 @@ export default abstract class Observer { ) let removed = 0 + const total = this.app.nodes.getNodeCount() while (walker.nextNode()) { removed += 1 this.app.nodes.unregisterNode(walker.currentNode) } - this.app.send(RemovedNodesCount(removed)) + this.app.send(UnbindNodes(Math.floor((removed / total) * 100))) } } From fbab981614641aa2f7d3f4938f27eb3d13512106 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Mar 2023 11:15:57 +0100 Subject: [PATCH 105/253] change(tracker/backend): regenerate messages --- backend/pkg/messages/messages.go | 14 +++++++------- backend/pkg/messages/read-message.go | 8 ++++---- ee/connectors/msgcodec/messages.py | 6 +++--- frontend/app/player/web/messages/tracker.gen.ts | 6 +++--- tracker/tracker/src/common/messages.gen.ts | 10 +++++----- tracker/tracker/src/main/app/messages.gen.ts | 10 +++++----- .../tracker/src/webworker/MessageEncoder.gen.ts | 2 +- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 232f116c0..20410182a 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -79,7 +79,7 @@ const ( MsgInputChange = 112 MsgSelectionChange = 113 MsgMouseThrashing = 114 - MsgRemovedNodesCount = 115 + MsgUnbindNodes = 115 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 @@ -2102,24 +2102,24 @@ func (msg *MouseThrashing) TypeID() int { return 114 } -type RemovedNodesCount struct { +type UnbindNodes struct { message - NodesCount uint64 + TotalRemovedPercent uint64 } -func (msg *RemovedNodesCount) Encode() []byte { +func (msg *UnbindNodes) Encode() []byte { buf := make([]byte, 11) buf[0] = 115 p := 1 - p = WriteUint(msg.NodesCount, buf, p) + p = WriteUint(msg.TotalRemovedPercent, buf, p) return buf[:p] } -func (msg *RemovedNodesCount) Decode() Message { +func (msg *UnbindNodes) Decode() Message { return msg } -func (msg *RemovedNodesCount) TypeID() int { +func (msg *UnbindNodes) TypeID() int { return 115 } diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 715f81d98..f3bc525a9 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -1269,10 +1269,10 @@ func DecodeMouseThrashing(reader BytesReader) (Message, error) { return msg, err } -func DecodeRemovedNodesCount(reader BytesReader) (Message, error) { +func DecodeUnbindNodes(reader BytesReader) (Message, error) { var err error = nil - msg := &RemovedNodesCount{} - if msg.NodesCount, err = reader.ReadUint(); err != nil { + msg := &UnbindNodes{} + if msg.TotalRemovedPercent, err = reader.ReadUint(); err != nil { return nil, err } return msg, err @@ -1888,7 +1888,7 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { case 114: return DecodeMouseThrashing(reader) case 115: - return DecodeRemovedNodesCount(reader) + return DecodeUnbindNodes(reader) case 125: return DecodeIssueEvent(reader) case 126: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 75cc5309f..7c2638ec6 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -736,11 +736,11 @@ class MouseThrashing(Message): self.timestamp = timestamp -class RemovedNodesCount(Message): +class UnbindNodes(Message): __id__ = 115 - def __init__(self, nodes_count): - self.nodes_count = nodes_count + def __init__(self, total_removed_percent): + self.total_removed_percent = total_removed_percent class IssueEvent(Message): diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index fbfe3c6c2..f012084ec 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -451,13 +451,13 @@ type TrMouseThrashing = [ timestamp: number, ] -type TrRemovedNodesCount = [ +type TrUnbindNodes = [ type: 115, - nodesCount: number, + totalRemovedPercent: number, ] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrRemovedNodesCount +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 2578c00a3..6b4f5b1c3 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -66,7 +66,7 @@ export declare const enum Type { InputChange = 112, SelectionChange = 113, MouseThrashing = 114, - RemovedNodesCount = 115, + UnbindNodes = 115, } @@ -516,11 +516,11 @@ export type MouseThrashing = [ /*timestamp:*/ number, ] -export type RemovedNodesCount = [ - /*type:*/ Type.RemovedNodesCount, - /*nodesCount:*/ number, +export type UnbindNodes = [ + /*type:*/ Type.UnbindNodes, + /*totalRemovedPercent:*/ number, ] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | RemovedNodesCount +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | UnbindNodes export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index 589442c2e..c522d719f 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -833,12 +833,12 @@ export function MouseThrashing( ] } -export function RemovedNodesCount( - nodesCount: number, -): Messages.RemovedNodesCount { +export function UnbindNodes( + totalRemovedPercent: number, +): Messages.UnbindNodes { return [ - Messages.Type.RemovedNodesCount, - nodesCount, + Messages.Type.UnbindNodes, + totalRemovedPercent, ] } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 97fca51e0..d49f18610 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -266,7 +266,7 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) break - case Messages.Type.RemovedNodesCount: + case Messages.Type.UnbindNodes: return this.uint(msg[1]) break From 570662ca7a43eb0ccc7d14f3cfe163638a708cf8 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 17 Mar 2023 11:23:00 +0100 Subject: [PATCH 106/253] feat(backend): debug log for unbindNodes event --- backend/internal/db/datasaver/saver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index 1a017fa6f..f8f9b05b0 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -101,6 +101,8 @@ func (s *saverImpl) handleMessage(msg Message) error { return s.pg.InsertIOSScreenEnter(m) case *IOSCrash: return s.pg.InsertIOSCrash(m) + case *UnbindNodes: + log.Printf("UnbineNodes: %+v", m) } return nil } From 181756ae59271dfa55b5020af53fdc1e1cdb6387 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 17 Mar 2023 11:36:06 +0100 Subject: [PATCH 107/253] feat(backend): added unbindNodes event to db filter --- backend/cmd/db/main.go | 3 ++- backend/internal/db/datasaver/saver.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 6086d9491..82ece145f 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -38,7 +38,8 @@ func main() { messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, messages.MsgSetInputTarget, messages.MsgSetInputValue, messages.MsgCreateDocument, messages.MsgMouseClick, messages.MsgSetPageLocation, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming, - messages.MsgInputEvent, messages.MsgPageEvent, messages.MsgMouseThrashing, messages.MsgInputChange} + messages.MsgInputEvent, messages.MsgPageEvent, messages.MsgMouseThrashing, messages.MsgInputChange, + messages.MsgUnbindNodes} // Init consumer consumer := queue.NewConsumer( diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index f8f9b05b0..bafae34a6 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -102,7 +102,7 @@ func (s *saverImpl) handleMessage(msg Message) error { case *IOSCrash: return s.pg.InsertIOSCrash(m) case *UnbindNodes: - log.Printf("UnbineNodes: %+v", m) + log.Printf("UnbindNodes: %+v", m) } return nil } From 784dcf53ab3f92cbea92324e7991de04ec4b8e91 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Mar 2023 12:00:03 +0100 Subject: [PATCH 108/253] change(tracker): set a 30 perc cutoff for dom drop --- tracker/tracker/src/main/app/observer/observer.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts index 0d7030855..4fabdf3e5 100644 --- a/tracker/tracker/src/main/app/observer/observer.ts +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -237,12 +237,17 @@ export default abstract class Observer { ) let removed = 0 - const total = this.app.nodes.getNodeCount() + const totalBeforeRemove = this.app.nodes.getNodeCount() + while (walker.nextNode()) { removed += 1 this.app.nodes.unregisterNode(walker.currentNode) } - this.app.send(UnbindNodes(Math.floor((removed / total) * 100))) + + const removedPercent = Math.floor((removed / totalBeforeRemove) * 100) + if (removedPercent > 30) { + this.app.send(UnbindNodes(removedPercent)) + } } } From 9110e9cd6dbd96e56f89bb947794c9e489252e59 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Fri, 17 Mar 2023 12:16:42 +0100 Subject: [PATCH 109/253] feat(backend): removed debug logs --- backend/internal/db/datasaver/saver.go | 2 -- backend/pkg/db/postgres/messages-web.go | 5 ----- 2 files changed, 7 deletions(-) diff --git a/backend/internal/db/datasaver/saver.go b/backend/internal/db/datasaver/saver.go index bafae34a6..1a017fa6f 100644 --- a/backend/internal/db/datasaver/saver.go +++ b/backend/internal/db/datasaver/saver.go @@ -101,8 +101,6 @@ func (s *saverImpl) handleMessage(msg Message) error { return s.pg.InsertIOSScreenEnter(m) case *IOSCrash: return s.pg.InsertIOSCrash(m) - case *UnbindNodes: - log.Printf("UnbindNodes: %+v", m) } return nil } diff --git a/backend/pkg/db/postgres/messages-web.go b/backend/pkg/db/postgres/messages-web.go index ebba00168..6b39f0eca 100644 --- a/backend/pkg/db/postgres/messages-web.go +++ b/backend/pkg/db/postgres/messages-web.go @@ -90,8 +90,6 @@ func (conn *Conn) InsertWebInputEvent(sessionID uint64, projectID uint32, e *Inp } func (conn *Conn) InsertWebInputDuration(sessionID uint64, projectID uint32, e *InputChange) error { - // Debug log - log.Printf("new InputDuration event: %v", e) if e.Label == "" { return nil } @@ -165,9 +163,6 @@ func (conn *Conn) InsertSessionReferrer(sessionID uint64, referrer string) error } func (conn *Conn) InsertMouseThrashing(sessionID uint64, projectID uint32, e *MouseThrashing) error { - // Debug log - log.Printf("new MouseThrashing event: %v", e) - // issueID := hashid.MouseThrashingID(projectID, sessionID, e.Timestamp) if err := conn.bulks.Get("webIssues").Append(projectID, issueID, "mouse_thrashing", e.Url); err != nil { log.Printf("insert web issue err: %s", err) From 5408e6b31f2c6ff85a00fdca38cd1a0e2d6e7c03 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Mar 2023 12:34:21 +0100 Subject: [PATCH 110/253] change(tracker): make sure testing deps are up to date --- tracker/tracker-testing-playground/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tracker/tracker-testing-playground/package.json b/tracker/tracker-testing-playground/package.json index ccc98d80d..fe51df630 100644 --- a/tracker/tracker-testing-playground/package.json +++ b/tracker/tracker-testing-playground/package.json @@ -4,10 +4,10 @@ "private": true, "dependencies": { "@openreplay/tracker": "file:../../../openreplay/tracker/tracker", - "@openreplay/tracker-axios": "3.6.1", - "@openreplay/tracker-fetch": "3.6.1", - "@openreplay/tracker-redux": "3.5.1", - "@openreplay/tracker-zustand": "1.0.2", + "@openreplay/tracker-axios": "^3.6.2", + "@openreplay/tracker-fetch": "^3.6.2", + "@openreplay/tracker-redux": "^3.5.1", + "@openreplay/tracker-zustand": "^1.0.3", "@tanstack/react-table": "^8.2.6", "@types/node": "^12.0.0", "@types/react": "^17.0.0", From e3d24c275420ef7aa8e0d425adaff201c5282dde Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Mar 2023 12:47:16 +0100 Subject: [PATCH 111/253] change(tracker): update changelog for 5.0.2 --- tracker/tracker/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index 6908f5bd4..89458de18 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,3 +1,9 @@ +# 5.0.2 + +- Capture mouse thrashing, input hesitation+duration, click hesitation +- Capture DOM node drop event (>30% nodes removed) +- Capture iframe network requests + ## 5.0.1 - Re-init worker after device sleep/hybernation From ab16a4e93aa3480631f9feba6eba02a446f61091 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 17 Mar 2023 14:57:40 +0100 Subject: [PATCH 112/253] chore(build): Adding script to update helm tag (#1047) * chore(build): Adding script to update helm tag * fix(script): check for ee tag Signed-off-by: rjshrjndrn --- api/build.sh | 26 +++++++++++++++++++++++--- api/build_alerts.sh | 23 ++++++++++++++++++++++- backend/build.sh | 23 ++++++++++++++++++++++- frontend/build.sh | 23 +++++++++++++++++++++-- peers/build.sh | 20 ++++++++++++++++++++ sourcemap-reader/build.sh | 23 +++++++++++++++++++++-- utilities/build.sh | 20 ++++++++++++++++++++ 7 files changed, 149 insertions(+), 9 deletions(-) diff --git a/api/build.sh b/api/build.sh index bd4c35861..946dbf3b6 100644 --- a/api/build.sh +++ b/api/build.sh @@ -10,7 +10,7 @@ # Helper function exit_err() { err_code=$1 - if [[ err_code != 0 ]]; then + if [[ $err_code != 0 ]]; then exit $err_code fi } @@ -27,13 +27,32 @@ check_prereq() { return } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_api(){ destination="_api" [[ $1 == "ee" ]] && { destination="_api_ee" } cp -R ../api ../${destination} - cd ../${destination} + cd ../${destination} || exit_err 100 tag="" # Copy enterprise code [[ $1 == "ee" ]] && { @@ -43,7 +62,7 @@ function build_api(){ } mv Dockerfile.dockerignore .dockerignore docker build -f ./Dockerfile --build-arg envarg=$envarg --build-arg GIT_SHA=$git_sha -t ${DOCKER_REPO:-'local'}/chalice:${image_tag} . - cd ../api + cd ../api || exit_err 100 rm -rf ../${destination} [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/chalice:${image_tag} @@ -67,3 +86,4 @@ echo buil_complete # exit_err $? # rm build_crons.sh #} || true +[[ $PATCH -eq 1 ]] && update_helm_release chalice diff --git a/api/build_alerts.sh b/api/build_alerts.sh index b3c738b99..5047780fe 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -8,7 +8,7 @@ # Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh git_sha=$(git rev-parse --short HEAD) -image_tag=${IMAGE_TAG:-git_sha} +image_tag=${IMAGE_TAG:-$git_sha} envarg="default-foss" check_prereq() { which docker || { @@ -17,6 +17,26 @@ check_prereq() { } } + +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_alerts(){ destination="_alerts" [[ $1 == "ee" ]] && { @@ -48,3 +68,4 @@ function build_alerts(){ check_prereq build_alerts $1 +[[ $PATCH -eq 1 ]] && update_helm_release alerts diff --git a/backend/build.sh b/backend/build.sh index 95a833139..5c0b1b771 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -10,7 +10,7 @@ set -e git_sha=$(git rev-parse --short HEAD) -image_tag=${IMAGE_TAG:-git_sha} +image_tag=${IMAGE_TAG:-$git_sha} ee="false" check_prereq() { which docker || { @@ -20,6 +20,25 @@ check_prereq() { return } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_service() { image="$1" echo "BUILDING $image" @@ -48,6 +67,7 @@ function build_api(){ } [[ $2 != "" ]] && { build_service $2 + [[ $PATCH -eq 1 ]] && update_helm_release $2 cd ../backend rm -rf ../${destination} return @@ -56,6 +76,7 @@ function build_api(){ do build_service $image echo "::set-output name=image::${DOCKER_REPO:-'local'}/$image:${image_tag}" + [[ $PATCH -eq 1 ]] && update_helm_release $image done cd ../backend rm -rf ../${destination} diff --git a/frontend/build.sh b/frontend/build.sh index 55d295746..ecb2fe845 100644 --- a/frontend/build.sh +++ b/frontend/build.sh @@ -9,8 +9,7 @@ # Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh git_sha=$(git rev-parse --short HEAD) -image_tag=${IMAGE_TAG:-git_sha} -ee="false" +image_tag=${IMAGE_TAG:-$git_sha} check_prereq() { which docker || { echo "Docker not installed, please install docker." @@ -18,6 +17,25 @@ check_prereq() { } } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + # https://github.com/docker/cli/issues/1134#issuecomment-613516912 export DOCKER_BUILDKIT=1 function build(){ @@ -34,3 +52,4 @@ function build(){ check_prereq build $1 +[[ $PATCH -eq 1 ]] && update_helm_release frontend diff --git a/peers/build.sh b/peers/build.sh index 45cc97892..dfff188cb 100644 --- a/peers/build.sh +++ b/peers/build.sh @@ -15,6 +15,25 @@ check_prereq() { } } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_api(){ destination="_peers" [[ $1 == "ee" ]] && { @@ -43,3 +62,4 @@ function build_api(){ check_prereq build_api $1 +[[ $PATCH -eq 1 ]] && update_helm_release peers diff --git a/sourcemap-reader/build.sh b/sourcemap-reader/build.sh index fbe8762e2..7169403e7 100644 --- a/sourcemap-reader/build.sh +++ b/sourcemap-reader/build.sh @@ -11,9 +11,8 @@ set -e image_name="sourcemaps-reader" git_sha=$(git rev-parse --short HEAD) -image_tag=${IMAGE_TAG:-git_sha} +image_tag=${IMAGE_TAG:-$git_sha} envarg="default-foss" -tmp_folder_name="${image_name}_${RANDOM}" check_prereq() { which docker || { @@ -23,6 +22,25 @@ check_prereq() { return } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_api(){ destination="_smr" [[ $1 == "ee" ]] && { @@ -55,3 +73,4 @@ function build_api(){ check_prereq build_api $1 echo buil_complete +[[ $PATCH -eq 1 ]] && update_helm_release sourcemapreader diff --git a/utilities/build.sh b/utilities/build.sh index 87ff7f3e6..861d37596 100644 --- a/utilities/build.sh +++ b/utilities/build.sh @@ -15,6 +15,25 @@ check_prereq() { } } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_api(){ destination="_utilities" [[ $1 == "ee" ]] && { @@ -44,3 +63,4 @@ function build_api(){ check_prereq build_api $1 +[[ $PATCH -eq 1 ]] && update_helm_release assist From c313119363e11dd059fcb09235b908748d506369 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Mar 2023 15:32:20 +0100 Subject: [PATCH 113/253] change(tracker): add new events for assist; assist 5.0.2 --- tracker/tracker-assist/CHANGELOG.md | 8 +++++++ tracker/tracker-assist/package.json | 2 +- tracker/tracker-assist/src/Assist.ts | 24 +++++++++++++------ tracker/tracker-assist/src/RemoteControl.ts | 8 +++---- .../src/ScreenRecordingState.ts | 2 +- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/tracker/tracker-assist/CHANGELOG.md b/tracker/tracker-assist/CHANGELOG.md index f61aad123..5df2617c9 100644 --- a/tracker/tracker-assist/CHANGELOG.md +++ b/tracker/tracker-assist/CHANGELOG.md @@ -1,3 +1,11 @@ +## 5.0.2 + +- Added `onCallDeny`, `onRemoteControlDeny` and `onRecordingDeny` callbacks to signal denial of user's consent to call/control/recording + +## 5.0.1 + +- dependency updates + ## 5.0.0 - fix recording state import diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 4c8c98d53..1bda6b785 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "5.0.1", + "version": "5.0.2", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 8d14740a5..f31f85c22 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -25,6 +25,9 @@ export interface Options { onCallStart: StartEndCallback; onRemoteControlStart: StartEndCallback; onRecordingRequest?: (agentInfo: Record) => any; + onCallDeny?: () => any; + onRemoteControlDeny?: (agentInfo: Record) => any; + onRecordingDeny?: (agentInfo: Record) => any; session_calling_peer_key: string; session_control_peer_key: string; callConfirm: ConfirmOptions; @@ -46,7 +49,7 @@ enum CallingState { Requesting, True, False, -}; +} // TODO typing???? @@ -84,7 +87,7 @@ export default class Assist { onAgentConnect: ()=>{}, onRemoteControlStart: ()=>{}, callConfirm: {}, - controlConfirm: {}, // TODO: clear options passing/merging/overriting + controlConfirm: {}, // TODO: clear options passing/merging/overwriting recordingConfirm: {}, }, options, @@ -194,7 +197,7 @@ export default class Assist { annot.mount() return callingAgents.get(id) }, - id => { + (id, isDenied) => { if (id) { const cb = this.agents[id].onControlReleased delete this.agents[id].onControlReleased @@ -210,14 +213,20 @@ export default class Assist { callUI?.remove() callUI = null } + if (isDenied) { + const info = id ? this.agents[id]?.agentInfo : {} + this.options.onRemoteControlDeny?.(info || {}) + } }, ) const onAcceptRecording = () => { socket.emit('recording_accepted') } - const onRejectRecording = () => { + const onRejectRecording = (agentData) => { socket.emit('recording_rejected') + + this.options.onRecordingDeny?.(agentData || {}) } const recordingState = new ScreenRecordingState(this.options.recordingConfirm) @@ -296,7 +305,7 @@ export default class Assist { socket.on('request_recording', (id, agentData) => { if (!recordingState.isActive) { this.options.onRecordingRequest?.(JSON.parse(agentData)) - recordingState.requestRecording(id, onAcceptRecording, onRejectRecording) + recordingState.requestRecording(id, onAcceptRecording, () => onRejectRecording(agentData)) } else { this.emit('recording_busy') } @@ -365,7 +374,7 @@ export default class Assist { }) } - const handleCallEnd = () => { // Completle stop and clear all calls + const handleCallEnd = () => { // Complete stop and clear all calls // Streams Object.values(calls).forEach(call => call.close()) Object.keys(calls).forEach(peerId => { @@ -417,6 +426,7 @@ export default class Assist { confirmAnswer.then(async agreed => { if (!agreed) { initiateCallEnd() + this.options.onCallDeny?.() return } // Request local stream for the new connection @@ -428,7 +438,7 @@ export default class Assist { } calls[call.peer] = call } catch (e) { - app.debug.error('Audio mediadevice request error:', e) + app.debug.error('Audio media device request error:', e) initiateCallEnd() return } diff --git a/tracker/tracker-assist/src/RemoteControl.ts b/tracker/tracker-assist/src/RemoteControl.ts index 017918238..c56ed3222 100644 --- a/tracker/tracker-assist/src/RemoteControl.ts +++ b/tracker/tracker-assist/src/RemoteControl.ts @@ -25,7 +25,7 @@ export default class RemoteControl { constructor( private readonly options: AssistOptions, private readonly onGrand: (id: string) => string | undefined, - private readonly onRelease: (id?: string | null) => void) {} + private readonly onRelease: (id?: string | null, isDenied?: boolean) => void) {} reconnect(ids: string[]) { const storedID = sessionStorage.getItem(this.options.session_control_peer_key) @@ -55,7 +55,7 @@ export default class RemoteControl { this.grantControl(id) } else { this.confirm?.remove() - this.releaseControl() + this.releaseControl(true) } }) .then(() => { @@ -67,7 +67,7 @@ export default class RemoteControl { }) } - releaseControl = () => { + releaseControl = (isDenied?: boolean) => { if (this.confirm) { this.confirm.remove() this.confirm = null @@ -75,7 +75,7 @@ export default class RemoteControl { this.resetMouse() this.status = RCStatus.Disabled sessionStorage.removeItem(this.options.session_control_peer_key) - this.onRelease(this.agentID) + this.onRelease(this.agentID, isDenied) this.agentID = null } diff --git a/tracker/tracker-assist/src/ScreenRecordingState.ts b/tracker/tracker-assist/src/ScreenRecordingState.ts index a962e85ac..87464f18a 100644 --- a/tracker/tracker-assist/src/ScreenRecordingState.ts +++ b/tracker/tracker-assist/src/ScreenRecordingState.ts @@ -109,6 +109,6 @@ export default class ScreenRecordingState { this.status = RecordingState.Off this.overlayAdded = false - this.uiComponents.forEach((el) => el.parentElement?.removeChild(el)) + this.uiComponents?.forEach((el) => el.parentElement?.removeChild(el)) } } From 0516d58632a873f6459de474d6dbbfa3ed47b38a Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Mar 2023 16:14:49 +0100 Subject: [PATCH 114/253] change(ui): signal reject event for call, recording and control --- .../Assist/components/AssistActions/AssistActions.tsx | 5 +++++ .../components/Session_/ScreenRecorder/ScreenRecorder.tsx | 3 ++- frontend/app/player/web/assist/AssistManager.ts | 3 +++ frontend/app/player/web/assist/Call.ts | 4 +--- frontend/app/player/web/assist/RemoteControl.ts | 7 ++++++- frontend/app/player/web/assist/ScreenRecording.ts | 5 ++++- 6 files changed, 21 insertions(+), 6 deletions(-) diff --git a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx index 18c717cf4..71a117868 100644 --- a/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx +++ b/frontend/app/components/Assist/components/AssistActions/AssistActions.tsx @@ -20,6 +20,9 @@ import ScreenRecorder from 'App/components/Session_/ScreenRecorder/ScreenRecorde function onReject() { toast.info(`Call was rejected.`); } +function onControlReject() { + toast.info('Remote control request was rejected by user') +} function onError(e: any) { console.log(e); @@ -52,6 +55,7 @@ function AssistActions({ setCallArgs, requestReleaseRemoteControl, toggleAnnotation, + setRemoteControlCallbacks }, toggleUserName, } = player @@ -153,6 +157,7 @@ function AssistActions({ }; const requestControl = () => { + setRemoteControlCallbacks({ onReject: onControlReject }) if (callRequesting || remoteRequesting) return; requestReleaseRemoteControl(); }; diff --git a/frontend/app/components/Session_/ScreenRecorder/ScreenRecorder.tsx b/frontend/app/components/Session_/ScreenRecorder/ScreenRecorder.tsx index 4f7d9f0bf..97c3db78d 100644 --- a/frontend/app/components/Session_/ScreenRecorder/ScreenRecorder.tsx +++ b/frontend/app/components/Session_/ScreenRecorder/ScreenRecorder.tsx @@ -100,7 +100,8 @@ function ScreenRecorder({ }; const recordingRequest = () => { - player.assistManager.requestRecording(); + const onDeny = () => toast.info('Recording request was rejected by user') + player.assistManager.requestRecording({ onDeny }); }; if (!isSupported() || !isEnterprise) { diff --git a/frontend/app/player/web/assist/AssistManager.ts b/frontend/app/player/web/assist/AssistManager.ts index 13de4ebdd..e75db1388 100644 --- a/frontend/app/player/web/assist/AssistManager.ts +++ b/frontend/app/player/web/assist/AssistManager.ts @@ -250,6 +250,9 @@ export default class AssistManager { requestReleaseRemoteControl = (...args: Parameters) => { return this.remoteControl?.requestReleaseRemoteControl(...args) } + setRemoteControlCallbacks = (...args: Parameters) => { + return this.remoteControl?.setCallbacks(...args) + } releaseRemoteControl = (...args: Parameters) => { return this.remoteControl?.releaseRemoteControl(...args) } diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index a771794fa..2119b9935 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -147,13 +147,11 @@ export default class Call { //this.toggleAnnotation(false) } private onRemoteCallEnd = () => { - if (this.store.get().calling === CallingState.Requesting) { + if ([CallingState.Requesting, CallingState.Connecting].includes(this.store.get().calling)) { this.callArgs && this.callArgs.onReject() this.callConnection[0] && this.callConnection[0].close() this.store.update({ calling: CallingState.NoCall }) this.callArgs = null - // TODO: We have it separated, right? (check) - //this.toggleAnnotation(false) } else { this.handleCallEnd() } diff --git a/frontend/app/player/web/assist/RemoteControl.ts b/frontend/app/player/web/assist/RemoteControl.ts index e262e75b8..56870995d 100644 --- a/frontend/app/player/web/assist/RemoteControl.ts +++ b/frontend/app/player/web/assist/RemoteControl.ts @@ -3,7 +3,6 @@ import type { Socket } from './types' import type Screen from '../Screen/Screen' import type { Store } from '../../common/types' - export enum RemoteControlStatus { Disabled = 0, Requesting, @@ -20,6 +19,7 @@ export default class RemoteControl { remoteControl: RemoteControlStatus.Disabled, annotating: false, } + onReject: () => void = () => {} constructor( private store: Store, @@ -33,6 +33,7 @@ export default class RemoteControl { }) socket.on("control_rejected", id => { id === socket.id && this.toggleRemoteControl(false) + this.onReject() }) socket.on('SESSION_DISCONNECTED', () => { if (this.store.get().remoteControl === RemoteControlStatus.Requesting) { @@ -59,6 +60,10 @@ export default class RemoteControl { this.socket.emit("scroll", [ e.deltaX, e.deltaY ]) } + public setCallbacks = ({ onReject }: { onReject: () => void }) => { + this.onReject = onReject + } + private onMouseClick = (e: MouseEvent): void => { if (this.store.get().annotating) { return; } // ignore clicks while annotating diff --git a/frontend/app/player/web/assist/ScreenRecording.ts b/frontend/app/player/web/assist/ScreenRecording.ts index 90ad76020..933fcc02b 100644 --- a/frontend/app/player/web/assist/ScreenRecording.ts +++ b/frontend/app/player/web/assist/ScreenRecording.ts @@ -15,6 +15,7 @@ export interface State { } export default class ScreenRecording { + onDeny: () => void = () => {} static readonly INITIAL_STATE: Readonly = { recordingState: SessionRecordingStatus.Off, } @@ -29,6 +30,7 @@ export default class ScreenRecording { }) socket.on('recording_rejected', () => { this.toggleRecording(false) + this.onDeny() }) socket.on('recording_busy', () => { this.onRecordingBusy() @@ -39,7 +41,8 @@ export default class ScreenRecording { toast.error("This session is already being recorded by another agent") } - requestRecording = () => { + requestRecording = ({ onDeny }: { onDeny: () => void }) => { + this.onDeny = onDeny const recordingState = this.store.get().recordingState if (recordingState === SessionRecordingStatus.Requesting) return; From 9774c823c3c8203c31c9bee1a07a70081f22ce6a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 17 Mar 2023 17:40:54 +0100 Subject: [PATCH 115/253] feat(chalice): changed assist-credentials response --- ee/api/chalicelib/utils/assist_helper.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ee/api/chalicelib/utils/assist_helper.py b/ee/api/chalicelib/utils/assist_helper.py index d182226c0..061b329ef 100644 --- a/ee/api/chalicelib/utils/assist_helper.py +++ b/ee/api/chalicelib/utils/assist_helper.py @@ -37,13 +37,16 @@ def get_full_config(): if __get_secret() is not None: for i in range(len(servers)): url = servers[i].split(",")[0] - servers[i] = {"url": url} if url.lower().startswith("stun") else {"url": url, **credentials} + # servers[i] = {"url": url} if url.lower().startswith("stun") else {"url": url, **credentials} + servers[i] = {"urls": url} if url.lower().startswith("stun") else {"urls": url, **credentials} else: for i in range(len(servers)): s = servers[i].split(",") if len(s) == 3: - servers[i] = {"url": s[0], "username": s[1], "credential": s[2]} + # servers[i] = {"url": s[0], "username": s[1], "credential": s[2]} + servers[i] = {"urls": s[0], "username": s[1], "credential": s[2]} else: - servers[i] = {"url": s[0]} + # servers[i] = {"url": s[0]} + servers[i] = {"urls": s[0]} return servers From de7257b6c44088a00d61bf7359f796f759c7ac30 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 17 Mar 2023 17:54:24 +0100 Subject: [PATCH 116/253] feat(chalice): ignore kafka health-check --- ee/api/chalicelib/core/health.py | 4 ++-- ee/api/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 80abdacd7..e00747288 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -2,7 +2,7 @@ from urllib.parse import urlparse import redis import requests -from confluent_kafka.admin import AdminClient +# from confluent_kafka.admin import AdminClient from decouple import config from chalicelib.utils import pg_client, ch_client @@ -152,7 +152,7 @@ def get_health(): }, "ingestionPipeline": { "redis": __check_redis, - "kafka": __check_kafka + # "kafka": __check_kafka }, "backendServices": { "alerts": __check_be_service("alerts"), diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index cad05e873..0ba6659c0 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -19,4 +19,4 @@ python3-saml==1.15.0 python-multipart==0.0.5 redis==4.5.1 -confluent-kafka==2.0.2 \ No newline at end of file +#confluent-kafka==2.0.2 \ No newline at end of file From e69bcbeff8482a62d033c798a923a4c3a47e92cb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 20 Mar 2023 10:25:33 +0100 Subject: [PATCH 117/253] feat(chalice): fixed new replay response --- ee/api/chalicelib/core/sessions_replay.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ee/api/chalicelib/core/sessions_replay.py b/ee/api/chalicelib/core/sessions_replay.py index 993855637..319eb13b6 100644 --- a/ee/api/chalicelib/core/sessions_replay.py +++ b/ee/api/chalicelib/core/sessions_replay.py @@ -133,7 +133,8 @@ def get_replay(project_id, session_id, context: schemas.CurrentContext, full_dat else: data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id) data['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session_id) - data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id) + data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id, + context=context) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, From 13986a3c55b6423256a3746036945ef2bbe52c22 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 20 Mar 2023 13:39:50 +0100 Subject: [PATCH 118/253] feat(DB): stop upgrade on version hopping --- .../db/init_dbs/postgresql/1.11.0/1.11.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.11.0/1.11.0.sql | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql index cf80dcad1..21544f62c 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.10.0-ee'; + next_version CONSTANT text := 'v1.11.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql b/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql index 41521a886..0fde93c48 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.10.0'; + next_version CONSTANT text := 'v1.11.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS From 1815d7b6be50f3d24d471fcf97427f47633c8d45 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 20 Mar 2023 13:31:35 +0100 Subject: [PATCH 119/253] feat(DB): stop upgrade on version hopping --- .../db/init_dbs/postgresql/1.10.0/1.10.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.1/1.5.1.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.2/1.5.2.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.3/1.5.3.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.4/1.5.4.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.6.0/1.6.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.7.0/1.7.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.8.0/1.8.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.8.1/1.8.1.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.9.0/1.9.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.10.0/1.10.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.1/1.5.1.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.2/1.5.2.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.3/1.5.3.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.4/1.5.4.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.6.0/1.6.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.7.0/1.7.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.8.0/1.8.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.8.1/1.8.1.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.9.0/1.9.0.sql | 17 +++++++++++++++++ 22 files changed, 374 insertions(+) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index 044b24176..173114586 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.9.0-ee'; + next_version CONSTANT text := 'v1.10.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql index 03af6067f..a73032e41 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.4.0-ee'; + next_version CONSTANT text := 'v1.5.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql index de01efd53..e5f19444c 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.0-ee'; + next_version CONSTANT text := 'v1.5.1-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql index 8ec804b02..675ecb871 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.1-ee'; + next_version CONSTANT text := 'v1.5.2-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql index 0d68e46b9..55494b9c9 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.2-ee'; + next_version CONSTANT text := 'v1.5.3-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql index 1a640b4be..fcb9cd832 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -1,4 +1,21 @@ \set ON_ERROR_STOP true +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.3-ee'; + next_version CONSTANT text := 'v1.5.4-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + SET client_min_messages TO NOTICE; BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql index bb0d7b7c0..1f545b1d6 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.4-ee'; + next_version CONSTANT text := 'v1.6.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql index cdf316fa4..fc62e3767 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.6.0-ee'; + next_version CONSTANT text := 'v1.7.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql index 8347a5c78..aeb87b3ff 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.7.0-ee'; + next_version CONSTANT text := 'v1.8.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql index f02f9e0a8..59c8819a7 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.8.0-ee'; + next_version CONSTANT text := 'v1.8.1-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql index c3483579d..8e969aeb6 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.8.1-ee'; + next_version CONSTANT text := 'v1.9.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql b/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index f2fb3f839..3a7344837 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.9.0'; + next_version CONSTANT text := 'v1.10.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql index fa72d27da..f0a3203b7 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.4.0'; + next_version CONSTANT text := 'v1.5.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql b/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql index 92fc44afe..c637b072d 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.0'; + next_version CONSTANT text := 'v1.5.1'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql b/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql index f4e26f93a..a94c1c3e9 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.1'; + next_version CONSTANT text := 'v1.5.2'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql b/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql index 159761b74..5b5bb92c4 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.2'; + next_version CONSTANT text := 'v1.5.3'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql index e7be94997..68aa530d3 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -1,4 +1,21 @@ \set ON_ERROR_STOP true +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.3'; + next_version CONSTANT text := 'v1.5.4'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + SET client_min_messages TO NOTICE; BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() diff --git a/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql b/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql index d11cad5be..cf5c88eab 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.4'; + next_version CONSTANT text := 'v1.6.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql b/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql index edc751da9..88a7acca7 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.6.0'; + next_version CONSTANT text := 'v1.7.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql b/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql index b14b14f91..93e05f01f 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.7.0'; + next_version CONSTANT text := 'v1.8.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql b/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql index c621da9c7..3b906c9ea 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.8.0'; + next_version CONSTANT text := 'v1.8.1'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql b/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql index 20a8bab1f..1b483432c 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.8.1'; + next_version CONSTANT text := 'v1.9.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS From e4a3062423ed3aabfe7d8cb3ee0df84fa5e5ca6f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 20 Mar 2023 13:31:35 +0100 Subject: [PATCH 120/253] feat(DB): stop upgrade on version hopping --- .../db/init_dbs/postgresql/1.10.0/1.10.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.1/1.5.1.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.2/1.5.2.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.3/1.5.3.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.4/1.5.4.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.6.0/1.6.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.7.0/1.7.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.8.0/1.8.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.8.1/1.8.1.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.9.0/1.9.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.10.0/1.10.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.1/1.5.1.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.2/1.5.2.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.3/1.5.3.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.5.4/1.5.4.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.6.0/1.6.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.7.0/1.7.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.8.0/1.8.0.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.8.1/1.8.1.sql | 17 +++++++++++++++++ .../db/init_dbs/postgresql/1.9.0/1.9.0.sql | 17 +++++++++++++++++ 22 files changed, 374 insertions(+) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index 044b24176..173114586 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.9.0-ee'; + next_version CONSTANT text := 'v1.10.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql index 03af6067f..a73032e41 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.4.0-ee'; + next_version CONSTANT text := 'v1.5.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql index de01efd53..e5f19444c 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.0-ee'; + next_version CONSTANT text := 'v1.5.1-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql index 8ec804b02..675ecb871 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.1-ee'; + next_version CONSTANT text := 'v1.5.2-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql index 0d68e46b9..55494b9c9 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.2-ee'; + next_version CONSTANT text := 'v1.5.3-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql index 1a640b4be..fcb9cd832 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -1,4 +1,21 @@ \set ON_ERROR_STOP true +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.3-ee'; + next_version CONSTANT text := 'v1.5.4-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + SET client_min_messages TO NOTICE; BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql index bb0d7b7c0..1f545b1d6 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.4-ee'; + next_version CONSTANT text := 'v1.6.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql index cdf316fa4..fc62e3767 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.6.0-ee'; + next_version CONSTANT text := 'v1.7.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql index 8347a5c78..aeb87b3ff 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.7.0-ee'; + next_version CONSTANT text := 'v1.8.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql index f02f9e0a8..59c8819a7 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.8.0-ee'; + next_version CONSTANT text := 'v1.8.1-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql index c3483579d..8e969aeb6 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.8.1-ee'; + next_version CONSTANT text := 'v1.9.0-ee'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql b/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index f2fb3f839..3a7344837 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.9.0'; + next_version CONSTANT text := 'v1.10.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql index fa72d27da..f0a3203b7 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.0/1.5.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.4.0'; + next_version CONSTANT text := 'v1.5.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql b/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql index 92fc44afe..c637b072d 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.1/1.5.1.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.0'; + next_version CONSTANT text := 'v1.5.1'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql b/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql index f4e26f93a..a94c1c3e9 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.2/1.5.2.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.1'; + next_version CONSTANT text := 'v1.5.2'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql b/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql index 159761b74..5b5bb92c4 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.3/1.5.3.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.2'; + next_version CONSTANT text := 'v1.5.3'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql b/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql index e7be94997..68aa530d3 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.5.4/1.5.4.sql @@ -1,4 +1,21 @@ \set ON_ERROR_STOP true +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.3'; + next_version CONSTANT text := 'v1.5.4'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + SET client_min_messages TO NOTICE; BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() diff --git a/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql b/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql index d11cad5be..cf5c88eab 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.6.0/1.6.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.5.4'; + next_version CONSTANT text := 'v1.6.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql b/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql index edc751da9..88a7acca7 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.7.0/1.7.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.6.0'; + next_version CONSTANT text := 'v1.7.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql b/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql index b14b14f91..93e05f01f 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.8.0/1.8.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.7.0'; + next_version CONSTANT text := 'v1.8.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql b/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql index c621da9c7..3b906c9ea 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.8.1/1.8.1.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.8.0'; + next_version CONSTANT text := 'v1.8.1'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS diff --git a/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql b/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql index 20a8bab1f..1b483432c 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql @@ -1,3 +1,20 @@ +DO +$$ + DECLARE + previous_version CONSTANT text := 'v1.8.1'; + next_version CONSTANT text := 'v1.9.0'; + BEGIN + IF (SELECT openreplay_version()) = previous_version THEN + raise notice 'valid previous DB version'; + ELSEIF (SELECT openreplay_version()) = next_version THEN + raise notice 'new version detected, nothing to do'; + ELSE + RAISE EXCEPTION 'upgrade to % failed, invalid previous version, expected %, got %', next_version,previous_version,(SELECT openreplay_version()); + END IF; + END ; +$$ +LANGUAGE plpgsql; + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS From 18d569c12b0032edd61f687cfa1227c32616fdfa Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 20 Mar 2023 14:51:58 +0100 Subject: [PATCH 121/253] feat(chalice): upgraded fastapi --- api/requirements.txt | 2 +- ee/api/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/requirements.txt b/api/requirements.txt index 27b95f17e..490a147df 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -8,7 +8,7 @@ jira==3.4.1 -fastapi==0.94.1 +fastapi==0.95.0 uvicorn[standard]==0.20.0 python-decouple==3.7 pydantic[email]==1.10.4 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 0ba6659c0..b5da59c4b 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -8,7 +8,7 @@ jira==3.4.1 -fastapi==0.94.1 +fastapi==0.95.0 uvicorn[standard]==0.20.0 python-decouple==3.7 pydantic[email]==1.10.4 From 74d8f91367c96271fe40bf13365885bda9b42771 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 21 Mar 2023 09:58:05 +0100 Subject: [PATCH 122/253] change(tracker): add option to disable click maps --- tracker/tracker/src/main/index.ts | 5 ++++- tracker/tracker/src/main/modules/mouse.ts | 11 ++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index 0688a22bf..bc4ff0775 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -34,6 +34,8 @@ import type { Options as InputOptions } from './modules/input.js' import type { Options as PerformanceOptions } from './modules/performance.js' import type { Options as TimingOptions } from './modules/timing.js' import type { Options as NetworkOptions } from './modules/network.js' +import type { MouseHandlerOptions } from './modules/mouse.js' + import type { StartOptions } from './app/index.js' //TODO: unique options init import type { StartPromiseReturn } from './app/index.js' @@ -47,6 +49,7 @@ export type Options = Partial< respectDoNotTrack?: boolean autoResetOnWindowOpen?: boolean network?: NetworkOptions + mouse?: MouseHandlerOptions // dev only __DISABLE_SECURE_MODE?: boolean } @@ -125,7 +128,7 @@ export default class API { Exception(app, options) Img(app) Input(app, options) - Mouse(app) + Mouse(app, options.mouse) Timing(app, options) Performance(app, options) Scroll(app) diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 032792b6f..fb69bef08 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -5,7 +5,7 @@ import { MouseMove, MouseClick, MouseThrashing } from '../app/messages.gen.js' import { getInputLabel } from './input.js' import { finder } from '@medv/finder' -function _getSelector(target: Element, document: Document): string { +function _getSelector(target: Element, document: Document) { const selector = finder(target, { root: document.body, seedMinLength: 3, @@ -73,7 +73,12 @@ function _getTarget(target: Element, document: Document): Element | null { return target === document.documentElement ? null : target } -export default function (app: App): void { +export interface MouseHandlerOptions { + disableClickmaps?: boolean +} + +export default function (app: App, options?: MouseHandlerOptions): void { + const { disableClickmaps = false } = options || {} function getTargetLabel(target: Element): string { const dl = getLabelAttribute(target) if (dl !== null) { @@ -197,7 +202,7 @@ export default function (app: App): void { id, mouseTarget === target ? Math.round(performance.now() - mouseTargetTime) : 0, getTargetLabel(target), - isClickable(target) ? getSelector(id, target) : '', + isClickable(target) && !disableClickmaps ? getSelector(id, target) : '', ), true, ) From 6742435bc973f211098c5401fc9126b7870583a8 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 21 Mar 2023 10:00:38 +0100 Subject: [PATCH 123/253] change(tracker): changelog for 5.0.2 --- tracker/tracker/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index 89458de18..a1050fb85 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -3,6 +3,7 @@ - Capture mouse thrashing, input hesitation+duration, click hesitation - Capture DOM node drop event (>30% nodes removed) - Capture iframe network requests +- added `{ network: { disableClickmaps: boolean } }` to disable calculating el. selectors ## 5.0.1 From f8737b84f732dc14efcefcf4f6d2bd4721f5f912 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 21 Mar 2023 13:16:06 +0100 Subject: [PATCH 124/253] feat(chalice): changed clickhouse-client config --- ee/api/chalicelib/utils/ch_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ee/api/chalicelib/utils/ch_client.py b/ee/api/chalicelib/utils/ch_client.py index 576bbc590..ef1839189 100644 --- a/ee/api/chalicelib/utils/ch_client.py +++ b/ee/api/chalicelib/utils/ch_client.py @@ -20,8 +20,9 @@ class ClickHouseClient: def __init__(self): self.__client = clickhouse_driver.Client(host=config("ch_host"), - database=config("ch_database",default="default", cast=str), - password=config("ch_password",default="", cast=str), + database=config("ch_database", default="default"), + user=config("ch_user", default="default"), + password=config("ch_password", default=""), port=config("ch_port", cast=int), settings=settings) \ if self.__client is None else self.__client From a128060248e42c0867f9fc94d4038c4655b297fa Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 21 Mar 2023 15:59:06 +0100 Subject: [PATCH 125/253] change(ui): tracker beta v, improvements for assets loading after click on timeline (player.jump) --- frontend/app/components/Session/WebPlayer.tsx | 2 +- frontend/app/player/common/types.ts | 2 +- frontend/app/player/player/Animator.ts | 8 +- frontend/app/player/web/MessageManager.ts | 8 +- .../app/player/web/managers/DOM/DOMManager.ts | 80 +++++++++++++++---- .../app/player/web/managers/DOM/VirtualDOM.ts | 7 +- .../app/player/web/managers/PagesManager.ts | 4 +- tracker/tracker/package.json | 2 +- 8 files changed, 82 insertions(+), 31 deletions(-) diff --git a/frontend/app/components/Session/WebPlayer.tsx b/frontend/app/components/Session/WebPlayer.tsx index 16b3d54ee..8111f11af 100644 --- a/frontend/app/components/Session/WebPlayer.tsx +++ b/frontend/app/components/Session/WebPlayer.tsx @@ -66,7 +66,7 @@ function WebPlayer(props: any) { React.useEffect(() => { if (session.events.length > 0 || session.errors.length > 0) { - contextValue.player.updateLists(session) + contextValue.player?.updateLists?.(session) } }, [session.events, session.errors]) diff --git a/frontend/app/player/common/types.ts b/frontend/app/player/common/types.ts index 308ec0659..7df4f6f6b 100644 --- a/frontend/app/player/common/types.ts +++ b/frontend/app/player/common/types.ts @@ -7,7 +7,7 @@ export interface Indexed { } export interface Moveable { - move(time: number): void + move(time: number, isJump?: boolean): void } export interface Cleanable { diff --git a/frontend/app/player/player/Animator.ts b/frontend/app/player/player/Animator.ts index e60b28d78..9423b5785 100644 --- a/frontend/app/player/player/Animator.ts +++ b/frontend/app/player/player/Animator.ts @@ -60,12 +60,12 @@ export default class Animator { window.playerJump = this.jump.bind(this) } - private setTime(time: number) { + private setTime(time: number, isJump?: boolean) { this.store.update({ time, completed: false, }) - this.mm.move(time) + this.mm.move(time, isJump) } private startAnimation() { @@ -183,11 +183,11 @@ export default class Animator { jump = (time: number) => { if (this.store.get().playing) { cancelAnimationFrame(this.animationFrameRequestId) - this.setTime(time) + this.setTime(time, true) this.startAnimation() this.store.update({ livePlay: time === this.store.get().endTime }) } else { - this.setTime(time) + this.setTime(time, true) this.store.update({ livePlay: time === this.store.get().endTime }) } } diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index 1fd0e419d..f5916d5a8 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -228,7 +228,7 @@ export default class MessageManager { sorted.forEach(msg => { if (indx > msg._index) outOfOrderCounter++ else indx = msg._index - this.distributeMessage(msg, msg._index) + this.distributeMessage(msg) }) if (outOfOrderCounter > 0) console.warn("Unsorted mob file, error count: ", outOfOrderCounter) @@ -287,7 +287,7 @@ export default class MessageManager { this.activityManager = new ActivityManager(this.session.duration.milliseconds); } - move(t: number, index?: number): void { + move(t: number, isJump?: boolean, index?: number): void { const stateToUpdate: Partial = {}; /* == REFACTOR_ME == */ const lastLoadedLocationMsg = this.loadedLocationManager.moveGetLast(t, index); @@ -337,7 +337,7 @@ export default class MessageManager { if (!!lastResize) { this.setSize(lastResize) } - this.pagesManager.moveReady(t).then(() => { + this.pagesManager.moveReady(t, isJump).then(() => { const lastScroll = this.scrollManager.moveGetLast(t, index); if (!!lastScroll && this.screen.window) { @@ -374,7 +374,7 @@ export default class MessageManager { return { ...msg, ...decoded }; } - distributeMessage(msg: Message, index: number): void { + distributeMessage(msg: Message): void { const lastMessageTime = Math.max(msg.time, this.lastMessageTime) this.lastMessageTime = lastMessageTime this.state.update({ lastMessageTime }) diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index 7b773860c..ec51401f6 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -43,7 +43,7 @@ export default class DOMManager extends ListWalker { private styleSheets: Map = new Map() private ppStyleSheets: Map = new Map() private stringDict: Record = {} - + private attrsBacktrack: Message[] = [] private upperBodyId: number = -1; private nodeScrollManagers: Map> = new Map() @@ -143,6 +143,7 @@ export default class DOMManager extends ListWalker { let { name, value } = msg; const vn = this.vElements.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } + if (vn.node.tagName === "INPUT" && name === "name") { // Otherwise binds local autocomplete values (maybe should ignore on the tracker level) return @@ -152,9 +153,13 @@ export default class DOMManager extends ListWalker { // if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { // value = value.replace("?", "%3F"); // } - if (!value.startsWith("http")) { return } - // blob:... value happened here. https://foss.openreplay.com/3/session/7013553567419137 - // that resulted in that link being unable to load and having 4sec timeout in the below function. + if (!value.startsWith("http")) { + return + } + // blob:... value can happen here for some reason. + // which will result in that link being unable to load and having 4sec timeout in the below function. + + // TODO: check if node actually exists on the page, not just in memory this.stylesManager.setStyleHandlers(vn.node as HTMLLinkElement, value); } if (vn.node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) { @@ -164,8 +169,7 @@ export default class DOMManager extends ListWalker { this.removeBodyScroll(msg.id, vn) } - private applyMessage = (msg: Message): Promise | undefined => { - let node: Node | undefined + private applyMessage = (msg: Message, isJump?: boolean): Promise | undefined => { let vn: VNode | undefined let doc: Document | null let styleSheet: CSSStyleSheet | PostponedStyleSheet | undefined @@ -229,9 +233,12 @@ export default class DOMManager extends ListWalker { if (!vn) { logger.error("Node not found", msg); return } if (!vn.parentNode) { logger.error("Parent node not found", msg); return } vn.parentNode.removeChild(vn) + this.vElements.delete(msg.id) + this.vTexts.delete(msg.id) return case MType.SetNodeAttribute: - this.setNodeAttribute(msg) + if (isJump && msg.name === 'href') this.attrsBacktrack.push(msg) + else this.setNodeAttribute(msg) return case MType.StringDict: this.stringDict[msg.key] = msg.value @@ -240,11 +247,14 @@ export default class DOMManager extends ListWalker { this.stringDict[msg.nameKey] === undefined && logger.error("No dictionary key for msg 'name': ", msg) this.stringDict[msg.valueKey] === undefined && logger.error("No dictionary key for msg 'value': ", msg) if (this.stringDict[msg.nameKey] === undefined || this.stringDict[msg.valueKey] === undefined ) { return } - this.setNodeAttribute({ - id: msg.id, - name: this.stringDict[msg.nameKey], - value: this.stringDict[msg.valueKey], - }) + if (isJump && this.stringDict[msg.nameKey] === 'href') this.attrsBacktrack.push(msg) + else { + this.setNodeAttribute({ + id: msg.id, + name: this.stringDict[msg.nameKey], + value: this.stringDict[msg.valueKey], + }) + } return case MType.RemoveNodeAttribute: vn = this.vElements.get(msg.id) @@ -422,13 +432,53 @@ export default class DOMManager extends ListWalker { } } - async moveReady(t: number): Promise { + applyBacktrack(msg: Message) { + // @ts-ignore + const target = this.vElements.get(msg.id) + if (!target) { + return + } + + switch (msg.tp) { + case MType.SetNodeAttribute: { + this.setNodeAttribute(msg) + return + } + case MType.SetNodeAttributeDict: { + this.stringDict[msg.nameKey] === undefined && logger.error("No dictionary key for msg 'name': ", msg) + this.stringDict[msg.valueKey] === undefined && logger.error("No dictionary key for msg 'value': ", msg) + if (this.stringDict[msg.nameKey] === undefined || this.stringDict[msg.valueKey] === undefined) { + return + } + this.setNodeAttribute({ + id: msg.id, + name: this.stringDict[msg.nameKey], + value: this.stringDict[msg.valueKey], + }) + return; + } + } + } + + async moveReady(t: number, isJump?: boolean): Promise { // MBTODO (back jump optimisation): // - store intemediate virtual dom state // - cancel previous moveReady tasks (is it possible?) if new timestamp is less // This function autoresets pointer if necessary (better name?) - - await this.moveWait(t, this.applyMessage) + + /** + * Basically just skipping all set attribute with attrs being "href" if user is 'jumping' + * to the other point of replay to save time on NOT downloading any resources before the dom tree changes + * are applied, so it won't try to download and then cancel when node is created in msg N and removed in msg N+2 + * which produces weird bug when asset is cached (10-25ms delay) + * */ + await this.moveWait(t, (msg) => this.applyMessage(msg, isJump)) + if (isJump) { + this.attrsBacktrack.forEach(msg => { + this.applyBacktrack(msg) + }) + this.attrsBacktrack = [] + } this.vRoots.forEach(rt => rt.applyChanges()) // MBTODO (optimisation): affected set // Thinkabout (read): css preload diff --git a/frontend/app/player/web/managers/DOM/VirtualDOM.ts b/frontend/app/player/web/managers/DOM/VirtualDOM.ts index 91b75eb24..d9fd3f77a 100644 --- a/frontend/app/player/web/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/web/managers/DOM/VirtualDOM.ts @@ -108,8 +108,8 @@ export class VElement extends VParent { } else { try { this.node.setAttribute(key, value) - } catch { - // log err + } catch (e) { + console.error(e) } } }) @@ -134,7 +134,8 @@ export class VStyleElement extends VElement { this.stylesheetCallbacks.forEach(cb => cb(sheet)) this.stylesheetCallbacks = [] } else { - console.warn("Style onload: sheet is null") + // console.warn("Style onload: sheet is null") ? + // sometimes logs shit ton of errors for some reason } this.loaded = true } diff --git a/frontend/app/player/web/managers/PagesManager.ts b/frontend/app/player/web/managers/PagesManager.ts index dbc64bb72..b30f40372 100644 --- a/frontend/app/player/web/managers/PagesManager.ts +++ b/frontend/app/player/web/managers/PagesManager.ts @@ -33,14 +33,14 @@ export default class PagesManager extends ListWalker { this.forEach(page => page.sort(comparator)) } - moveReady(t: number): Promise { + moveReady(t: number, isJump?: boolean): Promise { const requiredPage = this.moveGetLast(t) if (requiredPage != null) { this.currentPage = requiredPage this.currentPage.reset() // Otherwise it won't apply create_document } if (this.currentPage != null) { - return this.currentPage.moveReady(t) + return this.currentPage.moveReady(t, isJump) } return Promise.resolve() } diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index c22de968a..7705d49c9 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "5.0.1", + "version": "5.0.2-beta.2", "keywords": [ "logging", "replay" From 788a70fc62a7f1c9b7505daf17eebafaea6f5e98 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 21 Mar 2023 16:33:13 +0100 Subject: [PATCH 126/253] feat(peers): upgraded dependencies --- peers/package-lock.json | 8 ++++---- peers/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/peers/package-lock.json b/peers/package-lock.json index b18dca820..fd230847f 100644 --- a/peers/package-lock.json +++ b/peers/package-lock.json @@ -10,7 +10,7 @@ "license": "Elastic License 2.0 (ELv2)", "dependencies": { "express": "^4.18.2", - "peer": "^v1.0.0-rc.9" + "peer": "^v1.0.0" } }, "node_modules/@types/body-parser": { @@ -57,9 +57,9 @@ "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" }, "node_modules/@types/node": { - "version": "18.15.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.1.tgz", - "integrity": "sha512-U2TWca8AeHSmbpi314QBESRk7oPjSZjDsR+c+H4ECC1l+kFgpZf8Ydhv3SJpPy51VyZHHqxlb6mTTqYNNRVAIw==" + "version": "18.15.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.5.tgz", + "integrity": "sha512-Ark2WDjjZO7GmvsyFFf81MXuGTA/d6oP38anyxWOL6EREyBKAxKoFHwBhaZxCfLRLpO8JgVXwqOwSwa7jRcjew==" }, "node_modules/@types/qs": { "version": "6.9.7", diff --git a/peers/package.json b/peers/package.json index 82fd0ddf6..d77cf5910 100644 --- a/peers/package.json +++ b/peers/package.json @@ -19,6 +19,6 @@ "homepage": "https://github.com/openreplay/openreplay#readme", "dependencies": { "express": "^4.18.2", - "peer": "^v1.0.0-rc.9" + "peer": "^v1.0.0" } } From d86b0939f7f808a3f053637cb64e6fe1cab75384 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 21 Mar 2023 16:40:45 +0100 Subject: [PATCH 127/253] feat(peers): changed build script --- peers/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/peers/build.sh b/peers/build.sh index 746a12f9d..0f01a292f 100644 --- a/peers/build.sh +++ b/peers/build.sh @@ -23,6 +23,7 @@ function build_api(){ cp -R ../peers ../${destination} cd ../${destination} cp -R ../assist/utils . + cp ../sourcemap-reader/utils/health.js ./utils/. # Copy enterprise code [[ $1 == "ee" ]] && { cp -rf ../ee/peers/* ./ From 853ac7c277c15140da1e02024e03a90e7458e850 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 21 Mar 2023 17:18:31 +0100 Subject: [PATCH 128/253] feat(backend): added CH creds support --- ee/backend/pkg/db/clickhouse/connector.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 0ee0658ef..12ed1fb3f 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -10,6 +10,7 @@ import ( "openreplay/backend/pkg/hashid" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/url" + "os" "strings" "time" @@ -52,14 +53,25 @@ type connectorImpl struct { finished chan struct{} } +func getEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + func NewConnector(url string) Connector { license.CheckLicense() url = strings.TrimPrefix(url, "tcp://") url = strings.TrimSuffix(url, "/default") + userName := getEnv("CH_USERNAME", "default") + password := getEnv("CH_PASSWORD", "") conn, err := clickhouse.Open(&clickhouse.Options{ Addr: []string{url}, Auth: clickhouse.Auth{ Database: "default", + Username: userName, + Password: password, }, MaxOpenConns: 20, MaxIdleConns: 15, @@ -67,7 +79,6 @@ func NewConnector(url string) Connector { Compression: &clickhouse.Compression{ Method: clickhouse.CompressionLZ4, }, - // Debug: true, }) if err != nil { log.Fatal(err) From 5cf662240b10289f2e127f231d03856c2be4111e Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 22 Mar 2023 09:29:11 +0100 Subject: [PATCH 129/253] chore(cli): Adding edit option for cli --- scripts/helmcharts/openreplay-cli | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index 4a1b473d7..0c8aaad74 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -118,6 +118,7 @@ log info ' [ -U | --deprecated-upgrade /path/to/old_vars.yaml] [ -r | --restart ] [ -R | --Reload ] + [ -e | --edit ] [ -p | --install-packages ] [ -l | --logs SERVICE ] Services: alerts assets assist chalice @@ -236,7 +237,7 @@ function clean_tmp_dir() { install_packages } -PARSED_ARGUMENTS=$(busybox getopt -a -n openreplay -o Rrvpiuhsl:U: --long reload,restart,verbose,install-packages,install,upgrade,help,status,logs,deprecated-upgrade: -- "$@") +PARSED_ARGUMENTS=$(busybox getopt -a -n openreplay -o Rrevpiuhsl:U: --long reload,edit,restart,verbose,install-packages,install,upgrade,help,status,logs,deprecated-upgrade: -- "$@") VALID_ARGUMENTS=$? if [[ "$VALID_ARGUMENTS" != "0" ]]; then help @@ -278,6 +279,17 @@ do clean_tmp_dir exit 0 ;; + -e | --edit) + log title "Editing OpenReplay" + sudo vim -n /var/lib/openreplay/vars.yaml + /var/lib/openreplay/yq 'true' /var/lib/openreplay/vars.yaml || { + log debug "seems like the edit is not correct. Rerun ${BWHITE}openreplay -e${YELLOW} and fix the issue." + exit 100 + } + reload + clean_tmp_dir + exit 0 + ;; -s | --status) log title "Checking OpenReplay Components Status" status From 01723fa856a5cc7c019e6ffc1e3cc3e0315624b3 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 14 Feb 2023 12:43:16 +0100 Subject: [PATCH 130/253] change(tracker): detect if resource is loaded from cache --- .../DevTools/NetworkPanel/NetworkPanel.tsx | 18 ++++++++++++++- mobs/run.rb | 1 - tracker/tracker/src/main/modules/network.ts | 22 +++++++++++++++++-- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx index b7207ddef..38ebdf365 100644 --- a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx +++ b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx @@ -128,6 +128,21 @@ export function renderDuration(r: any) { ); } +function renderStatus({ status }) { + + return ( + <> + {parseInt(status, 10) === 200 ? ( + +
+ {status} + +
+ + ) : status} + + ) +} function NetworkPanel({ startedAt }: { startedAt: number }) { const { player, store } = React.useContext(PlayerContext) @@ -348,7 +363,8 @@ function NetworkPanel({ startedAt }: { startedAt: number }) { { label: 'Status', dataKey: 'status', - width: 70, + width: 90, + render: renderStatus, }, { label: 'Type', diff --git a/mobs/run.rb b/mobs/run.rb index 8d481e3b3..398068f95 100644 --- a/mobs/run.rb +++ b/mobs/run.rb @@ -113,7 +113,6 @@ $ids = [] $messages = [] def message(id, name, opts = {}, &block) raise "id duplicated #{name}" if $ids.include? id - raise "id is too big #{name}" if id > 127 $ids << id opts[:id] = id opts[:name] = name diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts index fdaed16c6..a85a0bb1b 100644 --- a/tracker/tracker/src/main/modules/network.ts +++ b/tracker/tracker/src/main/modules/network.ts @@ -35,6 +35,16 @@ type FetchRequestBody = RequestInit['body'] // } // } +function checkCacheByPerformanceTimings(requestUrl: string) { + if (performance) { + const timings = performance.getEntriesByName(requestUrl)[0] + // @ts-ignore - weird ts typings, please refer to https://developer.mozilla.org/en-US/docs/Web/API/PerformanceNavigationTiming + if (timings.transferSize === 0 || timings.responseStart - timings.requestStart < 10) { + return true + } + } else return false +} + interface RequestData { body: XHRRequestBody | FetchRequestBody headers: Record @@ -208,6 +218,10 @@ export default function (app: App, opts: Partial = {}) { return } + const isCached = + r.status === 304 || + reqHs['x-cache'].includes('Hit') || + checkCacheByPerformanceTimings(reqResInfo.url) app.send( NetworkRequest( 'fetch', @@ -215,7 +229,7 @@ export default function (app: App, opts: Partial = {}) { String(reqResInfo.url), stringify(reqResInfo.request), stringify(reqResInfo.response), - r.status, + isCached ? 304 : r.status, startTime + getTimeOrigin(), duration, ), @@ -274,6 +288,10 @@ export default function (app: App, opts: Partial = {}) { return } + const isCached = + xhr.status === 304 || + reqHs['x-cache'].includes('Hit') || + (xhr.status < 400 && checkCacheByPerformanceTimings(reqResInfo.url)) app.send( NetworkRequest( 'xhr', @@ -281,7 +299,7 @@ export default function (app: App, opts: Partial = {}) { String(reqResInfo.url), stringify(reqResInfo.request), stringify(reqResInfo.response), - xhr.status, + isCached ? 304 : xhr.status, startTime + getTimeOrigin(), duration, ), From e5a93d4ca0e1bfae91a05e11dcdd6a43232f907b Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Tue, 14 Feb 2023 14:04:31 +0100 Subject: [PATCH 131/253] feat(backend): reader maintains multibyte type --- backend/pkg/messages/primitives.go | 10 ++++++++++ backend/pkg/messages/reader.go | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/backend/pkg/messages/primitives.go b/backend/pkg/messages/primitives.go index 3e47a3943..921239287 100644 --- a/backend/pkg/messages/primitives.go +++ b/backend/pkg/messages/primitives.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "math" ) var ( @@ -50,6 +51,15 @@ func WriteUint(v uint64, buf []byte, p int) int { return p + 1 } +func ByteSizeUint(v uint64) int { + if v == 0 { + return 1 + } + nBits := math.Floor(math.Log2(float64(v))) + 1 + nBytes := math.Ceil(nBits / 7) + return int(nBytes) +} + func ReadInt(reader io.Reader) (int64, error) { ux, err := ReadUint(reader) x := int64(ux >> 1) diff --git a/backend/pkg/messages/reader.go b/backend/pkg/messages/reader.go index 1e9fa42db..436876e55 100644 --- a/backend/pkg/messages/reader.go +++ b/backend/pkg/messages/reader.go @@ -70,13 +70,15 @@ func (m *messageReaderImpl) Parse() (err error) { } // Dirty hack to avoid extra memory allocation - m.data[curr-1] = uint8(m.msgType) + mTypeByteSize := ByteSizeUint(m.msgType) + from := int(curr) - mTypeByteSize + WriteUint(m.msgType, m.data, from) // Add message meta to list m.list = append(m.list, &MessageMeta{ msgType: m.msgType, msgSize: m.msgSize + 1, - msgFrom: uint64(curr - 1), + msgFrom: uint64(from), }) // Update data pointer From 1afd130c901e836c71e02d7fe22d050cacde92c8 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 14 Feb 2023 14:42:03 +0100 Subject: [PATCH 132/253] change(ui): fixes for type --- .../components/shared/DevTools/NetworkPanel/NetworkPanel.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx index 38ebdf365..dbd25cede 100644 --- a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx +++ b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx @@ -128,8 +128,7 @@ export function renderDuration(r: any) { ); } -function renderStatus({ status }) { - +function renderStatus({ status }: { status: string }) { return ( <> {parseInt(status, 10) === 200 ? ( @@ -143,6 +142,7 @@ function renderStatus({ status }) { ) } + function NetworkPanel({ startedAt }: { startedAt: number }) { const { player, store } = React.useContext(PlayerContext) From 2f1b2330701c80340b6be97a0a5d5a371fcbb481 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 14 Feb 2023 14:46:54 +0100 Subject: [PATCH 133/253] change(tracker/player): create new network msg for cached req --- backend/pkg/messages/messages.go | 48 +++++++++++++++++-- backend/pkg/messages/read-message.go | 41 ++++++++++++++-- ee/connectors/msgcodec/messages.py | 17 ++++++- ee/connectors/msgcodec/msgcodec.py | 15 +++++- .../DevTools/NetworkPanel/NetworkPanel.tsx | 4 +- mobs/messages.rb | 14 +++++- tracker/tracker/src/main/modules/network.ts | 13 ++--- 7 files changed, 132 insertions(+), 20 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 20410182a..a77486a52 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -22,7 +22,7 @@ const ( MsgSetInputValue = 18 MsgSetInputChecked = 19 MsgMouseMove = 20 - MsgNetworkRequest = 21 + MsgLegacyNetworkRequest = 21 MsgConsoleLog = 22 MsgPageLoadTiming = 23 MsgPageRenderTiming = 24 @@ -83,6 +83,7 @@ const ( MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 + MsgNetworkRequest = 128 MsgIOSBatchMeta = 107 MsgIOSSessionStart = 90 MsgIOSSessionEnd = 91 @@ -601,7 +602,7 @@ func (msg *MouseMove) TypeID() int { return 20 } -type NetworkRequest struct { +type LegacyNetworkRequest struct { message Type string Method string @@ -613,7 +614,7 @@ type NetworkRequest struct { Duration uint64 } -func (msg *NetworkRequest) Encode() []byte { +func (msg *LegacyNetworkRequest) Encode() []byte { buf := make([]byte, 81+len(msg.Type)+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response)) buf[0] = 21 p := 1 @@ -628,11 +629,11 @@ func (msg *NetworkRequest) Encode() []byte { return buf[:p] } -func (msg *NetworkRequest) Decode() Message { +func (msg *LegacyNetworkRequest) Decode() Message { return msg } -func (msg *NetworkRequest) TypeID() int { +func (msg *LegacyNetworkRequest) TypeID() int { return 21 } @@ -2202,6 +2203,43 @@ func (msg *SessionSearch) TypeID() int { return 127 } +type NetworkRequest struct { + message + Type string + Method string + URL string + Request string + Response string + Status uint64 + Timestamp uint64 + Duration uint64 + Cached bool +} + +func (msg *NetworkRequest) Encode() []byte { + buf := make([]byte, 91+len(msg.Type)+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response)) + buf[0] = 128 + p := 1 + p = WriteString(msg.Type, buf, p) + p = WriteString(msg.Method, buf, p) + p = WriteString(msg.URL, buf, p) + p = WriteString(msg.Request, buf, p) + p = WriteString(msg.Response, buf, p) + p = WriteUint(msg.Status, buf, p) + p = WriteUint(msg.Timestamp, buf, p) + p = WriteUint(msg.Duration, buf, p) + p = WriteBoolean(msg.Cached, buf, p) + return buf[:p] +} + +func (msg *NetworkRequest) Decode() Message { + return msg +} + +func (msg *NetworkRequest) TypeID() int { + return 128 +} + type IOSBatchMeta struct { message Timestamp uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index f3bc525a9..e7968dd6c 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -300,9 +300,9 @@ func DecodeMouseMove(reader BytesReader) (Message, error) { return msg, err } -func DecodeNetworkRequest(reader BytesReader) (Message, error) { +func DecodeLegacyNetworkRequest(reader BytesReader) (Message, error) { var err error = nil - msg := &NetworkRequest{} + msg := &LegacyNetworkRequest{} if msg.Type, err = reader.ReadString(); err != nil { return nil, err } @@ -1329,6 +1329,39 @@ func DecodeSessionSearch(reader BytesReader) (Message, error) { return msg, err } +func DecodeNetworkRequest(reader BytesReader) (Message, error) { + var err error = nil + msg := &NetworkRequest{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Method, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Request, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Response, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Status, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Duration, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Cached, err = reader.ReadBoolean(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIOSBatchMeta(reader BytesReader) (Message, error) { var err error = nil msg := &IOSBatchMeta{} @@ -1774,7 +1807,7 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { case 20: return DecodeMouseMove(reader) case 21: - return DecodeNetworkRequest(reader) + return DecodeLegacyNetworkRequest(reader) case 22: return DecodeConsoleLog(reader) case 23: @@ -1895,6 +1928,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeSessionEnd(reader) case 127: return DecodeSessionSearch(reader) + case 128: + return DecodeNetworkRequest(reader) case 107: return DecodeIOSBatchMeta(reader) case 90: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 7c2638ec6..d189c7e2a 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -185,7 +185,7 @@ class MouseMove(Message): self.y = y -class NetworkRequest(Message): +class LegacyNetworkRequest(Message): __id__ = 21 def __init__(self, type, method, url, request, response, status, timestamp, duration): @@ -772,6 +772,21 @@ class SessionSearch(Message): self.partition = partition +class NetworkRequest(Message): + __id__ = 128 + + def __init__(self, type, method, url, request, response, status, timestamp, duration, cached): + self.type = type + self.method = method + self.url = url + self.request = request + self.response = response + self.status = status + self.timestamp = timestamp + self.duration = duration + self.cached = cached + + class IOSBatchMeta(Message): __id__ = 107 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index df6133cbb..bdf81ed96 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -216,7 +216,7 @@ class MessageCodec(Codec): ) if message_id == 21: - return NetworkRequest( + return LegacyNetworkRequest( type=self.read_string(reader), method=self.read_string(reader), url=self.read_string(reader), @@ -680,6 +680,19 @@ class MessageCodec(Codec): partition=self.read_uint(reader) ) + if message_id == 128: + return NetworkRequest( + type=self.read_string(reader), + method=self.read_string(reader), + url=self.read_string(reader), + request=self.read_string(reader), + response=self.read_string(reader), + status=self.read_uint(reader), + timestamp=self.read_uint(reader), + duration=self.read_uint(reader), + cached=self.read_boolean(reader) + ) + if message_id == 107: return IOSBatchMeta( timestamp=self.read_uint(reader), diff --git a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx index dbd25cede..9f7e9470a 100644 --- a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx +++ b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx @@ -128,10 +128,10 @@ export function renderDuration(r: any) { ); } -function renderStatus({ status }: { status: string }) { +function renderStatus({ status, cached }: { status: string, cached: boolean }) { return ( <> - {parseInt(status, 10) === 200 ? ( + {cached ? (
{status} diff --git a/mobs/messages.rb b/mobs/messages.rb index a107d5dcc..4c68918dc 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -104,7 +104,7 @@ message 20, 'MouseMove' do uint 'X' uint 'Y' end -message 21, 'NetworkRequest', :replayer => :devtools do +message 21, 'LegacyNetworkRequest', :replayer => :devtools do string 'Type' # fetch/xhr/anythingElse(axios,gql,fonts,image?) string 'Method' string 'URL' @@ -490,4 +490,14 @@ message 127, 'SessionSearch', :tracker => false, :replayer => false do uint 'Partition' end -# since tracker 4.1.10 +message 128, 'NetworkRequest', :replayer => :devtools do + string 'Type' # fetch/xhr/anythingElse(axios,gql,fonts,image?) + string 'Method' + string 'URL' + string 'Request' + string 'Response' + uint 'Status' + uint 'Timestamp' + uint 'Duration' + boolean 'Cached' +end \ No newline at end of file diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts index a85a0bb1b..eddad5873 100644 --- a/tracker/tracker/src/main/modules/network.ts +++ b/tracker/tracker/src/main/modules/network.ts @@ -39,10 +39,9 @@ function checkCacheByPerformanceTimings(requestUrl: string) { if (performance) { const timings = performance.getEntriesByName(requestUrl)[0] // @ts-ignore - weird ts typings, please refer to https://developer.mozilla.org/en-US/docs/Web/API/PerformanceNavigationTiming - if (timings.transferSize === 0 || timings.responseStart - timings.requestStart < 10) { - return true - } - } else return false + return timings.transferSize === 0 || timings.responseStart - timings.requestStart < 10 + } + return false } interface RequestData { @@ -229,9 +228,10 @@ export default function (app: App, opts: Partial = {}) { String(reqResInfo.url), stringify(reqResInfo.request), stringify(reqResInfo.response), - isCached ? 304 : r.status, + r.status, startTime + getTimeOrigin(), duration, + isCached, ), ) }) @@ -299,9 +299,10 @@ export default function (app: App, opts: Partial = {}) { String(reqResInfo.url), stringify(reqResInfo.request), stringify(reqResInfo.response), - isCached ? 304 : xhr.status, + xhr.status, startTime + getTimeOrigin(), duration, + isCached, ), ) }), From 89ec4b67f114eebd12ce2ef2a0704a767954424c Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Mar 2023 16:50:04 +0100 Subject: [PATCH 134/253] change(tracker): regen messages; add trans size and isCache to resource timing message, remove x-cached header detection --- backend/pkg/messages/messages.go | 126 ++++++++++++------ backend/pkg/messages/read-message.go | 114 ++++++++++------ ee/connectors/msgcodec/messages.py | 48 ++++--- ee/connectors/msgcodec/msgcodec.py | 42 ++++-- .../web/messages/RawMessageReader.gen.ts | 54 +++++++- .../app/player/web/messages/message.gen.ts | 14 +- frontend/app/player/web/messages/raw.gen.ts | 43 +++++- .../player/web/messages/tracker-legacy.gen.ts | 6 +- .../app/player/web/messages/tracker.gen.ts | 68 +++++++++- mobs/messages.rb | 41 ++++-- tracker/tracker/src/common/messages.gen.ts | 43 +++++- tracker/tracker/src/main/app/messages.gen.ts | 64 ++++++++- tracker/tracker/src/main/modules/network.ts | 9 +- tracker/tracker/src/main/modules/timing.ts | 3 + .../src/webworker/MessageEncoder.gen.ts | 12 +- 15 files changed, 521 insertions(+), 166 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index a77486a52..4fb53be08 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -48,7 +48,7 @@ const ( MsgPerformanceTrack = 49 MsgStringDict = 50 MsgSetNodeAttributeDict = 51 - MsgResourceTiming = 53 + MsgResourceTimingLegacy = 53 MsgConnectionInformation = 54 MsgSetPageVisibility = 55 MsgPerformanceTrackAggr = 56 @@ -80,10 +80,11 @@ const ( MsgSelectionChange = 113 MsgMouseThrashing = 114 MsgUnbindNodes = 115 + MsgResourceTiming = 116 + MsgNetworkRequest = 117 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 - MsgNetworkRequest = 128 MsgIOSBatchMeta = 107 MsgIOSSessionStart = 90 MsgIOSSessionEnd = 91 @@ -1296,7 +1297,7 @@ func (msg *SetNodeAttributeDict) TypeID() int { return 51 } -type ResourceTiming struct { +type ResourceTimingLegacy struct { message Timestamp uint64 Duration uint64 @@ -1308,7 +1309,7 @@ type ResourceTiming struct { Initiator string } -func (msg *ResourceTiming) Encode() []byte { +func (msg *ResourceTimingLegacy) Encode() []byte { buf := make([]byte, 81+len(msg.URL)+len(msg.Initiator)) buf[0] = 53 p := 1 @@ -1323,11 +1324,11 @@ func (msg *ResourceTiming) Encode() []byte { return buf[:p] } -func (msg *ResourceTiming) Decode() Message { +func (msg *ResourceTimingLegacy) Decode() Message { return msg } -func (msg *ResourceTiming) TypeID() int { +func (msg *ResourceTimingLegacy) TypeID() int { return 53 } @@ -2124,6 +2125,82 @@ func (msg *UnbindNodes) TypeID() int { return 115 } +type ResourceTiming struct { + message + Timestamp uint64 + Duration uint64 + TTFB uint64 + HeaderSize uint64 + EncodedBodySize uint64 + DecodedBodySize uint64 + URL string + Initiator string + TransferredSize uint64 + Cached bool +} + +func (msg *ResourceTiming) Encode() []byte { + buf := make([]byte, 101+len(msg.URL)+len(msg.Initiator)) + buf[0] = 116 + p := 1 + p = WriteUint(msg.Timestamp, buf, p) + p = WriteUint(msg.Duration, buf, p) + p = WriteUint(msg.TTFB, buf, p) + p = WriteUint(msg.HeaderSize, buf, p) + p = WriteUint(msg.EncodedBodySize, buf, p) + p = WriteUint(msg.DecodedBodySize, buf, p) + p = WriteString(msg.URL, buf, p) + p = WriteString(msg.Initiator, buf, p) + p = WriteUint(msg.TransferredSize, buf, p) + p = WriteBoolean(msg.Cached, buf, p) + return buf[:p] +} + +func (msg *ResourceTiming) Decode() Message { + return msg +} + +func (msg *ResourceTiming) TypeID() int { + return 116 +} + +type NetworkRequest struct { + message + Type string + Method string + URL string + Request string + Response string + Status uint64 + Timestamp uint64 + Duration uint64 + Cached bool +} + +func (msg *NetworkRequest) Encode() []byte { + buf := make([]byte, 91+len(msg.Type)+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response)) + buf[0] = 117 + p := 1 + p = WriteString(msg.Type, buf, p) + p = WriteString(msg.Method, buf, p) + p = WriteString(msg.URL, buf, p) + p = WriteString(msg.Request, buf, p) + p = WriteString(msg.Response, buf, p) + p = WriteUint(msg.Status, buf, p) + p = WriteUint(msg.Timestamp, buf, p) + p = WriteUint(msg.Duration, buf, p) + p = WriteBoolean(msg.Cached, buf, p) + return buf[:p] +} + +func (msg *NetworkRequest) Decode() Message { + return msg +} + +func (msg *NetworkRequest) TypeID() int { + return 117 +} + type IssueEvent struct { message MessageID uint64 @@ -2203,43 +2280,6 @@ func (msg *SessionSearch) TypeID() int { return 127 } -type NetworkRequest struct { - message - Type string - Method string - URL string - Request string - Response string - Status uint64 - Timestamp uint64 - Duration uint64 - Cached bool -} - -func (msg *NetworkRequest) Encode() []byte { - buf := make([]byte, 91+len(msg.Type)+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response)) - buf[0] = 128 - p := 1 - p = WriteString(msg.Type, buf, p) - p = WriteString(msg.Method, buf, p) - p = WriteString(msg.URL, buf, p) - p = WriteString(msg.Request, buf, p) - p = WriteString(msg.Response, buf, p) - p = WriteUint(msg.Status, buf, p) - p = WriteUint(msg.Timestamp, buf, p) - p = WriteUint(msg.Duration, buf, p) - p = WriteBoolean(msg.Cached, buf, p) - return buf[:p] -} - -func (msg *NetworkRequest) Decode() Message { - return msg -} - -func (msg *NetworkRequest) TypeID() int { - return 128 -} - type IOSBatchMeta struct { message Timestamp uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index e7968dd6c..adb80d2ed 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -756,9 +756,9 @@ func DecodeSetNodeAttributeDict(reader BytesReader) (Message, error) { return msg, err } -func DecodeResourceTiming(reader BytesReader) (Message, error) { +func DecodeResourceTimingLegacy(reader BytesReader) (Message, error) { var err error = nil - msg := &ResourceTiming{} + msg := &ResourceTimingLegacy{} if msg.Timestamp, err = reader.ReadUint(); err != nil { return nil, err } @@ -1278,6 +1278,75 @@ func DecodeUnbindNodes(reader BytesReader) (Message, error) { return msg, err } +func DecodeResourceTiming(reader BytesReader) (Message, error) { + var err error = nil + msg := &ResourceTiming{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Duration, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.TTFB, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.HeaderSize, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.EncodedBodySize, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.DecodedBodySize, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Initiator, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.TransferredSize, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Cached, err = reader.ReadBoolean(); err != nil { + return nil, err + } + return msg, err +} + +func DecodeNetworkRequest(reader BytesReader) (Message, error) { + var err error = nil + msg := &NetworkRequest{} + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Method, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Request, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Response, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Status, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Duration, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Cached, err = reader.ReadBoolean(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1329,39 +1398,6 @@ func DecodeSessionSearch(reader BytesReader) (Message, error) { return msg, err } -func DecodeNetworkRequest(reader BytesReader) (Message, error) { - var err error = nil - msg := &NetworkRequest{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Request, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Response, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Cached, err = reader.ReadBoolean(); err != nil { - return nil, err - } - return msg, err -} - func DecodeIOSBatchMeta(reader BytesReader) (Message, error) { var err error = nil msg := &IOSBatchMeta{} @@ -1859,7 +1895,7 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { case 51: return DecodeSetNodeAttributeDict(reader) case 53: - return DecodeResourceTiming(reader) + return DecodeResourceTimingLegacy(reader) case 54: return DecodeConnectionInformation(reader) case 55: @@ -1922,14 +1958,16 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeMouseThrashing(reader) case 115: return DecodeUnbindNodes(reader) + case 116: + return DecodeResourceTiming(reader) + case 117: + return DecodeNetworkRequest(reader) case 125: return DecodeIssueEvent(reader) case 126: return DecodeSessionEnd(reader) case 127: return DecodeSessionSearch(reader) - case 128: - return DecodeNetworkRequest(reader) case 107: return DecodeIOSBatchMeta(reader) case 90: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index d189c7e2a..4e384cb03 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -441,7 +441,7 @@ class SetNodeAttributeDict(Message): self.value_key = value_key -class ResourceTiming(Message): +class ResourceTimingLegacy(Message): __id__ = 53 def __init__(self, timestamp, duration, ttfb, header_size, encoded_body_size, decoded_body_size, url, initiator): @@ -743,6 +743,37 @@ class UnbindNodes(Message): self.total_removed_percent = total_removed_percent +class ResourceTiming(Message): + __id__ = 116 + + def __init__(self, timestamp, duration, ttfb, header_size, encoded_body_size, decoded_body_size, url, initiator, transferred_size, cached): + self.timestamp = timestamp + self.duration = duration + self.ttfb = ttfb + self.header_size = header_size + self.encoded_body_size = encoded_body_size + self.decoded_body_size = decoded_body_size + self.url = url + self.initiator = initiator + self.transferred_size = transferred_size + self.cached = cached + + +class NetworkRequest(Message): + __id__ = 117 + + def __init__(self, type, method, url, request, response, status, timestamp, duration, cached): + self.type = type + self.method = method + self.url = url + self.request = request + self.response = response + self.status = status + self.timestamp = timestamp + self.duration = duration + self.cached = cached + + class IssueEvent(Message): __id__ = 125 @@ -772,21 +803,6 @@ class SessionSearch(Message): self.partition = partition -class NetworkRequest(Message): - __id__ = 128 - - def __init__(self, type, method, url, request, response, status, timestamp, duration, cached): - self.type = type - self.method = method - self.url = url - self.request = request - self.response = response - self.status = status - self.timestamp = timestamp - self.duration = duration - self.cached = cached - - class IOSBatchMeta(Message): __id__ = 107 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index bdf81ed96..d4af91836 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -420,7 +420,7 @@ class MessageCodec(Codec): ) if message_id == 53: - return ResourceTiming( + return ResourceTimingLegacy( timestamp=self.read_uint(reader), duration=self.read_uint(reader), ttfb=self.read_uint(reader), @@ -657,6 +657,33 @@ class MessageCodec(Codec): total_removed_percent=self.read_uint(reader) ) + if message_id == 116: + return ResourceTiming( + timestamp=self.read_uint(reader), + duration=self.read_uint(reader), + ttfb=self.read_uint(reader), + header_size=self.read_uint(reader), + encoded_body_size=self.read_uint(reader), + decoded_body_size=self.read_uint(reader), + url=self.read_string(reader), + initiator=self.read_string(reader), + transferred_size=self.read_uint(reader), + cached=self.read_boolean(reader) + ) + + if message_id == 117: + return NetworkRequest( + type=self.read_string(reader), + method=self.read_string(reader), + url=self.read_string(reader), + request=self.read_string(reader), + response=self.read_string(reader), + status=self.read_uint(reader), + timestamp=self.read_uint(reader), + duration=self.read_uint(reader), + cached=self.read_boolean(reader) + ) + if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), @@ -680,19 +707,6 @@ class MessageCodec(Codec): partition=self.read_uint(reader) ) - if message_id == 128: - return NetworkRequest( - type=self.read_string(reader), - method=self.read_string(reader), - url=self.read_string(reader), - request=self.read_string(reader), - response=self.read_string(reader), - status=self.read_uint(reader), - timestamp=self.read_uint(reader), - duration=self.read_uint(reader), - cached=self.read_boolean(reader) - ) - if message_id == 107: return IOSBatchMeta( timestamp=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 1d0e0e7cd..72d6e2ff9 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -211,7 +211,7 @@ export default class RawMessageReader extends PrimitiveReader { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const duration = this.readUint(); if (duration === null) { return resetPointer() } return { - tp: MType.NetworkRequest, + tp: MType.LegacyNetworkRequest, type, method, url, @@ -403,7 +403,7 @@ export default class RawMessageReader extends PrimitiveReader { const url = this.readString(); if (url === null) { return resetPointer() } const initiator = this.readString(); if (initiator === null) { return resetPointer() } return { - tp: MType.ResourceTiming, + tp: MType.ResourceTimingLegacy, timestamp, duration, ttfb, @@ -647,6 +647,56 @@ export default class RawMessageReader extends PrimitiveReader { }; } + case 116: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + const ttfb = this.readUint(); if (ttfb === null) { return resetPointer() } + const headerSize = this.readUint(); if (headerSize === null) { return resetPointer() } + const encodedBodySize = this.readUint(); if (encodedBodySize === null) { return resetPointer() } + const decodedBodySize = this.readUint(); if (decodedBodySize === null) { return resetPointer() } + const url = this.readString(); if (url === null) { return resetPointer() } + const initiator = this.readString(); if (initiator === null) { return resetPointer() } + const transferredSize = this.readUint(); if (transferredSize === null) { return resetPointer() } + const cached = this.readBoolean(); if (cached === null) { return resetPointer() } + return { + tp: MType.ResourceTiming, + timestamp, + duration, + ttfb, + headerSize, + encodedBodySize, + decodedBodySize, + url, + initiator, + transferredSize, + cached, + }; + } + + case 117: { + const type = this.readString(); if (type === null) { return resetPointer() } + const method = this.readString(); if (method === null) { return resetPointer() } + const url = this.readString(); if (url === null) { return resetPointer() } + const request = this.readString(); if (request === null) { return resetPointer() } + const response = this.readString(); if (response === null) { return resetPointer() } + const status = this.readUint(); if (status === null) { return resetPointer() } + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + const cached = this.readBoolean(); if (cached === null) { return resetPointer() } + return { + tp: MType.NetworkRequest, + type, + method, + url, + request, + response, + status, + timestamp, + duration, + cached, + }; + } + case 90: { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const projectID = this.readUint(); if (projectID === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index e39d02584..4a1474f23 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -21,7 +21,7 @@ import type { RawSetInputValue, RawSetInputChecked, RawMouseMove, - RawNetworkRequest, + RawLegacyNetworkRequest, RawConsoleLog, RawCssInsertRule, RawCssDeleteRule, @@ -36,7 +36,7 @@ import type { RawPerformanceTrack, RawStringDict, RawSetNodeAttributeDict, - RawResourceTiming, + RawResourceTimingLegacy, RawConnectionInformation, RawSetPageVisibility, RawLoadFontFace, @@ -57,6 +57,8 @@ import type { RawZustand, RawSelectionChange, RawMouseThrashing, + RawResourceTiming, + RawNetworkRequest, RawIosSessionStart, RawIosCustomEvent, RawIosScreenChanges, @@ -103,7 +105,7 @@ export type SetInputChecked = RawSetInputChecked & Timed export type MouseMove = RawMouseMove & Timed -export type NetworkRequest = RawNetworkRequest & Timed +export type LegacyNetworkRequest = RawLegacyNetworkRequest & Timed export type ConsoleLog = RawConsoleLog & Timed @@ -133,7 +135,7 @@ export type StringDict = RawStringDict & Timed export type SetNodeAttributeDict = RawSetNodeAttributeDict & Timed -export type ResourceTiming = RawResourceTiming & Timed +export type ResourceTimingLegacy = RawResourceTimingLegacy & Timed export type ConnectionInformation = RawConnectionInformation & Timed @@ -175,6 +177,10 @@ export type SelectionChange = RawSelectionChange & Timed export type MouseThrashing = RawMouseThrashing & Timed +export type ResourceTiming = RawResourceTiming & Timed + +export type NetworkRequest = RawNetworkRequest & Timed + export type IosSessionStart = RawIosSessionStart & Timed export type IosCustomEvent = RawIosCustomEvent & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 67f65ab35..86aba859c 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -19,7 +19,7 @@ export const enum MType { SetInputValue = 18, SetInputChecked = 19, MouseMove = 20, - NetworkRequest = 21, + LegacyNetworkRequest = 21, ConsoleLog = 22, CssInsertRule = 37, CssDeleteRule = 38, @@ -34,7 +34,7 @@ export const enum MType { PerformanceTrack = 49, StringDict = 50, SetNodeAttributeDict = 51, - ResourceTiming = 53, + ResourceTimingLegacy = 53, ConnectionInformation = 54, SetPageVisibility = 55, LoadFontFace = 57, @@ -55,6 +55,8 @@ export const enum MType { Zustand = 79, SelectionChange = 113, MouseThrashing = 114, + ResourceTiming = 116, + NetworkRequest = 117, IosSessionStart = 90, IosCustomEvent = 93, IosScreenChanges = 96, @@ -173,8 +175,8 @@ export interface RawMouseMove { y: number, } -export interface RawNetworkRequest { - tp: MType.NetworkRequest, +export interface RawLegacyNetworkRequest { + tp: MType.LegacyNetworkRequest, type: string, method: string, url: string, @@ -284,8 +286,8 @@ export interface RawSetNodeAttributeDict { valueKey: number, } -export interface RawResourceTiming { - tp: MType.ResourceTiming, +export interface RawResourceTimingLegacy { + tp: MType.ResourceTimingLegacy, timestamp: number, duration: number, ttfb: number, @@ -432,6 +434,33 @@ export interface RawMouseThrashing { timestamp: number, } +export interface RawResourceTiming { + tp: MType.ResourceTiming, + timestamp: number, + duration: number, + ttfb: number, + headerSize: number, + encodedBodySize: number, + decodedBodySize: number, + url: string, + initiator: string, + transferredSize: number, + cached: boolean, +} + +export interface RawNetworkRequest { + tp: MType.NetworkRequest, + type: string, + method: string, + url: string, + request: string, + response: string, + status: number, + timestamp: number, + duration: number, + cached: boolean, +} + export interface RawIosSessionStart { tp: MType.IosSessionStart, timestamp: number, @@ -503,4 +532,4 @@ export interface RawIosNetworkCall { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawMouseThrashing | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawLegacyNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingLegacy | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawNetworkRequest | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index ce69f34b3..236a33da5 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -20,7 +20,7 @@ export const TP_MAP = { 18: MType.SetInputValue, 19: MType.SetInputChecked, 20: MType.MouseMove, - 21: MType.NetworkRequest, + 21: MType.LegacyNetworkRequest, 22: MType.ConsoleLog, 37: MType.CssInsertRule, 38: MType.CssDeleteRule, @@ -35,7 +35,7 @@ export const TP_MAP = { 49: MType.PerformanceTrack, 50: MType.StringDict, 51: MType.SetNodeAttributeDict, - 53: MType.ResourceTiming, + 53: MType.ResourceTimingLegacy, 54: MType.ConnectionInformation, 55: MType.SetPageVisibility, 57: MType.LoadFontFace, @@ -56,6 +56,8 @@ export const TP_MAP = { 79: MType.Zustand, 113: MType.SelectionChange, 114: MType.MouseThrashing, + 116: MType.ResourceTiming, + 117: MType.NetworkRequest, 90: MType.IosSessionStart, 93: MType.IosCustomEvent, 96: MType.IosScreenChanges, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index f012084ec..31acd4fe7 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -113,7 +113,7 @@ type TrMouseMove = [ y: number, ] -type TrNetworkRequest = [ +type TrLegacyNetworkRequest = [ type: 21, type: string, method: string, @@ -271,7 +271,7 @@ type TrSetNodeAttributeDict = [ valueKey: number, ] -type TrResourceTiming = [ +type TrResourceTimingLegacy = [ type: 53, timestamp: number, duration: number, @@ -456,8 +456,35 @@ type TrUnbindNodes = [ totalRemovedPercent: number, ] +type TrResourceTiming = [ + type: 116, + timestamp: number, + duration: number, + ttfb: number, + headerSize: number, + encodedBodySize: number, + decodedBodySize: number, + url: string, + initiator: string, + transferredSize: number, + cached: boolean, +] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTiming | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes +type TrNetworkRequest = [ + type: 117, + type: string, + method: string, + url: string, + request: string, + response: string, + status: number, + timestamp: number, + duration: number, + cached: boolean, +] + + +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrLegacyNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingLegacy | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming | TrNetworkRequest export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -598,7 +625,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { case 21: { return { - tp: MType.NetworkRequest, + tp: MType.LegacyNetworkRequest, type: tMsg[1], method: tMsg[2], url: tMsg[3], @@ -739,7 +766,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { case 53: { return { - tp: MType.ResourceTiming, + tp: MType.ResourceTimingLegacy, timestamp: tMsg[1], duration: tMsg[2], ttfb: tMsg[3], @@ -910,6 +937,37 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } + case 116: { + return { + tp: MType.ResourceTiming, + timestamp: tMsg[1], + duration: tMsg[2], + ttfb: tMsg[3], + headerSize: tMsg[4], + encodedBodySize: tMsg[5], + decodedBodySize: tMsg[6], + url: tMsg[7], + initiator: tMsg[8], + transferredSize: tMsg[9], + cached: tMsg[10], + } + } + + case 117: { + return { + tp: MType.NetworkRequest, + type: tMsg[1], + method: tMsg[2], + url: tMsg[3], + request: tMsg[4], + response: tMsg[5], + status: tMsg[6], + timestamp: tMsg[7], + duration: tMsg[8], + cached: tMsg[9], + } + } + default: return null } diff --git a/mobs/messages.rb b/mobs/messages.rb index 4c68918dc..1f75a8b5e 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -265,7 +265,7 @@ message 51, "SetNodeAttributeDict" do uint 'NameKey' uint 'ValueKey' end -message 53, 'ResourceTiming', :replayer => :devtools do +message 53, 'ResourceTimingLegacy', :replayer => :devtools do uint 'Timestamp' uint 'Duration' uint 'TTFB' @@ -471,6 +471,33 @@ message 115, 'UnbindNodes', :replayer => false do uint 'TotalRemovedPercent' end +message 116, 'ResourceTiming', :replayer => :devtools do + uint 'Timestamp' + uint 'Duration' + uint 'TTFB' + uint 'HeaderSize' + uint 'EncodedBodySize' + uint 'DecodedBodySize' + string 'URL' + string 'Initiator' + uint 'TransferredSize' + boolean 'Cached' +end + +message 117, 'NetworkRequest', :replayer => :devtools do + string 'Type' # fetch/xhr/anythingElse(axios,gql,fonts,image?) + string 'Method' + string 'URL' + string 'Request' + string 'Response' + uint 'Status' + uint 'Timestamp' + uint 'Duration' + boolean 'Cached' +end + + + ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do uint 'MessageID' @@ -489,15 +516,3 @@ message 127, 'SessionSearch', :tracker => false, :replayer => false do uint 'Timestamp' uint 'Partition' end - -message 128, 'NetworkRequest', :replayer => :devtools do - string 'Type' # fetch/xhr/anythingElse(axios,gql,fonts,image?) - string 'Method' - string 'URL' - string 'Request' - string 'Response' - uint 'Status' - uint 'Timestamp' - uint 'Duration' - boolean 'Cached' -end \ No newline at end of file diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 6b4f5b1c3..8018810a4 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -19,7 +19,7 @@ export declare const enum Type { SetInputValue = 18, SetInputChecked = 19, MouseMove = 20, - NetworkRequest = 21, + LegacyNetworkRequest = 21, ConsoleLog = 22, PageLoadTiming = 23, PageRenderTiming = 24, @@ -41,7 +41,7 @@ export declare const enum Type { PerformanceTrack = 49, StringDict = 50, SetNodeAttributeDict = 51, - ResourceTiming = 53, + ResourceTimingLegacy = 53, ConnectionInformation = 54, SetPageVisibility = 55, LoadFontFace = 57, @@ -67,6 +67,8 @@ export declare const enum Type { SelectionChange = 113, MouseThrashing = 114, UnbindNodes = 115, + ResourceTiming = 116, + NetworkRequest = 117, } @@ -178,8 +180,8 @@ export type MouseMove = [ /*y:*/ number, ] -export type NetworkRequest = [ - /*type:*/ Type.NetworkRequest, +export type LegacyNetworkRequest = [ + /*type:*/ Type.LegacyNetworkRequest, /*type:*/ string, /*method:*/ string, /*url:*/ string, @@ -336,8 +338,8 @@ export type SetNodeAttributeDict = [ /*valueKey:*/ number, ] -export type ResourceTiming = [ - /*type:*/ Type.ResourceTiming, +export type ResourceTimingLegacy = [ + /*type:*/ Type.ResourceTimingLegacy, /*timestamp:*/ number, /*duration:*/ number, /*ttfb:*/ number, @@ -521,6 +523,33 @@ export type UnbindNodes = [ /*totalRemovedPercent:*/ number, ] +export type ResourceTiming = [ + /*type:*/ Type.ResourceTiming, + /*timestamp:*/ number, + /*duration:*/ number, + /*ttfb:*/ number, + /*headerSize:*/ number, + /*encodedBodySize:*/ number, + /*decodedBodySize:*/ number, + /*url:*/ string, + /*initiator:*/ string, + /*transferredSize:*/ number, + /*cached:*/ boolean, +] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTiming | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | UnbindNodes +export type NetworkRequest = [ + /*type:*/ Type.NetworkRequest, + /*type:*/ string, + /*method:*/ string, + /*url:*/ string, + /*request:*/ string, + /*response:*/ string, + /*status:*/ number, + /*timestamp:*/ number, + /*duration:*/ number, + /*cached:*/ boolean, +] + + +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | LegacyNetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingLegacy | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming | NetworkRequest export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index c522d719f..30332a5fe 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -204,7 +204,7 @@ export function MouseMove( ] } -export function NetworkRequest( +export function LegacyNetworkRequest( type: string, method: string, url: string, @@ -213,9 +213,9 @@ export function NetworkRequest( status: number, timestamp: number, duration: number, -): Messages.NetworkRequest { +): Messages.LegacyNetworkRequest { return [ - Messages.Type.NetworkRequest, + Messages.Type.LegacyNetworkRequest, type, method, url, @@ -498,7 +498,7 @@ export function SetNodeAttributeDict( ] } -export function ResourceTiming( +export function ResourceTimingLegacy( timestamp: number, duration: number, ttfb: number, @@ -507,9 +507,9 @@ export function ResourceTiming( decodedBodySize: number, url: string, initiator: string, -): Messages.ResourceTiming { +): Messages.ResourceTimingLegacy { return [ - Messages.Type.ResourceTiming, + Messages.Type.ResourceTimingLegacy, timestamp, duration, ttfb, @@ -842,3 +842,55 @@ export function UnbindNodes( ] } +export function ResourceTiming( + timestamp: number, + duration: number, + ttfb: number, + headerSize: number, + encodedBodySize: number, + decodedBodySize: number, + url: string, + initiator: string, + transferredSize: number, + cached: boolean, +): Messages.ResourceTiming { + return [ + Messages.Type.ResourceTiming, + timestamp, + duration, + ttfb, + headerSize, + encodedBodySize, + decodedBodySize, + url, + initiator, + transferredSize, + cached, + ] +} + +export function NetworkRequest( + type: string, + method: string, + url: string, + request: string, + response: string, + status: number, + timestamp: number, + duration: number, + cached: boolean, +): Messages.NetworkRequest { + return [ + Messages.Type.NetworkRequest, + type, + method, + url, + request, + response, + status, + timestamp, + duration, + cached, + ] +} + diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts index eddad5873..b0ad1453a 100644 --- a/tracker/tracker/src/main/modules/network.ts +++ b/tracker/tracker/src/main/modules/network.ts @@ -217,10 +217,7 @@ export default function (app: App, opts: Partial = {}) { return } - const isCached = - r.status === 304 || - reqHs['x-cache'].includes('Hit') || - checkCacheByPerformanceTimings(reqResInfo.url) + const isCached = r.status === 304 || checkCacheByPerformanceTimings(reqResInfo.url) app.send( NetworkRequest( 'fetch', @@ -289,9 +286,7 @@ export default function (app: App, opts: Partial = {}) { } const isCached = - xhr.status === 304 || - reqHs['x-cache'].includes('Hit') || - (xhr.status < 400 && checkCacheByPerformanceTimings(reqResInfo.url)) + xhr.status === 304 || (xhr.status < 400 && checkCacheByPerformanceTimings(reqResInfo.url)) app.send( NetworkRequest( 'xhr', diff --git a/tracker/tracker/src/main/modules/timing.ts b/tracker/tracker/src/main/modules/timing.ts index 54c094126..299db2221 100644 --- a/tracker/tracker/src/main/modules/timing.ts +++ b/tracker/tracker/src/main/modules/timing.ts @@ -118,6 +118,9 @@ export default function (app: App, opts: Partial): void { entry.decodedBodySize || 0, entry.name, entry.initiatorType, + entry.transferSize, + // @ts-ignore + (entry.responseStatus && entry.responseStatus === 304) || entry.transferSize === 0, ), ) } diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index d49f18610..5c54c6390 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -78,7 +78,7 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) break - case Messages.Type.NetworkRequest: + case Messages.Type.LegacyNetworkRequest: return this.string(msg[1]) && this.string(msg[2]) && this.string(msg[3]) && this.string(msg[4]) && this.string(msg[5]) && this.uint(msg[6]) && this.uint(msg[7]) && this.uint(msg[8]) break @@ -166,7 +166,7 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) break - case Messages.Type.ResourceTiming: + case Messages.Type.ResourceTimingLegacy: return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) && this.uint(msg[4]) && this.uint(msg[5]) && this.uint(msg[6]) && this.string(msg[7]) && this.string(msg[8]) break @@ -270,6 +270,14 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) break + case Messages.Type.ResourceTiming: + return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) && this.uint(msg[4]) && this.uint(msg[5]) && this.uint(msg[6]) && this.string(msg[7]) && this.string(msg[8]) && this.uint(msg[9]) && this.boolean(msg[10]) + break + + case Messages.Type.NetworkRequest: + return this.string(msg[1]) && this.string(msg[2]) && this.string(msg[3]) && this.string(msg[4]) && this.string(msg[5]) && this.uint(msg[6]) && this.uint(msg[7]) && this.uint(msg[8]) && this.boolean(msg[9]) + break + } } From d65e789b5b00095f4fa834d0285679af9658c624 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 20 Mar 2023 11:40:00 +0100 Subject: [PATCH 135/253] change(ui): regen messages; wait for player initialization; insert event lists instead of creating new ones --- backend/pkg/messages/messages.go | 10 +++++----- backend/pkg/messages/read-message.go | 6 +++--- ee/connectors/msgcodec/messages.py | 2 +- ee/connectors/msgcodec/msgcodec.py | 2 +- frontend/app/components/Session/WebPlayer.tsx | 2 +- frontend/app/player/web/MessageManager.ts | 6 ++++-- .../app/player/web/messages/RawMessageReader.gen.ts | 2 +- frontend/app/player/web/messages/message.gen.ts | 4 ++-- frontend/app/player/web/messages/raw.gen.ts | 8 ++++---- frontend/app/player/web/messages/tracker-legacy.gen.ts | 2 +- frontend/app/player/web/messages/tracker.gen.ts | 6 +++--- mobs/messages.rb | 2 +- tracker/tracker/src/common/messages.gen.ts | 8 ++++---- tracker/tracker/src/main/app/messages.gen.ts | 6 +++--- tracker/tracker/src/main/modules/img.ts | 2 +- tracker/tracker/src/webworker/MessageEncoder.gen.ts | 2 +- 16 files changed, 36 insertions(+), 34 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 4fb53be08..bde18cc58 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -48,7 +48,7 @@ const ( MsgPerformanceTrack = 49 MsgStringDict = 50 MsgSetNodeAttributeDict = 51 - MsgResourceTimingLegacy = 53 + MsgResourceTimingDeprecated = 53 MsgConnectionInformation = 54 MsgSetPageVisibility = 55 MsgPerformanceTrackAggr = 56 @@ -1297,7 +1297,7 @@ func (msg *SetNodeAttributeDict) TypeID() int { return 51 } -type ResourceTimingLegacy struct { +type ResourceTimingDeprecated struct { message Timestamp uint64 Duration uint64 @@ -1309,7 +1309,7 @@ type ResourceTimingLegacy struct { Initiator string } -func (msg *ResourceTimingLegacy) Encode() []byte { +func (msg *ResourceTimingDeprecated) Encode() []byte { buf := make([]byte, 81+len(msg.URL)+len(msg.Initiator)) buf[0] = 53 p := 1 @@ -1324,11 +1324,11 @@ func (msg *ResourceTimingLegacy) Encode() []byte { return buf[:p] } -func (msg *ResourceTimingLegacy) Decode() Message { +func (msg *ResourceTimingDeprecated) Decode() Message { return msg } -func (msg *ResourceTimingLegacy) TypeID() int { +func (msg *ResourceTimingDeprecated) TypeID() int { return 53 } diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index adb80d2ed..888fee077 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -756,9 +756,9 @@ func DecodeSetNodeAttributeDict(reader BytesReader) (Message, error) { return msg, err } -func DecodeResourceTimingLegacy(reader BytesReader) (Message, error) { +func DecodeResourceTimingDeprecated(reader BytesReader) (Message, error) { var err error = nil - msg := &ResourceTimingLegacy{} + msg := &ResourceTimingDeprecated{} if msg.Timestamp, err = reader.ReadUint(); err != nil { return nil, err } @@ -1895,7 +1895,7 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { case 51: return DecodeSetNodeAttributeDict(reader) case 53: - return DecodeResourceTimingLegacy(reader) + return DecodeResourceTimingDeprecated(reader) case 54: return DecodeConnectionInformation(reader) case 55: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 4e384cb03..63074d920 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -441,7 +441,7 @@ class SetNodeAttributeDict(Message): self.value_key = value_key -class ResourceTimingLegacy(Message): +class ResourceTimingDeprecated(Message): __id__ = 53 def __init__(self, timestamp, duration, ttfb, header_size, encoded_body_size, decoded_body_size, url, initiator): diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index d4af91836..773dc4eb7 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -420,7 +420,7 @@ class MessageCodec(Codec): ) if message_id == 53: - return ResourceTimingLegacy( + return ResourceTimingDeprecated( timestamp=self.read_uint(reader), duration=self.read_uint(reader), ttfb=self.read_uint(reader), diff --git a/frontend/app/components/Session/WebPlayer.tsx b/frontend/app/components/Session/WebPlayer.tsx index 8111f11af..509add261 100644 --- a/frontend/app/components/Session/WebPlayer.tsx +++ b/frontend/app/components/Session/WebPlayer.tsx @@ -68,7 +68,7 @@ function WebPlayer(props: any) { if (session.events.length > 0 || session.errors.length > 0) { contextValue.player?.updateLists?.(session) } - }, [session.events, session.errors]) + }, [session.events, session.errors, contextValue.player]) const isPlayerReady = contextValue.store?.get().ready diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index f5916d5a8..2772ccbd2 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -139,8 +139,10 @@ export default class MessageManager { } public updateLists(lists: Partial) { - this.lists = new Lists(lists) - + Object.keys(lists).forEach((key: 'event' | 'stack' | 'exceptions') => { + const currentList = this.lists.lists[key] + lists[key]!.forEach(item => currentList.insert(item)) + }) lists?.event?.forEach((e: Record) => { if (e.type === EVENT_TYPES.LOCATION) { this.locationEventManager.append(e); diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 72d6e2ff9..3f220a133 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -403,7 +403,7 @@ export default class RawMessageReader extends PrimitiveReader { const url = this.readString(); if (url === null) { return resetPointer() } const initiator = this.readString(); if (initiator === null) { return resetPointer() } return { - tp: MType.ResourceTimingLegacy, + tp: MType.ResourceTimingDeprecated, timestamp, duration, ttfb, diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index 4a1474f23..ea8f9902e 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -36,7 +36,7 @@ import type { RawPerformanceTrack, RawStringDict, RawSetNodeAttributeDict, - RawResourceTimingLegacy, + RawResourceTimingDeprecated, RawConnectionInformation, RawSetPageVisibility, RawLoadFontFace, @@ -135,7 +135,7 @@ export type StringDict = RawStringDict & Timed export type SetNodeAttributeDict = RawSetNodeAttributeDict & Timed -export type ResourceTimingLegacy = RawResourceTimingLegacy & Timed +export type ResourceTimingDeprecated = RawResourceTimingDeprecated & Timed export type ConnectionInformation = RawConnectionInformation & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 86aba859c..01ed0cdbe 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -34,7 +34,7 @@ export const enum MType { PerformanceTrack = 49, StringDict = 50, SetNodeAttributeDict = 51, - ResourceTimingLegacy = 53, + ResourceTimingDeprecated = 53, ConnectionInformation = 54, SetPageVisibility = 55, LoadFontFace = 57, @@ -286,8 +286,8 @@ export interface RawSetNodeAttributeDict { valueKey: number, } -export interface RawResourceTimingLegacy { - tp: MType.ResourceTimingLegacy, +export interface RawResourceTimingDeprecated { + tp: MType.ResourceTimingDeprecated, timestamp: number, duration: number, ttfb: number, @@ -532,4 +532,4 @@ export interface RawIosNetworkCall { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawLegacyNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingLegacy | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawNetworkRequest | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawLegacyNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingDeprecated | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawNetworkRequest | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index 236a33da5..e83b0eec5 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -35,7 +35,7 @@ export const TP_MAP = { 49: MType.PerformanceTrack, 50: MType.StringDict, 51: MType.SetNodeAttributeDict, - 53: MType.ResourceTimingLegacy, + 53: MType.ResourceTimingDeprecated, 54: MType.ConnectionInformation, 55: MType.SetPageVisibility, 57: MType.LoadFontFace, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index 31acd4fe7..bff29e429 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -271,7 +271,7 @@ type TrSetNodeAttributeDict = [ valueKey: number, ] -type TrResourceTimingLegacy = [ +type TrResourceTimingDeprecated = [ type: 53, timestamp: number, duration: number, @@ -484,7 +484,7 @@ type TrNetworkRequest = [ ] -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrLegacyNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingLegacy | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming | TrNetworkRequest +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrLegacyNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingDeprecated | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming | TrNetworkRequest export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -766,7 +766,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { case 53: { return { - tp: MType.ResourceTimingLegacy, + tp: MType.ResourceTimingDeprecated, timestamp: tMsg[1], duration: tMsg[2], ttfb: tMsg[3], diff --git a/mobs/messages.rb b/mobs/messages.rb index 1f75a8b5e..ad9cad41d 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -265,7 +265,7 @@ message 51, "SetNodeAttributeDict" do uint 'NameKey' uint 'ValueKey' end -message 53, 'ResourceTimingLegacy', :replayer => :devtools do +message 53, 'ResourceTimingDeprecated', :replayer => :devtools do uint 'Timestamp' uint 'Duration' uint 'TTFB' diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 8018810a4..06a85410f 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -41,7 +41,7 @@ export declare const enum Type { PerformanceTrack = 49, StringDict = 50, SetNodeAttributeDict = 51, - ResourceTimingLegacy = 53, + ResourceTimingDeprecated = 53, ConnectionInformation = 54, SetPageVisibility = 55, LoadFontFace = 57, @@ -338,8 +338,8 @@ export type SetNodeAttributeDict = [ /*valueKey:*/ number, ] -export type ResourceTimingLegacy = [ - /*type:*/ Type.ResourceTimingLegacy, +export type ResourceTimingDeprecated = [ + /*type:*/ Type.ResourceTimingDeprecated, /*timestamp:*/ number, /*duration:*/ number, /*ttfb:*/ number, @@ -551,5 +551,5 @@ export type NetworkRequest = [ ] -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | LegacyNetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingLegacy | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming | NetworkRequest +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | LegacyNetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingDeprecated | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming | NetworkRequest export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index 30332a5fe..dcbc6f718 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -498,7 +498,7 @@ export function SetNodeAttributeDict( ] } -export function ResourceTimingLegacy( +export function ResourceTimingDeprecated( timestamp: number, duration: number, ttfb: number, @@ -507,9 +507,9 @@ export function ResourceTimingLegacy( decodedBodySize: number, url: string, initiator: string, -): Messages.ResourceTimingLegacy { +): Messages.ResourceTimingDeprecated { return [ - Messages.Type.ResourceTimingLegacy, + Messages.Type.ResourceTimingDeprecated, timestamp, duration, ttfb, diff --git a/tracker/tracker/src/main/modules/img.ts b/tracker/tracker/src/main/modules/img.ts index b5dddb293..0099f6969 100644 --- a/tracker/tracker/src/main/modules/img.ts +++ b/tracker/tracker/src/main/modules/img.ts @@ -60,7 +60,7 @@ export default function (app: App): void { const sendImgError = app.safe(function (img: HTMLImageElement): void { const resolvedSrc = resolveURL(img.src || '') // Src type is null sometimes. - is it true? if (isURL(resolvedSrc)) { - app.send(ResourceTiming(app.timestamp(), 0, 0, 0, 0, 0, resolvedSrc, 'img')) + app.send(ResourceTiming(app.timestamp(), 0, 0, 0, 0, 0, resolvedSrc, 'img', 0, false)) } }) diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index 5c54c6390..b88fec754 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -166,7 +166,7 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) break - case Messages.Type.ResourceTimingLegacy: + case Messages.Type.ResourceTimingDeprecated: return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) && this.uint(msg[4]) && this.uint(msg[5]) && this.uint(msg[6]) && this.string(msg[7]) && this.string(msg[8]) break From 04a768ee805d292180447a5dbf72a04d326ed719 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 20 Mar 2023 11:41:12 +0100 Subject: [PATCH 136/253] change(tracker): update changelog for tracker --- tracker/tracker/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index a1050fb85..90c68a91f 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -3,6 +3,7 @@ - Capture mouse thrashing, input hesitation+duration, click hesitation - Capture DOM node drop event (>30% nodes removed) - Capture iframe network requests +- Detect cached requests to img, css and js resources; send transferred size - added `{ network: { disableClickmaps: boolean } }` to disable calculating el. selectors ## 5.0.1 From 702e2f3def5e6814584085fcd4abf0bdb03da6d2 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 20 Mar 2023 13:53:07 +0100 Subject: [PATCH 137/253] change(tracker): fix tracker crash on performance entry missing (/browser api issue?) --- tracker/tracker/src/main/modules/network.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts index b0ad1453a..43f961b26 100644 --- a/tracker/tracker/src/main/modules/network.ts +++ b/tracker/tracker/src/main/modules/network.ts @@ -38,8 +38,10 @@ type FetchRequestBody = RequestInit['body'] function checkCacheByPerformanceTimings(requestUrl: string) { if (performance) { const timings = performance.getEntriesByName(requestUrl)[0] - // @ts-ignore - weird ts typings, please refer to https://developer.mozilla.org/en-US/docs/Web/API/PerformanceNavigationTiming - return timings.transferSize === 0 || timings.responseStart - timings.requestStart < 10 + if (timings) { + // @ts-ignore - weird ts typings, please refer to https://developer.mozilla.org/en-US/docs/Web/API/PerformanceNavigationTiming + return timings.transferSize === 0 || timings.responseStart - timings.requestStart < 10 + } } return false } From 7156fa0fc4db7f6f90374c7368d12bd87b028e8d Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 21 Mar 2023 17:01:17 +0100 Subject: [PATCH 138/253] change(tracker): fix msgs format --- backend/pkg/messages/messages.go | 48 ++----------------- backend/pkg/messages/read-message.go | 41 ++-------------- ee/connectors/msgcodec/messages.py | 17 +------ ee/connectors/msgcodec/msgcodec.py | 15 +----- .../web/messages/RawMessageReader.gen.ts | 26 +--------- .../app/player/web/messages/message.gen.ts | 7 +-- frontend/app/player/web/messages/raw.gen.ts | 22 ++------- .../player/web/messages/tracker-legacy.gen.ts | 3 +- .../app/player/web/messages/tracker.gen.ts | 34 ++----------- mobs/messages.rb | 16 +------ tracker/tracker/src/common/messages.gen.ts | 22 ++------- tracker/tracker/src/main/app/messages.gen.ts | 31 ++---------- tracker/tracker/src/main/modules/network.ts | 5 -- .../src/webworker/MessageEncoder.gen.ts | 6 +-- 14 files changed, 30 insertions(+), 263 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index bde18cc58..7a51c6ac9 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -22,7 +22,7 @@ const ( MsgSetInputValue = 18 MsgSetInputChecked = 19 MsgMouseMove = 20 - MsgLegacyNetworkRequest = 21 + MsgNetworkRequest = 21 MsgConsoleLog = 22 MsgPageLoadTiming = 23 MsgPageRenderTiming = 24 @@ -81,7 +81,6 @@ const ( MsgMouseThrashing = 114 MsgUnbindNodes = 115 MsgResourceTiming = 116 - MsgNetworkRequest = 117 MsgIssueEvent = 125 MsgSessionEnd = 126 MsgSessionSearch = 127 @@ -603,7 +602,7 @@ func (msg *MouseMove) TypeID() int { return 20 } -type LegacyNetworkRequest struct { +type NetworkRequest struct { message Type string Method string @@ -615,7 +614,7 @@ type LegacyNetworkRequest struct { Duration uint64 } -func (msg *LegacyNetworkRequest) Encode() []byte { +func (msg *NetworkRequest) Encode() []byte { buf := make([]byte, 81+len(msg.Type)+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response)) buf[0] = 21 p := 1 @@ -630,11 +629,11 @@ func (msg *LegacyNetworkRequest) Encode() []byte { return buf[:p] } -func (msg *LegacyNetworkRequest) Decode() Message { +func (msg *NetworkRequest) Decode() Message { return msg } -func (msg *LegacyNetworkRequest) TypeID() int { +func (msg *NetworkRequest) TypeID() int { return 21 } @@ -2164,43 +2163,6 @@ func (msg *ResourceTiming) TypeID() int { return 116 } -type NetworkRequest struct { - message - Type string - Method string - URL string - Request string - Response string - Status uint64 - Timestamp uint64 - Duration uint64 - Cached bool -} - -func (msg *NetworkRequest) Encode() []byte { - buf := make([]byte, 91+len(msg.Type)+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response)) - buf[0] = 117 - p := 1 - p = WriteString(msg.Type, buf, p) - p = WriteString(msg.Method, buf, p) - p = WriteString(msg.URL, buf, p) - p = WriteString(msg.Request, buf, p) - p = WriteString(msg.Response, buf, p) - p = WriteUint(msg.Status, buf, p) - p = WriteUint(msg.Timestamp, buf, p) - p = WriteUint(msg.Duration, buf, p) - p = WriteBoolean(msg.Cached, buf, p) - return buf[:p] -} - -func (msg *NetworkRequest) Decode() Message { - return msg -} - -func (msg *NetworkRequest) TypeID() int { - return 117 -} - type IssueEvent struct { message MessageID uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 888fee077..a51200dc0 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -300,9 +300,9 @@ func DecodeMouseMove(reader BytesReader) (Message, error) { return msg, err } -func DecodeLegacyNetworkRequest(reader BytesReader) (Message, error) { +func DecodeNetworkRequest(reader BytesReader) (Message, error) { var err error = nil - msg := &LegacyNetworkRequest{} + msg := &NetworkRequest{} if msg.Type, err = reader.ReadString(); err != nil { return nil, err } @@ -1314,39 +1314,6 @@ func DecodeResourceTiming(reader BytesReader) (Message, error) { return msg, err } -func DecodeNetworkRequest(reader BytesReader) (Message, error) { - var err error = nil - msg := &NetworkRequest{} - if msg.Type, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Method, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.URL, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Request, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Response, err = reader.ReadString(); err != nil { - return nil, err - } - if msg.Status, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Timestamp, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Duration, err = reader.ReadUint(); err != nil { - return nil, err - } - if msg.Cached, err = reader.ReadBoolean(); err != nil { - return nil, err - } - return msg, err -} - func DecodeIssueEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IssueEvent{} @@ -1843,7 +1810,7 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { case 20: return DecodeMouseMove(reader) case 21: - return DecodeLegacyNetworkRequest(reader) + return DecodeNetworkRequest(reader) case 22: return DecodeConsoleLog(reader) case 23: @@ -1960,8 +1927,6 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeUnbindNodes(reader) case 116: return DecodeResourceTiming(reader) - case 117: - return DecodeNetworkRequest(reader) case 125: return DecodeIssueEvent(reader) case 126: diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index 63074d920..663ba8f99 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -185,7 +185,7 @@ class MouseMove(Message): self.y = y -class LegacyNetworkRequest(Message): +class NetworkRequest(Message): __id__ = 21 def __init__(self, type, method, url, request, response, status, timestamp, duration): @@ -759,21 +759,6 @@ class ResourceTiming(Message): self.cached = cached -class NetworkRequest(Message): - __id__ = 117 - - def __init__(self, type, method, url, request, response, status, timestamp, duration, cached): - self.type = type - self.method = method - self.url = url - self.request = request - self.response = response - self.status = status - self.timestamp = timestamp - self.duration = duration - self.cached = cached - - class IssueEvent(Message): __id__ = 125 diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 773dc4eb7..9aef2b475 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -216,7 +216,7 @@ class MessageCodec(Codec): ) if message_id == 21: - return LegacyNetworkRequest( + return NetworkRequest( type=self.read_string(reader), method=self.read_string(reader), url=self.read_string(reader), @@ -671,19 +671,6 @@ class MessageCodec(Codec): cached=self.read_boolean(reader) ) - if message_id == 117: - return NetworkRequest( - type=self.read_string(reader), - method=self.read_string(reader), - url=self.read_string(reader), - request=self.read_string(reader), - response=self.read_string(reader), - status=self.read_uint(reader), - timestamp=self.read_uint(reader), - duration=self.read_uint(reader), - cached=self.read_boolean(reader) - ) - if message_id == 125: return IssueEvent( message_id=self.read_uint(reader), diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 3f220a133..135960cef 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -211,7 +211,7 @@ export default class RawMessageReader extends PrimitiveReader { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const duration = this.readUint(); if (duration === null) { return resetPointer() } return { - tp: MType.LegacyNetworkRequest, + tp: MType.NetworkRequest, type, method, url, @@ -673,30 +673,6 @@ export default class RawMessageReader extends PrimitiveReader { }; } - case 117: { - const type = this.readString(); if (type === null) { return resetPointer() } - const method = this.readString(); if (method === null) { return resetPointer() } - const url = this.readString(); if (url === null) { return resetPointer() } - const request = this.readString(); if (request === null) { return resetPointer() } - const response = this.readString(); if (response === null) { return resetPointer() } - const status = this.readUint(); if (status === null) { return resetPointer() } - const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } - const duration = this.readUint(); if (duration === null) { return resetPointer() } - const cached = this.readBoolean(); if (cached === null) { return resetPointer() } - return { - tp: MType.NetworkRequest, - type, - method, - url, - request, - response, - status, - timestamp, - duration, - cached, - }; - } - case 90: { const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } const projectID = this.readUint(); if (projectID === null) { return resetPointer() } diff --git a/frontend/app/player/web/messages/message.gen.ts b/frontend/app/player/web/messages/message.gen.ts index ea8f9902e..6a8916cd4 100644 --- a/frontend/app/player/web/messages/message.gen.ts +++ b/frontend/app/player/web/messages/message.gen.ts @@ -21,7 +21,7 @@ import type { RawSetInputValue, RawSetInputChecked, RawMouseMove, - RawLegacyNetworkRequest, + RawNetworkRequest, RawConsoleLog, RawCssInsertRule, RawCssDeleteRule, @@ -58,7 +58,6 @@ import type { RawSelectionChange, RawMouseThrashing, RawResourceTiming, - RawNetworkRequest, RawIosSessionStart, RawIosCustomEvent, RawIosScreenChanges, @@ -105,7 +104,7 @@ export type SetInputChecked = RawSetInputChecked & Timed export type MouseMove = RawMouseMove & Timed -export type LegacyNetworkRequest = RawLegacyNetworkRequest & Timed +export type NetworkRequest = RawNetworkRequest & Timed export type ConsoleLog = RawConsoleLog & Timed @@ -179,8 +178,6 @@ export type MouseThrashing = RawMouseThrashing & Timed export type ResourceTiming = RawResourceTiming & Timed -export type NetworkRequest = RawNetworkRequest & Timed - export type IosSessionStart = RawIosSessionStart & Timed export type IosCustomEvent = RawIosCustomEvent & Timed diff --git a/frontend/app/player/web/messages/raw.gen.ts b/frontend/app/player/web/messages/raw.gen.ts index 01ed0cdbe..ecd88631c 100644 --- a/frontend/app/player/web/messages/raw.gen.ts +++ b/frontend/app/player/web/messages/raw.gen.ts @@ -19,7 +19,7 @@ export const enum MType { SetInputValue = 18, SetInputChecked = 19, MouseMove = 20, - LegacyNetworkRequest = 21, + NetworkRequest = 21, ConsoleLog = 22, CssInsertRule = 37, CssDeleteRule = 38, @@ -56,7 +56,6 @@ export const enum MType { SelectionChange = 113, MouseThrashing = 114, ResourceTiming = 116, - NetworkRequest = 117, IosSessionStart = 90, IosCustomEvent = 93, IosScreenChanges = 96, @@ -175,8 +174,8 @@ export interface RawMouseMove { y: number, } -export interface RawLegacyNetworkRequest { - tp: MType.LegacyNetworkRequest, +export interface RawNetworkRequest { + tp: MType.NetworkRequest, type: string, method: string, url: string, @@ -448,19 +447,6 @@ export interface RawResourceTiming { cached: boolean, } -export interface RawNetworkRequest { - tp: MType.NetworkRequest, - type: string, - method: string, - url: string, - request: string, - response: string, - status: number, - timestamp: number, - duration: number, - cached: boolean, -} - export interface RawIosSessionStart { tp: MType.IosSessionStart, timestamp: number, @@ -532,4 +518,4 @@ export interface RawIosNetworkCall { } -export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawLegacyNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingDeprecated | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawNetworkRequest | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; +export type RawMessage = RawTimestamp | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawNetworkRequest | RawConsoleLog | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawStringDict | RawSetNodeAttributeDict | RawResourceTimingDeprecated | RawConnectionInformation | RawSetPageVisibility | RawLoadFontFace | RawSetNodeFocus | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawAdoptedSsReplaceURLBased | RawAdoptedSsReplace | RawAdoptedSsInsertRuleURLBased | RawAdoptedSsInsertRule | RawAdoptedSsDeleteRule | RawAdoptedSsAddOwner | RawAdoptedSsRemoveOwner | RawZustand | RawSelectionChange | RawMouseThrashing | RawResourceTiming | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/web/messages/tracker-legacy.gen.ts b/frontend/app/player/web/messages/tracker-legacy.gen.ts index e83b0eec5..7215b8c3c 100644 --- a/frontend/app/player/web/messages/tracker-legacy.gen.ts +++ b/frontend/app/player/web/messages/tracker-legacy.gen.ts @@ -20,7 +20,7 @@ export const TP_MAP = { 18: MType.SetInputValue, 19: MType.SetInputChecked, 20: MType.MouseMove, - 21: MType.LegacyNetworkRequest, + 21: MType.NetworkRequest, 22: MType.ConsoleLog, 37: MType.CssInsertRule, 38: MType.CssDeleteRule, @@ -57,7 +57,6 @@ export const TP_MAP = { 113: MType.SelectionChange, 114: MType.MouseThrashing, 116: MType.ResourceTiming, - 117: MType.NetworkRequest, 90: MType.IosSessionStart, 93: MType.IosCustomEvent, 96: MType.IosScreenChanges, diff --git a/frontend/app/player/web/messages/tracker.gen.ts b/frontend/app/player/web/messages/tracker.gen.ts index bff29e429..a8f9a7f14 100644 --- a/frontend/app/player/web/messages/tracker.gen.ts +++ b/frontend/app/player/web/messages/tracker.gen.ts @@ -113,7 +113,7 @@ type TrMouseMove = [ y: number, ] -type TrLegacyNetworkRequest = [ +type TrNetworkRequest = [ type: 21, type: string, method: string, @@ -470,21 +470,8 @@ type TrResourceTiming = [ cached: boolean, ] -type TrNetworkRequest = [ - type: 117, - type: string, - method: string, - url: string, - request: string, - response: string, - status: number, - timestamp: number, - duration: number, - cached: boolean, -] - -export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrLegacyNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingDeprecated | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming | TrNetworkRequest +export type TrackerMessage = TrTimestamp | TrSetPageLocation | TrSetViewportSize | TrSetViewportScroll | TrCreateDocument | TrCreateElementNode | TrCreateTextNode | TrMoveNode | TrRemoveNode | TrSetNodeAttribute | TrRemoveNodeAttribute | TrSetNodeData | TrSetNodeScroll | TrSetInputTarget | TrSetInputValue | TrSetInputChecked | TrMouseMove | TrNetworkRequest | TrConsoleLog | TrPageLoadTiming | TrPageRenderTiming | TrCustomEvent | TrUserID | TrUserAnonymousID | TrMetadata | TrCSSInsertRule | TrCSSDeleteRule | TrFetch | TrProfiler | TrOTable | TrStateAction | TrRedux | TrVuex | TrMobX | TrNgRx | TrGraphQL | TrPerformanceTrack | TrStringDict | TrSetNodeAttributeDict | TrResourceTimingDeprecated | TrConnectionInformation | TrSetPageVisibility | TrLoadFontFace | TrSetNodeFocus | TrLongTask | TrSetNodeAttributeURLBased | TrSetCSSDataURLBased | TrTechnicalInfo | TrCustomIssue | TrCSSInsertRuleURLBased | TrMouseClick | TrCreateIFrameDocument | TrAdoptedSSReplaceURLBased | TrAdoptedSSInsertRuleURLBased | TrAdoptedSSDeleteRule | TrAdoptedSSAddOwner | TrAdoptedSSRemoveOwner | TrJSException | TrZustand | TrBatchMetadata | TrPartitionedMessage | TrInputChange | TrSelectionChange | TrMouseThrashing | TrUnbindNodes | TrResourceTiming export default function translate(tMsg: TrackerMessage): RawMessage | null { switch(tMsg[0]) { @@ -625,7 +612,7 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { case 21: { return { - tp: MType.LegacyNetworkRequest, + tp: MType.NetworkRequest, type: tMsg[1], method: tMsg[2], url: tMsg[3], @@ -953,21 +940,6 @@ export default function translate(tMsg: TrackerMessage): RawMessage | null { } } - case 117: { - return { - tp: MType.NetworkRequest, - type: tMsg[1], - method: tMsg[2], - url: tMsg[3], - request: tMsg[4], - response: tMsg[5], - status: tMsg[6], - timestamp: tMsg[7], - duration: tMsg[8], - cached: tMsg[9], - } - } - default: return null } diff --git a/mobs/messages.rb b/mobs/messages.rb index ad9cad41d..5ac7b6ff2 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -104,7 +104,7 @@ message 20, 'MouseMove' do uint 'X' uint 'Y' end -message 21, 'LegacyNetworkRequest', :replayer => :devtools do +message 21, 'NetworkRequest', :replayer => :devtools do string 'Type' # fetch/xhr/anythingElse(axios,gql,fonts,image?) string 'Method' string 'URL' @@ -484,20 +484,6 @@ message 116, 'ResourceTiming', :replayer => :devtools do boolean 'Cached' end -message 117, 'NetworkRequest', :replayer => :devtools do - string 'Type' # fetch/xhr/anythingElse(axios,gql,fonts,image?) - string 'Method' - string 'URL' - string 'Request' - string 'Response' - uint 'Status' - uint 'Timestamp' - uint 'Duration' - boolean 'Cached' -end - - - ## Backend-only message 125, 'IssueEvent', :replayer => false, :tracker => false do uint 'MessageID' diff --git a/tracker/tracker/src/common/messages.gen.ts b/tracker/tracker/src/common/messages.gen.ts index 06a85410f..a96343098 100644 --- a/tracker/tracker/src/common/messages.gen.ts +++ b/tracker/tracker/src/common/messages.gen.ts @@ -19,7 +19,7 @@ export declare const enum Type { SetInputValue = 18, SetInputChecked = 19, MouseMove = 20, - LegacyNetworkRequest = 21, + NetworkRequest = 21, ConsoleLog = 22, PageLoadTiming = 23, PageRenderTiming = 24, @@ -68,7 +68,6 @@ export declare const enum Type { MouseThrashing = 114, UnbindNodes = 115, ResourceTiming = 116, - NetworkRequest = 117, } @@ -180,8 +179,8 @@ export type MouseMove = [ /*y:*/ number, ] -export type LegacyNetworkRequest = [ - /*type:*/ Type.LegacyNetworkRequest, +export type NetworkRequest = [ + /*type:*/ Type.NetworkRequest, /*type:*/ string, /*method:*/ string, /*url:*/ string, @@ -537,19 +536,6 @@ export type ResourceTiming = [ /*cached:*/ boolean, ] -export type NetworkRequest = [ - /*type:*/ Type.NetworkRequest, - /*type:*/ string, - /*method:*/ string, - /*url:*/ string, - /*request:*/ string, - /*response:*/ string, - /*status:*/ number, - /*timestamp:*/ number, - /*duration:*/ number, - /*cached:*/ boolean, -] - -type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | LegacyNetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingDeprecated | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming | NetworkRequest +type Message = Timestamp | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetNodeScroll | SetInputTarget | SetInputValue | SetInputChecked | MouseMove | NetworkRequest | ConsoleLog | PageLoadTiming | PageRenderTiming | CustomEvent | UserID | UserAnonymousID | Metadata | CSSInsertRule | CSSDeleteRule | Fetch | Profiler | OTable | StateAction | Redux | Vuex | MobX | NgRx | GraphQL | PerformanceTrack | StringDict | SetNodeAttributeDict | ResourceTimingDeprecated | ConnectionInformation | SetPageVisibility | LoadFontFace | SetNodeFocus | LongTask | SetNodeAttributeURLBased | SetCSSDataURLBased | TechnicalInfo | CustomIssue | CSSInsertRuleURLBased | MouseClick | CreateIFrameDocument | AdoptedSSReplaceURLBased | AdoptedSSInsertRuleURLBased | AdoptedSSDeleteRule | AdoptedSSAddOwner | AdoptedSSRemoveOwner | JSException | Zustand | BatchMetadata | PartitionedMessage | InputChange | SelectionChange | MouseThrashing | UnbindNodes | ResourceTiming export default Message diff --git a/tracker/tracker/src/main/app/messages.gen.ts b/tracker/tracker/src/main/app/messages.gen.ts index dcbc6f718..46d672d2d 100644 --- a/tracker/tracker/src/main/app/messages.gen.ts +++ b/tracker/tracker/src/main/app/messages.gen.ts @@ -204,7 +204,7 @@ export function MouseMove( ] } -export function LegacyNetworkRequest( +export function NetworkRequest( type: string, method: string, url: string, @@ -213,9 +213,9 @@ export function LegacyNetworkRequest( status: number, timestamp: number, duration: number, -): Messages.LegacyNetworkRequest { +): Messages.NetworkRequest { return [ - Messages.Type.LegacyNetworkRequest, + Messages.Type.NetworkRequest, type, method, url, @@ -869,28 +869,3 @@ export function ResourceTiming( ] } -export function NetworkRequest( - type: string, - method: string, - url: string, - request: string, - response: string, - status: number, - timestamp: number, - duration: number, - cached: boolean, -): Messages.NetworkRequest { - return [ - Messages.Type.NetworkRequest, - type, - method, - url, - request, - response, - status, - timestamp, - duration, - cached, - ] -} - diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts index 43f961b26..a1a3e2c9c 100644 --- a/tracker/tracker/src/main/modules/network.ts +++ b/tracker/tracker/src/main/modules/network.ts @@ -219,7 +219,6 @@ export default function (app: App, opts: Partial = {}) { return } - const isCached = r.status === 304 || checkCacheByPerformanceTimings(reqResInfo.url) app.send( NetworkRequest( 'fetch', @@ -230,7 +229,6 @@ export default function (app: App, opts: Partial = {}) { r.status, startTime + getTimeOrigin(), duration, - isCached, ), ) }) @@ -287,8 +285,6 @@ export default function (app: App, opts: Partial = {}) { return } - const isCached = - xhr.status === 304 || (xhr.status < 400 && checkCacheByPerformanceTimings(reqResInfo.url)) app.send( NetworkRequest( 'xhr', @@ -299,7 +295,6 @@ export default function (app: App, opts: Partial = {}) { xhr.status, startTime + getTimeOrigin(), duration, - isCached, ), ) }), diff --git a/tracker/tracker/src/webworker/MessageEncoder.gen.ts b/tracker/tracker/src/webworker/MessageEncoder.gen.ts index b88fec754..69fc7b35f 100644 --- a/tracker/tracker/src/webworker/MessageEncoder.gen.ts +++ b/tracker/tracker/src/webworker/MessageEncoder.gen.ts @@ -78,7 +78,7 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) break - case Messages.Type.LegacyNetworkRequest: + case Messages.Type.NetworkRequest: return this.string(msg[1]) && this.string(msg[2]) && this.string(msg[3]) && this.string(msg[4]) && this.string(msg[5]) && this.uint(msg[6]) && this.uint(msg[7]) && this.uint(msg[8]) break @@ -274,10 +274,6 @@ export default class MessageEncoder extends PrimitiveEncoder { return this.uint(msg[1]) && this.uint(msg[2]) && this.uint(msg[3]) && this.uint(msg[4]) && this.uint(msg[5]) && this.uint(msg[6]) && this.string(msg[7]) && this.string(msg[8]) && this.uint(msg[9]) && this.boolean(msg[10]) break - case Messages.Type.NetworkRequest: - return this.string(msg[1]) && this.string(msg[2]) && this.string(msg[3]) && this.string(msg[4]) && this.string(msg[5]) && this.uint(msg[6]) && this.uint(msg[7]) && this.uint(msg[8]) && this.boolean(msg[9]) - break - } } From 9f71d1a6b367d1f7d72ffa8661bc43e6d27c76db Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 21 Mar 2023 17:10:18 +0100 Subject: [PATCH 139/253] feat(backend): added support for new version of ResourceTiming --- backend/pkg/messages/legacy-message-transform.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/backend/pkg/messages/legacy-message-transform.go b/backend/pkg/messages/legacy-message-transform.go index 223178d15..a267d1f26 100644 --- a/backend/pkg/messages/legacy-message-transform.go +++ b/backend/pkg/messages/legacy-message-transform.go @@ -30,6 +30,19 @@ func transformDeprecated(msg Message) Message { Payload: m.Payload, URL: "", } + case *ResourceTimingDeprecated: + return &ResourceTiming{ + Timestamp: m.Timestamp, + Duration: m.Duration, + TTFB: m.TTFB, + HeaderSize: m.HeaderSize, + EncodedBodySize: m.EncodedBodySize, + DecodedBodySize: m.DecodedBodySize, + URL: m.URL, + Initiator: m.Initiator, + TransferredSize: 0, + Cached: false, + } } return msg } From 2af8f65b97d281aadabada7b4fb5090d8293e2f0 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 22 Mar 2023 10:15:14 +0100 Subject: [PATCH 140/253] change(ui) - text change in preferences --- .../app/components/Header/PreferencesView/PreferencesView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/Header/PreferencesView/PreferencesView.tsx b/frontend/app/components/Header/PreferencesView/PreferencesView.tsx index f9da1b933..735fdeaa9 100644 --- a/frontend/app/components/Header/PreferencesView/PreferencesView.tsx +++ b/frontend/app/components/Header/PreferencesView/PreferencesView.tsx @@ -18,7 +18,7 @@ function PreferencesView(props: Props) {
- Changes applied at organization level + Updates are be applied at organization level.
); From 078c7fbff268c8a4cda8f997c9c2f4b5707026e0 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 22 Mar 2023 11:26:19 +0100 Subject: [PATCH 141/253] chore(cli): Don't reload toolings Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index 0c8aaad74..52a6b28b2 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -154,10 +154,12 @@ function status() { function or_helm_upgrade() { set -o pipefail log_file="${tmp_dir}/helm.log" + state=$1 chart_names=( toolings openreplay ) + [[ $state == "reload" ]] && chart_names=( openreplay ) for chart in "${chart_names[@]}"; do [[ -z $OR_VERSION ]] || HELM_OPTIONS="--set dbMigrationUpstreamBranch=${OR_VERSION}" if ! helm upgrade --install "$chart" ./"$chart" -n "$APP_NS" --wait -f ./vars.yaml --atomic --debug $HELM_OPTIONS 2>&1 | tee -a "${log_file}"; then @@ -224,7 +226,7 @@ function upgrade() { function reload() { err_cd $OR_DIR/openreplay/scripts/helmcharts sudo cp -f $OR_DIR/vars.yaml . - or_helm_upgrade + or_helm_upgrade reload return } From 517b98197a15127e6f2f518a21277fe093224c4e Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 22 Mar 2023 12:54:27 +0100 Subject: [PATCH 142/253] feat(backend/assets): skip text/html content type files --- backend/internal/assets/cacher/cacher.go | 19 +++++++++++++------ backend/pkg/storage/s3.go | 4 ++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/backend/internal/assets/cacher/cacher.go b/backend/internal/assets/cacher/cacher.go index 1df32ca26..513d40c47 100644 --- a/backend/internal/assets/cacher/cacher.go +++ b/backend/internal/assets/cacher/cacher.go @@ -62,11 +62,11 @@ func NewCacher(cfg *config.Config) *cacher { caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) tlsConfig = &tls.Config{ - InsecureSkipVerify: true, - Certificates: []tls.Certificate{cert}, - RootCAs: caCertPool, - } - + InsecureSkipVerify: true, + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + } + } c := &cacher{ @@ -75,7 +75,7 @@ func NewCacher(cfg *config.Config) *cacher { httpClient: &http.Client{ Timeout: time.Duration(6) * time.Second, Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, + Proxy: http.ProxyFromEnvironment, TLSClientConfig: tlsConfig, }, }, @@ -135,6 +135,13 @@ func (c *cacher) cacheURL(t *Task) { if contentType == "" { contentType = mime.TypeByExtension(filepath.Ext(res.Request.URL.Path)) } + + // Skip html file (usually it's a CDN mock for 404 error) + if strings.HasPrefix(contentType, "text/html") { + c.Errors <- errors.Wrap(fmt.Errorf("context type is text/html, sessID: %d", t.sessionID), t.urlContext) + return + } + isCSS := strings.HasPrefix(contentType, "text/css") strData := string(data) diff --git a/backend/pkg/storage/s3.go b/backend/pkg/storage/s3.go index 2e97673d3..53dd02700 100644 --- a/backend/pkg/storage/s3.go +++ b/backend/pkg/storage/s3.go @@ -33,6 +33,10 @@ func NewS3(region string, bucket string) *S3 { func (s3 *S3) Upload(reader io.Reader, key string, contentType string, gzipped bool) error { cacheControl := "max-age=2628000, immutable, private" + // For asset files we set max-age to 1 day + if !gzipped { + cacheControl = "max-age=86400, immutable, private" + } var contentEncoding *string if gzipped { gzipStr := "gzip" From 561f3338700e47e61c081d1dd5b82db5eabf6a18 Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 22 Mar 2023 12:59:02 +0100 Subject: [PATCH 143/253] feat(backend/assets): set 30 days cache age for all s3 files --- backend/pkg/storage/s3.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/pkg/storage/s3.go b/backend/pkg/storage/s3.go index 53dd02700..2e97673d3 100644 --- a/backend/pkg/storage/s3.go +++ b/backend/pkg/storage/s3.go @@ -33,10 +33,6 @@ func NewS3(region string, bucket string) *S3 { func (s3 *S3) Upload(reader io.Reader, key string, contentType string, gzipped bool) error { cacheControl := "max-age=2628000, immutable, private" - // For asset files we set max-age to 1 day - if !gzipped { - cacheControl = "max-age=86400, immutable, private" - } var contentEncoding *string if gzipped { gzipStr := "gzip" From 516b90c5e1b11d8528ca4008523e148d2346e140 Mon Sep 17 00:00:00 2001 From: Dayan Graham Date: Wed, 22 Mar 2023 16:29:53 +0000 Subject: [PATCH 144/253] (feat): Chalice - Allow option to set ssl verification to false for the Chalice API --- api/chalicelib/utils/s3.py | 6 ++++-- api/env.default | 1 + ee/api/env.default | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/utils/s3.py b/api/chalicelib/utils/s3.py index 366a5d181..8a2369647 100644 --- a/api/chalicelib/utils/s3.py +++ b/api/chalicelib/utils/s3.py @@ -16,7 +16,8 @@ else: aws_access_key_id=config("S3_KEY"), aws_secret_access_key=config("S3_SECRET"), config=Config(signature_version='s3v4'), - region_name=config("sessions_region")) + region_name=config("sessions_region"), + verify=(False if config("S3_DISABLE_SSL_VERIFY") else True)) def __get_s3_resource(): @@ -26,7 +27,8 @@ def __get_s3_resource(): aws_access_key_id=config("S3_KEY"), aws_secret_access_key=config("S3_SECRET"), config=Config(signature_version='s3v4'), - region_name=config("sessions_region")) + region_name=config("sessions_region"), + verify=(False if config("S3_DISABLE_SSL_VERIFY") else True)) def exists(bucket, key): diff --git a/api/env.default b/api/env.default index 78acd001c..6fc4752ad 100644 --- a/api/env.default +++ b/api/env.default @@ -10,6 +10,7 @@ EMAIL_USE_TLS=true S3_HOST= S3_KEY= S3_SECRET= +S3_DISABLE_SSL_VERIFY= SITE_URL= announcement_url= captcha_key= diff --git a/ee/api/env.default b/ee/api/env.default index cdbc3d256..075b53eca 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -11,6 +11,7 @@ LICENSE_KEY= S3_HOST= S3_KEY= S3_SECRET= +S3_DISABLE_SSL_VERIFY= SAML2_MD_URL= SITE_URL= announcement_url= From 03b92e891058eaf32b0932531c5057e22d1c47f6 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Thu, 23 Mar 2023 18:20:49 +0100 Subject: [PATCH 145/253] CLI improvements (#1059) * chore(cli): Adding option to install OpenReplay Signed-off-by: rjshrjndrn * chore(cli): Don't install if existing OR found Signed-off-by: rjshrjndrn * chore(cli): Adding data cleanup option Signed-off-by: rjshrjndrn * chore(cli): Cleanup data minio Signed-off-by: rjshrjndrn * chore(cli): Adding info for cleanup Signed-off-by: rjshrjndrn * chore(cli): remove unnecessary logs. Signed-off-by: rjshrjndrn * chore(cli): Adding confirmation message Signed-off-by: rjshrjndrn * chore(cli): Clenaup comments Signed-off-by: rjshrjndrn --------- Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 91 +++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index 52a6b28b2..4cde9a54d 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -15,6 +15,9 @@ tmp_dir=$(mktemp -d) sudo mkdir $OR_DIR } export PATH=/var/lib/openreplay:$PATH +function xargs() { + /var/lib/openreplay/busybox xargs +} tools=( zyedidia/eget @@ -114,10 +117,12 @@ echo -e ${NC} log info ' Usage: openreplay [ -h | --help ] [ -s | --status ] + [ -i | --install DOMAIN_NAME ] [ -u | --upgrade ] [ -U | --deprecated-upgrade /path/to/old_vars.yaml] [ -r | --restart ] [ -R | --Reload ] + [ -c | --cleanup N(in days) ] [ -e | --edit ] [ -p | --install-packages ] [ -l | --logs SERVICE ] @@ -184,6 +189,73 @@ function upgrade_old() { upgrade } +function clone_repo() { + err_cd "$tmp_dir" + log info "Working directory $tmp_dir" + git_options="-b ${OR_VERSION:-main}" + eval git clone "${OR_REPO}" --depth 1 $git_options + return +} + +function install() { + domain_name=$1 + # Check existing installation + [[ -f ${OR_DIR}/vars.yaml ]] && { + or_version=$(busybox awk '/fromVersion/{print $2}' < "${OR_DIR}/vars.yaml") + log err "Openreplay installation ${BWHITE}${or_version}${RED} found. If you want to upgrade, run ${BWHITE}openreplay -u${RED}" + } + # Installing OR + log title "Installing OpenReplay" + clone_repo + err_cd "$tmp_dir/openreplay/scripts/helmcharts" + DOMAIN_NAME=$domain_name bash init.sh + return +} + +function cleanup() { + # Confirmation for deletion. Do you want to delete Postgres/Minio(session) data before $date ? + delete_from_number_days=$1 + delete_from_date=$(date +%Y-%m-%d -d "$delete_from_number_days day ago") + log debug "Do you want to delete the data captured on and before ${BWHITE}$delete_from_date${YELLOW}?" + read -p "Are you sure[y/n]? " -n 1 -r + echo # (optional) move to a new line + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log err "Cancelling data deletion" + fi + + # Run pg cleanup + pguser=$(awk '/postgresqlUser/{print $2}' < "${OR_DIR}/vars.yaml" | xargs) + pgpassword=$(awk '/postgresqlPassword/{print $2}' < "${OR_DIR}/vars.yaml" | xargs) + pghost=$(awk '/postgresqlHost/{print $2}' < "${OR_DIR}/vars.yaml" | xargs) + pgport=$(awk '/postgresqlPort/{print $2}' < "${OR_DIR}/vars.yaml" | xargs) + pgdatabase=$(awk '/postgresqlDatabase/{print $2}' < "${OR_DIR}/vars.yaml" | xargs) + kubectl delete po -n ${APP_NS} pg-cleanup &> /dev/null || true + kubectl run pg-cleanup -n ${APP_NS} \ + --restart=Never \ + --env PGHOST=$pghost \ + --env PGUSER=$pguser \ + --env PGDATABASE=$pgdatabase \ + --env PGPASSWORD=$pgpassword \ + --env PGPORT=$pgport \ + --image bitnami/postgresql -- psql -c "DELETE FROM public.sessions WHERE start_ts < extract(epoch from '${delete_from_date}'::date) * 1000;" + # Run minio cleanup + MINIO_ACCESS_KEY=$(awk '/accessKey/{print $NF}' < "${OR_DIR}/vars.yaml" | tail -n1 | xargs) + MINIO_SECRET_KEY=$(awk '/secretKey/{print $NF}' < "${OR_DIR}/vars.yaml" | tail -n1 | xargs) + MINIO_HOST=$(awk '/endpoint/{print $NF}' < "${OR_DIR}/vars.yaml" | tail -n1 | xargs) + kubectl delete po -n ${APP_NS} minio-cleanup &> /dev/null || true + kubectl run minio-cleanup -n ${APP_NS} \ + --restart=Never \ + --env MINIO_HOST=$pghost \ + --image bitnami/minio:2020.10.9-debian-10-r6 -- /bin/sh -c " + mc alias set minio $MINIO_HOST $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && + mc rm --recursive --dangerous --force --older-than ${delete_from_number_days}d minio/mobs + " + log info "Postgres data cleanup process initiated. Postgres will automatically vacuum deleted rows when the database is idle. This may take up a few days to free the disk space." + log info "Minio (where recordings are stored) cleanup process initiated." + log info "Run ${BWHITE}openreplay -s${GREEN} to check the status of the cleanup process and available disk space." + return +} + function upgrade() { # TODO: # 1. store vars.yaml in central place. @@ -191,15 +263,12 @@ function upgrade() { # 3. How to update package. Because openreplay -u will be done from old update script # 4. Update from Version exists git || log err "Git not found. Please install" - log info "Working directory $tmp_dir" - err_cd "$tmp_dir" or_version=$(busybox awk '/fromVersion/{print $2}' < "${OR_DIR}/vars.yaml") # Creating backup dir of current installation [[ -d "$OR_DIR/openreplay" ]] && sudo cp -rfb "$OR_DIR/openreplay" "$OR_DIR/openreplay_${or_version//\"}" && sudo rm -rf ${OR_DIR}/openreplay - git_options="-b ${OR_VERSION:-main}" - eval git clone "${OR_REPO}" --depth 1 $git_options + clone_repo err_cd openreplay/scripts/helmcharts install_packages [[ -d /openreplay ]] && sudo chown -R 1001:1001 /openreplay @@ -239,7 +308,7 @@ function clean_tmp_dir() { install_packages } -PARSED_ARGUMENTS=$(busybox getopt -a -n openreplay -o Rrevpiuhsl:U: --long reload,edit,restart,verbose,install-packages,install,upgrade,help,status,logs,deprecated-upgrade: -- "$@") +PARSED_ARGUMENTS=$(busybox getopt -a -n openreplay -o Rrevpi:uhsl:U:c: --long reload,edit,restart,verbose,install-packages,install:,upgrade,help,status,logs,deprecated-upgrade:,cleanup: -- "$@") VALID_ARGUMENTS=$? if [[ "$VALID_ARGUMENTS" != "0" ]]; then help @@ -256,6 +325,12 @@ do clean_tmp_dir exit 0 ;; + -i | --install) + log title "Installing OpenReplay" + install "$2" + clean_tmp_dir + exit 0 + ;; -u | --upgrade) log title "Upgrading OpenReplay" upgrade @@ -268,6 +343,12 @@ do clean_tmp_dir exit 0 ;; + -c | --cleanup) + log title "Cleaning up data older than $2 days" + cleanup "$2" + clean_tmp_dir + exit 0 + ;; -r | --restart) log title "Restarting OpenReplay Components" kubectl rollout restart deployment -n "${APP_NS}" From 09e788a29fe7440bfd7a1d1ebfd3c000b03d7649 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 24 Mar 2023 10:13:52 +0100 Subject: [PATCH 146/253] change(ui): testing more optimisations --- .../app/player/web/managers/DOM/DOMManager.ts | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index ec51401f6..ab839118b 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -257,9 +257,13 @@ export default class DOMManager extends ListWalker { } return case MType.RemoveNodeAttribute: - vn = this.vElements.get(msg.id) - if (!vn) { logger.error("Node not found", msg); return } - vn.removeAttribute(msg.name) + if (isJump) { + this.attrsBacktrack = this.attrsBacktrack.filter(m => m.id !== msg.id && m.name !== msg.name) + } else { + vn = this.vElements.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + vn.removeAttribute(msg.name) + } return case MType.SetInputValue: vn = this.vElements.get(msg.id) @@ -472,7 +476,25 @@ export default class DOMManager extends ListWalker { * are applied, so it won't try to download and then cancel when node is created in msg N and removed in msg N+2 * which produces weird bug when asset is cached (10-25ms delay) * */ - await this.moveWait(t, (msg) => this.applyMessage(msg, isJump)) + // http://0.0.0.0:3333/5/session/8452905874437457 + // 70 iframe, 8 create element - STYLE tag + console.time('moveWait') + let t0 = performance.now() + let t1 = t0 + const timings = [] + await this.moveWait(t, (msg) => { + t0 = performance.now() + this.applyMessage(msg, isJump) + t1 = performance.now() + timings.push({ t: t1 - t0, m: msg.tp, msg }) + }) + + console.timeEnd('moveWait') + console.log( + timings.sort((a, b) => b.t - a.t), + timings.filter(t => t.msg.tag === 'STYLE').length, + ) + if (isJump) { this.attrsBacktrack.forEach(msg => { this.applyBacktrack(msg) @@ -480,7 +502,6 @@ export default class DOMManager extends ListWalker { this.attrsBacktrack = [] } this.vRoots.forEach(rt => rt.applyChanges()) // MBTODO (optimisation): affected set - // Thinkabout (read): css preload // What if we go back before it is ready? We'll have two handlres? return this.stylesManager.moveReady(t).then(() => { From b2ce9a7aff8a9eb7035eeba2a4435b35fc39e6b2 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Feb 2023 12:30:20 +0100 Subject: [PATCH 147/253] feat(ui): health status widget --- frontend/app/components/Header/Header.js | 3 + .../HealthStatus/HealthModal/Footer.tsx | 38 ++++ .../HealthStatus/HealthModal/HealthModal.tsx | 66 ++++++ .../Header/HealthStatus/HealthStatus.tsx | 92 +++++++++ .../components/Header/HealthStatus/index.ts | 1 + .../BugReport/components/MetaInfo.tsx | 4 +- frontend/app/components/ui/SVG.tsx | 4 +- frontend/app/svg/cheers.svg | 193 ++++++++++++++++++ .../app/svg/icons/exclamation-circle-fill.svg | 10 + frontend/app/svg/icons/pulse.svg | 3 + frontend/app/theme/colors.js | 1 + 11 files changed, 411 insertions(+), 4 deletions(-) create mode 100644 frontend/app/components/Header/HealthStatus/HealthModal/Footer.tsx create mode 100644 frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx create mode 100644 frontend/app/components/Header/HealthStatus/HealthStatus.tsx create mode 100644 frontend/app/components/Header/HealthStatus/index.ts create mode 100644 frontend/app/svg/cheers.svg create mode 100644 frontend/app/svg/icons/exclamation-circle-fill.svg create mode 100644 frontend/app/svg/icons/pulse.svg diff --git a/frontend/app/components/Header/Header.js b/frontend/app/components/Header/Header.js index 7ef0028c9..021f96df3 100644 --- a/frontend/app/components/Header/Header.js +++ b/frontend/app/components/Header/Header.js @@ -19,6 +19,7 @@ import UserMenu from './UserMenu'; import SettingsMenu from './SettingsMenu'; import DefaultMenuView from './DefaultMenuView'; import PreferencesView from './PreferencesView'; +import HealthStatus from './HealthStatus' const CLIENT_PATH = client(CLIENT_DEFAULT_TAB); @@ -78,6 +79,8 @@ const Header = (props) => {
+ +
diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/Footer.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/Footer.tsx new file mode 100644 index 000000000..43bd434fc --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/HealthModal/Footer.tsx @@ -0,0 +1,38 @@ +import React from 'react'; +import { Icon } from 'UI'; + +function Footer() { + return ( + + ); +} + +export default Footer; diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx new file mode 100644 index 000000000..86b597ed7 --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx @@ -0,0 +1,66 @@ +import React from 'react'; +import slide from 'App/svg/cheers.svg'; +import { Icon, Button } from 'UI'; +import Footer from './Footer' + +function Category({ name, healthOk }: { name: string; healthOk: boolean }) { + const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); + return ( +
+ + {name} +
+ ) +} + +function HealthModal({ healthOk }: { healthOk: boolean }) { + + return ( +
+
+
Installation Status
+ +
+ +
+
+ + + + +
+
+ +
+
+
+ +
+
+
+ ); +} + +export default HealthModal; diff --git a/frontend/app/components/Header/HealthStatus/HealthStatus.tsx b/frontend/app/components/Header/HealthStatus/HealthStatus.tsx new file mode 100644 index 000000000..24e4c281e --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/HealthStatus.tsx @@ -0,0 +1,92 @@ +import React from 'react'; +import { Icon, Tooltip } from 'UI'; +import cn from 'classnames'; +import HealthModal from 'Components/Header/HealthStatus/HealthModal/HealthModal'; + +function HealthStatus() { + const [healthOk, setHealth] = React.useState(false); + + const icon = healthOk ? 'pulse' : ('exclamation-circle-fill' as const); + return ( +
+
+
+ +
+
+ + + +
+ ); +} + +function HealthMenu({ healthOk, setHealth }: { healthOk: boolean; setHealth: any }) { + const title = healthOk ? 'All Systems Operational' : 'Service disruption'; + const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); + return ( +
+
+
+ + {title} +
+
+ Last checked 22 mins. ago +
setHealth(!healthOk)}> + +
+
+
+ +
+
+
Version
+
+ 123 123 +
+
+ + {healthOk ? ( + <> +
+
Sessions
+
+ 10 000 +
+
+
+
Events
+
+ 90 000 +
+
+ + ) : ( +
Observed installation Issue with the following
+ )} +
+
+
+ ); +} + +export default HealthStatus; diff --git a/frontend/app/components/Header/HealthStatus/index.ts b/frontend/app/components/Header/HealthStatus/index.ts new file mode 100644 index 000000000..1f4ce8576 --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/index.ts @@ -0,0 +1 @@ +export { default } from './HealthStatus' \ No newline at end of file diff --git a/frontend/app/components/Session_/BugReport/components/MetaInfo.tsx b/frontend/app/components/Session_/BugReport/components/MetaInfo.tsx index 09746dcfb..2eecf4ea7 100644 --- a/frontend/app/components/Session_/BugReport/components/MetaInfo.tsx +++ b/frontend/app/components/Session_/BugReport/components/MetaInfo.tsx @@ -24,11 +24,9 @@ export default function MetaInfo({ {Object.keys(envObject).map((envTag) => (
{envTag}
-
- +
{/* @ts-ignore */} {envObject[envTag]} -
))} diff --git a/frontend/app/components/ui/SVG.tsx b/frontend/app/components/ui/SVG.tsx index eeaeae5dd..95254b16c 100644 --- a/frontend/app/components/ui/SVG.tsx +++ b/frontend/app/components/ui/SVG.tsx @@ -1,7 +1,7 @@ import React from 'react'; -export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'click-hesitation' | 'click-rage' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'cursor-trash' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope' | 'errors-icon' | 'event/click' | 'event/click_hesitation' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/input_hesitation' | 'event/link' | 'event/location' | 'event/mouse_thrashing' | 'event/resize' | 'event/view' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'input-hesitation' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; +export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'click-hesitation' | 'click-rage' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'cursor-trash' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope' | 'errors-icon' | 'event/click' | 'event/click_hesitation' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/input_hesitation' | 'event/link' | 'event/location' | 'event/mouse_thrashing' | 'event/resize' | 'event/view' | 'exclamation-circle-fill' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'input-hesitation' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'pulse' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; interface Props { name: IconNames; @@ -170,6 +170,7 @@ const SVG = (props: Props) => { case 'event/mouse_thrashing': return ; case 'event/resize': return ; case 'event/view': return ; + case 'exclamation-circle-fill': return ; case 'exclamation-circle': return ; case 'expand-wide': return ; case 'explosion': return ; @@ -368,6 +369,7 @@ const SVG = (props: Props) => { case 'plus': return ; case 'pointer-sessions-search': return ; case 'prev1': return ; + case 'pulse': return ; case 'puzzle-piece': return ; case 'puzzle': return ; case 'question-circle': return ; diff --git a/frontend/app/svg/cheers.svg b/frontend/app/svg/cheers.svg new file mode 100644 index 000000000..1341b27a2 --- /dev/null +++ b/frontend/app/svg/cheers.svg @@ -0,0 +1,193 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/icons/exclamation-circle-fill.svg b/frontend/app/svg/icons/exclamation-circle-fill.svg new file mode 100644 index 000000000..eebbd6833 --- /dev/null +++ b/frontend/app/svg/icons/exclamation-circle-fill.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/frontend/app/svg/icons/pulse.svg b/frontend/app/svg/icons/pulse.svg new file mode 100644 index 000000000..5075d1cab --- /dev/null +++ b/frontend/app/svg/icons/pulse.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/app/theme/colors.js b/frontend/app/theme/colors.js index 3986cf3d5..d9671f6fe 100644 --- a/frontend/app/theme/colors.js +++ b/frontend/app/theme/colors.js @@ -57,5 +57,6 @@ module.exports = { 'text-disabled': 'rgba(0,0,0, 0.38)', 'text-primary': 'rgba(0,0,0, 0.87)', 'outlined-border': 'rgba(0,0,0, 0.23)', + 'divider': 'rgba(0, 0, 0, 0.12)', } } From 0f1232f3a78d575b373b7eeb0ff72f803036dc1f Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Feb 2023 13:02:39 +0100 Subject: [PATCH 148/253] feat(ui): change hovers --- .../Header/HealthStatus/HealthModal/HealthModal.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx index 86b597ed7..5cce8dc9b 100644 --- a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx @@ -7,10 +7,12 @@ function Category({ name, healthOk }: { name: string; healthOk: boolean }) { const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); return (
{name} + +
) } From f3efa296df3cdacf503533caff1d6021c4703864 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 17 Feb 2023 13:03:34 +0100 Subject: [PATCH 149/253] feat(ui): remove warnings --- .../components/Header/HealthStatus/HealthModal/HealthModal.tsx | 3 ++- frontend/app/components/Header/HealthStatus/HealthStatus.tsx | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx index 5cce8dc9b..85cfd2dcf 100644 --- a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx @@ -1,4 +1,5 @@ import React from 'react'; +// @ts-ignore import slide from 'App/svg/cheers.svg'; import { Icon, Button } from 'UI'; import Footer from './Footer' @@ -17,7 +18,7 @@ function Category({ name, healthOk }: { name: string; healthOk: boolean }) { ) } -function HealthModal({ healthOk }: { healthOk: boolean }) { +function HealthModal() { return (
Date: Tue, 21 Feb 2023 15:08:41 +0100 Subject: [PATCH 150/253] change(ui): mock back response --- .../Header/HealthStatus/HealthModal/mock.ts | 187 ++++++++++++++++++ .../Header/HealthStatus/HealthStatus.tsx | 34 ++++ 2 files changed, 221 insertions(+) create mode 100644 frontend/app/components/Header/HealthStatus/HealthModal/mock.ts diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/mock.ts b/frontend/app/components/Header/HealthStatus/HealthModal/mock.ts new file mode 100644 index 000000000..8211126f6 --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/HealthModal/mock.ts @@ -0,0 +1,187 @@ +export const response = { + databases: { + postgres: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + clickhouse: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + }, + ingestionPipeline: { + redis: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + kafka: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + }, + backendServices: { + alerts: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + assets: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + assist: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + chalice: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + db: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + ender: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + frontend: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + heuristics: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + http: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + 'ingress-nginx': { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + integrations: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + peers: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + quickwit: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + sink: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + sourcemapreader: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + storage: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + utilities: { + health: true, + details: { + version: 'v1.13', + schema: 'v1.10', + lastUpdatedOn: '12jan2023', + }, + }, + }, + overall: { + health: true, + details: { + numberOfEventCaptured: 123000, + numberOfSessionsCaptured: 25678, + }, + labels: { + parent: 'information', + }, + }, + ssl: true, +}; diff --git a/frontend/app/components/Header/HealthStatus/HealthStatus.tsx b/frontend/app/components/Header/HealthStatus/HealthStatus.tsx index 5296594c2..361e49be3 100644 --- a/frontend/app/components/Header/HealthStatus/HealthStatus.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthStatus.tsx @@ -2,6 +2,40 @@ import React from 'react'; import { Icon } from 'UI'; import cn from 'classnames'; import HealthModal from 'Components/Header/HealthStatus/HealthModal/HealthModal'; +import { response } from './HealthModal/mock' + +function mapResponse(resp) { + const dbKeys = Object.keys(resp.databases) + const ingestKeys = Object.keys(resp.ingestionPipeline) + const backendKeys = Object.keys(resp.backendServices) + + if (!resp.overall.health) { + const dbHealth: Record = { + overall: true, + } + const ingestHealth: Record = { + overall: true, + } + const backHealth: Record = { + overall: true, + } + dbKeys.forEach(key => { + const dbStatus = resp.databases[key].health + if (!dbStatus) dbHealth.overall = false + dbHealth[key] = resp.databases.key + }) + ingestKeys.forEach(key => { + const ingestStatus = resp.ingestionPipeline[key].health + if (!ingestStatus) ingestHealth.overall = false + ingestHealth[key] = resp.ingestionPipeline.key + }) + backendKeys.forEach(key => { + const backendStatus = resp.backendServices[key].health + if (!backendStatus) backHealth.overall = false + backHealth[key] = resp.backendServices.key + }) + } +} function HealthStatus() { const [healthOk, setHealth] = React.useState(false); From 620489e57d17dc9e81a48e2a1ea4507e70578db5 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Mon, 13 Mar 2023 16:06:58 +0100 Subject: [PATCH 151/253] change(ui): remove mock, connect api to health status comp --- .../HealthStatus/HealthModal/HealthModal.tsx | 16 +- .../Header/HealthStatus/HealthModal/mock.ts | 187 ------------------ .../Header/HealthStatus/HealthStatus.tsx | 179 +++++++++++------ frontend/app/services/HealthService.ts | 10 + frontend/app/services/index.ts | 4 + 5 files changed, 141 insertions(+), 255 deletions(-) delete mode 100644 frontend/app/components/Header/HealthStatus/HealthModal/mock.ts create mode 100644 frontend/app/services/HealthService.ts diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx index 85cfd2dcf..e9aef0e71 100644 --- a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx @@ -4,11 +4,12 @@ import slide from 'App/svg/cheers.svg'; import { Icon, Button } from 'UI'; import Footer from './Footer' -function Category({ name, healthOk }: { name: string; healthOk: boolean }) { +export function Category({ name, healthOk, onClick }: { name: string; healthOk: boolean; onClick: (args: any) => void }) { const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); return (
{name} @@ -18,7 +19,7 @@ function Category({ name, healthOk }: { name: string; healthOk: boolean }) { ) } -function HealthModal() { +function HealthModal({ getHealth, isLoading, healthResponse }: { getHealth: () => void; isLoading: boolean; healthResponse: Record }) { return (
Installation Status
-
@@ -46,8 +47,8 @@ function HealthModal() {
- - + + {/**/}
= { - overall: true, - } - const ingestHealth: Record = { - overall: true, - } - const backHealth: Record = { - overall: true, - } - dbKeys.forEach(key => { - const dbStatus = resp.databases[key].health - if (!dbStatus) dbHealth.overall = false - dbHealth[key] = resp.databases.key - }) - ingestKeys.forEach(key => { - const ingestStatus = resp.ingestionPipeline[key].health - if (!ingestStatus) ingestHealth.overall = false - ingestHealth[key] = resp.ingestionPipeline.key - }) - backendKeys.forEach(key => { - const backendStatus = resp.backendServices[key].health - if (!backendStatus) backHealth.overall = false - backHealth[key] = resp.backendServices.key - }) - } +const categoryKeyNames = { + backendServices: 'Backend Services', + databases: 'Databases', + ingestionPipeline: 'Ingestion Pipeline', + ssl: 'SSL', } -function HealthStatus() { - const [healthOk, setHealth] = React.useState(false); +function mapResponse(resp: Record) { + const services = Object.keys(resp); + const healthMap: Record = {} + services.forEach(service => { + healthMap[service] = { + // @ts-ignore + name: categoryKeyNames[service], + healthOk: true, + subservices: resp[service], + } + Object.values(healthMap[service].subservices).forEach((subservice: Record) => { + if (!subservice?.health) healthMap[service].healthOk = false; + }) + }) - const icon = healthOk ? 'pulse' : ('exclamation-circle-fill' as const); + const overallHealth = Object.values(healthMap).every((service: Record) => service.healthOk); + + return { overallHealth, healthMap } +} + + +function HealthStatus() { + const lastAskedKey = '__openreplay_health_status'; + const healthResponseKey = '__openreplay_health_response'; + const healthResponseSaved = localStorage.getItem(healthResponseKey) || '{}'; + const [healthResponse, setHealthResponse] = React.useState(JSON.parse(healthResponseSaved)); + const [isLoading, setIsLoading] = React.useState(false); + const lastAskedSaved = localStorage.getItem(lastAskedKey); + const [lastAsked, setLastAsked] = React.useState(lastAskedSaved); + const [showModal, setShowModal] = React.useState(false); + + const getHealth = async () => { + if (isLoading) return; + try { + setIsLoading(true); + const r = await healthService.fetchStatus(); + const healthMap = mapResponse(r) + setHealthResponse(healthMap); + const asked = new Date().getTime(); + localStorage.setItem(healthResponseKey, JSON.stringify(healthMap)) + localStorage.setItem(lastAskedKey, asked.toString()); + setLastAsked(asked.toString()); + } catch (e) { + console.error(e); + } finally { + setIsLoading(false); + } + }; + + React.useEffect(() => { + const now = new Date(); + const lastAskedDate = lastAsked ? new Date(parseInt(lastAsked, 10)) : null; + const diff = lastAskedDate ? now.getTime() - lastAskedDate.getTime() : 0; + const diffInMinutes = Math.round(diff / 1000 / 60); + if (Object.keys(healthResponse).length === 0 || !lastAskedDate || diffInMinutes > 10) { + void getHealth(); + } + }, []); + + const icon = healthResponse?.overallHealth ? 'pulse' : ('exclamation-circle-fill' as const); return (
- - + + {showModal ? () : null}
); } -function HealthMenu({ healthOk, setHealth }: { healthOk: boolean; setHealth: any }) { +function HealthMenu({ + healthResponse, + getHealth, + isLoading, + lastAsked, + setShowModal, +}: { + healthResponse: Record; + getHealth: Function; + isLoading: boolean; + lastAsked: string | null; + setShowModal: (visible: boolean) => void; +}) { + const [lastAskedDiff, setLastAskedDiff] = React.useState(0); + const healthOk = healthResponse?.overallHealth; + + React.useEffect(() => { + const now = new Date(); + const lastAskedDate = lastAsked ? new Date(parseInt(lastAsked, 10)) : null; + const diff = lastAskedDate ? now.getTime() - lastAskedDate.getTime() : 0; + const diffInMinutes = Math.round(diff / 1000 / 60); + setLastAskedDiff(diffInMinutes); + }, [lastAsked]); + const title = healthOk ? 'All Systems Operational' : 'Service disruption'; const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); + + const problematicServices = Object.values(healthResponse?.healthMap || {}).filter( + (service: Record) => !service.healthOk + ) as Record[]; return (
{title}
- Last checked 22 mins. ago -
setHealth(!healthOk)}> + Last checked {lastAskedDiff} mins. ago +
getHealth()} + >
-
-
Version
-
- 123 123 -
-
+ {/*
*/} + {/*
Version
*/} + {/*
*/} + {/* 123 123*/} + {/*
*/} + {/*
*/} - {healthOk ? ( + {!healthOk ? ( <> -
-
Sessions
-
- 10 000 -
-
-
-
Events
-
- 90 000 -
-
+
Observed installation Issue with the following
+ {problematicServices.map(service => setShowModal(true)} healthOk={false} name={service.name} />)} - ) : ( -
Observed installation Issue with the following
- )} + ) : null}
diff --git a/frontend/app/services/HealthService.ts b/frontend/app/services/HealthService.ts new file mode 100644 index 000000000..019863bb3 --- /dev/null +++ b/frontend/app/services/HealthService.ts @@ -0,0 +1,10 @@ +import BaseService from './BaseService'; + +export default class HealthService extends BaseService { + fetchStatus(): Promise { + return this.client.get('/health') + .then(r => r.json()) + .then(j => j.data || {}) + .catch(Promise.reject) + } +} \ No newline at end of file diff --git a/frontend/app/services/index.ts b/frontend/app/services/index.ts index 816113e68..32e216127 100644 --- a/frontend/app/services/index.ts +++ b/frontend/app/services/index.ts @@ -10,6 +10,7 @@ import RecordingsService from "./RecordingsService"; import ConfigService from './ConfigService' import AlertsService from './AlertsService' import WebhookService from './WebhookService' +import HealthService from "./HealthService"; export const dashboardService = new DashboardService(); export const metricService = new MetricService(); @@ -24,6 +25,8 @@ export const configService = new ConfigService(); export const alertsService = new AlertsService(); export const webhookService = new WebhookService(); +export const healthService = new HealthService(); + export const services = [ dashboardService, metricService, @@ -37,4 +40,5 @@ export const services = [ configService, alertsService, webhookService, + healthService, ] \ No newline at end of file From 12d7ff4f99a1f52ffe864a309df07250d89f5348 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 14 Mar 2023 12:17:50 +0100 Subject: [PATCH 152/253] change(ui): remove keys warning, create service health type, keep response in localstorage for 15 mins --- .../HealthStatus/HealthModal/HealthModal.tsx | 156 +++++++++++------ .../Header/HealthStatus/HealthStatus.tsx | 159 +++++------------- .../Header/HealthStatus/HealthWidget.tsx | 93 ++++++++++ .../Header/HealthStatus/ServiceCategory.tsx | 43 +++++ .../SubserviceHealth/SubserviceHealth.tsx | 52 ++++++ .../components/Header/HealthStatus/const.ts | 6 + 6 files changed, 346 insertions(+), 163 deletions(-) create mode 100644 frontend/app/components/Header/HealthStatus/HealthWidget.tsx create mode 100644 frontend/app/components/Header/HealthStatus/ServiceCategory.tsx create mode 100644 frontend/app/components/Header/HealthStatus/SubserviceHealth/SubserviceHealth.tsx create mode 100644 frontend/app/components/Header/HealthStatus/const.ts diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx index e9aef0e71..b0bbb8969 100644 --- a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx @@ -1,73 +1,131 @@ import React from 'react'; // @ts-ignore import slide from 'App/svg/cheers.svg'; -import { Icon, Button } from 'UI'; -import Footer from './Footer' +import { Button } from 'UI'; +import Footer from './Footer'; +import { getHighest } from 'App/constants/zindex'; +import Category from 'Components/Header/HealthStatus/ServiceCategory'; +import SubserviceHealth from 'Components/Header/HealthStatus/SubserviceHealth/SubserviceHealth'; +import { IServiceStats } from '../HealthStatus'; -export function Category({ name, healthOk, onClick }: { name: string; healthOk: boolean; onClick: (args: any) => void }) { - const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); - return ( -
- - {name} +function HealthModal({ + getHealth, + isLoading, + healthResponse, + setShowModal, +}: { + getHealth: () => void; + isLoading: boolean; + healthResponse: { overallHealth: boolean; healthMap: Record }; + setShowModal: (isOpen: boolean) => void; +}) { + const [selectedService, setSelectedService] = React.useState(''); - -
- ) -} - -function HealthModal({ getHealth, isLoading, healthResponse }: { getHealth: () => void; isLoading: boolean; healthResponse: Record }) { + React.useEffect(() => { + if (!healthResponse.overallHealth) { + setSelectedService( + Object.keys(healthResponse.healthMap).filter( + (s) => !healthResponse.healthMap[s].healthOk + )[0] + ); + } + }, [healthResponse]); + const handleClose = () => { + setShowModal(false); + }; return (
e.stopPropagation()} + className={'flex flex-col bg-white rounded border border-figmaColors-divider'} > -
Installation Status
- -
- -
-
- - - - {/**/} -
- +
Installation Status
+
+ +
+
+ {Object.keys(healthResponse.healthMap).map((service) => ( + + setSelectedService(service)} + healthOk={healthResponse.healthMap[service].healthOk} + name={healthResponse.healthMap[service].name} + isSelectable + isSelected={selectedService === service} + /> + + ))} +
+
+ {selectedService ? ( + + ) : ( + + )} +
+
+
+ +
+
-
- -
-
); } - - +function ServiceStatus({ service }: { service: Record }) { + const { subservices } = service; + return ( +
+
+ {Object.keys(subservices).map((subservice: string) => ( + + + + ))} +
+
+ ); +} export default HealthModal; diff --git a/frontend/app/components/Header/HealthStatus/HealthStatus.tsx b/frontend/app/components/Header/HealthStatus/HealthStatus.tsx index 930419f63..7732bca9d 100644 --- a/frontend/app/components/Header/HealthStatus/HealthStatus.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthStatus.tsx @@ -1,38 +1,46 @@ import React from 'react'; import { Icon } from 'UI'; -import cn from 'classnames'; -import HealthModal, { Category } from 'Components/Header/HealthStatus/HealthModal/HealthModal'; +import HealthModal from 'Components/Header/HealthStatus/HealthModal/HealthModal'; import { healthService } from 'App/services'; +import { categoryKeyNames } from './const'; +import HealthWidget from "Components/Header/HealthStatus/HealthWidget"; - -const categoryKeyNames = { - backendServices: 'Backend Services', - databases: 'Databases', - ingestionPipeline: 'Ingestion Pipeline', - ssl: 'SSL', +export interface IServiceStats { + name: 'backendServices' | 'databases' | 'ingestionPipeline' | 'ssl'; + serviceName: string; + healthOk: boolean; + subservices: { + health: boolean; + details?: { + errors?: string[]; + version?: string; + } + }[] } function mapResponse(resp: Record) { const services = Object.keys(resp); - const healthMap: Record = {} - services.forEach(service => { + const healthMap: Record = {}; + services.forEach((service) => { healthMap[service] = { // @ts-ignore name: categoryKeyNames[service], healthOk: true, subservices: resp[service], - } + serviceName: service, + }; Object.values(healthMap[service].subservices).forEach((subservice: Record) => { if (!subservice?.health) healthMap[service].healthOk = false; - }) - }) + }); + }); - const overallHealth = Object.values(healthMap).every((service: Record) => service.healthOk); + const overallHealth = Object.values(healthMap).every( + (service: Record) => service.healthOk + ); - return { overallHealth, healthMap } + return { overallHealth, healthMap }; } - function HealthStatus() { const lastAskedKey = '__openreplay_health_status'; const healthResponseKey = '__openreplay_health_response'; @@ -48,10 +56,10 @@ function HealthStatus() { try { setIsLoading(true); const r = await healthService.fetchStatus(); - const healthMap = mapResponse(r) + const healthMap = mapResponse(r); setHealthResponse(healthMap); const asked = new Date().getTime(); - localStorage.setItem(healthResponseKey, JSON.stringify(healthMap)) + localStorage.setItem(healthResponseKey, JSON.stringify(healthMap)); localStorage.setItem(lastAskedKey, asked.toString()); setLastAsked(asked.toString()); } catch (e) { @@ -73,109 +81,32 @@ function HealthStatus() { const icon = healthResponse?.overallHealth ? 'pulse' : ('exclamation-circle-fill' as const); return ( -
-
-
- -
-
- - - {showModal ? () : null} -
- ); -} - -function HealthMenu({ - healthResponse, - getHealth, - isLoading, - lastAsked, - setShowModal, -}: { - healthResponse: Record; - getHealth: Function; - isLoading: boolean; - lastAsked: string | null; - setShowModal: (visible: boolean) => void; -}) { - const [lastAskedDiff, setLastAskedDiff] = React.useState(0); - const healthOk = healthResponse?.overallHealth; - - React.useEffect(() => { - const now = new Date(); - const lastAskedDate = lastAsked ? new Date(parseInt(lastAsked, 10)) : null; - const diff = lastAskedDate ? now.getTime() - lastAskedDate.getTime() : 0; - const diffInMinutes = Math.round(diff / 1000 / 60); - setLastAskedDiff(diffInMinutes); - }, [lastAsked]); - - const title = healthOk ? 'All Systems Operational' : 'Service disruption'; - const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); - - const problematicServices = Object.values(healthResponse?.healthMap || {}).filter( - (service: Record) => !service.healthOk - ) as Record[]; - return ( -
-
+ <> +
- - {title} -
-
- Last checked {lastAskedDiff} mins. ago -
getHealth()} - > - +
+
-
-
- {/*
*/} - {/*
Version
*/} - {/*
*/} - {/* 123 123*/} - {/*
*/} - {/*
*/} - - {!healthOk ? ( - <> -
Observed installation Issue with the following
- {problematicServices.map(service => setShowModal(true)} healthOk={false} name={service.name} />)} - - ) : null} -
+
-
+ {showModal ? ( + + ) : null} + ); } + export default HealthStatus; diff --git a/frontend/app/components/Header/HealthStatus/HealthWidget.tsx b/frontend/app/components/Header/HealthStatus/HealthWidget.tsx new file mode 100644 index 000000000..c6372540b --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/HealthWidget.tsx @@ -0,0 +1,93 @@ +import React from 'react' +import { Icon } from "UI"; +import ServiceCategory from "Components/Header/HealthStatus/ServiceCategory"; +import cn from 'classnames' +import { IServiceStats } from './HealthStatus' + +function HealthWidget({ + healthResponse, + getHealth, + isLoading, + lastAsked, + setShowModal, +}: { + healthResponse: { overallHealth: boolean; healthMap: Record }; + getHealth: Function; + isLoading: boolean; + lastAsked: string | null; + setShowModal: (visible: boolean) => void; +}) { + const [lastAskedDiff, setLastAskedDiff] = React.useState(0); + const healthOk = healthResponse?.overallHealth; + + React.useEffect(() => { + const now = new Date(); + const lastAskedDate = lastAsked ? new Date(parseInt(lastAsked, 10)) : null; + const diff = lastAskedDate ? now.getTime() - lastAskedDate.getTime() : 0; + const diffInMinutes = Math.round(diff / 1000 / 60); + setLastAskedDiff(diffInMinutes); + }, [lastAsked]); + + const title = healthOk ? 'All Systems Operational' : 'Service disruption'; + const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); + + const problematicServices = Object.values(healthResponse?.healthMap || {}).filter( + (service: Record) => !service.healthOk + ) + + return ( +
+
+
+ + {title} +
+
+ Last checked {lastAskedDiff} mins. ago +
getHealth()} + > + +
+
+
+ +
+ {!healthOk ? ( + <> +
+ Observed installation Issue with the following +
+ {problematicServices.map((service) => ( + + setShowModal(true)} + healthOk={false} + name={service.name} + /> + + ))} + + ) : null} +
+
+
+ ); +} + +export default HealthWidget \ No newline at end of file diff --git a/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx b/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx new file mode 100644 index 000000000..3c9259c39 --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx @@ -0,0 +1,43 @@ +import { Icon } from 'UI'; +import React from 'react'; +import cn from 'classnames'; + +function Category({ + name, + healthOk, + onClick, + isSelectable, + isExpandable, + isExpanded, + isSelected, +}: { + name: string; + healthOk: boolean; + onClick: (args: any) => void; + isSelectable?: boolean; + isExpandable?: boolean; + isExpanded?: boolean; + isSelected?: boolean; +}) { + const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); + return ( +
+ + {name} + + {isSelectable ? : null} + {isExpandable ? ( + + ) : null} +
+ ); +} + +export default Category \ No newline at end of file diff --git a/frontend/app/components/Header/HealthStatus/SubserviceHealth/SubserviceHealth.tsx b/frontend/app/components/Header/HealthStatus/SubserviceHealth/SubserviceHealth.tsx new file mode 100644 index 000000000..4de64ffbe --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/SubserviceHealth/SubserviceHealth.tsx @@ -0,0 +1,52 @@ +import React from 'react'; +import Category from 'Components/Header/HealthStatus/ServiceCategory'; +import cn from 'classnames'; + +function SubserviceHealth({ + subservice, + name, +}: { + name: string; + subservice: { health: boolean; details: { errors?: string[]; version?: string } }; +}) { + const [isExpanded, setIsExpanded] = React.useState(!subservice?.health); + + const isExpandable = subservice?.details && Object.keys(subservice?.details).length > 0; + return ( +
+ (isExpandable ? setIsExpanded(!isExpanded) : null)} + name={name} + healthOk={subservice?.health} + isExpandable={isExpandable} + isExpanded={isExpanded} + /> + {isExpanded ? ( +
+ {subservice?.details?.version ? ( +
+
Version
+
+ {subservice?.details?.version} +
+
+ ) : null} + {subservice?.details?.errors?.length ? ( +
+
Error log:
+ {subservice.details.errors.map((err: string, i) => ( +
+ {i + 1}. {err} +
+ ))} +
+ ) : subservice?.health ? null : ( + 'Service not responding' + )} +
+ ) : null} +
+ ); +} + +export default SubserviceHealth; diff --git a/frontend/app/components/Header/HealthStatus/const.ts b/frontend/app/components/Header/HealthStatus/const.ts new file mode 100644 index 000000000..3c13c52dd --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/const.ts @@ -0,0 +1,6 @@ +export const categoryKeyNames = { + backendServices: 'Backend Services', + databases: 'Databases', + ingestionPipeline: 'Ingestion Pipeline', + ssl: 'SSL', +} as const \ No newline at end of file From cbd4e4f6693b90e36dee1daabdc338bb712b05e7 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 15 Mar 2023 16:01:48 +0100 Subject: [PATCH 153/253] change(ui): small ui fixes --- .../components/Header/HealthStatus/HealthModal/HealthModal.tsx | 3 ++- frontend/app/components/Header/HealthStatus/HealthWidget.tsx | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx index b0bbb8969..a78ba4ad1 100644 --- a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx @@ -34,6 +34,7 @@ function HealthModal({ const handleClose = () => { setShowModal(false); }; + return (
-
diff --git a/frontend/app/components/Header/HealthStatus/HealthWidget.tsx b/frontend/app/components/Header/HealthStatus/HealthWidget.tsx index c6372540b..50b4de76d 100644 --- a/frontend/app/components/Header/HealthStatus/HealthWidget.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthWidget.tsx @@ -79,6 +79,7 @@ function HealthWidget({ onClick={() => setShowModal(true)} healthOk={false} name={service.name} + isSelectable /> ))} From b02bf8c23d794ca6b8d845a131d9f295d002483c Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 23 Mar 2023 13:52:08 +0100 Subject: [PATCH 154/253] change(ui): fix loading animation, add health modal to signup page (/cached in localstorage) --- .../HealthStatus/HealthModal/Footer.tsx | 9 +- .../HealthStatus/HealthModal/HealthModal.tsx | 85 ++++++++++++------- .../Header/HealthStatus/HealthStatus.tsx | 45 +++------- .../Header/HealthStatus/ServiceCategory.tsx | 10 ++- .../components/Header/HealthStatus/const.ts | 5 +- .../Header/HealthStatus/getHealth.ts | 36 ++++++++ frontend/app/components/Signup/Signup.js | 38 +++++++++ frontend/app/components/ui/SVG.tsx | 3 +- frontend/app/svg/icons/book-doc.svg | 10 +++ 9 files changed, 172 insertions(+), 69 deletions(-) create mode 100644 frontend/app/components/Header/HealthStatus/getHealth.ts create mode 100644 frontend/app/svg/icons/book-doc.svg diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/Footer.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/Footer.tsx index 43bd434fc..0daf5cf56 100644 --- a/frontend/app/components/Header/HealthStatus/HealthModal/Footer.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthModal/Footer.tsx @@ -1,9 +1,14 @@ import React from 'react'; import { Icon } from 'UI'; +import cn from 'classnames' -function Footer() { +function Footer({ isSetup }: { isSetup?: boolean }) { return ( -
+
void; isLoading: boolean; healthResponse: { overallHealth: boolean; healthMap: Record }; setShowModal: (isOpen: boolean) => void; + setPassed?: () => void; }) { const [selectedService, setSelectedService] = React.useState(''); React.useEffect(() => { - if (!healthResponse.overallHealth) { - setSelectedService( - Object.keys(healthResponse.healthMap).filter( - (s) => !healthResponse.healthMap[s].healthOk - )[0] - ); + if (!healthResponse?.overallHealth) { + if (healthResponse?.healthMap) { + setSelectedService( + Object.keys(healthResponse.healthMap).filter( + (s) => !healthResponse.healthMap[s].healthOk + )[0] + ); + } } }, [healthResponse]); @@ -35,6 +40,8 @@ function HealthModal({ setShowModal(false); }; + const isSetup = document.location.pathname.includes('/signup') + return (
e.stopPropagation()} @@ -78,37 +85,57 @@ function HealthModal({
- {Object.keys(healthResponse.healthMap).map((service) => ( - - setSelectedService(service)} - healthOk={healthResponse.healthMap[service].healthOk} - name={healthResponse.healthMap[service].name} - isSelectable - isSelected={selectedService === service} - /> - - ))} + {isLoading ? ( + null} name={"Loading health status"} isLoading /> + ) + : Object.keys(healthResponse.healthMap).map((service) => ( + + setSelectedService(service)} + healthOk={healthResponse.healthMap[service].healthOk} + name={healthResponse.healthMap[service].name} + isSelectable + isSelected={selectedService === service} + /> + + ))}
- {selectedService ? ( + {isLoading ? ( +
+ +
+ ) : selectedService ? ( - ) : ( - - )} + ) : + }
-
- -
-
+ {isSetup ? ( +
+ +
+ ) : null} +
); diff --git a/frontend/app/components/Header/HealthStatus/HealthStatus.tsx b/frontend/app/components/Header/HealthStatus/HealthStatus.tsx index 7732bca9d..317b36279 100644 --- a/frontend/app/components/Header/HealthStatus/HealthStatus.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthStatus.tsx @@ -1,9 +1,9 @@ import React from 'react'; import { Icon } from 'UI'; import HealthModal from 'Components/Header/HealthStatus/HealthModal/HealthModal'; -import { healthService } from 'App/services'; -import { categoryKeyNames } from './const'; +import { lastAskedKey, healthResponseKey } from './const'; import HealthWidget from "Components/Header/HealthStatus/HealthWidget"; +import { getHealthRequest } from './getHealth' export interface IServiceStats { name: 'backendServices' | 'databases' | 'ingestionPipeline' | 'ssl'; @@ -18,32 +18,8 @@ export interface IServiceStats { }[] } -function mapResponse(resp: Record) { - const services = Object.keys(resp); - const healthMap: Record = {}; - services.forEach((service) => { - healthMap[service] = { - // @ts-ignore - name: categoryKeyNames[service], - healthOk: true, - subservices: resp[service], - serviceName: service, - }; - Object.values(healthMap[service].subservices).forEach((subservice: Record) => { - if (!subservice?.health) healthMap[service].healthOk = false; - }); - }); - - const overallHealth = Object.values(healthMap).every( - (service: Record) => service.healthOk - ); - - return { overallHealth, healthMap }; -} function HealthStatus() { - const lastAskedKey = '__openreplay_health_status'; - const healthResponseKey = '__openreplay_health_response'; const healthResponseSaved = localStorage.getItem(healthResponseKey) || '{}'; const [healthResponse, setHealthResponse] = React.useState(JSON.parse(healthResponseSaved)); const [isLoading, setIsLoading] = React.useState(false); @@ -55,12 +31,8 @@ function HealthStatus() { if (isLoading) return; try { setIsLoading(true); - const r = await healthService.fetchStatus(); - const healthMap = mapResponse(r); + const { healthMap, asked } = await getHealthRequest(); setHealthResponse(healthMap); - const asked = new Date().getTime(); - localStorage.setItem(healthResponseKey, JSON.stringify(healthMap)); - localStorage.setItem(lastAskedKey, asked.toString()); setLastAsked(asked.toString()); } catch (e) { console.error(e); @@ -82,10 +54,10 @@ function HealthStatus() { const icon = healthResponse?.overallHealth ? 'pulse' : ('exclamation-circle-fill' as const); return ( <> -
+
@@ -102,7 +74,12 @@ function HealthStatus() { />
{showModal ? ( - + ) : null} ); diff --git a/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx b/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx index 3c9259c39..be5edec1f 100644 --- a/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx +++ b/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx @@ -1,6 +1,7 @@ import { Icon } from 'UI'; import React from 'react'; import cn from 'classnames'; +import AnimatedSVG, { ICONS } from 'Shared/AnimatedSVG/AnimatedSVG'; function Category({ name, @@ -10,15 +11,18 @@ function Category({ isExpandable, isExpanded, isSelected, + isLoading, }: { name: string; - healthOk: boolean; + healthOk?: boolean; + isLoading?: boolean; onClick: (args: any) => void; isSelectable?: boolean; isExpandable?: boolean; isExpanded?: boolean; isSelected?: boolean; }) { + const icon = healthOk ? ('check-circle-fill' as const) : ('exclamation-circle-fill' as const); return (
- + {isLoading ? ( + + ) : } {name} {isSelectable ? : null} diff --git a/frontend/app/components/Header/HealthStatus/const.ts b/frontend/app/components/Header/HealthStatus/const.ts index 3c13c52dd..69b5b1c5e 100644 --- a/frontend/app/components/Header/HealthStatus/const.ts +++ b/frontend/app/components/Header/HealthStatus/const.ts @@ -3,4 +3,7 @@ export const categoryKeyNames = { databases: 'Databases', ingestionPipeline: 'Ingestion Pipeline', ssl: 'SSL', -} as const \ No newline at end of file +} as const + +export const lastAskedKey = '__openreplay_health_status'; +export const healthResponseKey = '__openreplay_health_response'; \ No newline at end of file diff --git a/frontend/app/components/Header/HealthStatus/getHealth.ts b/frontend/app/components/Header/HealthStatus/getHealth.ts new file mode 100644 index 000000000..70bd8914c --- /dev/null +++ b/frontend/app/components/Header/HealthStatus/getHealth.ts @@ -0,0 +1,36 @@ +import { healthService } from 'App/services'; +import { categoryKeyNames, lastAskedKey, healthResponseKey } from "Components/Header/HealthStatus/const"; +import { IServiceStats } from "Components/Header/HealthStatus/HealthStatus"; + + +function mapResponse(resp: Record) { + const services = Object.keys(resp); + const healthMap: Record = {}; + services.forEach((service) => { + healthMap[service] = { + // @ts-ignore + name: categoryKeyNames[service], + healthOk: true, + subservices: resp[service], + serviceName: service, + }; + Object.values(healthMap[service].subservices).forEach((subservice: Record) => { + if (!subservice?.health) healthMap[service].healthOk = false; + }); + }); + + const overallHealth = Object.values(healthMap).every( + (service: Record) => service.healthOk + ); + + return { overallHealth, healthMap }; +} + +export async function getHealthRequest() { + const r = await healthService.fetchStatus(); + const healthMap = mapResponse(r); + const asked = new Date().getTime(); + localStorage.setItem(healthResponseKey, JSON.stringify(healthMap)); + localStorage.setItem(lastAskedKey, asked.toString()); + return { healthMap, asked } +} \ No newline at end of file diff --git a/frontend/app/components/Signup/Signup.js b/frontend/app/components/Signup/Signup.js index 83a658ec1..f5d61564f 100644 --- a/frontend/app/components/Signup/Signup.js +++ b/frontend/app/components/Signup/Signup.js @@ -6,6 +6,8 @@ import stl from './signup.module.css'; import cn from 'classnames'; import SignupForm from './SignupForm'; import RegisterBg from '../../svg/register.svg'; +import HealthModal from 'Components/Header/HealthStatus/HealthModal/HealthModal'; +import { getHealthRequest } from 'Components/Header/HealthStatus/getHealth'; const BulletItem = ({ text }) => (
@@ -15,9 +17,45 @@ const BulletItem = ({ text }) => (
{text}
); + +const healthStatusCheck_key = '__or__healthStatusCheck_key' + @withPageTitle('Signup - OpenReplay') export default class Signup extends React.Component { + state = { + healthModalPassed: localStorage.getItem(healthStatusCheck_key === 'true'), + healthStatusLoading: true, + healthStatus: null, + } + + getHealth = async () => { + this.setState({ healthStatusLoading: true }); + const { healthMap } = await getHealthRequest(); + this.setState({ healthStatus: healthMap, healthStatusLoading: false }); + } + + componentDidMount() { + if (!this.state.healthModalPassed) void this.getHealth(); + } + + setHealthModalPassed = () => { + localStorage.setItem(healthStatusCheck_key, 'true'); + this.setState({ healthModalPassed: true }); + } + render() { + if (!this.state.healthModalPassed) { + return ( + null} + healthResponse={this.state.healthStatus} + getHealth={this.getHealth} + isLoading={this.state.healthStatusLoading} + setPassed={this.setHealthModalPassed} + /> + ) + } + return (
diff --git a/frontend/app/components/ui/SVG.tsx b/frontend/app/components/ui/SVG.tsx index 95254b16c..a2322c730 100644 --- a/frontend/app/components/ui/SVG.tsx +++ b/frontend/app/components/ui/SVG.tsx @@ -1,7 +1,7 @@ import React from 'react'; -export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'click-hesitation' | 'click-rage' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'cursor-trash' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope' | 'errors-icon' | 'event/click' | 'event/click_hesitation' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/input_hesitation' | 'event/link' | 'event/location' | 'event/mouse_thrashing' | 'event/resize' | 'event/view' | 'exclamation-circle-fill' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'input-hesitation' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'pulse' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; +export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book-doc' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'click-hesitation' | 'click-rage' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'cursor-trash' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope' | 'errors-icon' | 'event/click' | 'event/click_hesitation' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/input_hesitation' | 'event/link' | 'event/location' | 'event/mouse_thrashing' | 'event/resize' | 'event/view' | 'exclamation-circle-fill' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'input-hesitation' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'pulse' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; interface Props { name: IconNames; @@ -78,6 +78,7 @@ const SVG = (props: Props) => { case 'bell-slash': return ; case 'bell': return ; case 'binoculars': return ; + case 'book-doc': return ; case 'book': return ; case 'browser/browser': return ; case 'browser/chrome': return ; diff --git a/frontend/app/svg/icons/book-doc.svg b/frontend/app/svg/icons/book-doc.svg new file mode 100644 index 000000000..7e6f2a680 --- /dev/null +++ b/frontend/app/svg/icons/book-doc.svg @@ -0,0 +1,10 @@ + + + + + + + + + + From 4bdb30daa3d11679e7f341244a26eb257e020078 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 23 Mar 2023 13:54:14 +0100 Subject: [PATCH 155/253] change(ui): fix error printing --- .../Header/HealthStatus/HealthModal/HealthModal.tsx | 2 +- .../HealthStatus/SubserviceHealth/SubserviceHealth.tsx | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx index 0587e5423..b0a6fb9fa 100644 --- a/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx +++ b/frontend/app/components/Header/HealthStatus/HealthModal/HealthModal.tsx @@ -135,7 +135,7 @@ function HealthModal({
) : null} -
+
); diff --git a/frontend/app/components/Header/HealthStatus/SubserviceHealth/SubserviceHealth.tsx b/frontend/app/components/Header/HealthStatus/SubserviceHealth/SubserviceHealth.tsx index 4de64ffbe..6fd91031b 100644 --- a/frontend/app/components/Header/HealthStatus/SubserviceHealth/SubserviceHealth.tsx +++ b/frontend/app/components/Header/HealthStatus/SubserviceHealth/SubserviceHealth.tsx @@ -34,11 +34,7 @@ function SubserviceHealth({ {subservice?.details?.errors?.length ? (
Error log:
- {subservice.details.errors.map((err: string, i) => ( -
- {i + 1}. {err} -
- ))} + {subservice.details.errors.toString()}
) : subservice?.health ? null : ( 'Service not responding' From c1568b0929df6e05aa5381171e3ef6879b1c024a Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 24 Mar 2023 11:13:27 +0100 Subject: [PATCH 156/253] change(ui): fix logs --- .../app/player/web/managers/DOM/DOMManager.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index ab839118b..a5c2f2c2f 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -257,13 +257,9 @@ export default class DOMManager extends ListWalker { } return case MType.RemoveNodeAttribute: - if (isJump) { - this.attrsBacktrack = this.attrsBacktrack.filter(m => m.id !== msg.id && m.name !== msg.name) - } else { vn = this.vElements.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } vn.removeAttribute(msg.name) - } return case MType.SetInputValue: vn = this.vElements.get(msg.id) @@ -478,23 +474,10 @@ export default class DOMManager extends ListWalker { * */ // http://0.0.0.0:3333/5/session/8452905874437457 // 70 iframe, 8 create element - STYLE tag - console.time('moveWait') - let t0 = performance.now() - let t1 = t0 - const timings = [] await this.moveWait(t, (msg) => { - t0 = performance.now() this.applyMessage(msg, isJump) - t1 = performance.now() - timings.push({ t: t1 - t0, m: msg.tp, msg }) }) - console.timeEnd('moveWait') - console.log( - timings.sort((a, b) => b.t - a.t), - timings.filter(t => t.msg.tag === 'STYLE').length, - ) - if (isJump) { this.attrsBacktrack.forEach(msg => { this.applyBacktrack(msg) From 5ddb0bbad4cb9bb33f73622dcd698d6a30a9cbd8 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 8 Mar 2023 09:37:48 +0100 Subject: [PATCH 157/253] feat(tracker): track network requests in iframes --- tracker/tracker/src/main/app/index.ts | 5 ++ .../src/main/app/observer/iframe_observer.ts | 4 ++ tracker/tracker/src/main/modules/network.ts | 64 +++++++++++++++---- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index faca22b9d..4c770f664 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -14,6 +14,8 @@ import type { Options as ObserverOptions } from './observer/top_observer.js' import type { Options as SanitizerOptions } from './sanitizer.js' import type { Options as LoggerOptions } from './logger.js' import type { Options as SessOptions } from './session.js' +import type { Options as NetworkOptions } from '../modules/network.js' + import type { Options as WebworkerOptions, ToWorkerData, @@ -75,6 +77,7 @@ type AppOptions = { // @deprecated onStart?: StartCallback + network?: NetworkOptions } & WebworkerOptions & SessOptions @@ -99,6 +102,7 @@ export default class App { private readonly stopCallbacks: Array<() => any> = [] private readonly commitCallbacks: Array = [] private readonly options: AppOptions + public readonly networkOptions?: NetworkOptions private readonly revID: string private activityState: ActivityState = ActivityState.NotActive private readonly version = 'TRACKER_VERSION' // TODO: version compatability check inside each plugin. @@ -109,6 +113,7 @@ export default class App { // } ?? maybe onStart is good this.projectKey = projectKey + this.networkOptions = options.network this.options = Object.assign( { revID: '', diff --git a/tracker/tracker/src/main/app/observer/iframe_observer.ts b/tracker/tracker/src/main/app/observer/iframe_observer.ts index 05df1fe54..e3b82b1d9 100644 --- a/tracker/tracker/src/main/app/observer/iframe_observer.ts +++ b/tracker/tracker/src/main/app/observer/iframe_observer.ts @@ -1,10 +1,13 @@ import Observer from './observer.js' import { CreateIFrameDocument } from '../messages.gen.js' +import Network from '../../modules/network.js' export default class IFrameObserver extends Observer { observe(iframe: HTMLIFrameElement) { const doc = iframe.contentDocument + const iWindow = iframe.contentWindow const hostID = this.app.nodes.getID(iframe) + console.log(iframe) if (!doc || hostID === undefined) { return } //log TODO common app.logger @@ -16,6 +19,7 @@ export default class IFrameObserver extends Observer { return } this.app.send(CreateIFrameDocument(hostID, docID)) + Network(this.app, this.app.networkOptions, iWindow!) }) } } diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts index a1a3e2c9c..097648d54 100644 --- a/tracker/tracker/src/main/modules/network.ts +++ b/tracker/tracker/src/main/modules/network.ts @@ -92,7 +92,7 @@ export interface Options { sanitizer?: Sanitizer } -export default function (app: App, opts: Partial = {}) { +export default function (app: App, opts: Partial = {}, customEnv?: Record) { const options: Options = Object.assign( { failuresOnly: false, @@ -150,8 +150,11 @@ export default function (app: App, opts: Partial = {}) { } /* ====== Fetch ====== */ - const origFetch = window.fetch.bind(window) as WindowFetch - window.fetch = (input, init = {}) => { + const origFetch = customEnv + ? (customEnv.fetch.bind(customEnv) as WindowFetch) + : (window.fetch.bind(window) as WindowFetch) + + const trackFetch = (input: RequestInfo | URL, init: RequestInit = {}) => { if (!(typeof input === 'string' || input instanceof URL) || app.isServiceURL(String(input))) { return origFetch(input, init) } @@ -237,12 +240,23 @@ export default function (app: App, opts: Partial = {}) { return response }) } + + if (customEnv) { + customEnv.fetch = trackFetch + } else { + window.fetch = trackFetch + } /* ====== <> ====== */ /* ====== XHR ====== */ - const nativeOpen = XMLHttpRequest.prototype.open - XMLHttpRequest.prototype.open = function (initMethod, url) { - const xhr = this + + const nativeOpen = customEnv + ? customEnv.XMLHttpRequest.prototype.open + : XMLHttpRequest.prototype.open + + function trackXMLHttpReqOpen(initMethod: string, url: string | URL) { + // @ts-ignore ??? this -> XMLHttpRequest + const xhr = this as XMLHttpRequest setSessionTokenHeader((name, value) => xhr.setRequestHeader(name, value)) let startTime = 0 @@ -302,23 +316,47 @@ export default function (app: App, opts: Partial = {}) { //TODO: handle error (though it has no Error API nor any useful information) //xhr.addEventListener('error', (e) => {}) - return nativeOpen.apply(this, arguments) + // @ts-ignore ??? this -> XMLHttpRequest + return nativeOpen.apply(this as XMLHttpRequest, arguments) } + if (customEnv) { + customEnv.XMLHttpRequest.prototype.open = trackXMLHttpReqOpen.bind(customEnv) + } else { + XMLHttpRequest.prototype.open = trackXMLHttpReqOpen + } + const nativeSend = XMLHttpRequest.prototype.send - XMLHttpRequest.prototype.send = function (body) { - const rdo = getXHRRequestDataObject(this) + function trackXHRSend(body: Document | XMLHttpRequestBodyInit | null | undefined) { + // @ts-ignore ??? this -> XMLHttpRequest + const rdo = getXHRRequestDataObject(this as XMLHttpRequest) rdo.body = body - return nativeSend.apply(this, arguments) + // @ts-ignore ??? this -> XMLHttpRequest + return nativeSend.apply(this as XMLHttpRequest, arguments) } + + if (customEnv) { + customEnv.XMLHttpRequest.prototype.send = trackXHRSend.bind(customEnv) + } else { + XMLHttpRequest.prototype.send = trackXHRSend + } + const nativeSetRequestHeader = XMLHttpRequest.prototype.setRequestHeader - XMLHttpRequest.prototype.setRequestHeader = function (name, value) { + + function trackSetReqHeader(name: string, value: string) { if (!isHIgnored(name)) { - const rdo = getXHRRequestDataObject(this) + // @ts-ignore ??? this -> XMLHttpRequest + const rdo = getXHRRequestDataObject(this as XMLHttpRequest) rdo.headers[name] = value } + // @ts-ignore ??? this -> XMLHttpRequest + return nativeSetRequestHeader.apply(this as XMLHttpRequest, arguments) + } - return nativeSetRequestHeader.apply(this, arguments) + if (customEnv) { + customEnv.XMLHttpRequest.prototype.setRequestHeader = trackSetReqHeader.bind(customEnv) + } else { + XMLHttpRequest.prototype.setRequestHeader = trackSetReqHeader } /* ====== <> ====== */ } From bd935b2f979b38d1309f93f464f6a7031960629d Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 9 Mar 2023 16:45:33 +0100 Subject: [PATCH 158/253] fix(tracker): iframe network tracking --- frontend/app/logger/index.js | 23 +++++++++++++------ .../app/player/web/messages/MFileReader.ts | 6 ++--- .../src/main/app/observer/iframe_observer.ts | 4 ---- .../src/main/app/observer/top_observer.ts | 2 ++ 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/frontend/app/logger/index.js b/frontend/app/logger/index.js index 353f186e9..caf6d7bee 100644 --- a/frontend/app/logger/index.js +++ b/frontend/app/logger/index.js @@ -24,18 +24,27 @@ function error(...args) { } let groupTm = {}; +let groupedLogs = {}; function group(groupName, ...args) { if (!window.env.PRODUCTION || options.verbose) { - if (!groupTm[groupName]) { - groupTm[groupName] = setTimeout(() => { - console.groupEnd() - delete groupTm[groupName] - }, 500); - console.groupCollapsed(groupName); + if (groupTm[groupName]) { + clearTimeout(groupTm[groupName]) + groupTm[groupName] = null + } else { + groupedLogs[groupName] = [] } - console.log(...args); + groupedLogs[groupName].push(args); + groupTm[groupName] = setTimeout(() => { + console.groupCollapsed(groupName) + groupedLogs[groupName].forEach((log) => { + console.log(...log) + }) + console.groupEnd() + delete groupTm[groupName] + delete groupedLogs[groupName] + }, 500) options.exceptionsLogs.push(args) } } diff --git a/frontend/app/player/web/messages/MFileReader.ts b/frontend/app/player/web/messages/MFileReader.ts index d1b131595..b5fdde85c 100644 --- a/frontend/app/player/web/messages/MFileReader.ts +++ b/frontend/app/player/web/messages/MFileReader.ts @@ -3,7 +3,7 @@ import type { RawMessage } from './raw.gen'; import { MType } from './raw.gen'; import RawMessageReader from './RawMessageReader.gen'; import resolveURL from './urlBasedResolver' - +import Logger from 'App/logger' // TODO: composition instead of inheritance // needSkipMessage() and next() methods here use buf and p protected properties, @@ -59,10 +59,8 @@ export default class MFileReader extends RawMessageReader { if (!skippedMessage) { return null } - this.logger.group("Openreplay: Skipping messages ", skippedMessage) - + Logger.group("Openreplay: Skipping messages ", skippedMessage) } - this.pLastMessageID = this.p const rMsg = this.readRawMessage() diff --git a/tracker/tracker/src/main/app/observer/iframe_observer.ts b/tracker/tracker/src/main/app/observer/iframe_observer.ts index e3b82b1d9..05df1fe54 100644 --- a/tracker/tracker/src/main/app/observer/iframe_observer.ts +++ b/tracker/tracker/src/main/app/observer/iframe_observer.ts @@ -1,13 +1,10 @@ import Observer from './observer.js' import { CreateIFrameDocument } from '../messages.gen.js' -import Network from '../../modules/network.js' export default class IFrameObserver extends Observer { observe(iframe: HTMLIFrameElement) { const doc = iframe.contentDocument - const iWindow = iframe.contentWindow const hostID = this.app.nodes.getID(iframe) - console.log(iframe) if (!doc || hostID === undefined) { return } //log TODO common app.logger @@ -19,7 +16,6 @@ export default class IFrameObserver extends Observer { return } this.app.send(CreateIFrameDocument(hostID, docID)) - Network(this.app, this.app.networkOptions, iWindow!) }) } } diff --git a/tracker/tracker/src/main/app/observer/top_observer.ts b/tracker/tracker/src/main/app/observer/top_observer.ts index 38944c5c9..7eb15c15b 100644 --- a/tracker/tracker/src/main/app/observer/top_observer.ts +++ b/tracker/tracker/src/main/app/observer/top_observer.ts @@ -1,5 +1,6 @@ import Observer from './observer.js' import { isElementNode, hasTag } from '../guards.js' +import Network from '../../modules/network.js' import IFrameObserver from './iframe_observer.js' import ShadowRootObserver from './shadow_root_observer.js' @@ -92,6 +93,7 @@ export default class TopObserver extends Observer { //TODO: more explicit logic ) { this.contextsSet.add(currentWin) + Network(this.app, this.app.networkOptions, currentWin) //@ts-ignore https://github.com/microsoft/TypeScript/issues/41684 this.contextCallbacks.forEach((cb) => cb(currentWin)) } From 1cf9e54f5a9383053da534650a009f6ccecb5772 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Fri, 24 Mar 2023 12:07:20 +0100 Subject: [PATCH 159/253] change(tracker): more configs for mouse module --- tracker/tracker/CHANGELOG.md | 3 ++- tracker/tracker/src/main/modules/mouse.ts | 33 ++++++++++++++++++----- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index 90c68a91f..fa8ab398a 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -4,7 +4,8 @@ - Capture DOM node drop event (>30% nodes removed) - Capture iframe network requests - Detect cached requests to img, css and js resources; send transferred size -- added `{ network: { disableClickmaps: boolean } }` to disable calculating el. selectors +- added `{ mouse: { disableClickmaps: boolean } }` to disable calculating el. selectors +- added `{ mouse: { minSelectorDepth?: number; nthThreshold?: number; maxOptimiseTries?: number }` for selector finding optimisations ## 5.0.1 diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index fb69bef08..5bba2ddda 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -5,13 +5,13 @@ import { MouseMove, MouseClick, MouseThrashing } from '../app/messages.gen.js' import { getInputLabel } from './input.js' import { finder } from '@medv/finder' -function _getSelector(target: Element, document: Document) { +function _getSelector(target: Element, document: Document, options?: MouseHandlerOptions): string { const selector = finder(target, { root: document.body, seedMinLength: 3, - optimizedMinLength: 2, - threshold: 1000, - maxNumberOfTries: 10_000, + optimizedMinLength: options?.minSelectorDepth || 2, + threshold: options?.nthThreshold || 1000, + maxNumberOfTries: options?.maxOptimiseTries || 10_000, }) return selector @@ -75,6 +75,25 @@ function _getTarget(target: Element, document: Document): Element | null { export interface MouseHandlerOptions { disableClickmaps?: boolean + /** minimum length of an optimised selector. + * + * body > div > div > p => body > p for example + * + * default 2 + * */ + minSelectorDepth?: number + /** how many selectors to try before falling back to nth-child selectors + * performance expensive operation + * + * default 1000 + * */ + nthThreshold?: number + /** + * how many tries to optimise and shorten the selector + * + * default 10_000 + * */ + maxOptimiseTries?: number } export default function (app: App, options?: MouseHandlerOptions): void { @@ -155,8 +174,8 @@ export default function (app: App, options?: MouseHandlerOptions): void { } const patchDocument = (document: Document, topframe = false) => { - function getSelector(id: number, target: Element): string { - return (selectorMap[id] = selectorMap[id] || _getSelector(target, document)) + function getSelector(id: number, target: Element, options?: MouseHandlerOptions): string { + return (selectorMap[id] = selectorMap[id] || _getSelector(target, document, options)) } const attachListener = topframe @@ -202,7 +221,7 @@ export default function (app: App, options?: MouseHandlerOptions): void { id, mouseTarget === target ? Math.round(performance.now() - mouseTargetTime) : 0, getTargetLabel(target), - isClickable(target) && !disableClickmaps ? getSelector(id, target) : '', + isClickable(target) && !disableClickmaps ? getSelector(id, target, options) : '', ), true, ) From ff762b9853bcc939cdd945f8c51cbf1a21257bae Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 24 Mar 2023 13:30:00 +0100 Subject: [PATCH 160/253] feat(chalice): changed health check endpoint --- api/chalicelib/core/health.py | 17 +------- ee/api/chalicelib/core/health.py | 68 +++++++++++++------------------- 2 files changed, 30 insertions(+), 55 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index f4e2abc6c..a9a54977c 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -14,7 +14,6 @@ if config("LOCAL_DEV", cast=bool, default=False): "chalice": "http://127.0.0.1:8888/metrics", "db": "http://127.0.0.1:8888/metrics", "ender": "http://127.0.0.1:8888/metrics", - "frontend": "http://127.0.0.1:8888/metrics", "heuristics": "http://127.0.0.1:8888/metrics", "http": "http://127.0.0.1:8888/metrics", "ingress-nginx": "http://127.0.0.1:8888/metrics", @@ -35,7 +34,6 @@ else: "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", - "frontend": "http://frontend-openreplay.app.svc.cluster.local:8888/metrics", "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics", "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", @@ -157,7 +155,7 @@ def get_health(): "chalice": __always_healthy_with_version, "db": __check_be_service("db"), "ender": __check_be_service("ender"), - "frontend": __check_be_service("frontend"), + "frontend": __always_healthy, "heuristics": __check_be_service("heuristics"), "http": __check_be_service("http"), "ingress-nginx": __always_healthy, @@ -166,18 +164,7 @@ def get_health(): "sink": __check_be_service("sink"), "sourcemapreader": __check_be_service("sourcemapreader"), "storage": __check_be_service("storage") - }, - # "overall": { - # "health": "na", - # "details": { - # "numberOfEventCaptured": "int", - # "numberOfSessionsCaptured": "int" - # }, - # "labels": { - # "parent": "information" - # } - # }, - # "ssl": True + } } for parent_key in health_map.keys(): for element_key in health_map[parent_key]: diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index e00747288..4b95888d1 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -15,7 +15,6 @@ if config("LOCAL_DEV", cast=bool, default=False): "chalice": "http://127.0.0.1:8888/metrics", "db": "http://127.0.0.1:8888/metrics", "ender": "http://127.0.0.1:8888/metrics", - "frontend": "http://127.0.0.1:8888/metrics", "heuristics": "http://127.0.0.1:8888/metrics", "http": "http://127.0.0.1:8888/metrics", "ingress-nginx": "http://127.0.0.1:8888/metrics", @@ -36,7 +35,6 @@ else: "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", - "frontend": "http://frontend-openreplay.app.svc.cluster.local:8888/metrics", "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics", "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", @@ -153,6 +151,7 @@ def get_health(): "ingestionPipeline": { "redis": __check_redis, # "kafka": __check_kafka + "kafka": __always_healthy }, "backendServices": { "alerts": __check_be_service("alerts"), @@ -161,7 +160,7 @@ def get_health(): "chalice": __always_healthy_with_version, "db": __check_be_service("db"), "ender": __check_be_service("ender"), - "frontend": __check_be_service("frontend"), + "frontend": __always_healthy, "heuristics": __check_be_service("heuristics"), "http": __check_be_service("http"), "ingress-nginx": __always_healthy, @@ -171,18 +170,7 @@ def get_health(): "sink": __check_be_service("sink"), "sourcemapreader": __check_be_service("sourcemapreader"), "storage": __check_be_service("storage") - }, - # "overall": { - # "health": "na", - # "details": { - # "numberOfEventCaptured": "int", - # "numberOfSessionsCaptured": "int" - # }, - # "labels": { - # "parent": "information" - # } - # }, - # "ssl": True + } } for parent_key in health_map.keys(): for element_key in health_map[parent_key]: @@ -213,28 +201,28 @@ def __check_database_ch(): } -def __check_kafka(): - fail_response = { - "health": False, - "details": {"errors": ["server health-check failed"]} - } - if config("KAFKA_SERVERS", default=None) is None: - fail_response["details"]["errors"].append("KAFKA_SERVERS not defined in env-vars") - return fail_response - - try: - a = AdminClient({'bootstrap.servers': config("KAFKA_SERVERS"), "socket.connection.setup.timeout.ms": 3000}) - topics = a.list_topics().topics - if not topics: - raise Exception('topics not found') - - except Exception as e: - print("!! Issue getting kafka-health response") - print(str(e)) - fail_response["details"]["errors"].append(str(e)) - return fail_response - - return { - "health": True, - "details": {} - } +# def __check_kafka(): +# fail_response = { +# "health": False, +# "details": {"errors": ["server health-check failed"]} +# } +# if config("KAFKA_SERVERS", default=None) is None: +# fail_response["details"]["errors"].append("KAFKA_SERVERS not defined in env-vars") +# return fail_response +# +# try: +# a = AdminClient({'bootstrap.servers': config("KAFKA_SERVERS"), "socket.connection.setup.timeout.ms": 3000}) +# topics = a.list_topics().topics +# if not topics: +# raise Exception('topics not found') +# +# except Exception as e: +# print("!! Issue getting kafka-health response") +# print(str(e)) +# fail_response["details"]["errors"].append(str(e)) +# return fail_response +# +# return { +# "health": True, +# "details": {} +# } From 7adf87a32362c2b71dbecc967f5c5e1f39aab367 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 24 Mar 2023 15:58:07 +0100 Subject: [PATCH 161/253] change(ui) - login changes --- .../ForgotPassword/ForgotPassword.js | 293 ++++++++++-------- frontend/app/components/Login/Login.js | 43 +-- frontend/app/components/Signup/Signup.js | 27 +- .../Signup/SignupForm/SignupForm.js | 152 +++++---- frontend/app/components/ui/Button/Button.tsx | 2 +- frontend/app/components/ui/SVG.tsx | 5 +- frontend/app/svg/icons/buildings.svg | 4 + frontend/app/svg/icons/envelope-check.svg | 4 + frontend/app/svg/icons/key.svg | 4 + 9 files changed, 294 insertions(+), 240 deletions(-) create mode 100644 frontend/app/svg/icons/buildings.svg create mode 100644 frontend/app/svg/icons/envelope-check.svg create mode 100644 frontend/app/svg/icons/key.svg diff --git a/frontend/app/components/ForgotPassword/ForgotPassword.js b/frontend/app/components/ForgotPassword/ForgotPassword.js index 1cd40c679..1ab158c1a 100644 --- a/frontend/app/components/ForgotPassword/ForgotPassword.js +++ b/frontend/app/components/ForgotPassword/ForgotPassword.js @@ -14,21 +14,23 @@ const LOGIN = loginRoute(); const recaptchaRef = React.createRef(); const ERROR_DONT_MATCH = "Passwords don't match."; const MIN_LENGTH = 8; -const PASSWORD_POLICY = `Password should contain at least ${ MIN_LENGTH } symbols.`; +const PASSWORD_POLICY = `Password should contain at least ${MIN_LENGTH} symbols.`; const checkDontMatch = (newPassword, newPasswordRepeat) => newPasswordRepeat.length > 0 && newPasswordRepeat !== newPassword; @connect( (state, props) => ({ - errors: state.getIn([ 'user', 'requestResetPassowrd', 'errors' ]), - resetErrors: state.getIn([ 'user', 'resetPassword', 'errors' ]), - loading: state.getIn([ 'user', 'requestResetPassowrd', 'loading' ]) || state.getIn([ 'user', 'resetPassword', 'loading' ]), - params: new URLSearchParams(props.location.search) + errors: state.getIn(['user', 'requestResetPassowrd', 'errors']), + resetErrors: state.getIn(['user', 'resetPassword', 'errors']), + loading: + state.getIn(['user', 'requestResetPassowrd', 'loading']) || + state.getIn(['user', 'resetPassword', 'loading']), + params: new URLSearchParams(props.location.search), }), - { requestResetPassword, resetPassword, resetErrors }, + { requestResetPassword, resetPassword, resetErrors } ) -@withPageTitle("Password Reset - OpenReplay") +@withPageTitle('Password Reset - OpenReplay') @withRouter export default class ForgotPassword extends React.PureComponent { state = { @@ -45,15 +47,17 @@ export default class ForgotPassword extends React.PureComponent { const { email, password } = this.state; const { params } = this.props; - const pass = params.get('pass') - const invitation = params.get('invitation') - const resetting = pass && invitation + const pass = params.get('pass'); + const invitation = params.get('invitation'); + const resetting = pass && invitation; if (!resetting) { - this.props.requestResetPassword({ email: email.trim(), 'g-recaptcha-response': token }).then(() => { - const { errors } = this.props; - if (!errors) this.setState({ requested: true }); - }); + this.props + .requestResetPassword({ email: email.trim(), 'g-recaptcha-response': token }) + .then(() => { + const { errors } = this.props; + if (!errors) this.setState({ requested: true }); + }); } else { if (this.isSubmitDisabled()) return; this.props.resetPassword({ email: email.trim(), invitation, pass, password }).then(() => { @@ -61,16 +65,15 @@ export default class ForgotPassword extends React.PureComponent { if (!resetErrors) this.setState({ updated: true }); }); } - } + }; isSubmitDisabled() { const { password, passwordRepeat } = this.state; - if (password !== passwordRepeat || - password.length < MIN_LENGTH) return true; + if (password !== passwordRepeat || password.length < MIN_LENGTH) return true; return false; } - write = ({ target: { value, name } }) => this.setState({ [ name ]: value }) + write = ({ target: { value, name } }) => this.setState({ [name]: value }); shouldShouwPolicy() { const { password } = this.state; @@ -79,152 +82,170 @@ export default class ForgotPassword extends React.PureComponent { return true; } - onSubmit = e => { + onSubmit = (e) => { e.preventDefault(); const { CAPTCHA_ENABLED } = this.state; if (CAPTCHA_ENABLED && recaptchaRef.current) { - recaptchaRef.current.execute() + recaptchaRef.current.execute(); } else if (!CAPTCHA_ENABLED) { this.handleSubmit(); } - } + }; componentWillUnmount() { - this.props.resetErrors() + this.props.resetErrors(); } render() { const { CAPTCHA_ENABLED } = this.state; const { errors, loading, params } = this.props; const { requested, updated, password, passwordRepeat, email } = this.state; - const dontMatch = checkDontMatch(password, passwordRepeat); - - const pass = params.get('pass') - const invitation = params.get('invitation') - const resetting = pass && invitation - const validEmail = validateEmail(email) + const dontMatch = checkDontMatch(password, passwordRepeat); + + const pass = params.get('pass'); + const invitation = params.get('invitation'); + const resetting = pass && invitation; + const validEmail = validateEmail(email); return ( -
-
-
- +
+
+
+
-
-
-
Welcome Back!
-
-
-
-
-
-
-

{`${resetting ? 'Create' : 'Reset'} Password`}

-
- -
- { CAPTCHA_ENABLED && ( -
- this.handleSubmit(token) } - /> -
- )} +
+

+ { resetting ? "Create Password" : "Reset Password" } +

- { !resetting && !requested && - - - - - } - - { - requested && !errors && ( -
Reset password link has been sent to your email.
- ) - } - - { - resetting && ( - - - - + {resetting &&
Provide your email address, so we can send you a link to reset your password.
} + + {/*
+

{`${ + resetting ? 'Create' : 'Reset' + } Password`}

+
*/} + +
+ {CAPTCHA_ENABLED && ( +
+ this.handleSubmit(token)} /> - -
- { PASSWORD_POLICY }
+ )} + + {!resetting && !requested && ( - + - - ) - } + )} - -
- -
- { errors && -
- { errors.map(error => { error }
) } + {requested && !errors && ( +
+
+ +
+
Alright! a reset link was emailed to {email}. Click on it to reset your account password.
+
+ )} + + {resetting && ( + + + + + +
+ {PASSWORD_POLICY} +
+ + + + + + )} + + +
+ +
+ {errors && ( +
+ {errors.map((error) => ( + + {error} +
+ + ))} +
+ )} +
+ + {'Your password has been updated sucessfully.'} +
- } -
- - { 'Your password has been updated sucessfully.' } -
-
- {/*
*/} - {!(updated || requested) && ( - - )} -
- - { updated && ()} -
{'Back to Login'}
- -
- {/*
*/} - + {!(updated || requested) && ( + + )} + +
+ + {updated && ( + + )} +
{'Back to Login'}
+ +
+ +
+
); diff --git a/frontend/app/components/Login/Login.js b/frontend/app/components/Login/Login.js index 05702a2fe..5fce3d153 100644 --- a/frontend/app/components/Login/Login.js +++ b/frontend/app/components/Login/Login.js @@ -69,18 +69,16 @@ class Login extends React.Component { const { CAPTCHA_ENABLED } = this.state; return ( -
-
-
- +
+
+
+
- -
-
-
+
-
-

Login to OpenReplay

+

Login to your account

+
+ {!authDetails.tenants && (
Don't have an account?{' '} @@ -99,19 +97,19 @@ class Login extends React.Component { onChange={(token) => this.handleSubmit(token)} /> )} -
+
- +
@@ -142,15 +140,16 @@ class Login extends React.Component { ))}
) : null} - {/*
*/} - + +
+ -
- {'Forgot your password?'} +
+ Having trouble logging in? {'Reset password'} +
- {/*
*/}
@@ -182,6 +181,8 @@ class Login extends React.Component { )}
+ +
diff --git a/frontend/app/components/Signup/Signup.js b/frontend/app/components/Signup/Signup.js index 83a658ec1..6ca511f32 100644 --- a/frontend/app/components/Signup/Signup.js +++ b/frontend/app/components/Signup/Signup.js @@ -19,24 +19,19 @@ const BulletItem = ({ text }) => ( export default class Signup extends React.Component { render() { return ( -
-
-
- -
- -
+
+
+
-
- OpenReplay Cloud{' '} -
- +
+
+ +
{' '} +
+ Cloud
-
OpenReplay Cloud is the hosted version of our open-source project.
+
OpenReplay Cloud is the hosted version of our open-source project.
We’ll manage hosting, scaling and upgrades.
@@ -47,7 +42,7 @@ export default class Signup extends React.Component {
-
+
diff --git a/frontend/app/components/Signup/SignupForm/SignupForm.js b/frontend/app/components/Signup/SignupForm/SignupForm.js index f110d2dae..3641dd36d 100644 --- a/frontend/app/components/Signup/SignupForm/SignupForm.js +++ b/frontend/app/components/Signup/SignupForm/SignupForm.js @@ -1,26 +1,25 @@ -import React from 'react' -import { Form, Input, Icon, Button, Link } from 'UI' -import { login } from 'App/routes' -import ReCAPTCHA from 'react-google-recaptcha' -import stl from './signup.module.css' +import React from 'react'; +import { Form, Input, Icon, Button, Link } from 'UI'; +import { login } from 'App/routes'; +import ReCAPTCHA from 'react-google-recaptcha'; +import stl from './signup.module.css'; import { signup } from 'Duck/user'; -import { connect } from 'react-redux' -import Select from 'Shared/Select' +import { connect } from 'react-redux'; +import Select from 'Shared/Select'; import { SITE_ID_STORAGE_KEY } from 'App/constants/storageKeys'; -const LOGIN_ROUTE = login() -const recaptchaRef = React.createRef() +const LOGIN_ROUTE = login(); +const recaptchaRef = React.createRef(); @connect( - state => ({ + (state) => ({ tenants: state.getIn(['user', 'tenants']), - errors: state.getIn([ 'user', 'signupRequest', 'errors' ]), - loading: state.getIn([ 'user', 'signupRequest', 'loading' ]), + errors: state.getIn(['user', 'signupRequest', 'errors']), + loading: state.getIn(['user', 'signupRequest', 'loading']), }), - { signup }, + { signup } ) export default class SignupForm extends React.Component { - state = { tenantId: '', fullname: '', @@ -36,21 +35,30 @@ export default class SignupForm extends React.Component { if (props.errors && props.errors.size > 0 && state.reload) { recaptchaRef.current.reset(); return { - reload: false - } - } + reload: false, + }; + } return null; } handleSubmit = (token) => { const { tenantId, fullname, password, email, projectName, organizationName, auth } = this.state; - localStorage.removeItem(SITE_ID_STORAGE_KEY) - this.props.signup({ tenantId, fullname, password, email, projectName, organizationName, auth, 'g-recaptcha-response': token }) - this.setState({ reload: true }) - } + localStorage.removeItem(SITE_ID_STORAGE_KEY); + this.props.signup({ + tenantId, + fullname, + password, + email, + projectName, + organizationName, + auth, + 'g-recaptcha-response': token, + }); + this.setState({ reload: true }); + }; - write = ({ target: { value, name } }) => this.setState({ [ name ]: value }) - writeOption = ({ name, value }) => this.setState({ [ name ]: value.value }); + write = ({ target: { value, name } }) => this.setState({ [name]: value }); + writeOption = ({ name, value }) => this.setState({ [name]: value.value }); onSubmit = (e) => { e.preventDefault(); @@ -60,52 +68,51 @@ export default class SignupForm extends React.Component { } else if (!CAPTCHA_ENABLED) { this.handleSubmit(); } - } + }; render() { const { loading, errors, tenants } = this.props; const { CAPTCHA_ENABLED } = this.state; return ( -
+
-

Get Started

-
Already having an account? Sign in
+

Create Account

<> - { CAPTCHA_ENABLED && ( + {CAPTCHA_ENABLED && ( this.handleSubmit(token) } + sitekey={window.env.CAPTCHA_SITE_KEY} + onChange={(token) => this.handleSubmit(token)} /> )} -
- { tenants.length > 0 && ( +
+ {tenants.length > 0 && ( @@ -115,57 +122,72 @@ export default class SignupForm extends React.Component { placeholder="Min 8 Characters" minLength="8" name="password" - onChange={ this.write } - className={ stl.password } + onChange={this.write} required="true" + icon="key" /> - - - - - - { errors && -
- { errors.map(error => ( + {errors && ( +
+ {errors.map((error) => (
- - { error }
 + + + {error} +
+
- )) } + ))} +
+ )} +
+ Already having an account?{' '} + + Login +
- } -
- -
- ) + ); } -} \ No newline at end of file +} diff --git a/frontend/app/components/ui/Button/Button.tsx b/frontend/app/components/ui/Button/Button.tsx index 3b08f55f7..924a70141 100644 --- a/frontend/app/components/ui/Button/Button.tsx +++ b/frontend/app/components/ui/Button/Button.tsx @@ -60,7 +60,7 @@ export default (props: Props) => { } const render = () => ( - @@ -170,12 +168,12 @@ class Login extends React.Component { placement="top" > From 1f85f0cf7180dcce262afb44e695c3b449b82eae Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 27 Mar 2023 14:43:39 +0200 Subject: [PATCH 173/253] chore(helm): Updating the log message Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index 4cde9a54d..c0b3192ac 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -366,7 +366,7 @@ do log title "Editing OpenReplay" sudo vim -n /var/lib/openreplay/vars.yaml /var/lib/openreplay/yq 'true' /var/lib/openreplay/vars.yaml || { - log debug "seems like the edit is not correct. Rerun ${BWHITE}openreplay -e${YELLOW} and fix the issue." + log debug "seems like the edit is not correct. Rerun ${BWHITE}openreplay -e${YELLOW} after fixing the issue." exit 100 } reload From 1ea0a3d9c5568903481d703bd2c947c16788dde7 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 27 Mar 2023 15:13:19 +0200 Subject: [PATCH 174/253] fix(build): script for api Signed-off-by: rjshrjndrn --- api/build.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/build.sh b/api/build.sh index 946dbf3b6..afda47f13 100644 --- a/api/build.sh +++ b/api/build.sh @@ -78,12 +78,12 @@ function build_api(){ check_prereq build_api $environment echo buil_complete -#IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO SIGN_IMAGE=$SIGN_IMAGE SIGN_KEY=$SIGN_KEY bash build_alerts.sh $1 -# -#[[ $environment == "ee" ]] && { -# cp ../ee/api/build_crons.sh . -# IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO SIGN_IMAGE=$SIGN_IMAGE SIGN_KEY=$SIGN_KEY bash build_crons.sh $1 -# exit_err $? -# rm build_crons.sh -#} || true [[ $PATCH -eq 1 ]] && update_helm_release chalice +IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO SIGN_IMAGE=$SIGN_IMAGE SIGN_KEY=$SIGN_KEY bash build_alerts.sh $1 + +[[ $environment == "ee" ]] && { + cp ../ee/api/build_crons.sh . + IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO SIGN_IMAGE=$SIGN_IMAGE SIGN_KEY=$SIGN_KEY bash build_crons.sh $1 + exit_err $? + rm build_crons.sh +} || true From 73df10e1d62401dc0d466ff242d51476556d4acf Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 27 Mar 2023 16:43:42 +0200 Subject: [PATCH 175/253] fix(ui) - country flag height --- frontend/app/components/ui/CountryFlag/countryFlag.module.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/ui/CountryFlag/countryFlag.module.css b/frontend/app/components/ui/CountryFlag/countryFlag.module.css index aa331ee77..fdf55ff32 100644 --- a/frontend/app/components/ui/CountryFlag/countryFlag.module.css +++ b/frontend/app/components/ui/CountryFlag/countryFlag.module.css @@ -1,6 +1,6 @@ .default { width: 22px !important; - height: 12px !important; + height: 15px !important; } .label { From 723f2b933f403a14fba89fbf04b12c6148e4c263 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Mar 2023 15:48:50 +0100 Subject: [PATCH 176/253] feat(DB): remove unnecessary columns --- .../db/init_dbs/clickhouse/1.11.0/1.11.0.sql | 78 +++++++++++++++++++ .../clickhouse/create/init_schema.sql | 8 -- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql index 1962fde10..8191f8fd1 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.11.0/1.11.0.sql @@ -6,3 +6,81 @@ ALTER TABLE experimental.events ALTER TABLE experimental.issues MODIFY COLUMN type Enum8('click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20); +DROP TABLE IF EXISTS experimental.js_errors_sessions_mv; +DROP TABLE IF EXISTS experimental.events_l7d_mv; + +ALTER TABLE experimental.events + DROP COLUMN IF EXISTS container_id, + DROP COLUMN IF EXISTS container_name, + DROP COLUMN IF EXISTS container_src, + DROP COLUMN IF EXISTS container_type; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS experimental.events_l7d_mv + ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMMDD(datetime) + ORDER BY (project_id, datetime, event_type, session_id, message_id) + TTL datetime + INTERVAL 7 DAY + POPULATE +AS +SELECT session_id, + project_id, + event_type, + datetime, + label, + hesitation_time, + name, + payload, + level, + source, + message, + error_id, + duration, + context, + url, + url_host, + url_path, + url_hostpath, + request_start, + response_start, + response_end, + dom_content_loaded_event_start, + dom_content_loaded_event_end, + load_event_start, + load_event_end, + first_paint, + first_contentful_paint_time, + speed_index, + visually_complete, + time_to_interactive, + ttfb, + ttlb, + response_time, + dom_building_time, + dom_content_loaded_event_time, + load_event_time, + min_fps, + avg_fps, + max_fps, + min_cpu, + avg_cpu, + max_cpu, + min_total_js_heap_size, + avg_total_js_heap_size, + max_total_js_heap_size, + min_used_js_heap_size, + avg_used_js_heap_size, + max_used_js_heap_size, + method, + status, + success, + request_body, + response_body, + issue_type, + issue_id, + error_tags_keys, + error_tags_values, + message_id, + _timestamp +FROM experimental.events +WHERE datetime >= now() - INTERVAL 7 DAY; \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 9536307d8..f04e5d8c6 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -28,10 +28,6 @@ CREATE TABLE IF NOT EXISTS experimental.events error_id Nullable(String), duration Nullable(UInt16), context Nullable(Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8)), - container_type Nullable(Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3)), - container_id Nullable(String), - container_name Nullable(String), - container_src Nullable(String), url Nullable(String), url_host Nullable(String) MATERIALIZED lower(domain(url)), url_path Nullable(String) MATERIALIZED lower(pathFull(url)), @@ -232,10 +228,6 @@ SELECT session_id, error_id, duration, context, - container_type, - container_id, - container_name, - container_src, url, url_host, url_path, From 95369ce8ef9b652f778b6cdd75f9e4e3040bc0e6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Mar 2023 15:53:50 +0100 Subject: [PATCH 177/253] feat(sourcemaps-reader): changed build script --- sourcemap-reader/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sourcemap-reader/build.sh b/sourcemap-reader/build.sh index 3025e05e4..bcbd49128 100644 --- a/sourcemap-reader/build.sh +++ b/sourcemap-reader/build.sh @@ -30,7 +30,7 @@ function build_api(){ } cp -R ../sourcemap-reader ../${destination} cd ../${destination} - cp -R ../assist/utils . + cp -R ../assist/utils/* ./utils/. tag="" # Copy enterprise code [[ $1 == "ee" ]] && { From d6bbd7685425ed28d1864ab733318790c8441cfe Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Mar 2023 16:11:41 +0100 Subject: [PATCH 178/253] feat(assist): centralized health check feat(assist): changed build feat(peers): centralized health check feat(peers): changed build feat(sourcemaps-reader): centralized health check feat(sourcemaps-reader): changed build --- assist/servers/websocket.js | 18 +------- assist/utils/health.js | 4 +- ee/assist/.gitignore | 1 + ee/assist/clean-dev.sh | 1 + ee/assist/servers/websocket-cluster.js | 17 ------- ee/assist/servers/websocket.js | 18 -------- ee/assist/utils/health.js | 61 -------------------------- peers/build.sh | 1 - peers/prepare-dev.sh | 3 +- sourcemap-reader/.gitignore | 5 +-- sourcemap-reader/build.sh | 2 +- sourcemap-reader/clean-dev.sh | 5 +-- sourcemap-reader/utils/health.js | 52 ---------------------- 13 files changed, 8 insertions(+), 180 deletions(-) delete mode 100644 ee/assist/utils/health.js delete mode 100644 sourcemap-reader/utils/health.js diff --git a/assist/servers/websocket.js b/assist/servers/websocket.js index 0fdda85f2..4c4a657bb 100644 --- a/assist/servers/websocket.js +++ b/assist/servers/websocket.js @@ -45,22 +45,7 @@ const respond = function (res, data) { res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify({"data": data})); } -const countSessions = async function () { - let count = 0; - try { - const arr = Array.from(io.sockets.adapter.rooms); - const filtered = arr.filter(room => !room[1].has(room[0])); - for (let i of filtered) { - let {projectKey, sessionId} = extractPeerId(i[0]); - if (projectKey !== null && sessionId !== null) { - count++; - } - } - } catch (e) { - console.error(e); - } - return count; -} + const socketsList = async function (req, res) { debug && console.log("[WS]looking for all available sessions"); let filters = extractPayloadFromRequest(req); @@ -375,7 +360,6 @@ module.exports = { socketConnexionTimeout(io); }, - countSessions, handlers: { socketsList, socketsListByProject, diff --git a/assist/utils/health.js b/assist/utils/health.js index d71864e71..0b89dd1d8 100644 --- a/assist/utils/health.js +++ b/assist/utils/health.js @@ -1,5 +1,4 @@ const express = require('express'); -const socket = require("../servers/websocket"); const HOST = process.env.LISTEN_HOST || '0.0.0.0'; const PORT = process.env.HEALTH_PORT || 8888; @@ -17,8 +16,7 @@ const check_health = async function (req, res) { respond(res, { "health": true, "details": { - "version": process.env.npm_package_version, - "connectedSessions": await socket.countSessions() + "version": process.env.npm_package_version } }); } diff --git a/ee/assist/.gitignore b/ee/assist/.gitignore index cd68b1ffb..98f9f8386 100644 --- a/ee/assist/.gitignore +++ b/ee/assist/.gitignore @@ -11,6 +11,7 @@ servers/peerjs-server.js servers/sourcemaps-handler.js servers/sourcemaps-server.js /utils/geoIP.js +/utils/health.js /utils/HeapSnapshot.js /utils/helper.js /utils/assistHelper.js diff --git a/ee/assist/clean-dev.sh b/ee/assist/clean-dev.sh index ec1aaeae4..eff1f6308 100755 --- a/ee/assist/clean-dev.sh +++ b/ee/assist/clean-dev.sh @@ -1,4 +1,5 @@ rm -rf ./utils/geoIP.js +rm -rf ./utils/health.js rm -rf ./utils/HeapSnapshot.js rm -rf ./utils/helper.js rm -rf ./utils/assistHelper.js diff --git a/ee/assist/servers/websocket-cluster.js b/ee/assist/servers/websocket-cluster.js index 4618a6184..a1f389685 100644 --- a/ee/assist/servers/websocket-cluster.js +++ b/ee/assist/servers/websocket-cluster.js @@ -83,22 +83,6 @@ const respond = function (res, data) { } } -const countSessions = async function () { - let count = 0; - try { - let rooms = await io.of('/').adapter.allRooms(); - for (let i of rooms) { - let {projectKey, sessionId} = extractPeerId(i); - if (projectKey !== undefined && sessionId !== undefined) { - count++; - } - } - } catch (e) { - console.error(e); - } - return count; -} - const socketsList = async function (req, res) { debug && console.log("[WS]looking for all available sessions"); let filters = await extractPayloadFromRequest(req, res); @@ -433,7 +417,6 @@ module.exports = { process.exit(2); }); }, - countSessions, handlers: { socketsList, socketsListByProject, diff --git a/ee/assist/servers/websocket.js b/ee/assist/servers/websocket.js index 7fb1c9684..330361df3 100644 --- a/ee/assist/servers/websocket.js +++ b/ee/assist/servers/websocket.js @@ -66,23 +66,6 @@ const respond = function (res, data) { } } -const countSessions = async function () { - let count = 0; - try { - const arr = Array.from(io.sockets.adapter.rooms); - const filtered = arr.filter(room => !room[1].has(room[0])); - for (let i of filtered) { - let {projectKey, sessionId} = extractPeerId(i[0]); - if (projectKey !== null && sessionId !== null) { - count++; - } - } - } catch (e) { - console.error(e); - } - return count; -} - const socketsList = async function (req, res) { debug && console.log("[WS]looking for all available sessions"); let filters = await extractPayloadFromRequest(req, res); @@ -396,7 +379,6 @@ module.exports = { socketConnexionTimeout(io); }, - countSessions, handlers: { socketsList, socketsListByProject, diff --git a/ee/assist/utils/health.js b/ee/assist/utils/health.js deleted file mode 100644 index bcb64f61c..000000000 --- a/ee/assist/utils/health.js +++ /dev/null @@ -1,61 +0,0 @@ -const express = require('express'); -let socket; -if (process.env.redis === "true") { - socket = require("../servers/websocket-cluster"); -} else { - socket = require("../servers/websocket"); -} -const HOST = process.env.LISTEN_HOST || '0.0.0.0'; -const PORT = process.env.HEALTH_PORT || 8888; - - -const {request_logger} = require("./helper"); -const debug = process.env.debug === "1"; -const respond = function (res, data) { - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({"data": data})); -} - -const check_health = async function (req, res) { - debug && console.log("[WS]looking for all available sessions"); - respond(res, { - "health": true, - "details": { - "version": process.env.npm_package_version, - "connectedSessions": await socket.countSessions(), - "uWebSocket": process.env.uws === "true", - "redis": process.env.redis === "true" - } - }); -} - - -const healthApp = express(); -healthApp.use(express.json()); -healthApp.use(express.urlencoded({extended: true})); -healthApp.use(request_logger("[healthApp]")); -healthApp.get(['/'], (req, res) => { - res.statusCode = 200; - res.end("healthApp ok!"); - } -); -healthApp.get('/health', check_health); -healthApp.get('/shutdown', (req, res) => { - console.log("Requested shutdown"); - res.statusCode = 200; - res.end("ok!"); - process.kill(1, "SIGTERM"); - } -); - -const listen_cb = async function () { - console.log(`Health App listening on http://${HOST}:${PORT}`); - console.log('Press Ctrl+C to quit.'); -} - -module.exports = { - healthApp, - PORT, - listen_cb -}; diff --git a/peers/build.sh b/peers/build.sh index 0f01a292f..746a12f9d 100644 --- a/peers/build.sh +++ b/peers/build.sh @@ -23,7 +23,6 @@ function build_api(){ cp -R ../peers ../${destination} cd ../${destination} cp -R ../assist/utils . - cp ../sourcemap-reader/utils/health.js ./utils/. # Copy enterprise code [[ $1 == "ee" ]] && { cp -rf ../ee/peers/* ./ diff --git a/peers/prepare-dev.sh b/peers/prepare-dev.sh index d4825a3d0..78a315946 100755 --- a/peers/prepare-dev.sh +++ b/peers/prepare-dev.sh @@ -1,3 +1,2 @@ #!/bin/bash -rsync -avr --exclude=".*" --ignore-existing ../assist/utils ./ -cp ../sourcemap-reader/utils/health.js ./utils/. \ No newline at end of file +rsync -avr --exclude=".*" --ignore-existing ../assist/utils ./ \ No newline at end of file diff --git a/sourcemap-reader/.gitignore b/sourcemap-reader/.gitignore index f2686decf..f2604ef2a 100644 --- a/sourcemap-reader/.gitignore +++ b/sourcemap-reader/.gitignore @@ -3,8 +3,5 @@ node_modules npm-debug.log .cache test.html -/utils/assistHelper.js -/utils/geoIP.js -/utils/HeapSnapshot.js -/utils/helper.js +utils mappings.wasm diff --git a/sourcemap-reader/build.sh b/sourcemap-reader/build.sh index bcbd49128..3025e05e4 100644 --- a/sourcemap-reader/build.sh +++ b/sourcemap-reader/build.sh @@ -30,7 +30,7 @@ function build_api(){ } cp -R ../sourcemap-reader ../${destination} cd ../${destination} - cp -R ../assist/utils/* ./utils/. + cp -R ../assist/utils . tag="" # Copy enterprise code [[ $1 == "ee" ]] && { diff --git a/sourcemap-reader/clean-dev.sh b/sourcemap-reader/clean-dev.sh index ebc1c36c6..a0cb5c9ed 100755 --- a/sourcemap-reader/clean-dev.sh +++ b/sourcemap-reader/clean-dev.sh @@ -1,6 +1,3 @@ #!/bin/bash -rm -rf ./utils/assistHelper.js -rm -rf ./utils/geoIP.js -rm -rf ./utils/HeapSnapshot.js -rm -rf ./utils/helper.js \ No newline at end of file +rm -rf ./utils \ No newline at end of file diff --git a/sourcemap-reader/utils/health.js b/sourcemap-reader/utils/health.js deleted file mode 100644 index 0b89dd1d8..000000000 --- a/sourcemap-reader/utils/health.js +++ /dev/null @@ -1,52 +0,0 @@ -const express = require('express'); -const HOST = process.env.LISTEN_HOST || '0.0.0.0'; -const PORT = process.env.HEALTH_PORT || 8888; - - -const {request_logger} = require("./helper"); -const debug = process.env.debug === "1"; -const respond = function (res, data) { - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({"data": data})); -} - -const check_health = async function (req, res) { - debug && console.log("[WS]looking for all available sessions"); - respond(res, { - "health": true, - "details": { - "version": process.env.npm_package_version - } - }); -} - - -const healthApp = express(); -healthApp.use(express.json()); -healthApp.use(express.urlencoded({extended: true})); -healthApp.use(request_logger("[healthApp]")); -healthApp.get(['/'], (req, res) => { - res.statusCode = 200; - res.end("healthApp ok!"); - } -); -healthApp.get('/health', check_health); -healthApp.get('/shutdown', (req, res) => { - console.log("Requested shutdown"); - res.statusCode = 200; - res.end("ok!"); - process.kill(1, "SIGTERM"); - } -); - -const listen_cb = async function () { - console.log(`Health App listening on http://${HOST}:${PORT}`); - console.log('Press Ctrl+C to quit.'); -} - -module.exports = { - healthApp, - PORT, - listen_cb -}; From 559c17472929ecbd62a2dea48d2ee0d14e23741d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Mar 2023 16:22:17 +0100 Subject: [PATCH 179/253] chore(actions): changed sourcemaps actions --- .github/workflows/sourcemaps-reader.yaml | 38 ++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sourcemaps-reader.yaml b/.github/workflows/sourcemaps-reader.yaml index 5b7c11d01..fb3face2f 100644 --- a/.github/workflows/sourcemaps-reader.yaml +++ b/.github/workflows/sourcemaps-reader.yaml @@ -1,6 +1,11 @@ # This action will push the sourcemapreader changes to aws on: workflow_dispatch: + inputs: + skip_security_checks: + description: 'Skip Security checks if there is a unfixable vuln or error. Value: true/false' + required: false + default: 'false' push: branches: - dev @@ -48,8 +53,26 @@ jobs: IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} ENVIRONMENT: staging run: | + skip_security_checks=${{ github.event.inputs.skip_security_checks }} cd sourcemap-reader - PUSH_IMAGE=1 bash build.sh + PUSH_IMAGE=0 bash -x ./build.sh + [[ "x$skip_security_checks" == "xtrue" ]] || { + curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ + images=("sourcemaps-reader") + for image in ${images[*]};do + ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG + done + err_code=$? + [[ $err_code -ne 0 ]] && { + exit $err_code + } + } && { + echo "Skipping Security Checks" + } + images=("sourcemaps-reader") + for image in ${images[*]};do + docker push $DOCKER_REPO/$image:$IMAGE_TAG + done - name: Creating old image input run: | # @@ -96,6 +119,17 @@ jobs: IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} ENVIRONMENT: staging + - name: Alert slack + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_CHANNEL: foss + SLACK_TITLE: "Failed ${{ github.workflow }}" + SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff' + SLACK_WEBHOOK: ${{ secrets.SLACK_WEB_HOOK }} + SLACK_USERNAME: "OR Bot" + SLACK_MESSAGE: 'Build failed :bomb:' + # - name: Debug Job # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 @@ -103,4 +137,4 @@ jobs: # DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} # IMAGE_TAG: ${{ github.sha }} # ENVIRONMENT: staging - # + From 18e9aa901de81439f3d6ae01eaa2aa0649647b80 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 27 Mar 2023 17:43:34 +0200 Subject: [PATCH 180/253] fix(ui) - player-hover icon --- frontend/app/components/ui/SVG.tsx | 4 ++-- frontend/app/svg/icons/play-hover.svg | 15 +++------------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/frontend/app/components/ui/SVG.tsx b/frontend/app/components/ui/SVG.tsx index 227488667..a30aae574 100644 --- a/frontend/app/components/ui/SVG.tsx +++ b/frontend/app/components/ui/SVG.tsx @@ -1,7 +1,7 @@ import React from 'react'; -export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'buildings' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'click-hesitation' | 'click-rage' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'cursor-trash' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope-check' | 'envelope' | 'errors-icon' | 'event/click' | 'event/click_hesitation' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/input_hesitation' | 'event/link' | 'event/location' | 'event/mouse_thrashing' | 'event/resize' | 'event/view' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'input-hesitation' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'key' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; +export type IconNames = 'activity' | 'alarm-clock' | 'alarm-plus' | 'all-sessions' | 'analytics' | 'anchor' | 'arrow-alt-square-right' | 'arrow-bar-left' | 'arrow-clockwise' | 'arrow-counterclockwise' | 'arrow-down-short' | 'arrow-down' | 'arrow-repeat' | 'arrow-right-short' | 'arrow-square-left' | 'arrow-square-right' | 'arrow-up-short' | 'arrow-up' | 'arrows-angle-extend' | 'avatar/icn_bear' | 'avatar/icn_beaver' | 'avatar/icn_bird' | 'avatar/icn_bison' | 'avatar/icn_camel' | 'avatar/icn_chameleon' | 'avatar/icn_deer' | 'avatar/icn_dog' | 'avatar/icn_dolphin' | 'avatar/icn_elephant' | 'avatar/icn_fish' | 'avatar/icn_fox' | 'avatar/icn_gorilla' | 'avatar/icn_hippo' | 'avatar/icn_horse' | 'avatar/icn_hyena' | 'avatar/icn_kangaroo' | 'avatar/icn_lemur' | 'avatar/icn_mammel' | 'avatar/icn_monkey' | 'avatar/icn_moose' | 'avatar/icn_panda' | 'avatar/icn_penguin' | 'avatar/icn_porcupine' | 'avatar/icn_quail' | 'avatar/icn_rabbit' | 'avatar/icn_rhino' | 'avatar/icn_sea_horse' | 'avatar/icn_sheep' | 'avatar/icn_snake' | 'avatar/icn_squirrel' | 'avatar/icn_tapir' | 'avatar/icn_turtle' | 'avatar/icn_vulture' | 'avatar/icn_wild1' | 'avatar/icn_wild_bore' | 'ban' | 'bar-chart-line' | 'bar-pencil' | 'bell-fill' | 'bell-plus' | 'bell-slash' | 'bell' | 'binoculars' | 'book-doc' | 'book' | 'browser/browser' | 'browser/chrome' | 'browser/edge' | 'browser/electron' | 'browser/facebook' | 'browser/firefox' | 'browser/ie' | 'browser/opera' | 'browser/safari' | 'buildings' | 'bullhorn' | 'business-time' | 'calendar-alt' | 'calendar-check' | 'calendar-day' | 'calendar' | 'call' | 'camera-alt' | 'camera-video-off' | 'camera-video' | 'camera' | 'card-checklist' | 'card-text' | 'caret-down-fill' | 'caret-left-fill' | 'caret-right-fill' | 'caret-up-fill' | 'chat-dots' | 'chat-right-text' | 'chat-square-quote' | 'check-circle-fill' | 'check-circle' | 'check' | 'chevron-double-left' | 'chevron-double-right' | 'chevron-down' | 'chevron-left' | 'chevron-right' | 'chevron-up' | 'circle-fill' | 'circle' | 'click-hesitation' | 'click-rage' | 'clipboard-list-check' | 'clock' | 'close' | 'cloud-fog2-fill' | 'code' | 'cog' | 'cogs' | 'collection' | 'columns-gap-filled' | 'columns-gap' | 'console/error' | 'console/exception' | 'console/info' | 'console/warning' | 'console' | 'controller' | 'cookies' | 'copy' | 'credit-card-front' | 'cross' | 'cubes' | 'cursor-trash' | 'dash' | 'dashboard-icn' | 'desktop' | 'device' | 'diagram-3' | 'dizzy' | 'door-closed' | 'doublecheck' | 'download' | 'drag' | 'edit' | 'ellipsis-v' | 'enter' | 'envelope-check' | 'envelope' | 'errors-icon' | 'event/click' | 'event/click_hesitation' | 'event/clickrage' | 'event/code' | 'event/i-cursor' | 'event/input' | 'event/input_hesitation' | 'event/link' | 'event/location' | 'event/mouse_thrashing' | 'event/resize' | 'event/view' | 'exclamation-circle-fill' | 'exclamation-circle' | 'expand-wide' | 'explosion' | 'external-link-alt' | 'eye-slash-fill' | 'eye-slash' | 'eye' | 'fetch' | 'file-code' | 'file-medical-alt' | 'file-pdf' | 'file' | 'files' | 'filter' | 'filters/arrow-return-right' | 'filters/browser' | 'filters/click' | 'filters/clickrage' | 'filters/code' | 'filters/console' | 'filters/country' | 'filters/cpu-load' | 'filters/custom' | 'filters/device' | 'filters/dom-complete' | 'filters/duration' | 'filters/error' | 'filters/fetch-failed' | 'filters/fetch' | 'filters/file-code' | 'filters/graphql' | 'filters/i-cursor' | 'filters/input' | 'filters/lcpt' | 'filters/link' | 'filters/location' | 'filters/memory-load' | 'filters/metadata' | 'filters/os' | 'filters/perfromance-network-request' | 'filters/platform' | 'filters/referrer' | 'filters/resize' | 'filters/rev-id' | 'filters/state-action' | 'filters/ttfb' | 'filters/user-alt' | 'filters/userid' | 'filters/view' | 'flag-na' | 'folder-plus' | 'folder2' | 'fullscreen' | 'funnel/cpu-fill' | 'funnel/cpu' | 'funnel/dizzy' | 'funnel/emoji-angry-fill' | 'funnel/emoji-angry' | 'funnel/emoji-dizzy-fill' | 'funnel/exclamation-circle-fill' | 'funnel/exclamation-circle' | 'funnel/file-earmark-break-fill' | 'funnel/file-earmark-break' | 'funnel/file-earmark-minus-fill' | 'funnel/file-earmark-minus' | 'funnel/file-medical-alt' | 'funnel/file-x' | 'funnel/hdd-fill' | 'funnel/hourglass-top' | 'funnel/image-fill' | 'funnel/image' | 'funnel/microchip' | 'funnel/mouse' | 'funnel/patch-exclamation-fill' | 'funnel/sd-card' | 'funnel-fill' | 'funnel-new' | 'funnel' | 'gear-fill' | 'gear' | 'geo-alt-fill-custom' | 'github' | 'graph-up-arrow' | 'graph-up' | 'grid-1x2' | 'grid-3x3' | 'grid-check' | 'grid-horizontal' | 'grid' | 'grip-horizontal' | 'hash' | 'hdd-stack' | 'headset' | 'heart-rate' | 'high-engagement' | 'history' | 'hourglass-start' | 'ic-errors' | 'ic-network' | 'ic-rage' | 'ic-resources' | 'id-card' | 'image' | 'info-circle-fill' | 'info-circle' | 'info-square' | 'info' | 'input-hesitation' | 'inspect' | 'integrations/assist' | 'integrations/bugsnag-text' | 'integrations/bugsnag' | 'integrations/cloudwatch-text' | 'integrations/cloudwatch' | 'integrations/datadog' | 'integrations/elasticsearch-text' | 'integrations/elasticsearch' | 'integrations/github' | 'integrations/graphql' | 'integrations/jira-text' | 'integrations/jira' | 'integrations/mobx' | 'integrations/newrelic-text' | 'integrations/newrelic' | 'integrations/ngrx' | 'integrations/openreplay-text' | 'integrations/openreplay' | 'integrations/redux' | 'integrations/rollbar-text' | 'integrations/rollbar' | 'integrations/segment' | 'integrations/sentry-text' | 'integrations/sentry' | 'integrations/slack-bw' | 'integrations/slack' | 'integrations/stackdriver' | 'integrations/sumologic-text' | 'integrations/sumologic' | 'integrations/teams-white' | 'integrations/teams' | 'integrations/vuejs' | 'journal-code' | 'key' | 'layer-group' | 'lightbulb-on' | 'lightbulb' | 'link-45deg' | 'list-alt' | 'list-arrow' | 'list-ul' | 'list' | 'lock-alt' | 'magic' | 'map-marker-alt' | 'memory' | 'mic-mute' | 'mic' | 'minus' | 'mobile' | 'mouse-alt' | 'network' | 'next1' | 'no-dashboard' | 'no-metrics-chart' | 'no-metrics' | 'no-recordings' | 'os/android' | 'os/chrome_os' | 'os/fedora' | 'os/ios' | 'os/linux' | 'os/mac_os_x' | 'os/other' | 'os/ubuntu' | 'os/windows' | 'os' | 'pause-fill' | 'pause' | 'pdf-download' | 'pencil-stop' | 'pencil' | 'percent' | 'performance-icon' | 'person-fill' | 'person' | 'pie-chart-fill' | 'pin-fill' | 'play-circle-bold' | 'play-circle-light' | 'play-circle' | 'play-fill-new' | 'play-fill' | 'play-hover' | 'play' | 'plus-circle' | 'plus-lg' | 'plus' | 'pointer-sessions-search' | 'prev1' | 'pulse' | 'puzzle-piece' | 'puzzle' | 'question-circle' | 'question-lg' | 'quote-left' | 'quote-right' | 'quotes' | 'record-circle' | 'redo-back' | 'redo' | 'remote-control' | 'replay-10' | 'resources-icon' | 'safe-fill' | 'safe' | 'sandglass' | 'search' | 'search_notification' | 'server' | 'share-alt' | 'shield-lock' | 'signpost-split' | 'signup' | 'skip-forward-fill' | 'skip-forward' | 'slack' | 'slash-circle' | 'sliders' | 'social/slack' | 'social/trello' | 'speedometer2' | 'spinner' | 'star-solid' | 'star' | 'step-forward' | 'stickies' | 'stop-record-circle' | 'stopwatch' | 'store' | 'sync-alt' | 'table-new' | 'table' | 'tablet-android' | 'tachometer-slow' | 'tachometer-slowest' | 'tags' | 'team-funnel' | 'telephone-fill' | 'telephone' | 'text-paragraph' | 'tools' | 'trash' | 'turtle' | 'user-alt' | 'user-circle' | 'user-friends' | 'users' | 'vendors/graphql' | 'vendors/mobx' | 'vendors/ngrx' | 'vendors/redux' | 'vendors/vuex' | 'web-vitals' | 'wifi' | 'window-alt' | 'window-restore' | 'window-x' | 'window' | 'zoom-in'; interface Props { name: IconNames; @@ -366,7 +366,7 @@ const SVG = (props: Props) => { case 'play-circle': return ; case 'play-fill-new': return ; case 'play-fill': return ; - case 'play-hover': return ; + case 'play-hover': return ; case 'play': return ; case 'plus-circle': return ; case 'plus-lg': return ; diff --git a/frontend/app/svg/icons/play-hover.svg b/frontend/app/svg/icons/play-hover.svg index 13b3c271b..4fe1084a9 100644 --- a/frontend/app/svg/icons/play-hover.svg +++ b/frontend/app/svg/icons/play-hover.svg @@ -1,12 +1,3 @@ - - - - - - - - - - - - + + + \ No newline at end of file From 985af6cf613c19b06bdfeeb5c4ae6031b0895f9b Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 27 Mar 2023 17:46:31 +0200 Subject: [PATCH 181/253] fix(build): if else error Signed-off-by: rjshrjndrn --- peers/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/peers/build.sh b/peers/build.sh index d3686b2ff..7109004b6 100644 --- a/peers/build.sh +++ b/peers/build.sh @@ -62,4 +62,6 @@ function build_api(){ check_prereq build_api $1 -[[ $PATCH -eq 1 ]] && update_helm_release peers +if [[ $PATCH -eq 1 ]]; then + update_helm_release peers +fi From 8ed2ca7792e56e41527e1cfdfb3ae9cbbe741229 Mon Sep 17 00:00:00 2001 From: Dayan Graham Date: Mon, 27 Mar 2023 16:55:25 +0000 Subject: [PATCH 182/253] (feat): chalice - S3_DISABLE_SSL_VERIFY cast bool --- api/chalicelib/utils/s3.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/utils/s3.py b/api/chalicelib/utils/s3.py index 8a2369647..655628602 100644 --- a/api/chalicelib/utils/s3.py +++ b/api/chalicelib/utils/s3.py @@ -17,7 +17,7 @@ else: aws_secret_access_key=config("S3_SECRET"), config=Config(signature_version='s3v4'), region_name=config("sessions_region"), - verify=(False if config("S3_DISABLE_SSL_VERIFY") else True)) + verify=not config("S3_DISABLE_SSL_VERIFY", default=False, cast=bool)) def __get_s3_resource(): @@ -28,7 +28,7 @@ def __get_s3_resource(): aws_secret_access_key=config("S3_SECRET"), config=Config(signature_version='s3v4'), region_name=config("sessions_region"), - verify=(False if config("S3_DISABLE_SSL_VERIFY") else True)) + verify=not config("S3_DISABLE_SSL_VERIFY", default=False, cast=bool)) def exists(bucket, key): @@ -83,7 +83,8 @@ def get_presigned_url_for_upload_secure(bucket, expires_in, key, conditions=None Conditions=conditions, ) req = PreparedRequest() - req.prepare_url(f"{url_parts['url']}/{url_parts['fields']['key']}", url_parts['fields']) + req.prepare_url( + f"{url_parts['url']}/{url_parts['fields']['key']}", url_parts['fields']) return req.url @@ -103,7 +104,8 @@ def get_file(source_bucket, source_key): def rename(source_bucket, source_key, target_bucket, target_key): s3 = __get_s3_resource() - s3.Object(target_bucket, target_key).copy_from(CopySource=f'{source_bucket}/{source_key}') + s3.Object(target_bucket, target_key).copy_from( + CopySource=f'{source_bucket}/{source_key}') s3.Object(source_bucket, source_key).delete() From 63b2e54dbfb08182e96d5ec34a10705bbbc903d1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Mar 2023 09:13:00 +0100 Subject: [PATCH 183/253] feat(peers): cherry pick build changes --- peers/build.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/peers/build.sh b/peers/build.sh index 746a12f9d..51ea3f734 100644 --- a/peers/build.sh +++ b/peers/build.sh @@ -15,6 +15,25 @@ check_prereq() { } } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_api(){ destination="_peers" [[ $1 == "ee" ]] && { @@ -43,3 +62,6 @@ function build_api(){ check_prereq build_api $1 +if [[ $PATCH -eq 1 ]]; then + update_helm_release peers +fi From efa0049607bf09e9d22b9db4b57834becc7408ba Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 22 Mar 2023 10:37:52 +0100 Subject: [PATCH 184/253] feat(chalice): cherry pick build changes feat(assist): cherry pick build changes feat(alerts): cherry pick build changes --- api/build.sh | 36 +++++++++++++++++++++++++----------- api/build_alerts.sh | 23 +++++++++++++++++++++++ assist/build.sh | 22 ++++++++++++++++++++++ sourcemap-reader/build.sh | 23 ++++++++++++++++++++++- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/api/build.sh b/api/build.sh index bd4c35861..668ce537a 100644 --- a/api/build.sh +++ b/api/build.sh @@ -10,7 +10,7 @@ # Helper function exit_err() { err_code=$1 - if [[ err_code != 0 ]]; then + if [[ $err_code != 0 ]]; then exit $err_code fi } @@ -27,13 +27,32 @@ check_prereq() { return } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_api(){ destination="_api" [[ $1 == "ee" ]] && { destination="_api_ee" } cp -R ../api ../${destination} - cd ../${destination} + cd ../${destination} || exit_err 100 tag="" # Copy enterprise code [[ $1 == "ee" ]] && { @@ -43,7 +62,7 @@ function build_api(){ } mv Dockerfile.dockerignore .dockerignore docker build -f ./Dockerfile --build-arg envarg=$envarg --build-arg GIT_SHA=$git_sha -t ${DOCKER_REPO:-'local'}/chalice:${image_tag} . - cd ../api + cd ../api || exit_err 100 rm -rf ../${destination} [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/chalice:${image_tag} @@ -59,11 +78,6 @@ function build_api(){ check_prereq build_api $environment echo buil_complete -#IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO SIGN_IMAGE=$SIGN_IMAGE SIGN_KEY=$SIGN_KEY bash build_alerts.sh $1 -# -#[[ $environment == "ee" ]] && { -# cp ../ee/api/build_crons.sh . -# IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO SIGN_IMAGE=$SIGN_IMAGE SIGN_KEY=$SIGN_KEY bash build_crons.sh $1 -# exit_err $? -# rm build_crons.sh -#} || true +if [[ $PATCH -eq 1 ]]; then + update_helm_release chalice +fi \ No newline at end of file diff --git a/api/build_alerts.sh b/api/build_alerts.sh index b3c738b99..82ea53e00 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -17,6 +17,26 @@ check_prereq() { } } + +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_alerts(){ destination="_alerts" [[ $1 == "ee" ]] && { @@ -48,3 +68,6 @@ function build_alerts(){ check_prereq build_alerts $1 +if [[ $PATCH -eq 1 ]]; then + update_helm_release alerts +fi diff --git a/assist/build.sh b/assist/build.sh index 7a780cc43..04c191f35 100644 --- a/assist/build.sh +++ b/assist/build.sh @@ -15,6 +15,25 @@ check_prereq() { } } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_api(){ destination="_assist" [[ $1 == "ee" ]] && { @@ -44,3 +63,6 @@ function build_api(){ check_prereq build_api $1 +if [[ $PATCH -eq 1 ]]; then + update_helm_release assist +fi \ No newline at end of file diff --git a/sourcemap-reader/build.sh b/sourcemap-reader/build.sh index 3025e05e4..8ea8f6167 100644 --- a/sourcemap-reader/build.sh +++ b/sourcemap-reader/build.sh @@ -13,7 +13,6 @@ image_name="sourcemaps-reader" git_sha=$(git rev-parse --short HEAD) image_tag=${IMAGE_TAG:-git_sha} envarg="default-foss" -tmp_folder_name="${image_name}_${RANDOM}" check_prereq() { which docker || { @@ -23,6 +22,25 @@ check_prereq() { return } +[[ $1 == ee ]] && ee=true +[[ $PATCH -eq 1 ]] && { + image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + [[ $ee == "true" ]] && { + image_tag="${image_tag}-ee" + } +} +update_helm_release() { + chart=$1 + HELM_TAG="$(grep -iER ^version ../scripts/helmcharts/openreplay/charts/$chart | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" + # Update the chart version + sed -i "s#^version.*#version: $HELM_TAG# g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Update image tags + sed -i "s#ppVersion.*#ppVersion: \"$image_tag\"#g" ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + # Commit the changes + git add ../scripts/helmcharts/openreplay/charts/$chart/Chart.yaml + git commit -m "chore(helm): Updating $chart image release" +} + function build_api(){ destination="_smr" [[ $1 == "ee" ]] && { @@ -55,3 +73,6 @@ function build_api(){ check_prereq build_api $1 echo buil_complete +if [[ $PATCH -eq 1 ]]; then + update_helm_release sourcemapreader +fi \ No newline at end of file From 6565d0fb548d7de6363e85225c8eb50db366e916 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Mar 2023 10:04:37 +0100 Subject: [PATCH 185/253] feat(chalice): upgraded dependencies feat(alerts): upgraded dependencies feat(crons): upgraded dependencies --- api/requirements-alerts.txt | 18 +++++++++--------- api/requirements.txt | 18 +++++++++--------- ee/api/requirements-alerts.txt | 18 +++++++++--------- ee/api/requirements-crons.txt | 10 +++++----- ee/api/requirements.txt | 20 ++++++++++---------- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index edb644c87..804f59cb4 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -1,15 +1,15 @@ requests==2.28.2 -urllib3==1.26.14 -boto3==1.26.70 +urllib3==1.26.15 +boto3==1.26.100 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.6.1 -jira==3.4.1 +elasticsearch==8.6.2 +jira==3.5.0 -fastapi==0.94.1 -uvicorn[standard]==0.20.0 -python-decouple==3.7 -pydantic[email]==1.10.4 -apscheduler==3.10.0 \ No newline at end of file +fastapi==0.95.0 +uvicorn[standard]==0.21.1 +python-decouple==3.8 +pydantic[email]==1.10.7 +apscheduler==3.10.1 \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt index 490a147df..2fb86ae9f 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,17 +1,17 @@ requests==2.28.2 -urllib3==1.26.14 -boto3==1.26.70 +urllib3==1.26.15 +boto3==1.26.100 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.6.1 -jira==3.4.1 +elasticsearch==8.6.2 +jira==3.5.0 fastapi==0.95.0 -uvicorn[standard]==0.20.0 -python-decouple==3.7 -pydantic[email]==1.10.4 -apscheduler==3.10.0 +uvicorn[standard]==0.21.1 +python-decouple==3.8 +pydantic[email]==1.10.7 +apscheduler==3.10.1 -redis==4.5.1 \ No newline at end of file +redis==4.5.3 \ No newline at end of file diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index 6b6901ca5..b6f49f6c1 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -1,18 +1,18 @@ requests==2.28.2 -urllib3==1.26.14 -boto3==1.26.70 +urllib3==1.26.15 +boto3==1.26.100 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.6.1 -jira==3.4.1 +elasticsearch==8.6.2 +jira==3.5.0 -fastapi==0.94.1 -uvicorn[standard]==0.20.0 -python-decouple==3.7 -pydantic[email]==1.10.4 -apscheduler==3.10.0 +fastapi==0.95.0 +uvicorn[standard]==0.21.1 +python-decouple==3.8 +pydantic[email]==1.10.7 +apscheduler==3.10.1 clickhouse-driver==0.2.5 python-multipart==0.0.5 \ No newline at end of file diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 5f3742cdd..616a3f9d4 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -1,13 +1,13 @@ requests==2.28.2 -urllib3==1.26.14 -boto3==1.26.70 +urllib3==1.26.15 +boto3==1.26.100 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.6.1 -jira==3.4.1 +elasticsearch==8.6.2 +jira==3.5.0 -apscheduler==3.10.0 +apscheduler==3.10.1 clickhouse-driver==0.2.5 \ No newline at end of file diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index b5da59c4b..d6a3a0dc1 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -1,22 +1,22 @@ requests==2.28.2 -urllib3==1.26.14 -boto3==1.26.70 +urllib3==1.26.15 +boto3==1.26.100 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.6.1 -jira==3.4.1 +elasticsearch==8.6.2 +jira==3.5.0 fastapi==0.95.0 -uvicorn[standard]==0.20.0 -python-decouple==3.7 -pydantic[email]==1.10.4 -apscheduler==3.10.0 +uvicorn[standard]==0.21.1 +python-decouple==3.8 +pydantic[email]==1.10.7 +apscheduler==3.10.1 clickhouse-driver==0.2.5 python3-saml==1.15.0 -python-multipart==0.0.5 +python-multipart==0.0.6 -redis==4.5.1 +redis==4.5.3 #confluent-kafka==2.0.2 \ No newline at end of file From 130f298544905ceb2978306ee338be4dce3108af Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 12:10:04 +0200 Subject: [PATCH 186/253] chore(build): Updating folder name for build Signed-off-by: rjshrjndrn --- scripts/helmcharts/build_deploy.sh | 2 +- scripts/helmcharts/build_deploy_parallel.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/build_deploy.sh b/scripts/helmcharts/build_deploy.sh index f37f26c94..1df0d685a 100644 --- a/scripts/helmcharts/build_deploy.sh +++ b/scripts/helmcharts/build_deploy.sh @@ -24,7 +24,7 @@ echo $DOCKER_REPO docker login $DOCKER_REPO cd ../../backend bash build.sh $@ - cd ../utilities + cd ../assist bash build.sh $@ cd ../peers bash build.sh $@ diff --git a/scripts/helmcharts/build_deploy_parallel.sh b/scripts/helmcharts/build_deploy_parallel.sh index 268811a34..b91eac316 100644 --- a/scripts/helmcharts/build_deploy_parallel.sh +++ b/scripts/helmcharts/build_deploy_parallel.sh @@ -22,7 +22,7 @@ echo $DOCKER_REPO docker login $DOCKER_REPO # tmux set-option remain-on-exit on tmux split-window "cd ../../backend && IMAGE_TAG=$IMAGE_TAG DOCKER_REPO=$DOCKER_REPO PUSH_IMAGE=1 bash build.sh $@" - tmux split-window "cd ../../utilities && IMAGE_TAG=$IMAGE_TAG DOCKER_REPO=$DOCKER_REPO PUSH_IMAGE=1 bash build.sh $@" + tmux split-window "cd ../../assist && IMAGE_TAG=$IMAGE_TAG DOCKER_REPO=$DOCKER_REPO PUSH_IMAGE=1 bash build.sh $@" tmux select-layout tiled tmux split-window "cd ../../peers && IMAGE_TAG=$IMAGE_TAG DOCKER_REPO=$DOCKER_REPO PUSH_IMAGE=1 bash build.sh $@" tmux split-window "cd ../../frontend && IMAGE_TAG=$IMAGE_TAG DOCKER_REPO=$DOCKER_REPO PUSH_IMAGE=1 bash build.sh $@" From c9c28bc2381afd05b2b6c3496ddafd0a741fdc63 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 12:52:33 +0200 Subject: [PATCH 187/253] chore(ing): Adding proxy support for tracker script Signed-off-by: rjshrjndrn --- .../charts/frontend/templates/ingress.yaml | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/frontend/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/frontend/templates/ingress.yaml index a27fd1444..f573d614f 100644 --- a/scripts/helmcharts/openreplay/charts/frontend/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/frontend/templates/ingress.yaml @@ -44,5 +44,48 @@ spec: number: 8080 path: /(api|assist|ws-assist)/(private|sockets-list|sockets-live|peers)(.*) pathType: Prefix - +--- +kind: Service +apiVersion: v1 +metadata: + name: openreplay-static +spec: + type: ExternalName + externalName: static.openreplay.com +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: openreplay-static-ingress + annotations: + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + nginx.ingress.kubernetes.io/backend-protocol: https + nginx.ingress.kubernetes.io/upstream-vhost: static.openreplay.com + nginx.ingress.kubernetes.io/rewrite-target: '/$1/openreplay.js' + nginx.ingress.kubernetes.io/proxy-body-size: 8m + nginx.ingress.kubernetes.io/proxy-buffering: "on" + nginx.ingress.kubernetes.io/server-snippet: | + proxy_ssl_name static.openreplay.com; + proxy_ssl_server_name on; +spec: + ingressClassName: openreplay + tls: + - hosts: + - {{ .Values.global.domainName }} + {{- if .Values.ingress.tls.secretName}} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end}} + rules: + - host: {{ .Values.global.domainName }} + http: + paths: + - path: /script/(.*)/openreplay.js + pathType: Prefix + backend: + service: + name: openreplay-static + port: + number: 443 {{- end }} From 5900e009835e88d8733ec07dc31360f1d2ab540c Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Tue, 28 Mar 2023 13:29:01 +0200 Subject: [PATCH 188/253] feat(backend): upgrade /x/net library --- backend/go.mod | 6 +++--- backend/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 9633f2b18..4e6647a02 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -24,7 +24,7 @@ require ( github.com/sethvargo/go-envconfig v0.7.0 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce github.com/ua-parser/uap-go v0.0.0-20200325213135-e1c09f13e2fe - golang.org/x/net v0.1.1-0.20221104162952-702349b0e862 + golang.org/x/net v0.8.0 google.golang.org/api v0.81.0 ) @@ -61,8 +61,8 @@ require ( golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect diff --git a/backend/go.sum b/backend/go.sum index 676cf479b..d9c07010c 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -589,8 +589,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.1.1-0.20221104162952-702349b0e862 h1:KrLJ+iz8J6j6VVr/OCfULAcK+xozUmWE43fKpMR4MlI= -golang.org/x/net v0.1.1-0.20221104162952-702349b0e862/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -702,8 +702,8 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -715,8 +715,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From b7577c9bd96e4801f35c62a7cced5fead57812d8 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 14:12:44 +0200 Subject: [PATCH 189/253] chore(helm): adding tracker host env variable Signed-off-by: rjshrjndrn --- .../charts/frontend/templates/deployment.yaml | 64 +------------------ 1 file changed, 2 insertions(+), 62 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml index f685b76bc..9a604e01a 100644 --- a/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/frontend/templates/deployment.yaml @@ -43,68 +43,8 @@ spec: {{- .Values.healthCheck | toYaml | nindent 10}} {{- end}} env: - - name: AWS_ACCESS_KEY_ID - {{- if .Values.global.s3.existingSecret }} - valueFrom: - secretKeyRef: - name: {{ .Values.global.s3.existingSecret }} - key: access-key - {{- else }} - value: {{ .Values.global.s3.accessKey }} - {{- end }} - - name: AWS_SECRET_ACCESS_KEY - {{- if .Values.global.s3.existingSecret }} - valueFrom: - secretKeyRef: - name: {{ .Values.global.s3.existingSecret }} - key: secret-key - {{- else }} - value: {{ .Values.global.s3.secretKey }} - {{- end }} - - name: AWS_REGION - value: '{{ .Values.global.s3.region }}' - - name: LICENSE_KEY - value: '{{ .Values.global.enterpriseEditionLicense }}' - - name: KAFKA_SERVERS - value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' - - name: KAFKA_USE_SSL - value: '{{ .Values.global.kafka.kafkaUseSsl }}' - - name: pg_password - {{- if .Values.global.postgresql.existingSecret }} - valueFrom: - secretKeyRef: - name: {{ .Values.global.postgresql.existingSecret }} - key: postgresql-postgres-password - {{- else }} - value: '{{ .Values.global.postgresql.postgresqlPassword }}' - {{- end}} - - name: POSTGRES_STRING - value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:$(pg_password)@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' - # We need to check what is the object store endpoint. - # There can be 4 options - # 1. Using minio inside kube clster - # 2. Using minio managed external cluster, like aws minio offering - # 3. Using GCP or other object stores compatible with s3 apis - # 4. Using AWS itself. - # AWS uses bucketname.endpoint/object while others use endpoint/bucketname/object - - name: ASSETS_ORIGIN - {{- if contains "minio" .Values.global.s3.endpoint }} - # Local minio Installation - value: 'frontends://{{ .Values.global.domainName }}:{{.Values.global.ingress.controller.service.ports.https}}/{{.Values.global.s3.assetsBucket}}' - {{- else if contains "amazonaws.com" .Values.global.s3.endpoint }} - # AWS S3 - # Ref: frontends://stackoverflow.com/questions/53634583/go-template-split-string-by-delimiter - # We need frontends://bucketname.s3endpoint - value: {{ (split "://" .Values.global.s3.endpoint)._0 }}://{{.Values.global.s3.assetsBucket}}.{{ (split "://" .Values.global.s3.endpoint)._1 }} - {{- else }} - # S3 compatible storage - value: '{{ .Values.global.s3.endpoint }}/{{.Values.global.s3.assetsBucket}}' - {{- end }} - {{- include "openreplay.env.redis_string" .Values.global.redis | nindent 12 }} - {{- range $key, $val := .Values.global.env }} - - name: {{ $key }} - value: '{{ $val }}' - {{- end }} + - name: TRACKER_HOST + value: '{{ ternary "https" "http" .Values.global.ORSecureAccess}}://{{ .Values.global.domainName }}/script' {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' From 7d835a652dada9dd4dcf1e2e4ca7d4a0bac34f1a Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 28 Mar 2023 15:09:41 +0200 Subject: [PATCH 190/253] change(ui): use or script path from .env --- .../components/Client/Integrations/AssistDoc/AssistScript.tsx | 2 +- .../OnboardingTabs/ProjectCodeSnippet/ProjectCodeSnippet.js | 2 +- frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/Client/Integrations/AssistDoc/AssistScript.tsx b/frontend/app/components/Client/Integrations/AssistDoc/AssistScript.tsx index 107a33351..28b55e6fa 100644 --- a/frontend/app/components/Client/Integrations/AssistDoc/AssistScript.tsx +++ b/frontend/app/components/Client/Integrations/AssistDoc/AssistScript.tsx @@ -23,7 +23,7 @@ function AssistScript(props) { r.issue=function(k,p){r.push([6,k,p])}; r.isActive=function(){return false}; r.getSessionToken=function(){}; -})(0, "${props.projectKey}", "//static.openreplay.com/3.4.9/openreplay-assist.js",1,28); +})(0, "${props.projectKey}", "${window.env.TRACKER_HOST || '//static.openreplay.com'}/${window.env.TRACKER_VERSION}/openreplay-assist.js", 1, 28); `}
diff --git a/frontend/app/components/Onboarding/components/OnboardingTabs/ProjectCodeSnippet/ProjectCodeSnippet.js b/frontend/app/components/Onboarding/components/OnboardingTabs/ProjectCodeSnippet/ProjectCodeSnippet.js index 7bf9ef5bb..837281e8f 100644 --- a/frontend/app/components/Onboarding/components/OnboardingTabs/ProjectCodeSnippet/ProjectCodeSnippet.js +++ b/frontend/app/components/Onboarding/components/OnboardingTabs/ProjectCodeSnippet/ProjectCodeSnippet.js @@ -56,7 +56,7 @@ const ProjectCodeSnippet = props => { r.issue=function(k,p){r.push([6,k,p])}; r.isActive=function(){return false}; r.getSessionToken=function(){}; - })("//static.openreplay.com/${window.env.TRACKER_VERSION}/openreplay.js",1,0,initOpts,startOpts); + })("${window.env.TRACKER_HOST || '//static.openreplay.com'}/${window.env.TRACKER_VERSION}/openreplay.js",1,0,initOpts,startOpts); `; const saveGDPR = (value) => { diff --git a/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx b/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx index 116392534..5fa0fdd96 100644 --- a/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx +++ b/frontend/app/components/shared/CodeSnippet/CodeSnippet.tsx @@ -44,7 +44,7 @@ function CodeSnippet(props: Props) { r.issue=function(k,p){r.push([6,k,p])}; r.isActive=function(){return false}; r.getSessionToken=function(){}; - })("//static.openreplay.com/${window.env.TRACKER_VERSION}/openreplay.js",1,0,initOpts,startOpts); + })("${window.env.TRACKER_HOST || '//static.openreplay.com'}/${window.env.TRACKER_VERSION}/openreplay.js",1,0,initOpts,startOpts); `; return ( From ecfa8af8cb7a8c921d131fcebc55fbba32d522fa Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 15:31:42 +0200 Subject: [PATCH 191/253] chore(helm): Renaming OR tracker proxy ingress Signed-off-by: rjshrjndrn --- .../openreplay/charts/frontend/templates/ingress.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/frontend/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/frontend/templates/ingress.yaml index f573d614f..46b4f7ed4 100644 --- a/scripts/helmcharts/openreplay/charts/frontend/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/frontend/templates/ingress.yaml @@ -48,7 +48,7 @@ spec: kind: Service apiVersion: v1 metadata: - name: openreplay-static + name: openreplay-tracker spec: type: ExternalName externalName: static.openreplay.com @@ -56,7 +56,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: openreplay-static-ingress + name: openreplay-tracker-ingress annotations: {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} @@ -85,7 +85,7 @@ spec: pathType: Prefix backend: service: - name: openreplay-static + name: openreplay-tracker port: number: 443 {{- end }} From ccb608298bcf9bbb378c2bdede0943138d4d53c8 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 15:49:57 +0200 Subject: [PATCH 192/253] chore(cli): Adding log for restart Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index c0b3192ac..424da9ae7 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -351,8 +351,10 @@ do ;; -r | --restart) log title "Restarting OpenReplay Components" - kubectl rollout restart deployment -n "${APP_NS}" - kubectl rollout status deployment -n "${APP_NS}" + kubecolor rollout restart deployment -n "${APP_NS}" + log info "Waiting for restart to finish" + sleep 10 + kubecolor rollout status deployment -n "${APP_NS}" clean_tmp_dir exit 0 ;; From c68761dfba3a148d339c8fbec3fa736be319dfe4 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 15:54:29 +0200 Subject: [PATCH 193/253] chore(helm): Upgrading kubectl Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index 424da9ae7..ce9fed689 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -95,7 +95,7 @@ function install_packages() { log info Installing helm sudo /var/lib/openreplay/eget -q --upgrade-only --to "$OR_DIR" https://get.helm.sh/helm-v3.10.2-linux-amd64.tar.gz -f helm log info Installing kubectl - sudo /var/lib/openreplay/eget -q --upgrade-only --to "$OR_DIR" https://dl.k8s.io/release/v1.20.0/bin/linux/amd64/kubectl + sudo /var/lib/openreplay/eget -q --upgrade-only --to "$OR_DIR" https://dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubectl log info Installing Busybox sudo /var/lib/openreplay/eget -q --upgrade-only --to "$OR_DIR" https://busybox.net/downloads/binaries/1.35.0-x86_64-linux-musl/busybox date | sudo tee $OR_DIR/packages.lock &> /dev/null From 64768b27a4b9bd641ce63db96d17f952d82a8f3a Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 16:00:38 +0200 Subject: [PATCH 194/253] chore(cli): Adding install packages flag Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index ce9fed689..6f0125543 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -331,6 +331,12 @@ do clean_tmp_dir exit 0 ;; + -p | --install-packages) + log title "Updating/Installing dependency packages" + install_packages + clean_tmp_dir + exit 0 + ;; -u | --upgrade) log title "Upgrading OpenReplay" upgrade From 2b5281c08184b50b9fdf158072792d679f20b6fc Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 16:01:40 +0200 Subject: [PATCH 195/253] chore(cli): Removing unnecessary log Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index 6f0125543..008371c35 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -358,8 +358,6 @@ do -r | --restart) log title "Restarting OpenReplay Components" kubecolor rollout restart deployment -n "${APP_NS}" - log info "Waiting for restart to finish" - sleep 10 kubecolor rollout status deployment -n "${APP_NS}" clean_tmp_dir exit 0 From b6c654f91908f83200b36ea681a057b70cb835e3 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 28 Mar 2023 16:02:05 +0200 Subject: [PATCH 196/253] fix(ui) - play time null --- frontend/app/components/Session_/Player/Controls/Time.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/Session_/Player/Controls/Time.js b/frontend/app/components/Session_/Player/Controls/Time.js index 85291009f..e55f0b731 100644 --- a/frontend/app/components/Session_/Player/Controls/Time.js +++ b/frontend/app/components/Session_/Player/Controls/Time.js @@ -5,7 +5,7 @@ import { PlayTime } from 'App/player-ui' const ReduxTime = observer(({ format, name, isCustom }) => { const { store } = React.useContext(PlayerContext) - const time = store.get()[name] + const time = store.get()[name] || 0 return }) From 43d612e10039d029d4c3b547680690d6faf67e42 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Mar 2023 15:46:10 +0100 Subject: [PATCH 197/253] feat(chalice): support search for mouse_thrashing issues --- api/schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/schemas.py b/api/schemas.py index 5cae3a31a..992729870 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -500,6 +500,7 @@ class IssueType(str, Enum): crash = 'crash' custom = 'custom' js_exception = 'js_exception' + mouse_thrashing = 'mouse_thrashing' class MetricFormatType(str, Enum): From d1bf40b65d014583b21a013a83dfad4147b346ad Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 28 Mar 2023 17:05:38 +0200 Subject: [PATCH 198/253] change(ui): skip failed events/notes request --- frontend/app/duck/sessions.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/frontend/app/duck/sessions.ts b/frontend/app/duck/sessions.ts index 0da12dd1c..90cf7b8f0 100644 --- a/frontend/app/duck/sessions.ts +++ b/frontend/app/duck/sessions.ts @@ -405,7 +405,7 @@ export const fetch = export const fetchV2 = (sessionId: string) => (dispatch, getState) => { const apiClient = new APIClient() - const apiGet = (url: string, dispatch: any, FAILURE: string) => apiClient.get(url) + const apiGet = (url: string, dispatch: any, FAILURE?: string) => apiClient.get(url) .then(async (response) => { if (response.status === 403) { dispatch({ type: FETCH_ACCOUNT.FAILURE }); @@ -420,7 +420,7 @@ export const fetchV2 = (sessionId: string) => .catch(async (e) => { const data = await e.response?.json(); logger.error('Error during API request. ', e); - return dispatch({ type: FAILURE, errors: data ? parseError(data.errors) : [] }); + return FAILURE && dispatch({ type: FAILURE, errors: data ? parseError(data.errors) : [] }); }); const filter = getState().getIn(['filters', 'appliedFilter']) @@ -432,11 +432,15 @@ export const fetchV2 = (sessionId: string) => dispatch({ type: FETCHV2.SUCCESS, data, ...filter }); let [events, notes] = await Promise.all([ - apiGet(`/sessions/${sessionId}/events`, dispatch, FETCH_EVENTS.FAILURE), - apiGet(`/sessions/${sessionId}/notes`, dispatch, FETCH_NOTES.FAILURE), + apiGet(`/sessions/${sessionId}/events`, dispatch), + apiGet(`/sessions/${sessionId}/notes`, dispatch,), ]); - dispatch({ type: FETCH_EVENTS.SUCCESS, data: events.data, filter }); - dispatch({ type: FETCH_NOTES.SUCCESS, data: notes.data }); + if (notes) { + dispatch({ type: FETCH_NOTES.SUCCESS, data: notes.data }); + } + if (events) { + dispatch({ type: FETCH_EVENTS.SUCCESS, data: events.data, filter }); + } } if (jwt) { dispatch({ type: UPDATE_JWT, data: jwt }); From 0e156f4883757b691596fa6054b9b559d883d6a1 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 28 Mar 2023 17:10:02 +0200 Subject: [PATCH 199/253] change(ui): typo fix --- frontend/app/duck/sessions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/duck/sessions.ts b/frontend/app/duck/sessions.ts index 90cf7b8f0..f83209a8d 100644 --- a/frontend/app/duck/sessions.ts +++ b/frontend/app/duck/sessions.ts @@ -433,7 +433,7 @@ export const fetchV2 = (sessionId: string) => let [events, notes] = await Promise.all([ apiGet(`/sessions/${sessionId}/events`, dispatch), - apiGet(`/sessions/${sessionId}/notes`, dispatch,), + apiGet(`/sessions/${sessionId}/notes`, dispatch), ]); if (notes) { dispatch({ type: FETCH_NOTES.SUCCESS, data: notes.data }); From 6a54c1a26345618c28e9666bd2d6d6e030fb319b Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Tue, 28 Mar 2023 17:56:04 +0200 Subject: [PATCH 200/253] chore(init): suppres verbose logging Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index 008371c35..140c80ac3 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -81,7 +81,7 @@ function install_packages() { [[ -e "$OR_DIR/eget" ]] || { cd "$tmp_dir" || log err "Not able to cd to tmp dir $tmp_dir" curl --version &> /dev/null || log err "curl not found. Please install" - curl https://zyedidia.github.io/eget.sh | sh + curl -SsL https://zyedidia.github.io/eget.sh | sh - > /dev/null sudo mv eget $OR_DIR err_cd - } From 154944ebe9466a541803307017b41d4b4235b301 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Mar 2023 16:59:47 +0100 Subject: [PATCH 201/253] feat(chalice): changed health response --- api/chalicelib/core/health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index a9a54977c..f9be1cbd9 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -109,7 +109,7 @@ def __check_be_service(service_name): return fail_response return { "health": True, - "details": {} + "details": results.json().get("data", {}) } return fn From e57e04fcb40eaada614001639764843e5262381a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Mar 2023 17:16:21 +0100 Subject: [PATCH 202/253] feat(chalice): changed health response --- api/chalicelib/core/health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index f9be1cbd9..a9a54977c 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -109,7 +109,7 @@ def __check_be_service(service_name): return fail_response return { "health": True, - "details": results.json().get("data", {}) + "details": {} } return fn From 00f205dbbc01591ea58094cf8b81c90182c6d89b Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 28 Mar 2023 17:57:36 +0200 Subject: [PATCH 203/253] change(ui) - notes header alignment --- .../shared/SessionListContainer/components/Notes/NoteTags.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/shared/SessionListContainer/components/Notes/NoteTags.tsx b/frontend/app/components/shared/SessionListContainer/components/Notes/NoteTags.tsx index 2831824b3..b7a4f6101 100644 --- a/frontend/app/components/shared/SessionListContainer/components/Notes/NoteTags.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/Notes/NoteTags.tsx @@ -18,7 +18,7 @@ function NoteTags() { const { notesStore } = useStore(); return ( -
+
notesStore.toggleTag()} @@ -35,7 +35,7 @@ function NoteTags() { />
))} -
+
+ + )} - - - )} - - - - - - - - - - - - - - - - - - -
-
- By signing up, you agree to our{' '} - - terms of service - {' '} - and{' '} - - privacy policy - - . + + + + + + + + + + + + + + +
+
+ By signing up, you agree to our{' '} + + terms of service + {' '} + and{' '} + + privacy policy + + . +
-
- - {errors && ( -
- {errors.map((error) => ( -
- - - {error} -
- -
- ))} -
- )} -
+ + {errors && ( +
+ {errors.map((error) => ( +
+ + + {error} +
+ +
+ ))} +
+ )} + + +
Already having an account?{' '} Login
- +
); } } From 98ef275aa117e51c361bc240e90f3705451f14f0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 29 Mar 2023 10:53:31 +0100 Subject: [PATCH 211/253] feat(chalice): changed health-check behaviour --- api/routers/subs/health.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/api/routers/subs/health.py b/api/routers/subs/health.py index 2a1588a34..fdef52509 100644 --- a/api/routers/subs/health.py +++ b/api/routers/subs/health.py @@ -5,14 +5,16 @@ from routers.base import get_routers public_app, app, app_apikey = get_routers() -health_router = public_app -if tenants.tenants_exists(use_pool=False): - health_router = app - - -@health_router.get('/health', tags=["health-check"]) +@app.get('/health', tags=["health-check"]) def get_global_health_status(): - if tenants.tenants_exists(): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Not Found") return {"data": health.get_health()} + + +if not tenants.tenants_exists(use_pool=False): + @public_app.get('/health', tags=["health-check"]) + def get_public_health_status(): + if tenants.tenants_exists(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Not Found") + + return get_global_health_status() From 7d48c43af4c43dc24038eb2f7036020aada785fe Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 29 Mar 2023 11:01:47 +0100 Subject: [PATCH 212/253] feat(chalice): refactored health-check --- api/chalicelib/core/health.py | 55 ++++++++++-------------------- ee/api/chalicelib/core/health.py | 57 ++++++++++---------------------- 2 files changed, 35 insertions(+), 77 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index 139335b1a..ec5eeeb8f 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -6,43 +6,22 @@ from decouple import config from chalicelib.utils import pg_client -if config("LOCAL_DEV", cast=bool, default=False): - HEALTH_ENDPOINTS = { - "alerts": "http://127.0.0.1:8888/metrics", - "assets": "http://127.0.0.1:8888/metrics", - "assist": "http://127.0.0.1:8888/metrics", - "chalice": "http://127.0.0.1:8888/metrics", - "db": "http://127.0.0.1:8888/metrics", - "ender": "http://127.0.0.1:8888/metrics", - "heuristics": "http://127.0.0.1:8888/metrics", - "http": "http://127.0.0.1:8888/metrics", - "ingress-nginx": "http://127.0.0.1:8888/metrics", - "integrations": "http://127.0.0.1:8888/metrics", - "peers": "http://127.0.0.1:8888/metrics", - "quickwit": "http://127.0.0.1:8888/metrics", - "sink": "http://127.0.0.1:8888/metrics", - "sourcemapreader": "http://127.0.0.1:8888/metrics", - "storage": "http://127.0.0.1:8888/metrics", - "utilities": "http://127.0.0.1:8888/metrics" - } - -else: - HEALTH_ENDPOINTS = { - "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health", - "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", - "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", - "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", - "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", - "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", - "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics", - "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", - "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", - "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", - "peers": "http://peers-openreplay.app.svc.cluster.local:8888/health", - "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", - "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health", - "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", - } +HEALTH_ENDPOINTS = { + "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health", + "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", + "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", + "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", + "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", + "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", + "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics", + "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", + "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", + "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", + "peers": "http://peers-openreplay.app.svc.cluster.local:8888/health", + "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", + "sourcemaps-reader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health", + "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", +} def __check_database_pg(): @@ -173,7 +152,7 @@ def get_health(): "integrations": __check_be_service("integrations"), "peers": __check_be_service("peers"), "sink": __check_be_service("sink"), - "sourcemapreader": __check_be_service("sourcemapreader"), + "sourcemaps-reader": __check_be_service("sourcemapreader"), "storage": __check_be_service("storage") } } diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index a14458193..4d75deb7c 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -7,44 +7,23 @@ from decouple import config from chalicelib.utils import pg_client, ch_client -if config("LOCAL_DEV", cast=bool, default=False): - HEALTH_ENDPOINTS = { - "alerts": "http://127.0.0.1:8888/metrics", - "assets": "http://127.0.0.1:8888/metrics", - "assist": "http://127.0.0.1:8888/metrics", - "chalice": "http://127.0.0.1:8888/metrics", - "db": "http://127.0.0.1:8888/metrics", - "ender": "http://127.0.0.1:8888/metrics", - "heuristics": "http://127.0.0.1:8888/metrics", - "http": "http://127.0.0.1:8888/metrics", - "ingress-nginx": "http://127.0.0.1:8888/metrics", - "integrations": "http://127.0.0.1:8888/metrics", - "peers": "http://127.0.0.1:8888/metrics", - "quickwit": "http://127.0.0.1:8888/metrics", - "sink": "http://127.0.0.1:8888/metrics", - "sourcemapreader": "http://127.0.0.1:8888/metrics", - "storage": "http://127.0.0.1:8888/metrics", - "utilities": "http://127.0.0.1:8888/metrics" - } - -else: - HEALTH_ENDPOINTS = { - "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health", - "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", - "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", - "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", - "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", - "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", - "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics", - "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", - "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", - "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", - "peers": "http://peers-openreplay.app.svc.cluster.local:8888/health", - "quickwit": "http://quickwit-openreplay.app.svc.cluster.local:8888/metrics", - "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", - "sourcemapreader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health", - "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", - } +HEALTH_ENDPOINTS = { + "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health", + "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", + "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", + "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", + "db": "http://db-openreplay.app.svc.cluster.local:8888/metrics", + "ender": "http://ender-openreplay.app.svc.cluster.local:8888/metrics", + "heuristics": "http://heuristics-openreplay.app.svc.cluster.local:8888/metrics", + "http": "http://http-openreplay.app.svc.cluster.local:8888/metrics", + "ingress-nginx": "http://ingress-nginx-openreplay.app.svc.cluster.local:8888/metrics", + "integrations": "http://integrations-openreplay.app.svc.cluster.local:8888/metrics", + "peers": "http://peers-openreplay.app.svc.cluster.local:8888/health", + "quickwit": "http://quickwit-openreplay.app.svc.cluster.local:8888/metrics", + "sink": "http://sink-openreplay.app.svc.cluster.local:8888/metrics", + "sourcemaps-reader": "http://sourcemapreader-openreplay.app.svc.cluster.local:8888/health", + "storage": "http://storage-openreplay.app.svc.cluster.local:8888/metrics", +} def __check_database_pg(): @@ -179,7 +158,7 @@ def get_health(): "peers": __check_be_service("peers"), "quickwit": __check_be_service("quickwit"), "sink": __check_be_service("sink"), - "sourcemapreader": __check_be_service("sourcemapreader"), + "sourcemaps-reader": __check_be_service("sourcemapreader"), "storage": __check_be_service("storage") } } From b734fbd3ad759bff6f4a412a24298619c8f84ac8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 29 Mar 2023 11:23:26 +0100 Subject: [PATCH 213/253] feat(chalice): changed health-check response --- ee/api/chalicelib/core/health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 4d75deb7c..1a84a9715 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -95,7 +95,7 @@ def __check_be_service(service_name): # fail_response["details"]["errors"].append(results.text) except: print("couldn't get response") - fail_response["details"]["errors"].append(str(e)) + # fail_response["details"]["errors"].append(str(e)) return fail_response return { "health": True, From 0855466c47874f565bc769e4c4663cd2191d00d2 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 29 Mar 2023 12:24:49 +0200 Subject: [PATCH 214/253] fix(build): proper image tag Signed-off-by: rjshrjndrn --- .github/workflows/peers-ee.yaml | 2 +- peers/build.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/peers-ee.yaml b/.github/workflows/peers-ee.yaml index 564c5cf6d..ce014a45f 100644 --- a/.github/workflows/peers-ee.yaml +++ b/.github/workflows/peers-ee.yaml @@ -51,7 +51,7 @@ jobs: id: build-image env: DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee ENVIRONMENT: staging run: | skip_security_checks=${{ github.event.inputs.skip_security_checks }} diff --git a/peers/build.sh b/peers/build.sh index 51ea3f734..4c5f01679 100644 --- a/peers/build.sh +++ b/peers/build.sh @@ -16,6 +16,7 @@ check_prereq() { } [[ $1 == ee ]] && ee=true + [[ $PATCH -eq 1 ]] && { image_tag="$(grep -ER ^.ppVersion ../scripts/helmcharts/openreplay/charts/$chart | xargs | awk '{print $2}' | awk -F. -v OFS=. '{$NF += 1 ; print}')" [[ $ee == "true" ]] && { From e7c3547de472ba07f4293b1eb760fe7ba5e26bd7 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 29 Mar 2023 12:41:03 +0200 Subject: [PATCH 215/253] chore(actions): Create sourcemaps-reader-ee actions Signed-off-by: rjshrjndrn --- .github/workflows/sourcemaps-reader-ee.yaml | 141 ++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 .github/workflows/sourcemaps-reader-ee.yaml diff --git a/.github/workflows/sourcemaps-reader-ee.yaml b/.github/workflows/sourcemaps-reader-ee.yaml new file mode 100644 index 000000000..7e04ed0be --- /dev/null +++ b/.github/workflows/sourcemaps-reader-ee.yaml @@ -0,0 +1,141 @@ +# This action will push the sourcemapreader changes to aws +on: + workflow_dispatch: + inputs: + skip_security_checks: + description: 'Skip Security checks if there is a unfixable vuln or error. Value: true/false' + required: false + default: 'false' + push: + branches: + - dev + - api-* + paths: + - "sourcemap-reader/**" + - "!sourcemap-reader/.gitignore" + - "!sourcemap-reader/*-dev.sh" + +name: Build and Deploy sourcemap-reader + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + # We need to diff with old commit + # to see which workers got changed. + fetch-depth: 2 + + - name: Docker login + run: | + docker login ${{ secrets.EE_REGISTRY_URL }} -u ${{ secrets.EE_DOCKER_USERNAME }} -p "${{ secrets.EE_REGISTRY_TOKEN }}" + + - uses: azure/k8s-set-context@v1 + with: + method: kubeconfig + kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret. + id: setcontext + + # Caching docker images + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + + + - name: Building and Pushing sourcemaps-reader image + id: build-image + env: + DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee + ENVIRONMENT: staging + run: | + skip_security_checks=${{ github.event.inputs.skip_security_checks }} + cd sourcemap-reader + PUSH_IMAGE=0 bash -x ./build.sh + [[ "x$skip_security_checks" == "xtrue" ]] || { + curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ + images=("sourcemaps-reader") + for image in ${images[*]};do + ./trivy image --exit-code 1 --security-checks vuln --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG + done + err_code=$? + [[ $err_code -ne 0 ]] && { + exit $err_code + } + } && { + echo "Skipping Security Checks" + } + images=("sourcemaps-reader") + for image in ${images[*]};do + docker push $DOCKER_REPO/$image:$IMAGE_TAG + done + - name: Creating old image input + run: | + # + # Create yaml with existing image tags + # + kubectl get pods -n app -o jsonpath="{.items[*].spec.containers[*].image}" |\ + tr -s '[[:space:]]' '\n' | sort | uniq -c | grep '/foss/' | cut -d '/' -f3 > /tmp/image_tag.txt + + echo > /tmp/image_override.yaml + + for line in `cat /tmp/image_tag.txt`; + do + image_array=($(echo "$line" | tr ':' '\n')) + cat <> /tmp/image_override.yaml + ${image_array[0]}: + image: + tag: ${image_array[1]} + EOF + done + + - name: Deploy to kubernetes + run: | + cd scripts/helmcharts/ + + ## Update secerts + sed -i "s#openReplayContainerRegistry.*#openReplayContainerRegistry: \"${{ secrets.OSS_REGISTRY_URL }}\"#g" vars.yaml + sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"${{ secrets.EE_PG_PASSWORD }}\"/g" vars.yaml + sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"${{ secrets.EE_MINIO_ACCESS_KEY }}\"/g" vars.yaml + sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"${{ secrets.EE_MINIO_SECRET_KEY }}\"/g" vars.yaml + sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"${{ secrets.EE_JWT_SECRET }}\"/g" vars.yaml + sed -i "s/domainName: \"\"/domainName: \"${{ secrets.EE_DOMAIN_NAME }}\"/g" vars.yaml + sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.EE_LICENSE_KEY }}\"/g" vars.yaml + + # Update changed image tag + sed -i "/sourcemapreader/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + + cat /tmp/image_override.yaml + # Deploy command + mv openreplay/charts/{ingress-nginx,sourcemapreader,quickwit} /tmp + rm -rf openreplay/charts/* + mv /tmp/{ingress-nginx,sourcemapreader,quickwit} openreplay/charts/ + helm template openreplay -n app openreplay -f vars.yaml -f /tmp/image_override.yaml --set ingress-nginx.enabled=false --set skipMigration=true --no-hooks | kubectl apply -n app -f - + env: + DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} + ENVIRONMENT: staging + + - name: Alert slack + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_CHANNEL: foss + SLACK_TITLE: "Failed ${{ github.workflow }}" + SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff' + SLACK_WEBHOOK: ${{ secrets.SLACK_WEB_HOOK }} + SLACK_USERNAME: "OR Bot" + SLACK_MESSAGE: 'Build failed :bomb:' + + # - name: Debug Job + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # env: + # DOCKER_REPO: ${{ secrets.EE_REGISTRY_URL }} + # IMAGE_TAG: ${{ github.sha }} + # ENVIRONMENT: staging + From f5702741bcee891abad907b3ecce59337c00c4e6 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 12:46:07 +0200 Subject: [PATCH 216/253] change(ui) - illustrations --- frontend/app/svg/ca-no-alerts.svg | 89 ++++++----- frontend/app/svg/ca-no-dashboards.svg | 95 ++++++------ frontend/app/svg/ca-no-live-sessions.svg | 153 ++++++++++--------- frontend/app/svg/ca-no-metrics.svg | 45 ++++++ frontend/app/svg/ca-no-notes.svg | 87 +++++++---- frontend/app/svg/ca-no-recordings.svg | 109 ++++++------- frontend/app/svg/ca-no-search-results.svg | 84 +++++----- frontend/app/svg/ca-no-sessions-in-vault.svg | 104 ++++++++----- frontend/app/svg/ca-no-sessions.svg | 96 +++++++----- 9 files changed, 496 insertions(+), 366 deletions(-) create mode 100644 frontend/app/svg/ca-no-metrics.svg diff --git a/frontend/app/svg/ca-no-alerts.svg b/frontend/app/svg/ca-no-alerts.svg index cfe825f14..5864bbaba 100644 --- a/frontend/app/svg/ca-no-alerts.svg +++ b/frontend/app/svg/ca-no-alerts.svg @@ -1,42 +1,63 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + diff --git a/frontend/app/svg/ca-no-dashboards.svg b/frontend/app/svg/ca-no-dashboards.svg index d4abf5f2a..7eedeff65 100644 --- a/frontend/app/svg/ca-no-dashboards.svg +++ b/frontend/app/svg/ca-no-dashboards.svg @@ -1,53 +1,52 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + diff --git a/frontend/app/svg/ca-no-live-sessions.svg b/frontend/app/svg/ca-no-live-sessions.svg index 927e914f7..8997dbbfd 100644 --- a/frontend/app/svg/ca-no-live-sessions.svg +++ b/frontend/app/svg/ca-no-live-sessions.svg @@ -1,69 +1,86 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/ca-no-metrics.svg b/frontend/app/svg/ca-no-metrics.svg new file mode 100644 index 000000000..ea6fe261e --- /dev/null +++ b/frontend/app/svg/ca-no-metrics.svg @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/ca-no-notes.svg b/frontend/app/svg/ca-no-notes.svg index 03df7c155..3ea082746 100644 --- a/frontend/app/svg/ca-no-notes.svg +++ b/frontend/app/svg/ca-no-notes.svg @@ -1,37 +1,66 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/ca-no-recordings.svg b/frontend/app/svg/ca-no-recordings.svg index 4c5c3b173..49732b5e4 100644 --- a/frontend/app/svg/ca-no-recordings.svg +++ b/frontend/app/svg/ca-no-recordings.svg @@ -1,71 +1,52 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - + + diff --git a/frontend/app/svg/ca-no-search-results.svg b/frontend/app/svg/ca-no-search-results.svg index d595ee688..af808254e 100644 --- a/frontend/app/svg/ca-no-search-results.svg +++ b/frontend/app/svg/ca-no-search-results.svg @@ -1,52 +1,42 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + diff --git a/frontend/app/svg/ca-no-sessions-in-vault.svg b/frontend/app/svg/ca-no-sessions-in-vault.svg index 001c7759f..69470f8f6 100644 --- a/frontend/app/svg/ca-no-sessions-in-vault.svg +++ b/frontend/app/svg/ca-no-sessions-in-vault.svg @@ -1,42 +1,72 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/app/svg/ca-no-sessions.svg b/frontend/app/svg/ca-no-sessions.svg index 6a0de60e1..68179338e 100644 --- a/frontend/app/svg/ca-no-sessions.svg +++ b/frontend/app/svg/ca-no-sessions.svg @@ -1,40 +1,58 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cd3cdb815ac5fe71b059b80b9911ec42a5709f4c Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 12:52:31 +0200 Subject: [PATCH 217/253] change(ui) - illustrations --- frontend/app/svg/ca-no-cards.svg | 93 +++++++++++++----------------- frontend/app/svg/ca-no-metrics.svg | 45 --------------- 2 files changed, 40 insertions(+), 98 deletions(-) delete mode 100644 frontend/app/svg/ca-no-metrics.svg diff --git a/frontend/app/svg/ca-no-cards.svg b/frontend/app/svg/ca-no-cards.svg index d5d30a58c..ea6fe261e 100644 --- a/frontend/app/svg/ca-no-cards.svg +++ b/frontend/app/svg/ca-no-cards.svg @@ -1,58 +1,45 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + diff --git a/frontend/app/svg/ca-no-metrics.svg b/frontend/app/svg/ca-no-metrics.svg deleted file mode 100644 index ea6fe261e..000000000 --- a/frontend/app/svg/ca-no-metrics.svg +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From dbfa7f3ded9b4b019f71c61217400746ea702776 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 29 Mar 2023 12:46:36 +0200 Subject: [PATCH 218/253] fix(ui): properly clean player inst on unmount --- frontend/app/components/Session/WebPlayer.tsx | 6 +++++- frontend/app/player/web/MessageManager.ts | 6 +++++- frontend/app/player/web/Screen/Screen.ts | 6 ++++++ frontend/app/player/web/WebPlayer.ts | 5 ++++- frontend/app/player/web/managers/DOM/DOMManager.ts | 4 ++-- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/Session/WebPlayer.tsx b/frontend/app/components/Session/WebPlayer.tsx index 509add261..53fe57bf8 100644 --- a/frontend/app/components/Session/WebPlayer.tsx +++ b/frontend/app/components/Session/WebPlayer.tsx @@ -61,7 +61,11 @@ function WebPlayer(props: any) { WebPlayerInst.freeze() } - return () => WebPlayerInst.clean(); + return () => { + WebPlayerInst.clean(); + // @ts-ignore + setContextValue(defaultContextValue); + } }, [session.sessionId]); React.useEffect(() => { diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index b343eef96..d37ac6fa5 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -223,7 +223,11 @@ export default class MessageManager { for (let msg = fileReader.readNext();msg !== null;msg = fileReader.readNext()) { msgs.push(msg) } - const sorted = msgs.sort((m1, m2) => m1.time - m2.time) + const sorted = msgs.sort((m1, m2) => { + // @ts-ignore + if (m1.time === m2.time) return m1._index - m2._index + return m1.time - m2.time + }) let indx = sorted[0]._index let outOfOrderCounter = 0 diff --git a/frontend/app/player/web/Screen/Screen.ts b/frontend/app/player/web/Screen/Screen.ts index f27a251f1..d85857b01 100644 --- a/frontend/app/player/web/Screen/Screen.ts +++ b/frontend/app/player/web/Screen/Screen.ts @@ -82,6 +82,12 @@ export default class Screen { this.cursor = new Cursor(this.overlay, isMobile) // TODO: move outside } + clean() { + this.screen.removeChild(this.iframe) + this.screen.removeChild(this.overlay) + this.screen.remove(); + } + attach(parentElement: HTMLElement) { if (this.parentElement) { this.parentElement = null diff --git a/frontend/app/player/web/WebPlayer.ts b/frontend/app/player/web/WebPlayer.ts index d307b4308..63007214e 100644 --- a/frontend/app/player/web/WebPlayer.ts +++ b/frontend/app/player/web/WebPlayer.ts @@ -22,7 +22,7 @@ export default class WebPlayer extends Player { } private readonly inspectorController: InspectorController - protected readonly screen: Screen + protected screen: Screen protected readonly messageManager: MessageManager private targetMarker: TargetMarker @@ -149,6 +149,9 @@ export default class WebPlayer extends Player { clean = () => { super.clean() + this.screen.clean() + // @ts-ignore + this.screen = undefined; window.removeEventListener('resize', this.scale) } } diff --git a/frontend/app/player/web/managers/DOM/DOMManager.ts b/frontend/app/player/web/managers/DOM/DOMManager.ts index d54781028..d5cbdd845 100644 --- a/frontend/app/player/web/managers/DOM/DOMManager.ts +++ b/frontend/app/player/web/managers/DOM/DOMManager.ts @@ -120,7 +120,7 @@ export default class DOMManager extends ListWalker { } const parent = this.vElements.get(parentID) || this.vRoots.get(parentID) if (!parent) { - logger.error("Insert error. Parent node not found", parentID); + logger.error("Insert error. Parent node not found", parentID, this.vElements, this.vRoots); return; } @@ -177,7 +177,7 @@ export default class DOMManager extends ListWalker { case MType.CreateDocument: doc = this.screen.document; if (!doc) { - logger.error("No root iframe document found", msg) + logger.error("No root iframe document found", msg, this.screen) return; } doc.open(); From e7e2655daedc50baf6a76e60fecb970f559c069b Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 29 Mar 2023 13:18:01 +0200 Subject: [PATCH 219/253] fix(ui): fix current session clearing --- frontend/app/components/Session/Session.js | 7 ++++++- .../SessionListContainer/SessionListContainer.tsx | 10 +++++++--- frontend/app/duck/sessions.ts | 4 +++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/Session/Session.js b/frontend/app/components/Session/Session.js index a218b4ca9..75a96433b 100644 --- a/frontend/app/components/Session/Session.js +++ b/frontend/app/components/Session/Session.js @@ -2,7 +2,7 @@ import React from 'react'; import { useEffect, useState } from 'react'; import { connect } from 'react-redux'; import usePageTitle from 'App/hooks/usePageTitle'; -import { fetchV2 } from "Duck/sessions"; +import { fetchV2, clearCurrentSession } from "Duck/sessions"; import { fetchList as fetchSlackList } from 'Duck/integrations/slack'; import { Link, NoContent, Loader } from 'UI'; import { sessions as sessionsRoute } from 'App/routes'; @@ -18,6 +18,7 @@ function Session({ loading, hasErrors, fetchV2, + clearCurrentSession, }) { usePageTitle("OpenReplay Session Player"); const [ initializing, setInitializing ] = useState(true) @@ -29,6 +30,9 @@ function Session({ console.error("No sessionID in route.") } setInitializing(false) + return () => { + clearCurrentSession(); + } },[ sessionId ]); useEffect(() => { @@ -65,4 +69,5 @@ export default withPermissions(['SESSION_REPLAY'], '', true)(connect((state, pro }, { fetchSlackList, fetchV2, + clearCurrentSession, })(Session)); diff --git a/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx b/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx index 67b1e12e7..e3bc98b62 100644 --- a/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx +++ b/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx @@ -3,18 +3,22 @@ import SessionList from './components/SessionList'; import SessionHeader from './components/SessionHeader'; import NotesList from './components/Notes/NoteList'; import { connect } from 'react-redux'; -import { fetchList as fetchMembers } from 'Duck/member'; import LatestSessionsMessage from './components/LatestSessionsMessage'; +import { clearCurrentSession } from "Duck/sessions"; function SessionListContainer({ activeTab, - fetchMembers, members, + clearCurrentSession, }: { activeTab: string; fetchMembers: () => void; members: object[]; + clearCurrentSession: () => void; }) { + React.useEffect(() => { + clearCurrentSession() + }, []) return (
@@ -32,5 +36,5 @@ export default connect( // @ts-ignore members: state.getIn(['members', 'list']), }), - { fetchMembers } + { clearCurrentSession } )(SessionListContainer); diff --git a/frontend/app/duck/sessions.ts b/frontend/app/duck/sessions.ts index f83209a8d..576d8bc0f 100644 --- a/frontend/app/duck/sessions.ts +++ b/frontend/app/duck/sessions.ts @@ -134,7 +134,9 @@ const reducer = (state = initialState, action: IAction) => { return state.set('filteredEvents', filteredEvents).set('eventsQuery', query); } case CLEAR_CURRENT_SESSION: { - return state.set('current', new Session()) + const session = new Session(); + + return state.set('current', session) .set('eventsIndex', []) .set('visitedEvents', List()) .set('host', ''); From 92335f5cf5bd775a1d2a4b9762a33b26688862bd Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 13:41:03 +0200 Subject: [PATCH 220/253] fix(ui) - check for api errors on project create --- .../components/Client/Sites/NewSiteForm.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/frontend/app/components/Client/Sites/NewSiteForm.js b/frontend/app/components/Client/Sites/NewSiteForm.js index 6ce0f7d4c..0527668c3 100644 --- a/frontend/app/components/Client/Sites/NewSiteForm.js +++ b/frontend/app/components/Client/Sites/NewSiteForm.js @@ -58,9 +58,7 @@ export default class NewSiteForm extends React.PureComponent { siteList, location: { pathname }, } = this.props; - if (!site.exists() && siteList.some(({ name }) => name === site.name)) { - return this.setState({ existsError: true }); - } + if (site.exists()) { this.props.update(this.props.site, this.props.site.id).then((response) => { if (!response || !response.errors || response.errors.size === 0) { @@ -72,11 +70,16 @@ export default class NewSiteForm extends React.PureComponent { } }); } else { - this.props.save(this.props.site).then(() => { - this.props.onClose(null); - this.props.clearSearch(); - this.props.clearSearchLive(); - this.props.mstore.initClient(); + this.props.save(this.props.site).then((response) => { + if (!response || !response.errors || response.errors.size === 0) { + this.props.onClose(null); + this.props.clearSearch(); + this.props.clearSearchLive(); + this.props.mstore.initClient(); + toast.success('Project added successfully'); + } else { + toast.error(response.errors[0]); + } }); } }; From 4e4fa774048ef9a36b9525e325628c42c97d74aa Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 29 Mar 2023 13:47:29 +0200 Subject: [PATCH 221/253] fix(actions): Sourcemap reader image tag Signed-off-by: rjshrjndrn --- .github/workflows/sourcemaps-reader-ee.yaml | 3 ++- .github/workflows/sourcemaps-reader.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sourcemaps-reader-ee.yaml b/.github/workflows/sourcemaps-reader-ee.yaml index 7e04ed0be..19a60eba3 100644 --- a/.github/workflows/sourcemaps-reader-ee.yaml +++ b/.github/workflows/sourcemaps-reader-ee.yaml @@ -107,7 +107,8 @@ jobs: sed -i "s/enterpriseEditionLicense: \"\"/enterpriseEditionLicense: \"${{ secrets.EE_LICENSE_KEY }}\"/g" vars.yaml # Update changed image tag - sed -i "/sourcemapreader/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + sed -i "/sourcemaps-reader/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + sed -i "/sourcemaps-reader/sourcemapreader/g" /tmp/image_override.yaml cat /tmp/image_override.yaml # Deploy command diff --git a/.github/workflows/sourcemaps-reader.yaml b/.github/workflows/sourcemaps-reader.yaml index fb3face2f..6f065b990 100644 --- a/.github/workflows/sourcemaps-reader.yaml +++ b/.github/workflows/sourcemaps-reader.yaml @@ -106,7 +106,8 @@ jobs: sed -i "s/domainName: \"\"/domainName: \"${{ secrets.OSS_DOMAIN_NAME }}\"/g" vars.yaml # Update changed image tag - sed -i "/sourcemapreader/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + sed -i "/sourcemaps-reader/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml + sed -i "/sourcemaps-reader/sourcemapreader/g" /tmp/image_override.yaml cat /tmp/image_override.yaml # Deploy command From 8a4ee2d31fb8adaab8713d5a2dbda777fd76b54b Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 13:49:14 +0200 Subject: [PATCH 222/253] change(ui) - sessions page max-width --- frontend/app/components/Overview/Overview.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/Overview/Overview.tsx b/frontend/app/components/Overview/Overview.tsx index 9d71b5702..775a8d0ae 100644 --- a/frontend/app/components/Overview/Overview.tsx +++ b/frontend/app/components/Overview/Overview.tsx @@ -9,16 +9,16 @@ import OverviewMenu from 'Shared/OverviewMenu'; function Overview() { return ( -
+
-
+
From 34d3ad419e457423b30d845dddf6b431926c8931 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 29 Mar 2023 13:51:40 +0200 Subject: [PATCH 223/253] fix(actions): smr image tag replacement Signed-off-by: rjshrjndrn --- .github/workflows/sourcemaps-reader-ee.yaml | 2 +- .github/workflows/sourcemaps-reader.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sourcemaps-reader-ee.yaml b/.github/workflows/sourcemaps-reader-ee.yaml index 19a60eba3..0bee8ba4e 100644 --- a/.github/workflows/sourcemaps-reader-ee.yaml +++ b/.github/workflows/sourcemaps-reader-ee.yaml @@ -108,7 +108,7 @@ jobs: # Update changed image tag sed -i "/sourcemaps-reader/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml - sed -i "/sourcemaps-reader/sourcemapreader/g" /tmp/image_override.yaml + sed -i "s/sourcemaps-reader/sourcemapreader/g" /tmp/image_override.yaml cat /tmp/image_override.yaml # Deploy command diff --git a/.github/workflows/sourcemaps-reader.yaml b/.github/workflows/sourcemaps-reader.yaml index 6f065b990..bbc7ae887 100644 --- a/.github/workflows/sourcemaps-reader.yaml +++ b/.github/workflows/sourcemaps-reader.yaml @@ -107,7 +107,7 @@ jobs: # Update changed image tag sed -i "/sourcemaps-reader/{n;n;s/.*/ tag: ${IMAGE_TAG}/}" /tmp/image_override.yaml - sed -i "/sourcemaps-reader/sourcemapreader/g" /tmp/image_override.yaml + sed -i "s/sourcemaps-reader/sourcemapreader/g" /tmp/image_override.yaml cat /tmp/image_override.yaml # Deploy command From f471fd3ae1af2a205ef60d4abd91a77eef3b9219 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 13:56:13 +0200 Subject: [PATCH 224/253] change(ui) - sessions page title --- .../SessionHeader/SessionHeader.tsx | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx b/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx index 9222b8183..dcc986f0f 100644 --- a/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import { numberWithCommas } from 'App/utils'; import { applyFilter } from 'Duck/search'; import Period from 'Types/app/period'; @@ -7,23 +7,9 @@ import SessionTags from '../SessionTags'; import NoteTags from '../Notes/NoteTags'; import { connect } from 'react-redux'; import SessionSort from '../SessionSort'; -import cn from 'classnames'; import { setActiveTab } from 'Duck/search'; import SessionSettingButton from '../SessionSettingButton'; -// @ts-ignore -const Tab = ({ addBorder, onClick, children }) => ( -
- {children} -
-); - interface Props { listCount: number; filter: any; @@ -41,6 +27,16 @@ function SessionHeader(props: Props) { const period = Period({ start: startDate, end: endDate, rangeName: rangeValue }); + const title = useMemo(() => { + if (activeTab === 'notes') { + return 'Notes'; + } + if (activeTab === 'bookmark') { + return isEnterprise? 'Vault' : 'Bookmarks'; + } + return 'Sessions'; + }, [activeTab]); + const onDateChange = (e: any) => { const dateValues = e.toJSON(); props.applyFilter(dateValues); @@ -48,6 +44,7 @@ function SessionHeader(props: Props) { return (
+

{title}

{activeTab !== 'notes' ? (
{activeTab !== 'bookmark' && ( From 3a3e1ee6885763437a9fd4a52bc2272ba3f4ece8 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 29 Mar 2023 14:24:14 +0200 Subject: [PATCH 225/253] fix(ui): fix clickmap crash --- frontend/app/player/web/Screen/Screen.ts | 6 +++--- frontend/app/player/web/WebPlayer.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/app/player/web/Screen/Screen.ts b/frontend/app/player/web/Screen/Screen.ts index d85857b01..92479a4af 100644 --- a/frontend/app/player/web/Screen/Screen.ts +++ b/frontend/app/player/web/Screen/Screen.ts @@ -83,9 +83,9 @@ export default class Screen { } clean() { - this.screen.removeChild(this.iframe) - this.screen.removeChild(this.overlay) - this.screen.remove(); + this.iframe?.remove?.(); + this.overlay?.remove?.(); + this.screen?.remove?.(); } attach(parentElement: HTMLElement) { diff --git a/frontend/app/player/web/WebPlayer.ts b/frontend/app/player/web/WebPlayer.ts index 63007214e..c44ddc7b2 100644 --- a/frontend/app/player/web/WebPlayer.ts +++ b/frontend/app/player/web/WebPlayer.ts @@ -137,7 +137,7 @@ export default class WebPlayer extends Player { } showClickmap = (...args: Parameters) => { - this.screen.overlay.remove() // hack. TODO: 1.split Screen functionalities (overlay, mounter) 2. separate ClickMapPlayer class that does not create overlay + this.screen?.overlay?.remove?.() // hack. TODO: 1.split Screen functionalities (overlay, mounter) 2. separate ClickMapPlayer class that does not create overlay this.freeze().then(() => { this.targetMarker.injectTargets(...args) }) From 57572b942c0b9860409c11a56e8875d027cd8e84 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 29 Mar 2023 13:31:08 +0100 Subject: [PATCH 226/253] feat(chalice): changed env-vars --- api/env.default | 2 +- ee/api/env.default | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/env.default b/api/env.default index 8d0131dd7..074d9b643 100644 --- a/api/env.default +++ b/api/env.default @@ -40,7 +40,7 @@ PG_POOL=true sessions_bucket=mobs sessions_region=us-east-1 sourcemaps_bucket=sourcemaps -sourcemaps_reader=http://sourcemaps-reader-openreplay.app.svc.cluster.local:9000/sourcemaps/%s/sourcemaps +sourcemaps_reader=http://sourcemapreader-openreplay.app.svc.cluster.local:9000/sourcemaps/%s/sourcemaps STAGE=default-foss version_number=1.4.0 FS_DIR=/mnt/efs diff --git a/ee/api/env.default b/ee/api/env.default index c5d1c95f7..ff82cb2af 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -51,7 +51,7 @@ ASSIST_RECORDS_BUCKET=records sessions_bucket=mobs sessions_region=us-east-1 sourcemaps_bucket=sourcemaps -sourcemaps_reader=http://sourcemaps-reader-openreplay.app.svc.cluster.local:9000/sourcemaps/%s/sourcemaps +sourcemaps_reader=http://sourcemapreader-openreplay.app.svc.cluster.local:9000/sourcemaps/%s/sourcemaps version_number=1.0.0 FS_DIR=/mnt/efs EXP_SESSIONS_SEARCH=false From b3b2b8d047e85772ef94e4757bb45eb413c2d67b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 29 Mar 2023 13:38:23 +0100 Subject: [PATCH 227/253] feat(chalice): ignore quickwit health-check --- ee/api/chalicelib/core/health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 1a84a9715..e530c4a64 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -156,7 +156,7 @@ def get_health(): "ingress-nginx": __always_healthy, "integrations": __check_be_service("integrations"), "peers": __check_be_service("peers"), - "quickwit": __check_be_service("quickwit"), + # "quickwit": __check_be_service("quickwit"), "sink": __check_be_service("sink"), "sourcemaps-reader": __check_be_service("sourcemapreader"), "storage": __check_be_service("storage") From 387cb44dd63bc477af6d6896b9b3e9f568465b68 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 14:39:00 +0200 Subject: [PATCH 228/253] fix(ui) - check for jira/github integrations --- frontend/app/components/Session_/Subheader.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/Session_/Subheader.js b/frontend/app/components/Session_/Subheader.js index 8a8d4d2f6..8c87cd184 100644 --- a/frontend/app/components/Session_/Subheader.js +++ b/frontend/app/components/Session_/Subheader.js @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import { Icon, Tooltip, Button } from 'UI'; import QueueControls from './QueueControls'; import Bookmark from 'Shared/Bookmark'; @@ -33,6 +33,15 @@ function SubHeader(props) { endTime, } = store.get(); + const enabledIntegration = useMemo(() => { + const { integrations } = props; + if (!integrations || !integrations.size) { + return false; + } + + return integrations.some((i) => i.token); + }) + const mappedResourceList = resourceList .filter((r) => r.isRed || r.isYellow) .concat(fetchList.filter((i) => parseInt(i.status) >= 400)) @@ -120,7 +129,7 @@ function SubHeader(props) { Create Bug Report - + {enabledIntegration && } ({ siteId: state.getIn(['site', 'siteId']) }))(observer(SubHeader)); +export default connect((state) => ({ + siteId: state.getIn(['site', 'siteId']), + integrations: state.getIn([ 'issues', 'list' ]) +}))(observer(SubHeader)); From d600c1ac2c191847e50f9ecc62df98c2c9a01746 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 29 Mar 2023 13:42:06 +0100 Subject: [PATCH 229/253] feat(chalice): changed health-check logs --- api/chalicelib/core/health.py | 4 ++-- ee/api/chalicelib/core/health.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index ec5eeeb8f..a40e6f9b9 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -77,7 +77,7 @@ def __check_be_service(service_name): try: results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2) if results.status_code != 200: - print(f"!! issue with the storage-health code:{results.status_code}") + print(f"!! issue with the {service_name}-health code:{results.status_code}") print(results.text) # fail_response["details"]["errors"].append(results.text) return fail_response @@ -86,7 +86,7 @@ def __check_be_service(service_name): # fail_response["details"]["errors"].append("timeout") return fail_response except Exception as e: - print("!! Issue getting storage-health response") + print(f"!! Issue getting {service_name}-health response") print(str(e)) try: print(results.text) diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index e530c4a64..d20e47dae 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -79,7 +79,7 @@ def __check_be_service(service_name): try: results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2) if results.status_code != 200: - print(f"!! issue with the storage-health code:{results.status_code}") + print(f"!! issue with the {service_name}-health code:{results.status_code}") print(results.text) # fail_response["details"]["errors"].append(results.text) return fail_response @@ -88,7 +88,7 @@ def __check_be_service(service_name): # fail_response["details"]["errors"].append("timeout") return fail_response except Exception as e: - print("!! Issue getting storage-health response") + print(f"!! Issue getting {service_name}-health response") print(str(e)) try: print(results.text) From 480dd0cb00c9176e662a7bc6a96cd2d8d850bd3b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 29 Mar 2023 13:47:20 +0100 Subject: [PATCH 230/253] feat(chalice): fixed health-check for sourcemaps-reader --- api/chalicelib/core/health.py | 2 +- ee/api/chalicelib/core/health.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index a40e6f9b9..76fcf35ca 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -152,7 +152,7 @@ def get_health(): "integrations": __check_be_service("integrations"), "peers": __check_be_service("peers"), "sink": __check_be_service("sink"), - "sourcemaps-reader": __check_be_service("sourcemapreader"), + "sourcemaps-reader": __check_be_service("sourcemaps-reader"), "storage": __check_be_service("storage") } } diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index d20e47dae..26b3a2b24 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -158,7 +158,7 @@ def get_health(): "peers": __check_be_service("peers"), # "quickwit": __check_be_service("quickwit"), "sink": __check_be_service("sink"), - "sourcemaps-reader": __check_be_service("sourcemapreader"), + "sourcemaps-reader": __check_be_service("sourcemaps-reader"), "storage": __check_be_service("storage") } } From 0fbca8558d48797f120a48f2ce3b169495dbed5a Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 29 Mar 2023 15:14:55 +0200 Subject: [PATCH 231/253] fix(ui): add thrashing to issue list for search --- frontend/app/constants/filterOptions.js | 1 + frontend/app/types/filter/filterType.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/frontend/app/constants/filterOptions.js b/frontend/app/constants/filterOptions.js index 3fc7e0bc1..139f96f7f 100644 --- a/frontend/app/constants/filterOptions.js +++ b/frontend/app/constants/filterOptions.js @@ -118,6 +118,7 @@ export const issueOptions = [ { label: 'Crash', value: IssueType.CRASH }, { label: 'Custom', value: IssueType.CUSTOM }, { label: 'Error', value: IssueType.JS_EXCEPTION }, + { label: 'Mouse Thrashing', value: IssueType.MOUSE_THRASHING } ] export const issueCategories = [ diff --git a/frontend/app/types/filter/filterType.ts b/frontend/app/types/filter/filterType.ts index 842b0df8e..879dc985e 100644 --- a/frontend/app/types/filter/filterType.ts +++ b/frontend/app/types/filter/filterType.ts @@ -147,6 +147,8 @@ export enum IssueType { CRASH = 'crash', CUSTOM = 'custom', JS_EXCEPTION = 'js_exception', + + MOUSE_THRASHING = 'mouse_thrashing', } export enum IssueCategory { From 57c30055485e817c62bde29632161d9a5671e605 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 15:13:13 +0200 Subject: [PATCH 232/253] change(ui) - illustrations --- frontend/app/svg/ca-no-live-sessions.svg | 136 +++++++++++------------ frontend/app/svg/ca-no-sessions.svg | 12 +- 2 files changed, 73 insertions(+), 75 deletions(-) diff --git a/frontend/app/svg/ca-no-live-sessions.svg b/frontend/app/svg/ca-no-live-sessions.svg index 8997dbbfd..51edffdeb 100644 --- a/frontend/app/svg/ca-no-live-sessions.svg +++ b/frontend/app/svg/ca-no-live-sessions.svg @@ -1,75 +1,76 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -77,10 +78,7 @@ - - - - + diff --git a/frontend/app/svg/ca-no-sessions.svg b/frontend/app/svg/ca-no-sessions.svg index 68179338e..5f2514cf0 100644 --- a/frontend/app/svg/ca-no-sessions.svg +++ b/frontend/app/svg/ca-no-sessions.svg @@ -17,9 +17,9 @@ - - - + + + @@ -33,7 +33,7 @@ - + @@ -43,8 +43,8 @@ - - + + From a79f7a4531a71a81c75798d4ee48792b76d42d5b Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 15:43:05 +0200 Subject: [PATCH 233/253] fix(ui) - flag height --- .../Player/ReplayPlayer/EventsBlock/UserCard/UserCard.js | 2 +- frontend/app/components/ui/CountryFlag/CountryFlag.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/Session/Player/ReplayPlayer/EventsBlock/UserCard/UserCard.js b/frontend/app/components/Session/Player/ReplayPlayer/EventsBlock/UserCard/UserCard.js index e6893ca3e..ef80fd0bf 100644 --- a/frontend/app/components/Session/Player/ReplayPlayer/EventsBlock/UserCard/UserCard.js +++ b/frontend/app/components/Session/Player/ReplayPlayer/EventsBlock/UserCard/UserCard.js @@ -79,7 +79,7 @@ function UserCard({ className, request, session, width, height, similarSessions, render={() => (
} + comp={} label={countries[userCountry]} value={{formatTimeOrDate(startedAt)}} /> diff --git a/frontend/app/components/ui/CountryFlag/CountryFlag.js b/frontend/app/components/ui/CountryFlag/CountryFlag.js index 09192f070..4e4ba4cba 100644 --- a/frontend/app/components/ui/CountryFlag/CountryFlag.js +++ b/frontend/app/components/ui/CountryFlag/CountryFlag.js @@ -4,7 +4,7 @@ import { countries } from 'App/constants'; import { Icon } from 'UI'; import stl from './countryFlag.module.css'; -const CountryFlag = ({ country = '', className = '', style = {}, label = false }) => { +const CountryFlag = ({ country = '', className = '', style = {}, label = false, width = 22, height = 15}) => { const knownCountry = !!country && country !== 'UN'; const countryFlag = knownCountry ? country.toLowerCase() : ''; const countryName = knownCountry ? countries[ country ] : 'Unknown Country'; @@ -12,7 +12,7 @@ const CountryFlag = ({ country = '', className = '', style = {}, label = false } return (
{knownCountry - ?
+ ?
: (
From 09934de91e38e7abed338693cf748d9df074ca12 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 15:47:58 +0200 Subject: [PATCH 234/253] fix(ui) - menu rounded --- frontend/app/components/Header/SettingsMenu/SettingsMenu.tsx | 2 +- frontend/app/components/Header/UserMenu/UserMenu.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/Header/SettingsMenu/SettingsMenu.tsx b/frontend/app/components/Header/SettingsMenu/SettingsMenu.tsx index 2faf5e128..3d628594c 100644 --- a/frontend/app/components/Header/SettingsMenu/SettingsMenu.tsx +++ b/frontend/app/components/Header/SettingsMenu/SettingsMenu.tsx @@ -36,7 +36,7 @@ function SettingsMenu(props: RouteComponentProps) { return (
{isAdmin && ( <> diff --git a/frontend/app/components/Header/UserMenu/UserMenu.tsx b/frontend/app/components/Header/UserMenu/UserMenu.tsx index c54d582b0..ddc3cc2f1 100644 --- a/frontend/app/components/Header/UserMenu/UserMenu.tsx +++ b/frontend/app/components/Header/UserMenu/UserMenu.tsx @@ -24,7 +24,7 @@ function UserMenu(props: RouteComponentProps) { return (
From d74568dfb5962e534848cc7b61d30690c714a7ef Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 29 Mar 2023 16:26:04 +0200 Subject: [PATCH 235/253] fix(ui): fix for hover for health modal --- .../app/components/Header/HealthStatus/ServiceCategory.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx b/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx index a3ecaf4d9..4a9e1bc15 100644 --- a/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx +++ b/frontend/app/components/Header/HealthStatus/ServiceCategory.tsx @@ -29,8 +29,8 @@ function Category({ return (
Date: Wed, 29 Mar 2023 15:30:04 +0100 Subject: [PATCH 236/253] feat(chalice): pull changes --- api/chalicelib/core/health.py | 6 +++--- ee/api/chalicelib/core/health.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index ec5eeeb8f..76fcf35ca 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -77,7 +77,7 @@ def __check_be_service(service_name): try: results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2) if results.status_code != 200: - print(f"!! issue with the storage-health code:{results.status_code}") + print(f"!! issue with the {service_name}-health code:{results.status_code}") print(results.text) # fail_response["details"]["errors"].append(results.text) return fail_response @@ -86,7 +86,7 @@ def __check_be_service(service_name): # fail_response["details"]["errors"].append("timeout") return fail_response except Exception as e: - print("!! Issue getting storage-health response") + print(f"!! Issue getting {service_name}-health response") print(str(e)) try: print(results.text) @@ -152,7 +152,7 @@ def get_health(): "integrations": __check_be_service("integrations"), "peers": __check_be_service("peers"), "sink": __check_be_service("sink"), - "sourcemaps-reader": __check_be_service("sourcemapreader"), + "sourcemaps-reader": __check_be_service("sourcemaps-reader"), "storage": __check_be_service("storage") } } diff --git a/ee/api/chalicelib/core/health.py b/ee/api/chalicelib/core/health.py index 1a84a9715..26b3a2b24 100644 --- a/ee/api/chalicelib/core/health.py +++ b/ee/api/chalicelib/core/health.py @@ -79,7 +79,7 @@ def __check_be_service(service_name): try: results = requests.get(HEALTH_ENDPOINTS.get(service_name), timeout=2) if results.status_code != 200: - print(f"!! issue with the storage-health code:{results.status_code}") + print(f"!! issue with the {service_name}-health code:{results.status_code}") print(results.text) # fail_response["details"]["errors"].append(results.text) return fail_response @@ -88,7 +88,7 @@ def __check_be_service(service_name): # fail_response["details"]["errors"].append("timeout") return fail_response except Exception as e: - print("!! Issue getting storage-health response") + print(f"!! Issue getting {service_name}-health response") print(str(e)) try: print(results.text) @@ -156,9 +156,9 @@ def get_health(): "ingress-nginx": __always_healthy, "integrations": __check_be_service("integrations"), "peers": __check_be_service("peers"), - "quickwit": __check_be_service("quickwit"), + # "quickwit": __check_be_service("quickwit"), "sink": __check_be_service("sink"), - "sourcemaps-reader": __check_be_service("sourcemapreader"), + "sourcemaps-reader": __check_be_service("sourcemaps-reader"), "storage": __check_be_service("storage") } } From 1c434d51dbf9f613f800441ae61c458629a8699b Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 16:41:10 +0200 Subject: [PATCH 237/253] change(ui) - tooltip text on dashboard --- .../DashboardHeader/DashboardHeader.tsx | 4 ++-- .../MetricViewHeader/MetricViewHeader.tsx | 16 ++++++++-------- .../components/WidgetName/WidgetName.tsx | 2 +- .../BugReport/components/ReportTitle.tsx | 2 +- frontend/app/components/ui/Button/Button.tsx | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/frontend/app/components/Dashboard/components/DashboardHeader/DashboardHeader.tsx b/frontend/app/components/Dashboard/components/DashboardHeader/DashboardHeader.tsx index 5f312d124..0d850038c 100644 --- a/frontend/app/components/Dashboard/components/DashboardHeader/DashboardHeader.tsx +++ b/frontend/app/components/Dashboard/components/DashboardHeader/DashboardHeader.tsx @@ -71,7 +71,7 @@ function DashboardHeader(props: Props) { + {dashboard?.name} } @@ -118,7 +118,7 @@ function DashboardHeader(props: Props) {
{/* @ts-ignore */} - +

onEdit(false)} diff --git a/frontend/app/components/Dashboard/components/MetricViewHeader/MetricViewHeader.tsx b/frontend/app/components/Dashboard/components/MetricViewHeader/MetricViewHeader.tsx index 081dd21e5..1e1775415 100644 --- a/frontend/app/components/Dashboard/components/MetricViewHeader/MetricViewHeader.tsx +++ b/frontend/app/components/Dashboard/components/MetricViewHeader/MetricViewHeader.tsx @@ -53,6 +53,13 @@ function MetricViewHeader() { isSearchable={true} /> + + metricStore.updateKey('filter', { ...filter, dashboard: value }) + } + /> + ) : ( // @ts-ignore - +
setEditing(true)} className={ diff --git a/frontend/app/components/Session_/BugReport/components/ReportTitle.tsx b/frontend/app/components/Session_/BugReport/components/ReportTitle.tsx index 93eb4148a..bb6bb327c 100644 --- a/frontend/app/components/Session_/BugReport/components/ReportTitle.tsx +++ b/frontend/app/components/Session_/BugReport/components/ReportTitle.tsx @@ -45,7 +45,7 @@ function ReportTitle() { /> ) : ( // @ts-ignore - +
{ default: 'bg-white hover:bg-gray-light border border-gray-light', primary: 'bg-teal color-white hover:bg-teal-dark', green: 'bg-green color-white hover:bg-green-dark', - text: 'bg-transparent color-gray-dark hover:bg-gray-light-shade hover:!text-teal hover-fill-teal', + text: 'bg-transparent color-gray-dark hover:bg-active-blue hover:!text-teal hover-fill-teal', 'text-primary': 'bg-transparent color-teal hover:bg-teal-light hover:color-teal-dark', 'text-red': 'bg-transparent color-red hover:bg-teal-light', outline: 'bg-white color-teal border border-teal hover:bg-teal-light', From 889fa4a519131c8246a321071f5718d133e2006d Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 29 Mar 2023 17:04:13 +0200 Subject: [PATCH 238/253] change(ui) - alignment and other changes in dashboard --- .../MetricViewHeader/MetricViewHeader.tsx | 14 +++++++------- .../components/MetricsList/MetricsList.tsx | 7 +++++-- frontend/app/svg/ca-no-cards.svg | 16 +++++++++++++++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/frontend/app/components/Dashboard/components/MetricViewHeader/MetricViewHeader.tsx b/frontend/app/components/Dashboard/components/MetricViewHeader/MetricViewHeader.tsx index 1e1775415..719677914 100644 --- a/frontend/app/components/Dashboard/components/MetricViewHeader/MetricViewHeader.tsx +++ b/frontend/app/components/Dashboard/components/MetricViewHeader/MetricViewHeader.tsx @@ -25,13 +25,8 @@ function MetricViewHeader() {

-
- - Create custom Cards to capture key interactions and track KPIs. -
-
- - + +
+
+ +
+ )} @@ -168,7 +169,7 @@ export default class ForgotPassword extends React.PureComponent { {resetting && ( - + {/* */}
@@ -193,6 +195,7 @@ export default class ForgotPassword extends React.PureComponent { onChange={this.write} className="w-full" icon="key" + required /> @@ -226,7 +229,7 @@ export default class ForgotPassword extends React.PureComponent { variant="primary" loading={loading} className="w-full" - disabled={(resetting && this.isSubmitDisabled()) || (!resetting && !validEmail)} + // disabled={(resetting && this.isSubmitDisabled()) || (!resetting && !validEmail)} > {resetting ? 'Create' : 'Email password reset link'} diff --git a/frontend/app/components/Login/Login.js b/frontend/app/components/Login/Login.js index c8683a1ee..e4dc763b3 100644 --- a/frontend/app/components/Login/Login.js +++ b/frontend/app/components/Login/Login.js @@ -114,10 +114,11 @@ class Login extends React.Component { Date: Wed, 29 Mar 2023 17:50:27 +0200 Subject: [PATCH 244/253] fix(ui) - dashboard card select all restrict to 30 --- .../Dashboard/components/DashboardHeader/DashboardHeader.tsx | 2 +- .../components/Dashboard/components/MetricsList/MetricsList.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/Dashboard/components/DashboardHeader/DashboardHeader.tsx b/frontend/app/components/Dashboard/components/DashboardHeader/DashboardHeader.tsx index 0d850038c..072e2bf44 100644 --- a/frontend/app/components/Dashboard/components/DashboardHeader/DashboardHeader.tsx +++ b/frontend/app/components/Dashboard/components/DashboardHeader/DashboardHeader.tsx @@ -19,7 +19,7 @@ interface IProps { } type Props = IProps & RouteComponentProps; -const MAX_CARDS = 30 +const MAX_CARDS = 29; function DashboardHeader(props: Props) { const { siteId, dashboardId } = props; const { dashboardStore } = useStore(); diff --git a/frontend/app/components/Dashboard/components/MetricsList/MetricsList.tsx b/frontend/app/components/Dashboard/components/MetricsList/MetricsList.tsx index 1f618544f..ab412676f 100644 --- a/frontend/app/components/Dashboard/components/MetricsList/MetricsList.tsx +++ b/frontend/app/components/Dashboard/components/MetricsList/MetricsList.tsx @@ -73,7 +73,7 @@ function MetricsList({ toggleSelection={toggleMetricSelection} allSelected={cards.length === selectedMetrics.length} toggleAll={({ target: { checked, name } }) => - setSelectedMetrics(checked ? cards.map((i: any) => i.metricId) : []) + setSelectedMetrics(checked ? cards.map((i: any) => i.metricId).slice(0, 30 - existingCardIds!.length) : []) } /> ) : ( From ea838c171bad2284cdc607e050f657428b15b6ad Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Wed, 29 Mar 2023 17:57:50 +0200 Subject: [PATCH 245/253] chore(cli): Adding more log info for editing Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay-cli | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli index 140c80ac3..d518cc186 100755 --- a/scripts/helmcharts/openreplay-cli +++ b/scripts/helmcharts/openreplay-cli @@ -370,7 +370,14 @@ do ;; -e | --edit) log title "Editing OpenReplay" - sudo vim -n /var/lib/openreplay/vars.yaml + [[ -f ${OR_DIR}/vars.yaml ]] || { + log err " + Couldn't open ${BWHITE}${OR_DIR}/vars.yaml${RED}. Seems like a custom installation. + Edit the proper ${BWHITE}vars.yaml${RED} and run ${BWHITE}openreplay -R${RED} + Or ${BWHITE}helm upgrade openreplay -n app openreplay/scripts/helmcharts/openreplay -f openreplay/scripts/helmcharts/vars.yaml --debug --atomic" + exit 100 + } + sudo vim -n ${OR_DIR}/vars.yaml /var/lib/openreplay/yq 'true' /var/lib/openreplay/vars.yaml || { log debug "seems like the edit is not correct. Rerun ${BWHITE}openreplay -e${YELLOW} after fixing the issue." exit 100 From 725b0d56ed77d9d001d0e3c726af857f29a54d23 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Wed, 29 Mar 2023 18:25:56 +0200 Subject: [PATCH 246/253] change(tracker): 6.0.0 --- frontend/app/player/web/MessageManager.ts | 2 +- frontend/app/player/web/messages/RawMessageReader.gen.ts | 5 +++-- ...ntend~app~player~web~messages~RawMessageReader.gen.ts.erb | 5 +++-- tracker/tracker/CHANGELOG.md | 2 +- tracker/tracker/package.json | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index d37ac6fa5..38a28a0aa 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -221,7 +221,7 @@ export default class MessageManager { fileReader.append(b) const msgs: Array = [] for (let msg = fileReader.readNext();msg !== null;msg = fileReader.readNext()) { - msgs.push(msg) + msg && msgs.push(msg) } const sorted = msgs.sort((m1, m2) => { // @ts-ignore diff --git a/frontend/app/player/web/messages/RawMessageReader.gen.ts b/frontend/app/player/web/messages/RawMessageReader.gen.ts index 135960cef..0ae41e9a4 100644 --- a/frontend/app/player/web/messages/RawMessageReader.gen.ts +++ b/frontend/app/player/web/messages/RawMessageReader.gen.ts @@ -800,8 +800,9 @@ export default class RawMessageReader extends PrimitiveReader { } default: - throw new Error(`Unrecognizable message type: ${ tp }; Pointer at the position ${this.p} of ${this.buf.length}`) - return null; + console.error(`Unrecognizable message type: ${ tp }; Pointer at the position ${this.p} of ${this.buf.length}`) + // skipping unrecognized messages + return false; } } } diff --git a/mobs/templates/frontend~app~player~web~messages~RawMessageReader.gen.ts.erb b/mobs/templates/frontend~app~player~web~messages~RawMessageReader.gen.ts.erb index 3c9268366..4b59b9d59 100644 --- a/mobs/templates/frontend~app~player~web~messages~RawMessageReader.gen.ts.erb +++ b/mobs/templates/frontend~app~player~web~messages~RawMessageReader.gen.ts.erb @@ -30,8 +30,9 @@ export default class RawMessageReader extends PrimitiveReader { } <% end %> default: - throw new Error(`Unrecognizable message type: ${ tp }; Pointer at the position ${this.p} of ${this.buf.length}`) - return null; + console.error(`Unrecognizable message type: ${ tp }; Pointer at the position ${this.p} of ${this.buf.length}`) + // skipping unrecognized messages + return false; } } } diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index e0ba20c30..a9f91b5c5 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,4 +1,4 @@ -# 5.0.2 +# 6.0.0 - Capture mouse thrashing, input hesitation+duration, click hesitation - Capture DOM node drop event (>30% nodes removed) diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 7ac8ba10f..a5f2b4ab4 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "5.0.2", + "version": "6.0.0", "keywords": [ "logging", "replay" From ed21439a059cfcf37654174b087e6636caf41d87 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 30 Mar 2023 11:01:54 +0100 Subject: [PATCH 247/253] feat(DB): changed deltas --- .../schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql | 12 ++++++------ .../schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql index a8b7a297b..6fd5c9dc7 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql @@ -25,16 +25,16 @@ $$ LANGUAGE sql IMMUTABLE; ALTER TYPE issue_type ADD VALUE IF NOT EXISTS 'mouse_thrashing'; LOCK TABLE ONLY events.inputs IN ACCESS EXCLUSIVE MODE; -ALTER TABLE events.inputs - ADD COLUMN duration integer NULL, - ADD COLUMN hesitation integer NULL; +ALTER TABLE IF EXISTS events.inputs + ADD COLUMN IF NOT EXISTS duration integer NULL, + ADD COLUMN IF NOT EXISTS hesitation integer NULL; LOCK TABLE ONLY events.clicks IN ACCESS EXCLUSIVE MODE; -ALTER TABLE events.clicks - ADD COLUMN hesitation integer NULL; +ALTER TABLE IF EXISTS events.clicks + ADD COLUMN IF NOT EXISTS hesitation integer NULL; LOCK TABLE ONLY public.projects IN ACCESS EXCLUSIVE MODE; -ALTER TABLE public.projects +ALTER TABLE IF EXISTS public.projects ALTER COLUMN gdpr SET DEFAULT '{ "maskEmails": true, "sampleRate": 33, diff --git a/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql b/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql index d0bfd18e2..cdab6f1ad 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.11.0/1.11.0.sql @@ -25,16 +25,16 @@ $$ LANGUAGE sql IMMUTABLE; ALTER TYPE issue_type ADD VALUE IF NOT EXISTS 'mouse_thrashing'; LOCK TABLE ONLY events.inputs IN ACCESS EXCLUSIVE MODE; -ALTER TABLE events.inputs - ADD COLUMN duration integer NULL, - ADD COLUMN hesitation integer NULL; +ALTER TABLE IF EXISTS events.inputs + ADD COLUMN IF NOT EXISTS duration integer NULL, + ADD COLUMN IF NOT EXISTS hesitation integer NULL; LOCK TABLE ONLY events.clicks IN ACCESS EXCLUSIVE MODE; -ALTER TABLE events.clicks - ADD COLUMN hesitation integer NULL; +ALTER TABLE IF EXISTS events.clicks + ADD COLUMN IF NOT EXISTS hesitation integer NULL; LOCK TABLE ONLY public.projects IN ACCESS EXCLUSIVE MODE; -ALTER TABLE public.projects +ALTER TABLE IF EXISTS public.projects ALTER COLUMN gdpr SET DEFAULT '{ "maskEmails": true, "sampleRate": 33, From 6dd22030d8268fffdb57dd55663377879b5e4299 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 30 Mar 2023 12:28:40 +0200 Subject: [PATCH 248/253] change(tracker): update changelog for 5 0 2 --- tracker/tracker/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index a9f91b5c5..a021c6ae6 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -6,6 +6,9 @@ - Detect cached requests to img, css and js resources; send transferred size - added `{ mouse: { disableClickmaps: boolean } }` to disable calculating el. selectors - added `{ mouse: { minSelectorDepth?: number; nthThreshold?: number; maxOptimiseTries?: number }` for selector finding optimisations + +## 5.0.2 + - fixed inline css loading in specific cases when assets gets around min flush size ## 5.0.1 From fdbc3e646f9bb0a79281c950161c398374a74869 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 30 Mar 2023 12:58:39 +0200 Subject: [PATCH 249/253] fix(ui): loader for player, remove thrashing tag from session list header --- .../Session/Player/ReplayPlayer/PlayerBlockHeader.tsx | 3 ++- frontend/app/components/Session/WebPlayer.tsx | 8 ++++---- .../shared/SessionListContainer/SessionListContainer.tsx | 6 ------ .../components/SessionTags/SessionTags.tsx | 2 +- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/frontend/app/components/Session/Player/ReplayPlayer/PlayerBlockHeader.tsx b/frontend/app/components/Session/Player/ReplayPlayer/PlayerBlockHeader.tsx index 162f92237..7fc8eabe3 100644 --- a/frontend/app/components/Session/Player/ReplayPlayer/PlayerBlockHeader.tsx +++ b/frontend/app/components/Session/Player/ReplayPlayer/PlayerBlockHeader.tsx @@ -24,7 +24,8 @@ function PlayerBlockHeader(props: any) { const [hideBack, setHideBack] = React.useState(false); const { player, store } = React.useContext(PlayerContext); - const { width, height, showEvents } = store.get(); + const playerState = store?.get?.() || { width: 0, height: 0, showEvents: false } + const { width = 0, height = 0, showEvents = false } = playerState const { session, diff --git a/frontend/app/components/Session/WebPlayer.tsx b/frontend/app/components/Session/WebPlayer.tsx index 53fe57bf8..07fab2877 100644 --- a/frontend/app/components/Session/WebPlayer.tsx +++ b/frontend/app/components/Session/WebPlayer.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from 'react'; import { connect } from 'react-redux'; -import { Modal } from 'UI'; +import { Modal, Loader } from 'UI'; import { toggleFullscreen, closeBottomBlock } from 'Duck/components/player'; import { fetchList } from 'Duck/integrations'; import { createWebPlayer } from 'Player'; @@ -100,7 +100,7 @@ function WebPlayer(props: any) { contextValue.player.play(); }; - if (!contextValue.player || !session) return null; + if (!session) return ; return ( @@ -112,12 +112,12 @@ function WebPlayer(props: any) { fullscreen={fullscreen} /> {/* @ts-ignore */} - + /> : } {showNoteModal ? ( void; members: object[]; clearCurrentSession: () => void; }) { - React.useEffect(() => { - clearCurrentSession() - }, []) return (
@@ -36,5 +31,4 @@ export default connect( // @ts-ignore members: state.getIn(['members', 'list']), }), - { clearCurrentSession } )(SessionListContainer); diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionTags/SessionTags.tsx b/frontend/app/components/shared/SessionListContainer/components/SessionTags/SessionTags.tsx index b8eb23d1e..aa35b48f9 100644 --- a/frontend/app/components/shared/SessionListContainer/components/SessionTags/SessionTags.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/SessionTags/SessionTags.tsx @@ -38,7 +38,7 @@ export default connect( const isEnterprise = state.getIn(['user', 'account', 'edition']) === 'ee'; return { activeTab: state.getIn(['search', 'activeTab']), - tags: issues_types.filter((tag: any) => (isEnterprise ? tag.type !== 'bookmark' : tag.type !== 'vault')), + tags: issues_types.filter((tag: any) => tag.type !== 'mouse_thrashing' && (isEnterprise ? tag.type !== 'bookmark' : tag.type !== 'vault')), total: state.getIn(['sessions', 'total']) || 0, }; }, From c6b97151c9228c6f5e392776af13622fb87d3a6a Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 30 Mar 2023 14:12:09 +0200 Subject: [PATCH 250/253] fix(ui): fix duplicate assist creation on /credentials call --- .../app/components/Session/LivePlayer.tsx | 45 ++++++++++--------- .../SessionListContainer.tsx | 2 - frontend/app/player/create.ts | 2 +- frontend/app/player/web/WebLivePlayer.ts | 2 +- .../app/player/web/assist/AssistManager.ts | 2 +- frontend/app/player/web/assist/Call.ts | 2 +- 6 files changed, 27 insertions(+), 28 deletions(-) diff --git a/frontend/app/components/Session/LivePlayer.tsx b/frontend/app/components/Session/LivePlayer.tsx index cace337e8..f558c3103 100644 --- a/frontend/app/components/Session/LivePlayer.tsx +++ b/frontend/app/components/Session/LivePlayer.tsx @@ -1,7 +1,6 @@ import React from 'react'; import { useEffect, useState } from 'react'; import { connect } from 'react-redux'; -import withRequest from 'HOCs/withRequest'; import withPermissions from 'HOCs/withPermissions'; import { PlayerContext, defaultContextValue, ILivePlayerContext } from './playerContext'; import { makeAutoObservable } from 'mobx'; @@ -11,6 +10,7 @@ import PlayerBlock from './Player/LivePlayer/LivePlayerBlock'; import styles from '../Session_/session.module.css'; import Session from 'App/mstore/types/session'; import withLocationHandlers from 'HOCs/withLocationHandlers'; +import APIClient from 'App/api_client'; interface Props { session: Session; @@ -28,14 +28,12 @@ interface Props { function LivePlayer({ session, loadingCredentials, - assistCredentials, - request, - isEnterprise, userEmail, userName, isMultiview, customSession, - query + query, + isEnterprise }: Props) { // @ts-ignore const [contextValue, setContextValue] = useState(defaultContextValue); @@ -52,13 +50,21 @@ function LivePlayer({ name: userName, }, }; - const [player, store] = createLiveWebPlayer(sessionWithAgentData, assistCredentials, (state) => - makeAutoObservable(state) - ); - setContextValue({ player, store }); - - return () => player.clean(); - }, [session.sessionId, assistCredentials]); + if (isEnterprise) { + new APIClient().get('/config/assist/credentials').then(r => r.json()) + .then(({ data }) => { + const [player, store] = createLiveWebPlayer(sessionWithAgentData, data, (state) => + makeAutoObservable(state) + ); + setContextValue({ player, store }); + }) + } else { + const [player, store] = createLiveWebPlayer(sessionWithAgentData, null, (state) => + makeAutoObservable(state) + ); + setContextValue({ player, store }); + } + }, [session.sessionId]); // LAYOUT (TODO: local layout state - useContext or something..) useEffect(() => { @@ -70,8 +76,10 @@ function LivePlayer({ setFullView(true); } - if (isEnterprise) { - request(); + return () => { + contextValue.player?.clean?.(); + // @ts-ignore default empty + setContextValue(defaultContextValue) } }, []); @@ -98,13 +106,7 @@ function LivePlayer({ ); } -export default withRequest({ - initialData: null, - endpoint: '/assist/credentials', - dataName: 'assistCredentials', - loadingName: 'loadingCredentials', -})( - withPermissions( +export default withPermissions( ['ASSIST_LIVE'], '', true @@ -121,4 +123,3 @@ export default withRequest({ } )(withLocationHandlers()(LivePlayer)) ) -); diff --git a/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx b/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx index 1f02679b1..bd340e4b0 100644 --- a/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx +++ b/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx @@ -12,9 +12,7 @@ function SessionListContainer({ clearCurrentSession, }: { activeTab: string; - fetchMembers: () => void; members: object[]; - clearCurrentSession: () => void; }) { React.useEffect(() => { clearCurrentSession() diff --git a/frontend/app/player/create.ts b/frontend/app/player/create.ts index 016b8eb52..2d46c5b0e 100644 --- a/frontend/app/player/create.ts +++ b/frontend/app/player/create.ts @@ -39,7 +39,7 @@ export function createClickMapPlayer(session: Record, wrapStore?: ( return [player, store] } -export function createLiveWebPlayer(session: Record, config: RTCIceServer[], wrapStore?: (s:IWebLivePlayerStore) => IWebLivePlayerStore): [IWebLivePlayer, IWebLivePlayerStore] { +export function createLiveWebPlayer(session: Record, config: RTCIceServer[] | null, wrapStore?: (s:IWebLivePlayerStore) => IWebLivePlayerStore): [IWebLivePlayer, IWebLivePlayerStore] { let store: WebLivePlayerStore = new SimpleStore({ ...WebLivePlayer.INITIAL_STATE, }) diff --git a/frontend/app/player/web/WebLivePlayer.ts b/frontend/app/player/web/WebLivePlayer.ts index 7ed1e3400..4d02fa1f8 100644 --- a/frontend/app/player/web/WebLivePlayer.ts +++ b/frontend/app/player/web/WebLivePlayer.ts @@ -23,7 +23,7 @@ export default class WebLivePlayer extends WebPlayer { private lastMessageInFileTime = 0 private lastMessageInFileIndex = 0 - constructor(wpState: Store, private session:any, config: RTCIceServer[]) { + constructor(wpState: Store, private session:any, config: RTCIceServer[] | null) { super(wpState, session, true) this.assistManager = new AssistManager( diff --git a/frontend/app/player/web/assist/AssistManager.ts b/frontend/app/player/web/assist/AssistManager.ts index e75db1388..413de0857 100644 --- a/frontend/app/player/web/assist/AssistManager.ts +++ b/frontend/app/player/web/assist/AssistManager.ts @@ -66,7 +66,7 @@ export default class AssistManager { private setMessagesLoading: (flag: boolean) => void, private handleMessage: (m: Message, index: number) => void, private screen: Screen, - private config: RTCIceServer[], + private config: RTCIceServer[] | null, private store: Store, ) {} diff --git a/frontend/app/player/web/assist/Call.ts b/frontend/app/player/web/assist/Call.ts index 2119b9935..2cb136243 100644 --- a/frontend/app/player/web/assist/Call.ts +++ b/frontend/app/player/web/assist/Call.ts @@ -33,7 +33,7 @@ export default class Call { constructor( private store: Store, private socket: Socket, - private config: RTCIceServer[], + private config: RTCIceServer[] | null, private peerID: string, ) { socket.on('call_end', this.onRemoteCallEnd) From 5435f536fce539fa6ad85e1115b3081367a619da Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 30 Mar 2023 14:23:35 +0200 Subject: [PATCH 251/253] fix(ui): fix ui issue --- .../shared/SessionListContainer/SessionListContainer.tsx | 5 ----- 1 file changed, 5 deletions(-) diff --git a/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx b/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx index bd340e4b0..f5144a9a6 100644 --- a/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx +++ b/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx @@ -4,19 +4,14 @@ import SessionHeader from './components/SessionHeader'; import NotesList from './components/Notes/NoteList'; import { connect } from 'react-redux'; import LatestSessionsMessage from './components/LatestSessionsMessage'; -import { clearCurrentSession } from "Duck/sessions"; function SessionListContainer({ activeTab, members, - clearCurrentSession, }: { activeTab: string; members: object[]; }) { - React.useEffect(() => { - clearCurrentSession() - }, []) return (
From 1f3344275638e919e9d9eb0a9d34c97ab2a6ef4c Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Thu, 30 Mar 2023 12:36:54 +0200 Subject: [PATCH 252/253] change(ui) - no session message placement --- frontend/app/components/Overview/Overview.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/app/components/Overview/Overview.tsx b/frontend/app/components/Overview/Overview.tsx index 775a8d0ae..1ef003f7a 100644 --- a/frontend/app/components/Overview/Overview.tsx +++ b/frontend/app/components/Overview/Overview.tsx @@ -16,9 +16,8 @@ function Overview() {
- -
+ From 60134bfe7acce8281a26e4d037499f5d47b3ff0d Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Thu, 30 Mar 2023 16:13:08 +0200 Subject: [PATCH 253/253] fix(ui) - url search page reset --- .../SessionSearchQueryParamHandler.tsx | 2 +- frontend/app/duck/search.js | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx b/frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx index 5bd5d739d..e38729b2a 100644 --- a/frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx +++ b/frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx @@ -17,7 +17,7 @@ const SessionSearchQueryParamHandler = (props: Props) => { const applyFilterFromQuery = () => { const filter = getFiltersFromQuery(history.location.search, appliedFilter); - props.updateFilter(filter, true); + props.updateFilter(filter, true, false); }; const generateUrlQuery = () => { diff --git a/frontend/app/duck/search.js b/frontend/app/duck/search.js index 31f028bc3..1f68d1994 100644 --- a/frontend/app/duck/search.js +++ b/frontend/app/duck/search.js @@ -71,8 +71,11 @@ function reducer(state = initialState, action = {}) { case EDIT: return state.mergeIn(['instance'], action.instance).set('currentPage', 1); case APPLY: - - return action.fromUrl ? state.set('instance', Filter(action.filter)) : state.mergeIn(['instance'], action.filter).set('currentPage', 1); + state.mergeIn(['instance'], action.filter).set('currentPage', 1); + if (action.resetPage) { + state.set('currentPage', 1) + } + return state case success(FETCH): return state.set('instance', action.data); case success(FETCH_LIST): @@ -243,10 +246,11 @@ export const applyFilter = reduceThenFetchResource((filter, force = false) => ({ force, })); -export const updateFilter = (filter, force = false) => ({ +export const updateFilter = (filter, force = false, resetPage = true) => ({ type: APPLY, filter, force, + resetPage, }); export const updateCurrentPage = reduceThenFetchResource((page) => ({