From 130775a968efc528056cbba8b84192fb4d25ee63 Mon Sep 17 00:00:00 2001 From: Eric Chan Date: Wed, 13 Jul 2022 11:41:43 -0400 Subject: [PATCH 01/29] avoid accessing localStorage and sessionStorage before override --- tracker/tracker/src/main/app/index.ts | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index be03a968c..81b99aae6 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -23,8 +23,8 @@ export interface StartOptions { } interface OnStartInfo { - sessionID: string, - sessionToken: string, + sessionID: string, + sessionToken: string, userUUID: string, } const CANCELED = "canceled" as const @@ -60,8 +60,8 @@ type AppOptions = { __is_snippet: boolean; __debug_report_edp: string | null; __debug__?: LoggerOptions; - localStorage: Storage; - sessionStorage: Storage; + localStorage: Storage | null; + sessionStorage: Storage | null; // @deprecated onStart?: StartCallback; @@ -117,8 +117,8 @@ export default class App { verbose: false, __is_snippet: false, __debug_report_edp: null, - localStorage: window.localStorage, - sessionStorage: window.sessionStorage, + localStorage: null, + sessionStorage: null, }, options, ); @@ -140,8 +140,8 @@ export default class App { Object.entries(metadata).forEach(([key, value]) => this.send(new Metadata(key, value))) } }) - this.localStorage = this.options.localStorage; - this.sessionStorage = this.options.sessionStorage; + this.localStorage = this.options.localStorage ?? window.localStorage; + this.sessionStorage = this.options.sessionStorage ?? window.sessionStorage; if (sessionToken != null) { this.sessionStorage.setItem(this.options.session_token_key, sessionToken); @@ -175,7 +175,7 @@ export default class App { this.attachEventListener(document.body, 'mouseleave', alertWorker, false, false); // TODO: stop session after inactivity timeout (make configurable) this.attachEventListener(document, 'visibilitychange', alertWorker, false); - } catch (e) { + } catch (e) { this._debug("worker_start", e); } } @@ -197,9 +197,9 @@ export default class App { send(message: Message, urgent = false): void { if (this.activityState === ActivityState.NotActive) { return } this.messages.push(message); - // TODO: commit on start if there were `urgent` sends; + // TODO: commit on start if there were `urgent` sends; // Clearify where urgent can be used for; - // Clearify workflow for each type of message in case it was sent before start + // Clearify workflow for each type of message in case it was sent before start // (like Fetch before start; maybe add an option "preCapture: boolean" or sth alike) if (this.activityState === ActivityState.Active && urgent) { this.commit(); @@ -339,8 +339,8 @@ export default class App { if (!this.worker) { return Promise.resolve(UnsuccessfulStart("No worker found: perhaps, CSP is not set.")) } - if (this.activityState !== ActivityState.NotActive) { - return Promise.resolve(UnsuccessfulStart("OpenReplay: trying to call `start()` on the instance that has been started already.")) + if (this.activityState !== ActivityState.NotActive) { + return Promise.resolve(UnsuccessfulStart("OpenReplay: trying to call `start()` on the instance that has been started already.")) } this.activityState = ActivityState.Starting; @@ -364,7 +364,7 @@ export default class App { this.worker.postMessage(startWorkerMsg) this.session.update({ // TODO: transparent "session" module logic AND explicit internal api for plugins. - // "updating" with old metadata in order to trigger session's UpdateCallbacks. + // "updating" with old metadata in order to trigger session's UpdateCallbacks. // (for the case of internal .start() calls, like on "restart" webworker signal or assistent connection in tracker-assist ) metadata: startOpts.metadata || this.session.getInfo().metadata, userID: startOpts.userID, @@ -391,7 +391,7 @@ export default class App { if (r.status === 200) { return r.json() } else { - return r.text().then(text => text === CANCELED + return r.text().then(text => text === CANCELED ? Promise.reject(CANCELED) : Promise.reject(`Server error: ${r.status}. ${text}`) ); @@ -418,7 +418,7 @@ export default class App { this.worker.postMessage(startWorkerMsg) this.activityState = ActivityState.Active - + const onStartInfo = { sessionToken: token, userUUID, sessionID }; this.startCallbacks.forEach((cb) => cb(onStartInfo)); // TODO: start as early as possible (before receiving the token) @@ -432,7 +432,7 @@ export default class App { } return SuccessfulStart(onStartInfo) }) - .catch(reason => { + .catch(reason => { this.sessionStorage.removeItem(this.options.session_token_key) this.stop() if (reason === CANCELED) { return UnsuccessfulStart(CANCELED) } From 690df1772623c4c86649d19bc752f06070d73911 Mon Sep 17 00:00:00 2001 From: Eric Chan Date: Fri, 22 Jul 2022 11:13:13 -0400 Subject: [PATCH 02/29] add comment about #490, #637 --- tracker/tracker/src/main/app/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 81b99aae6..f7b2ddb0b 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -140,6 +140,8 @@ export default class App { Object.entries(metadata).forEach(([key, value]) => this.send(new Metadata(key, value))) } }) + + // window.localStorage and window.sessionStorage should only be accessed if required, see #490, #637 this.localStorage = this.options.localStorage ?? window.localStorage; this.sessionStorage = this.options.sessionStorage ?? window.sessionStorage; From 41797730d3cbc4914d7dff7dfeaa70a0c394d85f Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 29 Jun 2022 10:44:14 +0200 Subject: [PATCH 03/29] fix(frontend/player):add Select to set_input_value --- .../app/player/MessageDistributor/managers/DOMManager.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOMManager.ts index 43c7a274c..37d24aa44 100644 --- a/frontend/app/player/MessageDistributor/managers/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOMManager.ts @@ -209,9 +209,12 @@ export default class DOMManager extends ListWalker { case "set_input_value": node = this.nl[ msg.id ] if (!node) { logger.error("Node not found", msg); return } - if (!(node instanceof HTMLInputElement || node instanceof HTMLTextAreaElement)) { - logger.error("Trying to set value of non-Input element", msg) - return + if (!(node instanceof HTMLInputElement + || node instanceof HTMLTextAreaElement + || node instanceof HTMLSelectElement) + ) { + logger.error("Trying to set value of non-Input element", msg) + return } const val = msg.mask > 0 ? '*'.repeat(msg.mask) : msg.value doc = this.screen.document From 8b00543d8bfa0deb7083967963478d9409b2d37f Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Tue, 5 Jul 2022 18:33:52 +0200 Subject: [PATCH 04/29] feat(frontend/player): virtualisation of the DOM operations --- .../managers/DOM/DOMManager.ts | 305 ++++++++++++++++ .../managers/{ => DOM}/StylesManager.ts | 12 +- .../managers/DOM/VirtualDOM.ts | 143 ++++++++ .../MessageDistributor/managers/DOMManager.ts | 325 ------------------ .../managers/PagesManager.ts | 2 +- 5 files changed, 455 insertions(+), 332 deletions(-) create mode 100644 frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts rename frontend/app/player/MessageDistributor/managers/{ => DOM}/StylesManager.ts (90%) create mode 100644 frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts delete mode 100644 frontend/app/player/MessageDistributor/managers/DOMManager.ts diff --git a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts new file mode 100644 index 000000000..278e97ba4 --- /dev/null +++ b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts @@ -0,0 +1,305 @@ +import logger from 'App/logger'; + +import type StatedScreen from '../../StatedScreen'; +import type { Message, SetNodeScroll, CreateElementNode } from '../../messages'; + +import ListWalker from '../ListWalker'; +import StylesManager, { rewriteNodeStyleSheet } from './StylesManager'; +import { VElement, VText, VFragment, VDocument, VNode, VStyleElement } from './VirtualDOM'; +import type { StyleElement } from './VirtualDOM'; + + +type HTMLElementWithValue = HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement + +const IGNORED_ATTRS = [ "autocomplete", "name" ]; +const ATTR_NAME_REGEXP = /([^\t\n\f \/>"'=]+)/; // regexp costs ~ + +export default class DOMManager extends ListWalker { + private isMobile: boolean; + private screen: StatedScreen; + private vTexts: Map = new Map() // map vs object here? + private vElements: Map = new Map() + private vRoots: Map = new Map() + + + private upperBodyId: number = -1; + private nodeScrollManagers: Array> = [] + private stylesManager: StylesManager + + + constructor(screen: StatedScreen, isMobile: boolean, public readonly time: number) { + super(); + this.isMobile = isMobile; + this.screen = screen; + this.stylesManager = new StylesManager(screen); + } + + append(m: Message): void { + switch (m.tp) { + case "set_node_scroll": + if (!this.nodeScrollManagers[ m.id ]) { + this.nodeScrollManagers[ m.id ] = new ListWalker(); + } + this.nodeScrollManagers[ m.id ].append(m); + return; + default: + if (m.tp === "create_element_node") { + if(m.tag === "BODY" && this.upperBodyId === -1) { + this.upperBodyId = m.id + } + } else if (m.tp === "set_node_attribute" && + (IGNORED_ATTRS.includes(m.name) || !ATTR_NAME_REGEXP.test(m.name))) { + logger.log("Ignorring message: ", m) + return; // Ignoring + } + super.append(m); + } + } + + private removeBodyScroll(id: number, vn: VElement): void { + if (this.isMobile && this.upperBodyId === id) { // Need more type safety! + (vn.node as HTMLBodyElement).style.overflow = "hidden" + } + } + + // May be make it as a message on message add? + private removeAutocomplete(node: Element): boolean { + const tag = node.tagName + if ([ "FORM", "TEXTAREA", "SELECT" ].includes(tag)) { + node.setAttribute("autocomplete", "off"); + return true; + } + if (tag === "INPUT") { + node.setAttribute("autocomplete", "new-password"); + return true; + } + return false; + } + + private insertNode({ parentID, id, index }: { parentID: number, id: number, index: number }): void { + const child = this.vElements.get(id) || this.vTexts.get(id) + if (!child) { + logger.error("Insert error. Node not found", id); + return; + } + const parent = this.vElements.get(parentID) || this.vRoots.get(parentID) + if (!parent) { + logger.error("Insert error. Parent node not found", parentID); + return; + } + + const pNode = parent.node + if ((pNode instanceof HTMLStyleElement) && // TODO: correct ordering OR filter in tracker + pNode.sheet && + pNode.sheet.cssRules && + pNode.sheet.cssRules.length > 0 && + pNode.innerText.trim().length === 0 + ) { + logger.log("Trying to insert child to a style tag with virtual rules: ", parent, child); + return; + } + + parent.insertChildAt(child, index) + } + + private applyMessage = (msg: Message): void => { + let node: Node | undefined + let vn: VNode | undefined + let doc: Document | null + switch (msg.tp) { + case "create_document": + doc = this.screen.document; + if (!doc) { + logger.error("No iframe document found", msg) + return; + } + doc.open(); + doc.write(""); + doc.close(); + const fRoot = doc.documentElement; + fRoot.innerText = ''; + + vn = new VElement(fRoot) + this.vElements = new Map([[0, vn ]]) + this.stylesManager.reset(); + return + case "create_text_node": + vn = new VText() + this.vTexts.set(msg.id, vn) + this.insertNode(msg) + return + case "create_element_node": + let element: Element + if (msg.svg) { + element = document.createElementNS('http://www.w3.org/2000/svg', msg.tag) + } else { + element = document.createElement(msg.tag) + } + if (msg.tag === "STYLE" || msg.tag === "style") { + vn = new VStyleElement(element as StyleElement) + } else { + vn = new VElement(element) + } + this.vElements.set(msg.id, vn) + this.insertNode(msg) + this.removeBodyScroll(msg.id, vn) + this.removeAutocomplete(element) + return + case "move_node": + this.insertNode(msg); + return + case "remove_node": + vn = this.vElements.get(msg.id) || this.vTexts.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + if (!vn.parentNode) { logger.error("Parent node not found", msg); return } + vn.parentNode.removeChild(vn) + return + case "set_node_attribute": + let { name, value } = msg; + vn = this.vElements.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + if (name === "href" && vn.node.tagName === "LINK") { + // @ts-ignore TODO: global ENV type // Hack for queries in rewrited urls (don't we do that in backend?) + if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { + value = value.replace("?", "%3F"); + } + this.stylesManager.setStyleHandlers(vn.node as HTMLLinkElement, value); + } + if (vn.node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) { + value = "url(#" + (value.split("#")[1] ||")") + } + vn.setAttribute(name, value) + this.removeBodyScroll(msg.id, vn) + return + case "remove_node_attribute": + vn = this.vElements.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + vn.removeAttribute(msg.name) + return + case "set_input_value": + vn = this.vElements.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + const nodeWithValue = vn.node + if (!(nodeWithValue instanceof HTMLInputElement + || nodeWithValue instanceof HTMLTextAreaElement + || nodeWithValue instanceof HTMLSelectElement) + ) { + logger.error("Trying to set value of non-Input element", msg) + return + } + const val = msg.mask > 0 ? '*'.repeat(msg.mask) : msg.value + doc = this.screen.document + if (doc && nodeWithValue === doc.activeElement) { + // For the case of Remote Control + nodeWithValue.onblur = () => { nodeWithValue.value = val } + return + } + nodeWithValue.value = val + return + case "set_input_checked": + vn = this.vElements.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + (vn.node as HTMLInputElement).checked = msg.checked + return + case "set_node_data": + case "set_css_data": // TODO: remove css transitions when timeflow is not natural (on jumps) + vn = this.vTexts.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + vn.setData(msg.data) + if (vn.node instanceof HTMLStyleElement) { + doc = this.screen.document + // TODO: move to message parsing + doc && rewriteNodeStyleSheet(doc, vn.node) + } + return + case "css_insert_rule": + vn = this.vElements.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + if (!(vn instanceof VStyleElement)) { + logger.warn("Non-style node in CSS rules message (or sheet is null)", msg, node.sheet); + return + } + vn.onStyleSheet(sheet => { + try { + sheet.insertRule(msg.rule, msg.index) + } catch (e) { + logger.warn(e, msg) + try { + sheet.insertRule(msg.rule) + } catch (e) { + logger.warn("Cannot insert rule.", e, msg) + } + } + }) + return + case "css_delete_rule": + vn = this.vElements.get(msg.id) + if (!vn) { logger.error("Node not found", msg); return } + if (!(vn instanceof VStyleElement)) { + logger.warn("Non-style node in CSS rules message (or sheet is null)", msg, node.sheet); + return + } + vn.onStyleSheet(sheet => { + try { + sheet.deleteRule(msg.index) + } catch (e) { + logger.warn(e, msg) + } + }) + return + case "create_i_frame_document": + vn = this.vElements.get(msg.frameID) + if (!vn) { logger.error("Node not found", msg); return } + const host = vn.node + if (host instanceof HTMLIFrameElement) { + const vDoc = new VDocument() + this.vRoots.set(msg.id, vDoc) + host.onload = () => { + const doc = host.contentDocument + if (!doc) { + logger.warn("No iframe doc onload", msg, host) + return + } + vDoc.setDocument(doc) + vDoc.applyChanges() + } + return; + } else if (host instanceof Element) { // shadow DOM + try { + const shadowRoot = host.attachShadow({ mode: 'open' }) + vn = new VFragment(shadowRoot) + this.vRoots.set(msg.id, vn) + } catch(e) { + logger.warn("Can not attach shadow dom", e, msg) + } + } else { + logger.warn("Context message host is not Element", msg) + } + return + } + } + + moveReady(t: number): Promise { + this.moveApply(t, this.applyMessage) // This function autoresets pointer if necessary (better name?) + + // @ts-ignore + this.vElements.get(0).applyChanges() + this.vRoots.forEach(rt => rt.applyChanges()) + + // Thinkabout (read): css preload + // What if we go back before it is ready? We'll have two handlres? + return this.stylesManager.moveReady(t).then(() => { + // Apply all scrolls after the styles got applied + this.nodeScrollManagers.forEach(manager => { + const msg = manager.moveGetLast(t) + if (msg) { + const vElm = this.vElements.get(msg.id) + if (vElm) { + vElm.node.scrollLeft = msg.x + vElm.node.scrollTop = msg.y + } + } + }) + }) + } +} \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/managers/StylesManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts similarity index 90% rename from frontend/app/player/MessageDistributor/managers/StylesManager.ts rename to frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts index 3f5ee1b86..139ebd95c 100644 --- a/frontend/app/player/MessageDistributor/managers/StylesManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts @@ -1,10 +1,10 @@ -import type StatedScreen from '../StatedScreen'; -import type { CssInsertRule, CssDeleteRule } from '../messages'; +import type StatedScreen from '../../StatedScreen'; +import type { CssInsertRule, CssDeleteRule } from '../../messages'; type CSSRuleMessage = CssInsertRule | CssDeleteRule; import logger from 'App/logger'; -import ListWalker from './ListWalker'; +import ListWalker from '../ListWalker'; const HOVER_CN = "-openreplay-hover"; @@ -40,7 +40,7 @@ export default class StylesManager extends ListWalker { } setStyleHandlers(node: HTMLLinkElement, value: string): void { - let timeoutId; + let timeoutId: ReturnType | undefined; const promise = new Promise((resolve) => { if (this.skipCSSLinks.includes(value)) resolve(null); this.linkLoadingCount++; @@ -49,8 +49,8 @@ export default class StylesManager extends ListWalker { this.skipCSSLinks.push(value); // watch out resolve(null); } - timeoutId = setTimeout(addSkipAndResolve, 4000); - + timeoutId = setTimeout(addSkipAndResolve, 4000000); + console.log(node.getAttribute("href")) node.onload = () => { const doc = this.screen.document; doc && rewriteNodeStyleSheet(doc, node); diff --git a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts new file mode 100644 index 000000000..3df1b5a3e --- /dev/null +++ b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts @@ -0,0 +1,143 @@ +type VChild = VElement | VText + +export type VNode = VDocument | VFragment | VElement | VText + +abstract class VParent { + abstract node: Node | null + protected children: VChild[] = [] + insertChildAt(child: VChild, index: number) { + if (child.parentNode) { + child.parentNode.removeChild(child) + } + this.children.splice(index, 0, child) + child.parentNode = this + } + + removeChild(child: VChild) { + this.children = this.children.filter(ch => ch !== child) + child.parentNode = null + } + applyChanges() { + const node = this.node + if (!node) { + // log err + console.error("No node found", this) + return + } + const realChildren = node.childNodes + for (let i = 0; i < this.children.length; i++) { + const ch = this.children[i] + ch.applyChanges() + if (ch.node.parentNode !== node) { + const nextSibling = realChildren[i] + node.insertBefore(ch.node, nextSibling || null) + } + if (realChildren[i] !== ch.node) { + node.removeChild(realChildren[i]) + } + } + } +} + +export class VDocument extends VParent { + constructor(public node: Document | null = null) { super() } + setDocument(doc: Document) { + this.node = doc + } + applyChanges() { + if (this.children.length > 1) { + // log err + } + if (!this.node) { + // iframe not mounted yet + return + } + const htmlNode = this.children[0].node + if (htmlNode.parentNode !== this.node) { + this.node.replaceChild(htmlNode, this.node.documentElement) + } + } +} + +export class VFragment extends VParent { + constructor(public readonly node: DocumentFragment) { super() } +} + +export class VElement extends VParent { + parentNode: VParent | null = null + private newAttributes: Record = {} + //private props: Record + constructor(public readonly node: Element) { super() } + setAttribute(name: string, value: string) { + this.newAttributes[name] = value + } + removeAttribute(name: string) { + this.newAttributes[name] = false + } + + applyChanges() { + Object.entries(this.newAttributes).forEach(([key, value]) => { + if (value === false) { + this.node.removeAttribute(key) + } else { + try { + this.node.setAttribute(key, value) + } catch { + // log err + } + } + }) + this.newAttributes = {} + super.applyChanges() + } +} + + +type StyleSheetCallback = (s: CSSStyleSheet) => void +export type StyleElement = HTMLStyleElement | SVGStyleElement +export class VStyleElement extends VElement { + private loaded = false + private stylesheetCallbacks: StyleSheetCallback[] = [] + constructor(public readonly node: StyleElement) { + super(node) // Is it compiled correctly or with 2 node assignments? + node.onload = () => { + const sheet = node.sheet + if (sheet) { + this.stylesheetCallbacks.forEach(cb => cb(sheet)) + } else { + console.warn("Style onload: sheet is null") + } + this.loaded = true + } + } + + onStyleSheet(cb: StyleSheetCallback) { + if (this.loaded) { + if (!this.node.sheet) { + console.warn("Style tag is loaded, but sheet is null") + return + } + cb(this.node.sheet) + } else { + this.stylesheetCallbacks.push(cb) + } + } +} + +export class VText { + parentNode: VParent | null = null + constructor(public readonly node: Text = new Text()) {} + private data: string = "" + private changed: boolean = false + setData(data: string) { + this.data = data + this.changed = true + } + applyChanges() { + if (this.changed) { + this.node.data = this.data + this.changed = false + } + } +} + diff --git a/frontend/app/player/MessageDistributor/managers/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOMManager.ts deleted file mode 100644 index 37d24aa44..000000000 --- a/frontend/app/player/MessageDistributor/managers/DOMManager.ts +++ /dev/null @@ -1,325 +0,0 @@ -import type StatedScreen from '../StatedScreen'; -import type { Message, SetNodeScroll, CreateElementNode } from '../messages'; - -import logger from 'App/logger'; -import StylesManager, { rewriteNodeStyleSheet } from './StylesManager'; -import ListWalker from './ListWalker'; - -const IGNORED_ATTRS = [ "autocomplete", "name" ]; - -const ATTR_NAME_REGEXP = /([^\t\n\f \/>"'=]+)/; // regexp costs ~ - -export default class DOMManager extends ListWalker { - private isMobile: boolean; - private screen: StatedScreen; - private nl: Array = []; - private isLink: Array = []; // Optimisations - private bodyId: number = -1; - private postponedBodyMessage: CreateElementNode | null = null; - private nodeScrollManagers: Array> = []; - - private stylesManager: StylesManager; - - private startTime: number; - - constructor(screen: StatedScreen, isMobile: boolean, startTime: number) { - super(); - this.startTime = startTime; - this.isMobile = isMobile; - this.screen = screen; - this.stylesManager = new StylesManager(screen); - } - - get time(): number { - return this.startTime; - } - - append(m: Message): void { - switch (m.tp) { - case "set_node_scroll": - if (!this.nodeScrollManagers[ m.id ]) { - this.nodeScrollManagers[ m.id ] = new ListWalker(); - } - this.nodeScrollManagers[ m.id ].append(m); - return; - //case "css_insert_rule": // || //set_css_data ??? - //case "css_delete_rule": - // (m.tp === "set_node_attribute" && this.isLink[ m.id ] && m.key === "href")) { - // this.stylesManager.append(m); - // return; - default: - if (m.tp === "create_element_node") { - switch(m.tag) { - case "LINK": - this.isLink[ m.id ] = true; - break; - case "BODY": - this.bodyId = m.id; // Can be several body nodes at one document session? - break; - } - } else if (m.tp === "set_node_attribute" && - (IGNORED_ATTRS.includes(m.name) || !ATTR_NAME_REGEXP.test(m.name))) { - logger.log("Ignorring message: ", m) - return; // Ignoring... - } - super.append(m); - } - - } - - private removeBodyScroll(id: number): void { - if (this.isMobile && this.bodyId === id) { - (this.nl[ id ] as HTMLBodyElement).style.overflow = "hidden"; - } - } - - // May be make it as a message on message add? - private removeAutocomplete({ id, tag }: CreateElementNode): boolean { - const node = this.nl[ id ] as HTMLElement; - if ([ "FORM", "TEXTAREA", "SELECT" ].includes(tag)) { - node.setAttribute("autocomplete", "off"); - return true; - } - if (tag === "INPUT") { - node.setAttribute("autocomplete", "new-password"); - return true; - } - return false; - } - - // type = NodeMessage ? - private insertNode({ parentID, id, index }: { parentID: number, id: number, index: number }): void { - if (!this.nl[ id ]) { - logger.error("Insert error. Node not found", id); - return; - } - if (!this.nl[ parentID ]) { - logger.error("Insert error. Parent node not found", parentID); - return; - } - // WHAT if text info contains some rules and the ordering is just wrong??? - const el = this.nl[ parentID ] - if ((el instanceof HTMLStyleElement) && // TODO: correct ordering OR filter in tracker - el.sheet && - el.sheet.cssRules && - el.sheet.cssRules.length > 0 && - el.innerText.trim().length === 0) { - logger.log("Trying to insert child to a style tag with virtual rules: ", this.nl[ parentID ], this.nl[ id ]); - return; - } - - const childNodes = this.nl[ parentID ].childNodes; - if (!childNodes) { - logger.error("Node has no childNodes", this.nl[ parentID ]); - return; - } - - if (this.nl[ id ] instanceof HTMLHtmlElement) { - // What if some exotic cases? - this.nl[ parentID ].replaceChild(this.nl[ id ], childNodes[childNodes.length-1]) - return - } - - this.nl[ parentID ] - .insertBefore(this.nl[ id ], childNodes[ index ]) - } - - private applyMessage = (msg: Message): void => { - let node; - let doc: Document | null; - switch (msg.tp) { - case "create_document": - doc = this.screen.document; - if (!doc) { - logger.error("No iframe document found", msg) - return; - } - doc.open(); - doc.write(""); - doc.close(); - const fRoot = doc.documentElement; - fRoot.innerText = ''; - this.nl = [ fRoot ]; - - // the last load event I can control - //if (this.document.fonts) { - // this.document.fonts.onloadingerror = () => this.marker.redraw(); - // this.document.fonts.onloadingdone = () => this.marker.redraw(); - //} - - //this.screen.setDisconnected(false); - this.stylesManager.reset(); - return - case "create_text_node": - this.nl[ msg.id ] = document.createTextNode(''); - this.insertNode(msg); - return - case "create_element_node": - if (msg.svg) { - this.nl[ msg.id ] = document.createElementNS('http://www.w3.org/2000/svg', msg.tag); - } else { - this.nl[ msg.id ] = document.createElement(msg.tag); - } - if (this.bodyId === msg.id) { // there are several bodies in iframes TODO: optimise & cache prebuild - this.postponedBodyMessage = msg; - } else { - this.insertNode(msg); - } - this.removeBodyScroll(msg.id); - this.removeAutocomplete(msg); - return - case "move_node": - this.insertNode(msg); - return - case "remove_node": - node = this.nl[ msg.id ] - if (!node) { logger.error("Node not found", msg); return } - if (!node.parentElement) { logger.error("Parent node not found", msg); return } - node.parentElement.removeChild(node); - return - case "set_node_attribute": - let { id, name, value } = msg; - node = this.nl[ id ]; - if (!node) { logger.error("Node not found", msg); return } - if (this.isLink[ id ] && name === "href") { - // @ts-ignore TODO: global ENV type - if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { // Hack for queries in rewrited urls - value = value.replace("?", "%3F"); - } - this.stylesManager.setStyleHandlers(node, value); - } - if (node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) { - value = "url(#" + (value.split("#")[1] ||")") - } - try { - node.setAttribute(name, value); - } catch(e) { - logger.error(e, msg); - } - this.removeBodyScroll(msg.id); - return - case "remove_node_attribute": - if (!this.nl[ msg.id ]) { logger.error("Node not found", msg); return } - try { - (this.nl[ msg.id ] as HTMLElement).removeAttribute(msg.name); - } catch(e) { - logger.error(e, msg); - } - return - case "set_input_value": - node = this.nl[ msg.id ] - if (!node) { logger.error("Node not found", msg); return } - if (!(node instanceof HTMLInputElement - || node instanceof HTMLTextAreaElement - || node instanceof HTMLSelectElement) - ) { - logger.error("Trying to set value of non-Input element", msg) - return - } - const val = msg.mask > 0 ? '*'.repeat(msg.mask) : msg.value - doc = this.screen.document - if (doc && node === doc.activeElement) { - // For the case of Remote Control - node.onblur = () => { node.value = val } - return - } - node.value = val - return - case "set_input_checked": - node = this.nl[ msg.id ]; - if (!node) { logger.error("Node not found", msg); return } - (node as HTMLInputElement).checked = msg.checked; - return - case "set_node_data": - case "set_css_data": - node = this.nl[ msg.id ] - if (!node) { logger.error("Node not found", msg); return } - // @ts-ignore - node.data = msg.data; - if (node instanceof HTMLStyleElement) { - doc = this.screen.document - doc && rewriteNodeStyleSheet(doc, node) - } - return - case "css_insert_rule": - node = this.nl[ msg.id ]; - if (!node) { logger.error("Node not found", msg); return } - if (!(node instanceof HTMLStyleElement) // link or null - || node.sheet == null) { - logger.warn("Non-style node in CSS rules message (or sheet is null)", msg); - return - } - try { - node.sheet.insertRule(msg.rule, msg.index) - } catch (e) { - logger.warn(e, msg) - try { - node.sheet.insertRule(msg.rule) - } catch (e) { - logger.warn("Cannot insert rule.", e, msg) - } - } - return - case "css_delete_rule": - node = this.nl[ msg.id ]; - if (!node) { logger.error("Node not found", msg); return } - if (!(node instanceof HTMLStyleElement) // link or null - || node.sheet == null) { - logger.warn("Non-style node in CSS rules message (or sheet is null)", msg); - return - } - try { - node.sheet.deleteRule(msg.index) - } catch (e) { - logger.warn(e, msg) - } - return - case "create_i_frame_document": - node = this.nl[ msg.frameID ]; - // console.log('ifr', msg, node) - - if (node instanceof HTMLIFrameElement) { - doc = node.contentDocument; - if (!doc) { - logger.warn("No iframe doc", msg, node, node.contentDocument); - return; - } - this.nl[ msg.id ] = doc.documentElement - return; - } else if (node instanceof Element) { // shadow DOM - try { - this.nl[ msg.id ] = node.attachShadow({ mode: 'open' }) - } catch(e) { - logger.warn("Can not attach shadow dom", e, msg) - } - } else { - logger.warn("Context message host is not Element", msg) - } - return - } - } - - moveReady(t: number): Promise { - this.moveApply(t, this.applyMessage) // This function autoresets pointer if necessary (better name?) - - /* Mount body as late as possible */ - if (this.postponedBodyMessage != null) { - this.insertNode(this.postponedBodyMessage) - this.postponedBodyMessage = null - } - - // Thinkabout (read): css preload - // What if we go back before it is ready? We'll have two handlres? - return this.stylesManager.moveReady(t).then(() => { - // Apply all scrolls after the styles got applied - this.nodeScrollManagers.forEach(manager => { - const msg = manager.moveGetLast(t) - if (!!msg && !!this.nl[msg.id]) { - const node = this.nl[msg.id] as HTMLElement - node.scrollLeft = msg.x - node.scrollTop = msg.y - } - }) - }) - } -} \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/managers/PagesManager.ts b/frontend/app/player/MessageDistributor/managers/PagesManager.ts index 0a463fe97..9a4398246 100644 --- a/frontend/app/player/MessageDistributor/managers/PagesManager.ts +++ b/frontend/app/player/MessageDistributor/managers/PagesManager.ts @@ -2,7 +2,7 @@ import type StatedScreen from '../StatedScreen'; import type { Message } from '../messages'; import ListWalker from './ListWalker'; -import DOMManager from './DOMManager'; +import DOMManager from './DOM/DOMManager'; export default class PagesManager extends ListWalker { From e74fb2051b00aea41a59d2f371d42397b69ea6cc Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Tue, 5 Jul 2022 18:43:23 +0200 Subject: [PATCH 05/29] fix(frontend/player): non-dev style timeout --- .../player/MessageDistributor/managers/DOM/StylesManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts index 139ebd95c..14a2ee8a4 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts @@ -49,8 +49,8 @@ export default class StylesManager extends ListWalker { this.skipCSSLinks.push(value); // watch out resolve(null); } - timeoutId = setTimeout(addSkipAndResolve, 4000000); - console.log(node.getAttribute("href")) + timeoutId = setTimeout(addSkipAndResolve, 4000); + node.onload = () => { const doc = this.screen.document; doc && rewriteNodeStyleSheet(doc, node); From 7cd514eb23951cf2869cb7db9dc00641c3c00253 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 6 Jul 2022 16:40:52 +0200 Subject: [PATCH 06/29] style(frontend/player): codefixes --- .../managers/DOM/DOMManager.ts | 58 +++++++++---------- .../managers/DOM/StylesManager.ts | 8 +-- .../managers/DOM/VirtualDOM.ts | 10 ++-- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts index 278e97ba4..924333eea 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts @@ -15,45 +15,45 @@ const IGNORED_ATTRS = [ "autocomplete", "name" ]; const ATTR_NAME_REGEXP = /([^\t\n\f \/>"'=]+)/; // regexp costs ~ export default class DOMManager extends ListWalker { - private isMobile: boolean; - private screen: StatedScreen; private vTexts: Map = new Map() // map vs object here? private vElements: Map = new Map() private vRoots: Map = new Map() private upperBodyId: number = -1; - private nodeScrollManagers: Array> = [] + private nodeScrollManagers: Map> = new Map() private stylesManager: StylesManager - constructor(screen: StatedScreen, isMobile: boolean, public readonly time: number) { - super(); - this.isMobile = isMobile; - this.screen = screen; - this.stylesManager = new StylesManager(screen); + constructor( + private readonly screen: StatedScreen, + private readonly isMobile: boolean, + public readonly time: number + ) { + super() + this.stylesManager = new StylesManager(screen) } append(m: Message): void { - switch (m.tp) { - case "set_node_scroll": - if (!this.nodeScrollManagers[ m.id ]) { - this.nodeScrollManagers[ m.id ] = new ListWalker(); + if (m.tp === "set_node_scroll") { + let scrollManager = this.nodeScrollManagers.get(m.id) + if (!scrollManager) { + scrollManager = new ListWalker() + this.nodeScrollManagers.set(m.id, scrollManager) } - this.nodeScrollManagers[ m.id ].append(m); - return; - default: - if (m.tp === "create_element_node") { - if(m.tag === "BODY" && this.upperBodyId === -1) { - this.upperBodyId = m.id - } - } else if (m.tp === "set_node_attribute" && - (IGNORED_ATTRS.includes(m.name) || !ATTR_NAME_REGEXP.test(m.name))) { - logger.log("Ignorring message: ", m) - return; // Ignoring - } - super.append(m); + scrollManager.append(m) + return } + if (m.tp === "create_element_node") { + if(m.tag === "BODY" && this.upperBodyId === -1) { + this.upperBodyId = m.id + } + } else if (m.tp === "set_node_attribute" && + (IGNORED_ATTRS.includes(m.name) || !ATTR_NAME_REGEXP.test(m.name))) { + logger.log("Ignorring message: ", m) + return; // Ignoring + } + super.append(m) } private removeBodyScroll(id: number, vn: VElement): void { @@ -160,9 +160,9 @@ export default class DOMManager extends ListWalker { if (!vn) { logger.error("Node not found", msg); return } if (name === "href" && vn.node.tagName === "LINK") { // @ts-ignore TODO: global ENV type // Hack for queries in rewrited urls (don't we do that in backend?) - if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { - value = value.replace("?", "%3F"); - } + // if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { + // value = value.replace("?", "%3F"); + // } this.stylesManager.setStyleHandlers(vn.node as HTMLLinkElement, value); } if (vn.node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) { @@ -236,7 +236,7 @@ export default class DOMManager extends ListWalker { vn = this.vElements.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } if (!(vn instanceof VStyleElement)) { - logger.warn("Non-style node in CSS rules message (or sheet is null)", msg, node.sheet); + logger.warn("Non-style node in CSS rules message (or sheet is null)", msg, vn); return } vn.onStyleSheet(sheet => { diff --git a/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts index 14a2ee8a4..c6b674c4e 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/StylesManager.ts @@ -41,20 +41,20 @@ export default class StylesManager extends ListWalker { setStyleHandlers(node: HTMLLinkElement, value: string): void { let timeoutId: ReturnType | undefined; - const promise = new Promise((resolve) => { - if (this.skipCSSLinks.includes(value)) resolve(null); + const promise = new Promise((resolve) => { + if (this.skipCSSLinks.includes(value)) resolve(); this.linkLoadingCount++; this.screen.setCSSLoading(true); const addSkipAndResolve = () => { this.skipCSSLinks.push(value); // watch out - resolve(null); + resolve() } timeoutId = setTimeout(addSkipAndResolve, 4000); node.onload = () => { const doc = this.screen.document; doc && rewriteNodeStyleSheet(doc, node); - resolve(null); + resolve(); } node.onerror = addSkipAndResolve; }).then(() => { diff --git a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts index 3df1b5a3e..12009c21c 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts @@ -65,18 +65,18 @@ export class VFragment extends VParent { export class VElement extends VParent { parentNode: VParent | null = null - private newAttributes: Record = {} + private newAttributes: Map = new Map() //private props: Record constructor(public readonly node: Element) { super() } setAttribute(name: string, value: string) { - this.newAttributes[name] = value + this.newAttributes.set(name, value) } removeAttribute(name: string) { - this.newAttributes[name] = false + this.newAttributes.set(name, false) } applyChanges() { - Object.entries(this.newAttributes).forEach(([key, value]) => { + this.newAttributes.forEach((value, key) => { if (value === false) { this.node.removeAttribute(key) } else { @@ -87,7 +87,7 @@ export class VElement extends VParent { } } }) - this.newAttributes = {} + this.newAttributes.clear() super.applyChanges() } } From 3c719f4839cdb160333c8d071551d741066f2b03 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 6 Jul 2022 17:34:07 +0200 Subject: [PATCH 07/29] fix(frontend-player): ignore non-http urls in links --- .../app/player/MessageDistributor/managers/DOM/DOMManager.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts index 924333eea..005713e15 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts @@ -159,10 +159,13 @@ export default class DOMManager extends ListWalker { vn = this.vElements.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } if (name === "href" && vn.node.tagName === "LINK") { - // @ts-ignore TODO: global ENV type // Hack for queries in rewrited urls (don't we do that in backend?) + // @ts-ignore TODO: global ENV type // It'd done on backend (remove after testing in saas) // if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { // value = value.replace("?", "%3F"); // } + if (!value.startsWith("http")) { return } + // blob:... value happened here. https://foss.openreplay.com/3/session/7013553567419137 + // that resulted in that link being unable to load and having 4sec timeout in the below function. this.stylesManager.setStyleHandlers(vn.node as HTMLLinkElement, value); } if (vn.node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) { From 5366f2b79876879aef4bf7134a7c9f6a9008c639 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 27 Jul 2022 17:18:46 +0200 Subject: [PATCH 08/29] fix(frontend): log error in API middleware --- frontend/app/api_middleware.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/app/api_middleware.js b/frontend/app/api_middleware.js index a29a22eb6..8f9965ec5 100644 --- a/frontend/app/api_middleware.js +++ b/frontend/app/api_middleware.js @@ -1,3 +1,4 @@ +import logger from 'App/logger'; import APIClient from './api_client'; import { UPDATE, DELETE } from './duck/jwt'; @@ -28,8 +29,9 @@ export default store => next => (action) => { next({ type: UPDATE, data: jwt }); } }) - .catch(() => { - return next({ type: FAILURE, errors: [ 'Connection error' ] }); + .catch((e) => { + logger.error("Error during API request. ", e) + return next({ type: FAILURE, errors: [ "Connection error", String(e) ] }); }); }; From 683b0b3ceb21d8736db3f0d9e31f1c90b67017dd Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 27 Jul 2022 17:20:39 +0200 Subject: [PATCH 09/29] fix(frontend): uncomment ussue types, don't break on unknown --- frontend/app/types/session/issue.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frontend/app/types/session/issue.js b/frontend/app/types/session/issue.js index db220f8cc..673467fd2 100644 --- a/frontend/app/types/session/issue.js +++ b/frontend/app/types/session/issue.js @@ -10,11 +10,11 @@ export const issues_types = List([ { 'type': 'memory', 'visible': true, 'order': 4, 'name': 'High Memory', 'icon': 'funnel/sd-card' }, { 'type': 'vault', 'visible': true, 'order': 5, 'name': 'Vault', 'icon': 'safe' }, { 'type': 'bookmark', 'visible': true, 'order': 5, 'name': 'Bookmarks', 'icon': 'safe' }, - // { 'type': 'bad_request', 'visible': true, 'order': 1, 'name': 'Bad Requests', 'icon': 'funnel/file-medical-alt' }, - // { 'type': 'missing_resource', 'visible': true, 'order': 2, 'name': 'Missing Images', 'icon': 'funnel/image' }, - // { 'type': 'dead_click', 'visible': true, 'order': 4, 'name': 'Dead Clicks', 'icon': 'funnel/dizzy' }, - // { 'type': 'cpu', 'visible': true, 'order': 6, 'name': 'High CPU', 'icon': 'funnel/cpu' }, - // { 'type': 'custom', 'visible': false, 'order': 8, 'name': 'Custom', 'icon': 'funnel/exclamation-circle' } + { 'type': 'bad_request', 'visible': true, 'order': 1, 'name': 'Bad Requests', 'icon': 'funnel/file-medical-alt' }, + { 'type': 'missing_resource', 'visible': true, 'order': 2, 'name': 'Missing Images', 'icon': 'funnel/image' }, + { 'type': 'dead_click', 'visible': true, 'order': 4, 'name': 'Dead Clicks', 'icon': 'funnel/dizzy' }, + { 'type': 'cpu', 'visible': true, 'order': 6, 'name': 'High CPU', 'icon': 'funnel/cpu' }, + { 'type': 'custom', 'visible': false, 'order': 8, 'name': 'Custom', 'icon': 'funnel/exclamation-circle' } ]).map(Watchdog) export const issues_types_map = {} @@ -40,7 +40,7 @@ export default Record({ fromJS: ({ type, ...rest }) => ({ ...rest, type, - icon: issues_types_map[type].icon, - name: issues_types_map[type].name, + icon: issues_types_map[type]?.icon, + name: issues_types_map[type]?.name, }), }); From dabe7f0e444441eede2f9bc5b2ecaae3977373f5 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 27 Jul 2022 17:28:51 +0200 Subject: [PATCH 10/29] fix (frontend/player): instant insertion of style elements & correct children deletion --- .../managers/DOM/DOMManager.ts | 15 ++++- .../managers/DOM/VirtualDOM.ts | 63 +++++++++++-------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts index 005713e15..19e35d8f6 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts @@ -14,6 +14,16 @@ type HTMLElementWithValue = HTMLInputElement | HTMLTextAreaElement | HTMLSelectE const IGNORED_ATTRS = [ "autocomplete", "name" ]; const ATTR_NAME_REGEXP = /([^\t\n\f \/>"'=]+)/; // regexp costs ~ + +// TODO: filter out non-relevant prefixes +// function replaceCSSPrefixes(css: string) { +// return css +// .replace(/\-ms\-/g, "") +// .replace(/\-webkit\-/g, "") +// .replace(/\-moz\-/g, "") +// .replace(/\-webkit\-/g, "") +// } + export default class DOMManager extends ListWalker { private vTexts: Map = new Map() // map vs object here? private vElements: Map = new Map() @@ -144,6 +154,9 @@ export default class DOMManager extends ListWalker { this.insertNode(msg) this.removeBodyScroll(msg.id, vn) this.removeAutocomplete(element) + if (['STYLE', 'style', 'link'].includes(msg.tag)) { + vn.enforceInsertion() + } return case "move_node": this.insertNode(msg); @@ -287,7 +300,7 @@ export default class DOMManager extends ListWalker { // @ts-ignore this.vElements.get(0).applyChanges() - this.vRoots.forEach(rt => rt.applyChanges()) + this.vRoots.forEach(rt => rt.applyChanges()) // MBTODO (optimisation): affected set // Thinkabout (read): css preload // What if we go back before it is ready? We'll have two handlres? diff --git a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts index 12009c21c..c25f07caf 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts @@ -17,6 +17,7 @@ abstract class VParent { this.children = this.children.filter(ch => ch !== child) child.parentNode = null } + applyChanges() { const node = this.node if (!node) { @@ -25,17 +26,20 @@ abstract class VParent { return } const realChildren = node.childNodes - for (let i = 0; i < this.children.length; i++) { - const ch = this.children[i] - ch.applyChanges() - if (ch.node.parentNode !== node) { - const nextSibling = realChildren[i] - node.insertBefore(ch.node, nextSibling || null) - } - if (realChildren[i] !== ch.node) { - node.removeChild(realChildren[i]) + let i: number + // apply correct children order + for (i = 0; i < this.children.length; i++) { + const child = this.children[i] + child.applyChanges() + //while (realChildren[i] shouldn't be there) remove it //optimal way + if (realChildren[i] !== child.node) { + node.insertBefore(child.node, realChildren[i] || null) } } + // remove rest + while(realChildren[i]) { + node.removeChild(realChildren[i]) + } } } @@ -52,7 +56,9 @@ export class VDocument extends VParent { // iframe not mounted yet return } - const htmlNode = this.children[0].node + const child = this.children[0] + child.applyChanges() + const htmlNode = child.node if (htmlNode.parentNode !== this.node) { this.node.replaceChild(htmlNode, this.node.documentElement) } @@ -66,7 +72,6 @@ export class VFragment extends VParent { export class VElement extends VParent { parentNode: VParent | null = null private newAttributes: Map = new Map() - //private props: Record constructor(public readonly node: Element) { super() } setAttribute(name: string, value: string) { this.newAttributes.set(name, value) @@ -75,6 +80,14 @@ export class VElement extends VParent { this.newAttributes.set(name, false) } + enforceInsertion() { + let vNode: VElement = this + while (vNode.parentNode instanceof VElement) { + vNode = vNode.parentNode + } + (vNode.parentNode || vNode).applyChanges() + } + applyChanges() { this.newAttributes.forEach((value, key) => { if (value === false) { @@ -96,31 +109,31 @@ export class VElement extends VParent { type StyleSheetCallback = (s: CSSStyleSheet) => void export type StyleElement = HTMLStyleElement | SVGStyleElement export class VStyleElement extends VElement { - private loaded = false + // private loaded = false private stylesheetCallbacks: StyleSheetCallback[] = [] constructor(public readonly node: StyleElement) { super(node) // Is it compiled correctly or with 2 node assignments? - node.onload = () => { - const sheet = node.sheet - if (sheet) { - this.stylesheetCallbacks.forEach(cb => cb(sheet)) - } else { - console.warn("Style onload: sheet is null") - } - this.loaded = true - } + // node.onload = () => { + // const sheet = node.sheet + // if (sheet) { + // this.stylesheetCallbacks.forEach(cb => cb(sheet)) + // } else { + // console.warn("Style onload: sheet is null") + // } + // this.loaded = true + // } } onStyleSheet(cb: StyleSheetCallback) { - if (this.loaded) { + // if (this.loaded) { if (!this.node.sheet) { console.warn("Style tag is loaded, but sheet is null") return } cb(this.node.sheet) - } else { - this.stylesheetCallbacks.push(cb) - } + // } else { + // this.stylesheetCallbacks.push(cb) + // } } } From cdcabf5f0883acb21f202ff4bda77c5003db1f86 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 27 Jul 2022 17:37:42 +0200 Subject: [PATCH 11/29] fix(frontend/player): store main vDoc inside the vRoots Map --- .../player/MessageDistributor/managers/DOM/DOMManager.ts | 9 +++++---- .../player/MessageDistributor/managers/DOM/VirtualDOM.ts | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts index 19e35d8f6..9e7b4a5f3 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts @@ -130,8 +130,11 @@ export default class DOMManager extends ListWalker { fRoot.innerText = ''; vn = new VElement(fRoot) - this.vElements = new Map([[0, vn ]]) - this.stylesManager.reset(); + this.vElements = new Map([[0, vn]]) + const vDoc = new VDocument(doc) + vDoc.insertChildAt(vn, 0) + this.vRoots = new Map([[-1, vDoc]]) // todo: start from 0 (sync logic with tracker) + this.stylesManager.reset() return case "create_text_node": vn = new VText() @@ -298,8 +301,6 @@ export default class DOMManager extends ListWalker { moveReady(t: number): Promise { this.moveApply(t, this.applyMessage) // This function autoresets pointer if necessary (better name?) - // @ts-ignore - this.vElements.get(0).applyChanges() this.vRoots.forEach(rt => rt.applyChanges()) // MBTODO (optimisation): affected set // Thinkabout (read): css preload diff --git a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts index c25f07caf..76f2cdf67 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts @@ -80,7 +80,7 @@ export class VElement extends VParent { this.newAttributes.set(name, false) } - enforceInsertion() { + enforceInsertion() { // mbtodo: priority insertion instead let vNode: VElement = this while (vNode.parentNode instanceof VElement) { vNode = vNode.parentNode From 2bcc1f3f1e92e63325f34f3f28b874ebaad6704a Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 27 Jul 2022 17:46:39 +0200 Subject: [PATCH 12/29] fix(frontend/player): styles in priority (style tag text as well) --- .../MessageDistributor/managers/DOM/DOMManager.ts | 11 +++++++---- .../MessageDistributor/managers/DOM/VirtualDOM.ts | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts index 9e7b4a5f3..b44e8ac4e 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts @@ -157,7 +157,7 @@ export default class DOMManager extends ListWalker { this.insertNode(msg) this.removeBodyScroll(msg.id, vn) this.removeAutocomplete(element) - if (['STYLE', 'style', 'link'].includes(msg.tag)) { + if (['STYLE', 'style', 'LINK'].includes(msg.tag)) { // Styles in priority vn.enforceInsertion() } return @@ -175,7 +175,7 @@ export default class DOMManager extends ListWalker { vn = this.vElements.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } if (name === "href" && vn.node.tagName === "LINK") { - // @ts-ignore TODO: global ENV type // It'd done on backend (remove after testing in saas) + // @ts-ignore ?global ENV type // It've been done on backend (remove after testing in saas) // if (value.startsWith(window.env.ASSETS_HOST || window.location.origin + '/assets')) { // value = value.replace("?", "%3F"); // } @@ -221,7 +221,7 @@ export default class DOMManager extends ListWalker { (vn.node as HTMLInputElement).checked = msg.checked return case "set_node_data": - case "set_css_data": // TODO: remove css transitions when timeflow is not natural (on jumps) + case "set_css_data": // mbtodo: remove css transitions when timeflow is not natural (on jumps) vn = this.vTexts.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } vn.setData(msg.data) @@ -230,12 +230,15 @@ export default class DOMManager extends ListWalker { // TODO: move to message parsing doc && rewriteNodeStyleSheet(doc, vn.node) } + if (msg.tp === "set_css_data") { // Styles in priority (do we need inlines as well?) + vn.applyChanges() + } return case "css_insert_rule": vn = this.vElements.get(msg.id) if (!vn) { logger.error("Node not found", msg); return } if (!(vn instanceof VStyleElement)) { - logger.warn("Non-style node in CSS rules message (or sheet is null)", msg, node.sheet); + logger.warn("Non-style node in CSS rules message (or sheet is null)", msg, vn); return } vn.onStyleSheet(sheet => { diff --git a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts index 76f2cdf67..256b92bb4 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts @@ -80,7 +80,10 @@ export class VElement extends VParent { this.newAttributes.set(name, false) } - enforceInsertion() { // mbtodo: priority insertion instead + // mbtodo: priority insertion instead. + // rn this is for styles that should be inserted as prior, + // otherwise it will show visual styling lag if there is a transition CSS property) + enforceInsertion() { let vNode: VElement = this while (vNode.parentNode instanceof VElement) { vNode = vNode.parentNode From a1d8848e01fc12dbc901cc1e6aa02ad1ba89f5b3 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Wed, 27 Jul 2022 18:48:02 +0200 Subject: [PATCH 13/29] style(player):future optimisation comments --- .../app/player/MessageDistributor/managers/DOM/DOMManager.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts index b44e8ac4e..378021203 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/DOMManager.ts @@ -302,6 +302,9 @@ export default class DOMManager extends ListWalker { } moveReady(t: number): Promise { + // MBTODO (back jump optimisation): + // - store intemediate virtual dom state + // - cancel previous moveReady tasks (is it possible?) if new timestamp is less this.moveApply(t, this.applyMessage) // This function autoresets pointer if necessary (better name?) this.vRoots.forEach(rt => rt.applyChanges()) // MBTODO (optimisation): affected set From 0d82d7feacdbd2972a4958b3a740fa468078e192 Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 28 Jul 2022 16:34:44 +0200 Subject: [PATCH 14/29] DB improvements (#647) * feat(backend/db): updated ClickHouse library version from 1.5.4 to 2.2.0 * feat(backend/db): refactored ClickHouse connector * feat(backend/db): rewritten batch implementation for ClickHouse inserts * feat(backend/db): found and fix memory leak in db service --- backend/cmd/db/main.go | 11 +- backend/go.mod | 14 +- backend/go.sum | 52 +- ee/backend/internal/db/datasaver/stats.go | 2 +- ee/backend/pkg/db/clickhouse/bulk.go | 45 -- ee/backend/pkg/db/clickhouse/connector.go | 500 +++++++++++++++---- ee/backend/pkg/db/clickhouse/helpers.go | 34 -- ee/backend/pkg/db/clickhouse/messages-web.go | 243 --------- 8 files changed, 447 insertions(+), 454 deletions(-) delete mode 100644 ee/backend/pkg/db/clickhouse/bulk.go delete mode 100644 ee/backend/pkg/db/clickhouse/helpers.go delete mode 100644 ee/backend/pkg/db/clickhouse/messages-web.go diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index 2e962cb1b..1712b8a3f 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -122,11 +122,18 @@ func main() { os.Exit(0) case <-commitTick: // Send collected batches to db + start := time.Now() pg.CommitBatches() + pgDur := time.Now().Sub(start).Milliseconds() + + start = time.Now() if err := saver.CommitStats(); err != nil { log.Printf("Error on stats commit: %v", err) } - // TODO?: separate stats & regular messages + chDur := time.Now().Sub(start).Milliseconds() + log.Printf("commit duration(ms), pg: %d, ch: %d", pgDur, chDur) + + // TODO: use commit worker to save time each tick if err := consumer.Commit(); err != nil { log.Printf("Error on consumer commit: %v", err) } @@ -134,7 +141,7 @@ func main() { // Handle new message from queue err := consumer.ConsumeNext() if err != nil { - log.Fatalf("Error on consumption: %v", err) // TODO: is always fatal? + log.Fatalf("Error on consumption: %v", err) } } } diff --git a/backend/go.mod b/backend/go.mod index a15e23196..caaf1bf83 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,12 +4,12 @@ go 1.18 require ( cloud.google.com/go/logging v1.4.2 - github.com/ClickHouse/clickhouse-go v1.5.4 + github.com/ClickHouse/clickhouse-go/v2 v2.2.0 github.com/aws/aws-sdk-go v1.35.23 github.com/btcsuite/btcutil v1.0.2 github.com/elastic/go-elasticsearch/v7 v7.13.1 github.com/go-redis/redis v6.15.9+incompatible - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/jackc/pgconn v1.6.0 github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 @@ -36,7 +36,6 @@ require ( cloud.google.com/go/storage v1.14.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/confluentinc/confluent-kafka-go v1.9.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -50,15 +49,19 @@ require ( github.com/jackc/pgproto3/v2 v2.0.2 // indirect github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 // indirect github.com/jackc/pgtype v1.3.0 // indirect - github.com/jackc/puddle v1.1.0 // indirect + github.com/jackc/puddle v1.2.2-0.20220404125616-4e959849469a // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.11.9 // indirect + github.com/klauspost/compress v1.15.7 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/paulmach/orb v0.7.1 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + github.com/stretchr/testify v1.8.0 // indirect go.opencensus.io v0.23.0 // indirect go.opentelemetry.io/otel/sdk v1.7.0 // indirect go.opentelemetry.io/otel/trace v1.7.0 // indirect @@ -73,5 +76,4 @@ require ( google.golang.org/grpc v1.46.2 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index 433f2b895..6b76d1278 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -61,9 +61,11 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/ClickHouse/clickhouse-go/v2 v2.2.0 h1:dj00TDKY+xwuTJdbpspCSmTLFyWzRJerTHwaBxut1C0= +github.com/ClickHouse/clickhouse-go/v2 v2.2.0/go.mod h1:8f2XZUi7XoeU+uPIytSi1cvx8fmJxi7vIgqpvYTF1+o= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -79,7 +81,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -100,7 +101,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -151,6 +151,8 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -158,6 +160,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -230,8 +233,9 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -240,8 +244,10 @@ github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/Oth github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -290,8 +296,9 @@ github.com/jackc/pgx/v4 v4.6.0 h1:Fh0O9GdlG4gYpjpwOqjdEodJUQM9jzN3Hdv7PN0xmm0= github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.0 h1:musOWczZC/rSbqut475Vfcczg7jJsdUQf0D6oKPLgNU= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.2-0.20220404125616-4e959849469a h1:oH7y/b+q2BEerCnARr/HZc1NxOYbKSJor4MqQXlhh+s= +github.com/jackc/puddle v1.2.2-0.20220404125616-4e959849469a/go.mod h1:ZQuO1Un86Xpe1ShKl08ERTzYhzWq+OvrvotbpeE3XO0= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -308,10 +315,11 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.11.9 h1:5OCMOdde1TCT2sookEuVeEZzA8bmRSFV3AwPDZAG8AA= -github.com/klauspost/compress v1.11.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok= +github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -339,6 +347,7 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -353,8 +362,12 @@ github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/oschwald/maxminddb-golang v1.7.0 h1:JmU4Q1WBv5Q+2KZy5xJI+98aUwTIrPPxZUkd5Cwr8Zc= github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/paulmach/orb v0.7.1 h1:Zha++Z5OX/l168sqHK3k4z18LDvr+YAO/VjK0ReQ9rU= +github.com/paulmach/orb v0.7.1/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -393,8 +406,12 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sethvargo/go-envconfig v0.7.0 h1:P/ljQXSRjgAgsnIripHs53Jg/uNVXu2FYQ9yLSDappA= github.com/sethvargo/go-envconfig v0.7.0/go.mod h1:00S1FAhRUuTNJazWBWcJGvEHOM+NO6DhoRMAOX7FY5o= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= +github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -403,14 +420,19 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= github.com/ua-parser/uap-go v0.0.0-20200325213135-e1c09f13e2fe h1:aj/vX5epIlQQBEocKoM9nSAiNpakdQzElc8SaRFPu+I= @@ -420,6 +442,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -594,8 +617,10 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -649,6 +674,7 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -706,6 +732,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -714,6 +741,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -927,8 +955,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/ee/backend/internal/db/datasaver/stats.go b/ee/backend/internal/db/datasaver/stats.go index d5bd74f83..7fa2fb9d0 100644 --- a/ee/backend/internal/db/datasaver/stats.go +++ b/ee/backend/internal/db/datasaver/stats.go @@ -10,7 +10,7 @@ import ( . "openreplay/backend/pkg/messages" ) -var ch *clickhouse.Connector +var ch clickhouse.Connector var finalizeTicker <-chan time.Time func (si *Saver) InitStats() { diff --git a/ee/backend/pkg/db/clickhouse/bulk.go b/ee/backend/pkg/db/clickhouse/bulk.go deleted file mode 100644 index 121cdbbf0..000000000 --- a/ee/backend/pkg/db/clickhouse/bulk.go +++ /dev/null @@ -1,45 +0,0 @@ -package clickhouse - -import ( - "errors" - "database/sql" -) - -type bulk struct { - db *sql.DB - query string - tx *sql.Tx - stmt *sql.Stmt -} - -func newBulk(db *sql.DB, query string) *bulk { - return &bulk{ - db: db, - query: query, - } -} - -func (b *bulk) prepare() error { - var err error - b.tx, err = b.db.Begin() - if err != nil { - return err - } - b.stmt, err = b.tx.Prepare(b.query) - if err != nil { - return err - } - return nil -} - -func (b *bulk) commit() error { - return b.tx.Commit() -} - -func (b *bulk) exec(args ...interface{}) error { - if b.stmt == nil { - return errors.New("Bulk is not prepared.") - } - _, err := b.stmt.Exec(args...) - return err -} diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index cc0d20497..1fd6e5d1e 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -1,138 +1,416 @@ package clickhouse import ( - "database/sql" - _ "github.com/ClickHouse/clickhouse-go" + "context" + "errors" + "fmt" + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "log" + "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/hashid" + "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/url" + "strings" + "time" "openreplay/backend/pkg/license" ) -type Connector struct { - sessions *bulk - metadata *bulk // TODO: join sessions, sessions_metadata & sessions_ios - resources *bulk - pages *bulk - clicks *bulk - inputs *bulk - errors *bulk - performance *bulk - longtasks *bulk - db *sql.DB +var CONTEXT_MAP = map[uint64]string{0: "unknown", 1: "self", 2: "same-origin-ancestor", 3: "same-origin-descendant", 4: "same-origin", 5: "cross-origin-ancestor", 6: "cross-origin-descendant", 7: "cross-origin-unreachable", 8: "multiple-contexts"} +var CONTAINER_TYPE_MAP = map[uint64]string{0: "window", 1: "iframe", 2: "embed", 3: "object"} + +type Connector interface { + Prepare() error + Commit() error + FinaliseSessionsTable() error + InsertWebSession(session *types.Session) error + InsertWebResourceEvent(session *types.Session, msg *messages.ResourceEvent) error + InsertWebPageEvent(session *types.Session, msg *messages.PageEvent) error + InsertWebClickEvent(session *types.Session, msg *messages.ClickEvent) error + InsertWebInputEvent(session *types.Session, msg *messages.InputEvent) error + InsertWebErrorEvent(session *types.Session, msg *messages.ErrorEvent) error + InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error + InsertLongtask(session *types.Session, msg *messages.LongTask) error } -func NewConnector(url string) *Connector { +type connectorImpl struct { + conn driver.Conn + batches map[string]driver.Batch +} + +func NewConnector(url string) Connector { license.CheckLicense() - - db, err := sql.Open("clickhouse", url) + url = strings.TrimPrefix(url, "tcp://") + url = strings.TrimSuffix(url, "/default") + conn, err := clickhouse.Open(&clickhouse.Options{ + Addr: []string{url}, + Auth: clickhouse.Auth{ + Database: "default", + }, + MaxOpenConns: 20, + MaxIdleConns: 15, + ConnMaxLifetime: 3 * time.Minute, + Compression: &clickhouse.Compression{ + Method: clickhouse.CompressionLZ4, + }, + // Debug: true, + }) if err != nil { - log.Fatalln(err) + log.Fatal(err) } - return &Connector{ - db: db, - sessions: newBulk(db, ` - INSERT INTO sessions (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, datetime, duration, pages_count, events_count, errors_count, user_browser, user_browser_version) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - // TODO: join sessions, sessions_metadata & sessions_ios - metadata: newBulk(db, ` - INSERT INTO sessions_metadata (session_id, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, datetime) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - resources: newBulk(db, ` - INSERT INTO resources (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - pages: newBulk(db, ` - INSERT INTO pages (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - clicks: newBulk(db, ` - INSERT INTO clicks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label, hesitation_time) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - inputs: newBulk(db, ` - INSERT INTO inputs (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - errors: newBulk(db, ` - INSERT INTO errors (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, source, name, message, error_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - performance: newBulk(db, ` - INSERT INTO performance (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - longtasks: newBulk(db, ` - INSERT INTO longtasks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, context, container_type, container_id, container_name, container_src) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), + + c := &connectorImpl{ + conn: conn, + batches: make(map[string]driver.Batch, 9), } + return c } -func (conn *Connector) Prepare() error { - if err := conn.sessions.prepare(); err != nil { - return err +func (c *connectorImpl) newBatch(name, query string) error { + batch, err := c.conn.PrepareBatch(context.Background(), query) + if err != nil { + return fmt.Errorf("can't create new batch: %s", err) } - if err := conn.metadata.prepare(); err != nil { - return err + if _, ok := c.batches[name]; ok { + delete(c.batches, name) } - if err := conn.resources.prepare(); err != nil { - return err - } - if err := conn.pages.prepare(); err != nil { - return err - } - if err := conn.clicks.prepare(); err != nil { - return err - } - if err := conn.inputs.prepare(); err != nil { - return err - } - if err := conn.errors.prepare(); err != nil { - return err - } - if err := conn.performance.prepare(); err != nil { - return err - } - if err := conn.longtasks.prepare(); err != nil { - return err + c.batches[name] = batch + return nil +} + +var batches = map[string]string{ + "sessions": "INSERT INTO sessions (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, datetime, duration, pages_count, events_count, errors_count, user_browser, user_browser_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "metadata": "INSERT INTO sessions_metadata (session_id, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, datetime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "resources": "INSERT INTO resources (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "pages": "INSERT INTO pages (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "clicks": "INSERT INTO clicks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label, hesitation_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "inputs": "INSERT INTO inputs (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, label) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "errors": "INSERT INTO errors (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, source, name, message, error_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "performance": "INSERT INTO performance (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "longtasks": "INSERT INTO longtasks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, context, container_type, container_id, container_name, container_src) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", +} + +func (c *connectorImpl) Prepare() error { + for table, query := range batches { + if err := c.newBatch(table, query); err != nil { + return fmt.Errorf("can't create %s batch: %s", table, err) + } } return nil } -func (conn *Connector) Commit() error { - if err := conn.sessions.commit(); err != nil { - return err - } - if err := conn.metadata.commit(); err != nil { - return err - } - if err := conn.resources.commit(); err != nil { - return err - } - if err := conn.pages.commit(); err != nil { - return err - } - if err := conn.clicks.commit(); err != nil { - return err - } - if err := conn.inputs.commit(); err != nil { - return err - } - if err := conn.errors.commit(); err != nil { - return err - } - if err := conn.performance.commit(); err != nil { - return err - } - if err := conn.longtasks.commit(); err != nil { - return err +func (c *connectorImpl) Commit() error { + for _, b := range c.batches { + if err := b.Send(); err != nil { + return fmt.Errorf("can't send batch: %s", err) + } } return nil } -func (conn *Connector) FinaliseSessionsTable() error { - _, err := conn.db.Exec("OPTIMIZE TABLE sessions FINAL") - return err +func (c *connectorImpl) FinaliseSessionsTable() error { + if err := c.conn.Exec(context.Background(), "OPTIMIZE TABLE sessions FINAL"); err != nil { + return fmt.Errorf("can't finalise sessions table: %s", err) + } + return nil +} + +func (c *connectorImpl) checkError(name string, err error) { + if err != clickhouse.ErrBatchAlreadySent { + if batchErr := c.newBatch(name, batches[name]); batchErr != nil { + log.Printf("can't create %s batch after failed append operation: %s", name, batchErr) + } + } +} + +func (c *connectorImpl) InsertWebSession(session *types.Session) error { + if session.Duration == nil { + return errors.New("trying to insert session with nil duration") + } + if err := c.batches["sessions"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(session.Timestamp), + uint32(*session.Duration), + uint16(session.PagesCount), + uint16(session.EventsCount), + uint16(session.ErrorsCount), + // Web unique columns + session.UserBrowser, + nullableString(session.UserBrowserVersion), + ); err != nil { + c.checkError("sessions", err) + return fmt.Errorf("can't append to sessions batch: %s", err) + } + if err := c.batches["metadata"].Append( + session.SessionID, + session.UserID, + session.UserAnonymousID, + session.Metadata1, + session.Metadata2, + session.Metadata3, + session.Metadata4, + session.Metadata5, + session.Metadata6, + session.Metadata7, + session.Metadata8, + session.Metadata9, + session.Metadata10, + datetime(session.Timestamp), + ); err != nil { + c.checkError("metadata", err) + return fmt.Errorf("can't append to metadata batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebResourceEvent(session *types.Session, msg *messages.ResourceEvent) error { + var method interface{} = url.EnsureMethod(msg.Method) + if method == "" { + method = nil + } + if err := c.batches["resources"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + url.DiscardURLQuery(msg.URL), + msg.Type, + nullableUint16(uint16(msg.Duration)), + nullableUint16(uint16(msg.TTFB)), + nullableUint16(uint16(msg.HeaderSize)), + nullableUint32(uint32(msg.EncodedBodySize)), + nullableUint32(uint32(msg.DecodedBodySize)), + msg.Success, + ); err != nil { + c.checkError("resources", err) + return fmt.Errorf("can't append to resources batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebPageEvent(session *types.Session, msg *messages.PageEvent) error { + if err := c.batches["pages"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + url.DiscardURLQuery(msg.URL), + nullableUint16(uint16(msg.RequestStart)), + nullableUint16(uint16(msg.ResponseStart)), + nullableUint16(uint16(msg.ResponseEnd)), + nullableUint16(uint16(msg.DomContentLoadedEventStart)), + nullableUint16(uint16(msg.DomContentLoadedEventEnd)), + nullableUint16(uint16(msg.LoadEventStart)), + nullableUint16(uint16(msg.LoadEventEnd)), + nullableUint16(uint16(msg.FirstPaint)), + nullableUint16(uint16(msg.FirstContentfulPaint)), + nullableUint16(uint16(msg.SpeedIndex)), + nullableUint16(uint16(msg.VisuallyComplete)), + nullableUint16(uint16(msg.TimeToInteractive)), + ); err != nil { + c.checkError("pages", err) + return fmt.Errorf("can't append to pages batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebClickEvent(session *types.Session, msg *messages.ClickEvent) error { + if msg.Label == "" { + return nil + } + if err := c.batches["clicks"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + msg.Label, + nullableUint32(uint32(msg.HesitationTime)), + ); err != nil { + c.checkError("clicks", err) + return fmt.Errorf("can't append to clicks batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebInputEvent(session *types.Session, msg *messages.InputEvent) error { + if msg.Label == "" { + return nil + } + if err := c.batches["inputs"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + msg.Label, + ); err != nil { + c.checkError("inputs", err) + return fmt.Errorf("can't append to inputs batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebErrorEvent(session *types.Session, msg *messages.ErrorEvent) error { + if err := c.batches["errors"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + msg.Source, + nullableString(msg.Name), + msg.Message, + hashid.WebErrorID(session.ProjectID, msg), + ); err != nil { + c.checkError("errors", err) + return fmt.Errorf("can't append to errors batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebPerformanceTrackAggr(session *types.Session, msg *messages.PerformanceTrackAggr) error { + var timestamp uint64 = (msg.TimestampStart + msg.TimestampEnd) / 2 + if err := c.batches["performance"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(timestamp), + uint8(msg.MinFPS), + uint8(msg.AvgFPS), + uint8(msg.MaxFPS), + uint8(msg.MinCPU), + uint8(msg.AvgCPU), + uint8(msg.MaxCPU), + msg.MinTotalJSHeapSize, + msg.AvgTotalJSHeapSize, + msg.MaxTotalJSHeapSize, + msg.MinUsedJSHeapSize, + msg.AvgUsedJSHeapSize, + msg.MaxUsedJSHeapSize, + ); err != nil { + c.checkError("performance", err) + return fmt.Errorf("can't append to performance batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertLongtask(session *types.Session, msg *messages.LongTask) error { + if err := c.batches["longtasks"].Append( + session.SessionID, + session.ProjectID, + session.TrackerVersion, + nullableString(session.RevID), + session.UserUUID, + session.UserOS, + nullableString(session.UserOSVersion), + session.UserBrowser, + nullableString(session.UserBrowserVersion), + nullableString(session.UserDevice), + session.UserDeviceType, + session.UserCountry, + datetime(msg.Timestamp), + CONTEXT_MAP[msg.Context], + CONTAINER_TYPE_MAP[msg.ContainerType], + msg.ContainerId, + msg.ContainerName, + msg.ContainerSrc, + ); err != nil { + c.checkError("longtasks", err) + return fmt.Errorf("can't append to longtasks batch: %s", err) + } + return nil +} + +func nullableUint16(v uint16) *uint16 { + var p *uint16 = nil + if v != 0 { + p = &v + } + return p +} + +func nullableUint32(v uint32) *uint32 { + var p *uint32 = nil + if v != 0 { + p = &v + } + return p +} + +func nullableString(v string) *string { + var p *string = nil + if v != "" { + p = &v + } + return p +} + +func datetime(timestamp uint64) time.Time { + t := time.Unix(int64(timestamp/1e3), 0) + // Temporal solution for not correct timestamps in performance messages + if t.Year() < 2022 || t.Year() > 2025 { + return time.Now() + } + return t } diff --git a/ee/backend/pkg/db/clickhouse/helpers.go b/ee/backend/pkg/db/clickhouse/helpers.go deleted file mode 100644 index 37e30518c..000000000 --- a/ee/backend/pkg/db/clickhouse/helpers.go +++ /dev/null @@ -1,34 +0,0 @@ -package clickhouse - -import ( - "time" -) - - -func nullableUint16(v uint16) *uint16 { - var p *uint16 = nil - if v != 0 { - p = &v - } - return p -} - -func nullableUint32(v uint32) *uint32 { - var p *uint32 = nil - if v != 0 { - p = &v - } - return p -} - -func nullableString(v string) *string { - var p *string = nil - if v != "" { - p = &v - } - return p -} - -func datetime(timestamp uint64) time.Time { - return time.Unix(int64(timestamp/1e3), 0) -} diff --git a/ee/backend/pkg/db/clickhouse/messages-web.go b/ee/backend/pkg/db/clickhouse/messages-web.go deleted file mode 100644 index adfa38655..000000000 --- a/ee/backend/pkg/db/clickhouse/messages-web.go +++ /dev/null @@ -1,243 +0,0 @@ -package clickhouse - -import ( - "errors" - - . "openreplay/backend/pkg/db/types" - "openreplay/backend/pkg/hashid" - . "openreplay/backend/pkg/messages" - "openreplay/backend/pkg/url" -) - -func (conn *Connector) InsertWebSession(session *Session) error { - if session.Duration == nil { - return errors.New("Clickhouse: trying to insert session with ") - } - - if err := conn.sessions.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(session.Timestamp), - uint32(*session.Duration), - session.PagesCount, - session.EventsCount, - session.ErrorsCount, - // Web unique columns - session.UserBrowser, - nullableString(session.UserBrowserVersion), - ); err != nil { - return err - } - // TODO: join sessions, sessions_metadata & sessions_ios - return conn.metadata.exec( - session.SessionID, - session.UserID, - session.UserAnonymousID, - session.Metadata1, - session.Metadata2, - session.Metadata3, - session.Metadata4, - session.Metadata5, - session.Metadata6, - session.Metadata7, - session.Metadata8, - session.Metadata9, - session.Metadata10, - datetime(session.Timestamp), - ) -} - -func (conn *Connector) InsertWebResourceEvent(session *Session, msg *ResourceEvent) error { - // nullableString causes error "unexpected type *string" on Nullable Enum type - // (apparently, a clickhouse-go bug) https://github.com/ClickHouse/clickhouse-go/pull/204 - var method interface{} = url.EnsureMethod(msg.Method) - if method == "" { - method = nil - } - return conn.resources.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - url.DiscardURLQuery(msg.URL), - msg.Type, - nullableUint16(uint16(msg.Duration)), - nullableUint16(uint16(msg.TTFB)), - nullableUint16(uint16(msg.HeaderSize)), - nullableUint32(uint32(msg.EncodedBodySize)), - nullableUint32(uint32(msg.DecodedBodySize)), - msg.Success, - ) -} - -func (conn *Connector) InsertWebPageEvent(session *Session, msg *PageEvent) error { - return conn.pages.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - url.DiscardURLQuery(msg.URL), - nullableUint16(uint16(msg.RequestStart)), - nullableUint16(uint16(msg.ResponseStart)), - nullableUint16(uint16(msg.ResponseEnd)), - nullableUint16(uint16(msg.DomContentLoadedEventStart)), - nullableUint16(uint16(msg.DomContentLoadedEventEnd)), - nullableUint16(uint16(msg.LoadEventStart)), - nullableUint16(uint16(msg.LoadEventEnd)), - nullableUint16(uint16(msg.FirstPaint)), - nullableUint16(uint16(msg.FirstContentfulPaint)), - nullableUint16(uint16(msg.SpeedIndex)), - nullableUint16(uint16(msg.VisuallyComplete)), - nullableUint16(uint16(msg.TimeToInteractive)), - ) -} - -func (conn *Connector) InsertWebClickEvent(session *Session, msg *ClickEvent) error { - if msg.Label == "" { - return nil - } - return conn.clicks.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - msg.Label, - nullableUint32(uint32(msg.HesitationTime)), - ) -} - -func (conn *Connector) InsertWebInputEvent(session *Session, msg *InputEvent) error { - if msg.Label == "" { - return nil - } - return conn.inputs.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - msg.Label, - ) -} - -func (conn *Connector) InsertWebErrorEvent(session *Session, msg *ErrorEvent) error { - return conn.errors.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - msg.Source, - nullableString(msg.Name), - msg.Message, - hashid.WebErrorID(session.ProjectID, msg), - ) -} - -func (conn *Connector) InsertWebPerformanceTrackAggr(session *Session, msg *PerformanceTrackAggr) error { - var timestamp uint64 = (msg.TimestampStart + msg.TimestampEnd) / 2 - return conn.performance.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(timestamp), - uint8(msg.MinFPS), - uint8(msg.AvgFPS), - uint8(msg.MaxFPS), - uint8(msg.MinCPU), - uint8(msg.AvgCPU), - uint8(msg.MaxCPU), - msg.MinTotalJSHeapSize, - msg.AvgTotalJSHeapSize, - msg.MaxTotalJSHeapSize, - msg.MinUsedJSHeapSize, - msg.AvgUsedJSHeapSize, - msg.MaxUsedJSHeapSize, - ) -} - -// TODO: make enum message type -var CONTEXT_MAP = map[uint64]string{0: "unknown", 1: "self", 2: "same-origin-ancestor", 3: "same-origin-descendant", 4: "same-origin", 5: "cross-origin-ancestor", 6: "cross-origin-descendant", 7: "cross-origin-unreachable", 8: "multiple-contexts"} -var CONTAINER_TYPE_MAP = map[uint64]string{0: "window", 1: "iframe", 2: "embed", 3: "object"} - -func (conn *Connector) InsertLongtask(session *Session, msg *LongTask) error { - return conn.longtasks.exec( - session.SessionID, - session.ProjectID, - session.TrackerVersion, - nullableString(session.RevID), - session.UserUUID, - session.UserOS, - nullableString(session.UserOSVersion), - session.UserBrowser, - nullableString(session.UserBrowserVersion), - nullableString(session.UserDevice), - session.UserDeviceType, - session.UserCountry, - datetime(msg.Timestamp), - CONTEXT_MAP[msg.Context], - CONTAINER_TYPE_MAP[msg.ContainerType], - msg.ContainerId, - msg.ContainerName, - msg.ContainerSrc, - ) -} From eb48f14a19d46c73e2e33681090092804adbb2fc Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Thu, 28 Jul 2022 22:24:55 +0200 Subject: [PATCH 15/29] fix(frontend/player): (re)insert only necessary children --- .../managers/DOM/VirtualDOM.ts | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts index 256b92bb4..9fa151e61 100644 --- a/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts +++ b/frontend/app/player/MessageDistributor/managers/DOM/VirtualDOM.ts @@ -5,16 +5,20 @@ export type VNode = VDocument | VFragment | VElement | VText abstract class VParent { abstract node: Node | null protected children: VChild[] = [] + private insertedChildren: Set = new Set() + insertChildAt(child: VChild, index: number) { if (child.parentNode) { child.parentNode.removeChild(child) } this.children.splice(index, 0, child) + this.insertedChildren.add(child) child.parentNode = this } removeChild(child: VChild) { this.children = this.children.filter(ch => ch !== child) + this.insertedChildren.delete(child) child.parentNode = null } @@ -25,20 +29,26 @@ abstract class VParent { console.error("No node found", this) return } - const realChildren = node.childNodes - let i: number - // apply correct children order - for (i = 0; i < this.children.length; i++) { + // inserting + for (let i = this.children.length-1; i >= 0; i--) { const child = this.children[i] child.applyChanges() - //while (realChildren[i] shouldn't be there) remove it //optimal way - if (realChildren[i] !== child.node) { - node.insertBefore(child.node, realChildren[i] || null) + if (this.insertedChildren.has(child)) { + const nextVSibling = this.children[i+1] + node.insertBefore(child.node, nextVSibling ? nextVSibling.node : null) } } - // remove rest - while(realChildren[i]) { - node.removeChild(realChildren[i]) + this.insertedChildren.clear() + // removing + const realChildren = node.childNodes + for(let j = 0; j < this.children.length; j++) { + while (realChildren[j] !== this.children[j].node) { + node.removeChild(realChildren[j]) + } + } + // removing rest + while(realChildren.length > this.children.length) { + node.removeChild(node.lastChild) } } } From 2088d16c79577bc2590c555c0a96dfabeb99503a Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 29 Jul 2022 18:27:17 +0200 Subject: [PATCH 16/29] feat(backend&mobs):new batch metadata + PartitionedMessage service msg --- backend/pkg/messages/messages.go | 44 +++++++++++++++++++ backend/pkg/messages/read-message.go | 33 ++++++++++++-- mobs/messages.rb | 18 ++++++++ .../backend~pkg~messages~read-message.go.erb | 4 -- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 6c4d75bfc..9f0136ab0 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -22,6 +22,50 @@ func (msg *BatchMeta) TypeID() int { return 80 } +type BatchMetadata struct { + message + Version uint64 + PageNo uint64 + FirstIndex uint64 + Timestamp int64 + Location string +} + +func (msg *BatchMetadata) Encode() []byte { + buf := make([]byte, 51+len(msg.Location)) + buf[0] = 81 + p := 1 + p = WriteUint(msg.Version, buf, p) + p = WriteUint(msg.PageNo, buf, p) + p = WriteUint(msg.FirstIndex, buf, p) + p = WriteInt(msg.Timestamp, buf, p) + p = WriteString(msg.Location, buf, p) + return buf[:p] +} + +func (msg *BatchMetadata) TypeID() int { + return 81 +} + +type PartitionedMessage struct { + message + PartNo uint64 + PartTotal uint64 +} + +func (msg *PartitionedMessage) Encode() []byte { + buf := make([]byte, 21) + buf[0] = 82 + p := 1 + p = WriteUint(msg.PartNo, buf, p) + p = WriteUint(msg.PartTotal, buf, p) + return buf[:p] +} + +func (msg *PartitionedMessage) TypeID() int { + return 82 +} + type Timestamp struct { message Timestamp uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 5009994f5..89b02bfcd 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -7,10 +7,6 @@ import ( ) func ReadMessage(reader io.Reader) (Message, error) { - t, err := ReadUint(reader) - if err != nil { - return nil, err - } switch t { case 80: @@ -26,6 +22,35 @@ func ReadMessage(reader io.Reader) (Message, error) { } return msg, nil + case 81: + msg := &BatchMetadata{} + if msg.Version, err = ReadUint(reader); err != nil { + return nil, err + } + if msg.PageNo, err = ReadUint(reader); err != nil { + return nil, err + } + if msg.FirstIndex, err = ReadUint(reader); err != nil { + return nil, err + } + if msg.Timestamp, err = ReadInt(reader); err != nil { + return nil, err + } + if msg.Location, err = ReadString(reader); err != nil { + return nil, err + } + return msg, nil + + case 82: + msg := &PartitionedMessage{} + if msg.PartNo, err = ReadUint(reader); err != nil { + return nil, err + } + if msg.PartTotal, err = ReadUint(reader); err != nil { + return nil, err + } + return msg, nil + case 0: msg := &Timestamp{} if msg.Timestamp, err = ReadUint(reader); err != nil { diff --git a/mobs/messages.rb b/mobs/messages.rb index 0f97f3db5..e50d02be4 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -1,9 +1,27 @@ # Special one for Batch Meta. Message id could define the version +# Depricated since tracker 3.6.0 message 80, 'BatchMeta', :replayer => false do uint 'PageNo' uint 'FirstIndex' int 'Timestamp' end + +# since tracker 3.6.0 +message 81, 'BatchMetadata', :replayer => false do + uint 'Version' + uint 'PageNo' + uint 'FirstIndex' + int 'Timestamp' + string 'Location' +end + +# since tracker 3.6.0 +message 82, 'PartitionedMessage', :replayer => false do + uint 'PartNo' + uint 'PartTotal' +end + + message 0, 'Timestamp' do uint 'Timestamp' end diff --git a/mobs/templates/backend~pkg~messages~read-message.go.erb b/mobs/templates/backend~pkg~messages~read-message.go.erb index 2e8920747..c797e2e74 100644 --- a/mobs/templates/backend~pkg~messages~read-message.go.erb +++ b/mobs/templates/backend~pkg~messages~read-message.go.erb @@ -7,10 +7,6 @@ import ( ) func ReadMessage(reader io.Reader) (Message, error) { - t, err := ReadUint(reader) - if err != nil { - return nil, err - } switch t { <% $messages.each do |msg| %> case <%= msg.id %>: From d0e486233a97c62148c02c8758c514961f3065d2 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 29 Jul 2022 19:18:44 +0200 Subject: [PATCH 17/29] fix(backend): no breaking changes to dev --- backend/pkg/messages/messages.go | 44 ------------------- backend/pkg/messages/read-message.go | 33 ++------------ mobs/messages.rb | 18 -------- .../backend~pkg~messages~read-message.go.erb | 4 ++ 4 files changed, 8 insertions(+), 91 deletions(-) diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 9f0136ab0..6c4d75bfc 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -22,50 +22,6 @@ func (msg *BatchMeta) TypeID() int { return 80 } -type BatchMetadata struct { - message - Version uint64 - PageNo uint64 - FirstIndex uint64 - Timestamp int64 - Location string -} - -func (msg *BatchMetadata) Encode() []byte { - buf := make([]byte, 51+len(msg.Location)) - buf[0] = 81 - p := 1 - p = WriteUint(msg.Version, buf, p) - p = WriteUint(msg.PageNo, buf, p) - p = WriteUint(msg.FirstIndex, buf, p) - p = WriteInt(msg.Timestamp, buf, p) - p = WriteString(msg.Location, buf, p) - return buf[:p] -} - -func (msg *BatchMetadata) TypeID() int { - return 81 -} - -type PartitionedMessage struct { - message - PartNo uint64 - PartTotal uint64 -} - -func (msg *PartitionedMessage) Encode() []byte { - buf := make([]byte, 21) - buf[0] = 82 - p := 1 - p = WriteUint(msg.PartNo, buf, p) - p = WriteUint(msg.PartTotal, buf, p) - return buf[:p] -} - -func (msg *PartitionedMessage) TypeID() int { - return 82 -} - type Timestamp struct { message Timestamp uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 89b02bfcd..5009994f5 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -7,6 +7,10 @@ import ( ) func ReadMessage(reader io.Reader) (Message, error) { + t, err := ReadUint(reader) + if err != nil { + return nil, err + } switch t { case 80: @@ -22,35 +26,6 @@ func ReadMessage(reader io.Reader) (Message, error) { } return msg, nil - case 81: - msg := &BatchMetadata{} - if msg.Version, err = ReadUint(reader); err != nil { - return nil, err - } - if msg.PageNo, err = ReadUint(reader); err != nil { - return nil, err - } - if msg.FirstIndex, err = ReadUint(reader); err != nil { - return nil, err - } - if msg.Timestamp, err = ReadInt(reader); err != nil { - return nil, err - } - if msg.Location, err = ReadString(reader); err != nil { - return nil, err - } - return msg, nil - - case 82: - msg := &PartitionedMessage{} - if msg.PartNo, err = ReadUint(reader); err != nil { - return nil, err - } - if msg.PartTotal, err = ReadUint(reader); err != nil { - return nil, err - } - return msg, nil - case 0: msg := &Timestamp{} if msg.Timestamp, err = ReadUint(reader); err != nil { diff --git a/mobs/messages.rb b/mobs/messages.rb index e50d02be4..0f97f3db5 100644 --- a/mobs/messages.rb +++ b/mobs/messages.rb @@ -1,27 +1,9 @@ # Special one for Batch Meta. Message id could define the version -# Depricated since tracker 3.6.0 message 80, 'BatchMeta', :replayer => false do uint 'PageNo' uint 'FirstIndex' int 'Timestamp' end - -# since tracker 3.6.0 -message 81, 'BatchMetadata', :replayer => false do - uint 'Version' - uint 'PageNo' - uint 'FirstIndex' - int 'Timestamp' - string 'Location' -end - -# since tracker 3.6.0 -message 82, 'PartitionedMessage', :replayer => false do - uint 'PartNo' - uint 'PartTotal' -end - - message 0, 'Timestamp' do uint 'Timestamp' end diff --git a/mobs/templates/backend~pkg~messages~read-message.go.erb b/mobs/templates/backend~pkg~messages~read-message.go.erb index c797e2e74..2e8920747 100644 --- a/mobs/templates/backend~pkg~messages~read-message.go.erb +++ b/mobs/templates/backend~pkg~messages~read-message.go.erb @@ -7,6 +7,10 @@ import ( ) func ReadMessage(reader io.Reader) (Message, error) { + t, err := ReadUint(reader) + if err != nil { + return nil, err + } switch t { <% $messages.each do |msg| %> case <%= msg.id %>: From b66923a468d30d86222e1b21cf27565461bba3ff Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 28 Jul 2022 14:58:08 +0200 Subject: [PATCH 18/29] chore(helm): disable minio migration if s3 is enabled Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/templates/job.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index 8925b83d8..c0d7f0a45 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -103,6 +103,7 @@ spec: mountPath: /opt/openreplay - name: dbmigrationscript mountPath: /opt/migrations/ + {{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }} - name: minio image: bitnami/minio:2020.10.9-debian-10-r6 env: @@ -128,6 +129,7 @@ spec: mountPath: /opt/openreplay - name: dbmigrationscript mountPath: /opt/migrations/ + {{- end}} {{- if .Values.global.enterpriseEditionLicense }} # Enterprise migration - name: clickhouse From b36a0a09ea2568b872380f9970e1b5a6d93aaea7 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 1 Aug 2022 17:47:09 +0200 Subject: [PATCH 19/29] feat(ui) - session list - tabs and settings --- .../SessionHeader/SessionHeader.tsx | 38 ++++++++++++++++--- .../SessionSettingButton.tsx | 20 ++++++++++ .../components/SessionSettingButton/index.ts | 1 + frontend/app/types/session/issue.js | 14 +++---- 4 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 frontend/app/components/shared/SessionListContainer/components/SessionSettingButton/SessionSettingButton.tsx create mode 100644 frontend/app/components/shared/SessionListContainer/components/SessionSettingButton/index.ts diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx b/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx index 9efcf7e6e..a3af10ff4 100644 --- a/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx @@ -6,14 +6,25 @@ import SelectDateRange from 'Shared/SelectDateRange'; import SessionTags from '../SessionTags'; import { connect } from 'react-redux'; import SessionSort from '../SessionSort'; +import cn from 'classnames'; +import { setActiveTab } from 'Duck/search'; +import SessionSettingButton from '../SessionSettingButton'; interface Props { listCount: number; filter: any; + isBookmark: any; + isEnterprise: boolean; applyFilter: (filter: any) => void; + setActiveTab: (tab: any) => void; } function SessionHeader(props: Props) { - const { listCount, filter: { startDate, endDate, rangeValue } } = props; + const { + listCount, + filter: { startDate, endDate, rangeValue }, + isBookmark, + isEnterprise, + } = props; const period = Period({ start: startDate, end: endDate, rangeName: rangeValue }); const onDateChange = (e: any) => { @@ -23,17 +34,30 @@ function SessionHeader(props: Props) { return (
-
-
- Sessions {listCount} +
+
+
props.setActiveTab({ type: 'all' })} + > + Sessions {listCount} +
+
props.setActiveTab({ type: 'bookmark' })} + > + {`${isEnterprise ? 'Vault' : 'Bookmarks'}`} +
-
+ {!isBookmark && } +
+
); @@ -43,6 +67,8 @@ export default connect( (state: any) => ({ filter: state.getIn(['search', 'instance']), listCount: numberWithCommas(state.getIn(['sessions', 'total'])), + isBookmark: state.getIn(['search', 'activeTab', 'type']) === 'bookmark', + isEnterprise: state.getIn(['user', 'account', 'edition']) === 'ee', }), - { applyFilter } + { applyFilter, setActiveTab } )(SessionHeader); diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionSettingButton/SessionSettingButton.tsx b/frontend/app/components/shared/SessionListContainer/components/SessionSettingButton/SessionSettingButton.tsx new file mode 100644 index 000000000..897a59754 --- /dev/null +++ b/frontend/app/components/shared/SessionListContainer/components/SessionSettingButton/SessionSettingButton.tsx @@ -0,0 +1,20 @@ +import { useModal } from 'App/components/Modal'; +import React from 'react'; +import SessionSettings from 'Shared/SessionSettings'; +import { Button } from 'UI'; + +function SessionSettingButton(props: any) { + const { showModal } = useModal(); + + const handleClick = () => { + showModal(, { right: true }); + }; + + return ( +
+
+ ); +} + +export default SessionSettingButton; diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionSettingButton/index.ts b/frontend/app/components/shared/SessionListContainer/components/SessionSettingButton/index.ts new file mode 100644 index 000000000..dffed3cc0 --- /dev/null +++ b/frontend/app/components/shared/SessionListContainer/components/SessionSettingButton/index.ts @@ -0,0 +1 @@ +export { default } from './SessionSettingButton'; \ No newline at end of file diff --git a/frontend/app/types/session/issue.js b/frontend/app/types/session/issue.js index 673467fd2..d2afff190 100644 --- a/frontend/app/types/session/issue.js +++ b/frontend/app/types/session/issue.js @@ -8,13 +8,13 @@ export const issues_types = List([ { 'type': 'click_rage', 'visible': true, 'order': 2, 'name': 'Click Rage', 'icon': 'funnel/emoji-angry' }, { 'type': 'crash', 'visible': true, 'order': 3, 'name': 'Crashes', 'icon': 'funnel/file-earmark-break' }, { 'type': 'memory', 'visible': true, 'order': 4, 'name': 'High Memory', 'icon': 'funnel/sd-card' }, - { 'type': 'vault', 'visible': true, 'order': 5, 'name': 'Vault', 'icon': 'safe' }, - { 'type': 'bookmark', 'visible': true, 'order': 5, 'name': 'Bookmarks', 'icon': 'safe' }, - { 'type': 'bad_request', 'visible': true, 'order': 1, 'name': 'Bad Requests', 'icon': 'funnel/file-medical-alt' }, - { 'type': 'missing_resource', 'visible': true, 'order': 2, 'name': 'Missing Images', 'icon': 'funnel/image' }, - { 'type': 'dead_click', 'visible': true, 'order': 4, 'name': 'Dead Clicks', 'icon': 'funnel/dizzy' }, - { 'type': 'cpu', 'visible': true, 'order': 6, 'name': 'High CPU', 'icon': 'funnel/cpu' }, - { 'type': 'custom', 'visible': false, 'order': 8, 'name': 'Custom', 'icon': 'funnel/exclamation-circle' } + // { 'type': 'vault', 'visible': true, 'order': 5, 'name': 'Vault', 'icon': 'safe' }, + // { 'type': 'bookmark', 'visible': true, 'order': 5, 'name': 'Bookmarks', 'icon': 'safe' }, + // { 'type': 'bad_request', 'visible': true, 'order': 1, 'name': 'Bad Requests', 'icon': 'funnel/file-medical-alt' }, + // { 'type': 'missing_resource', 'visible': true, 'order': 2, 'name': 'Missing Images', 'icon': 'funnel/image' }, + // { 'type': 'dead_click', 'visible': true, 'order': 4, 'name': 'Dead Clicks', 'icon': 'funnel/dizzy' }, + // { 'type': 'cpu', 'visible': true, 'order': 6, 'name': 'High CPU', 'icon': 'funnel/cpu' }, + // { 'type': 'custom', 'visible': false, 'order': 8, 'name': 'Custom', 'icon': 'funnel/exclamation-circle' } ]).map(Watchdog) export const issues_types_map = {} From a0344608386e474a892672735e2e9cc2b1fe5499 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 1 Aug 2022 18:01:16 +0200 Subject: [PATCH 20/29] feat(ui) - session list - tabs and settings --- .../SessionListContainer/SessionListContainer.tsx | 6 +++--- .../components/SessionHeader/SessionHeader.tsx | 14 +++++++++----- .../components/SessionList/SessionList.tsx | 5 ++--- frontend/app/theme/colors.js | 1 + 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx b/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx index 156f845c4..0a13707c7 100644 --- a/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx +++ b/frontend/app/components/shared/SessionListContainer/SessionListContainer.tsx @@ -2,12 +2,12 @@ import React from 'react'; import SessionList from './components/SessionList'; import SessionHeader from './components/SessionHeader'; -interface Props {} -function SessionListContainer(props: Props) { +function SessionListContainer() { return (
-
+
+
diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx b/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx index a3af10ff4..f0fe2b7d2 100644 --- a/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/SessionHeader/SessionHeader.tsx @@ -33,20 +33,24 @@ function SessionHeader(props: Props) { }; return ( -
+
props.setActiveTab({ type: 'all' })} > - Sessions {listCount} + SESSIONS {listCount}
props.setActiveTab({ type: 'bookmark' })} > - {`${isEnterprise ? 'Vault' : 'Bookmarks'}`} + {`${isEnterprise ? 'VAULT' : 'BOOKMARKS'}`}
diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionList/SessionList.tsx b/frontend/app/components/shared/SessionListContainer/components/SessionList/SessionList.tsx index 60d1aec6c..b5a475073 100644 --- a/frontend/app/components/shared/SessionListContainer/components/SessionList/SessionList.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/SessionList/SessionList.tsx @@ -60,7 +60,7 @@ function SessionList(props: Props) { show={!loading && list.size === 0} > {list.map((session: any) => ( - +
-
- +
))} diff --git a/frontend/app/theme/colors.js b/frontend/app/theme/colors.js index 97c65d0c5..51be4289d 100644 --- a/frontend/app/theme/colors.js +++ b/frontend/app/theme/colors.js @@ -42,5 +42,6 @@ module.exports = { default: '#DDDDDD', 'gray-light-shade': '#EEEEEE', 'primary': '#3490dc', + 'transparent': 'transparent', } } From f51519bc8656a4288881b81f9584c590dc2da7a9 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 7 Jul 2022 17:15:08 +0200 Subject: [PATCH 21/29] chore(helm): Adding PV support for deployments --- .../charts/alerts/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/alerts/values.yaml | 12 ++++++++++++ .../charts/assets/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/assets/values.yaml | 12 ++++++++++++ .../charts/assist/templates/deployment.yaml | 10 +++++++++- .../helmcharts/openreplay/charts/assist/values.yaml | 12 ++++++++++++ .../charts/chalice/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/chalice/values.yaml | 12 ++++++++++++ .../openreplay/charts/db/templates/deployment.yaml | 8 ++++++++ scripts/helmcharts/openreplay/charts/db/values.yaml | 12 ++++++++++++ .../charts/ender/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/ender/values.yaml | 12 ++++++++++++ .../openreplay/charts/http/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/http/values.yaml | 12 ++++++++++++ .../charts/integrations/templates/deployment.yaml | 8 ++++++++ .../openreplay/charts/integrations/values.yaml | 12 ++++++++++++ .../charts/peers/templates/deployment.yaml | 8 ++++++++ .../helmcharts/openreplay/charts/peers/values.yaml | 12 ++++++++++++ 18 files changed, 181 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml index eac304df0..afb7aedc5 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml @@ -96,8 +96,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/alerts/values.yaml b/scripts/helmcharts/openreplay/charts/alerts/values.yaml index 8efd36a80..4bcc516c8 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/values.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/values.yaml @@ -99,3 +99,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml index ce0c41c99..5fbd084c0 100644 --- a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml @@ -90,8 +90,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/assets/values.yaml b/scripts/helmcharts/openreplay/charts/assets/values.yaml index e590f1b3c..2597ed36c 100644 --- a/scripts/helmcharts/openreplay/charts/assets/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/values.yaml @@ -96,3 +96,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml index 2a776ab58..ed4ec5d4a 100644 --- a/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/deployment.yaml @@ -62,10 +62,18 @@ spec: {{- range $key, $val := .Values.service.ports }} - name: {{ $key }} containerPort: {{ $val }} - protocol: TCP {{- end }} + protocol: TCP + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/assist/values.yaml b/scripts/helmcharts/openreplay/charts/assist/values.yaml index 4ffd45a0d..4ffaf88e1 100644 --- a/scripts/helmcharts/openreplay/charts/assist/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/values.yaml @@ -97,3 +97,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index d2aafc35a..4491a82e4 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -106,8 +106,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 99acdbf76..2c9d75040 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -120,3 +120,15 @@ healthCheck: initialDelaySeconds: 100 periodSeconds: 15 timeoutSeconds: 10 + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml index 7afbf0e7d..2c18179df 100644 --- a/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml @@ -62,8 +62,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/db/values.yaml b/scripts/helmcharts/openreplay/charts/db/values.yaml index 0da7ab913..7d375c594 100644 --- a/scripts/helmcharts/openreplay/charts/db/values.yaml +++ b/scripts/helmcharts/openreplay/charts/db/values.yaml @@ -98,3 +98,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml index 368c3ee29..a313415c8 100644 --- a/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml @@ -62,8 +62,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/ender/values.yaml b/scripts/helmcharts/openreplay/charts/ender/values.yaml index 2d3d2b65b..c751680d4 100644 --- a/scripts/helmcharts/openreplay/charts/ender/values.yaml +++ b/scripts/helmcharts/openreplay/charts/ender/values.yaml @@ -97,3 +97,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml index 44574d1f8..eaa5d7ed1 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml @@ -88,8 +88,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/http/values.yaml b/scripts/helmcharts/openreplay/charts/http/values.yaml index 72a8acb7d..7a96d525d 100644 --- a/scripts/helmcharts/openreplay/charts/http/values.yaml +++ b/scripts/helmcharts/openreplay/charts/http/values.yaml @@ -97,3 +97,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml index 5e63e5153..e0f3fff60 100644 --- a/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml @@ -62,8 +62,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/integrations/values.yaml b/scripts/helmcharts/openreplay/charts/integrations/values.yaml index b9086900b..191ed7047 100644 --- a/scripts/helmcharts/openreplay/charts/integrations/values.yaml +++ b/scripts/helmcharts/openreplay/charts/integrations/values.yaml @@ -98,3 +98,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl diff --git a/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml index 6f1a379d8..ac673fd08 100644 --- a/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/templates/deployment.yaml @@ -54,8 +54,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/peers/values.yaml b/scripts/helmcharts/openreplay/charts/peers/values.yaml index 721c09db9..4643a75a7 100644 --- a/scripts/helmcharts/openreplay/charts/peers/values.yaml +++ b/scripts/helmcharts/openreplay/charts/peers/values.yaml @@ -93,3 +93,15 @@ nodeSelector: {} tolerations: [] affinity: {} + + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl From 76acafdd125129f4ec35fbf007b9c73c470df2fe Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 1 Aug 2022 19:04:30 +0200 Subject: [PATCH 22/29] fix(nginx): fixing base config Signed-off-by: rjshrjndrn --- frontend/nginx.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 0138027f0..0fc356092 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -5,5 +5,8 @@ server { index index.html; location / { try_files $uri $uri/ =404; + rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /index.html break; + proxy_intercept_errors on; # see frontend://nginx.org/en/docs/frontend/ngx_frontend_proxy_module.html#proxy_intercept_errors + error_page 404 =200 /index.html; } } From 26c8d769b78f18320e0446bd598b6fb144647ff4 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 1 Aug 2022 19:03:15 +0200 Subject: [PATCH 23/29] chore(nginx): removing ipv6, as some machines won't have support for that. Signed-off-by: rjshrjndrn --- frontend/nginx.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 0fc356092..b886096da 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,6 +1,5 @@ server { listen 8080 default_server; - listen [::]:8080 default_server; root /var/www/openreplay; index index.html; location / { From 7dedafa7b70676e2d4f643c6c8a80f237a0f7ec4 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Mon, 1 Aug 2022 19:11:34 +0200 Subject: [PATCH 24/29] chore(helm): heuristics mount persistence Signed-off-by: rjshrjndrn --- .../charts/heuristics/templates/deployment.yaml | 8 ++++++++ .../openreplay/charts/heuristics/values.yaml | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml index 995e8eac2..58059f58d 100644 --- a/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/heuristics/templates/deployment.yaml @@ -60,8 +60,16 @@ spec: containerPort: {{ $val }} protocol: TCP {{- end }} + {{- with .Values.persistence.mounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.persistence.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/scripts/helmcharts/openreplay/charts/heuristics/values.yaml b/scripts/helmcharts/openreplay/charts/heuristics/values.yaml index 12d2346a3..ec8400866 100644 --- a/scripts/helmcharts/openreplay/charts/heuristics/values.yaml +++ b/scripts/helmcharts/openreplay/charts/heuristics/values.yaml @@ -97,3 +97,14 @@ nodeSelector: {} tolerations: [] affinity: {} + +persistence: {} + # # Spec of spec.template.spec.containers[*].volumeMounts + # mounts: + # - name: kafka-ssl + # mountPath: /opt/kafka/ssl + # # Spec of spec.template.spec.volumes + # volumes: + # - name: kafka-ssl + # secret: + # secretName: kafka-ssl From 4794a513eda31577bc132584e677164620ed9864 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 2 Aug 2022 11:29:49 +0200 Subject: [PATCH 25/29] feat(ui) - session list - tabs and settings --- .../components/SessionTags/SessionTags.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/app/components/shared/SessionListContainer/components/SessionTags/SessionTags.tsx b/frontend/app/components/shared/SessionListContainer/components/SessionTags/SessionTags.tsx index 1c8f5c926..4d49ab83b 100644 --- a/frontend/app/components/shared/SessionListContainer/components/SessionTags/SessionTags.tsx +++ b/frontend/app/components/shared/SessionListContainer/components/SessionTags/SessionTags.tsx @@ -54,12 +54,11 @@ function TagItem({ isActive, onClick, label, icon = '', disabled = false }: any) onClick={onClick} className={cn('transition group rounded ml-2 px-2 py-1 flex items-center uppercase text-sm hover:bg-teal hover:text-white', { 'bg-teal text-white': isActive, - 'bg-active-blue color-teal': !isActive, 'disabled': disabled, })} > - {icon && } - {label} + {icon && } + {label}
); From 01ec7865b7d083afe8ba928528be1f67ce74e0ca Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 2 Aug 2022 16:00:39 +0200 Subject: [PATCH 26/29] feat(chalice): EFS sessions --- api/.gitignore | 3 ++- api/app.py | 3 ++- api/chalicelib/core/assist.py | 10 +++++++++- api/env.default | 3 ++- api/routers/core.py | 15 +++++++++++++-- ee/api/app.py | 4 +++- ee/api/env.default | 1 + 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/api/.gitignore b/api/.gitignore index 6a46fedcb..68797b56a 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -174,4 +174,5 @@ logs*.txt SUBNETS.json ./chalicelib/.configs -README/* \ No newline at end of file +README/* +.local \ No newline at end of file diff --git a/api/app.py b/api/app.py index 4fd042d1a..cf00c747b 100644 --- a/api/app.py +++ b/api/app.py @@ -4,6 +4,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware from starlette.responses import StreamingResponse from chalicelib.utils import helper @@ -14,7 +15,7 @@ from routers.crons import core_dynamic_crons from routers.subs import dashboard, insights, metrics, v1_api app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) - +app.add_middleware(GZipMiddleware, minimum_size=1000) @app.middleware('http') async def or_middleware(request: Request, call_next): diff --git a/api/chalicelib/core/assist.py b/api/chalicelib/core/assist.py index 6fc8bcd90..e382fe348 100644 --- a/api/chalicelib/core/assist.py +++ b/api/chalicelib/core/assist.py @@ -1,6 +1,6 @@ import requests from decouple import config - +from os.path import exists import schemas from chalicelib.core import projects @@ -158,3 +158,11 @@ def autocomplete(project_id, q: str, key: str = None): def get_ice_servers(): return config("iceServers") if config("iceServers", default=None) is not None \ and len(config("iceServers")) > 0 else None + + +def get_raw_mob_by_id(project_id, session_id): + path_to_file = config("FS_DIR") + "/" + str(session_id) + + if exists(path_to_file): + return path_to_file + return None diff --git a/api/env.default b/api/env.default index aa14fc993..c99e1dc05 100644 --- a/api/env.default +++ b/api/env.default @@ -47,4 +47,5 @@ sessions_region=us-east-1 sourcemaps_bucket=sourcemaps sourcemaps_reader=http://127.0.0.1:9000/sourcemaps stage=default-foss -version_number=1.4.0 \ No newline at end of file +version_number=1.4.0 +FS_DIR=/mnt/efs \ No newline at end of file diff --git a/api/routers/core.py b/api/routers/core.py index f9629d8bb..816ffbd4a 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -2,6 +2,7 @@ from typing import Union, Optional from decouple import config from fastapi import Depends, Body, BackgroundTasks, HTTPException +from fastapi.responses import FileResponse from starlette import status import schemas @@ -183,8 +184,8 @@ def session_top_filter_values(projectId: int, context: schemas.CurrentContext = @app.get('/{projectId}/integrations', tags=["integrations"]) def get_integrations_status(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): data = integrations_global.get_global_integrations_status(tenant_id=context.tenant_id, - user_id=context.user_id, - project_id=projectId) + user_id=context.user_id, + project_id=projectId) return {"data": data} @@ -895,6 +896,16 @@ def get_live_session(projectId: int, sessionId: str, background_tasks: Backgroun return {'data': data} +@app.get('/{projectId}/assist/sessions/{sessionId}/replay', tags=["assist"]) +def get_live_session_replay_file(projectId: int, sessionId: str, + context: schemas.CurrentContext = Depends(OR_context)): + path = assist.get_raw_mob_by_id(project_id=projectId, session_id=sessionId) + if path is None: + return {"errors": ["Replay file not found"]} + + return FileResponse(path=path, media_type="application/octet-stream") + + @app.post('/{projectId}/heatmaps/url', tags=["heatmaps"]) def get_heatmaps_by_url(projectId: int, data: schemas.GetHeatmapPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): diff --git a/ee/api/app.py b/ee/api/app.py index 1e12e6015..9f2f9a306 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -5,18 +5,20 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware from starlette import status from starlette.responses import StreamingResponse, JSONResponse from chalicelib.utils import helper from chalicelib.utils import pg_client from routers import core, core_dynamic, ee, saml -from routers.subs import v1_api from routers.crons import core_crons from routers.crons import core_dynamic_crons from routers.subs import dashboard, insights, metrics, v1_api_ee +from routers.subs import v1_api app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) +app.add_middleware(GZipMiddleware, minimum_size=1000) @app.middleware('http') diff --git a/ee/api/env.default b/ee/api/env.default index 7687566d7..70941ab99 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -57,3 +57,4 @@ sourcemaps_bucket=sourcemaps sourcemaps_reader=http://127.0.0.1:9000/sourcemaps stage=default-ee version_number=1.0.0 +FS_DIR=/mnt/efs \ No newline at end of file From 7af26795ace6ac23d44380c976166bf4e8d83c86 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Tue, 2 Aug 2022 16:01:44 +0200 Subject: [PATCH 27/29] feat(chalice): EFS sessions (#651) --- api/.gitignore | 3 ++- api/app.py | 3 ++- api/chalicelib/core/assist.py | 10 +++++++++- api/env.default | 3 ++- api/routers/core.py | 15 +++++++++++++-- ee/api/app.py | 4 +++- ee/api/env.default | 1 + 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/api/.gitignore b/api/.gitignore index 6a46fedcb..68797b56a 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -174,4 +174,5 @@ logs*.txt SUBNETS.json ./chalicelib/.configs -README/* \ No newline at end of file +README/* +.local \ No newline at end of file diff --git a/api/app.py b/api/app.py index 4fd042d1a..cf00c747b 100644 --- a/api/app.py +++ b/api/app.py @@ -4,6 +4,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware from starlette.responses import StreamingResponse from chalicelib.utils import helper @@ -14,7 +15,7 @@ from routers.crons import core_dynamic_crons from routers.subs import dashboard, insights, metrics, v1_api app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) - +app.add_middleware(GZipMiddleware, minimum_size=1000) @app.middleware('http') async def or_middleware(request: Request, call_next): diff --git a/api/chalicelib/core/assist.py b/api/chalicelib/core/assist.py index 6fc8bcd90..e382fe348 100644 --- a/api/chalicelib/core/assist.py +++ b/api/chalicelib/core/assist.py @@ -1,6 +1,6 @@ import requests from decouple import config - +from os.path import exists import schemas from chalicelib.core import projects @@ -158,3 +158,11 @@ def autocomplete(project_id, q: str, key: str = None): def get_ice_servers(): return config("iceServers") if config("iceServers", default=None) is not None \ and len(config("iceServers")) > 0 else None + + +def get_raw_mob_by_id(project_id, session_id): + path_to_file = config("FS_DIR") + "/" + str(session_id) + + if exists(path_to_file): + return path_to_file + return None diff --git a/api/env.default b/api/env.default index aa14fc993..c99e1dc05 100644 --- a/api/env.default +++ b/api/env.default @@ -47,4 +47,5 @@ sessions_region=us-east-1 sourcemaps_bucket=sourcemaps sourcemaps_reader=http://127.0.0.1:9000/sourcemaps stage=default-foss -version_number=1.4.0 \ No newline at end of file +version_number=1.4.0 +FS_DIR=/mnt/efs \ No newline at end of file diff --git a/api/routers/core.py b/api/routers/core.py index f9629d8bb..816ffbd4a 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -2,6 +2,7 @@ from typing import Union, Optional from decouple import config from fastapi import Depends, Body, BackgroundTasks, HTTPException +from fastapi.responses import FileResponse from starlette import status import schemas @@ -183,8 +184,8 @@ def session_top_filter_values(projectId: int, context: schemas.CurrentContext = @app.get('/{projectId}/integrations', tags=["integrations"]) def get_integrations_status(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): data = integrations_global.get_global_integrations_status(tenant_id=context.tenant_id, - user_id=context.user_id, - project_id=projectId) + user_id=context.user_id, + project_id=projectId) return {"data": data} @@ -895,6 +896,16 @@ def get_live_session(projectId: int, sessionId: str, background_tasks: Backgroun return {'data': data} +@app.get('/{projectId}/assist/sessions/{sessionId}/replay', tags=["assist"]) +def get_live_session_replay_file(projectId: int, sessionId: str, + context: schemas.CurrentContext = Depends(OR_context)): + path = assist.get_raw_mob_by_id(project_id=projectId, session_id=sessionId) + if path is None: + return {"errors": ["Replay file not found"]} + + return FileResponse(path=path, media_type="application/octet-stream") + + @app.post('/{projectId}/heatmaps/url', tags=["heatmaps"]) def get_heatmaps_by_url(projectId: int, data: schemas.GetHeatmapPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): diff --git a/ee/api/app.py b/ee/api/app.py index 1e12e6015..9f2f9a306 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -5,18 +5,20 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware from starlette import status from starlette.responses import StreamingResponse, JSONResponse from chalicelib.utils import helper from chalicelib.utils import pg_client from routers import core, core_dynamic, ee, saml -from routers.subs import v1_api from routers.crons import core_crons from routers.crons import core_dynamic_crons from routers.subs import dashboard, insights, metrics, v1_api_ee +from routers.subs import v1_api app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) +app.add_middleware(GZipMiddleware, minimum_size=1000) @app.middleware('http') diff --git a/ee/api/env.default b/ee/api/env.default index 7687566d7..70941ab99 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -57,3 +57,4 @@ sourcemaps_bucket=sourcemaps sourcemaps_reader=http://127.0.0.1:9000/sourcemaps stage=default-ee version_number=1.0.0 +FS_DIR=/mnt/efs \ No newline at end of file From e867c18300d3dff9518e47b4dd0da1097d467a09 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 2 Aug 2022 17:08:32 +0200 Subject: [PATCH 28/29] feat(chalice): changed EFS endpoint --- api/routers/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/routers/core.py b/api/routers/core.py index 816ffbd4a..91e07e493 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -896,6 +896,7 @@ def get_live_session(projectId: int, sessionId: str, background_tasks: Backgroun return {'data': data} +@app.get('/{projectId}/unprocessed/{sessionId}', tags=["assist"]) @app.get('/{projectId}/assist/sessions/{sessionId}/replay', tags=["assist"]) def get_live_session_replay_file(projectId: int, sessionId: str, context: schemas.CurrentContext = Depends(OR_context)): From 4ddbe3884519e8c0a2a28386925963034092ebb3 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Tue, 2 Aug 2022 17:09:47 +0200 Subject: [PATCH 29/29] Api dev v1.8.0 (#653) * feat(chalice): EFS sessions * feat(chalice): changed EFS endpoint --- api/routers/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/routers/core.py b/api/routers/core.py index 816ffbd4a..91e07e493 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -896,6 +896,7 @@ def get_live_session(projectId: int, sessionId: str, background_tasks: Backgroun return {'data': data} +@app.get('/{projectId}/unprocessed/{sessionId}', tags=["assist"]) @app.get('/{projectId}/assist/sessions/{sessionId}/replay', tags=["assist"]) def get_live_session_replay_file(projectId: int, sessionId: str, context: schemas.CurrentContext = Depends(OR_context)):