From 9dd8a351d9ea417c6fe69c29305b22d1660913ca Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 6 Jan 2023 11:56:12 +0100 Subject: [PATCH 01/15] change(ui) - filter series to check for empty flag --- .../components/FilterSeries/FilterSeries.tsx | 6 ++++-- .../Dashboard/components/WidgetForm/WidgetForm.tsx | 1 + .../shared/Filters/FilterItem/FilterItem.tsx | 13 ++++++------- .../shared/Filters/FilterList/FilterList.tsx | 5 ++++- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/frontend/app/components/Dashboard/components/FilterSeries/FilterSeries.tsx b/frontend/app/components/Dashboard/components/FilterSeries/FilterSeries.tsx index 12fbab009..c0de60d2d 100644 --- a/frontend/app/components/Dashboard/components/FilterSeries/FilterSeries.tsx +++ b/frontend/app/components/Dashboard/components/FilterSeries/FilterSeries.tsx @@ -11,7 +11,7 @@ interface Props { series: any; onRemoveSeries: (seriesIndex: any) => void; canDelete?: boolean; - + supportsEmpty?: boolean; hideHeader?: boolean; emptyMessage?: any; observeChanges?: () => void; @@ -23,7 +23,8 @@ function FilterSeries(props: Props) { }, canDelete, hideHeader = false, - emptyMessage = 'Add user event or filter to define the series by clicking Add Step.' + emptyMessage = 'Add user event or filter to define the series by clicking Add Step.', + supportsEmpty = true, } = props; const [expanded, setExpanded] = useState(true) const { series, seriesIndex } = props; @@ -74,6 +75,7 @@ function FilterSeries(props: Props) { onUpdateFilter={onUpdateFilter} onRemoveFilter={onRemoveFilter} onChangeEventsOrder={onChangeEventsOrder} + supportsEmpty={supportsEmpty} /> ) : (
{emptyMessage}
diff --git a/frontend/app/components/Dashboard/components/WidgetForm/WidgetForm.tsx b/frontend/app/components/Dashboard/components/WidgetForm/WidgetForm.tsx index bc8429d87..a24db7391 100644 --- a/frontend/app/components/Dashboard/components/WidgetForm/WidgetForm.tsx +++ b/frontend/app/components/Dashboard/components/WidgetForm/WidgetForm.tsx @@ -201,6 +201,7 @@ function WidgetForm(props: Props) { .map((series: any, index: number) => (
metric.updateKey('hasChanged', true)} hideHeader={isTable || isClickmap} seriesIndex={index} diff --git a/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx b/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx index fae8d9f1d..c76cc7938 100644 --- a/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx +++ b/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx @@ -2,7 +2,7 @@ import React from 'react'; import FilterOperator from '../FilterOperator'; import FilterSelection from '../FilterSelection'; import FilterValue from '../FilterValue'; -import { Icon } from 'UI'; +import { Icon, Button } from 'UI'; import FilterSource from '../FilterSource'; import { FilterKey, FilterType } from 'App/types/filter/filterType'; import SubFilterItem from '../SubFilterItem'; @@ -14,9 +14,10 @@ interface Props { onRemoveFilter: () => void; isFilter?: boolean; saveRequestPayloads?: boolean; + disableDelete?: boolean } function FilterItem(props: Props) { - const { isFilter = false, filterIndex, filter, saveRequestPayloads } = props; + const { isFilter = false, filterIndex, filter, saveRequestPayloads, disableDelete = false } = props; const canShowValues = !(filter.operator === 'isAny' || filter.operator === 'onAny' || filter.operator === 'isUndefined'); const isSubFilter = filter.type === FilterType.SUB_FILTERS; @@ -49,7 +50,7 @@ function FilterItem(props: Props) { }; return ( -
+
{!isFilter && (
@@ -102,10 +103,8 @@ function FilterItem(props: Props) {
)}
-
-
- -
+
+
); diff --git a/frontend/app/components/shared/Filters/FilterList/FilterList.tsx b/frontend/app/components/shared/Filters/FilterList/FilterList.tsx index f6cbbec94..afdd02823 100644 --- a/frontend/app/components/shared/Filters/FilterList/FilterList.tsx +++ b/frontend/app/components/shared/Filters/FilterList/FilterList.tsx @@ -12,13 +12,15 @@ interface Props { hideEventsOrder?: boolean; observeChanges?: () => void; saveRequestPayloads?: boolean; + supportsEmpty?: boolean } function FilterList(props: Props) { - const { observeChanges = () => {}, filter, hideEventsOrder = false, saveRequestPayloads } = props; + const { observeChanges = () => {}, filter, hideEventsOrder = false, saveRequestPayloads, supportsEmpty = true } = props; const filters = List(filter.filters); const hasEvents = filters.filter((i: any) => i.isEvent).size > 0; const hasFilters = filters.filter((i: any) => !i.isEvent).size > 0; let rowIndex = 0; + const cannotDeleteFilter = filters.size === 1 && !supportsEmpty; useEffect(observeChanges, [filters]); @@ -69,6 +71,7 @@ function FilterList(props: Props) { onUpdate={(filter) => props.onUpdateFilter(filterIndex, filter)} onRemoveFilter={() => onRemoveFilter(filterIndex)} saveRequestPayloads={saveRequestPayloads} + disableDelete={cannotDeleteFilter} /> ) : null )} From 221e605cc476af8768d6a000211182c5c3c80b4a Mon Sep 17 00:00:00 2001 From: sylenien Date: Fri, 6 Jan 2023 13:15:53 +0100 Subject: [PATCH 02/15] fix(ui): fix default player data --- frontend/app/player/web/WebPlayer.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frontend/app/player/web/WebPlayer.ts b/frontend/app/player/web/WebPlayer.ts index 089ad94c5..e27432daf 100644 --- a/frontend/app/player/web/WebPlayer.ts +++ b/frontend/app/player/web/WebPlayer.ts @@ -26,19 +26,18 @@ export default class WebPlayer extends Player { private targetMarker: TargetMarker constructor(protected wpState: Store, session: any, live: boolean) { - console.log(session.events, session.stackEvents, session.resources, session.errors) let initialLists = live ? {} : { - event: session.events, + event: session.events || [], stack: session.stackEvents || [], resource: session.resources || [], // MBTODO: put ResourceTiming in file - exceptions: session.errors.map(({ time, errorId, name }: any) => + exceptions: session.errors?.map(({ time, errorId, name }: any) => Log({ level: LogLevel.ERROR, value: name, time, errorId, }) - ), + ) || [], } const screen = new Screen(session.isMobile) From a30da5ef77f96a77ad149c9c6b57fdbb220983a7 Mon Sep 17 00:00:00 2001 From: sylenien Date: Fri, 6 Jan 2023 14:55:57 +0100 Subject: [PATCH 03/15] fix(ui): fix alert modal --- .../components/shared/AlertTriggersModal/AlertTriggersModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/shared/AlertTriggersModal/AlertTriggersModal.tsx b/frontend/app/components/shared/AlertTriggersModal/AlertTriggersModal.tsx index b76666962..594fa0f8b 100644 --- a/frontend/app/components/shared/AlertTriggersModal/AlertTriggersModal.tsx +++ b/frontend/app/components/shared/AlertTriggersModal/AlertTriggersModal.tsx @@ -31,7 +31,7 @@ function AlertTriggersModal(props: Props) { }, []) return useObserver(() => ( -
+
Alerts
{ count > 0 && ( From a94d5f49046efca9ae8eae0a82bcba23e6649fb7 Mon Sep 17 00:00:00 2001 From: sylenien Date: Fri, 6 Jan 2023 17:48:39 +0100 Subject: [PATCH 04/15] fix(ui): fix method context --- frontend/app/api_middleware.js | 4 ++-- frontend/app/store.js | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/app/api_middleware.js b/frontend/app/api_middleware.js index 2b4fd4b4a..cf866f298 100644 --- a/frontend/app/api_middleware.js +++ b/frontend/app/api_middleware.js @@ -1,6 +1,6 @@ import logger from 'App/logger'; import APIClient from './api_client'; -import { UPDATE_JWT } from './duck/user'; +import { LOGIN, UPDATE_JWT } from './duck/user'; export default () => (next) => (action) => { const { types, call, ...rest } = action; @@ -14,7 +14,7 @@ export default () => (next) => (action) => { return call(client) .then(async (response) => { if (response.status === 403) { - next({ type: UPDATE_JWT, data: null }); + next({ type: LOGIN.FAILURE, data: null }); } if (!response.ok) { const text = await response.text(); diff --git a/frontend/app/store.js b/frontend/app/store.js index a2379496c..54b429ada 100644 --- a/frontend/app/store.js +++ b/frontend/app/store.js @@ -6,6 +6,9 @@ import apiMiddleware from './api_middleware'; import LocalStorage from './local_storage'; import { initialState as initUserState, UPDATE_JWT } from './duck/user' +// TODO @remove after few days +localStorage.removeItem('jwt') + const storage = new LocalStorage({ user: Object, }); From 9e7486c6b7309b7bc689253601cc39cb8f4ae846 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 6 Jan 2023 18:04:26 +0100 Subject: [PATCH 05/15] change(ui) - clickmaps disable first event selection and delete btn, also restrict the filters --- .../components/FilterSeries/FilterSeries.tsx | 4 ++ .../components/WidgetForm/WidgetForm.tsx | 6 +- .../shared/Filters/FilterItem/FilterItem.tsx | 7 ++- .../shared/Filters/FilterList/FilterList.tsx | 7 ++- .../Filters/FilterModal/FilterModal.tsx | 25 ++++++++- .../FilterSelection/FilterSelection.tsx | 56 ++++++++++++------- frontend/app/types/filter/newFilter.js | 2 + 7 files changed, 78 insertions(+), 29 deletions(-) diff --git a/frontend/app/components/Dashboard/components/FilterSeries/FilterSeries.tsx b/frontend/app/components/Dashboard/components/FilterSeries/FilterSeries.tsx index c0de60d2d..49f3659da 100644 --- a/frontend/app/components/Dashboard/components/FilterSeries/FilterSeries.tsx +++ b/frontend/app/components/Dashboard/components/FilterSeries/FilterSeries.tsx @@ -15,6 +15,7 @@ interface Props { hideHeader?: boolean; emptyMessage?: any; observeChanges?: () => void; + excludeFilterKeys?: Array } function FilterSeries(props: Props) { @@ -25,6 +26,7 @@ function FilterSeries(props: Props) { hideHeader = false, emptyMessage = 'Add user event or filter to define the series by clicking Add Step.', supportsEmpty = true, + excludeFilterKeys = [] } = props; const [expanded, setExpanded] = useState(true) const { series, seriesIndex } = props; @@ -76,6 +78,7 @@ function FilterSeries(props: Props) { onRemoveFilter={onRemoveFilter} onChangeEventsOrder={onChangeEventsOrder} supportsEmpty={supportsEmpty} + excludeFilterKeys={excludeFilterKeys} /> ) : (
{emptyMessage}
@@ -86,6 +89,7 @@ function FilterSeries(props: Props) { diff --git a/frontend/app/components/Dashboard/components/WidgetForm/WidgetForm.tsx b/frontend/app/components/Dashboard/components/WidgetForm/WidgetForm.tsx index a24db7391..d9eaf0e65 100644 --- a/frontend/app/components/Dashboard/components/WidgetForm/WidgetForm.tsx +++ b/frontend/app/components/Dashboard/components/WidgetForm/WidgetForm.tsx @@ -19,9 +19,8 @@ import { PERFORMANCE, WEB_VITALS, } from 'App/constants/card'; -import { clickmapFilter } from 'App/types/filter/newFilter'; +import { clickmapFilter, eventKeys } from 'App/types/filter/newFilter'; import { renderClickmapThumbnail } from './renderMap'; - interface Props { history: any; match: any; @@ -51,6 +50,8 @@ function WidgetForm(props: Props) { metric.metricType ); + const excludeFilterKeys = isClickmap ? eventKeys : [] + const writeOption = ({ value, name }: { value: any; name: any }) => { value = Array.isArray(value) ? value : value.value; const obj: any = { [name]: value }; @@ -202,6 +203,7 @@ function WidgetForm(props: Props) {
metric.updateKey('hasChanged', true)} hideHeader={isTable || isClickmap} seriesIndex={index} diff --git a/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx b/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx index c76cc7938..854a15b75 100644 --- a/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx +++ b/frontend/app/components/shared/Filters/FilterItem/FilterItem.tsx @@ -14,10 +14,11 @@ interface Props { onRemoveFilter: () => void; isFilter?: boolean; saveRequestPayloads?: boolean; - disableDelete?: boolean + disableDelete?: boolean; + excludeFilterKeys?: Array; } function FilterItem(props: Props) { - const { isFilter = false, filterIndex, filter, saveRequestPayloads, disableDelete = false } = props; + const { isFilter = false, filterIndex, filter, saveRequestPayloads, disableDelete = false, excludeFilterKeys = [] } = props; const canShowValues = !(filter.operator === 'isAny' || filter.operator === 'onAny' || filter.operator === 'isUndefined'); const isSubFilter = filter.type === FilterType.SUB_FILTERS; @@ -57,7 +58,7 @@ function FilterItem(props: Props) { {filterIndex + 1}
)} - + {/* Filter with Source */} {filter.hasSource && ( diff --git a/frontend/app/components/shared/Filters/FilterList/FilterList.tsx b/frontend/app/components/shared/Filters/FilterList/FilterList.tsx index afdd02823..521a70eeb 100644 --- a/frontend/app/components/shared/Filters/FilterList/FilterList.tsx +++ b/frontend/app/components/shared/Filters/FilterList/FilterList.tsx @@ -13,14 +13,15 @@ interface Props { observeChanges?: () => void; saveRequestPayloads?: boolean; supportsEmpty?: boolean + excludeFilterKeys?: Array } function FilterList(props: Props) { - const { observeChanges = () => {}, filter, hideEventsOrder = false, saveRequestPayloads, supportsEmpty = true } = props; + const { observeChanges = () => {}, filter, hideEventsOrder = false, saveRequestPayloads, supportsEmpty = true, excludeFilterKeys = [] } = props; const filters = List(filter.filters); const hasEvents = filters.filter((i: any) => i.isEvent).size > 0; const hasFilters = filters.filter((i: any) => !i.isEvent).size > 0; let rowIndex = 0; - const cannotDeleteFilter = filters.size === 1 && !supportsEmpty; + const cannotDeleteFilter = hasEvents && !supportsEmpty; useEffect(observeChanges, [filters]); @@ -72,6 +73,7 @@ function FilterList(props: Props) { onRemoveFilter={() => onRemoveFilter(filterIndex)} saveRequestPayloads={saveRequestPayloads} disableDelete={cannotDeleteFilter} + excludeFilterKeys={excludeFilterKeys} /> ) : null )} @@ -92,6 +94,7 @@ function FilterList(props: Props) { filter={filter} onUpdate={(filter) => props.onUpdateFilter(filterIndex, filter)} onRemoveFilter={() => onRemoveFilter(filterIndex)} + excludeFilterKeys={excludeFilterKeys} /> ) : null )} diff --git a/frontend/app/components/shared/Filters/FilterModal/FilterModal.tsx b/frontend/app/components/shared/Filters/FilterModal/FilterModal.tsx index 5106dd8c6..fa259b9f6 100644 --- a/frontend/app/components/shared/Filters/FilterModal/FilterModal.tsx +++ b/frontend/app/components/shared/Filters/FilterModal/FilterModal.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { Icon, Loader } from 'UI'; import { connect } from 'react-redux'; import cn from 'classnames'; @@ -6,10 +6,27 @@ import stl from './FilterModal.module.css'; import { filtersMap } from 'Types/filter/newFilter'; import AnimatedSVG, { ICONS } from 'Shared/AnimatedSVG/AnimatedSVG'; +function filterJson( + jsonObj: Record, + excludeKeys: string[] = [] +): Record { + let filtered: Record = {}; + + for (const key in jsonObj) { + const arr = jsonObj[key].filter((i: any) => !excludeKeys.includes(i.key)); + if (arr.length) { + filtered[key] = arr; + } + } + + return filtered; +} + export const getMatchingEntries = (searchQuery: string, filters: Record) => { const matchingCategories: string[] = []; const matchingFilters: Record = {}; const lowerCaseQuery = searchQuery.toLowerCase(); + if (lowerCaseQuery.length === 0) return { matchingCategories: Object.keys(filters), matchingFilters: filters, @@ -33,12 +50,13 @@ export const getMatchingEntries = (searchQuery: string, filters: Record void, + onFilterClick?: (filter: any) => void, filterSearchList: any, // metaOptions: any, isMainSearch?: boolean, fetchingFilterSearchList: boolean, searchQuery?: string, + excludeFilterKeys?: Array } function FilterModal(props: Props) { const { @@ -48,6 +66,7 @@ function FilterModal(props: Props) { isMainSearch = false, fetchingFilterSearchList, searchQuery = '', + excludeFilterKeys = [] } = props; const showSearchList = isMainSearch && searchQuery.length > 0; @@ -57,7 +76,7 @@ function FilterModal(props: Props) { onFilterClick(_filter); } - const { matchingCategories, matchingFilters } = getMatchingEntries(searchQuery, filters); + const { matchingCategories, matchingFilters } = getMatchingEntries(searchQuery, filterJson(filters, excludeFilterKeys)); const isResultEmpty = (!filterSearchList || Object.keys(filterSearchList).length === 0) && matchingCategories.length === 0 && Object.keys(matchingFilters).length === 0 diff --git a/frontend/app/components/shared/Filters/FilterSelection/FilterSelection.tsx b/frontend/app/components/shared/Filters/FilterSelection/FilterSelection.tsx index d4e20d322..e78904936 100644 --- a/frontend/app/components/shared/Filters/FilterSelection/FilterSelection.tsx +++ b/frontend/app/components/shared/Filters/FilterSelection/FilterSelection.tsx @@ -3,7 +3,8 @@ import FilterModal from '../FilterModal'; import OutsideClickDetectingDiv from 'Shared/OutsideClickDetectingDiv'; import { Icon } from 'UI'; import { connect } from 'react-redux'; -import { assist as assistRoute, isRoute } from "App/routes"; +import { assist as assistRoute, isRoute } from 'App/routes'; +import cn from 'classnames'; const ASSIST_ROUTE = assistRoute(); @@ -14,40 +15,54 @@ interface Props { onFilterClick: (filter: any) => void; children?: any; isLive?: boolean; + excludeFilterKeys?: Array + disabled?: boolean } function FilterSelection(props: Props) { - const { filter, onFilterClick, children } = props; + const { filter, onFilterClick, children, excludeFilterKeys = [], disabled = false } = props; const [showModal, setShowModal] = useState(false); return (
setTimeout(function() { - setShowModal(false) - }, 200)} + onClickOutside={() => + setTimeout(function () { + setShowModal(false); + }, 200) + } > - { children ? React.cloneElement(children, { onClick: (e) => { - e.stopPropagation(); - e.preventDefault(); - setShowModal(true); - }}) : ( + {children ? ( + React.cloneElement(children, { + onClick: (e) => { + e.stopPropagation(); + e.preventDefault(); + setShowModal(true); + }, + disabled: disabled + }) + ) : (
setShowModal(true)} > -
{filter.label}
+
+ {filter.label} +
- ) } + )} {showModal && (
)} @@ -55,8 +70,11 @@ function FilterSelection(props: Props) { ); } -export default connect((state: any) => ({ - filterList: state.getIn([ 'search', 'filterList' ]), - filterListLive: state.getIn([ 'search', 'filterListLive' ]), - isLive: state.getIn([ 'sessions', 'activeTab' ]).type === 'live', -}), { })(FilterSelection); \ No newline at end of file +export default connect( + (state: any) => ({ + filterList: state.getIn(['search', 'filterList']), + filterListLive: state.getIn(['search', 'filterListLive']), + isLive: state.getIn(['sessions', 'activeTab']).type === 'live', + }), + {} +)(FilterSelection); diff --git a/frontend/app/types/filter/newFilter.js b/frontend/app/types/filter/newFilter.js index 488eb0331..2f4ffe127 100644 --- a/frontend/app/types/filter/newFilter.js +++ b/frontend/app/types/filter/newFilter.js @@ -53,6 +53,8 @@ export const filters = [ { key: FilterKey.ISSUE, type: FilterType.ISSUE, category: FilterCategory.JAVASCRIPT, label: 'Issue', placeholder: 'Select an issue', operator: 'is', operatorOptions: filterOptions.getOperatorsByKeys(['is', 'isAny', 'isNot']), icon: 'filters/click', options: filterOptions.issueOptions }, ]; +export const eventKeys = filters.filter((i) => i.isEvent).map(i => i.key); + export const clickmapFilter = { key: FilterKey.LOCATION, type: FilterType.MULTIPLE, From 6be1e7b6620b222af187733401836bcd4df7c79f Mon Sep 17 00:00:00 2001 From: sylenien Date: Fri, 6 Jan 2023 18:05:42 +0100 Subject: [PATCH 06/15] fix(ui): fix jwt --- frontend/app/api_middleware.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/api_middleware.js b/frontend/app/api_middleware.js index cf866f298..9c71c6d19 100644 --- a/frontend/app/api_middleware.js +++ b/frontend/app/api_middleware.js @@ -14,7 +14,7 @@ export default () => (next) => (action) => { return call(client) .then(async (response) => { if (response.status === 403) { - next({ type: LOGIN.FAILURE, data: null }); + next({ type: LOGIN.FAILURE }); } if (!response.ok) { const text = await response.text(); From 8c00dfa1b6c6d06908a7a8738b431af1c44037de Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 5 Jan 2023 00:17:45 +0100 Subject: [PATCH 07/15] chore(helm): Disable linkerd injection for migration job Signed-off-by: rjshrjndrn --- scripts/helmcharts/openreplay/templates/job.yaml | 5 +++++ scripts/helmcharts/openreplay/values.yaml | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index aaadc2eb7..a30cca848 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -26,6 +26,11 @@ spec: template: metadata: name: postgresqlMigrate + {{- with .Values.migrationJob.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: spec: initContainers: - name: git diff --git a/scripts/helmcharts/openreplay/values.yaml b/scripts/helmcharts/openreplay/values.yaml index 67cf00405..cf76cb2d3 100644 --- a/scripts/helmcharts/openreplay/values.yaml +++ b/scripts/helmcharts/openreplay/values.yaml @@ -1,3 +1,7 @@ +migrationJob: + podAnnotations: + linkerd.io/inject: disabled + redis: &redis tls: enabled: false @@ -5,6 +9,10 @@ redis: &redis ingress-nginx: enabled: true controller: + admissionWebhooks: + patch: + podAnnotations: + linkerd.io/inject: disabled name: controller image: registry: k8s.gcr.io From cfce1eb84eda3635aa6efdca1af3418a2cb249d0 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 5 Jan 2023 00:29:31 +0100 Subject: [PATCH 08/15] chore(helm): Disabling linkerd for cronjobs --- .../openreplay/charts/utilities/templates/efs-cron.yaml | 5 +++++ .../openreplay/charts/utilities/templates/report-cron.yaml | 5 +++++ .../charts/utilities/templates/sessions-cleaner-cron.yaml | 5 +++++ .../charts/utilities/templates/telemetry-cron.yaml | 5 +++++ scripts/helmcharts/openreplay/charts/utilities/values.yaml | 3 +++ scripts/helmcharts/openreplay/templates/job.yaml | 1 - 6 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/efs-cron.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/efs-cron.yaml index e88aabe80..ea9413538 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/efs-cron.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/efs-cron.yaml @@ -11,6 +11,11 @@ spec: spec: backoffLimit: 0 # Don't restart the failed jobs template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 12 }} + {{- end }} spec: restartPolicy: Never containers: diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/report-cron.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/report-cron.yaml index 17fa52720..79b88f520 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/report-cron.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/report-cron.yaml @@ -12,6 +12,11 @@ spec: spec: backoffLimit: 0 # Don't restart the failed jobs template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 12 }} + {{- end }} spec: restartPolicy: Never containers: diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/sessions-cleaner-cron.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/sessions-cleaner-cron.yaml index 49bc8cfed..c6fcea0e6 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/sessions-cleaner-cron.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/sessions-cleaner-cron.yaml @@ -12,6 +12,11 @@ spec: spec: backoffLimit: 0 # Don't restart the failed jobs template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 12 }} + {{- end }} spec: restartPolicy: Never containers: diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/telemetry-cron.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/telemetry-cron.yaml index b9044664f..890bedf9a 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/telemetry-cron.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/telemetry-cron.yaml @@ -12,6 +12,11 @@ spec: spec: backoffLimit: 0 # Don't restart the failed jobs template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 12 }} + {{- end }} spec: restartPolicy: Never containers: diff --git a/scripts/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/utilities/values.yaml index 2076f8349..feba1be7c 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/values.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/values.yaml @@ -85,6 +85,9 @@ cron: "5 3 */3 * *" # Pod configurations +podAnnotations: + linkerd.io/inject: disabled + securityContext: runAsUser: 1001 runAsGroup: 1001 diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index a30cca848..ba406fa7a 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -30,7 +30,6 @@ spec: annotations: {{- toYaml . | nindent 8 }} {{- end }} - annotations: spec: initContainers: - name: git From 3e50efd6737d0639de40bb2eb0f63e8d8016d5f1 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 10 Jan 2023 16:06:43 +0100 Subject: [PATCH 09/15] change(ui) - fadeout inactive rows --- frontend/app/components/Session_/TimeTable/timeTable.module.css | 2 +- frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/Session_/TimeTable/timeTable.module.css b/frontend/app/components/Session_/TimeTable/timeTable.module.css index c2412ff8d..175958f21 100644 --- a/frontend/app/components/Session_/TimeTable/timeTable.module.css +++ b/frontend/app/components/Session_/TimeTable/timeTable.module.css @@ -108,5 +108,5 @@ $offset: 10px; } .inactiveRow { - opacity: 0.5; + opacity: 0.4; } \ No newline at end of file diff --git a/frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx b/frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx index bbb2b204e..06cfa2d5c 100644 --- a/frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx +++ b/frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx @@ -214,7 +214,7 @@ export default class TimeTable extends React.PureComponent { 'error color-red': !!row.isRed && row.isRed(), 'cursor-pointer': typeof onRowClick === 'function', [stl.activeRow]: activeIndex === index, - // [stl.inactiveRow]: !activeIndex || index > activeIndex, + [stl.inactiveRow]: !activeIndex || index > activeIndex, } )} onClick={typeof onRowClick === 'function' ? () => onRowClick(row, index) : undefined} From 2ec63b159194e2e58514c42bf06d02607d809b2f Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 10 Jan 2023 16:55:50 +0100 Subject: [PATCH 10/15] fix(ui): fix incorrect login error --- frontend/app/components/Login/Login.js | 6 +++--- frontend/app/duck/user.js | 15 +++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/frontend/app/components/Login/Login.js b/frontend/app/components/Login/Login.js index 70ee0f74e..06a2a8dd7 100644 --- a/frontend/app/components/Login/Login.js +++ b/frontend/app/components/Login/Login.js @@ -122,15 +122,15 @@ export default class Login extends React.Component {
- { errors && -
+ { errors.length ? + (
{ errors.map(error => (
{ error }
)) } -
+
) : null } {/*
*/} diff --git a/frontend/app/duck/user.js b/frontend/app/duck/user.js index f28fb7ed0..5814f082c 100644 --- a/frontend/app/duck/user.js +++ b/frontend/app/duck/user.js @@ -28,7 +28,11 @@ export const initialState = Map({ authDetails: {}, onboarding: false, sites: List(), - jwt: null + jwt: null, + loginRequest: { + loading: false, + errors: [] + }, }); const setClient = (state, data) => { @@ -50,10 +54,12 @@ const reducer = (state = initialState, action = {}) => { switch (action.type) { case UPDATE_JWT: return state.set('jwt', action.data); + case LOGIN.REQUEST: + return state.set('loginRequest', { loading: true, errors: [] }) case RESET_PASSWORD.SUCCESS: case UPDATE_PASSWORD.SUCCESS: case LOGIN.SUCCESS: - state.set('account', Account({...action.data.user })) + state.set('account', Account({...action.data.user })).set('loginRequest', { loading: false, errors: [] }) case SIGNUP.SUCCESS: state.set('account', Account(action.data.user)).set('onboarding', true); case REQUEST_RESET_PASSWORD.SUCCESS: @@ -67,6 +73,8 @@ const reducer = (state = initialState, action = {}) => { return state.set('passwordErrors', List(action.errors)) case FETCH_ACCOUNT.FAILURE: case LOGIN.FAILURE: + deleteCookie('jwt', '/', 'openreplay.com') + return state.set('loginRequest', { loading: false, errors: ['Invalid username or password'] }); case DELETE.SUCCESS: case DELETE.FAILURE: deleteCookie('jwt', '/', 'openreplay.com') @@ -86,7 +94,6 @@ const reducer = (state = initialState, action = {}) => { export default withRequestState({ - loginRequest: LOGIN, signupRequest: SIGNUP, updatePasswordRequest: UPDATE_PASSWORD, requestResetPassowrd: REQUEST_RESET_PASSWORD, @@ -96,7 +103,7 @@ export default withRequestState({ updateAccountRequest: UPDATE_ACCOUNT, }, reducer); -export const login = params => dispatch => dispatch({ +export const login = params => ({ types: LOGIN.toArray(), call: client => client.post('/login', params), }); From 269ea69206d302646eb0fc5c09e0cee256bd0c9f Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Tue, 10 Jan 2023 17:28:37 +0100 Subject: [PATCH 11/15] fix(ui): fix incorrect login error --- frontend/app/duck/user.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/duck/user.js b/frontend/app/duck/user.js index 5814f082c..fc627e214 100644 --- a/frontend/app/duck/user.js +++ b/frontend/app/duck/user.js @@ -71,10 +71,10 @@ const reducer = (state = initialState, action = {}) => { return state.set('authDetails', action.data); case UPDATE_PASSWORD.FAILURE: return state.set('passwordErrors', List(action.errors)) - case FETCH_ACCOUNT.FAILURE: case LOGIN.FAILURE: deleteCookie('jwt', '/', 'openreplay.com') return state.set('loginRequest', { loading: false, errors: ['Invalid username or password'] }); + case FETCH_ACCOUNT.FAILURE: case DELETE.SUCCESS: case DELETE.FAILURE: deleteCookie('jwt', '/', 'openreplay.com') From 44f9e4c1203055524a8b56e72a2acf181dd4d7e6 Mon Sep 17 00:00:00 2001 From: Alexander Date: Wed, 11 Jan 2023 16:23:10 +0100 Subject: [PATCH 12/15] [Storage] added sort operation for session messages (#921) * feat(backend): added sort operation for session messages --- backend/internal/config/storage/config.go | 1 + backend/internal/storage/storage.go | 45 +++++++++++++- backend/pkg/messages/bytes.go | 11 ++++ backend/pkg/messages/session-iterator.go | 71 +++++++++++++++++++++++ 4 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 backend/pkg/messages/session-iterator.go diff --git a/backend/internal/config/storage/config.go b/backend/internal/config/storage/config.go index 63c595f62..ca4ff8028 100644 --- a/backend/internal/config/storage/config.go +++ b/backend/internal/config/storage/config.go @@ -21,6 +21,7 @@ type Config struct { ProducerCloseTimeout int `env:"PRODUCER_CLOSE_TIMEOUT,default=15000"` UseFailover bool `env:"USE_FAILOVER,default=false"` MaxFileSize int64 `env:"MAX_FILE_SIZE,default=524288000"` + UseSort bool `env:"USE_SESSION_SORT,default=true"` UseProfiler bool `env:"PROFILER_ENABLED,default=false"` } diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go index 3b315561d..fbe9e2228 100644 --- a/backend/internal/storage/storage.go +++ b/backend/internal/storage/storage.go @@ -42,6 +42,8 @@ type Storage struct { sessionDEVSize syncfloat64.Histogram readingDOMTime syncfloat64.Histogram readingDEVTime syncfloat64.Histogram + sortingDOMTime syncfloat64.Histogram + sortingDEVTime syncfloat64.Histogram archivingDOMTime syncfloat64.Histogram archivingDEVTime syncfloat64.Histogram uploadingDOMTime syncfloat64.Histogram @@ -79,6 +81,14 @@ func New(cfg *config.Config, s3 *storage.S3, metrics *monitoring.Metrics) (*Stor if err != nil { log.Printf("can't create reading_duration metric: %s", err) } + sortingDOMTime, err := metrics.RegisterHistogram("sorting_duration") + if err != nil { + log.Printf("can't create reading_duration metric: %s", err) + } + sortingDEVTime, err := metrics.RegisterHistogram("sorting_dt_duration") + if err != nil { + log.Printf("can't create reading_duration metric: %s", err) + } archivingDOMTime, err := metrics.RegisterHistogram("archiving_duration") if err != nil { log.Printf("can't create archiving_duration metric: %s", err) @@ -104,6 +114,8 @@ func New(cfg *config.Config, s3 *storage.S3, metrics *monitoring.Metrics) (*Stor sessionDEVSize: sessionDevtoolsSize, readingDOMTime: readingDOMTime, readingDEVTime: readingDEVTime, + sortingDOMTime: sortingDOMTime, + sortingDEVTime: sortingDEVTime, archivingDOMTime: archivingDOMTime, archivingDEVTime: archivingDEVTime, uploadingDOMTime: uploadingDOMTime, @@ -156,14 +168,41 @@ func (s *Storage) Upload(msg *messages.SessionEnd) (err error) { return nil } -func (s *Storage) openSession(filePath string) ([]byte, error) { +func (s *Storage) openSession(filePath string, tp FileType) ([]byte, error) { // Check file size before download into memory info, err := os.Stat(filePath) if err == nil && info.Size() > s.cfg.MaxFileSize { return nil, fmt.Errorf("big file, size: %d", info.Size()) } // Read file into memory - return os.ReadFile(filePath) + raw, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + if !s.cfg.UseSort { + return raw, nil + } + start := time.Now() + res, err := s.sortSessionMessages(raw) + if err != nil { + return nil, fmt.Errorf("can't sort session, err: %s", err) + } + if tp == DOM { + s.sortingDOMTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds())) + } else { + s.sortingDEVTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds())) + } + return res, nil +} + +func (s *Storage) sortSessionMessages(raw []byte) ([]byte, error) { + // Parse messages, sort by index and save result into slice of bytes + unsortedMessages, err := messages.SplitMessages(raw) + if err != nil { + log.Printf("can't sort session, err: %s", err) + return raw, nil + } + return messages.MergeMessages(raw, messages.SortMessages(unsortedMessages)), nil } func (s *Storage) prepareSession(path string, tp FileType, task *Task) error { @@ -172,7 +211,7 @@ func (s *Storage) prepareSession(path string, tp FileType, task *Task) error { path += "devtools" } startRead := time.Now() - mob, err := s.openSession(path) + mob, err := s.openSession(path, tp) if err != nil { return err } diff --git a/backend/pkg/messages/bytes.go b/backend/pkg/messages/bytes.go index 0576201ea..00d161d97 100644 --- a/backend/pkg/messages/bytes.go +++ b/backend/pkg/messages/bytes.go @@ -1,6 +1,7 @@ package messages import ( + "encoding/binary" "errors" "fmt" "io" @@ -13,6 +14,7 @@ type BytesReader interface { ReadInt() (int64, error) ReadBoolean() (bool, error) ReadString() (string, error) + ReadIndex() (uint64, error) Data() []byte Pointer() int64 SetPointer(p int64) @@ -106,6 +108,15 @@ func (m *bytesReaderImpl) ReadString() (string, error) { return str, nil } +func (m *bytesReaderImpl) ReadIndex() (uint64, error) { + if len(m.data)-int(m.curr) < 8 { + return 0, fmt.Errorf("out of range") + } + size := binary.LittleEndian.Uint64(m.data[m.curr : m.curr+8]) + m.curr += 8 + return size, nil +} + func (m *bytesReaderImpl) Data() []byte { return m.data } diff --git a/backend/pkg/messages/session-iterator.go b/backend/pkg/messages/session-iterator.go new file mode 100644 index 000000000..45daae4b8 --- /dev/null +++ b/backend/pkg/messages/session-iterator.go @@ -0,0 +1,71 @@ +package messages + +import ( + "bytes" + "fmt" + "io" + "log" + "sort" +) + +type msgInfo struct { + index uint64 + start int64 + end int64 +} + +func SplitMessages(data []byte) ([]*msgInfo, error) { + messages := make([]*msgInfo, 0) + reader := NewBytesReader(data) + for { + // Get message start + msgStart := reader.Pointer() + if int(msgStart) >= len(data) { + return messages, nil + } + + // Read message index + msgIndex, err := reader.ReadIndex() + if err != nil { + if err != io.EOF { + log.Println(reader.Pointer(), msgStart) + return nil, fmt.Errorf("read message index err: %s", err) + } + return messages, nil + } + + // Read message type + msgType, err := reader.ReadUint() + if err != nil { + return nil, fmt.Errorf("read message type err: %s", err) + } + + // Read message body + _, err = ReadMessage(msgType, reader) + if err != nil { + return nil, fmt.Errorf("read message body err: %s", err) + } + + // Add new message info to messages slice + messages = append(messages, &msgInfo{ + index: msgIndex, + start: msgStart, + end: reader.Pointer(), + }) + } +} + +func SortMessages(messages []*msgInfo) []*msgInfo { + sort.SliceStable(messages, func(i, j int) bool { + return messages[i].index < messages[j].index + }) + return messages +} + +func MergeMessages(data []byte, messages []*msgInfo) []byte { + sortedSession := bytes.NewBuffer(make([]byte, 0, len(data))) + for _, info := range messages { + sortedSession.Write(data[info.start:info.end]) + } + return sortedSession.Bytes() +} From 6972f842756012f699159aab0c765468d9eca00c Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 11 Jan 2023 16:52:32 +0100 Subject: [PATCH 13/15] feat(backend): added batch and compression configuration for kafka producer --- backend/Dockerfile | 4 ++-- ee/backend/pkg/kafka/producer.go | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 043de51cd..3705600ff 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -76,8 +76,8 @@ ENV TZ=UTC \ USE_FAILOVER=false \ GROUP_STORAGE_FAILOVER=failover \ TOPIC_STORAGE_FAILOVER=storage-failover \ - PROFILER_ENABLED=false - + PROFILER_ENABLED=false \ + COMPRESSION_TYPE=zstd ARG SERVICE_NAME diff --git a/ee/backend/pkg/kafka/producer.go b/ee/backend/pkg/kafka/producer.go index 1ec241b8a..d3bc2cf51 100644 --- a/ee/backend/pkg/kafka/producer.go +++ b/ee/backend/pkg/kafka/producer.go @@ -15,13 +15,20 @@ type Producer struct { func NewProducer(messageSizeLimit int, useBatch bool) *Producer { kafkaConfig := &kafka.ConfigMap{ - "enable.idempotence": true, - "bootstrap.servers": env.String("KAFKA_SERVERS"), - "go.delivery.reports": true, - "security.protocol": "plaintext", - "go.batch.producer": useBatch, - "queue.buffering.max.ms": 100, - "message.max.bytes": messageSizeLimit, + "enable.idempotence": true, + "bootstrap.servers": env.String("KAFKA_SERVERS"), + "go.delivery.reports": true, + "security.protocol": "plaintext", + "go.batch.producer": useBatch, + "message.max.bytes": messageSizeLimit, // should be synced with broker config + "linger.ms": 1000, + "queue.buffering.max.ms": 1000, + "batch.num.messages": 1000, + "queue.buffering.max.messages": 1000, + "retries": 3, + "retry.backoff.ms": 100, + "max.in.flight.requests.per.connection": 1, + "compression.type": env.String("COMPRESSION_TYPE"), } // Apply ssl configuration if env.Bool("KAFKA_USE_SSL") { From d88896d649de2b0ceef11540784a52a108e48c5c Mon Sep 17 00:00:00 2001 From: Alexander Zavorotynskiy Date: Wed, 11 Jan 2023 17:53:36 +0100 Subject: [PATCH 14/15] feat(backend): added 503 error for assets retrier --- backend/internal/assets/cacher/cacher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/internal/assets/cacher/cacher.go b/backend/internal/assets/cacher/cacher.go index b56c97d74..8bbee092f 100644 --- a/backend/internal/assets/cacher/cacher.go +++ b/backend/internal/assets/cacher/cacher.go @@ -90,8 +90,8 @@ func (c *cacher) cacheURL(t *Task) { defer res.Body.Close() if res.StatusCode >= 400 { printErr := true - // Retry 403 error - if res.StatusCode == 403 && t.retries > 0 { + // Retry 403/503 errors + if (res.StatusCode == 403 || res.StatusCode == 503) && t.retries > 0 { c.workers.AddTask(t) printErr = false } From 572971b7fced36303dc739c8488ed66c5290827f Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Thu, 12 Jan 2023 12:09:20 +0100 Subject: [PATCH 15/15] Update Dockerfile --- backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 3705600ff..ad60c1b75 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,7 +27,7 @@ RUN adduser -u 1001 openreplay -D ENV TZ=UTC \ GIT_SHA=$GIT_SHA \ - FS_ULIMIT=1000 \ + FS_ULIMIT=10000 \ FS_DIR=/mnt/efs \ MAXMINDDB_FILE=/home/openreplay/geoip.mmdb \ UAPARSER_FILE=/home/openreplay/regexes.yaml \