From 96b5e2e0cce68ee55326b9c5c81bc779b48db11f Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 19 May 2025 17:11:42 +0200 Subject: [PATCH] feat(product_analytics): table of cards fixes --- backend/pkg/analytics/charts/metric_table.go | 61 +++++++++++++------- backend/pkg/analytics/charts/model.go | 16 ++--- backend/pkg/analytics/charts/query.go | 51 ++++++++++++++++ 3 files changed, 98 insertions(+), 30 deletions(-) diff --git a/backend/pkg/analytics/charts/metric_table.go b/backend/pkg/analytics/charts/metric_table.go index b1313d731..2f011a79e 100644 --- a/backend/pkg/analytics/charts/metric_table.go +++ b/backend/pkg/analytics/charts/metric_table.go @@ -37,18 +37,20 @@ const ( ) var propertySelectorMap = map[string]string{ - string(MetricOfTableBrowser): "main.$browser AS metric_value", - string(MetricOfTableDevice): "main.$device AS metric_value", - string(MetricOfTableCountry): "main.$country AS metric_value", + string(MetricOfTableLocation): "JSONExtractString(toString(main.$properties), 'url_path') AS metric_value", + //string(MetricOfTableUserId): "if(empty(sessions.user_id), 'Anonymous', sessions.user_id) AS metric_value", + string(MetricOfTableUserId): "if(empty(sessions.user_id) OR sessions.user_id IS NULL, 'Anonymous', sessions.user_id) AS metric_value", + string(MetricOfTableBrowser): "main.$browser AS metric_value", + //string(MetricOfTableDevice): "sessions.user_device AS metric_value", + string(MetricOfTableDevice): "if(empty(sessions.user_device) OR sessions.user_device IS NULL, 'Undefined', sessions.user_device) AS metric_value", + string(MetricOfTableCountry): "toString(sessions.user_country) AS metric_value", string(MetricOfTableReferrer): "main.$referrer AS metric_value", + string(MetricOfTableFetch): "JSONExtractString(toString(main.$properties), 'url_path') AS metric_value", } var mainColumns = map[string]string{ "userBrowser": "$browser", - "userDevice": "$device_type", - "userCountry": "$country", "referrer": "$referrer", - // TODO add more columns if needed } func (t TableQueryBuilder) Execute(p Payload, conn db.Connector) (interface{}, error) { @@ -77,12 +79,17 @@ func (t TableQueryBuilder) Execute(p Payload, conn db.Connector) (interface{}, e } defer rows.Close() - var ( - overallTotalMetricValues uint64 - overallCount uint64 - values []TableValue - firstRow = true - ) + var overallTotalMetricValues uint64 + var overallCount uint64 + values := make([]TableValue, 0) + firstRow := true + + //var ( + // overallTotalMetricValues uint64 + // overallCount uint64 + // values []TableValue + // firstRow = true + //) for rows.Next() { var ( @@ -127,20 +134,32 @@ func (t TableQueryBuilder) buildQuery(r Payload, metricFormat string) (string, e originalMetricOf := r.MetricOf propertyName = originalMetricOf - eventFilters := s.Filter.Filters + durationConds, eventFilters := buildDurationWhere(s.Filter.Filters) eventConds, eventNames := buildEventConditions(eventFilters, BuildConditionsOptions{ DefinedColumns: mainColumns, }) - baseWhereConditions := []string{ fmt.Sprintf("main.created_at >= toDateTime(%d/1000)", r.StartTimestamp), fmt.Sprintf("main.created_at <= toDateTime(%d/1000)", r.EndTimestamp), - "sessions.duration > 0", + fmt.Sprintf("main.project_id = %d", r.ProjectId), + } + baseWhereConditions = append(baseWhereConditions, durationConds...) + + if cond := eventNameCondition("", r.MetricOf); cond != "" { + baseWhereConditions = append(baseWhereConditions, cond) } - if r.ProjectId > 0 { - baseWhereConditions = append(baseWhereConditions, fmt.Sprintf("main.project_id = %d", r.ProjectId)) - } + //baseWhereConditions := []string{ + // fmt.Sprintf("main.created_at >= toDateTime(%d/1000)", r.StartTimestamp), + // fmt.Sprintf("main.created_at <= toDateTime(%d/1000)", r.EndTimestamp), + // "sessions.duration > 0", + //} + // + + // + //if r.ProjectId > 0 { + // baseWhereConditions = append(baseWhereConditions, fmt.Sprintf("main.project_id = %d", r.ProjectId)) + //} var aggregationExpression string var aggregationAlias = "aggregation_id" @@ -154,9 +173,7 @@ func (t TableQueryBuilder) buildQuery(r Payload, metricFormat string) (string, e aggregationExpression = "main.session_id" } - var propertySelector string - var ok bool - propertySelector, ok = propertySelectorMap[originalMetricOf] + propertySelector, ok := propertySelectorMap[originalMetricOf] if !ok { propertySelector = fmt.Sprintf("JSONExtractString(toString(main.$properties), '%s') AS metric_value", propertyName) } @@ -196,7 +213,7 @@ func (t TableQueryBuilder) buildQuery(r Payload, metricFormat string) (string, e metric_value AS name, countDistinct(%s) AS value_count FROM filtered_data - WHERE name IS NOT NULL AND name != '' + -- WHERE name IS NOT NULL AND name != '' GROUP BY name ) SELECT diff --git a/backend/pkg/analytics/charts/model.go b/backend/pkg/analytics/charts/model.go index 4345ccf7a..644eeaf44 100644 --- a/backend/pkg/analytics/charts/model.go +++ b/backend/pkg/analytics/charts/model.go @@ -82,12 +82,12 @@ type MetricPayload struct { type MetricOfTable string const ( - MetricOfTableLocation MetricOfTable = "url_path" // TOP Pages - MetricOfTableBrowser MetricOfTable = "user_browser" + MetricOfTableLocation MetricOfTable = "location" // TOP Pages + MetricOfTableBrowser MetricOfTable = "userBrowser" MetricOfTableReferrer MetricOfTable = "referrer" - MetricOfTableUserId MetricOfTable = "user_id" - MetricOfTableCountry MetricOfTable = "user_country" - MetricOfTableDevice MetricOfTable = "user_device" + MetricOfTableUserId MetricOfTable = "userId" + MetricOfTableCountry MetricOfTable = "userCountry" + MetricOfTableDevice MetricOfTable = "userDevice" MetricOfTableFetch MetricOfTable = "fetch" //MetricOfTableIssues MetricOfTable = "issues" @@ -136,9 +136,9 @@ const ( // Event filters const ( - FilterClick FilterType = "click" - FilterInput FilterType = "input" - FilterLocation FilterType = "location" + FilterClick FilterType = "CLICK" + FilterInput FilterType = "INPUT" + FilterLocation FilterType = "LOCATION" FilterTag FilterType = "tag" FilterCustom FilterType = "customEvent" FilterFetch FilterType = "fetch" diff --git a/backend/pkg/analytics/charts/query.go b/backend/pkg/analytics/charts/query.go index e1d0049bb..a770c7629 100644 --- a/backend/pkg/analytics/charts/query.go +++ b/backend/pkg/analytics/charts/query.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "openreplay/backend/pkg/analytics/db" + "strconv" "strings" ) @@ -48,6 +49,7 @@ type BuildConditionsOptions struct { var propertyKeyMap = map[string]filterConfig{ "LOCATION": {LogicalProperty: "url_path"}, + "FETCH": {LogicalProperty: "url_path"}, "CLICK": {LogicalProperty: "label"}, "INPUT": {LogicalProperty: "label"}, "fetchUrl": {LogicalProperty: "url_path"}, @@ -120,6 +122,10 @@ func buildEventConditions(filters []Filter, options ...BuildConditionsOptions) ( } } for _, f := range filters { + if f.Type == FilterDuration { + continue + } + fConds, fNames := addFilter(f, opts) if len(fConds) > 0 { conds = append(conds, fConds...) @@ -415,3 +421,48 @@ func reverseLookup(m map[string]string, value string) string { } return "" } + +func eventNameCondition(table, metricOf string) string { + if table == "" { + table = "main" + } + switch metricOf { + case string(MetricOfTableFetch): + return fmt.Sprintf("%s.`$event_name` = 'REQUEST'", table) + case string(MetricOfTableLocation): + return fmt.Sprintf("%s.`$event_name` = 'LOCATION'", table) + default: + return "" + } +} + +func buildDurationWhere(filters []Filter) ([]string, []Filter) { + var conds []string + var rest []Filter + for _, f := range filters { + if string(f.Type) == "duration" { + v := f.Value + if len(v) == 1 { + if v[0] != "" { + if d, err := strconv.ParseInt(v[0], 10, 64); err == nil { + conds = append(conds, fmt.Sprintf("sessions.duration >= %d", d)) + } + } + } else if len(v) >= 2 { + if v[0] != "" { + if d, err := strconv.ParseInt(v[0], 10, 64); err == nil { + conds = append(conds, fmt.Sprintf("sessions.duration >= %d", d)) + } + } + if v[1] != "" { + if d, err := strconv.ParseInt(v[1], 10, 64); err == nil { + conds = append(conds, fmt.Sprintf("sessions.duration <= %d", d)) + } + } + } + } else { + rest = append(rest, f) + } + } + return conds, rest +}