From 3f13eeef7567ed6d99180c9fcfe2878ec92fcd15 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Mon, 20 Jan 2025 12:35:25 +0100 Subject: [PATCH 1/5] Dev (#2959) * feat(chalice): autocomplete return top 10 with stats * fix(chalice): fixed autocomplete top 10 meta-filters * feat(chalice): changed user journey --- api/chalicelib/core/metrics/product_analytics.py | 13 +++++++------ api/chalicelib/core/metrics/product_analytics_ch.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/metrics/product_analytics.py b/api/chalicelib/core/metrics/product_analytics.py index 934cd33ad..ffa4fd319 100644 --- a/api/chalicelib/core/metrics/product_analytics.py +++ b/api/chalicelib/core/metrics/product_analytics.py @@ -11,9 +11,6 @@ logger = logging.getLogger(__name__) def __transform_journey(rows, reverse_path=False): - print("---------------------------------") - print(rows) - print("---------------------------------") total_100p = 0 number_of_step1 = 0 for r in rows: @@ -32,17 +29,21 @@ def __transform_journey(rows, reverse_path=False): source = f"{r['event_number_in_session']}_{r['event_type']}_{r['e_value']}" if source not in nodes: nodes.append(source) - nodes_values.append({"name": r['e_value'], "eventType": r['event_type']}) + nodes_values.append({"depth": r['event_number_in_session'] - 1, + "name": r['e_value'], + "eventType": r['event_type']}) # if r['next_value']: target = f"{r['event_number_in_session'] + 1}_{r['next_type']}_{r['next_value']}" if target not in nodes: nodes.append(target) - nodes_values.append({"name": r['next_value'], "eventType": r['next_type']}) + nodes_values.append({"depth": r['event_number_in_session'], + "name": r['next_value'], + "eventType": r['next_type']}) sr_idx = nodes.index(source) tg_idx = nodes.index(target) - link = {"eventType": r['event_type'], "sessionsCount": r["sessions_count"],"value": r["value"]} + link = {"eventType": r['event_type'], "sessionsCount": r["sessions_count"], "value": r["value"]} if not reverse_path: link["source"] = sr_idx link["target"] = tg_idx diff --git a/api/chalicelib/core/metrics/product_analytics_ch.py b/api/chalicelib/core/metrics/product_analytics_ch.py index d4f3a4432..4c5f3334d 100644 --- a/api/chalicelib/core/metrics/product_analytics_ch.py +++ b/api/chalicelib/core/metrics/product_analytics_ch.py @@ -253,7 +253,7 @@ FROM ( SELECT others_n.event_number_in_session, 'OTHER' AS event_type, NULL AS e_value, - 'OTHERS' AS next_type, + 'OTHER' AS next_type, NULL AS next_value, SUM(sessions_count) AS sessions_count FROM others_n From 9370a7a50ea14af747ddd09b6e9c405b7aab39f5 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 20 Jan 2025 14:21:57 +0100 Subject: [PATCH 2/5] Adapt CH client for a new PA events table (#2960) * feat(db): use a new CH events schema * feat(db): added a missing columns to issue events * feat(db): correct order of the issue's arguments * feat(db): crop for url related strings * feat(db): added missing values * feat(db): moved materialized columns to json * feat(db): use the latest ch library with JSON support * feat(db): added missing duration for requests event --- backend/go.mod | 12 +- backend/go.sum | 34 +- backend/pkg/db/clickhouse/connector.go | 755 +++++++++++++++-------- backend/pkg/db/clickhouse/insert_type.go | 2 +- backend/pkg/db/types/error-event.go | 64 +- backend/pkg/messages/message.go | 4 +- 6 files changed, 579 insertions(+), 292 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 45a38903b..d0193a399 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -5,7 +5,7 @@ go 1.23 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 - github.com/ClickHouse/clickhouse-go/v2 v2.30.0 + github.com/ClickHouse/clickhouse-go/v2 v2.30.1 github.com/DataDog/datadog-api-client-go/v2 v2.34.0 github.com/Masterminds/semver v1.5.0 github.com/andybalholm/brotli v1.1.1 @@ -14,6 +14,7 @@ require ( github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/docker/distribution v2.8.3+incompatible + github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/elastic/go-elasticsearch/v8 v8.17.0 github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611 github.com/getsentry/sentry-go v0.30.0 @@ -37,12 +38,12 @@ require ( github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce github.com/ua-parser/uap-go v0.0.0-20241012191800-bbb40edc15aa go.uber.org/zap v1.27.0 - golang.org/x/net v0.33.0 + golang.org/x/net v0.34.0 ) require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/ClickHouse/ch-go v0.63.1 // indirect github.com/DataDog/zstd v1.5.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -77,12 +78,13 @@ require ( go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 // indirect google.golang.org/protobuf v1.36.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + diff --git a/backend/go.sum b/backend/go.sum index 847b27241..b67ff7905 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -19,10 +19,10 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg6 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= -github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= -github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= +github.com/ClickHouse/ch-go v0.63.1 h1:s2JyZvWLTCSAGdtjMBBmAgQQHMco6pawLJMOXi0FODM= +github.com/ClickHouse/ch-go v0.63.1/go.mod h1:I1kJJCL3WJcBMGe1m+HVK0+nREaG+JOYYBWjrDrF3R0= github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.1/go.mod h1:szk8BMoQV/NgHXZ20ZbwDyvPWmpfhRKjFkc6wzASGxM= github.com/DataDog/datadog-api-client-go/v2 v2.34.0 h1:0VVmv8uZg8vdBuEpiF2nBGUezl2QITrxdEsLgh38j8M= github.com/DataDog/datadog-api-client-go/v2 v2.34.0/go.mod h1:d3tOEgUd2kfsr9uuHQdY+nXrWp4uikgTgVCPdKNK30U= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= @@ -130,8 +130,8 @@ github.com/docker/compose/v2 v2.28.1 h1:ORPfiVHrpnRQBDoC3F8JJyWAY8N5gWuo3Fgwyivx github.com/docker/compose/v2 v2.28.1/go.mod h1:wDtGQFHe99sPLCHXeVbCkc+Wsl4Y/2ZxiAJa/nga6rA= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v27.3.0+incompatible h1:BNb1QY6o4JdKpqwi9IB+HUYcRRrVN4aGFUTvDmWYK1A= -github.com/docker/docker v27.3.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.4.1+incompatible h1:ZJvcY7gfwHn1JF48PfbyXg7Jyt9ZCWDW+GGXOIxEwp4= +github.com/docker/docker v27.4.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0= @@ -146,6 +146,8 @@ github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJ github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= +github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-elasticsearch/v8 v8.17.0 h1:e9cWksE/Fr7urDRmGPGp47Nsp4/mvNOrU8As1l2HQQ0= github.com/elastic/go-elasticsearch/v8 v8.17.0/go.mod h1:lGMlgKIbYoRvay3xWBeKahAiJOgmFDsjZC39nmO3H64= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= @@ -500,8 +502,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= github.com/testcontainers/testcontainers-go/modules/compose v0.33.0 h1:PyrUOF+zG+xrS3p+FesyVxMI+9U+7pwhZhyFozH3jKY= @@ -542,6 +544,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zavorotynskiy/clickhouse-go/v2 v2.30.1-0.20250110230205-a22b6d524deb h1:3gQycbGOJreRQkKORHjCn+wEPpnlra6Ek5TnW78z5vc= +github.com/zavorotynskiy/clickhouse-go/v2 v2.30.1-0.20250110230205-a22b6d524deb/go.mod h1:szk8BMoQV/NgHXZ20ZbwDyvPWmpfhRKjFkc6wzASGxM= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= @@ -608,8 +612,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -632,8 +636,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -666,16 +670,16 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/backend/pkg/db/clickhouse/connector.go b/backend/pkg/db/clickhouse/connector.go index 71d94ab85..279b96bab 100644 --- a/backend/pkg/db/clickhouse/connector.go +++ b/backend/pkg/db/clickhouse/connector.go @@ -1,14 +1,18 @@ package clickhouse import ( + "encoding/binary" + "encoding/json" "errors" "fmt" + "hash/fnv" "log" "strings" "time" "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/google/uuid" "openreplay/backend/internal/config/common" "openreplay/backend/pkg/db/types" @@ -22,7 +26,6 @@ type Connector interface { Prepare() error Commit() error Stop() error - // Web InsertWebSession(session *sessions.Session) error InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error InsertWebClickEvent(session *sessions.Session, msg *messages.MouseClick) error @@ -35,7 +38,6 @@ type Connector interface { InsertIssue(session *sessions.Session, msg *messages.IssueEvent) error InsertWebInputDuration(session *sessions.Session, msg *messages.InputChange) error InsertMouseThrashing(session *sessions.Session, msg *messages.MouseThrashing) error - // Mobile InsertMobileSession(session *sessions.Session) error InsertMobileCustom(session *sessions.Session, msg *messages.MobileEvent) error InsertMobileClick(session *sessions.Session, msg *messages.MobileClickEvent) error @@ -101,27 +103,25 @@ func (c *connectorImpl) newBatch(name, query string) error { } var batches = map[string]string{ - // Web - "sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?)", - "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, SUBSTR(?, 1, 8000))", - "pages": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint_time, speed_index, visually_complete, time_to_interactive, url_path, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?)", - "clicks": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, hesitation_time, event_type, selector, normalized_x, normalized_y, url, url_path) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000))", - "inputs": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, label, event_type, duration, hesitation_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - "errors": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, source, name, message, error_id, event_type, error_tags_keys, error_tags_values) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - "performance": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, min_fps, avg_fps, max_fps, min_cpu, avg_cpu, max_cpu, min_total_js_heap_size, avg_total_js_heap_size, max_total_js_heap_size, min_used_js_heap_size, avg_used_js_heap_size, max_used_js_heap_size, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - "requests": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, url, request_body, response_body, status, method, duration, success, event_type, transfer_size, url_path) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000))", - "custom": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, name, payload, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)", - "graphql": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, name, request_body, response_body, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - "issuesEvents": "INSERT INTO experimental.events (session_id, project_id, message_id, datetime, issue_id, issue_type, event_type, url, url_path) VALUES (?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000))", - "issues": "INSERT INTO experimental.issues (project_id, issue_id, type, context_string) VALUES (?, ?, ?, ?)", - //Mobile - "ios_sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, platform, timezone) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?)", - "ios_custom": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, name, payload, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)", - "ios_clicks": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, label, event_type) VALUES (?, ?, ?, ?, ?, ?)", - "ios_swipes": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, label, direction, event_type) VALUES (?, ?, ?, ?, ?, ?, ?)", - "ios_inputs": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, label, event_type) VALUES (?, ?, ?, ?, ?, ?)", - "ios_requests": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, url, request_body, response_body, status, method, duration, success, event_type) VALUES (?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?)", - "ios_crashes": "INSERT INTO experimental.ios_events (session_id, project_id, message_id, datetime, name, reason, stacktrace, event_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + "sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, timezone, utm_source, utm_medium, utm_campaign) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?, ?, ?)", + "autocompletes": "INSERT INTO experimental.autocomplete (project_id, type, value) VALUES (?, ?, SUBSTR(?, 1, 8000))", + "pages": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "clicks": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$current_url", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "inputs": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "errors": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "performance": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "requests": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$duration_s", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "custom": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "graphql": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "issuesEvents": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", issue_type, issue_id, "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "issues": "INSERT INTO experimental.issues (project_id, issue_id, type, context_string) VALUES (?, ?, ?, ?)", + "mobile_sessions": "INSERT INTO experimental.sessions (session_id, project_id, user_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, user_state, user_city, datetime, duration, pages_count, events_count, errors_count, issue_score, referrer, issue_types, tracker_version, user_browser, user_browser_version, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, platform, timezone) VALUES (?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, SUBSTR(?, 1, 8000), ?, ?, ?, ?, SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), SUBSTR(?, 1, 8000), ?, ?)", + "mobile_custom": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "mobile_clicks": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "mobile_swipes": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "mobile_inputs": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "mobile_requests": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + "mobile_crashes": `INSERT INTO product_analytics.events (session_id, project_id, event_id, "$event_name", created_at, "$time", distinct_id, "$auto_captured", "$properties") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, } func (c *connectorImpl) Prepare() error { @@ -175,96 +175,6 @@ func (c *connectorImpl) worker() { } } -func (c *connectorImpl) checkError(name string, err error) { - if err != clickhouse.ErrBatchAlreadySent { - log.Printf("can't create %s batch after failed append operation: %s", name, err) - } -} - -func (c *connectorImpl) InsertWebInputDuration(session *sessions.Session, msg *messages.InputChange) error { - if msg.Label == "" { - return nil - } - if err := c.batches["inputs"].Append( - session.SessionID, - uint16(session.ProjectID), - msg.MsgID(), - datetime(msg.Timestamp), - msg.Label, - "INPUT", - nullableUint16(uint16(msg.InputDuration)), - nullableUint32(uint32(msg.HesitationTime)), - ); err != nil { - c.checkError("inputs", err) - return fmt.Errorf("can't append to inputs batch: %s", err) - } - return nil -} - -func (c *connectorImpl) InsertMouseThrashing(session *sessions.Session, msg *messages.MouseThrashing) error { - issueID := hashid.MouseThrashingID(session.ProjectID, session.SessionID, msg.Timestamp) - // Insert issue event to batches - if err := c.batches["issuesEvents"].Append( - session.SessionID, - uint16(session.ProjectID), - msg.MsgID(), - datetime(msg.Timestamp), - issueID, - "mouse_thrashing", - "ISSUE", - msg.Url, - extractUrlPath(msg.Url), - ); err != nil { - c.checkError("issuesEvents", err) - return fmt.Errorf("can't append to issuesEvents batch: %s", err) - } - if err := c.batches["issues"].Append( - uint16(session.ProjectID), - issueID, - "mouse_thrashing", - msg.Url, - ); err != nil { - c.checkError("issues", err) - return fmt.Errorf("can't append to issues batch: %s", err) - } - return nil -} - -func (c *connectorImpl) InsertIssue(session *sessions.Session, msg *messages.IssueEvent) error { - issueID := hashid.IssueID(session.ProjectID, msg) - // Check issue type before insert to avoid panic from clickhouse lib - switch msg.Type { - case "click_rage", "dead_click", "excessive_scrolling", "bad_request", "missing_resource", "memory", "cpu", "slow_resource", "slow_page_load", "crash", "ml_cpu", "ml_memory", "ml_dead_click", "ml_click_rage", "ml_mouse_thrashing", "ml_excessive_scrolling", "ml_slow_resources", "custom", "js_exception", "mouse_thrashing", "app_crash": - default: - return fmt.Errorf("unknown issueType: %s", msg.Type) - } - // Insert issue event to batches - if err := c.batches["issuesEvents"].Append( - session.SessionID, - uint16(session.ProjectID), - msg.MessageID, - datetime(msg.Timestamp), - issueID, - msg.Type, - "ISSUE", - msg.URL, - extractUrlPath(msg.URL), - ); err != nil { - c.checkError("issuesEvents", err) - return fmt.Errorf("can't append to issuesEvents batch: %s", err) - } - if err := c.batches["issues"].Append( - uint16(session.ProjectID), - issueID, - msg.Type, - msg.ContextString, - ); err != nil { - c.checkError("issues", err) - return fmt.Errorf("can't append to issues batch: %s", err) - } - return nil -} - func (c *connectorImpl) InsertWebSession(session *sessions.Session) error { if session.Duration == nil { return errors.New("trying to insert session with nil duration") @@ -313,40 +223,217 @@ func (c *connectorImpl) InsertWebSession(session *sessions.Session) error { return nil } -func extractUrlPath(fullUrl string) string { - _, path, query, err := url.GetURLParts(fullUrl) +func (c *connectorImpl) InsertAutocomplete(session *sessions.Session, msgType, msgValue string) error { + if len(msgValue) == 0 { + return nil + } + if err := c.batches["autocompletes"].Append( + uint16(session.ProjectID), + msgType, + msgValue, + ); err != nil { + c.checkError("autocompletes", err) + return fmt.Errorf("can't append to autocompletes batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertWebInputDuration(session *sessions.Session, msg *messages.InputChange) error { + if msg.Label == "" { + return nil + } + jsonString, err := json.Marshal(map[string]interface{}{ + "label": msg.Label, + "hesitation_time": nullableUint32(uint32(msg.HesitationTime)), + }) if err != nil { - log.Printf("can't parse url: %s", err) - return "" + return fmt.Errorf("can't marshal input event: %s", err) } - pathQuery := path - if query != "" { - pathQuery += "?" + query + eventTime := datetime(msg.Timestamp) + if err := c.batches["inputs"].Append( + session.SessionID, + uint16(session.ProjectID), + getUUID(msg), + "INPUT", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + nullableUint16(uint16(msg.InputDuration)), + jsonString, + ); err != nil { + c.checkError("inputs", err) + return fmt.Errorf("can't append to inputs batch: %s", err) } - return strings.ToLower(pathQuery) + return nil +} + +func (c *connectorImpl) InsertMouseThrashing(session *sessions.Session, msg *messages.MouseThrashing) error { + issueID := hashid.MouseThrashingID(session.ProjectID, session.SessionID, msg.Timestamp) + host, path, hostpath, err := extractUrlParts(msg.Url) + if err != nil { + return fmt.Errorf("can't extract url parts: %s", err) + } + jsonString, err := json.Marshal(map[string]interface{}{ + "issue_id": issueID, + "issue_type": "mouse_thrashing", + "url": cropString(msg.Url), + "url_host": host, + "url_path": path, + "url_hostpath": hostpath, + }) + if err != nil { + return fmt.Errorf("can't marshal issue event: %s", err) + } + eventTime := datetime(msg.Timestamp) + if err := c.batches["issuesEvents"].Append( + session.SessionID, + uint16(session.ProjectID), + getUUID(msg), + "ISSUE", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + "mouse_thrashing", + issueID, + jsonString, + ); err != nil { + c.checkError("issuesEvents", err) + return fmt.Errorf("can't append to issuesEvents batch: %s", err) + } + if err := c.batches["issues"].Append( + uint16(session.ProjectID), + issueID, + "mouse_thrashing", + msg.Url, + ); err != nil { + c.checkError("issues", err) + return fmt.Errorf("can't append to issues batch: %s", err) + } + return nil +} + +func (c *connectorImpl) InsertIssue(session *sessions.Session, msg *messages.IssueEvent) error { + issueID := hashid.IssueID(session.ProjectID, msg) + // Check issue type before insert to avoid panic from clickhouse lib + switch msg.Type { + case "click_rage", "dead_click", "excessive_scrolling", "bad_request", "missing_resource", "memory", "cpu", "slow_resource", "slow_page_load", "crash", "ml_cpu", "ml_memory", "ml_dead_click", "ml_click_rage", "ml_mouse_thrashing", "ml_excessive_scrolling", "ml_slow_resources", "custom", "js_exception", "mouse_thrashing", "app_crash": + default: + return fmt.Errorf("unknown issueType: %s", msg.Type) + } + host, path, hostpath, err := extractUrlParts(msg.Url) + if err != nil { + return fmt.Errorf("can't extract url parts: %s", err) + } + jsonString, err := json.Marshal(map[string]interface{}{ + "issue_id": issueID, + "issue_type": msg.Type, + "url": cropString(msg.Url), + "url_host": host, + "url_path": path, + "url_hostpath": hostpath, + }) + if err != nil { + return fmt.Errorf("can't marshal issue event: %s", err) + } + eventTime := datetime(msg.Timestamp) + if err := c.batches["issuesEvents"].Append( + session.SessionID, + uint16(session.ProjectID), + getUUID(msg), + "ISSUE", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + msg.Type, + issueID, + jsonString, + ); err != nil { + c.checkError("issuesEvents", err) + return fmt.Errorf("can't append to issuesEvents batch: %s", err) + } + if err := c.batches["issues"].Append( + uint16(session.ProjectID), + issueID, + msg.Type, + msg.ContextString, + ); err != nil { + c.checkError("issues", err) + return fmt.Errorf("can't append to issues batch: %s", err) + } + return nil } func (c *connectorImpl) InsertWebPageEvent(session *sessions.Session, msg *messages.PageEvent) error { + host, path, hostpath, err := extractUrlParts(msg.URL) + if err != nil { + return fmt.Errorf("can't extract url parts: %s", err) + } + ttfb := nullableUint16(0) + if msg.ResponseStart >= msg.RequestStart { + ttfb = nullableUint16(uint16(msg.ResponseStart - msg.RequestStart)) + } + ttlb := nullableUint16(0) + if msg.ResponseEnd >= msg.RequestStart { + ttlb = nullableUint16(uint16(msg.ResponseEnd - msg.RequestStart)) + } + responseTime := nullableUint16(0) + if msg.ResponseEnd >= msg.ResponseStart { + responseTime = nullableUint16(uint16(msg.ResponseEnd - msg.ResponseStart)) + } + domBuildingTime := nullableUint16(0) + if msg.DomContentLoadedEventStart >= msg.ResponseEnd { + domBuildingTime = nullableUint16(uint16(msg.DomContentLoadedEventStart - msg.ResponseEnd)) + } + domContentLoadedEventTime := nullableUint16(0) + if msg.DomContentLoadedEventEnd >= msg.DomContentLoadedEventStart { + domContentLoadedEventTime = nullableUint16(uint16(msg.DomContentLoadedEventEnd - msg.DomContentLoadedEventStart)) + } + loadEventTime := nullableUint16(0) + if msg.LoadEventEnd >= msg.LoadEventStart { + loadEventTime = nullableUint16(uint16(msg.LoadEventEnd - msg.LoadEventStart)) + } + jsonString, err := json.Marshal(map[string]interface{}{ + "request_start": nullableUint16(uint16(msg.RequestStart)), + "response_start": nullableUint16(uint16(msg.ResponseStart)), + "response_end": nullableUint16(uint16(msg.ResponseEnd)), + "dom_content_loaded_event_start": nullableUint16(uint16(msg.DomContentLoadedEventStart)), + "dom_content_loaded_event_end": nullableUint16(uint16(msg.DomContentLoadedEventEnd)), + "load_event_start": nullableUint16(uint16(msg.LoadEventStart)), + "load_event_end": nullableUint16(uint16(msg.LoadEventEnd)), + "first_paint": nullableUint16(uint16(msg.FirstPaint)), + "first_contentful_paint_time": nullableUint16(uint16(msg.FirstContentfulPaint)), + "speed_index": nullableUint16(uint16(msg.SpeedIndex)), + "visually_complete": nullableUint16(uint16(msg.VisuallyComplete)), + "time_to_interactive": nullableUint16(uint16(msg.TimeToInteractive)), + "url": cropString(msg.URL), + "url_host": host, + "url_path": path, + "url_hostpath": hostpath, + "ttfb": ttfb, + "ttlb": ttlb, + "response_time": responseTime, + "dom_building_time": domBuildingTime, + "dom_content_loaded_event_time": domContentLoadedEventTime, + "load_event_time": loadEventTime, + }) + if err != nil { + return fmt.Errorf("can't marshal page event: %s", err) + } + eventTime := datetime(msg.Timestamp) if err := c.batches["pages"].Append( session.SessionID, uint16(session.ProjectID), - msg.MessageID, - datetime(msg.Timestamp), - msg.URL, - nullableUint16(uint16(msg.RequestStart)), - nullableUint16(uint16(msg.ResponseStart)), - nullableUint16(uint16(msg.ResponseEnd)), - nullableUint16(uint16(msg.DomContentLoadedEventStart)), - nullableUint16(uint16(msg.DomContentLoadedEventEnd)), - nullableUint16(uint16(msg.LoadEventStart)), - nullableUint16(uint16(msg.LoadEventEnd)), - nullableUint16(uint16(msg.FirstPaint)), - nullableUint16(uint16(msg.FirstContentfulPaint)), - nullableUint16(uint16(msg.SpeedIndex)), - nullableUint16(uint16(msg.VisuallyComplete)), - nullableUint16(uint16(msg.TimeToInteractive)), - extractUrlPath(msg.URL), + getUUID(msg), "LOCATION", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + cropString(msg.URL), + jsonString, ); err != nil { c.checkError("pages", err) return fmt.Errorf("can't append to pages batch: %s", err) @@ -373,19 +460,36 @@ func (c *connectorImpl) InsertWebClickEvent(session *sessions.Session, msg *mess nYVal := normalizedY nY = &nYVal } + host, path, hostpath, err := extractUrlParts(msg.Url) + if err != nil { + return fmt.Errorf("can't extract url parts: %s", err) + } + jsonString, err := json.Marshal(map[string]interface{}{ + "label": msg.Label, + "hesitation_time": nullableUint32(uint32(msg.HesitationTime)), + "selector": msg.Selector, + "normalized_x": nX, + "normalized_y": nY, + "url": cropString(msg.Url), + "url_host": host, + "url_path": path, + "url_hostpath": hostpath, + }) + if err != nil { + return fmt.Errorf("can't marshal click event: %s", err) + } + eventTime := datetime(msg.Timestamp) if err := c.batches["clicks"].Append( session.SessionID, uint16(session.ProjectID), - msg.MsgID(), - datetime(msg.Timestamp), - msg.Label, - nullableUint32(uint32(msg.HesitationTime)), + getUUID(msg), "CLICK", - msg.Selector, - nX, - nY, - msg.Url, - extractUrlPath(msg.Url), + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + cropString(msg.Url), + jsonString, ); err != nil { c.checkError("clicks", err) return fmt.Errorf("can't append to clicks batch: %s", err) @@ -406,19 +510,28 @@ func (c *connectorImpl) InsertWebErrorEvent(session *sessions.Session, msg *type return fmt.Errorf("unknown error source: %s", msg.Source) } msgID, _ := msg.ID(session.ProjectID) - // Insert event to batch + jsonString, err := json.Marshal(map[string]interface{}{ + "source": msg.Source, + "name": nullableString(msg.Name), + "message": msg.Message, + "error_id": msgID, + "error_tags_keys": keys, + "error_tags_values": values, + }) + if err != nil { + return fmt.Errorf("can't marshal error event: %s", err) + } + eventTime := datetime(msg.Timestamp) if err := c.batches["errors"].Append( session.SessionID, uint16(session.ProjectID), - msg.MessageID, - datetime(msg.Timestamp), - msg.Source, - nullableString(msg.Name), - msg.Message, - msgID, + msg.GetUUID(session.SessionID), "ERROR", - keys, - values, + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { c.checkError("errors", err) return fmt.Errorf("can't append to errors batch: %s", err) @@ -427,26 +540,43 @@ func (c *connectorImpl) InsertWebErrorEvent(session *sessions.Session, msg *type } func (c *connectorImpl) InsertWebPerformanceTrackAggr(session *sessions.Session, msg *messages.PerformanceTrackAggr) error { - var timestamp uint64 = (msg.TimestampStart + msg.TimestampEnd) / 2 + var timestamp = (msg.TimestampStart + msg.TimestampEnd) / 2 + host, path, hostpath, err := extractUrlParts(msg.Url) + if err != nil { + return fmt.Errorf("can't extract url parts: %s", err) + } + jsonString, err := json.Marshal(map[string]interface{}{ + "url": cropString(msg.Url), + "url_host": host, + "url_path": path, + "url_hostpath": hostpath, + "min_fps": uint8(msg.MinFPS), + "avg_fps": uint8(msg.AvgFPS), + "max_fps": uint8(msg.MaxFPS), + "min_cpu": uint8(msg.MinCPU), + "avg_cpu": uint8(msg.AvgCPU), + "max_cpu": uint8(msg.MaxCPU), + "min_total_js_heap_size": msg.MinTotalJSHeapSize, + "avg_total_js_heap_size": msg.AvgTotalJSHeapSize, + "max_total_js_heap_size": msg.MaxTotalJSHeapSize, + "min_used_js_heap_size": msg.MinUsedJSHeapSize, + "avg_used_js_heap_size": msg.AvgUsedJSHeapSize, + "max_used_js_heap_size": msg.MaxUsedJSHeapSize, + }) + if err != nil { + return fmt.Errorf("can't marshal performance event: %s", err) + } + eventTime := datetime(timestamp) if err := c.batches["performance"].Append( session.SessionID, uint16(session.ProjectID), - uint64(0), // TODO: find messageID for performance events - datetime(timestamp), - nullableString(msg.Meta().Url), - uint8(msg.MinFPS), - uint8(msg.AvgFPS), - uint8(msg.MaxFPS), - uint8(msg.MinCPU), - uint8(msg.AvgCPU), - uint8(msg.MaxCPU), - msg.MinTotalJSHeapSize, - msg.AvgTotalJSHeapSize, - msg.MaxTotalJSHeapSize, - msg.MinUsedJSHeapSize, - msg.AvgUsedJSHeapSize, - msg.MaxUsedJSHeapSize, + getUUID(msg), "PERFORMANCE", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { c.checkError("performance", err) return fmt.Errorf("can't append to performance batch: %s", err) @@ -454,21 +584,6 @@ func (c *connectorImpl) InsertWebPerformanceTrackAggr(session *sessions.Session, return nil } -func (c *connectorImpl) InsertAutocomplete(session *sessions.Session, msgType, msgValue string) error { - if len(msgValue) == 0 { - return nil - } - if err := c.batches["autocompletes"].Append( - uint16(session.ProjectID), - msgType, - msgValue, - ); err != nil { - c.checkError("autocompletes", err) - return fmt.Errorf("can't append to autocompletes batch: %s", err) - } - return nil -} - func (c *connectorImpl) InsertRequest(session *sessions.Session, msg *messages.NetworkRequest, savePayload bool) error { urlMethod := url.EnsureMethod(msg.Method) if urlMethod == "" { @@ -479,21 +594,37 @@ func (c *connectorImpl) InsertRequest(session *sessions.Session, msg *messages.N request = &msg.Request response = &msg.Response } + host, path, hostpath, err := extractUrlParts(msg.URL) + if err != nil { + return fmt.Errorf("can't extract url parts: %s", err) + } + jsonString, err := json.Marshal(map[string]interface{}{ + "request_body": request, + "response_body": response, + "status": uint16(msg.Status), + "method": url.EnsureMethod(msg.Method), + "success": msg.Status < 400, + "transfer_size": uint32(msg.TransferredBodySize), + "url": cropString(msg.URL), + "url_host": host, + "url_path": path, + "url_hostpath": hostpath, + }) + if err != nil { + return fmt.Errorf("can't marshal request event: %s", err) + } + eventTime := datetime(msg.Timestamp) if err := c.batches["requests"].Append( session.SessionID, uint16(session.ProjectID), - msg.Meta().Index, - datetime(uint64(msg.Meta().Timestamp)), - msg.URL, - request, - response, - uint16(msg.Status), - url.EnsureMethod(msg.Method), - uint16(msg.Duration), - msg.Status < 400, + getUUID(msg), "REQUEST", - uint32(msg.TransferredBodySize), - extractUrlPath(msg.URL), + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + nullableUint16(uint16(msg.Duration)), + jsonString, ); err != nil { c.checkError("requests", err) return fmt.Errorf("can't append to requests batch: %s", err) @@ -502,14 +633,24 @@ func (c *connectorImpl) InsertRequest(session *sessions.Session, msg *messages.N } func (c *connectorImpl) InsertCustom(session *sessions.Session, msg *messages.CustomEvent) error { + jsonString, err := json.Marshal(map[string]interface{}{ + "name": msg.Name, + "payload": msg.Payload, + }) + if err != nil { + return fmt.Errorf("can't marshal custom event: %s", err) + } + eventTime := datetime(msg.Timestamp) if err := c.batches["custom"].Append( session.SessionID, uint16(session.ProjectID), - msg.Meta().Index, - datetime(uint64(msg.Meta().Timestamp)), - msg.Name, - msg.Payload, + getUUID(msg), "CUSTOM", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { c.checkError("custom", err) return fmt.Errorf("can't append to custom batch: %s", err) @@ -518,15 +659,25 @@ func (c *connectorImpl) InsertCustom(session *sessions.Session, msg *messages.Cu } func (c *connectorImpl) InsertGraphQL(session *sessions.Session, msg *messages.GraphQL) error { + jsonString, err := json.Marshal(map[string]interface{}{ + "name": msg.OperationName, + "request_body": nullableString(msg.Variables), + "response_body": nullableString(msg.Response), + }) + if err != nil { + return fmt.Errorf("can't marshal graphql event: %s", err) + } + eventTime := datetime(msg.Timestamp) if err := c.batches["graphql"].Append( session.SessionID, uint16(session.ProjectID), - msg.Meta().Index, - datetime(uint64(msg.Meta().Timestamp)), - msg.OperationName, - nullableString(msg.Variables), - nullableString(msg.Response), + getUUID(msg), "GRAPHQL", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { c.checkError("graphql", err) return fmt.Errorf("can't append to graphql batch: %s", err) @@ -540,7 +691,7 @@ func (c *connectorImpl) InsertMobileSession(session *sessions.Session) error { if session.Duration == nil { return errors.New("trying to insert mobile session with nil duration") } - if err := c.batches["ios_sessions"].Append( + if err := c.batches["mobile_sessions"].Append( session.SessionID, uint16(session.ProjectID), session.UserID, @@ -576,23 +727,33 @@ func (c *connectorImpl) InsertMobileSession(session *sessions.Session) error { "ios", session.Timezone, ); err != nil { - c.checkError("ios_sessions", err) + c.checkError("mobile_sessions", err) return fmt.Errorf("can't append to sessions batch: %s", err) } return nil } func (c *connectorImpl) InsertMobileCustom(session *sessions.Session, msg *messages.MobileEvent) error { - if err := c.batches["ios_custom"].Append( + jsonString, err := json.Marshal(map[string]interface{}{ + "name": msg.Name, + "payload": msg.Payload, + }) + if err != nil { + return fmt.Errorf("can't marshal mobile custom event: %s", err) + } + eventTime := datetime(msg.Timestamp) + if err := c.batches["mobile_custom"].Append( session.SessionID, uint16(session.ProjectID), - msg.Meta().Index, - datetime(uint64(msg.Meta().Timestamp)), - msg.Name, - msg.Payload, + getUUID(msg), "CUSTOM", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { - c.checkError("ios_custom", err) + c.checkError("mobile_custom", err) return fmt.Errorf("can't append to mobile custom batch: %s", err) } return nil @@ -602,15 +763,25 @@ func (c *connectorImpl) InsertMobileClick(session *sessions.Session, msg *messag if msg.Label == "" { return nil } - if err := c.batches["ios_clicks"].Append( + jsonString, err := json.Marshal(map[string]interface{}{ + "label": msg.Label, + }) + if err != nil { + return fmt.Errorf("can't marshal mobile clicks event: %s", err) + } + eventTime := datetime(msg.Timestamp) + if err := c.batches["mobile_clicks"].Append( session.SessionID, uint16(session.ProjectID), - msg.MsgID(), - datetime(msg.Timestamp), - msg.Label, + getUUID(msg), "TAP", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { - c.checkError("ios_clicks", err) + c.checkError("mobile_clicks", err) return fmt.Errorf("can't append to mobile clicks batch: %s", err) } return nil @@ -620,17 +791,27 @@ func (c *connectorImpl) InsertMobileSwipe(session *sessions.Session, msg *messag if msg.Label == "" { return nil } - if err := c.batches["ios_swipes"].Append( + jsonString, err := json.Marshal(map[string]interface{}{ + "label": msg.Label, + "direction": nullableString(msg.Direction), + }) + if err != nil { + return fmt.Errorf("can't marshal mobile swipe event: %s", err) + } + eventTime := datetime(msg.Timestamp) + if err := c.batches["mobile_swipes"].Append( session.SessionID, uint16(session.ProjectID), - msg.MsgID(), - datetime(msg.Timestamp), - msg.Label, - nullableString(msg.Direction), + getUUID(msg), "SWIPE", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { - c.checkError("ios_clicks", err) - return fmt.Errorf("can't append to mobile clicks batch: %s", err) + c.checkError("mobile_swipes", err) + return fmt.Errorf("can't append to mobile swipe batch: %s", err) } return nil } @@ -639,15 +820,25 @@ func (c *connectorImpl) InsertMobileInput(session *sessions.Session, msg *messag if msg.Label == "" { return nil } - if err := c.batches["ios_inputs"].Append( + jsonString, err := json.Marshal(map[string]interface{}{ + "label": msg.Label, + }) + if err != nil { + return fmt.Errorf("can't marshal mobile input event: %s", err) + } + eventTime := datetime(msg.Timestamp) + if err := c.batches["mobile_inputs"].Append( session.SessionID, uint16(session.ProjectID), - msg.MsgID(), - datetime(msg.Timestamp), - msg.Label, + getUUID(msg), "INPUT", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { - c.checkError("ios_inputs", err) + c.checkError("mobile_inputs", err) return fmt.Errorf("can't append to mobile inputs batch: %s", err) } return nil @@ -663,39 +854,103 @@ func (c *connectorImpl) InsertMobileRequest(session *sessions.Session, msg *mess request = &msg.Request response = &msg.Response } - if err := c.batches["ios_requests"].Append( + jsonString, err := json.Marshal(map[string]interface{}{ + "url": cropString(msg.URL), + "request_body": request, + "response_body": response, + "status": uint16(msg.Status), + "method": url.EnsureMethod(msg.Method), + "duration": uint16(msg.Duration), + "success": msg.Status < 400, + }) + if err != nil { + return fmt.Errorf("can't marshal mobile request event: %s", err) + } + eventTime := datetime(msg.Timestamp) + if err := c.batches["mobile_requests"].Append( session.SessionID, uint16(session.ProjectID), - msg.Meta().Index, - datetime(uint64(msg.Meta().Timestamp)), - msg.URL, - request, - response, - uint16(msg.Status), - url.EnsureMethod(msg.Method), - uint16(msg.Duration), - msg.Status < 400, + getUUID(msg), "REQUEST", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { - c.checkError("ios_requests", err) + c.checkError("mobile_requests", err) return fmt.Errorf("can't append to mobile requests batch: %s", err) } return nil } func (c *connectorImpl) InsertMobileCrash(session *sessions.Session, msg *messages.MobileCrash) error { - if err := c.batches["ios_crashes"].Append( + jsonString, err := json.Marshal(map[string]interface{}{ + "name": msg.Name, + "reason": msg.Reason, + "stacktrace": msg.Stacktrace, + }) + if err != nil { + return fmt.Errorf("can't marshal mobile crash event: %s", err) + } + eventTime := datetime(msg.Timestamp) + if err := c.batches["mobile_crashes"].Append( session.SessionID, uint16(session.ProjectID), - msg.MsgID(), - datetime(msg.Timestamp), - msg.Name, - msg.Reason, - msg.Stacktrace, + getUUID(msg), "CRASH", + eventTime, + eventTime.Unix(), + session.UserUUID, + true, + jsonString, ); err != nil { - c.checkError("ios_crashes", err) - return fmt.Errorf("can't append to mobile crashges batch: %s", err) + c.checkError("mobile_crashes", err) + return fmt.Errorf("can't append to mobile crashs batch: %s", err) } return nil } + +func (c *connectorImpl) checkError(name string, err error) { + if !errors.Is(err, clickhouse.ErrBatchAlreadySent) { + log.Printf("can't create %s batch after failed append operation: %s", name, err) + } +} + +func extractUrlParts(fullUrl string) (string, string, string, error) { + host, path, query, err := url.GetURLParts(strings.ToLower(fullUrl)) + if err != nil { + return "", "", "", err + } + pathQuery := path + if query != "" { + pathQuery += "?" + query + } + return cropString(host), cropString(pathQuery), cropString(host + pathQuery), nil +} + +func getUUID(m messages.Message) string { + hash := fnv.New128a() + hash.Write(uint64ToBytes(m.SessionID())) + hash.Write(uint64ToBytes(m.MsgID())) + hash.Write(uint64ToBytes(uint64(m.TypeID()))) + uuidObj, err := uuid.FromBytes(hash.Sum(nil)) + if err != nil { + fmt.Printf("can't create uuid from bytes: %s", err) + uuidObj = uuid.New() + } + return uuidObj.String() +} + +func uint64ToBytes(num uint64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, num) + return buf +} + +func cropString(s string) string { + if len(s) > 8000 { + return s[:8000] + } + return s +} diff --git a/backend/pkg/db/clickhouse/insert_type.go b/backend/pkg/db/clickhouse/insert_type.go index b83898d6f..153ca3602 100644 --- a/backend/pkg/db/clickhouse/insert_type.go +++ b/backend/pkg/db/clickhouse/insert_type.go @@ -31,7 +31,7 @@ func nullableString(v string) *string { func datetime(timestamp uint64) time.Time { t := time.Unix(int64(timestamp/1e3), 0) // Temporal solution for not correct timestamps in performance messages - if t.Year() < 2022 || t.Year() > 2025 { + if t.Year() < 2023 || t.Year() > 2026 { return time.Now() } return t diff --git a/backend/pkg/db/types/error-event.go b/backend/pkg/db/types/error-event.go index 70222556c..5fdb4e9ff 100644 --- a/backend/pkg/db/types/error-event.go +++ b/backend/pkg/db/types/error-event.go @@ -1,9 +1,11 @@ package types import ( + "encoding/binary" "encoding/hex" "encoding/json" "fmt" + "github.com/google/uuid" "hash/fnv" "strconv" @@ -13,13 +15,14 @@ import ( const SOURCE_JS = "js_exception" type ErrorEvent struct { - MessageID uint64 - Timestamp uint64 - Source string - Name string - Message string - Payload string - Tags map[string]*string + MessageID uint64 + Timestamp uint64 + Source string + Name string + Message string + Payload string + Tags map[string]*string + OriginType int } func unquote(s string) string { @@ -60,24 +63,26 @@ func parseTags(tagsJSON string) (tags map[string]*string, err error) { func WrapJSException(m *JSException) (*ErrorEvent, error) { meta, err := parseTags(m.Metadata) return &ErrorEvent{ - MessageID: m.Meta().Index, - Timestamp: m.Meta().Timestamp, - Source: SOURCE_JS, - Name: m.Name, - Message: m.Message, - Payload: m.Payload, - Tags: meta, + MessageID: m.Meta().Index, + Timestamp: m.Meta().Timestamp, + Source: SOURCE_JS, + Name: m.Name, + Message: m.Message, + Payload: m.Payload, + Tags: meta, + OriginType: m.TypeID(), }, err } func WrapIntegrationEvent(m *IntegrationEvent) *ErrorEvent { return &ErrorEvent{ - MessageID: m.Meta().Index, // This will be always 0 here since it's coming from backend TODO: find another way to index - Timestamp: m.Timestamp, - Source: m.Source, - Name: m.Name, - Message: m.Message, - Payload: m.Payload, + MessageID: m.Meta().Index, // This will be always 0 here since it's coming from backend TODO: find another way to index + Timestamp: m.Timestamp, + Source: m.Source, + Name: m.Name, + Message: m.Message, + Payload: m.Payload, + OriginType: m.TypeID(), } } @@ -129,3 +134,22 @@ func WrapCustomEvent(m *CustomEvent) *IssueEvent { msg.Meta().SetMeta(m.Meta()) return msg } + +func (e *ErrorEvent) GetUUID(sessID uint64) string { + hash := fnv.New128a() + hash.Write(Uint64ToBytes(sessID)) + hash.Write(Uint64ToBytes(e.MessageID)) + hash.Write(Uint64ToBytes(uint64(e.OriginType))) + uuidObj, err := uuid.FromBytes(hash.Sum(nil)) + if err != nil { + fmt.Printf("can't create uuid from bytes: %s", err) + uuidObj = uuid.New() + } + return uuidObj.String() +} + +func Uint64ToBytes(num uint64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, num) + return buf +} diff --git a/backend/pkg/messages/message.go b/backend/pkg/messages/message.go index 6ae02e6c5..348e8f5e0 100644 --- a/backend/pkg/messages/message.go +++ b/backend/pkg/messages/message.go @@ -1,6 +1,8 @@ package messages -import "fmt" +import ( + "fmt" +) type Message interface { Encode() []byte From 12a9448a8d67c06a2e83d8da74ce9bd89f65e92e Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 20 Jan 2025 14:47:56 +0100 Subject: [PATCH 3/5] feat(go): updated all golang imports --- backend/go.mod | 38 +++++++++++------------ backend/go.sum | 81 ++++++++++++++++++++++++-------------------------- 2 files changed, 57 insertions(+), 62 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index d0193a399..39cf40152 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,22 +3,20 @@ module openreplay/backend go 1.23 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 github.com/ClickHouse/clickhouse-go/v2 v2.30.1 github.com/DataDog/datadog-api-client-go/v2 v2.34.0 github.com/Masterminds/semver v1.5.0 github.com/andybalholm/brotli v1.1.1 - github.com/aws/aws-sdk-go v1.55.5 + github.com/aws/aws-sdk-go v1.55.6 github.com/btcsuite/btcutil v1.0.2 - github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 - github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/confluentinc/confluent-kafka-go/v2 v2.8.0 github.com/docker/distribution v2.8.3+incompatible github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/elastic/go-elasticsearch/v8 v8.17.0 - github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611 - github.com/getsentry/sentry-go v0.30.0 - github.com/go-playground/validator/v10 v10.23.0 + github.com/getsentry/sentry-go v0.31.1 + github.com/go-playground/validator/v10 v10.24.0 github.com/go-redis/redis v6.15.9+incompatible github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 @@ -34,7 +32,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.20.5 github.com/rs/xid v1.6.0 - github.com/sethvargo/go-envconfig v0.7.0 + github.com/sethvargo/go-envconfig v1.1.0 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce github.com/ua-parser/uap-go v0.0.0-20241012191800-bbb40edc15aa go.uber.org/zap v1.27.0 @@ -44,19 +42,19 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/ClickHouse/ch-go v0.63.1 // indirect - github.com/DataDog/zstd v1.5.2 // indirect + github.com/DataDog/zstd v1.5.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/goccy/go-json v0.10.2 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/goccy/go-json v0.10.4 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -69,22 +67,22 @@ require ( github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.32.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 // indirect - google.golang.org/protobuf v1.36.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/protobuf v1.36.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - diff --git a/backend/go.sum b/backend/go.sum index b67ff7905..3eb945a77 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -4,8 +4,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= @@ -21,12 +21,12 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/ch-go v0.63.1 h1:s2JyZvWLTCSAGdtjMBBmAgQQHMco6pawLJMOXi0FODM= github.com/ClickHouse/ch-go v0.63.1/go.mod h1:I1kJJCL3WJcBMGe1m+HVK0+nREaG+JOYYBWjrDrF3R0= -github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.1 h1:Dy0n0l+cMbPXs8hFkeeWGaPKrB+MDByUNQBSmRO3W6k= github.com/ClickHouse/clickhouse-go/v2 v2.30.1/go.mod h1:szk8BMoQV/NgHXZ20ZbwDyvPWmpfhRKjFkc6wzASGxM= github.com/DataDog/datadog-api-client-go/v2 v2.34.0 h1:0VVmv8uZg8vdBuEpiF2nBGUezl2QITrxdEsLgh38j8M= github.com/DataDog/datadog-api-client-go/v2 v2.34.0/go.mod h1:d3tOEgUd2kfsr9uuHQdY+nXrWp4uikgTgVCPdKNK30U= -github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= -github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= +github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= @@ -41,8 +41,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat6 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= -github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= +github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/config v1.27.10 h1:PS+65jThT0T/snC5WjyfHHyUgG+eBoupSDV+f838cro= @@ -91,8 +91,8 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/compose-spec/compose-go/v2 v2.1.3 h1:bD67uqLuL/XgkAK6ir3xZvNLFPxPScEi1KW7R5esrLE= github.com/compose-spec/compose-go/v2 v2.1.3/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 h1:XFkytnGvk/ZcY2qU0ql4E4h+ftBaGqkLO7tlZ4kRbr4= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.1/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= +github.com/confluentinc/confluent-kafka-go/v2 v2.8.0 h1:0HlcSNWg4LpLA9nIjzUMIqWHI+w0S68UN7alXAc3TeA= +github.com/confluentinc/confluent-kafka-go/v2 v2.8.0/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= @@ -118,8 +118,6 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/buildx v0.15.1 h1:1cO6JIc0rOoC8tlxfXoh1HH1uxaNvYH1q7J7kv5enhw= @@ -154,17 +152,15 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611 h1:JwYtKJ/DVEoIA5dH45OEU7uoryZY/gjd/BQiwwAOImM= -github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611/go.mod h1:zHMNeYgqrTpKyjawjitDg0Osd1P/FmeA0SZLYK3RfLQ= github.com/fsnotify/fsevents v0.2.0 h1:BRlvlqjvNTfogHfeBOFvSC9N0Ddy+wzQCQukyoD7o/c= github.com/fsnotify/fsevents v0.2.0/go.mod h1:B3eEk39i4hz8y1zaWS/wPrAP4O6wkIl7HQwKBr1qH/w= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo= github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/getsentry/sentry-go v0.30.0 h1:lWUwDnY7sKHaVIoZ9wYqRHJ5iEmoc0pqcRqFkosKzBo= -github.com/getsentry/sentry-go v0.30.0/go.mod h1:WU9B9/1/sHDqeV8T+3VwwbjeR5MSXs/6aqG3mqZrezA= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4= +github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= @@ -192,15 +188,15 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= -github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.24.0 h1:KHQckvo8G6hlWnrPX4NJJ+aBfWNAE/HH+qdL2cBpCmg= +github.com/go-playground/validator/v10 v10.24.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-viper/mapstructure/v2 v2.0.0 h1:dhn8MZ1gZ0mzeodTG3jt5Vj/o87xZKuNAprG2mQfMfc= github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= @@ -243,8 +239,9 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -442,8 +439,8 @@ github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+ github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o= @@ -451,8 +448,8 @@ github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNr github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= @@ -465,8 +462,8 @@ github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= -github.com/sethvargo/go-envconfig v0.7.0 h1:P/ljQXSRjgAgsnIripHs53Jg/uNVXu2FYQ9yLSDappA= -github.com/sethvargo/go-envconfig v0.7.0/go.mod h1:00S1FAhRUuTNJazWBWcJGvEHOM+NO6DhoRMAOX7FY5o= +github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog= +github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= @@ -544,18 +541,18 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zavorotynskiy/clickhouse-go/v2 v2.30.1-0.20250110230205-a22b6d524deb h1:3gQycbGOJreRQkKORHjCn+wEPpnlra6Ek5TnW78z5vc= -github.com/zavorotynskiy/clickhouse-go/v2 v2.30.1-0.20250110230205-a22b6d524deb/go.mod h1:szk8BMoQV/NgHXZ20ZbwDyvPWmpfhRKjFkc6wzASGxM= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 h1:gbhw/u49SS3gkPWiYweQNJGm/uJN5GkI/FrosxSHT7A= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM= @@ -568,14 +565,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqhe go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0= go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -638,8 +635,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -716,14 +713,14 @@ google.golang.org/genproto v0.0.0-20240325203815-454cdb8f5daa h1:ePqxpG3LVx+feAU google.golang.org/genproto v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:CnZenrTdRJb7jc+jOm0Rkywq+9wh0QC4U8tyiRbEPPM= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 h1:Z7FRVJPSMaHQxD0uXU8WdgFh8PseLM8Q8NzhnpMrBhQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= -google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From eb11f4bf58c3b0a80b64e2cc67d430ad7b29546a Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 20 Jan 2025 15:38:53 +0100 Subject: [PATCH 4/5] feat(ender): optimized logs --- backend/cmd/ender/main.go | 135 +++++++++++++++++++++++++++----------- 1 file changed, 95 insertions(+), 40 deletions(-) diff --git a/backend/cmd/ender/main.go b/backend/cmd/ender/main.go index c502a004f..ac4bce5a5 100644 --- a/backend/cmd/ender/main.go +++ b/backend/cmd/ender/main.go @@ -93,25 +93,7 @@ func main() { consumer.Close() os.Exit(0) case <-tick: - failedSessionEnds := make(map[uint64]uint64) - duplicatedSessionEnds := make(map[uint64]uint64) - negativeDuration := make(map[uint64]uint64) - shorterDuration := make(map[uint64]int64) - diffDuration := make(map[uint64]int64) - noSessionInDB := make(map[uint64]uint64) - updatedDurations := 0 - newSessionEnds := 0 - - type SessionEndType int - const ( - FailedSessionEnd SessionEndType = iota + 1 - DuplicatedSessionEnd - NegativeDuration - ShorterDuration - DiffDuration - NewSessionEnd - NoSessionInDB - ) + details := newDetails() // Find ended sessions and send notification to other services sessionEndGenerator.HandleEndedSessions(func(sessionID uint64, timestamp uint64) (bool, int) { @@ -128,12 +110,12 @@ func main() { newDur := timestamp - sess.Timestamp // Skip if session was ended before with same duration if currDuration == newDur { - duplicatedSessionEnds[sessionID] = currDuration + details.Duplicated[sessionID] = currDuration return true, int(DuplicatedSessionEnd) } // Skip if session was ended before with longer duration if currDuration > newDur { - shorterDuration[sessionID] = int64(currDuration) - int64(newDur) + details.Shorter[sessionID] = int64(currDuration) - int64(newDur) return true, int(ShorterDuration) } } @@ -141,15 +123,15 @@ func main() { if err != nil { if strings.Contains(err.Error(), "integer out of range") { // Skip session with broken duration - failedSessionEnds[sessionID] = timestamp + details.Failed[sessionID] = timestamp return true, int(FailedSessionEnd) } if strings.Contains(err.Error(), "is less than zero for uint64") { - negativeDuration[sessionID] = timestamp + details.Negative[sessionID] = timestamp return true, int(NegativeDuration) } if strings.Contains(err.Error(), "no rows in result set") { - noSessionInDB[sessionID] = timestamp + details.NotFound[sessionID] = timestamp return true, int(NoSessionInDB) } log.Error(sessCtx, "can't update session duration, err: %s", err) @@ -157,7 +139,7 @@ func main() { } // Check one more time just in case if currDuration == newDuration { - duplicatedSessionEnds[sessionID] = currDuration + details.Duplicated[sessionID] = currDuration return true, int(DuplicatedSessionEnd) } if cfg.UseEncryption { @@ -191,25 +173,14 @@ func main() { } if currDuration != 0 { - diffDuration[sessionID] = int64(newDuration) - int64(currDuration) - updatedDurations++ + details.Diff[sessionID] = int64(newDuration) - int64(currDuration) + details.Updated++ } else { - newSessionEnds++ + details.New++ } return true, int(NewSessionEnd) }) - if n := len(failedSessionEnds); n > 0 { - log.Info(ctx, "sessions with wrong duration: %d, %v", n, failedSessionEnds) - } - if n := len(negativeDuration); n > 0 { - log.Info(ctx, "sessions with negative duration: %d, %v", n, negativeDuration) - } - if n := len(noSessionInDB); n > 0 { - log.Info(ctx, "sessions without info in DB: %d, %v", n, noSessionInDB) - } - log.Info(ctx, "failed: %d, negative: %d, shorter: %d, same: %d, updated: %d, new: %d, not found: %d", - len(failedSessionEnds), len(negativeDuration), len(shorterDuration), len(duplicatedSessionEnds), - updatedDurations, newSessionEnds, len(noSessionInDB)) + details.Log(log, ctx) producer.Flush(cfg.ProducerTimeout) if err := consumer.CommitBack(intervals.EVENTS_BACK_COMMIT_GAP); err != nil { log.Error(ctx, "can't commit messages with offset: %s", err) @@ -231,3 +202,87 @@ func main() { } } } + +type logDetails struct { + Failed map[uint64]uint64 + Duplicated map[uint64]uint64 + Negative map[uint64]uint64 + Shorter map[uint64]int64 + NotFound map[uint64]uint64 + Diff map[uint64]int64 + Updated int + New int +} + +func newDetails() *logDetails { + return &logDetails{ + Failed: make(map[uint64]uint64), + Duplicated: make(map[uint64]uint64), + Negative: make(map[uint64]uint64), + Shorter: make(map[uint64]int64), + NotFound: make(map[uint64]uint64), + Diff: make(map[uint64]int64), + Updated: 0, + New: 0, + } +} + +func (l *logDetails) Log(log logger.Logger, ctx context.Context) { + if n := len(l.Failed); n > 0 { + log.Debug(ctx, "sessions with wrong duration: %d, %v", n, l.Failed) + } + if n := len(l.Negative); n > 0 { + log.Debug(ctx, "sessions with negative duration: %d, %v", n, l.Negative) + } + if n := len(l.NotFound); n > 0 { + log.Debug(ctx, "sessions without info in DB: %d, %v", n, l.NotFound) + } + var logBuilder strings.Builder + logValues := []interface{}{} + + if len(l.Failed) > 0 { + logBuilder.WriteString("failed: %d, ") + logValues = append(logValues, len(l.Failed)) + } + if len(l.Negative) > 0 { + logBuilder.WriteString("negative: %d, ") + logValues = append(logValues, len(l.Negative)) + } + if len(l.Shorter) > 0 { + logBuilder.WriteString("shorter: %d, ") + logValues = append(logValues, len(l.Shorter)) + } + if len(l.Duplicated) > 0 { + logBuilder.WriteString("same: %d, ") + logValues = append(logValues, len(l.Duplicated)) + } + if l.Updated > 0 { + logBuilder.WriteString("updated: %d, ") + logValues = append(logValues, l.Updated) + } + if l.New > 0 { + logBuilder.WriteString("new: %d, ") + logValues = append(logValues, l.New) + } + if len(l.NotFound) > 0 { + logBuilder.WriteString("not found: %d, ") + logValues = append(logValues, len(l.NotFound)) + } + + if logBuilder.Len() > 0 { + logMessage := logBuilder.String() + logMessage = logMessage[:len(logMessage)-2] + log.Info(ctx, logMessage, logValues...) + } +} + +type SessionEndType int + +const ( + FailedSessionEnd SessionEndType = iota + 1 + DuplicatedSessionEnd + NegativeDuration + ShorterDuration + NewSessionEnd + NoSessionInDB +) From 57e604794c8a5246db425c3cb1bbcfb955d539eb Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 21 Jan 2025 11:24:36 +0100 Subject: [PATCH 5/5] fix(ui): conditional capture metadata --- frontend/app/mstore/types/FeatureFlag.ts | 5 +++-- frontend/app/mstore/types/filterItem.ts | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frontend/app/mstore/types/FeatureFlag.ts b/frontend/app/mstore/types/FeatureFlag.ts index 7e4d9c327..de6b59310 100644 --- a/frontend/app/mstore/types/FeatureFlag.ts +++ b/frontend/app/mstore/types/FeatureFlag.ts @@ -9,10 +9,11 @@ export class Conditions { constructor(data?: Record, isConditional?: boolean, isMobile?: boolean) { makeAutoObservable(this); + console.log('data', data) this.name = data?.name; if (data && (data.rolloutPercentage || data.captureRate)) { this.rolloutPercentage = data.rolloutPercentage ?? data.captureRate; - this.filter = new Filter(isConditional, isMobile).fromJson(data); + this.filter = new Filter([], isConditional, isMobile).fromJson(data); } } @@ -213,4 +214,4 @@ export default class FeatureFlag { this.isActive = isEnabled; this.setHasChanged(true) } -} \ No newline at end of file +} diff --git a/frontend/app/mstore/types/filterItem.ts b/frontend/app/mstore/types/filterItem.ts index 8809aa16c..6082aae4b 100644 --- a/frontend/app/mstore/types/filterItem.ts +++ b/frontend/app/mstore/types/filterItem.ts @@ -85,12 +85,12 @@ export default class FilterItem { if (this.isConditional) { if (this.isMobile) { _filter = - mobileConditionalFiltersMap[json.type] || - mobileConditionalFiltersMap[json.source]; + mobileConditionalFiltersMap[_filter.key] || + mobileConditionalFiltersMap[_filter.source]; } else { _filter = - conditionalFiltersMap[json.type] || - conditionalFiltersMap[json.source]; + conditionalFiltersMap[_filter.key] || + conditionalFiltersMap[_filter.source]; } } if (mainFilterKey) {