From e26ce2e963c6f9a68eef5d79b9d1e88d913f2c3b Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Mon, 16 May 2022 14:41:44 +0200 Subject: [PATCH] fix(backend-ee/clickhouse): do not insert metod & status into resources as they are always unknown --- ee/backend/pkg/db/clickhouse/connector.go | 44 ++++++++++---------- ee/backend/pkg/db/clickhouse/messages-web.go | 23 +++++----- 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index a93e792e5..532389425 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -1,31 +1,31 @@ package clickhouse import ( - "log" "database/sql" _ "github.com/ClickHouse/clickhouse-go" + "log" "openreplay/backend/pkg/license" ) type Connector struct { - sessionsIOS *bulk + sessionsIOS *bulk //viewsIOS *bulk clicksIOS *bulk inputsIOS *bulk crashesIOS *bulk performanceIOS *bulk resourcesIOS *bulk - sessions *bulk - metadata *bulk // TODO: join sessions, sessions_metadata & sessions_ios - resources *bulk - pages *bulk - clicks *bulk - inputs *bulk - errors *bulk - performance *bulk - longtasks *bulk - db *sql.DB + sessions *bulk + metadata *bulk // TODO: join sessions, sessions_metadata & sessions_ios + resources *bulk + pages *bulk + clicks *bulk + inputs *bulk + errors *bulk + performance *bulk + longtasks *bulk + db *sql.DB } func NewConnector(url string) *Connector { @@ -38,9 +38,9 @@ func NewConnector(url string) *Connector { return &Connector{ db: db, // sessionsIOS: newBulk(db, ` - // INSERT INTO sessions_ios (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, datetime, duration, views_count, events_count, crashes_count, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) - // VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - // `), + // INSERT INTO sessions_ios (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, datetime, duration, views_count, events_count, crashes_count, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10) + // VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + // `), // viewsIOS: newBulk(db, ` // INSERT INTO views_ios (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive) // VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -66,18 +66,18 @@ func NewConnector(url string) *Connector { // VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, nullIf(?, ''), ?) // `), - sessions: newBulk(db, ` + sessions: newBulk(db, ` INSERT INTO sessions (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_device, user_device_type, user_country, datetime, duration, pages_count, events_count, errors_count, user_browser, user_browser_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), - // TODO: join sessions, sessions_metadata & sessions_ios - metadata: newBulk(db, ` + // TODO: join sessions, sessions_metadata & sessions_ios + metadata: newBulk(db, ` INSERT INTO sessions_metadata (session_id, user_id, user_anonymous_id, metadata_1, metadata_2, metadata_3, metadata_4, metadata_5, metadata_6, metadata_7, metadata_8, metadata_9, metadata_10, datetime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), resources: newBulk(db, ` - INSERT INTO resources (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success, method, status) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO resources (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, type, duration, ttfb, header_size, encoded_body_size, decoded_body_size, success) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), pages: newBulk(db, ` INSERT INTO pages (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, url, request_start, response_start, response_end, dom_content_loaded_event_start, dom_content_loaded_event_end, load_event_start, load_event_end, first_paint, first_contentful_paint, speed_index, visually_complete, time_to_interactive) @@ -103,7 +103,6 @@ func NewConnector(url string) *Connector { INSERT INTO longtasks (session_id, project_id, tracker_version, rev_id, user_uuid, user_os, user_os_version, user_browser, user_browser_version, user_device, user_device_type, user_country, datetime, context, container_type, container_id, container_name, container_src) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), - } } @@ -129,7 +128,7 @@ func (conn *Connector) Prepare() error { // if err := conn.resourcesIOS.prepare(); err != nil { // return err // } - if err := conn.sessions.prepare(); err != nil { + if err := conn.sessions.prepare(); err != nil { return err } if err := conn.metadata.prepare(); err != nil { @@ -211,7 +210,6 @@ func (conn *Connector) Commit() error { return nil } - func (conn *Connector) FinaliseSessionsTable() error { _, err := conn.db.Exec("OPTIMIZE TABLE sessions FINAL") return err diff --git a/ee/backend/pkg/db/clickhouse/messages-web.go b/ee/backend/pkg/db/clickhouse/messages-web.go index 0c4de6a7b..adfa38655 100644 --- a/ee/backend/pkg/db/clickhouse/messages-web.go +++ b/ee/backend/pkg/db/clickhouse/messages-web.go @@ -1,17 +1,16 @@ -package clickhouse +package clickhouse import ( "errors" - - "openreplay/backend/pkg/hashid" - "openreplay/backend/pkg/url" + . "openreplay/backend/pkg/db/types" + "openreplay/backend/pkg/hashid" . "openreplay/backend/pkg/messages" + "openreplay/backend/pkg/url" ) - func (conn *Connector) InsertWebSession(session *Session) error { - if (session.Duration == nil) { + if session.Duration == nil { return errors.New("Clickhouse: trying to insert session with ") } @@ -66,7 +65,7 @@ func (conn *Connector) InsertWebResourceEvent(session *Session, msg *ResourceEve return conn.resources.exec( session.SessionID, session.ProjectID, - session.TrackerVersion, + session.TrackerVersion, nullableString(session.RevID), session.UserUUID, session.UserOS, @@ -85,8 +84,6 @@ func (conn *Connector) InsertWebResourceEvent(session *Session, msg *ResourceEve nullableUint32(uint32(msg.EncodedBodySize)), nullableUint32(uint32(msg.DecodedBodySize)), msg.Success, - method, - nullableUint16(uint16(msg.Status)), ) } @@ -219,8 +216,8 @@ func (conn *Connector) InsertWebPerformanceTrackAggr(session *Session, msg *Perf } // TODO: make enum message type -var CONTEXT_MAP = map[uint64]string{0:"unknown",1:"self",2:"same-origin-ancestor",3:"same-origin-descendant",4:"same-origin",5:"cross-origin-ancestor",6:"cross-origin-descendant",7:"cross-origin-unreachable",8:"multiple-contexts"} -var CONTAINER_TYPE_MAP = map[uint64]string{0:"window",1:"iframe",2:"embed",3:"object"} +var CONTEXT_MAP = map[uint64]string{0: "unknown", 1: "self", 2: "same-origin-ancestor", 3: "same-origin-descendant", 4: "same-origin", 5: "cross-origin-ancestor", 6: "cross-origin-descendant", 7: "cross-origin-unreachable", 8: "multiple-contexts"} +var CONTAINER_TYPE_MAP = map[uint64]string{0: "window", 1: "iframe", 2: "embed", 3: "object"} func (conn *Connector) InsertLongtask(session *Session, msg *LongTask) error { return conn.longtasks.exec( @@ -237,8 +234,8 @@ func (conn *Connector) InsertLongtask(session *Session, msg *LongTask) error { session.UserDeviceType, session.UserCountry, datetime(msg.Timestamp), - CONTEXT_MAP[ msg.Context ], - CONTAINER_TYPE_MAP[ msg.ContainerType ], + CONTEXT_MAP[msg.Context], + CONTAINER_TYPE_MAP[msg.ContainerType], msg.ContainerId, msg.ContainerName, msg.ContainerSrc,