From d578f5e097ce3c4628955e1f755cf2445b2de396 Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Tue, 12 Jul 2022 12:16:08 +0200 Subject: [PATCH 01/14] Updated messages methods. New values: - [x] IOSIssueEvent (111) - [x] IOSBatchMeta (107) - [x] IOSNetworkCall (105) - [x] IOSPerformanceEvent (102) - [x] IOSInputEvent (101) - [x] IOSClickEvent (100) - [x] IOSScreenEnter (98) - [x] IOSCrash (97) - [x] IOSScreenChanges (96) - [x] IOSCustomEvent (93) - [x] CreateIFrameDocument (70) - [x] CSSInsertRuleURLBased (67) - [x] AssetCache (66) - [x] FetchEvent (51) - [x] MouseClick (69) <- MouseClickDepricated (21) --- ee/connectors/msgcodec/codec.py | 130 ++++++++++++++++++++++- ee/connectors/msgcodec/messages.py | 161 ++++++++++++++++++++++++++++- 2 files changed, 289 insertions(+), 2 deletions(-) diff --git a/ee/connectors/msgcodec/codec.py b/ee/connectors/msgcodec/codec.py index 18f074a33..da06b5262 100644 --- a/ee/connectors/msgcodec/codec.py +++ b/ee/connectors/msgcodec/codec.py @@ -215,7 +215,7 @@ class MessageCodec(Codec): ) if message_id == 21: - return MouseClick( + return MouseClickDepricated( id=self.read_uint(reader), hesitation_time=self.read_uint(reader), label=self.read_string(reader) @@ -461,6 +461,18 @@ class MessageCodec(Codec): name=self.read_string(reader) ) + if message_id == 51: + return FetchEvent( + message_id=self.read_uint(reader), + timestamp=self.read_uint(reader), + method=self.read_string(reader), + url=self.read_string(reader), + request=self.read_string(reader), + response=self.read_string(reader), + status=self.read_uint(reader), + duration=self.read_uint(reader) + ) + if message_id == 52: return DomDrop( timestamp=self.read_uint(reader) @@ -558,6 +570,33 @@ class MessageCodec(Codec): if message_id == 65: return PageClose() + if mesage_id == 66: + return AssetCache( + url=self.read_string(reader) + ) + + if message_id == 67: + return CSSInsertRuleURLBased( + id=self.read_uint(reader), + rule=self.read_string(reader), + index=self.read_uint(reader), + based_url=self.read_string(reader) + ) + + if message_id == 69: + return MouseClick( + id=self.read_uint(reader), + hesitation_time=self.read_uint(reader), + label=self.read_string(reader), + selector=self.read_string(reader) + ) + + if message_id == 70: + return CreateIFrameDocument( + frame_id=self.read_uint(reader), + id=self.read_uint(reader) + ) + if message_id == 90: return IOSSessionStart( timestamp=self.read_uint(reader), @@ -585,6 +624,14 @@ class MessageCodec(Codec): value=self.read_string(reader) ) + if message_id == 93: + return IOSCustomEvent( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + name=self.read_string(reader), + payload=self.read_string(reader) + ) + if message_id == 94: return IOSUserID( timestamp=self.read_uint(reader), @@ -599,6 +646,33 @@ class MessageCodec(Codec): value=self.read_string(reader) ) + if message_id == 96: + return IOSScreenChanges( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + x=self.read_uint(reader), + y=self.read_uint(reader), + width=self.read_uint(reader), + height=self.read_uint(reader) + ) + + if message_id == 97: + return IOSCrash( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + name=self.read_string(reader), + reason=self.read_string(reader), + stacktrace=self.read_string(reader) + ) + + if message_id == 98: + return IOSSCreenEnter( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + title=self.read_string(reader), + view_name=self.read_string(reader) + ) + if message_id == 99: return IOSScreenLeave( timestamp=self.read_uint(reader), @@ -607,6 +681,32 @@ class MessageCodec(Codec): view_name=self.read_string(reader) ) + if message_id == 100: + return IOSClickEvent( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + label=self.read_string(reader), + x=self.read_uint(reader), + y=self.read_uint(reader) + ) + + if message_if == 101: + return IOSInputEvent( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + value=self.read_string(reader), + value_masked=self.read_boolean(reader), + label=self.read_string(reader) + ) + + if message_id == 102: + return IOSPreformanceEvent( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + name=self.read_string(reader), + value=self.read_uint(reader) + ) + if message_id == 103: return IOSLog( timestamp=self.read_uint(reader), @@ -622,6 +722,26 @@ class MessageCodec(Codec): content=self.read_string(reader) ) + if message_id == 105: + return IOSNetworkCall( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + duration=self.read_uint(reader), + headers=self.read_string(reader), + body=self.read_string(reader), + url=self.read_string(reader), + success=self.read_boolean(reader), + method=self.read_string(reader), + status=self.read_uint(reader) + ) + + if message_id == 107: + return IOSBatchMeta( + timestamp=self.read_uint(reader), + length=self.read_uint(reader), + first_index=self.read_uint(reader) + ) + if message_id == 110: return IOSPerformanceAggregated( timestamp_start=self.read_uint(reader), @@ -639,6 +759,14 @@ class MessageCodec(Codec): avg_battery=self.read_uint(reader), max_battery=self.read_uint(reader) ) + if message_id == 111: + return IOSIssueEvent( + timestamp=self.read_uint(reader), + type=self.read_string(reader), + context_string=self.read_string(reader), + context=self.read_string(reader), + payload=self.read_string(reader) + ) def read_message_id(self, reader: io.BytesIO) -> int: """ diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index c6e53b445..c026202fb 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -194,7 +194,7 @@ class MouseMove(Message): self.y = y -class MouseClick(Message): +class MouseClickDepricated(Message): __id__ = 21 def __init__(self, id, hesitation_time, label: str): @@ -518,6 +518,21 @@ class GraphQLEvent(Message): self.name = name +class FetchEvent(Message): + __id__ = 51 + + def __init__(self, message_id, timestamp, method: str, url, request, response: str, + status, duration): + self.message_id = message_id + self.timestamp = timestamp + self.method = method + self.url = url + self.request = request + self.response = response + self.status = status + self.duration = duration + + class DomDrop(Message): __id__ = 52 @@ -646,6 +661,41 @@ class PageClose(Message): __id__ = 65 +class AssetCache(Message): + __id__ = 66 + + def __init__(self, url): + self.url = url + + +class CSSInsertRuleURLBased(Message): + __id__ = 67 + + def __init__(self, id, rule, index, base_url): + self.id = id + self.rule = rule + self.index = index + self.base_url = base_url + + +class MouseClick(Message): + __id__ = 69 + + def __init__(self, id, hesitation_time, label: str, selector): + self.id = id + self.hesitation_time = hesitation_time + self.label = label + self.selector = selector + + +class CreateIFrameDocument(Message): + __id__ = 70 + + def __init__(self, frame_id, id): + self.frame_id = frame_id + self.id = id + + class IOSSessionStart(Message): __id__ = 90 @@ -681,6 +731,16 @@ class IOSMetadata(Message): self.value = value +class IOSCustomEvent(Message): + __id__ = 93 + + def __init__(self, timestamp, length, name: str, payload: str): + self.timestamp = timestamp + self.length = length + self.name = name + self.payload = payload + + class IOSUserID(Message): __id__ = 94 @@ -699,6 +759,39 @@ class IOSUserAnonymousID(Message): self.value = value +class IOSScreenChanges(Message): + __id__ = 96 + + def __init__(self, timestamp, length, x, y, width, height): + self.timestamp = timestamp + self.length = length + self.x = x + self.y = y + self.width = width + self.height = height + + +class IOSCrash(Message): + __id__ = 97 + + def __init__(self, timestamp, length, name: str, reason: str, stacktrace): + self.timestamp = timestamp + self.length = length + self.name = name + self.reason = reason + self.stacktrace = stacktrace + + +class IOSScreenEnter(Message): + __id__ = 98 + + def __init__(self, timestamp, length, title, view_name): + self.timestamp = timestamp + self.length = length + self.title = title + self.view_name = view_name + + class IOSScreenLeave(Message): __id__ = 99 @@ -709,6 +802,37 @@ class IOSScreenLeave(Message): self.view_name = view_name +class IOSClickEvent(Message): + __id__ = 100 + + def __init__(self, timestamp, length, label, x, y): + self.timestamp = timestamp + self.length = length + self.label = label + self.x = x + self.y = y + + +class IOSInputEvent(Message): + __id__ = 101 + + def __init__(self, timestamp, length, value: str, value_masked: bool, label: str): + self.timestamp = timestamp + self.length = length + self.value_masked = value_masked + self.label = label + + +class IOSPerformanceEvent(Message): + __id__ = 102 + + def __init__(self, timestamp, length, name: str, value): + self.timestamp = timestamp + self.length = length + self.name = name + self.value = value + + class IOSLog(Message): __id__ = 103 @@ -728,6 +852,30 @@ class IOSInternalError(Message): self.content = content +class IOSNetworkCall(Message): + __id__ = 105 + + def __init__(self, timestamp, length, duration, headers, body, url, success: bool, method: str, status): + self.timestamp = timestamp + self.length = length + self.duration = duration + self.headers = headers + self.body = body + self.url = url + self.success = success + self.method = method + self.status = status + + +class IOSBatchMeta(Message): + __id__ = 107 + + def __init__(self, timestamp, length, first_index): + self.timestamp = timestamp + self.length = length + self.first_index = first_index + + class IOSPerformanceAggregated(Message): __id__ = 110 @@ -750,3 +898,14 @@ class IOSPerformanceAggregated(Message): self.min_battery = min_battery self.avg_battery = avg_battery self.max_battery = max_battery + + +class IOSIssueEvent(Message): + __id__ = 111 + + def __init__(self, timestamp, type: str, context_string: str, context: str, payload: str): + self.timestamp = timestamp + self.type = type + self.context_string = context_string + self.context = context + self.payload = payload From c271a0b64e2e212a9fdba58b5e8d5f44dce60aed Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Wed, 13 Jul 2022 12:39:55 +0200 Subject: [PATCH 02/14] Solved some properties issues --- ee/connectors/msgcodec/codec.py | 31 +++++++++++++++++++++++++----- ee/connectors/msgcodec/messages.py | 8 ++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/ee/connectors/msgcodec/codec.py b/ee/connectors/msgcodec/codec.py index da06b5262..e26653478 100644 --- a/ee/connectors/msgcodec/codec.py +++ b/ee/connectors/msgcodec/codec.py @@ -1,5 +1,5 @@ import io - +from typing import List from msgcodec.messages import * @@ -24,6 +24,8 @@ class Codec: i = 0 # n of byte (max 9 for uint64) while True: b = reader.read(1) + if len(b) == 0: + raise IndexError('bytes out of range') num = int.from_bytes(b, "big", signed=False) # print(i, x) @@ -72,8 +74,20 @@ class MessageCodec(Codec): def decode(self, b: bytes) -> Message: reader = io.BytesIO(b) - message_id = self.read_message_id(reader) + return self.read_head_message(reader) + def decode_detailed(self, b: bytes) -> List[Message]: + reader = io.BytesIO(b) + messages_list = list() + while True: + try: + messages_list.append(self.read_head_message(reader)) + except IndexError: + break + return messages_list + + def read_head_message(self, reader: io.BytesIO) -> Message: + message_id = self.read_message_id(reader) if message_id == 0: return Timestamp( timestamp=self.read_uint(reader) @@ -570,7 +584,7 @@ class MessageCodec(Codec): if message_id == 65: return PageClose() - if mesage_id == 66: + if message_id == 66: return AssetCache( url=self.read_string(reader) ) @@ -580,7 +594,7 @@ class MessageCodec(Codec): id=self.read_uint(reader), rule=self.read_string(reader), index=self.read_uint(reader), - based_url=self.read_string(reader) + base_url=self.read_string(reader) ) if message_id == 69: @@ -597,6 +611,13 @@ class MessageCodec(Codec): id=self.read_uint(reader) ) + if message_id == 80: + return BatchMeta( + page_no=self.read_uint(reader), + first_index=self.read_uint(reader), + timestamp=self.read_int(reader) + ) + if message_id == 90: return IOSSessionStart( timestamp=self.read_uint(reader), @@ -690,7 +711,7 @@ class MessageCodec(Codec): y=self.read_uint(reader) ) - if message_if == 101: + if message_id == 101: return IOSInputEvent( timestamp=self.read_uint(reader), length=self.read_uint(reader), diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index c026202fb..bc451b287 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -696,6 +696,14 @@ class CreateIFrameDocument(Message): self.id = id +class BatchMeta(Message): + __id__ = 80 + + def __init__(self, page_no, first_index, timestamp): + self.page_no = page_no + self.first_index = first_index + self.timestamp = timestamp + class IOSSessionStart(Message): __id__ = 90 From 2e8f582bf77b54fe99abf4751b5e168246648da3 Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Wed, 13 Jul 2022 12:44:51 +0200 Subject: [PATCH 03/14] consumer.py now working with new message types and BigQuery --- ee/connectors/consumer.py | 86 ++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/ee/connectors/consumer.py b/ee/connectors/consumer.py index dfa856501..f65633f7d 100644 --- a/ee/connectors/consumer.py +++ b/ee/connectors/consumer.py @@ -30,59 +30,61 @@ def main(): codec = MessageCodec() consumer = KafkaConsumer(security_protocol="SSL", - bootstrap_servers=[os.environ['KAFKA_SERVER_1'], - os.environ['KAFKA_SERVER_2']], - group_id=f"connector_{DATABASE}", + bootstrap_servers=os.environ['KAFKA_SERVER_2'], + os.environ['KAFKA_SERVER_1']], + group_id=f"my_test3_connector_{DATABASE}", auto_offset_reset="earliest", - enable_auto_commit=False) + enable_auto_commit=False + ) - consumer.subscribe(topics=["events", "messages"]) + consumer.subscribe(topics=["raw", "raw_ios"]) print("Kafka consumer subscribed") for msg in consumer: - message = codec.decode(msg.value) - if message is None: + messages = codec.decode_detailed(msg.value) + session_id = codec.decode_key(msg.key) + if messages is None: print('-') continue + for message in messages: + if LEVEL == 'detailed': + n = handle_message(message) + elif LEVEL == 'normal': + n = handle_normal_message(message) - if LEVEL == 'detailed': - n = handle_message(message) - elif LEVEL == 'normal': - n = handle_normal_message(message) - - session_id = codec.decode_key(msg.key) - sessions[session_id] = handle_session(sessions[session_id], message) - if sessions[session_id]: - sessions[session_id].sessionid = session_id - - # put in a batch for insertion if received a SessionEnd - if isinstance(message, SessionEnd): + #session_id = codec.decode_key(msg.key) + sessions[session_id] = handle_session(sessions[session_id], message) if sessions[session_id]: - sessions_batch.append(sessions[session_id]) + sessions[session_id].sessionid = session_id - # try to insert sessions - if len(sessions_batch) >= sessions_batch_size: - attempt_session_insert(sessions_batch) - for s in sessions_batch: - try: - del sessions[s.sessionid] - except KeyError as e: - print(repr(e)) - sessions_batch = [] + # put in a batch for insertion if received a SessionEnd + if isinstance(message, SessionEnd): + if sessions[session_id]: + sessions_batch.append(sessions[session_id]) - if n: - n.sessionid = session_id - n.received_at = int(datetime.now().timestamp() * 1000) - n.batch_order_number = len(batch) - batch.append(n) - else: - continue + # try to insert sessions + if len(sessions_batch) >= sessions_batch_size: + attempt_session_insert(sessions_batch) + for s in sessions_batch: + try: + del sessions[s.sessionid] + except KeyError as e: + print(repr(e)) + sessions_batch = [] - # insert a batch of events - if len(batch) >= batch_size: - attempt_batch_insert(batch) - batch = [] - consumer.commit() - print("sessions in cache:", len(sessions)) + if n: + n.sessionid = session_id + n.received_at = int(datetime.now().timestamp() * 1000) + n.batch_order_number = len(batch) + batch.append(n) + else: + continue + + # insert a batch of events + if len(batch) >= batch_size: + attempt_batch_insert(batch) + batch = [] + consumer.commit() + print("sessions in cache:", len(sessions)) def attempt_session_insert(sess_batch): From fb42eded2c059d47508e148af886685e16cc5afe Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Wed, 13 Jul 2022 13:36:08 +0200 Subject: [PATCH 04/14] fixed minor issue consumer.py --- ee/connectors/consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/connectors/consumer.py b/ee/connectors/consumer.py index f65633f7d..acf55f2d5 100644 --- a/ee/connectors/consumer.py +++ b/ee/connectors/consumer.py @@ -30,7 +30,7 @@ def main(): codec = MessageCodec() consumer = KafkaConsumer(security_protocol="SSL", - bootstrap_servers=os.environ['KAFKA_SERVER_2'], + bootstrap_servers=[os.environ['KAFKA_SERVER_2'], os.environ['KAFKA_SERVER_1']], group_id=f"my_test3_connector_{DATABASE}", auto_offset_reset="earliest", From eabc52b7a666fa5e85490b0f6f6bd75bc48e50e1 Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Fri, 15 Jul 2022 17:06:11 +0200 Subject: [PATCH 05/14] Fixed and updated detailed sessions --- ee/connectors/bigquery_utils/create_table.py | 240 +++++++++++++++++++ 1 file changed, 240 insertions(+) diff --git a/ee/connectors/bigquery_utils/create_table.py b/ee/connectors/bigquery_utils/create_table.py index 4b166e4ae..9fd7ef1cc 100644 --- a/ee/connectors/bigquery_utils/create_table.py +++ b/ee/connectors/bigquery_utils/create_table.py @@ -128,6 +128,246 @@ def create_events_table(creds_file, table_id): bigquery.SchemaField("batch_order_number", "INT64")] create_table(creds_file, table_id, schema) +def create_events_detailed_table(creds_file, table_id): + #TODO: change types according to field + schema = [ + bigquery.SchemaField("sessionid", "INT64", mode="REQUIRED"), + bigquery.SchemaField("clickevent_hesitationtime", "INT64"), + bigquery.SchemaField("clickevent_label", "STRING"), + bigquery.SchemaField("clickevent_messageid", "INT64"), + bigquery.SchemaField("clickevent_timestamp", "INT64"), + bigquery.SchemaField("connectioninformation_downlink", "INT64"), + bigquery.SchemaField("connectioninformation_type", "STRING"), + bigquery.SchemaField("consolelog_level", "STRING"), + bigquery.SchemaField("consolelog_value", "STRING"), + bigquery.SchemaField("cpuissue_duration", "INT64"), + bigquery.SchemaField("cpuissue_rate", "INT64"), + bigquery.SchemaField("cpuissue_timestamp", "INT64"), + bigquery.SchemaField("createdocument", "BOOL"), + bigquery.SchemaField("createelementnode_id", "INT64"), + bigquery.SchemaField("createelementnode_parentid", "INT64"), + bigquery.SchemaField("cssdeleterule_index", "INT64"), + bigquery.SchemaField("cssdeleterule_stylesheetid", "INT64"), + bigquery.SchemaField("cssinsertrule_index", "INT64"), + bigquery.SchemaField("cssinsertrule_rule", "STRING"), + bigquery.SchemaField("cssinsertrule_stylesheetid", "INT64"), + bigquery.SchemaField("customevent_messageid", "INT64"), + bigquery.SchemaField("customevent_name", "STRING"), + bigquery.SchemaField("customevent_payload", "STRING"), + bigquery.SchemaField("customevent_timestamp", "INT64"), + bigquery.SchemaField("domdrop_timestamp", "INT64"), + bigquery.SchemaField("errorevent_message", "STRING"), + bigquery.SchemaField("errorevent_messageid", "INT64"), + bigquery.SchemaField("errorevent_name", "STRING"), + bigquery.SchemaField("errorevent_payload", "STRING"), + bigquery.SchemaField("errorevent_source", "STRING"), + bigquery.SchemaField("errorevent_timestamp", "INT64"), + bigquery.SchemaField("fetch_duration", "INT64"), + bigquery.SchemaField("fetch_method", "STRING"), + bigquery.SchemaField("fetch_request", "STRING"), + bigquery.SchemaField("fetch_response", "STRING"), + bigquery.SchemaField("fetch_status", "INT64"), + bigquery.SchemaField("fetch_timestamp", "INT64"), + bigquery.SchemaField("fetch_url", "STRING"), + bigquery.SchemaField("graphql_operationkind", "STRING"), + bigquery.SchemaField("graphql_operationname", "STRING"), + bigquery.SchemaField("graphql_response", "STRING"), + bigquery.SchemaField("graphql_variables", "STRING"), + bigquery.SchemaField("graphqlevent_messageid", "INT64"), + bigquery.SchemaField("graphqlevent_name", "STRING"), + bigquery.SchemaField("graphqlevent_timestamp", "INT64"), + bigquery.SchemaField("inputevent_label", "STRING"), + bigquery.SchemaField("inputevent_messageid", "INT64"), + bigquery.SchemaField("inputevent_timestamp", "INT64"), + bigquery.SchemaField("inputevent_value", "STRING"), + bigquery.SchemaField("inputevent_valuemasked", "BOOL"), + bigquery.SchemaField("jsexception_message", "STRING"), + bigquery.SchemaField("jsexception_name", "STRING"), + bigquery.SchemaField("jsexception_payload", "STRING"), + bigquery.SchemaField("memoryissue_duration", "INT64"), + bigquery.SchemaField("memoryissue_rate", "INT64"), + bigquery.SchemaField("memoryissue_timestamp", "INT64"), + bigquery.SchemaField("metadata_key", "STRING"), + bigquery.SchemaField("metadata_value", "STRING"), + bigquery.SchemaField("mobx_payload", "STRING"), + bigquery.SchemaField("mobx_type", "STRING"), + bigquery.SchemaField("mouseclick_id", "INT64"), + bigquery.SchemaField("mouseclick_hesitationtime", "INT64"), + bigquery.SchemaField("mouseclick_label", "STRING"), + bigquery.SchemaField("mousemove_x", "INT64"), + bigquery.SchemaField("mousemove_y", "INT64"), + bigquery.SchemaField("movenode_id", "INT64"), + bigquery.SchemaField("movenode_index", "INT64"), + bigquery.SchemaField("movenode_parentid", "INT64"), + bigquery.SchemaField("ngrx_action", "STRING"), + bigquery.SchemaField("ngrx_duration", "INT64"), + bigquery.SchemaField("ngrx_state", "STRING"), + bigquery.SchemaField("otable_key", "STRING"), + bigquery.SchemaField("otable_value", "STRING"), + bigquery.SchemaField("pageevent_domcontentloadedeventend", "INT64"), + bigquery.SchemaField("pageevent_domcontentloadedeventstart", "INT64"), + bigquery.SchemaField("pageevent_firstcontentfulpaint", "INT64"), + bigquery.SchemaField("pageevent_firstpaint", "INT64"), + bigquery.SchemaField("pageevent_loaded", "BOOL"), + bigquery.SchemaField("pageevent_loadeventend", "INT64"), + bigquery.SchemaField("pageevent_loadeventstart", "INT64"), + bigquery.SchemaField("pageevent_messageid", "INT64"), + bigquery.SchemaField("pageevent_referrer", "STRING"), + bigquery.SchemaField("pageevent_requeststart", "INT64"), + bigquery.SchemaField("pageevent_responseend", "INT64"), + bigquery.SchemaField("pageevent_responsestart", "INT64"), + bigquery.SchemaField("pageevent_speedindex", "INT64"), + bigquery.SchemaField("pageevent_timestamp", "INT64"), + bigquery.SchemaField("pageevent_url", "STRING"), + bigquery.SchemaField("pageloadtiming_domcontentloadedeventend", "INT64"), + bigquery.SchemaField("pageloadtiming_domcontentloadedeventstart", "INT64"), + bigquery.SchemaField("pageloadtiming_firstcontentfulpaint", "INT64"), + bigquery.SchemaField("pageloadtiming_firstpaint", "INT64"), + bigquery.SchemaField("pageloadtiming_loadeventend", "INT64"), + bigquery.SchemaField("pageloadtiming_loadeventstart", "INT64"), + bigquery.SchemaField("pageloadtiming_requeststart", "INT64"), + bigquery.SchemaField("pageloadtiming_responseend", "INT64"), + bigquery.SchemaField("pageloadtiming_responsestart", "INT64"), + bigquery.SchemaField("pagerendertiming_speedindex", "INT64"), + bigquery.SchemaField("pagerendertiming_timetointeractive", "INT64"), + bigquery.SchemaField("pagerendertiming_visuallycomplete", "INT64"), + bigquery.SchemaField("performancetrack_frames", "INT64"), + bigquery.SchemaField("performancetrack_ticks", "INT64"), + bigquery.SchemaField("performancetrack_totaljsheapsize", "INT64"), + bigquery.SchemaField("performancetrack_usedjsheapsize", "INT64"), + bigquery.SchemaField("performancetrackaggr_avgcpu", "INT64"), + bigquery.SchemaField("performancetrackaggr_avgfps", "INT64"), + bigquery.SchemaField("performancetrackaggr_avgtotaljsheapsize", "INT64"), + bigquery.SchemaField("performancetrackaggr_avgusedjsheapsize", "INT64"), + bigquery.SchemaField("performancetrackaggr_maxcpu", "INT64"), + bigquery.SchemaField("performancetrackaggr_maxfps", "INT64"), + bigquery.SchemaField("performancetrackaggr_maxtotaljsheapsize", "INT64"), + bigquery.SchemaField("performancetrackaggr_maxusedjsheapsize", "INT64"), + bigquery.SchemaField("performancetrackaggr_mincpu", "INT64"), + bigquery.SchemaField("performancetrackaggr_minfps", "INT64"), + bigquery.SchemaField("performancetrackaggr_mintotaljsheapsize", "INT64"), + bigquery.SchemaField("performancetrackaggr_minusedjsheapsize", "INT64"), + bigquery.SchemaField("performancetrackaggr_timestampend", "INT64"), + bigquery.SchemaField("performancetrackaggr_timestampstart", "INT64"), + bigquery.SchemaField("profiler_args", "STRING"), + bigquery.SchemaField("profiler_duration", "INT64"), + bigquery.SchemaField("profiler_name", "STRING"), + bigquery.SchemaField("profiler_result", "STRING"), + bigquery.SchemaField("rawcustomevent_name", "STRING"), + bigquery.SchemaField("rawcustomevent_payload", "STRING"), + bigquery.SchemaField("rawerrorevent_message", "STRING"), + bigquery.SchemaField("rawerrorevent_name", "STRING"), + bigquery.SchemaField("rawerrorevent_payload", "STRING"), + bigquery.SchemaField("rawerrorevent_source", "STRING"), + bigquery.SchemaField("rawerrorevent_timestamp", "INT64"), + bigquery.SchemaField("redux_action", "STRING"), + bigquery.SchemaField("redux_duration", "INT64"), + bigquery.SchemaField("redux_state", "STRING"), + bigquery.SchemaField("removenode_id", "INT64"), + bigquery.SchemaField("removenodeattribute_id", "INT64"), + bigquery.SchemaField("removenodeattribute_name", "STRING"), + bigquery.SchemaField("resourceevent_decodedbodysize", "INT64"), + bigquery.SchemaField("resourceevent_duration", "INT64"), + bigquery.SchemaField("resourceevent_encodedbodysize", "INT64"), + bigquery.SchemaField("resourceevent_headersize", "INT64"), + bigquery.SchemaField("resourceevent_messageid", "INT64"), + bigquery.SchemaField("resourceevent_method", "STRING"), + bigquery.SchemaField("resourceevent_status", "INT64"), + bigquery.SchemaField("resourceevent_success", "BOOL"), + bigquery.SchemaField("resourceevent_timestamp", "INT64"), + bigquery.SchemaField("resourceevent_ttfb", "INT64"), + bigquery.SchemaField("resourceevent_type", "STRING"), + bigquery.SchemaField("resourceevent_url", "STRING"), + bigquery.SchemaField("resourcetiming_decodedbodysize", "INT64"), + bigquery.SchemaField("resourcetiming_duration", "INT64"), + bigquery.SchemaField("resourcetiming_encodedbodysize", "INT64"), + bigquery.SchemaField("resourcetiming_headersize", "INT64"), + bigquery.SchemaField("resourcetiming_initiator", "STRING"), + bigquery.SchemaField("resourcetiming_timestamp", "INT64"), + bigquery.SchemaField("resourcetiming_ttfb", "INT64"), + bigquery.SchemaField("resourcetiming_url", "STRING"), + bigquery.SchemaField("sessiondisconnect", "BOOL"), + bigquery.SchemaField("sessiondisconnect_timestamp", "INT64"), + bigquery.SchemaField("sessionend", "BOOL"), + bigquery.SchemaField("sessionend_timestamp", "INT64"), + bigquery.SchemaField("sessionstart_projectid", "INT64"), + bigquery.SchemaField("sessionstart_revid", "STRING"), + bigquery.SchemaField("sessionstart_timestamp", "INT64"), + bigquery.SchemaField("sessionstart_trackerversion", "STRING"), + bigquery.SchemaField("sessionstart_useragent", "STRING"), + bigquery.SchemaField("sessionstart_userbrowser", "STRING"), + bigquery.SchemaField("sessionstart_userbrowserversion", "STRING"), + bigquery.SchemaField("sessionstart_usercountry", "STRING"), + bigquery.SchemaField("sessionstart_userdevice", "STRING"), + bigquery.SchemaField("sessionstart_userdeviceheapsize", "INT64"), + bigquery.SchemaField("sessionstart_userdevicememorysize", "INT64"), + bigquery.SchemaField("sessionstart_userdevicetype", "STRING"), + bigquery.SchemaField("sessionstart_useros", "STRING"), + bigquery.SchemaField("sessionstart_userosversion", "STRING"), + bigquery.SchemaField("sessionstart_useruuid", "STRING"), + bigquery.SchemaField("setcssdata_data", "INT64"), + bigquery.SchemaField("setcssdata_id", "INT64"), + bigquery.SchemaField("setinputchecked_checked", "INT64"), + bigquery.SchemaField("setinputchecked_id", "INT64"), + bigquery.SchemaField("setinputtarget_id", "INT64"), + bigquery.SchemaField("setinputtarget_label", "INT64"), + bigquery.SchemaField("setinputvalue_id", "INT64"), + bigquery.SchemaField("setinputvalue_mask", "INT64"), + bigquery.SchemaField("setinputvalue_value", "INT64"), + bigquery.SchemaField("setnodeattribute_id", "INT64"), + bigquery.SchemaField("setnodeattribute_name", "INT64"), + bigquery.SchemaField("setnodeattribute_value", "INT64"), + bigquery.SchemaField("setnodedata_data", "INT64"), + bigquery.SchemaField("setnodedata_id", "INT64"), + bigquery.SchemaField("setnodescroll_id", "INT64"), + bigquery.SchemaField("setnodescroll_x", "INT64"), + bigquery.SchemaField("setnodescroll_y", "INT64"), + bigquery.SchemaField("setpagelocation_navigationstart", "INT64"), + bigquery.SchemaField("setpagelocation_referrer", "STRING"), + bigquery.SchemaField("setpagelocation_url", "STRING"), + bigquery.SchemaField("setpagevisibility_hidden", "BOOL"), + bigquery.SchemaField("setviewportscroll_x", "INT64"), + bigquery.SchemaField("setviewportscroll_y", "INT64"), + bigquery.SchemaField("setviewportsize_height", "INT64"), + bigquery.SchemaField("setviewportsize_width", "INT64"), + bigquery.SchemaField("stateaction_type", "STRING"), + bigquery.SchemaField("stateactionevent_messageid", "INT64"), + bigquery.SchemaField("stateactionevent_timestamp", "INT64"), + bigquery.SchemaField("stateactionevent_type", "STRING"), + bigquery.SchemaField("timestamp_timestamp", "INT64"), + bigquery.SchemaField("useranonymousid_id", "STRING"), + bigquery.SchemaField("userid_id", "STRING"), + bigquery.SchemaField("vuex_mutation", "STRING"), + bigquery.SchemaField("vuex_state", "STRING"), + bigquery.SchemaField("longtasks_timestamp", "INT64"), + bigquery.SchemaField("longtasks_duration", "INT64"), + bigquery.SchemaField("longtasks_context", "FLOAT64"), + bigquery.SchemaField("longtasks_containertype", "FLOAT64"), + bigquery.SchemaField("longtasks_containersrc", "STRING"), + bigquery.SchemaField("longtasks_containerid", "STRING"), + bigquery.SchemaField("longtasks_containername", "FLOAT64"), + bigquery.SchemaField("setnodeurlbasedattribute_id", "FLOAT64"), + bigquery.SchemaField("setnodeurlbasedattribute_name", "STRING"), + bigquery.SchemaField("setnodeurlbasedattribute_value", "STRING"), + bigquery.SchemaField("setnodeurlbasedattribute_baseurl", "STRING"), + bigquery.SchemaField("setstyledata_id", "FLOAT64"), + bigquery.SchemaField("setstyledata_data", "STRING"), #CAUSING ISSUES + bigquery.SchemaField("setstyledata_baseurl", "STRING"), #CAUSING ISSUES + bigquery.SchemaField("issueevent_messageid", "FLOAT64"), + bigquery.SchemaField("issueevent_timestamp", "FLOAT64"), + bigquery.SchemaField("issueevent_type", "FLOAT64"), + bigquery.SchemaField("issueevent_contextstring", "FLOAT64"), + bigquery.SchemaField("issueevent_context", "FLOAT64"), + bigquery.SchemaField("issueevent_payload", "FLOAT64"), + bigquery.SchemaField("technicalinfo_type", "FLOAT64"), + bigquery.SchemaField("technicalinfo_value", "FLOAT64"), + bigquery.SchemaField("customissue_name", "FLOAT64"), + bigquery.SchemaField("customissue_payload", "FLOAT64"), + bigquery.SchemaField("pageclose", "FLOAT64"), + bigquery.SchemaField("received_at", "INT64", mode="REQUIRED"), + bigquery.SchemaField("batch_order_number", "INT64", mode="REQUIRED") + ] + create_table(creds_file, table_id, schema) def create_table_negatives(creds_file, table_id): client = bigquery.Client.from_service_account_json(creds_file) From b446147cb8d0d8c883cf55e0c14692fe288621cf Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Fri, 15 Jul 2022 17:12:22 +0200 Subject: [PATCH 06/14] solved some minor issues --- ee/connectors/db/models.py | 14 ++-- ee/connectors/db/utils.py | 1 + ee/connectors/handler.py | 141 +++++++++++++++++++++++++++++++- ee/connectors/msgcodec/codec.py | 2 +- 4 files changed, 149 insertions(+), 9 deletions(-) diff --git a/ee/connectors/db/models.py b/ee/connectors/db/models.py index 82af90ad8..74cd23230 100644 --- a/ee/connectors/db/models.py +++ b/ee/connectors/db/models.py @@ -346,13 +346,13 @@ class DetailedEvent(Base): userid_id = Column(VARCHAR(5000)) vuex_mutation = Column(VARCHAR(5000)) vuex_state = Column(VARCHAR(5000)) - longtask_timestamp = Column(BigInteger) - longtask_duration = Column(BigInteger) - longtask_context = Column(BigInteger) - longtask_containertype = Column(BigInteger) - longtask_containersrc = Column(VARCHAR(5000)) - longtask_containerid = Column(VARCHAR(5000)) - longtask_containername = Column(VARCHAR(5000)) + longtasks_timestamp = Column(BigInteger) + longtasks_duration = Column(BigInteger) + longtasks_context = Column(BigInteger) + longtasks_containertype = Column(BigInteger) + longtasks_containersrc = Column(VARCHAR(5000)) + longtasks_containerid = Column(VARCHAR(5000)) + longtasks_containername = Column(VARCHAR(5000)) setnodeurlbasedattribute_id = Column(BigInteger) setnodeurlbasedattribute_name = Column(VARCHAR(5000)) setnodeurlbasedattribute_value = Column(VARCHAR(5000)) diff --git a/ee/connectors/db/utils.py b/ee/connectors/db/utils.py index 7c268c6b3..a51806289 100644 --- a/ee/connectors/db/utils.py +++ b/ee/connectors/db/utils.py @@ -336,6 +336,7 @@ def get_df_from_batch(batch, level): df = pd.DataFrame([b.__dict__ for b in batch], columns=detailed_events_col) if level == 'sessions': df = pd.DataFrame([b.__dict__ for b in batch], columns=sessions_col) + print('if error is after me, then problem is later') try: df = df.drop('_sa_instance_state', axis=1) diff --git a/ee/connectors/handler.py b/ee/connectors/handler.py index 5167c7800..6f6316545 100644 --- a/ee/connectors/handler.py +++ b/ee/connectors/handler.py @@ -49,6 +49,14 @@ def handle_normal_message(message: Message) -> Optional[Event]: n.mouseclick_hesitationtime = message.hesitation_time n.mouseclick_id = message.id n.mouseclick_label = message.label + n.mouseclick_selector = message.selector + return n + + if isinstance(message, MouseClickDepricated): + n.mouseclick_hesitationtime = message.hesitation_time + n.mouseclick_id = message.id + n.mouseclick_label = message.label + n.mouseclick_selector = '' return n if isinstance(message, PageEvent): @@ -132,6 +140,18 @@ def handle_session(n: Session, message: Message) -> Optional[Session]: pass return n + if isinstance(message, BatchMeta): + n.batchmeta_page_no = message.page_no + n.batchmeta_first_index = message.first_index + n.batchmeta_timestamp = message.timestamp + return n + + # if isinstance(message, IOSBatchMeta): + # n.iosbatchmeta_page_no = message.page_no + # n.iosbatchmeta_first_index = message.first_index + # n.iosbatchmeta_timestamp = message.timestamp + # return n + if isinstance(message, ConnectionInformation): n.connection_effective_bandwidth = message.downlink n.connection_type = message.type @@ -224,7 +244,7 @@ def handle_session(n: Session, message: Message) -> Optional[Session]: n.inputs_count = 1 return n - if isinstance(message, MouseClick): + if isinstance(message, MouseClickDepricated): try: n.inputs_count += 1 except TypeError: @@ -278,6 +298,11 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.sessionstart_usercountry = message.user_country return n + if isinstance(message, CreateIFrameDocument): + n.create_iframedocument_frame_id = message.frame_id + n.create_iframedocument_id = message.id + return n + if isinstance(message, SetViewportSize): n.setviewportsize_width = message.width n.setviewportsize_height = message.height @@ -360,6 +385,12 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.metadata_value = message.value return n + if isinstance(message, BatchMeta): + n.batchmeta_page_no = message.page_no + n.batchmeta_first_index = message.first_index + n.batchmeta_timestamp = message.timestamp + return n + if isinstance(message, PerformanceTrack): n.performancetrack_frames = message.frames n.performancetrack_ticks = message.ticks @@ -470,6 +501,12 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: # n.cssinsertrule_rule = message.rule # n.cssinsertrule_index = message.index # return n + + # if isinstance(message, CSSInsertRuleURLBased): + # n.cssinsertrule_urlbased_id = message.id + # n.cssinsertrule_urlbased_rule = message.rule + # n.cssinsertrule_urlbased_index = message.index + # n.cssinsertrule_urlbased_base_url = message.base_url # # if isinstance(message, CSSDeleteRule): # n.cssdeleterule_stylesheetid = message.id @@ -485,6 +522,17 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.fetch_duration = message.duration return n + if isinstance(message, FetchEvent): + n.fetch_event_message_id = message.message_id + n.fetch_event_timestamp = message.timestamp + n.fetch_event_method = message.method + n.fetch_event_url = message.url + n.fetch_event_request = message.request + n.fetch_event_response = message.response + n.fetch_event_status = message.status + n.fetch_event_duration = message.duration + return n + if isinstance(message, Profiler): n.profiler_name = message.name n.profiler_duration = message.duration @@ -513,6 +561,14 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.mouseclick_id = message.id n.mouseclick_hesitationtime = message.hesitation_time n.mouseclick_label = message.label + n.mouseclick_selector = message.selector + return n + + if isinstance(message, MouseClickDepricated): + n.mouseclick_id = message.id + n.mouseclick_hesitationtime = message.hesitation_time + n.mouseclick_label = message.label + n.mouseclick_selector = '' return n if isinstance(message, SetPageLocation): @@ -572,6 +628,10 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.pageclose = True return n + if isinstance(message, AssetCache): + n.asset_cache_url = message.url + return n + if isinstance(message, IOSSessionStart): n.iossessionstart_timestamp = message.timestamp n.iossessionstart_projectid = message.project_id @@ -596,6 +656,12 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.iosmetadata_value = message.value return n + if isinstance(message, IOSBatchMeta): + n.iosbatchmeta_page_no = message.page_no + n.iosbatchmeta_first_index = message.first_index + n.iosbatchmeta_timestamp = message.timestamp + return n + if isinstance(message, IOSUserID): n.iosuserid_timestamp = message.timestamp n.iosuserid_length = message.length @@ -608,6 +674,13 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.iosuseranonymousid_value = message.value return n + if isinstance(message, IOSScreenEnter): + n.iosscreenenter_timestamp = message.timestamp + n.iosscreenenter_length = message.length + n.iosscreenenter_title = message.title + n.iosscreenenter_view_name = message.view_name + return n + if isinstance(message, IOSScreenLeave): n.iosscreenleave_timestamp = message.timestamp n.iosscreenleave_length = message.length @@ -615,6 +688,30 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.iosscreenleave_viewname = message.view_name return n + if isinstance(message, IOSScreenChanges): + n.iosscreenchanges_timestamp = message.timestamp + n.iosscreenchanges_length = message.length + n.iosscreenchanges_x = message.x + n.iosscreenchanges_y = message.y + n.iosscreenchanges_width = message.width + n.iosscreenchanges_height = message.height + return n + + if isinstance(message, IOSClickEvent): + n.iosclickevent_timestamp = message.timestamp + n.iosclickevent_length = message.length + n.iosclickevent_label = message.label + n.iosclickevent_x = message.x + n.iosclickevent_y = message.y + return n + + if isinstance(message, IOSInputEvent): + n.iosinputevent_timestamp = message.timestamp + n.iosinputevent_length = message.length + n.iosinputevent_value_masked = message.value_masked + n.iosinputevent_label = message.label + return n + if isinstance(message, IOSLog): n.ioslog_timestamp = message.timestamp n.ioslog_length = message.length @@ -622,12 +719,54 @@ def handle_message(message: Message) -> Optional[DetailedEvent]: n.ioslog_content = message.content return n + if isinstance(message, IOSNetworkCall): + n.iosnetworkcall_timestamp = message.timestamp + n.iosnetworkcall_length = message.length + n.iosnetworkcall_duration = message.duration + n.iosnetworkcall_headers = message.headers + n.iosnetworkcall_body = message.body + n.iosnetworkcall_url = message.url + n.iosnetworkcall_success = message.success + n.iosnetworkcall_method = message.method + n.iosnetworkcall_status = message.status + return n + + if isinstance(message, IOSIssueEvent): + n.iosissueevent_timestamp = message.timestamp + n.iosissueevent_type = message.type + n.iosissueevent_context_string = message.context_string + n.iosissueevent_context = message.context + n.iosissueevent_payload = message.payload + return n + + if isinstance(message, IOSCustomEvent): + n.ioscustomevent_timestamp = message.timestamp + n.ioscustomevent_length = message.length + n.ioscustomevent_name = message.name + n.ioscustomevent_payload = message.payload + return n + if isinstance(message, IOSInternalError): n.iosinternalerror_timestamp = message.timestamp n.iosinternalerror_length = message.length n.iosinternalerror_content = message.content return n + if isinstance(message, IOSCrash): + n.ioscrash_timestamp = message.timestamp + n.ioscrash_length = message.length + n.ioscrash_name = message.name + n.ioscrash_reason = message.reason + n.ioscrash_stacktrace = message.stacktrace + return n + + if isinstance(message, IOSPerformanceEvent): + n.iosperformanceevent_timestamp = message.timestamp + n.iosperformanceevent_length = message.length + n.iosperformanceevent_name = message.name + n.iosperformanceevent_value = message.value + return n + if isinstance(message, IOSPerformanceAggregated): n.iosperformanceaggregated_timestampstart = message.timestamp_start n.iosperformanceaggregated_timestampend = message.timestamp_end diff --git a/ee/connectors/msgcodec/codec.py b/ee/connectors/msgcodec/codec.py index e26653478..7e1aabbf9 100644 --- a/ee/connectors/msgcodec/codec.py +++ b/ee/connectors/msgcodec/codec.py @@ -687,7 +687,7 @@ class MessageCodec(Codec): ) if message_id == 98: - return IOSSCreenEnter( + return IOSScreenEnter( timestamp=self.read_uint(reader), length=self.read_uint(reader), title=self.read_string(reader), From 02cbeca7712701ee816afdca0c936db52d35e1a9 Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Fri, 15 Jul 2022 17:15:56 +0200 Subject: [PATCH 07/14] deleted some prints --- ee/connectors/db/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ee/connectors/db/utils.py b/ee/connectors/db/utils.py index a51806289..7c268c6b3 100644 --- a/ee/connectors/db/utils.py +++ b/ee/connectors/db/utils.py @@ -336,7 +336,6 @@ def get_df_from_batch(batch, level): df = pd.DataFrame([b.__dict__ for b in batch], columns=detailed_events_col) if level == 'sessions': df = pd.DataFrame([b.__dict__ for b in batch], columns=sessions_col) - print('if error is after me, then problem is later') try: df = df.drop('_sa_instance_state', axis=1) From de1977b0de40548b4c8d2ff329dec36d4fe0915d Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Mon, 18 Jul 2022 14:38:14 +0200 Subject: [PATCH 08/14] Changed types of some messages and force type when pandas fails to recognize column type --- ee/connectors/bigquery_utils/create_table.py | 20 ++++++++++---------- ee/connectors/db/utils.py | 9 +++++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/ee/connectors/bigquery_utils/create_table.py b/ee/connectors/bigquery_utils/create_table.py index 9fd7ef1cc..5dc4e9b9a 100644 --- a/ee/connectors/bigquery_utils/create_table.py +++ b/ee/connectors/bigquery_utils/create_table.py @@ -351,18 +351,18 @@ def create_events_detailed_table(creds_file, table_id): bigquery.SchemaField("setnodeurlbasedattribute_value", "STRING"), bigquery.SchemaField("setnodeurlbasedattribute_baseurl", "STRING"), bigquery.SchemaField("setstyledata_id", "FLOAT64"), - bigquery.SchemaField("setstyledata_data", "STRING"), #CAUSING ISSUES - bigquery.SchemaField("setstyledata_baseurl", "STRING"), #CAUSING ISSUES + bigquery.SchemaField("setstyledata_data", "STRING"), + bigquery.SchemaField("setstyledata_baseurl", "STRING"), bigquery.SchemaField("issueevent_messageid", "FLOAT64"), bigquery.SchemaField("issueevent_timestamp", "FLOAT64"), - bigquery.SchemaField("issueevent_type", "FLOAT64"), - bigquery.SchemaField("issueevent_contextstring", "FLOAT64"), - bigquery.SchemaField("issueevent_context", "FLOAT64"), - bigquery.SchemaField("issueevent_payload", "FLOAT64"), - bigquery.SchemaField("technicalinfo_type", "FLOAT64"), - bigquery.SchemaField("technicalinfo_value", "FLOAT64"), - bigquery.SchemaField("customissue_name", "FLOAT64"), - bigquery.SchemaField("customissue_payload", "FLOAT64"), + bigquery.SchemaField("issueevent_type", "STRING"), + bigquery.SchemaField("issueevent_contextstring", "STRING"), + bigquery.SchemaField("issueevent_context", "STRING"), + bigquery.SchemaField("issueevent_payload", "STRING"), + bigquery.SchemaField("technicalinfo_type", "STRING"), + bigquery.SchemaField("technicalinfo_value", "STRING"), + bigquery.SchemaField("customissue_name", "STRING"), + bigquery.SchemaField("customissue_payload", "STRING"), bigquery.SchemaField("pageclose", "FLOAT64"), bigquery.SchemaField("received_at", "INT64", mode="REQUIRED"), bigquery.SchemaField("batch_order_number", "INT64", mode="REQUIRED") diff --git a/ee/connectors/db/utils.py b/ee/connectors/db/utils.py index 7c268c6b3..cdc650884 100644 --- a/ee/connectors/db/utils.py +++ b/ee/connectors/db/utils.py @@ -355,8 +355,17 @@ def get_df_from_batch(batch, level): df['issues'] = df['issues'].fillna('') df['urls'] = df['urls'].fillna('') + forced_type_cols_str = ['setnodeurlbasedattribute_name', 'setnodeurlbasedattribute_value', + 'setnodeurlbasedattribute_baseurl', 'setstyledata_data', 'setstyledata_baseurl', + 'customissue_payload', 'customissue_name', 'technicalinfo_value', 'technicalinfo_type', + 'issueevent_payload', 'issueevent_context', 'issueevent_contextstring', 'issueevent_type'] + forced_type_cols_float = ['setnodeurlbasedattribute_id'] for x in df.columns: try: + if x in forced_type_cols_str: + df[x] = df[x].apply(str) + if x in forced_type_cols_float: + df[x] = df[x].apply(float) if df[x].dtype == 'string': df[x] = df[x].str.slice(0, 255) df[x] = df[x].str.replace("|", "") From f51b067ed8db3dd62a764ec053b6aa445810e3c6 Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Tue, 19 Jul 2022 11:19:17 +0200 Subject: [PATCH 09/14] Updated PG connector events --- ee/connectors/db/tables.py | 5 + .../sql/postgres_events_detailed.sql | 238 ++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 ee/connectors/sql/postgres_events_detailed.sql diff --git a/ee/connectors/db/tables.py b/ee/connectors/db/tables.py index 0127cbbd1..e3298825f 100644 --- a/ee/connectors/db/tables.py +++ b/ee/connectors/db/tables.py @@ -36,6 +36,11 @@ def create_tables_postgres(db): db.engine.execute(q) print(f"`connector_sessions` table created succesfully.") + #with open(base_path / 'sql' / 'postgres_events_detailed.sql') as f: + # q = f.read() + #db.engine.execute(q) + #print(f"`connector_user_events_detailed` table created succesfully.") + def create_tables_snowflake(db): with open(base_path / 'sql' / 'snowflake_events.sql') as f: diff --git a/ee/connectors/sql/postgres_events_detailed.sql b/ee/connectors/sql/postgres_events_detailed.sql new file mode 100644 index 000000000..092a50ea3 --- /dev/null +++ b/ee/connectors/sql/postgres_events_detailed.sql @@ -0,0 +1,238 @@ +CREATE TABLE IF NOT EXISTS connector_events_detailed +( + sessionid bigint, + clickevent_hesitationtime bigint, + clickevent_label text, + clickevent_messageid bigint, + clickevent_timestamp bigint, + connectioninformation_downlink bigint, + connectioninformation_type text, + consolelog_level text, + consolelog_value text, + cpuissue_duration bigint, + cpuissue_rate bigint, + cpuissue_timestamp bigint, + createdocument boolean, + createelementnode_id bigint, + createelementnode_parentid bigint, + cssdeleterule_index bigint, + cssdeleterule_stylesheetid bigint, + cssinsertrule_index bigint, + cssinsertrule_rule text, + cssinsertrule_stylesheetid bigint, + customevent_messageid bigint, + customevent_name text, + customevent_payload text, + customevent_timestamp bigint, + domdrop_timestamp bigint, + errorevent_message text, + errorevent_messageid bigint, + errorevent_name text, + errorevent_payload text, + errorevent_source text, + errorevent_timestamp bigint, + fetch_duration bigint, + fetch_method text, + fetch_request text, + fetch_response text, + fetch_status bigint, + fetch_timestamp bigint, + fetch_url text, + graphql_operationkind text, + graphql_operationname text, + graphql_response text, + graphql_variables text, + graphqlevent_messageid bigint, + graphqlevent_name text, + graphqlevent_timestamp bigint, + inputevent_label text, + inputevent_messageid bigint, + inputevent_timestamp bigint, + inputevent_value text, + inputevent_valuemasked boolean, + jsexception_message text, + jsexception_name text, + jsexception_payload text, + memoryissue_duration bigint, + memoryissue_rate bigint, + memoryissue_timestamp bigint, + metadata_key text, + metadata_value text, + mobx_payload text, + mobx_type text, + mouseclick_id bigint, + mouseclick_hesitationtime bigint, + mouseclick_label text, + mousemove_x bigint, + mousemove_y bigint, + movenode_id bigint, + movenode_index bigint, + movenode_parentid bigint, + ngrx_action text, + ngrx_duration bigint, + ngrx_state text, + otable_key text, + otable_value text, + pageevent_domcontentloadedeventend bigint, + pageevent_domcontentloadedeventstart bigint, + pageevent_firstcontentfulpaint bigint, + pageevent_firstpaint bigint, + pageevent_loaded boolean, + pageevent_loadeventend bigint, + pageevent_loadeventstart bigint, + pageevent_messageid bigint, + pageevent_referrer text, + pageevent_requeststart bigint, + pageevent_responseend bigint, + pageevent_responsestart bigint, + pageevent_speedindex bigint, + pageevent_timestamp bigint, + pageevent_url text, + pageloadtiming_domcontentloadedeventend bigint, + pageloadtiming_domcontentloadedeventstart bigint, + pageloadtiming_firstcontentfulpaint bigint, + pageloadtiming_firstpaint bigint, + pageloadtiming_loadeventend bigint, + pageloadtiming_loadeventstart bigint, + pageloadtiming_requeststart bigint, + pageloadtiming_responseend bigint, + pageloadtiming_responsestart bigint, + pagerendertiming_speedindex bigint, + pagerendertiming_timetointeractive bigint, + pagerendertiming_visuallycomplete bigint, + performancetrack_frames bigint, + performancetrack_ticks bigint, + performancetrack_totaljsheapsize bigint, + performancetrack_usedjsheapsize bigint, + performancetrackaggr_avgcpu bigint, + performancetrackaggr_avgfps bigint, + performancetrackaggr_avgtotaljsheapsize bigint, + performancetrackaggr_avgusedjsheapsize bigint, + performancetrackaggr_maxcpu bigint, + performancetrackaggr_maxfps bigint, + performancetrackaggr_maxtotaljsheapsize bigint, + performancetrackaggr_maxusedjsheapsize bigint, + performancetrackaggr_mincpu bigint, + performancetrackaggr_minfps bigint, + performancetrackaggr_mintotaljsheapsize bigint, + performancetrackaggr_minusedjsheapsize bigint, + performancetrackaggr_timestampend bigint, + performancetrackaggr_timestampstart bigint, + profiler_args text, + profiler_duration bigint, + profiler_name text, + profiler_result text, + rawcustomevent_name text, + rawcustomevent_payload text, + rawerrorevent_message text, + rawerrorevent_name text, + rawerrorevent_payload text, + rawerrorevent_source text, + rawerrorevent_timestamp bigint, + redux_action text, + redux_duration bigint, + redux_state text, + removenode_id bigint, + removenodeattribute_id bigint, + removenodeattribute_name text, + resourceevent_decodedbodysize bigint, + resourceevent_duration bigint, + resourceevent_encodedbodysize bigint, + resourceevent_headersize bigint, + resourceevent_messageid bigint, + resourceevent_method text, + resourceevent_status bigint, + resourceevent_success boolean, + resourceevent_timestamp bigint, + resourceevent_ttfb bigint, + resourceevent_type text, + resourceevent_url text, + resourcetiming_decodedbodysize bigint, + resourcetiming_duration bigint, + resourcetiming_encodedbodysize bigint, + resourcetiming_headersize bigint, + resourcetiming_initiator text, + resourcetiming_timestamp bigint, + resourcetiming_ttfb bigint, + resourcetiming_url text, + sessiondisconnect boolean, + sessiondisconnect_timestamp bigint, + sessionend boolean, + sessionend_timestamp bigint, + sessionstart_projectid bigint, + sessionstart_revid text, + sessionstart_timestamp bigint, + sessionstart_trackerversion text, + sessionstart_useragent text, + sessionstart_userbrowser text, + sessionstart_userbrowserversion text, + sessionstart_usercountry text, + sessionstart_userdevice text, + sessionstart_userdeviceheapsize bigint, + sessionstart_userdevicememorysize bigint, + sessionstart_userdevicetype text, + sessionstart_useros text, + sessionstart_userosversion text, + sessionstart_useruuid text, + setcssdata_data bigint, + setcssdata_id bigint, + setinputchecked_checked bigint, + setinputchecked_id bigint, + setinputtarget_id bigint, + setinputtarget_label bigint, + setinputvalue_id bigint, + setinputvalue_mask bigint, + setinputvalue_value bigint, + setnodeattribute_id bigint, + setnodeattribute_name bigint, + setnodeattribute_value bigint, + setnodedata_data bigint, + setnodedata_id bigint, + setnodescroll_id bigint, + setnodescroll_x bigint, + setnodescroll_y bigint, + setpagelocation_navigationstart bigint, + setpagelocation_referrer text, + setpagelocation_url text, + setpagevisibility_hidden boolean, + setviewportscroll_x bigint, + setviewportscroll_y bigint, + setviewportsize_height bigint, + setviewportsize_width bigint, + stateaction_type text, + stateactionevent_messageid bigint, + stateactionevent_timestamp bigint, + stateactionevent_type text, + timestamp_timestamp bigint, + useranonymousid_id text, + userid_id text, + vuex_mutation text, + vuex_state text, + longtasks_timestamp bigint, + longtasks_duration bigint, + longtasks_context bigint, + longtasks_containertype bigint, + longtasks_containersrc text, + longtasks_containerid text, + longtasks_containername bigint, + setnodeurlbasedattribute_id bigint, + setnodeurlbasedattribute_name text, + setnodeurlbasedattribute_value text, + setnodeurlbasedattribute_baseurl text, + setstyledata_id bigint, + setstyledata_data text, + setstyledata_baseurl text, + issueevent_messageid bigint, + issueevent_timestamp bigint, + issueevent_type text, + issueevent_contextstring text, + issueevent_context text, + issueevent_payload text, + technicalinfo_type text, + technicalinfo_value text, + customissue_name text, + customissue_payload text, + pageclose bigint, + received_at bigint, + batch_order_number bigint +); From 7abe9f96808484dd3a885a8bb57928830823ad4b Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Wed, 20 Jul 2022 10:54:53 +0200 Subject: [PATCH 10/14] Updated CH connector and event types --- ee/connectors/db/tables.py | 10 + ee/connectors/db/utils.py | 28 +- .../sql/clickhouse_events_detailed.sql | 242 ++++++++++++++++++ .../sql/clickhouse_events_detailed_buffer.sql | 238 +++++++++++++++++ 4 files changed, 508 insertions(+), 10 deletions(-) create mode 100644 ee/connectors/sql/clickhouse_events_detailed.sql create mode 100644 ee/connectors/sql/clickhouse_events_detailed_buffer.sql diff --git a/ee/connectors/db/tables.py b/ee/connectors/db/tables.py index e3298825f..db1c2ca42 100644 --- a/ee/connectors/db/tables.py +++ b/ee/connectors/db/tables.py @@ -24,6 +24,16 @@ def create_tables_clickhouse(db): db.engine.execute(q) print(f"`connector_sessions_buffer` table created succesfully.") + #with open(base_path / 'sql' / 'clickhouse_events_detailed.sql') as f: + # q = f.read() + #db.engine.execute(q) + #print(f"`connector_user_events_detailed` table created succesfully.") + + #with open(base_path / 'sql' / 'clickhouse_events_detailed_buffer.sql') as f: + # q = f.read() + #db.engine.execute(q) + #print(f"`connector_user_events_detailed_buffer` table created succesfully.") + def create_tables_postgres(db): with open(base_path / 'sql' / 'postgres_events.sql') as f: diff --git a/ee/connectors/db/utils.py b/ee/connectors/db/utils.py index cdc650884..698a6043a 100644 --- a/ee/connectors/db/utils.py +++ b/ee/connectors/db/utils.py @@ -264,7 +264,24 @@ dtypes_detailed_events = { "vuex_mutation": "object", "vuex_state": "string", "received_at": "Int64", - "batch_order_number": "Int64" + "batch_order_number": "Int64", + + #NEW + 'setnodeurlbasedattribute_id': 'Int64', + 'setnodeurlbasedattribute_name': 'string', + 'setnodeurlbasedattribute_value': 'string', + 'setnodeurlbasedattribute_baseurl': 'string', + 'setstyledata_id': 'Int64', + 'setstyledata_data': 'string', + 'setstyledata_baseurl': 'string', + 'customissue_payload': 'string', + 'customissue_name': 'string', + 'technicalinfo_value': 'string', + 'technicalinfo_type': 'string', + 'issueevent_payload': 'string', + 'issueevent_context': 'string', + 'issueevent_contextstring': 'string', + 'issueevent_type': 'string' } dtypes_sessions = {'sessionid': 'Int64', 'user_agent': 'string', @@ -355,17 +372,8 @@ def get_df_from_batch(batch, level): df['issues'] = df['issues'].fillna('') df['urls'] = df['urls'].fillna('') - forced_type_cols_str = ['setnodeurlbasedattribute_name', 'setnodeurlbasedattribute_value', - 'setnodeurlbasedattribute_baseurl', 'setstyledata_data', 'setstyledata_baseurl', - 'customissue_payload', 'customissue_name', 'technicalinfo_value', 'technicalinfo_type', - 'issueevent_payload', 'issueevent_context', 'issueevent_contextstring', 'issueevent_type'] - forced_type_cols_float = ['setnodeurlbasedattribute_id'] for x in df.columns: try: - if x in forced_type_cols_str: - df[x] = df[x].apply(str) - if x in forced_type_cols_float: - df[x] = df[x].apply(float) if df[x].dtype == 'string': df[x] = df[x].str.slice(0, 255) df[x] = df[x].str.replace("|", "") diff --git a/ee/connectors/sql/clickhouse_events_detailed.sql b/ee/connectors/sql/clickhouse_events_detailed.sql new file mode 100644 index 000000000..f029184e3 --- /dev/null +++ b/ee/connectors/sql/clickhouse_events_detailed.sql @@ -0,0 +1,242 @@ +CREATE TABLE IF NOT EXISTS connector_events_detailed +( + sessionid UInt64, + clickevent_hesitationtime Nullable(UInt64), + clickevent_label Nullable(String), + clickevent_messageid Nullable(UInt64), + clickevent_timestamp Nullable(UInt64), + connectioninformation_downlink Nullable(UInt64), + connectioninformation_type Nullable(String), + consolelog_level Nullable(String), + consolelog_value Nullable(String), + cpuissue_duration Nullable(UInt64), + cpuissue_rate Nullable(UInt64), + cpuissue_timestamp Nullable(UInt64), + createdocument Nullable(Bool), + createelementnode_id Nullable(UInt64), + createelementnode_parentid Nullable(UInt64), + cssdeleterule_index Nullable(UInt64), + cssdeleterule_stylesheetid Nullable(UInt64), + cssinsertrule_index Nullable(UInt64), + cssinsertrule_rule Nullable(String), + cssinsertrule_stylesheetid Nullable(UInt64), + customevent_messageid Nullable(UInt64), + customevent_name Nullable(String), + customevent_payload Nullable(String), + customevent_timestamp Nullable(UInt64), + domdrop_timestamp Nullable(UInt64), + errorevent_message Nullable(String), + errorevent_messageid Nullable(UInt64), + errorevent_name Nullable(String), + errorevent_payload Nullable(String), + errorevent_source Nullable(String), + errorevent_timestamp Nullable(UInt64), + fetch_duration Nullable(UInt64), + fetch_method Nullable(String), + fetch_request Nullable(String), + fetch_response Nullable(String), + fetch_status Nullable(UInt64), + fetch_timestamp Nullable(UInt64), + fetch_url Nullable(String), + graphql_operationkind Nullable(String), + graphql_operationname Nullable(String), + graphql_response Nullable(String), + graphql_variables Nullable(String), + graphqlevent_messageid Nullable(UInt64), + graphqlevent_name Nullable(String), + graphqlevent_timestamp Nullable(UInt64), + inputevent_label Nullable(String), + inputevent_messageid Nullable(UInt64), + inputevent_timestamp Nullable(UInt64), + inputevent_value Nullable(String), + inputevent_valuemasked Nullable(Bool), + jsexception_message Nullable(String), + jsexception_name Nullable(String), + jsexception_payload Nullable(String), + memoryissue_duration Nullable(UInt64), + memoryissue_rate Nullable(UInt64), + memoryissue_timestamp Nullable(UInt64), + metadata_key Nullable(String), + metadata_value Nullable(String), + mobx_payload Nullable(String), + mobx_type Nullable(String), + mouseclick_id Nullable(UInt64), + mouseclick_hesitationtime Nullable(UInt64), + mouseclick_label Nullable(String), + mousemove_x Nullable(UInt64), + mousemove_y Nullable(UInt64), + movenode_id Nullable(UInt64), + movenode_index Nullable(UInt64), + movenode_parentid Nullable(UInt64), + ngrx_action Nullable(String), + ngrx_duration Nullable(UInt64), + ngrx_state Nullable(String), + otable_key Nullable(String), + otable_value Nullable(String), + pageevent_domcontentloadedeventend Nullable(UInt64), + pageevent_domcontentloadedeventstart Nullable(UInt64), + pageevent_firstcontentfulpaint Nullable(UInt64), + pageevent_firstpaint Nullable(UInt64), + pageevent_loaded Nullable(Bool), + pageevent_loadeventend Nullable(UInt64), + pageevent_loadeventstart Nullable(UInt64), + pageevent_messageid Nullable(UInt64), + pageevent_referrer Nullable(String), + pageevent_requeststart Nullable(UInt64), + pageevent_responseend Nullable(UInt64), + pageevent_responsestart Nullable(UInt64), + pageevent_speedindex Nullable(UInt64), + pageevent_timestamp Nullable(UInt64), + pageevent_url Nullable(String), + pageloadtiming_domcontentloadedeventend Nullable(UInt64), + pageloadtiming_domcontentloadedeventstart Nullable(UInt64), + pageloadtiming_firstcontentfulpaint Nullable(UInt64), + pageloadtiming_firstpaint Nullable(UInt64), + pageloadtiming_loadeventend Nullable(UInt64), + pageloadtiming_loadeventstart Nullable(UInt64), + pageloadtiming_requeststart Nullable(UInt64), + pageloadtiming_responseend Nullable(UInt64), + pageloadtiming_responsestart Nullable(UInt64), + pagerendertiming_speedindex Nullable(UInt64), + pagerendertiming_timetointeractive Nullable(UInt64), + pagerendertiming_visuallycomplete Nullable(UInt64), + performancetrack_frames Nullable(Int64), + performancetrack_ticks Nullable(Int64), + performancetrack_totaljsheapsize Nullable(UInt64), + performancetrack_usedjsheapsize Nullable(UInt64), + performancetrackaggr_avgcpu Nullable(UInt64), + performancetrackaggr_avgfps Nullable(UInt64), + performancetrackaggr_avgtotaljsheapsize Nullable(UInt64), + performancetrackaggr_avgusedjsheapsize Nullable(UInt64), + performancetrackaggr_maxcpu Nullable(UInt64), + performancetrackaggr_maxfps Nullable(UInt64), + performancetrackaggr_maxtotaljsheapsize Nullable(UInt64), + performancetrackaggr_maxusedjsheapsize Nullable(UInt64), + performancetrackaggr_mincpu Nullable(UInt64), + performancetrackaggr_minfps Nullable(UInt64), + performancetrackaggr_mintotaljsheapsize Nullable(UInt64), + performancetrackaggr_minusedjsheapsize Nullable(UInt64), + performancetrackaggr_timestampend Nullable(UInt64), + performancetrackaggr_timestampstart Nullable(UInt64), + profiler_args Nullable(String), + profiler_duration Nullable(UInt64), + profiler_name Nullable(String), + profiler_result Nullable(String), + rawcustomevent_name Nullable(String), + rawcustomevent_payload Nullable(String), + rawerrorevent_message Nullable(String), + rawerrorevent_name Nullable(String), + rawerrorevent_payload Nullable(String), + rawerrorevent_source Nullable(String), + rawerrorevent_timestamp Nullable(UInt64), + redux_action Nullable(String), + redux_duration Nullable(UInt64), + redux_state Nullable(String), + removenode_id Nullable(UInt64), + removenodeattribute_id Nullable(UInt64), + removenodeattribute_name Nullable(String), + resourceevent_decodedbodysize Nullable(UInt64), + resourceevent_duration Nullable(UInt64), + resourceevent_encodedbodysize Nullable(UInt64), + resourceevent_headersize Nullable(UInt64), + resourceevent_messageid Nullable(UInt64), + resourceevent_method Nullable(String), + resourceevent_status Nullable(UInt64), + resourceevent_success Nullable(Bool), + resourceevent_timestamp Nullable(UInt64), + resourceevent_ttfb Nullable(UInt64), + resourceevent_type Nullable(String), + resourceevent_url Nullable(String), + resourcetiming_decodedbodysize Nullable(UInt64), + resourcetiming_duration Nullable(UInt64), + resourcetiming_encodedbodysize Nullable(UInt64), + resourcetiming_headersize Nullable(UInt64), + resourcetiming_initiator Nullable(String), + resourcetiming_timestamp Nullable(UInt64), + resourcetiming_ttfb Nullable(UInt64), + resourcetiming_url Nullable(String), + sessiondisconnect Nullable(Bool), + sessiondisconnect_timestamp Nullable(UInt64), + sessionend Nullable(Bool), + sessionend_timestamp Nullable(UInt64), + sessionstart_projectid Nullable(UInt64), + sessionstart_revid Nullable(String), + sessionstart_timestamp Nullable(UInt64), + sessionstart_trackerversion Nullable(String), + sessionstart_useragent Nullable(String), + sessionstart_userbrowser Nullable(String), + sessionstart_userbrowserversion Nullable(String), + sessionstart_usercountry Nullable(String), + sessionstart_userdevice Nullable(String), + sessionstart_userdeviceheapsize Nullable(UInt64), + sessionstart_userdevicememorysize Nullable(UInt64), + sessionstart_userdevicetype Nullable(String), + sessionstart_useros Nullable(String), + sessionstart_userosversion Nullable(String), + sessionstart_useruuid Nullable(String), + setcssdata_data Nullable(UInt64), + setcssdata_id Nullable(UInt64), + setinputchecked_checked Nullable(UInt64), + setinputchecked_id Nullable(UInt64), + setinputtarget_id Nullable(UInt64), + setinputtarget_label Nullable(UInt64), + setinputvalue_id Nullable(UInt64), + setinputvalue_mask Nullable(UInt64), + setinputvalue_value Nullable(UInt64), + setnodeattribute_id Nullable(UInt64), + setnodeattribute_name Nullable(UInt64), + setnodeattribute_value Nullable(UInt64), + setnodedata_data Nullable(UInt64), + setnodedata_id Nullable(UInt64), + setnodescroll_id Nullable(UInt64), + setnodescroll_x Nullable(UInt64), + setnodescroll_y Nullable(UInt64), + setpagelocation_navigationstart Nullable(UInt64), + setpagelocation_referrer Nullable(String), + setpagelocation_url Nullable(String), + setpagevisibility_hidden Nullable(Bool), + setviewportscroll_x Nullable(UInt64), + setviewportscroll_y Nullable(UInt64), + setviewportsize_height Nullable(UInt64), + setviewportsize_width Nullable(UInt64), + stateaction_type Nullable(String), + stateactionevent_messageid Nullable(UInt64), + stateactionevent_timestamp Nullable(UInt64), + stateactionevent_type Nullable(String), + timestamp_timestamp Nullable(UInt64), + useranonymousid_id Nullable(String), + userid_id Nullable(String), + vuex_mutation Nullable(String), + vuex_state Nullable(String), + longtasks_timestamp Nullable(UInt64), + longtasks_duration Nullable(UInt64), + longtasks_context Nullable(UInt64), + longtasks_containertype Nullable(UInt64), + longtasks_containersrc Nullable(String), + longtasks_containerid Nullable(String), + longtasks_containername Nullable(UInt64), + setnodeurlbasedattribute_id Nullable(UInt64), + setnodeurlbasedattribute_name Nullable(String), + setnodeurlbasedattribute_value Nullable(String), + setnodeurlbasedattribute_baseurl Nullable(String), + setstyledata_id Nullable(UInt64), + setstyledata_data Nullable(String), + setstyledata_baseurl Nullable(String), + issueevent_messageid Nullable(UInt64), + issueevent_timestamp Nullable(UInt64), + issueevent_type Nullable(String), + issueevent_contextstring Nullable(String), + issueevent_context Nullable(String), + issueevent_payload Nullable(String), + technicalinfo_type Nullable(String), + technicalinfo_value Nullable(String), + customissue_name Nullable(String), + customissue_payload Nullable(String), + pageclose Nullable(UInt64), + received_at UInt64, + batch_order_number UInt64 +) ENGINE = MergeTree() +PARTITION BY intDiv(received_at, 100000) +ORDER BY (received_at, batch_order_number, sessionid) +PRIMARY KEY (received_at) +SETTINGS use_minimalistic_part_header_in_zookeeper=1, index_granularity=1000; diff --git a/ee/connectors/sql/clickhouse_events_detailed_buffer.sql b/ee/connectors/sql/clickhouse_events_detailed_buffer.sql new file mode 100644 index 000000000..fdaaf1520 --- /dev/null +++ b/ee/connectors/sql/clickhouse_events_detailed_buffer.sql @@ -0,0 +1,238 @@ +CREATE TABLE IF NOT EXISTS connector_events_detailed_buffer +( + sessionid UInt64, + clickevent_hesitationtime Nullable(UInt64), + clickevent_label Nullable(String), + clickevent_messageid Nullable(UInt64), + clickevent_timestamp Nullable(UInt64), + connectioninformation_downlink Nullable(UInt64), + connectioninformation_type Nullable(String), + consolelog_level Nullable(String), + consolelog_value Nullable(String), + cpuissue_duration Nullable(UInt64), + cpuissue_rate Nullable(UInt64), + cpuissue_timestamp Nullable(UInt64), + createdocument Nullable(Bool), + createelementnode_id Nullable(UInt64), + createelementnode_parentid Nullable(UInt64), + cssdeleterule_index Nullable(UInt64), + cssdeleterule_stylesheetid Nullable(UInt64), + cssinsertrule_index Nullable(UInt64), + cssinsertrule_rule Nullable(String), + cssinsertrule_stylesheetid Nullable(UInt64), + customevent_messageid Nullable(UInt64), + customevent_name Nullable(String), + customevent_payload Nullable(String), + customevent_timestamp Nullable(UInt64), + domdrop_timestamp Nullable(UInt64), + errorevent_message Nullable(String), + errorevent_messageid Nullable(UInt64), + errorevent_name Nullable(String), + errorevent_payload Nullable(String), + errorevent_source Nullable(String), + errorevent_timestamp Nullable(UInt64), + fetch_duration Nullable(UInt64), + fetch_method Nullable(String), + fetch_request Nullable(String), + fetch_response Nullable(String), + fetch_status Nullable(UInt64), + fetch_timestamp Nullable(UInt64), + fetch_url Nullable(String), + graphql_operationkind Nullable(String), + graphql_operationname Nullable(String), + graphql_response Nullable(String), + graphql_variables Nullable(String), + graphqlevent_messageid Nullable(UInt64), + graphqlevent_name Nullable(String), + graphqlevent_timestamp Nullable(UInt64), + inputevent_label Nullable(String), + inputevent_messageid Nullable(UInt64), + inputevent_timestamp Nullable(UInt64), + inputevent_value Nullable(String), + inputevent_valuemasked Nullable(Bool), + jsexception_message Nullable(String), + jsexception_name Nullable(String), + jsexception_payload Nullable(String), + memoryissue_duration Nullable(UInt64), + memoryissue_rate Nullable(UInt64), + memoryissue_timestamp Nullable(UInt64), + metadata_key Nullable(String), + metadata_value Nullable(String), + mobx_payload Nullable(String), + mobx_type Nullable(String), + mouseclick_id Nullable(UInt64), + mouseclick_hesitationtime Nullable(UInt64), + mouseclick_label Nullable(String), + mousemove_x Nullable(UInt64), + mousemove_y Nullable(UInt64), + movenode_id Nullable(UInt64), + movenode_index Nullable(UInt64), + movenode_parentid Nullable(UInt64), + ngrx_action Nullable(String), + ngrx_duration Nullable(UInt64), + ngrx_state Nullable(String), + otable_key Nullable(String), + otable_value Nullable(String), + pageevent_domcontentloadedeventend Nullable(UInt64), + pageevent_domcontentloadedeventstart Nullable(UInt64), + pageevent_firstcontentfulpaint Nullable(UInt64), + pageevent_firstpaint Nullable(UInt64), + pageevent_loaded Nullable(Bool), + pageevent_loadeventend Nullable(UInt64), + pageevent_loadeventstart Nullable(UInt64), + pageevent_messageid Nullable(UInt64), + pageevent_referrer Nullable(String), + pageevent_requeststart Nullable(UInt64), + pageevent_responseend Nullable(UInt64), + pageevent_responsestart Nullable(UInt64), + pageevent_speedindex Nullable(UInt64), + pageevent_timestamp Nullable(UInt64), + pageevent_url Nullable(String), + pageloadtiming_domcontentloadedeventend Nullable(UInt64), + pageloadtiming_domcontentloadedeventstart Nullable(UInt64), + pageloadtiming_firstcontentfulpaint Nullable(UInt64), + pageloadtiming_firstpaint Nullable(UInt64), + pageloadtiming_loadeventend Nullable(UInt64), + pageloadtiming_loadeventstart Nullable(UInt64), + pageloadtiming_requeststart Nullable(UInt64), + pageloadtiming_responseend Nullable(UInt64), + pageloadtiming_responsestart Nullable(UInt64), + pagerendertiming_speedindex Nullable(UInt64), + pagerendertiming_timetointeractive Nullable(UInt64), + pagerendertiming_visuallycomplete Nullable(UInt64), + performancetrack_frames Nullable(Int64), + performancetrack_ticks Nullable(Int64), + performancetrack_totaljsheapsize Nullable(UInt64), + performancetrack_usedjsheapsize Nullable(UInt64), + performancetrackaggr_avgcpu Nullable(UInt64), + performancetrackaggr_avgfps Nullable(UInt64), + performancetrackaggr_avgtotaljsheapsize Nullable(UInt64), + performancetrackaggr_avgusedjsheapsize Nullable(UInt64), + performancetrackaggr_maxcpu Nullable(UInt64), + performancetrackaggr_maxfps Nullable(UInt64), + performancetrackaggr_maxtotaljsheapsize Nullable(UInt64), + performancetrackaggr_maxusedjsheapsize Nullable(UInt64), + performancetrackaggr_mincpu Nullable(UInt64), + performancetrackaggr_minfps Nullable(UInt64), + performancetrackaggr_mintotaljsheapsize Nullable(UInt64), + performancetrackaggr_minusedjsheapsize Nullable(UInt64), + performancetrackaggr_timestampend Nullable(UInt64), + performancetrackaggr_timestampstart Nullable(UInt64), + profiler_args Nullable(String), + profiler_duration Nullable(UInt64), + profiler_name Nullable(String), + profiler_result Nullable(String), + rawcustomevent_name Nullable(String), + rawcustomevent_payload Nullable(String), + rawerrorevent_message Nullable(String), + rawerrorevent_name Nullable(String), + rawerrorevent_payload Nullable(String), + rawerrorevent_source Nullable(String), + rawerrorevent_timestamp Nullable(UInt64), + redux_action Nullable(String), + redux_duration Nullable(UInt64), + redux_state Nullable(String), + removenode_id Nullable(UInt64), + removenodeattribute_id Nullable(UInt64), + removenodeattribute_name Nullable(String), + resourceevent_decodedbodysize Nullable(UInt64), + resourceevent_duration Nullable(UInt64), + resourceevent_encodedbodysize Nullable(UInt64), + resourceevent_headersize Nullable(UInt64), + resourceevent_messageid Nullable(UInt64), + resourceevent_method Nullable(String), + resourceevent_status Nullable(UInt64), + resourceevent_success Nullable(Bool), + resourceevent_timestamp Nullable(UInt64), + resourceevent_ttfb Nullable(UInt64), + resourceevent_type Nullable(String), + resourceevent_url Nullable(String), + resourcetiming_decodedbodysize Nullable(UInt64), + resourcetiming_duration Nullable(UInt64), + resourcetiming_encodedbodysize Nullable(UInt64), + resourcetiming_headersize Nullable(UInt64), + resourcetiming_initiator Nullable(String), + resourcetiming_timestamp Nullable(UInt64), + resourcetiming_ttfb Nullable(UInt64), + resourcetiming_url Nullable(String), + sessiondisconnect Nullable(Bool), + sessiondisconnect_timestamp Nullable(UInt64), + sessionend Nullable(Bool), + sessionend_timestamp Nullable(UInt64), + sessionstart_projectid Nullable(UInt64), + sessionstart_revid Nullable(String), + sessionstart_timestamp Nullable(UInt64), + sessionstart_trackerversion Nullable(String), + sessionstart_useragent Nullable(String), + sessionstart_userbrowser Nullable(String), + sessionstart_userbrowserversion Nullable(String), + sessionstart_usercountry Nullable(String), + sessionstart_userdevice Nullable(String), + sessionstart_userdeviceheapsize Nullable(UInt64), + sessionstart_userdevicememorysize Nullable(UInt64), + sessionstart_userdevicetype Nullable(String), + sessionstart_useros Nullable(String), + sessionstart_userosversion Nullable(String), + sessionstart_useruuid Nullable(String), + setcssdata_data Nullable(UInt64), + setcssdata_id Nullable(UInt64), + setinputchecked_checked Nullable(UInt64), + setinputchecked_id Nullable(UInt64), + setinputtarget_id Nullable(UInt64), + setinputtarget_label Nullable(UInt64), + setinputvalue_id Nullable(UInt64), + setinputvalue_mask Nullable(UInt64), + setinputvalue_value Nullable(UInt64), + setnodeattribute_id Nullable(UInt64), + setnodeattribute_name Nullable(UInt64), + setnodeattribute_value Nullable(UInt64), + setnodedata_data Nullable(UInt64), + setnodedata_id Nullable(UInt64), + setnodescroll_id Nullable(UInt64), + setnodescroll_x Nullable(UInt64), + setnodescroll_y Nullable(UInt64), + setpagelocation_navigationstart Nullable(UInt64), + setpagelocation_referrer Nullable(String), + setpagelocation_url Nullable(String), + setpagevisibility_hidden Nullable(Bool), + setviewportscroll_x Nullable(UInt64), + setviewportscroll_y Nullable(UInt64), + setviewportsize_height Nullable(UInt64), + setviewportsize_width Nullable(UInt64), + stateaction_type Nullable(String), + stateactionevent_messageid Nullable(UInt64), + stateactionevent_timestamp Nullable(UInt64), + stateactionevent_type Nullable(String), + timestamp_timestamp Nullable(UInt64), + useranonymousid_id Nullable(String), + userid_id Nullable(String), + vuex_mutation Nullable(String), + vuex_state Nullable(String), + longtasks_timestamp Nullable(UInt64), + longtasks_duration Nullable(UInt64), + longtasks_context Nullable(UInt64), + longtasks_containertype Nullable(UInt64), + longtasks_containersrc Nullable(String), + longtasks_containerid Nullable(String), + longtasks_containername Nullable(UInt64), + setnodeurlbasedattribute_id Nullable(UInt64), + setnodeurlbasedattribute_name Nullable(String), + setnodeurlbasedattribute_value Nullable(String), + setnodeurlbasedattribute_baseurl Nullable(String), + setstyledata_id Nullable(UInt64), + setstyledata_data Nullable(String), + setstyledata_baseurl Nullable(String), + issueevent_messageid Nullable(UInt64), + issueevent_timestamp Nullable(UInt64), + issueevent_type Nullable(String), + issueevent_contextstring Nullable(String), + issueevent_context Nullable(String), + issueevent_payload Nullable(String), + technicalinfo_type Nullable(String), + technicalinfo_value Nullable(String), + customissue_name Nullable(String), + customissue_payload Nullable(String), + pageclose Nullable(UInt64), + received_at UInt64, + batch_order_number UInt64 +) ENGINE = Buffer(default, connector_events_detailed, 16, 10, 120, 10000, 1000000, 10000, 100000000); From 0b38f1dedbdea691eb8a3b4977ab7267ee93209e Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Wed, 20 Jul 2022 10:55:25 +0200 Subject: [PATCH 11/14] Updated types in BigQuery table --- ee/connectors/bigquery_utils/create_table.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ee/connectors/bigquery_utils/create_table.py b/ee/connectors/bigquery_utils/create_table.py index 5dc4e9b9a..e88c903ec 100644 --- a/ee/connectors/bigquery_utils/create_table.py +++ b/ee/connectors/bigquery_utils/create_table.py @@ -341,20 +341,20 @@ def create_events_detailed_table(creds_file, table_id): bigquery.SchemaField("vuex_state", "STRING"), bigquery.SchemaField("longtasks_timestamp", "INT64"), bigquery.SchemaField("longtasks_duration", "INT64"), - bigquery.SchemaField("longtasks_context", "FLOAT64"), - bigquery.SchemaField("longtasks_containertype", "FLOAT64"), + bigquery.SchemaField("longtasks_context", "INT64"), + bigquery.SchemaField("longtasks_containertype", "INT64"), bigquery.SchemaField("longtasks_containersrc", "STRING"), bigquery.SchemaField("longtasks_containerid", "STRING"), - bigquery.SchemaField("longtasks_containername", "FLOAT64"), - bigquery.SchemaField("setnodeurlbasedattribute_id", "FLOAT64"), + bigquery.SchemaField("longtasks_containername", "INT64"), + bigquery.SchemaField("setnodeurlbasedattribute_id", "INT64"), bigquery.SchemaField("setnodeurlbasedattribute_name", "STRING"), bigquery.SchemaField("setnodeurlbasedattribute_value", "STRING"), bigquery.SchemaField("setnodeurlbasedattribute_baseurl", "STRING"), - bigquery.SchemaField("setstyledata_id", "FLOAT64"), + bigquery.SchemaField("setstyledata_id", "INT64"), bigquery.SchemaField("setstyledata_data", "STRING"), bigquery.SchemaField("setstyledata_baseurl", "STRING"), - bigquery.SchemaField("issueevent_messageid", "FLOAT64"), - bigquery.SchemaField("issueevent_timestamp", "FLOAT64"), + bigquery.SchemaField("issueevent_messageid", "INT64"), + bigquery.SchemaField("issueevent_timestamp", "INT64"), bigquery.SchemaField("issueevent_type", "STRING"), bigquery.SchemaField("issueevent_contextstring", "STRING"), bigquery.SchemaField("issueevent_context", "STRING"), @@ -363,7 +363,7 @@ def create_events_detailed_table(creds_file, table_id): bigquery.SchemaField("technicalinfo_value", "STRING"), bigquery.SchemaField("customissue_name", "STRING"), bigquery.SchemaField("customissue_payload", "STRING"), - bigquery.SchemaField("pageclose", "FLOAT64"), + bigquery.SchemaField("pageclose", "INT64"), bigquery.SchemaField("received_at", "INT64", mode="REQUIRED"), bigquery.SchemaField("batch_order_number", "INT64", mode="REQUIRED") ] From ee80c7c03fd4c049867ba2a0ef557a220a45df15 Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Fri, 22 Jul 2022 15:05:42 +0200 Subject: [PATCH 12/14] RedShift operational --- ee/connectors/db/tables.py | 5 + ee/connectors/sql/redshift_events.sql | 4 +- .../sql/redshift_events_detailed.sql | 238 ++++++++++++++++++ ee/connectors/sql/redshift_sessions.sql | 4 +- 4 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 ee/connectors/sql/redshift_events_detailed.sql diff --git a/ee/connectors/db/tables.py b/ee/connectors/db/tables.py index db1c2ca42..58f57514c 100644 --- a/ee/connectors/db/tables.py +++ b/ee/connectors/db/tables.py @@ -74,3 +74,8 @@ def create_tables_redshift(db): q = f.read() db.engine.execute(q) print(f"`connector_sessions` table created succesfully.") + + #with open(base_path / 'sql' / 'redshift_events_detailed.sql') as f: + # q = f.read() + #db.engine.execute(q) + #print(f"`connector_user_events_detailed` table created succesfully.") diff --git a/ee/connectors/sql/redshift_events.sql b/ee/connectors/sql/redshift_events.sql index c310e3202..773444233 100644 --- a/ee/connectors/sql/redshift_events.sql +++ b/ee/connectors/sql/redshift_events.sql @@ -1,4 +1,4 @@ -CREATE TABLE connector_events +CREATE TABLE IF NOT EXISTS connector_events ( sessionid BIGINT, connectioninformation_downlink BIGINT, @@ -49,4 +49,4 @@ CREATE TABLE connector_events customissue_payload VARCHAR(300), received_at BIGINT, batch_order_number BIGINT -); \ No newline at end of file +); diff --git a/ee/connectors/sql/redshift_events_detailed.sql b/ee/connectors/sql/redshift_events_detailed.sql new file mode 100644 index 000000000..f50078828 --- /dev/null +++ b/ee/connectors/sql/redshift_events_detailed.sql @@ -0,0 +1,238 @@ +CREATE TABLE IF NOT EXISTS connector_events_detailed +( + sessionid BIGINT, + clickevent_hesitationtime BIGINT, + clickevent_label VARCHAR(300), + clickevent_messageid BIGINT, + clickevent_timestamp BIGINT, + connectioninformation_downlink BIGINT, + connectioninformation_type VARCHAR(300), + consolelog_level VARCHAR(300), + consolelog_value VARCHAR(300), + cpuissue_duration BIGINT, + cpuissue_rate BIGINT, + cpuissue_timestamp BIGINT, + createdocument BOOLEAN, + createelementnode_id BIGINT, + createelementnode_parentid BIGINT, + cssdeleterule_index BIGINT, + cssdeleterule_stylesheetid BIGINT, + cssinsertrule_index BIGINT, + cssinsertrule_rule VARCHAR(300), + cssinsertrule_stylesheetid BIGINT, + customevent_messageid BIGINT, + customevent_name VARCHAR(300), + customevent_payload VARCHAR(300), + customevent_timestamp BIGINT, + domdrop_timestamp BIGINT, + errorevent_message VARCHAR(300), + errorevent_messageid BIGINT, + errorevent_name VARCHAR(300), + errorevent_payload VARCHAR(300), + errorevent_source VARCHAR(300), + errorevent_timestamp BIGINT, + fetch_duration BIGINT, + fetch_method VARCHAR(300), + fetch_request VARCHAR(300), + fetch_response VARCHAR(300), + fetch_status BIGINT, + fetch_timestamp BIGINT, + fetch_url VARCHAR(300), + graphql_operationkind VARCHAR(300), + graphql_operationname VARCHAR(300), + graphql_response VARCHAR(300), + graphql_variables VARCHAR(300), + graphqlevent_messageid BIGINT, + graphqlevent_name VARCHAR(300), + graphqlevent_timestamp BIGINT, + inputevent_label VARCHAR(300), + inputevent_messageid BIGINT, + inputevent_timestamp BIGINT, + inputevent_value VARCHAR(300), + inputevent_valuemasked BOOLEAN, + jsexception_message VARCHAR(300), + jsexception_name VARCHAR(300), + jsexception_payload VARCHAR(300), + memoryissue_duration BIGINT, + memoryissue_rate BIGINT, + memoryissue_timestamp BIGINT, + metadata_key VARCHAR(300), + metadata_value VARCHAR(300), + mobx_payload VARCHAR(300), + mobx_type VARCHAR(300), + mouseclick_id BIGINT, + mouseclick_hesitationtime BIGINT, + mouseclick_label VARCHAR(300), + mousemove_x BIGINT, + mousemove_y BIGINT, + movenode_id BIGINT, + movenode_index BIGINT, + movenode_parentid BIGINT, + ngrx_action VARCHAR(300), + ngrx_duration BIGINT, + ngrx_state VARCHAR(300), + otable_key VARCHAR(300), + otable_value VARCHAR(300), + pageevent_domcontentloadedeventend BIGINT, + pageevent_domcontentloadedeventstart BIGINT, + pageevent_firstcontentfulpaint BIGINT, + pageevent_firstpaint BIGINT, + pageevent_loaded BOOLEAN, + pageevent_loadeventend BIGINT, + pageevent_loadeventstart BIGINT, + pageevent_messageid BIGINT, + pageevent_referrer VARCHAR(300), + pageevent_requeststart BIGINT, + pageevent_responseend BIGINT, + pageevent_responsestart BIGINT, + pageevent_speedindex BIGINT, + pageevent_timestamp BIGINT, + pageevent_url VARCHAR(300), + pageloadtiming_domcontentloadedeventend BIGINT, + pageloadtiming_domcontentloadedeventstart BIGINT, + pageloadtiming_firstcontentfulpaint BIGINT, + pageloadtiming_firstpaint BIGINT, + pageloadtiming_loadeventend BIGINT, + pageloadtiming_loadeventstart BIGINT, + pageloadtiming_requeststart BIGINT, + pageloadtiming_responseend BIGINT, + pageloadtiming_responsestart BIGINT, + pagerendertiming_speedindex BIGINT, + pagerendertiming_timetointeractive BIGINT, + pagerendertiming_visuallycomplete BIGINT, + performancetrack_frames BIGINT, + performancetrack_ticks BIGINT, + performancetrack_totaljsheapsize BIGINT, + performancetrack_usedjsheapsize BIGINT, + performancetrackaggr_avgcpu BIGINT, + performancetrackaggr_avgfps BIGINT, + performancetrackaggr_avgtotaljsheapsize BIGINT, + performancetrackaggr_avgusedjsheapsize BIGINT, + performancetrackaggr_maxcpu BIGINT, + performancetrackaggr_maxfps BIGINT, + performancetrackaggr_maxtotaljsheapsize BIGINT, + performancetrackaggr_maxusedjsheapsize BIGINT, + performancetrackaggr_mincpu BIGINT, + performancetrackaggr_minfps BIGINT, + performancetrackaggr_mintotaljsheapsize BIGINT, + performancetrackaggr_minusedjsheapsize BIGINT, + performancetrackaggr_timestampend BIGINT, + performancetrackaggr_timestampstart BIGINT, + profiler_args VARCHAR(300), + profiler_duration BIGINT, + profiler_name VARCHAR(300), + profiler_result VARCHAR(300), + rawcustomevent_name VARCHAR(300), + rawcustomevent_payload VARCHAR(300), + rawerrorevent_message VARCHAR(300), + rawerrorevent_name VARCHAR(300), + rawerrorevent_payload VARCHAR(300), + rawerrorevent_source VARCHAR(300), + rawerrorevent_timestamp BIGINT, + redux_action VARCHAR(300), + redux_duration BIGINT, + redux_state VARCHAR(300), + removenode_id BIGINT, + removenodeattribute_id BIGINT, + removenodeattribute_name VARCHAR(300), + resourceevent_decodedbodysize BIGINT, + resourceevent_duration BIGINT, + resourceevent_encodedbodysize BIGINT, + resourceevent_headersize BIGINT, + resourceevent_messageid BIGINT, + resourceevent_method VARCHAR(300), + resourceevent_status BIGINT, + resourceevent_success BOOLEAN, + resourceevent_timestamp BIGINT, + resourceevent_ttfb BIGINT, + resourceevent_type VARCHAR(300), + resourceevent_url VARCHAR(300), + resourcetiming_decodedbodysize BIGINT, + resourcetiming_duration BIGINT, + resourcetiming_encodedbodysize BIGINT, + resourcetiming_headersize BIGINT, + resourcetiming_initiator VARCHAR(300), + resourcetiming_timestamp BIGINT, + resourcetiming_ttfb BIGINT, + resourcetiming_url VARCHAR(300), + sessiondisconnect BOOLEAN, + sessiondisconnect_timestamp BIGINT, + sessionend BOOLEAN, + sessionend_timestamp BIGINT, + sessionstart_projectid BIGINT, + sessionstart_revid VARCHAR(300), + sessionstart_timestamp BIGINT, + sessionstart_trackerversion VARCHAR(300), + sessionstart_useragent VARCHAR(300), + sessionstart_userbrowser VARCHAR(300), + sessionstart_userbrowserversion VARCHAR(300), + sessionstart_usercountry VARCHAR(300), + sessionstart_userdevice VARCHAR(300), + sessionstart_userdeviceheapsize BIGINT, + sessionstart_userdevicememorysize BIGINT, + sessionstart_userdevicetype VARCHAR(300), + sessionstart_useros VARCHAR(300), + sessionstart_userosversion VARCHAR(300), + sessionstart_useruuid VARCHAR(300), + setcssdata_data BIGINT, + setcssdata_id BIGINT, + setinputchecked_checked BIGINT, + setinputchecked_id BIGINT, + setinputtarget_id BIGINT, + setinputtarget_label BIGINT, + setinputvalue_id BIGINT, + setinputvalue_mask BIGINT, + setinputvalue_value BIGINT, + setnodeattribute_id BIGINT, + setnodeattribute_name BIGINT, + setnodeattribute_value BIGINT, + setnodedata_data BIGINT, + setnodedata_id BIGINT, + setnodescroll_id BIGINT, + setnodescroll_x BIGINT, + setnodescroll_y BIGINT, + setpagelocation_navigationstart BIGINT, + setpagelocation_referrer VARCHAR(300), + setpagelocation_url VARCHAR(300), + setpagevisibility_hidden BOOLEAN, + setviewportscroll_x BIGINT, + setviewportscroll_y BIGINT, + setviewportsize_height BIGINT, + setviewportsize_width BIGINT, + stateaction_type VARCHAR(300), + stateactionevent_messageid BIGINT, + stateactionevent_timestamp BIGINT, + stateactionevent_type VARCHAR(300), + timestamp_timestamp BIGINT, + useranonymousid_id VARCHAR(300), + userid_id VARCHAR(300), + vuex_mutation VARCHAR(300), + vuex_state VARCHAR(300), + longtasks_timestamp BIGINT, + longtasks_duration BIGINT, + longtasks_context BIGINT, + longtasks_containertype BIGINT, + longtasks_containersrc VARCHAR(300), + longtasks_containerid VARCHAR(300), + longtasks_containername BIGINT, + setnodeurlbasedattribute_id BIGINT, + setnodeurlbasedattribute_name VARCHAR(300), + setnodeurlbasedattribute_value VARCHAR(300), + setnodeurlbasedattribute_baseurl VARCHAR(300), + setstyledata_id BIGINT, + setstyledata_data VARCHAR(300), + setstyledata_baseurl VARCHAR(300), + issueevent_messageid BIGINT, + issueevent_timestamp BIGINT, + issueevent_type VARCHAR(300), + issueevent_contextstring VARCHAR(300), + issueevent_context VARCHAR(300), + issueevent_payload VARCHAR(300), + technicalinfo_type VARCHAR(300), + technicalinfo_value VARCHAR(300), + customissue_name VARCHAR(300), + customissue_payload VARCHAR(300), + pageclose BIGINT, + received_at BIGINT, + batch_order_number BIGINT +); diff --git a/ee/connectors/sql/redshift_sessions.sql b/ee/connectors/sql/redshift_sessions.sql index f1750dcc2..ba1887ac1 100644 --- a/ee/connectors/sql/redshift_sessions.sql +++ b/ee/connectors/sql/redshift_sessions.sql @@ -1,4 +1,4 @@ -CREATE TABLE connector_user_sessions +CREATE TABLE IF NOT EXISTS connector_user_sessions ( -- SESSION METADATA sessionid bigint, @@ -47,4 +47,4 @@ CREATE TABLE connector_user_sessions issues VARCHAR, urls_count bigint, urls VARCHAR -); \ No newline at end of file +); From ab06c5c8e38135cb2d9b6bb428995107827871df Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Tue, 9 Aug 2022 10:40:14 +0200 Subject: [PATCH 13/14] added events detailed snowflakes --- .../sql/snowflake_events_detailed.sql | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 ee/connectors/sql/snowflake_events_detailed.sql diff --git a/ee/connectors/sql/snowflake_events_detailed.sql b/ee/connectors/sql/snowflake_events_detailed.sql new file mode 100644 index 000000000..092a50ea3 --- /dev/null +++ b/ee/connectors/sql/snowflake_events_detailed.sql @@ -0,0 +1,238 @@ +CREATE TABLE IF NOT EXISTS connector_events_detailed +( + sessionid bigint, + clickevent_hesitationtime bigint, + clickevent_label text, + clickevent_messageid bigint, + clickevent_timestamp bigint, + connectioninformation_downlink bigint, + connectioninformation_type text, + consolelog_level text, + consolelog_value text, + cpuissue_duration bigint, + cpuissue_rate bigint, + cpuissue_timestamp bigint, + createdocument boolean, + createelementnode_id bigint, + createelementnode_parentid bigint, + cssdeleterule_index bigint, + cssdeleterule_stylesheetid bigint, + cssinsertrule_index bigint, + cssinsertrule_rule text, + cssinsertrule_stylesheetid bigint, + customevent_messageid bigint, + customevent_name text, + customevent_payload text, + customevent_timestamp bigint, + domdrop_timestamp bigint, + errorevent_message text, + errorevent_messageid bigint, + errorevent_name text, + errorevent_payload text, + errorevent_source text, + errorevent_timestamp bigint, + fetch_duration bigint, + fetch_method text, + fetch_request text, + fetch_response text, + fetch_status bigint, + fetch_timestamp bigint, + fetch_url text, + graphql_operationkind text, + graphql_operationname text, + graphql_response text, + graphql_variables text, + graphqlevent_messageid bigint, + graphqlevent_name text, + graphqlevent_timestamp bigint, + inputevent_label text, + inputevent_messageid bigint, + inputevent_timestamp bigint, + inputevent_value text, + inputevent_valuemasked boolean, + jsexception_message text, + jsexception_name text, + jsexception_payload text, + memoryissue_duration bigint, + memoryissue_rate bigint, + memoryissue_timestamp bigint, + metadata_key text, + metadata_value text, + mobx_payload text, + mobx_type text, + mouseclick_id bigint, + mouseclick_hesitationtime bigint, + mouseclick_label text, + mousemove_x bigint, + mousemove_y bigint, + movenode_id bigint, + movenode_index bigint, + movenode_parentid bigint, + ngrx_action text, + ngrx_duration bigint, + ngrx_state text, + otable_key text, + otable_value text, + pageevent_domcontentloadedeventend bigint, + pageevent_domcontentloadedeventstart bigint, + pageevent_firstcontentfulpaint bigint, + pageevent_firstpaint bigint, + pageevent_loaded boolean, + pageevent_loadeventend bigint, + pageevent_loadeventstart bigint, + pageevent_messageid bigint, + pageevent_referrer text, + pageevent_requeststart bigint, + pageevent_responseend bigint, + pageevent_responsestart bigint, + pageevent_speedindex bigint, + pageevent_timestamp bigint, + pageevent_url text, + pageloadtiming_domcontentloadedeventend bigint, + pageloadtiming_domcontentloadedeventstart bigint, + pageloadtiming_firstcontentfulpaint bigint, + pageloadtiming_firstpaint bigint, + pageloadtiming_loadeventend bigint, + pageloadtiming_loadeventstart bigint, + pageloadtiming_requeststart bigint, + pageloadtiming_responseend bigint, + pageloadtiming_responsestart bigint, + pagerendertiming_speedindex bigint, + pagerendertiming_timetointeractive bigint, + pagerendertiming_visuallycomplete bigint, + performancetrack_frames bigint, + performancetrack_ticks bigint, + performancetrack_totaljsheapsize bigint, + performancetrack_usedjsheapsize bigint, + performancetrackaggr_avgcpu bigint, + performancetrackaggr_avgfps bigint, + performancetrackaggr_avgtotaljsheapsize bigint, + performancetrackaggr_avgusedjsheapsize bigint, + performancetrackaggr_maxcpu bigint, + performancetrackaggr_maxfps bigint, + performancetrackaggr_maxtotaljsheapsize bigint, + performancetrackaggr_maxusedjsheapsize bigint, + performancetrackaggr_mincpu bigint, + performancetrackaggr_minfps bigint, + performancetrackaggr_mintotaljsheapsize bigint, + performancetrackaggr_minusedjsheapsize bigint, + performancetrackaggr_timestampend bigint, + performancetrackaggr_timestampstart bigint, + profiler_args text, + profiler_duration bigint, + profiler_name text, + profiler_result text, + rawcustomevent_name text, + rawcustomevent_payload text, + rawerrorevent_message text, + rawerrorevent_name text, + rawerrorevent_payload text, + rawerrorevent_source text, + rawerrorevent_timestamp bigint, + redux_action text, + redux_duration bigint, + redux_state text, + removenode_id bigint, + removenodeattribute_id bigint, + removenodeattribute_name text, + resourceevent_decodedbodysize bigint, + resourceevent_duration bigint, + resourceevent_encodedbodysize bigint, + resourceevent_headersize bigint, + resourceevent_messageid bigint, + resourceevent_method text, + resourceevent_status bigint, + resourceevent_success boolean, + resourceevent_timestamp bigint, + resourceevent_ttfb bigint, + resourceevent_type text, + resourceevent_url text, + resourcetiming_decodedbodysize bigint, + resourcetiming_duration bigint, + resourcetiming_encodedbodysize bigint, + resourcetiming_headersize bigint, + resourcetiming_initiator text, + resourcetiming_timestamp bigint, + resourcetiming_ttfb bigint, + resourcetiming_url text, + sessiondisconnect boolean, + sessiondisconnect_timestamp bigint, + sessionend boolean, + sessionend_timestamp bigint, + sessionstart_projectid bigint, + sessionstart_revid text, + sessionstart_timestamp bigint, + sessionstart_trackerversion text, + sessionstart_useragent text, + sessionstart_userbrowser text, + sessionstart_userbrowserversion text, + sessionstart_usercountry text, + sessionstart_userdevice text, + sessionstart_userdeviceheapsize bigint, + sessionstart_userdevicememorysize bigint, + sessionstart_userdevicetype text, + sessionstart_useros text, + sessionstart_userosversion text, + sessionstart_useruuid text, + setcssdata_data bigint, + setcssdata_id bigint, + setinputchecked_checked bigint, + setinputchecked_id bigint, + setinputtarget_id bigint, + setinputtarget_label bigint, + setinputvalue_id bigint, + setinputvalue_mask bigint, + setinputvalue_value bigint, + setnodeattribute_id bigint, + setnodeattribute_name bigint, + setnodeattribute_value bigint, + setnodedata_data bigint, + setnodedata_id bigint, + setnodescroll_id bigint, + setnodescroll_x bigint, + setnodescroll_y bigint, + setpagelocation_navigationstart bigint, + setpagelocation_referrer text, + setpagelocation_url text, + setpagevisibility_hidden boolean, + setviewportscroll_x bigint, + setviewportscroll_y bigint, + setviewportsize_height bigint, + setviewportsize_width bigint, + stateaction_type text, + stateactionevent_messageid bigint, + stateactionevent_timestamp bigint, + stateactionevent_type text, + timestamp_timestamp bigint, + useranonymousid_id text, + userid_id text, + vuex_mutation text, + vuex_state text, + longtasks_timestamp bigint, + longtasks_duration bigint, + longtasks_context bigint, + longtasks_containertype bigint, + longtasks_containersrc text, + longtasks_containerid text, + longtasks_containername bigint, + setnodeurlbasedattribute_id bigint, + setnodeurlbasedattribute_name text, + setnodeurlbasedattribute_value text, + setnodeurlbasedattribute_baseurl text, + setstyledata_id bigint, + setstyledata_data text, + setstyledata_baseurl text, + issueevent_messageid bigint, + issueevent_timestamp bigint, + issueevent_type text, + issueevent_contextstring text, + issueevent_context text, + issueevent_payload text, + technicalinfo_type text, + technicalinfo_value text, + customissue_name text, + customissue_payload text, + pageclose bigint, + received_at bigint, + batch_order_number bigint +); From dab72fbb38b431486a302f8adcbde11cf8f4751f Mon Sep 17 00:00:00 2001 From: mauricio garcia suarez Date: Wed, 10 Aug 2022 12:23:42 +0200 Subject: [PATCH 14/14] snowflakes working --- ee/connectors/db/tables.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ee/connectors/db/tables.py b/ee/connectors/db/tables.py index 58f57514c..f9e6eaaef 100644 --- a/ee/connectors/db/tables.py +++ b/ee/connectors/db/tables.py @@ -63,6 +63,11 @@ def create_tables_snowflake(db): db.engine.execute(q) print(f"`connector_sessions` table created succesfully.") + #with open(base_path / 'sql' / 'snowflake_events_detailed.sql') as f: + # q = f.read() + #db.engine.execute(q) + #print(f"`connector_user_events_detailed` table created succesfully.") + def create_tables_redshift(db): with open(base_path / 'sql' / 'redshift_events.sql') as f: