diff --git a/api/chalicelib/core/product_analytics/autocomplete.py b/api/chalicelib/core/product_analytics/autocomplete.py new file mode 100644 index 000000000..5915a8ab6 --- /dev/null +++ b/api/chalicelib/core/product_analytics/autocomplete.py @@ -0,0 +1,57 @@ +from typing import Optional + +from chalicelib.utils import helper +from chalicelib.utils.ch_client import ClickHouseClient + + +def search_events(project_id: int, q: Optional[str] = None): + with ClickHouseClient() as ch_client: + full_args = {"project_id": project_id, "limit": 20} + + constraints = ["project_id = %(project_id)s", + "_timestamp >= now()-INTERVAL 1 MONTH"] + if q: + constraints += ["value ILIKE %(q)s"] + full_args["q"] = helper.string_to_sql_like(q) + query = ch_client.format( + f"""SELECT value,data_count + FROM product_analytics.autocomplete_events_grouped + WHERE {" AND ".join(constraints)} + ORDER BY data_count DESC + LIMIT %(limit)s;""", + parameters=full_args) + rows = ch_client.execute(query) + + return {"values": helper.list_to_camel_case(rows), "_src": 2} + + +def search_properties(project_id: int, property_name: Optional[str] = None, event_name: Optional[str] = None, + q: Optional[str] = None): + with ClickHouseClient() as ch_client: + select = "value" + full_args = {"project_id": project_id, "limit": 20, + "event_name": event_name, "property_name": property_name} + + constraints = ["project_id = %(project_id)s", + "_timestamp >= now()-INTERVAL 1 MONTH"] + if event_name: + constraints += ["event_name = %(event_name)s"] + if property_name and q: + constraints += ["property_name = %(property_name)s"] + elif property_name: + select = "DISTINCT ON(property_name) property_name AS value" + constraints += ["property_name ILIKE %(property_name)s"] + full_args["property_name"] = helper.string_to_sql_like(property_name) + if q: + constraints += ["value ILIKE %(q)s"] + full_args["q"] = helper.string_to_sql_like(q) + query = ch_client.format( + f"""SELECT {select},data_count + FROM product_analytics.autocomplete_event_properties_grouped + WHERE {" AND ".join(constraints)} + ORDER BY data_count DESC + LIMIT %(limit)s;""", + parameters=full_args) + rows = ch_client.execute(query) + + return {"values": helper.list_to_camel_case(rows), "_src": 2} diff --git a/api/chalicelib/core/product_analytics/events.py b/api/chalicelib/core/product_analytics/events.py index 41363e7c6..10e578c7d 100644 --- a/api/chalicelib/core/product_analytics/events.py +++ b/api/chalicelib/core/product_analytics/events.py @@ -7,30 +7,69 @@ from chalicelib.utils.ch_client import ClickHouseClient from chalicelib.utils.exp_ch_helper import get_sub_condition logger = logging.getLogger(__name__) +PREDEFINED_EVENTS = { + "CLICK": "String", + "INPUT": "String", + "LOCATION": "String", + "ERROR": "String", + "PERFORMANCE": "String", + "REQUEST": "String" +} def get_events(project_id: int, page: schemas.PaginatedSchema): with ClickHouseClient() as ch_client: r = ch_client.format( - """SELECT DISTINCT ON(event_name,auto_captured) - COUNT(1) OVER () AS total, - event_name AS name, display_name, description, - auto_captured - FROM product_analytics.all_events - WHERE project_id=%(project_id)s - ORDER BY auto_captured,display_name - LIMIT %(limit)s OFFSET %(offset)s;""", + """SELECT DISTINCT + ON(event_name,auto_captured) + COUNT (1) OVER () AS total, + event_name AS name, display_name, description, + auto_captured + FROM product_analytics.all_events + WHERE project_id=%(project_id)s + ORDER BY auto_captured, display_name + LIMIT %(limit)s + OFFSET %(offset)s;""", parameters={"project_id": project_id, "limit": page.limit, "offset": (page.page - 1) * page.limit}) rows = ch_client.execute(r) if len(rows) == 0: - return {"total": 0, "list": []} + return {"total": len(PREDEFINED_EVENTS), "list": [{ + "name": e, + "displayName": "", + "description": "", + "autoCaptured": True, + "id": "event_0", + "dataType": "string", + "possibleTypes": [ + "string" + ], + "_foundInPredefinedList": False + } for e in PREDEFINED_EVENTS]} total = rows[0]["total"] + rows = helper.list_to_camel_case(rows) for i, row in enumerate(rows): row["id"] = f"event_{i}" - row["icon"] = None + row["dataType"] = "string" row["possibleTypes"] = ["string"] + row["_foundInPredefinedList"] = True row.pop("total") - return {"total": total, "list": helper.list_to_camel_case(rows)} + keys = [r["name"] for r in rows] + for e in PREDEFINED_EVENTS: + if e not in keys: + total += 1 + rows.append({ + "name": e, + "displayName": "", + "description": "", + "autoCaptured": True, + "id": "event_0", + "dataType": "string", + "possibleTypes": [ + "string" + ], + "_foundInPredefinedList": False + }) + return {"total": total, "list": rows} def search_events(project_id: int, data: schemas.EventsSearchPayloadSchema): @@ -109,31 +148,33 @@ def search_events(project_id: int, data: schemas.EventsSearchPayloadSchema): parameters=full_args) rows = ch_client.execute(query) if len(rows) == 0: - return {"total": 0, "rows": [], "src": 2} + return {"total": 0, "rows": [], "_src": 2} total = rows[0]["total"] for r in rows: r.pop("total") - return {"total": total, "rows": rows, "src": 2} + return {"total": total, "rows": rows, "_src": 2} def get_lexicon(project_id: int, page: schemas.PaginatedSchema): with ClickHouseClient() as ch_client: r = ch_client.format( - """SELECT COUNT(1) OVER () AS total, - all_events.event_name AS name, - * - FROM product_analytics.all_events - WHERE project_id=%(project_id)s - ORDER BY display_name - LIMIT %(limit)s OFFSET %(offset)s;""", + """SELECT COUNT(1) OVER () AS total, all_events.event_name AS name, + * + FROM product_analytics.all_events + WHERE project_id = %(project_id)s + ORDER BY display_name + LIMIT %(limit)s + OFFSET %(offset)s;""", parameters={"project_id": project_id, "limit": page.limit, "offset": (page.page - 1) * page.limit}) rows = ch_client.execute(r) if len(rows) == 0: return {"total": 0, "list": []} total = rows[0]["total"] + rows = helper.list_to_camel_case(rows) for i, row in enumerate(rows): row["id"] = f"event_{i}" - row["icon"] = None + row["dataType"] = "string" row["possibleTypes"] = ["string"] + row["_foundInPredefinedList"] = True row.pop("total") - return {"total": total, "list": helper.list_to_camel_case(rows)} + return {"total": total, "list": rows} diff --git a/api/chalicelib/core/product_analytics/properties.py b/api/chalicelib/core/product_analytics/properties.py index 704f1794c..c88fe1c7d 100644 --- a/api/chalicelib/core/product_analytics/properties.py +++ b/api/chalicelib/core/product_analytics/properties.py @@ -1,46 +1,77 @@ -import re -from functools import cache - import schemas from chalicelib.utils import helper, exp_ch_helper from chalicelib.utils.ch_client import ClickHouseClient - -@cache -def get_predefined_property_types(): - with ClickHouseClient() as ch_client: - properties_type = ch_client.execute("""\ - SELECT type - FROM system.columns - WHERE database = 'product_analytics' - AND table = 'events' - AND name = '$properties';""") - if len(properties_type) == 0: - return {} - properties_type = properties_type[0]["type"] - - pattern = r'(\w+)\s+(Enum8\([^\)]+\)|[A-Za-z0-9_]+(?:\([^\)]+\))?)' - - # Find all matches - matches = re.findall(pattern, properties_type) - - # Create a dictionary of attribute names and types - attributes = {match[0]: match[1] for match in matches} - return attributes +PREDEFINED_PROPERTIES = { + "label": "String", + "hesitation_time": "UInt32", + "name": "String", + "payload": "String", + "level": "Enum8", + "source": "Enum8", + "message": "String", + "error_id": "String", + "duration": "UInt16", + "context": "Enum8", + "url_host": "String", + "url_path": "String", + "url_hostpath": "String", + "request_start": "UInt16", + "response_start": "UInt16", + "response_end": "UInt16", + "dom_content_loaded_event_start": "UInt16", + "dom_content_loaded_event_end": "UInt16", + "load_event_start": "UInt16", + "load_event_end": "UInt16", + "first_paint": "UInt16", + "first_contentful_paint_time": "UInt16", + "speed_index": "UInt16", + "visually_complete": "UInt16", + "time_to_interactive": "UInt16", + "ttfb": "UInt16", + "ttlb": "UInt16", + "response_time": "UInt16", + "dom_building_time": "UInt16", + "dom_content_loaded_event_time": "UInt16", + "load_event_time": "UInt16", + "min_fps": "UInt8", + "avg_fps": "UInt8", + "max_fps": "UInt8", + "min_cpu": "UInt8", + "avg_cpu": "UInt8", + "max_cpu": "UInt8", + "min_total_js_heap_size": "UInt64", + "avg_total_js_heap_size": "UInt64", + "max_total_js_heap_size": "UInt64", + "min_used_js_heap_size": "UInt64", + "avg_used_js_heap_size": "UInt64", + "max_used_js_heap_size": "UInt64", + "method": "Enum8", + "status": "UInt16", + "success": "UInt8", + "request_body": "String", + "response_body": "String", + "transfer_size": "UInt32", + "selector": "String", + "normalized_x": "Float32", + "normalized_y": "Float32", + "message_id": "UInt64" +} def get_all_properties(project_id: int, page: schemas.PaginatedSchema): with ClickHouseClient() as ch_client: r = ch_client.format( - """SELECT COUNT(1) OVER () AS total, - property_name AS name, display_name, - array_agg(DISTINCT event_properties.value_type) AS possible_types - FROM product_analytics.all_properties + """SELECT COUNT(1) OVER () AS total, property_name AS name, + display_name, + array_agg(DISTINCT event_properties.value_type) AS possible_types + FROM product_analytics.all_properties LEFT JOIN product_analytics.event_properties USING (project_id, property_name) - WHERE all_properties.project_id=%(project_id)s - GROUP BY property_name,display_name - ORDER BY display_name - LIMIT %(limit)s OFFSET %(offset)s;""", + WHERE all_properties.project_id = %(project_id)s + GROUP BY property_name, display_name + ORDER BY display_name + LIMIT %(limit)s + OFFSET %(offset)s;""", parameters={"project_id": project_id, "limit": page.limit, "offset": (page.page - 1) * page.limit}) @@ -49,56 +80,80 @@ def get_all_properties(project_id: int, page: schemas.PaginatedSchema): return {"total": 0, "list": []} total = properties[0]["total"] properties = helper.list_to_camel_case(properties) - predefined_properties = get_predefined_property_types() for i, p in enumerate(properties): p["id"] = f"prop_{i}" - p["icon"] = None - if p["name"] in predefined_properties: - p["possibleTypes"].insert(0, predefined_properties[p["name"]]) - p["possibleTypes"] = list(set(p["possibleTypes"])) - p["possibleTypes"] = exp_ch_helper.simplify_clickhouse_types(p["possibleTypes"]) + p["_foundInPredefinedList"] = False + if p["name"] in PREDEFINED_PROPERTIES: + p["dataType"] = exp_ch_helper.simplify_clickhouse_type(PREDEFINED_PROPERTIES[p["name"]]) + p["_foundInPredefinedList"] = True + p["possibleTypes"] = list(set(exp_ch_helper.simplify_clickhouse_types(p["possibleTypes"]))) p.pop("total") + keys = [p["name"] for p in properties] + for p in PREDEFINED_PROPERTIES: + if p not in keys: + total += 1 + properties.append({ + "name": p, + "displayName": "", + "possibleTypes": [ + ], + "id": f"prop_{len(properties) + 1}", + "_foundInPredefinedList": False, + "dataType": PREDEFINED_PROPERTIES[p] + }) return {"total": total, "list": properties} def get_event_properties(project_id: int, event_name): with ClickHouseClient() as ch_client: r = ch_client.format( - """SELECT all_properties.property_name, - all_properties.display_name - FROM product_analytics.event_properties - INNER JOIN product_analytics.all_properties USING (property_name) - WHERE event_properties.project_id=%(project_id)s - AND all_properties.project_id=%(project_id)s - AND event_properties.event_name=%(event_name)s - ORDER BY created_at;""", + """SELECT all_properties.property_name AS name, + all_properties.display_name, + array_agg(DISTINCT event_properties.value_type) AS possible_types + FROM product_analytics.event_properties + INNER JOIN product_analytics.all_properties USING (property_name) + WHERE event_properties.project_id = %(project_id)s + AND all_properties.project_id = %(project_id)s + AND event_properties.event_name = %(event_name)s + GROUP BY ALL + ORDER BY 1;""", parameters={"project_id": project_id, "event_name": event_name}) properties = ch_client.execute(r) + properties = helper.list_to_camel_case(properties) + for i, p in enumerate(properties): + p["id"] = f"prop_{i}" + p["_foundInPredefinedList"] = False + if p["name"] in PREDEFINED_PROPERTIES: + p["dataType"] = exp_ch_helper.simplify_clickhouse_type(PREDEFINED_PROPERTIES[p["name"]]) + p["_foundInPredefinedList"] = True + p["possibleTypes"] = list(set(exp_ch_helper.simplify_clickhouse_types(p["possibleTypes"]))) - return helper.list_to_camel_case(properties) + return properties def get_lexicon(project_id: int, page: schemas.PaginatedSchema): with ClickHouseClient() as ch_client: r = ch_client.format( - """SELECT COUNT(1) OVER () AS total, - all_properties.property_name AS name, - all_properties.*, - possible_types.values AS possible_types, - possible_values.values AS sample_values - FROM product_analytics.all_properties - LEFT JOIN (SELECT project_id, property_name, array_agg(DISTINCT value_type) AS values - FROM product_analytics.event_properties - WHERE project_id=%(project_id)s - GROUP BY 1, 2) AS possible_types - USING (project_id, property_name) - LEFT JOIN (SELECT project_id, property_name, array_agg(DISTINCT value) AS values - FROM product_analytics.property_values_samples - WHERE project_id=%(project_id)s - GROUP BY 1, 2) AS possible_values USING (project_id, property_name) - WHERE project_id=%(project_id)s - ORDER BY display_name - LIMIT %(limit)s OFFSET %(offset)s;""", + """SELECT COUNT(1) OVER () AS total, all_properties.property_name AS name, + all_properties.*, + possible_types.values AS possible_types, + possible_values.values AS sample_values + FROM product_analytics.all_properties + LEFT JOIN (SELECT project_id, property_name, array_agg(DISTINCT value_type) AS + values + FROM product_analytics.event_properties + WHERE project_id=%(project_id)s + GROUP BY 1, 2) AS possible_types + USING (project_id, property_name) + LEFT JOIN (SELECT project_id, property_name, array_agg(DISTINCT value) AS + values + FROM product_analytics.property_values_samples + WHERE project_id=%(project_id)s + GROUP BY 1, 2) AS possible_values USING (project_id, property_name) + WHERE project_id = %(project_id)s + ORDER BY display_name + LIMIT %(limit)s + OFFSET %(offset)s;""", parameters={"project_id": project_id, "limit": page.limit, "offset": (page.page - 1) * page.limit}) @@ -108,6 +163,5 @@ def get_lexicon(project_id: int, page: schemas.PaginatedSchema): total = properties[0]["total"] for i, p in enumerate(properties): p["id"] = f"prop_{i}" - p["icon"] = None p.pop("total") return {"total": total, "list": helper.list_to_camel_case(properties)} diff --git a/api/chalicelib/core/sessions/sessions_search_ch.py b/api/chalicelib/core/sessions/sessions_search_ch.py index 38ada500d..c0142bae4 100644 --- a/api/chalicelib/core/sessions/sessions_search_ch.py +++ b/api/chalicelib/core/sessions/sessions_search_ch.py @@ -73,7 +73,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project: schemas. return { 'total': 0, 'sessions': [], - 'src': 2 + '_src': 2 } if project.platform == "web": full_args, query_part = sessions.search_query_parts_ch(data=data, error_status=error_status, @@ -216,7 +216,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project: schemas. return { 'total': total, 'sessions': sessions_list, - 'src': 2 + '_src': 2 } diff --git a/api/chalicelib/core/sessions/sessions_search_pg.py b/api/chalicelib/core/sessions/sessions_search_pg.py index f28af757a..9036e2686 100644 --- a/api/chalicelib/core/sessions/sessions_search_pg.py +++ b/api/chalicelib/core/sessions/sessions_search_pg.py @@ -49,7 +49,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project: schemas. return { 'total': 0, 'sessions': [], - 'src': 1 + '_src': 1 } full_args, query_part = sessions_legacy.search_query_parts(data=data, error_status=error_status, errors_only=errors_only, @@ -177,7 +177,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project: schemas. return { 'total': total, 'sessions': helper.list_to_camel_case(sessions), - 'src': 1 + '_src': 1 } @@ -240,7 +240,7 @@ def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): cur.execute("\nUNION\n".join(sub_queries)) rows = cur.fetchall() for i in rows: - i["src"] = 1 + i["_src"] = 1 results[str(i["project_id"])]["sessions"].append(helper.dict_to_camel_case(i)) return results @@ -248,7 +248,7 @@ def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): def search_sessions_by_ids(project_id: int, session_ids: list, sort_by: str = 'session_id', ascending: bool = False) -> dict: if session_ids is None or len(session_ids) == 0: - return {"total": 0, "sessions": [], "src": 1} + return {"total": 0, "sessions": [], "_src": 1} with pg_client.PostgresClient() as cur: meta_keys = metadata.get(project_id=project_id) params = {"project_id": project_id, "session_ids": tuple(session_ids)} @@ -267,4 +267,4 @@ def search_sessions_by_ids(project_id: int, session_ids: list, sort_by: str = 's s["metadata"] = {} for m in meta_keys: s["metadata"][m["key"]] = s.pop(f'metadata_{m["index"]}') - return {"total": len(rows), "sessions": helper.list_to_camel_case(rows), "src": 1} + return {"total": len(rows), "sessions": helper.list_to_camel_case(rows), "_src": 1} diff --git a/api/chalicelib/utils/exp_ch_helper.py b/api/chalicelib/utils/exp_ch_helper.py index b2c061533..babef4d57 100644 --- a/api/chalicelib/utils/exp_ch_helper.py +++ b/api/chalicelib/utils/exp_ch_helper.py @@ -99,12 +99,13 @@ def simplify_clickhouse_type(ch_type: str) -> str: return "int" # Floats: Float32, Float64 - if re.match(r'^float(32|64)$', normalized_type): + if re.match(r'^float(32|64)|double$', normalized_type): return "float" # Decimal: Decimal(P, S) if normalized_type.startswith("decimal"): - return "decimal" + # return "decimal" + return "float" # Date/DateTime if normalized_type.startswith("date"): @@ -120,11 +121,13 @@ def simplify_clickhouse_type(ch_type: str) -> str: # UUID if normalized_type.startswith("uuid"): - return "uuid" + # return "uuid" + return "string" # Enums: Enum8(...) or Enum16(...) if normalized_type.startswith("enum8") or normalized_type.startswith("enum16"): - return "enum" + # return "enum" + return "string" # Arrays: Array(T) if normalized_type.startswith("array"): diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index d4cd202c3..cac321549 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -1,16 +1,16 @@ -urllib3==2.3.0 +urllib3==2.4.0 requests==2.32.3 -boto3==1.37.21 +boto3==1.38.10 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.6 -clickhouse-connect==0.8.15 -elasticsearch==8.17.2 +psycopg[pool,binary]==3.2.7 +clickhouse-connect==0.8.17 +elasticsearch==9.0.1 jira==3.8.0 cachetools==5.5.2 fastapi==0.115.12 -uvicorn[standard]==0.34.0 +uvicorn[standard]==0.34.2 python-decouple==3.8 -pydantic[email]==2.10.6 +pydantic[email]==2.11.4 apscheduler==3.11.0 diff --git a/api/requirements.txt b/api/requirements.txt index dca445128..de8243ad4 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,18 +1,18 @@ -urllib3==2.3.0 +urllib3==2.4.0 requests==2.32.3 -boto3==1.37.21 +boto3==1.38.10 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.6 -clickhouse-connect==0.8.15 -elasticsearch==8.17.2 +psycopg[pool,binary]==3.2.7 +clickhouse-connect==0.8.17 +elasticsearch==9.0.1 jira==3.8.0 cachetools==5.5.2 fastapi==0.115.12 -uvicorn[standard]==0.34.0 +uvicorn[standard]==0.34.2 python-decouple==3.8 -pydantic[email]==2.10.6 +pydantic[email]==2.11.4 apscheduler==3.11.0 -redis==5.2.1 +redis==6.0.0 diff --git a/api/routers/subs/product_analytics.py b/api/routers/subs/product_analytics.py index 5b18ca93e..d7dbcba23 100644 --- a/api/routers/subs/product_analytics.py +++ b/api/routers/subs/product_analytics.py @@ -4,9 +4,10 @@ from fastapi import Body, Depends, Query import schemas from chalicelib.core import metadata -from chalicelib.core.product_analytics import events, properties +from chalicelib.core.product_analytics import events, properties, autocomplete from or_dependencies import OR_context from routers.base import get_routers +from typing import Optional public_app, app, app_apikey = get_routers() @@ -53,3 +54,20 @@ def get_all_lexicon_events(projectId: int, filter_query: Annotated[schemas.Pagin def get_all_lexicon_properties(projectId: int, filter_query: Annotated[schemas.PaginatedSchema, Query()], context: schemas.CurrentContext = Depends(OR_context)): return {"data": properties.get_lexicon(project_id=projectId, page=filter_query)} + + +@app.get('/{projectId}/events/autocomplete', tags=["autocomplete"]) +def autocomplete_events(projectId: int, q: Optional[str] = None, + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": autocomplete.search_events(project_id=projectId, q=None if not q or len(q) == 0 else q)} + + +@app.get('/{projectId}/properties/autocomplete', tags=["autocomplete"]) +def autocomplete_properties(projectId: int, propertyName: str, eventName: Optional[str] = None, + q: Optional[str] = None, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": autocomplete.search_properties(project_id=projectId, + event_name=None if not eventName \ + or len(eventName) == 0 else eventName, + property_name=None if not propertyName \ + or len(propertyName) == 0 else propertyName, + q=None if not q or len(q) == 0 else q)} diff --git a/backend/cmd/db/main.go b/backend/cmd/db/main.go index ee9f8a536..77ad5edd8 100644 --- a/backend/cmd/db/main.go +++ b/backend/cmd/db/main.go @@ -66,7 +66,7 @@ func main() { messages.MsgMetadata, messages.MsgIssueEvent, messages.MsgSessionStart, messages.MsgSessionEnd, messages.MsgUserID, messages.MsgUserAnonymousID, messages.MsgIntegrationEvent, messages.MsgPerformanceTrackAggr, messages.MsgJSException, messages.MsgResourceTiming, messages.MsgCustomEvent, messages.MsgCustomIssue, - messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, messages.MsgMouseClick, + messages.MsgFetch, messages.MsgNetworkRequest, messages.MsgGraphQL, messages.MsgStateAction, messages.MsgMouseClick, messages.MsgMouseClickDeprecated, messages.MsgSetPageLocation, messages.MsgSetPageLocationDeprecated, messages.MsgPageLoadTiming, messages.MsgPageRenderTiming, messages.MsgPageEvent, messages.MsgPageEventDeprecated, messages.MsgMouseThrashing, messages.MsgInputChange, diff --git a/backend/cmd/sink/main.go b/backend/cmd/sink/main.go index 5ecf87ff3..012729fb3 100644 --- a/backend/cmd/sink/main.go +++ b/backend/cmd/sink/main.go @@ -100,6 +100,7 @@ func main() { // Process assets if msg.TypeID() == messages.MsgSetNodeAttributeURLBased || msg.TypeID() == messages.MsgSetCSSDataURLBased || + msg.TypeID() == messages.MsgCSSInsertRuleURLBased || msg.TypeID() == messages.MsgAdoptedSSReplaceURLBased || msg.TypeID() == messages.MsgAdoptedSSInsertRuleURLBased { m := msg.Decode() diff --git a/backend/internal/sink/assetscache/assets.go b/backend/internal/sink/assetscache/assets.go index 9f490752e..82ae4b9bc 100644 --- a/backend/internal/sink/assetscache/assets.go +++ b/backend/internal/sink/assetscache/assets.go @@ -133,6 +133,17 @@ func (e *AssetsCache) ParseAssets(msg messages.Message) messages.Message { } newMsg.SetMeta(msg.Meta()) return newMsg + case *messages.CSSInsertRuleURLBased: + if e.shouldSkipAsset(m.BaseURL) { + return msg + } + newMsg := &messages.CSSInsertRule{ + ID: m.ID, + Index: m.Index, + Rule: e.handleCSS(m.SessionID(), m.BaseURL, m.Rule), + } + newMsg.SetMeta(msg.Meta()) + return newMsg case *messages.AdoptedSSReplaceURLBased: if e.shouldSkipAsset(m.BaseURL) { return msg diff --git a/backend/pkg/db/clickhouse/connector.go b/backend/pkg/db/clickhouse/connector.go index 44ee1c5c4..3d09dd2c0 100644 --- a/backend/pkg/db/clickhouse/connector.go +++ b/backend/pkg/db/clickhouse/connector.go @@ -251,6 +251,7 @@ func (c *connectorImpl) InsertWebInputDuration(session *sessions.Session, msg *m "hesitation_time": nullableUint32(uint32(msg.HesitationTime)), "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal input event: %s", err) @@ -298,6 +299,7 @@ func (c *connectorImpl) InsertMouseThrashing(session *sessions.Session, msg *mes "url_hostpath": hostpath, "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal issue event: %s", err) @@ -361,6 +363,7 @@ func (c *connectorImpl) InsertIssue(session *sessions.Session, msg *messages.Iss "url_hostpath": hostpath, "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal issue event: %s", err) @@ -457,6 +460,7 @@ func (c *connectorImpl) InsertWebPageEvent(session *sessions.Session, msg *messa "load_event_time": loadEventTime, "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal page event: %s", err) @@ -523,6 +527,7 @@ func (c *connectorImpl) InsertWebClickEvent(session *sessions.Session, msg *mess "url_hostpath": hostpath, "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal click event: %s", err) @@ -568,6 +573,7 @@ func (c *connectorImpl) InsertWebErrorEvent(session *sessions.Session, msg *type "message": msg.Message, "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal error event: %s", err) @@ -625,6 +631,7 @@ func (c *connectorImpl) InsertWebPerformanceTrackAggr(session *sessions.Session, "max_used_js_heap_size": msg.MaxUsedJSHeapSize, "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal performance event: %s", err) @@ -683,6 +690,7 @@ func (c *connectorImpl) InsertRequest(session *sessions.Session, msg *messages.N "url_hostpath": hostpath, "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal request event: %s", err) @@ -721,6 +729,7 @@ func (c *connectorImpl) InsertCustom(session *sessions.Session, msg *messages.Cu "payload": msg.Payload, "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal custom event: %s", err) @@ -759,6 +768,7 @@ func (c *connectorImpl) InsertGraphQL(session *sessions.Session, msg *messages.G "response_body": nullableString(msg.Response), "user_device": session.UserDevice, "user_device_type": session.UserDeviceType, + "page_title ": msg.PageTitle, }) if err != nil { return fmt.Errorf("can't marshal graphql event: %s", err) diff --git a/backend/pkg/db/types/error-event.go b/backend/pkg/db/types/error-event.go index 550174cde..32608b222 100644 --- a/backend/pkg/db/types/error-event.go +++ b/backend/pkg/db/types/error-event.go @@ -25,6 +25,7 @@ type ErrorEvent struct { Tags map[string]*string OriginType int Url string + PageTitle string } func WrapJSException(m *JSException) (*ErrorEvent, error) { @@ -37,6 +38,7 @@ func WrapJSException(m *JSException) (*ErrorEvent, error) { Payload: m.Payload, OriginType: m.TypeID(), Url: m.Url, + PageTitle: m.PageTitle, }, nil } @@ -50,6 +52,7 @@ func WrapIntegrationEvent(m *IntegrationEvent) *ErrorEvent { Payload: m.Payload, OriginType: m.TypeID(), Url: m.Url, + PageTitle: m.PageTitle, } } diff --git a/backend/pkg/handlers/web/deadClick.go b/backend/pkg/handlers/web/deadClick.go index 78d2f4408..e2c9dc10c 100644 --- a/backend/pkg/handlers/web/deadClick.go +++ b/backend/pkg/handlers/web/deadClick.go @@ -77,6 +77,8 @@ func (d *DeadClickDetector) Handle(message Message, timestamp uint64) Message { *MoveNode, *RemoveNode, *SetCSSData, + *CSSInsertRule, + *CSSDeleteRule, *SetInputValue, *SetInputChecked: return d.Build() diff --git a/backend/pkg/messages/filters.go b/backend/pkg/messages/filters.go index 227b83c39..094586abf 100644 --- a/backend/pkg/messages/filters.go +++ b/backend/pkg/messages/filters.go @@ -2,7 +2,7 @@ package messages func IsReplayerType(id int) bool { - return 1 != id && 17 != id && 23 != id && 24 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 42 != id && 56 != id && 63 != id && 64 != id && 66 != id && 78 != id && 81 != id && 82 != id && 112 != id && 115 != id && 124 != id && 125 != id && 126 != id && 127 != id && 90 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 107 != id && 110 != id + return 1 != id && 3 != id && 17 != id && 23 != id && 24 != id && 25 != id && 26 != id && 27 != id && 28 != id && 29 != id && 30 != id && 31 != id && 32 != id && 33 != id && 42 != id && 56 != id && 62 != id && 63 != id && 64 != id && 66 != id && 78 != id && 80 != id && 81 != id && 82 != id && 112 != id && 115 != id && 124 != id && 125 != id && 126 != id && 127 != id && 90 != id && 91 != id && 92 != id && 94 != id && 95 != id && 97 != id && 98 != id && 107 != id && 110 != id } func IsMobileType(id int) bool { @@ -10,5 +10,5 @@ func IsMobileType(id int) bool { } func IsDOMType(id int) bool { - return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 34 == id || 35 == id || 49 == id || 50 == id || 51 == id || 43 == id || 52 == id || 54 == id || 55 == id || 57 == id || 58 == id || 60 == id || 61 == id || 68 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 117 == id || 118 == id || 119 == id || 122 == id || 93 == id || 96 == id || 100 == id || 101 == id || 102 == id || 103 == id || 104 == id || 105 == id || 106 == id || 111 == id -} \ No newline at end of file + return 0 == id || 4 == id || 5 == id || 6 == id || 7 == id || 8 == id || 9 == id || 10 == id || 11 == id || 12 == id || 13 == id || 14 == id || 15 == id || 16 == id || 18 == id || 19 == id || 20 == id || 34 == id || 35 == id || 37 == id || 38 == id || 49 == id || 50 == id || 51 == id || 43 == id || 52 == id || 54 == id || 55 == id || 57 == id || 58 == id || 59 == id || 60 == id || 61 == id || 67 == id || 68 == id || 69 == id || 70 == id || 71 == id || 72 == id || 73 == id || 74 == id || 75 == id || 76 == id || 77 == id || 113 == id || 114 == id || 117 == id || 118 == id || 119 == id || 122 == id || 93 == id || 96 == id || 100 == id || 101 == id || 102 == id || 103 == id || 104 == id || 105 == id || 106 == id || 111 == id +} diff --git a/backend/pkg/messages/iterator.go b/backend/pkg/messages/iterator.go index 104da38fa..304f15812 100644 --- a/backend/pkg/messages/iterator.go +++ b/backend/pkg/messages/iterator.go @@ -44,8 +44,9 @@ func NewMessageIterator(log logger.Logger, messageHandler MessageHandler, messag iter.filter = filter } iter.preFilter = map[int]struct{}{ - MsgBatchMetadata: {}, MsgTimestamp: {}, MsgSessionStart: {}, - MsgSessionEnd: {}, MsgSetPageLocation: {}, MsgMobileBatchMeta: {}, + MsgBatchMetadata: {}, MsgBatchMeta: {}, MsgTimestamp: {}, + MsgSessionStart: {}, MsgSessionEnd: {}, MsgSetPageLocation: {}, + MsgMobileBatchMeta: {}, } return iter } @@ -151,6 +152,20 @@ func (i *messageIteratorImpl) preprocessing(msg Message) error { i.version = m.Version i.batchInfo.version = m.Version + case *BatchMeta: // Is not required to be present in batch since Mobile doesn't have it (though we might change it) + if i.messageInfo.Index > 1 { // Might be several 0-0 BatchMeta in a row without an error though + return fmt.Errorf("batchMeta found at the end of the batch, info: %s", i.batchInfo.Info()) + } + i.messageInfo.Index = m.PageNo<<32 + m.FirstIndex // 2^32 is the maximum count of messages per page (ha-ha) + i.messageInfo.Timestamp = uint64(m.Timestamp) + if m.Timestamp == 0 { + i.zeroTsLog("BatchMeta") + } + // Try to get saved session's page url + if savedURL := i.urls.Get(i.messageInfo.batch.sessionID); savedURL != "" { + i.messageInfo.Url = savedURL + } + case *Timestamp: i.messageInfo.Timestamp = m.Timestamp if m.Timestamp == 0 { @@ -176,6 +191,7 @@ func (i *messageIteratorImpl) preprocessing(msg Message) error { case *SetPageLocation: i.messageInfo.Url = m.URL + i.messageInfo.PageTitle = m.DocumentTitle // Save session page url in cache for using in next batches i.urls.Set(i.messageInfo.batch.sessionID, m.URL) diff --git a/backend/pkg/messages/legacy-message-transform.go b/backend/pkg/messages/legacy-message-transform.go index 2d97e252b..23ab1e52d 100644 --- a/backend/pkg/messages/legacy-message-transform.go +++ b/backend/pkg/messages/legacy-message-transform.go @@ -2,6 +2,34 @@ package messages func transformDeprecated(msg Message) Message { switch m := msg.(type) { + case *JSExceptionDeprecated: + return &JSException{ + Name: m.Name, + Message: m.Message, + Payload: m.Payload, + Metadata: "{}", + } + case *Fetch: + return &NetworkRequest{ + Type: "fetch", + Method: m.Method, + URL: m.URL, + Request: m.Request, + Response: m.Response, + Status: m.Status, + Timestamp: m.Timestamp, + Duration: m.Duration, + } + case *IssueEventDeprecated: + return &IssueEvent{ + MessageID: m.MessageID, + Timestamp: m.Timestamp, + Type: m.Type, + ContextString: m.ContextString, + Context: m.Context, + Payload: m.Payload, + URL: "", + } case *ResourceTimingDeprecated: return &ResourceTiming{ Timestamp: m.Timestamp, diff --git a/backend/pkg/messages/message.go b/backend/pkg/messages/message.go index 348e8f5e0..83f8f5920 100644 --- a/backend/pkg/messages/message.go +++ b/backend/pkg/messages/message.go @@ -54,6 +54,7 @@ type message struct { Timestamp uint64 Index uint64 Url string + PageTitle string batch *BatchInfo } @@ -70,6 +71,7 @@ func (m *message) SetMeta(origin *message) { m.Timestamp = origin.Timestamp m.Index = origin.Index m.Url = origin.Url + m.PageTitle = origin.PageTitle } func (m *message) SessionID() uint64 { diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 87a4483e8..5759ea627 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -186,6 +186,27 @@ func (msg *SessionStart) TypeID() int { return 1 } +type SessionEndDeprecated struct { + message + Timestamp uint64 +} + +func (msg *SessionEndDeprecated) Encode() []byte { + buf := make([]byte, 11) + buf[0] = 3 + p := 1 + p = WriteUint(msg.Timestamp, buf, p) + return buf[:p] +} + +func (msg *SessionEndDeprecated) Decode() Message { + return msg +} + +func (msg *SessionEndDeprecated) TypeID() int { + return 3 +} + type SetPageLocationDeprecated struct { message URL string @@ -711,6 +732,31 @@ func (msg *PageRenderTiming) TypeID() int { return 24 } +type JSExceptionDeprecated struct { + message + Name string + Message string + Payload string +} + +func (msg *JSExceptionDeprecated) Encode() []byte { + buf := make([]byte, 31+len(msg.Name)+len(msg.Message)+len(msg.Payload)) + buf[0] = 25 + p := 1 + p = WriteString(msg.Name, buf, p) + p = WriteString(msg.Message, buf, p) + p = WriteString(msg.Payload, buf, p) + return buf[:p] +} + +func (msg *JSExceptionDeprecated) Decode() Message { + return msg +} + +func (msg *JSExceptionDeprecated) TypeID() int { + return 25 +} + type IntegrationEvent struct { message Timestamp uint64 @@ -1013,6 +1059,87 @@ func (msg *SetNodeAttributeDictGlobal) TypeID() int { return 35 } +type CSSInsertRule struct { + message + ID uint64 + Rule string + Index uint64 +} + +func (msg *CSSInsertRule) Encode() []byte { + buf := make([]byte, 31+len(msg.Rule)) + buf[0] = 37 + p := 1 + p = WriteUint(msg.ID, buf, p) + p = WriteString(msg.Rule, buf, p) + p = WriteUint(msg.Index, buf, p) + return buf[:p] +} + +func (msg *CSSInsertRule) Decode() Message { + return msg +} + +func (msg *CSSInsertRule) TypeID() int { + return 37 +} + +type CSSDeleteRule struct { + message + ID uint64 + Index uint64 +} + +func (msg *CSSDeleteRule) Encode() []byte { + buf := make([]byte, 21) + buf[0] = 38 + p := 1 + p = WriteUint(msg.ID, buf, p) + p = WriteUint(msg.Index, buf, p) + return buf[:p] +} + +func (msg *CSSDeleteRule) Decode() Message { + return msg +} + +func (msg *CSSDeleteRule) TypeID() int { + return 38 +} + +type Fetch struct { + message + Method string + URL string + Request string + Response string + Status uint64 + Timestamp uint64 + Duration uint64 +} + +func (msg *Fetch) Encode() []byte { + buf := make([]byte, 71+len(msg.Method)+len(msg.URL)+len(msg.Request)+len(msg.Response)) + buf[0] = 39 + p := 1 + p = WriteString(msg.Method, buf, p) + p = WriteString(msg.URL, buf, p) + p = WriteString(msg.Request, buf, p) + p = WriteString(msg.Response, buf, p) + p = WriteUint(msg.Status, buf, p) + p = WriteUint(msg.Timestamp, buf, p) + p = WriteUint(msg.Duration, buf, p) + return buf[:p] +} + +func (msg *Fetch) Decode() Message { + return msg +} + +func (msg *Fetch) TypeID() int { + return 39 +} + type Profiler struct { message Name string @@ -1506,6 +1633,39 @@ func (msg *SetNodeFocus) TypeID() int { return 58 } +type LongTask struct { + message + Timestamp uint64 + Duration uint64 + Context uint64 + ContainerType uint64 + ContainerSrc string + ContainerId string + ContainerName string +} + +func (msg *LongTask) Encode() []byte { + buf := make([]byte, 71+len(msg.ContainerSrc)+len(msg.ContainerId)+len(msg.ContainerName)) + buf[0] = 59 + p := 1 + p = WriteUint(msg.Timestamp, buf, p) + p = WriteUint(msg.Duration, buf, p) + p = WriteUint(msg.Context, buf, p) + p = WriteUint(msg.ContainerType, buf, p) + p = WriteString(msg.ContainerSrc, buf, p) + p = WriteString(msg.ContainerId, buf, p) + p = WriteString(msg.ContainerName, buf, p) + return buf[:p] +} + +func (msg *LongTask) Decode() Message { + return msg +} + +func (msg *LongTask) TypeID() int { + return 59 +} + type SetNodeAttributeURLBased struct { message ID uint64 @@ -1558,6 +1718,37 @@ func (msg *SetCSSDataURLBased) TypeID() int { return 61 } +type IssueEventDeprecated struct { + message + MessageID uint64 + Timestamp uint64 + Type string + ContextString string + Context string + Payload string +} + +func (msg *IssueEventDeprecated) Encode() []byte { + buf := make([]byte, 61+len(msg.Type)+len(msg.ContextString)+len(msg.Context)+len(msg.Payload)) + buf[0] = 62 + p := 1 + p = WriteUint(msg.MessageID, buf, p) + p = WriteUint(msg.Timestamp, buf, p) + p = WriteString(msg.Type, buf, p) + p = WriteString(msg.ContextString, buf, p) + p = WriteString(msg.Context, buf, p) + p = WriteString(msg.Payload, buf, p) + return buf[:p] +} + +func (msg *IssueEventDeprecated) Decode() Message { + return msg +} + +func (msg *IssueEventDeprecated) TypeID() int { + return 62 +} + type TechnicalInfo struct { message Type string @@ -1625,6 +1816,33 @@ func (msg *AssetCache) TypeID() int { return 66 } +type CSSInsertRuleURLBased struct { + message + ID uint64 + Rule string + Index uint64 + BaseURL string +} + +func (msg *CSSInsertRuleURLBased) Encode() []byte { + buf := make([]byte, 41+len(msg.Rule)+len(msg.BaseURL)) + buf[0] = 67 + p := 1 + p = WriteUint(msg.ID, buf, p) + p = WriteString(msg.Rule, buf, p) + p = WriteUint(msg.Index, buf, p) + p = WriteString(msg.BaseURL, buf, p) + return buf[:p] +} + +func (msg *CSSInsertRuleURLBased) Decode() Message { + return msg +} + +func (msg *CSSInsertRuleURLBased) TypeID() int { + return 67 +} + type MouseClick struct { message ID uint64 @@ -1925,6 +2143,31 @@ func (msg *Zustand) TypeID() int { return 79 } +type BatchMeta struct { + message + PageNo uint64 + FirstIndex uint64 + Timestamp int64 +} + +func (msg *BatchMeta) Encode() []byte { + buf := make([]byte, 31) + buf[0] = 80 + p := 1 + p = WriteUint(msg.PageNo, buf, p) + p = WriteUint(msg.FirstIndex, buf, p) + p = WriteInt(msg.Timestamp, buf, p) + return buf[:p] +} + +func (msg *BatchMeta) Decode() Message { + return msg +} + +func (msg *BatchMeta) TypeID() int { + return 80 +} + type BatchMetadata struct { message Version uint64 diff --git a/backend/pkg/messages/read-message.go b/backend/pkg/messages/read-message.go index 850d6b52b..d6a12ad38 100644 --- a/backend/pkg/messages/read-message.go +++ b/backend/pkg/messages/read-message.go @@ -68,6 +68,15 @@ func DecodeSessionStart(reader BytesReader) (Message, error) { return msg, err } +func DecodeSessionEndDeprecated(reader BytesReader) (Message, error) { + var err error = nil + msg := &SessionEndDeprecated{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + func DecodeSetPageLocationDeprecated(reader BytesReader) (Message, error) { var err error = nil msg := &SetPageLocationDeprecated{} @@ -381,6 +390,21 @@ func DecodePageRenderTiming(reader BytesReader) (Message, error) { return msg, err } +func DecodeJSExceptionDeprecated(reader BytesReader) (Message, error) { + var err error = nil + msg := &JSExceptionDeprecated{} + if msg.Name, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Message, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Payload, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + func DecodeIntegrationEvent(reader BytesReader) (Message, error) { var err error = nil msg := &IntegrationEvent{} @@ -609,6 +633,60 @@ func DecodeSetNodeAttributeDictGlobal(reader BytesReader) (Message, error) { return msg, err } +func DecodeCSSInsertRule(reader BytesReader) (Message, error) { + var err error = nil + msg := &CSSInsertRule{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Rule, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Index, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + +func DecodeCSSDeleteRule(reader BytesReader) (Message, error) { + var err error = nil + msg := &CSSDeleteRule{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Index, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + +func DecodeFetch(reader BytesReader) (Message, error) { + var err error = nil + msg := &Fetch{} + if msg.Method, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.URL, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Request, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Response, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Status, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Duration, err = reader.ReadUint(); err != nil { + return nil, err + } + return msg, err +} + func DecodeProfiler(reader BytesReader) (Message, error) { var err error = nil msg := &Profiler{} @@ -921,6 +999,33 @@ func DecodeSetNodeFocus(reader BytesReader) (Message, error) { return msg, err } +func DecodeLongTask(reader BytesReader) (Message, error) { + var err error = nil + msg := &LongTask{} + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Duration, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Context, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.ContainerType, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.ContainerSrc, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.ContainerId, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.ContainerName, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + func DecodeSetNodeAttributeURLBased(reader BytesReader) (Message, error) { var err error = nil msg := &SetNodeAttributeURLBased{} @@ -954,6 +1059,30 @@ func DecodeSetCSSDataURLBased(reader BytesReader) (Message, error) { return msg, err } +func DecodeIssueEventDeprecated(reader BytesReader) (Message, error) { + var err error = nil + msg := &IssueEventDeprecated{} + if msg.MessageID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Timestamp, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Type, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.ContextString, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Context, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Payload, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + func DecodeTechnicalInfo(reader BytesReader) (Message, error) { var err error = nil msg := &TechnicalInfo{} @@ -987,6 +1116,24 @@ func DecodeAssetCache(reader BytesReader) (Message, error) { return msg, err } +func DecodeCSSInsertRuleURLBased(reader BytesReader) (Message, error) { + var err error = nil + msg := &CSSInsertRuleURLBased{} + if msg.ID, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Rule, err = reader.ReadString(); err != nil { + return nil, err + } + if msg.Index, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.BaseURL, err = reader.ReadString(); err != nil { + return nil, err + } + return msg, err +} + func DecodeMouseClick(reader BytesReader) (Message, error) { var err error = nil msg := &MouseClick{} @@ -1167,6 +1314,21 @@ func DecodeZustand(reader BytesReader) (Message, error) { return msg, err } +func DecodeBatchMeta(reader BytesReader) (Message, error) { + var err error = nil + msg := &BatchMeta{} + if msg.PageNo, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.FirstIndex, err = reader.ReadUint(); err != nil { + return nil, err + } + if msg.Timestamp, err = reader.ReadInt(); err != nil { + return nil, err + } + return msg, err +} + func DecodeBatchMetadata(reader BytesReader) (Message, error) { var err error = nil msg := &BatchMetadata{} @@ -1941,6 +2103,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeTimestamp(reader) case 1: return DecodeSessionStart(reader) + case 3: + return DecodeSessionEndDeprecated(reader) case 4: return DecodeSetPageLocationDeprecated(reader) case 5: @@ -1983,6 +2147,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodePageLoadTiming(reader) case 24: return DecodePageRenderTiming(reader) + case 25: + return DecodeJSExceptionDeprecated(reader) case 26: return DecodeIntegrationEvent(reader) case 27: @@ -2003,6 +2169,12 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeStringDictGlobal(reader) case 35: return DecodeSetNodeAttributeDictGlobal(reader) + case 37: + return DecodeCSSInsertRule(reader) + case 38: + return DecodeCSSDeleteRule(reader) + case 39: + return DecodeFetch(reader) case 40: return DecodeProfiler(reader) case 41: @@ -2041,16 +2213,22 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeLoadFontFace(reader) case 58: return DecodeSetNodeFocus(reader) + case 59: + return DecodeLongTask(reader) case 60: return DecodeSetNodeAttributeURLBased(reader) case 61: return DecodeSetCSSDataURLBased(reader) + case 62: + return DecodeIssueEventDeprecated(reader) case 63: return DecodeTechnicalInfo(reader) case 64: return DecodeCustomIssue(reader) case 66: return DecodeAssetCache(reader) + case 67: + return DecodeCSSInsertRuleURLBased(reader) case 68: return DecodeMouseClick(reader) case 69: @@ -2075,6 +2253,8 @@ func ReadMessage(t uint64, reader BytesReader) (Message, error) { return DecodeJSException(reader) case 79: return DecodeZustand(reader) + case 80: + return DecodeBatchMeta(reader) case 81: return DecodeBatchMetadata(reader) case 82: diff --git a/backend/pkg/messages/reader.go b/backend/pkg/messages/reader.go index a72331d63..436876e55 100644 --- a/backend/pkg/messages/reader.go +++ b/backend/pkg/messages/reader.go @@ -89,13 +89,15 @@ func (m *messageReaderImpl) Parse() (err error) { if err != nil { return fmt.Errorf("read message err: %s", err) } - if m.msgType == MsgBatchMetadata { + if m.msgType == MsgBatchMeta || m.msgType == MsgBatchMetadata { if len(m.list) > 0 { return fmt.Errorf("batch meta not at the start of batch") } switch message := msg.(type) { case *BatchMetadata: m.version = int(message.Version) + case *BatchMeta: + m.version = 0 } if m.version != 1 { // Unsupported tracker version, reset reader diff --git a/backend/pkg/sessions/api/web/handlers.go b/backend/pkg/sessions/api/web/handlers.go index af509d8f6..a8538ba3f 100644 --- a/backend/pkg/sessions/api/web/handlers.go +++ b/backend/pkg/sessions/api/web/handlers.go @@ -79,34 +79,31 @@ func (e *handlersImpl) GetAll() []*api.Description { } } -func getSessionTimestamp(req *StartSessionRequest, startTimeMili int64) uint64 { +func getSessionTimestamp(req *StartSessionRequest, startTimeMili int64) (ts uint64) { + ts = uint64(req.Timestamp) if req.IsOffline { - return uint64(req.Timestamp) + return } - ts := uint64(startTimeMili) - if req.BufferDiff > 0 && req.BufferDiff < 5*60*1000 { - ts -= req.BufferDiff - } - return ts -} - -func validateTrackerVersion(ver string) error { - c, err := semver.NewConstraint(">=6.0.0") + c, err := semver.NewConstraint(">=4.1.6") if err != nil { - return err + return } + ver := req.TrackerVersion parts := strings.Split(ver, "-") if len(parts) > 1 { ver = parts[0] } v, err := semver.NewVersion(ver) if err != nil { - return err + return } - if !c.Check(v) { - return errors.New("unsupported tracker version") + if c.Check(v) { + ts = uint64(startTimeMili) + if req.BufferDiff > 0 && req.BufferDiff < 5*60*1000 { + ts -= req.BufferDiff + } } - return nil + return } func (e *handlersImpl) startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { diff --git a/ee/api/chalicelib/core/errors/errors_details_exp.py b/ee/api/chalicelib/core/errors/errors_details_exp.py index 2287c5215..2898493f2 100644 --- a/ee/api/chalicelib/core/errors/errors_details_exp.py +++ b/ee/api/chalicelib/core/errors/errors_details_exp.py @@ -71,7 +71,7 @@ def get_details(project_id, error_id, user_id, **data): MAIN_EVENTS_TABLE = exp_ch_helper.get_main_events_table(0) ch_basic_query = errors_helper.__get_basic_constraints_ch(time_constraint=False) - ch_basic_query.append("toString(`$properties`.error_id) = %(error_id)s") + ch_basic_query.append("error_id = %(error_id)s") with ch_client.ClickHouseClient() as ch: data["startDate24"] = TimeUTC.now(-1) @@ -95,7 +95,7 @@ def get_details(project_id, error_id, user_id, **data): "error_id": error_id} main_ch_query = f"""\ - WITH pre_processed AS (SELECT toString(`$properties`.error_id) AS error_id, + WITH pre_processed AS (SELECT error_id, toString(`$properties`.name) AS name, toString(`$properties`.message) AS message, session_id, @@ -183,7 +183,7 @@ def get_details(project_id, error_id, user_id, **data): AND `$event_name` = 'ERROR' AND events.created_at >= toDateTime(timestamp / 1000) AND events.created_at < toDateTime((timestamp + %(step_size24)s) / 1000) - AND toString(`$properties`.error_id) = %(error_id)s + AND error_id = %(error_id)s GROUP BY timestamp ORDER BY timestamp) AS chart_details ) AS chart_details24 ON TRUE @@ -196,7 +196,7 @@ def get_details(project_id, error_id, user_id, **data): AND `$event_name` = 'ERROR' AND events.created_at >= toDateTime(timestamp / 1000) AND events.created_at < toDateTime((timestamp + %(step_size30)s) / 1000) - AND toString(`$properties`.error_id) = %(error_id)s + AND error_id = %(error_id)s GROUP BY timestamp ORDER BY timestamp) AS chart_details ) AS chart_details30 ON TRUE;""" diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index f07a5a381..fff5eb968 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -1,18 +1,18 @@ -urllib3==2.3.0 +urllib3==2.4.0 requests==2.32.3 -boto3==1.37.21 +boto3==1.38.10 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.6 -clickhouse-connect==0.8.15 -elasticsearch==8.17.2 +psycopg[pool,binary]==3.2.7 +clickhouse-connect==0.8.17 +elasticsearch==9.0.1 jira==3.8.0 cachetools==5.5.2 fastapi==0.115.12 -uvicorn[standard]==0.34.0 +uvicorn[standard]==0.34.2 python-decouple==3.8 -pydantic[email]==2.10.6 +pydantic[email]==2.11.4 apscheduler==3.11.0 -azure-storage-blob==12.25.0 +azure-storage-blob==12.25.1 diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 5c51cabf6..f960d68c7 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -3,16 +3,16 @@ requests==2.32.3 boto3==1.37.21 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.6 -clickhouse-connect==0.8.15 -elasticsearch==8.17.2 +psycopg[pool,binary]==3.2.7 +clickhouse-connect==0.8.17 +elasticsearch==9.0.1 jira==3.8.0 cachetools==5.5.2 fastapi==0.115.12 python-decouple==3.8 -pydantic[email]==2.10.6 +pydantic[email]==2.11.4 apscheduler==3.11.0 -redis==5.2.1 -azure-storage-blob==12.25.0 +redis==6.0.0 +azure-storage-blob==12.25.1 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 65c3c78ac..075415335 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -1,19 +1,19 @@ -urllib3==2.3.0 +urllib3==2.4.0 requests==2.32.3 -boto3==1.37.21 +boto3==1.38.10 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.6 -clickhouse-connect==0.8.15 -elasticsearch==8.17.2 +psycopg[pool,binary]==3.2.7 +clickhouse-connect==0.8.17 +elasticsearch==9.0.1 jira==3.8.0 cachetools==5.5.2 fastapi==0.115.12 -uvicorn[standard]==0.34.0 +uvicorn[standard]==0.34.2 gunicorn==23.0.0 python-decouple==3.8 -pydantic[email]==2.10.6 +pydantic[email]==2.11.4 apscheduler==3.11.0 # TODO: enable after xmlsec fix https://github.com/xmlsec/python-xmlsec/issues/252 @@ -21,6 +21,6 @@ apscheduler==3.11.0 python3-saml==1.16.0 --no-binary=lxml python-multipart==0.0.20 -redis==5.2.1 +redis==6.0.0 #confluent-kafka==2.1.0 -azure-storage-blob==12.25.0 +azure-storage-blob==12.25.1 diff --git a/ee/connectors/msgcodec/messages.py b/ee/connectors/msgcodec/messages.py index c6b05e42f..e4de175ef 100644 --- a/ee/connectors/msgcodec/messages.py +++ b/ee/connectors/msgcodec/messages.py @@ -35,6 +35,13 @@ class SessionStart(Message): self.user_id = user_id +class SessionEndDeprecated(Message): + __id__ = 3 + + def __init__(self, timestamp): + self.timestamp = timestamp + + class SetPageLocationDeprecated(Message): __id__ = 4 @@ -224,6 +231,15 @@ class PageRenderTiming(Message): self.time_to_interactive = time_to_interactive +class JSExceptionDeprecated(Message): + __id__ = 25 + + def __init__(self, name, message, payload): + self.name = name + self.message = message + self.payload = payload + + class IntegrationEvent(Message): __id__ = 26 @@ -323,21 +339,34 @@ class PageEvent(Message): self.web_vitals = web_vitals -class StringDictGlobal(Message): - __id__ = 34 +class CSSInsertRule(Message): + __id__ = 37 - def __init__(self, key, value): - self.key = key - self.value = value - - -class SetNodeAttributeDictGlobal(Message): - __id__ = 35 - - def __init__(self, id, name, value): + def __init__(self, id, rule, index): self.id = id - self.name = name - self.value = value + self.rule = rule + self.index = index + + +class CSSDeleteRule(Message): + __id__ = 38 + + def __init__(self, id, index): + self.id = id + self.index = index + + +class Fetch(Message): + __id__ = 39 + + def __init__(self, method, url, request, response, status, timestamp, duration): + self.method = method + self.url = url + self.request = request + self.response = response + self.status = status + self.timestamp = timestamp + self.duration = duration class Profiler(Message): @@ -520,6 +549,19 @@ class SetNodeFocus(Message): self.id = id +class LongTask(Message): + __id__ = 59 + + def __init__(self, timestamp, duration, context, container_type, container_src, container_id, container_name): + self.timestamp = timestamp + self.duration = duration + self.context = context + self.container_type = container_type + self.container_src = container_src + self.container_id = container_id + self.container_name = container_name + + class SetNodeAttributeURLBased(Message): __id__ = 60 @@ -539,6 +581,18 @@ class SetCSSDataURLBased(Message): self.base_url = base_url +class IssueEventDeprecated(Message): + __id__ = 62 + + def __init__(self, message_id, timestamp, type, context_string, context, payload): + self.message_id = message_id + self.timestamp = timestamp + self.type = type + self.context_string = context_string + self.context = context + self.payload = payload + + class TechnicalInfo(Message): __id__ = 63 @@ -562,6 +616,16 @@ class AssetCache(Message): self.url = url +class CSSInsertRuleURLBased(Message): + __id__ = 67 + + def __init__(self, id, rule, index, base_url): + self.id = id + self.rule = rule + self.index = index + self.base_url = base_url + + class MouseClick(Message): __id__ = 68 @@ -670,6 +734,15 @@ class Zustand(Message): self.state = state +class BatchMeta(Message): + __id__ = 80 + + def __init__(self, page_no, first_index, timestamp): + self.page_no = page_no + self.first_index = first_index + self.timestamp = timestamp + + class BatchMetadata(Message): __id__ = 81 diff --git a/ee/connectors/msgcodec/messages.pyx b/ee/connectors/msgcodec/messages.pyx index 79f4e5639..d801cb3b1 100644 --- a/ee/connectors/msgcodec/messages.pyx +++ b/ee/connectors/msgcodec/messages.pyx @@ -58,6 +58,15 @@ cdef class SessionStart(PyMessage): self.user_id = user_id +cdef class SessionEndDeprecated(PyMessage): + cdef public int __id__ + cdef public unsigned long timestamp + + def __init__(self, unsigned long timestamp): + self.__id__ = 3 + self.timestamp = timestamp + + cdef class SetPageLocationDeprecated(PyMessage): cdef public int __id__ cdef public str url @@ -331,6 +340,19 @@ cdef class PageRenderTiming(PyMessage): self.time_to_interactive = time_to_interactive +cdef class JSExceptionDeprecated(PyMessage): + cdef public int __id__ + cdef public str name + cdef public str message + cdef public str payload + + def __init__(self, str name, str message, str payload): + self.__id__ = 25 + self.name = name + self.message = message + self.payload = payload + + cdef class IntegrationEvent(PyMessage): cdef public int __id__ cdef public unsigned long timestamp @@ -489,28 +511,49 @@ cdef class PageEvent(PyMessage): self.web_vitals = web_vitals -cdef class StringDictGlobal(PyMessage): - cdef public int __id__ - cdef public unsigned long key - cdef public str value - - def __init__(self, unsigned long key, str value): - self.__id__ = 34 - self.key = key - self.value = value - - -cdef class SetNodeAttributeDictGlobal(PyMessage): +cdef class CSSInsertRule(PyMessage): cdef public int __id__ cdef public unsigned long id - cdef public unsigned long name - cdef public unsigned long value + cdef public str rule + cdef public unsigned long index - def __init__(self, unsigned long id, unsigned long name, unsigned long value): - self.__id__ = 35 + def __init__(self, unsigned long id, str rule, unsigned long index): + self.__id__ = 37 self.id = id - self.name = name - self.value = value + self.rule = rule + self.index = index + + +cdef class CSSDeleteRule(PyMessage): + cdef public int __id__ + cdef public unsigned long id + cdef public unsigned long index + + def __init__(self, unsigned long id, unsigned long index): + self.__id__ = 38 + self.id = id + self.index = index + + +cdef class Fetch(PyMessage): + cdef public int __id__ + cdef public str method + cdef public str url + cdef public str request + cdef public str response + cdef public unsigned long status + cdef public unsigned long timestamp + cdef public unsigned long duration + + def __init__(self, str method, str url, str request, str response, unsigned long status, unsigned long timestamp, unsigned long duration): + self.__id__ = 39 + self.method = method + self.url = url + self.request = request + self.response = response + self.status = status + self.timestamp = timestamp + self.duration = duration cdef class Profiler(PyMessage): @@ -778,6 +821,27 @@ cdef class SetNodeFocus(PyMessage): self.id = id +cdef class LongTask(PyMessage): + cdef public int __id__ + cdef public unsigned long timestamp + cdef public unsigned long duration + cdef public unsigned long context + cdef public unsigned long container_type + cdef public str container_src + cdef public str container_id + cdef public str container_name + + def __init__(self, unsigned long timestamp, unsigned long duration, unsigned long context, unsigned long container_type, str container_src, str container_id, str container_name): + self.__id__ = 59 + self.timestamp = timestamp + self.duration = duration + self.context = context + self.container_type = container_type + self.container_src = container_src + self.container_id = container_id + self.container_name = container_name + + cdef class SetNodeAttributeURLBased(PyMessage): cdef public int __id__ cdef public unsigned long id @@ -806,6 +870,25 @@ cdef class SetCSSDataURLBased(PyMessage): self.base_url = base_url +cdef class IssueEventDeprecated(PyMessage): + cdef public int __id__ + cdef public unsigned long message_id + cdef public unsigned long timestamp + cdef public str type + cdef public str context_string + cdef public str context + cdef public str payload + + def __init__(self, unsigned long message_id, unsigned long timestamp, str type, str context_string, str context, str payload): + self.__id__ = 62 + self.message_id = message_id + self.timestamp = timestamp + self.type = type + self.context_string = context_string + self.context = context + self.payload = payload + + cdef class TechnicalInfo(PyMessage): cdef public int __id__ cdef public str type @@ -837,6 +920,21 @@ cdef class AssetCache(PyMessage): self.url = url +cdef class CSSInsertRuleURLBased(PyMessage): + cdef public int __id__ + cdef public unsigned long id + cdef public str rule + cdef public unsigned long index + cdef public str base_url + + def __init__(self, unsigned long id, str rule, unsigned long index, str base_url): + self.__id__ = 67 + self.id = id + self.rule = rule + self.index = index + self.base_url = base_url + + cdef class MouseClick(PyMessage): cdef public int __id__ cdef public unsigned long id @@ -993,6 +1091,19 @@ cdef class Zustand(PyMessage): self.state = state +cdef class BatchMeta(PyMessage): + cdef public int __id__ + cdef public unsigned long page_no + cdef public unsigned long first_index + cdef public long timestamp + + def __init__(self, unsigned long page_no, unsigned long first_index, long timestamp): + self.__id__ = 80 + self.page_no = page_no + self.first_index = first_index + self.timestamp = timestamp + + cdef class BatchMetadata(PyMessage): cdef public int __id__ cdef public unsigned long version diff --git a/ee/connectors/msgcodec/msgcodec.py b/ee/connectors/msgcodec/msgcodec.py index 3a79afb63..c069eb44a 100644 --- a/ee/connectors/msgcodec/msgcodec.py +++ b/ee/connectors/msgcodec/msgcodec.py @@ -118,6 +118,11 @@ class MessageCodec(Codec): user_id=self.read_string(reader) ) + if message_id == 3: + return SessionEndDeprecated( + timestamp=self.read_uint(reader) + ) + if message_id == 4: return SetPageLocationDeprecated( url=self.read_string(reader), @@ -265,6 +270,13 @@ class MessageCodec(Codec): time_to_interactive=self.read_uint(reader) ) + if message_id == 25: + return JSExceptionDeprecated( + name=self.read_string(reader), + message=self.read_string(reader), + payload=self.read_string(reader) + ) + if message_id == 26: return IntegrationEvent( timestamp=self.read_uint(reader), @@ -348,17 +360,28 @@ class MessageCodec(Codec): web_vitals=self.read_string(reader) ) - if message_id == 34: - return StringDictGlobal( - key=self.read_uint(reader), - value=self.read_string(reader) + if message_id == 37: + return CSSInsertRule( + id=self.read_uint(reader), + rule=self.read_string(reader), + index=self.read_uint(reader) ) - if message_id == 35: - return SetNodeAttributeDictGlobal( + if message_id == 38: + return CSSDeleteRule( id=self.read_uint(reader), - name=self.read_uint(reader), - value=self.read_uint(reader) + index=self.read_uint(reader) + ) + + if message_id == 39: + return Fetch( + method=self.read_string(reader), + url=self.read_string(reader), + request=self.read_string(reader), + response=self.read_string(reader), + status=self.read_uint(reader), + timestamp=self.read_uint(reader), + duration=self.read_uint(reader) ) if message_id == 40: @@ -503,6 +526,17 @@ class MessageCodec(Codec): id=self.read_int(reader) ) + if message_id == 59: + return LongTask( + timestamp=self.read_uint(reader), + duration=self.read_uint(reader), + context=self.read_uint(reader), + container_type=self.read_uint(reader), + container_src=self.read_string(reader), + container_id=self.read_string(reader), + container_name=self.read_string(reader) + ) + if message_id == 60: return SetNodeAttributeURLBased( id=self.read_uint(reader), @@ -518,6 +552,16 @@ class MessageCodec(Codec): base_url=self.read_string(reader) ) + if message_id == 62: + return IssueEventDeprecated( + message_id=self.read_uint(reader), + timestamp=self.read_uint(reader), + type=self.read_string(reader), + context_string=self.read_string(reader), + context=self.read_string(reader), + payload=self.read_string(reader) + ) + if message_id == 63: return TechnicalInfo( type=self.read_string(reader), @@ -535,6 +579,14 @@ class MessageCodec(Codec): url=self.read_string(reader) ) + if message_id == 67: + return CSSInsertRuleURLBased( + id=self.read_uint(reader), + rule=self.read_string(reader), + index=self.read_uint(reader), + base_url=self.read_string(reader) + ) + if message_id == 68: return MouseClick( id=self.read_uint(reader), @@ -619,6 +671,13 @@ class MessageCodec(Codec): state=self.read_string(reader) ) + if message_id == 80: + return BatchMeta( + page_no=self.read_uint(reader), + first_index=self.read_uint(reader), + timestamp=self.read_int(reader) + ) + if message_id == 81: return BatchMetadata( version=self.read_uint(reader), diff --git a/ee/connectors/msgcodec/msgcodec.pyx b/ee/connectors/msgcodec/msgcodec.pyx index 769cbb603..5320a66d3 100644 --- a/ee/connectors/msgcodec/msgcodec.pyx +++ b/ee/connectors/msgcodec/msgcodec.pyx @@ -216,6 +216,11 @@ cdef class MessageCodec: user_id=self.read_string(reader) ) + if message_id == 3: + return SessionEndDeprecated( + timestamp=self.read_uint(reader) + ) + if message_id == 4: return SetPageLocationDeprecated( url=self.read_string(reader), @@ -363,6 +368,13 @@ cdef class MessageCodec: time_to_interactive=self.read_uint(reader) ) + if message_id == 25: + return JSExceptionDeprecated( + name=self.read_string(reader), + message=self.read_string(reader), + payload=self.read_string(reader) + ) + if message_id == 26: return IntegrationEvent( timestamp=self.read_uint(reader), @@ -446,17 +458,28 @@ cdef class MessageCodec: web_vitals=self.read_string(reader) ) - if message_id == 34: - return StringDictGlobal( - key=self.read_uint(reader), - value=self.read_string(reader) + if message_id == 37: + return CSSInsertRule( + id=self.read_uint(reader), + rule=self.read_string(reader), + index=self.read_uint(reader) ) - if message_id == 35: - return SetNodeAttributeDictGlobal( + if message_id == 38: + return CSSDeleteRule( id=self.read_uint(reader), - name=self.read_uint(reader), - value=self.read_uint(reader) + index=self.read_uint(reader) + ) + + if message_id == 39: + return Fetch( + method=self.read_string(reader), + url=self.read_string(reader), + request=self.read_string(reader), + response=self.read_string(reader), + status=self.read_uint(reader), + timestamp=self.read_uint(reader), + duration=self.read_uint(reader) ) if message_id == 40: @@ -601,6 +624,17 @@ cdef class MessageCodec: id=self.read_int(reader) ) + if message_id == 59: + return LongTask( + timestamp=self.read_uint(reader), + duration=self.read_uint(reader), + context=self.read_uint(reader), + container_type=self.read_uint(reader), + container_src=self.read_string(reader), + container_id=self.read_string(reader), + container_name=self.read_string(reader) + ) + if message_id == 60: return SetNodeAttributeURLBased( id=self.read_uint(reader), @@ -616,6 +650,16 @@ cdef class MessageCodec: base_url=self.read_string(reader) ) + if message_id == 62: + return IssueEventDeprecated( + message_id=self.read_uint(reader), + timestamp=self.read_uint(reader), + type=self.read_string(reader), + context_string=self.read_string(reader), + context=self.read_string(reader), + payload=self.read_string(reader) + ) + if message_id == 63: return TechnicalInfo( type=self.read_string(reader), @@ -633,6 +677,14 @@ cdef class MessageCodec: url=self.read_string(reader) ) + if message_id == 67: + return CSSInsertRuleURLBased( + id=self.read_uint(reader), + rule=self.read_string(reader), + index=self.read_uint(reader), + base_url=self.read_string(reader) + ) + if message_id == 68: return MouseClick( id=self.read_uint(reader), @@ -717,6 +769,13 @@ cdef class MessageCodec: state=self.read_string(reader) ) + if message_id == 80: + return BatchMeta( + page_no=self.read_uint(reader), + first_index=self.read_uint(reader), + timestamp=self.read_int(reader) + ) + if message_id == 81: return BatchMetadata( version=self.read_uint(reader), diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql index dcd616e5f..953c86662 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql @@ -1,65 +1,5 @@ CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.23.0-ee'; -SET allow_experimental_json_type = 1; -SET enable_json_type = 1; -ALTER TABLE product_analytics.events - MODIFY COLUMN `$properties` JSON( -max_dynamic_paths=0, -label String , -hesitation_time UInt32 , -name String , -payload String , -level Enum8 ('info'=0, 'error'=1), -source Enum8 ('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9), -message String , -error_id String , -duration UInt16, -context Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8), -url_host String , -url_path String , -url_hostpath String , -request_start UInt16 , -response_start UInt16 , -response_end UInt16 , -dom_content_loaded_event_start UInt16 , -dom_content_loaded_event_end UInt16 , -load_event_start UInt16 , -load_event_end UInt16 , -first_paint UInt16 , -first_contentful_paint_time UInt16 , -speed_index UInt16 , -visually_complete UInt16 , -time_to_interactive UInt16, -ttfb UInt16, -ttlb UInt16, -response_time UInt16, -dom_building_time UInt16, -dom_content_loaded_event_time UInt16, -load_event_time UInt16, -min_fps UInt8, -avg_fps UInt8, -max_fps UInt8, -min_cpu UInt8, -avg_cpu UInt8, -max_cpu UInt8, -min_total_js_heap_size UInt64, -avg_total_js_heap_size UInt64, -max_total_js_heap_size UInt64, -min_used_js_heap_size UInt64, -avg_used_js_heap_size UInt64, -max_used_js_heap_size UInt64, -method Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8), -status UInt16, -success UInt8, -request_body String, -response_body String, -transfer_size UInt32, -selector String, -normalized_x Float32, -normalized_y Float32, -message_id UInt64 -) DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events'; - DROP TABLE IF EXISTS product_analytics.all_events; CREATE TABLE IF NOT EXISTS product_analytics.all_events ( @@ -225,3 +165,92 @@ FROM product_analytics.events WHERE randCanonical() < 0.5 -- This randomly skips inserts AND value != '' LIMIT 2 BY project_id,property_name; + +-- Autocomplete + +CREATE TABLE IF NOT EXISTS product_analytics.autocomplete_events +( + project_id UInt16, + value String COMMENT 'The $event_name', + _timestamp DateTime +) ENGINE = MergeTree() + ORDER BY (project_id, value, _timestamp) + TTL _timestamp + INTERVAL 1 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.autocomplete_events_mv + TO product_analytics.autocomplete_events AS +SELECT project_id, + `$event_name` AS value, + _timestamp +FROM product_analytics.events +WHERE _timestamp > now() - INTERVAL 1 MONTH; + +CREATE TABLE IF NOT EXISTS product_analytics.autocomplete_events_grouped +( + project_id UInt16, + value String COMMENT 'The $event_name', + data_count UInt16 COMMENT 'The number of appearance during the past month', + _timestamp DateTime +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, value) + TTL _timestamp + INTERVAL 1 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.autocomplete_events_grouped_mv + REFRESH EVERY 30 MINUTE TO product_analytics.autocomplete_events_grouped AS +SELECT project_id, + value, + count(1) AS data_count, + max(_timestamp) AS _timestamp +FROM product_analytics.autocomplete_events +WHERE autocomplete_events._timestamp > now() - INTERVAL 1 MONTH +GROUP BY project_id, value; + +CREATE TABLE IF NOT EXISTS product_analytics.autocomplete_event_properties +( + project_id UInt16, + event_name String COMMENT 'The $event_name', + property_name String, + value String COMMENT 'The property-value as a string', + _timestamp DateTime DEFAULT now() +) ENGINE = MergeTree() + ORDER BY (project_id, event_name, property_name, value, _timestamp) + TTL _timestamp + INTERVAL 1 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.autocomplete_event_properties_mv + TO product_analytics.autocomplete_event_properties AS +SELECT project_id, + `$event_name` AS event_name, + property_name, + JSONExtractString(toString(`$properties`), property_name) AS value, + _timestamp +FROM product_analytics.events + ARRAY JOIN JSONExtractKeys(toString(`$properties`)) as property_name +WHERE length(value) > 0 AND isNull(toFloat64OrNull(value)) + AND _timestamp > now() - INTERVAL 1 MONTH; + + +CREATE TABLE IF NOT EXISTS product_analytics.autocomplete_event_properties_grouped +( + project_id UInt16, + event_name String COMMENT 'The $event_name', + property_name String, + value String COMMENT 'The property-value as a string', + data_count UInt16 COMMENT 'The number of appearance during the past month', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, event_name, property_name, value) + TTL _timestamp + INTERVAL 1 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.autocomplete_event_properties_grouped_mv + REFRESH EVERY 30 MINUTE TO product_analytics.autocomplete_event_properties_grouped AS +SELECT project_id, + event_name, + property_name, + value, + count(1) AS data_count, + max(_timestamp) AS _timestamp +FROM product_analytics.autocomplete_event_properties +WHERE length(value) > 0 + AND autocomplete_event_properties._timestamp > now() - INTERVAL 1 MONTH +GROUP BY project_id, event_name, property_name, value; + diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 5f6a06511..6c32c70c6 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -431,62 +431,7 @@ CREATE TABLE IF NOT EXISTS product_analytics.events "$source" LowCardinality(String) DEFAULT '' COMMENT 'the name of the integration that sent the event', "$duration_s" UInt16 DEFAULT 0 COMMENT 'the duration from session-start in seconds', properties JSON DEFAULT '{}', - "$properties" JSON( -max_dynamic_paths=0, -label String , -hesitation_time UInt32 , -name String , -payload String , -level Enum8 ('info'=0, 'error'=1), -source Enum8 ('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9), -message String , -error_id String , -duration UInt16, -context Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8), -url_host String , -url_path String , -url_hostpath String , -request_start UInt16 , -response_start UInt16 , -response_end UInt16 , -dom_content_loaded_event_start UInt16 , -dom_content_loaded_event_end UInt16 , -load_event_start UInt16 , -load_event_end UInt16 , -first_paint UInt16 , -first_contentful_paint_time UInt16 , -speed_index UInt16 , -visually_complete UInt16 , -time_to_interactive UInt16, -ttfb UInt16, -ttlb UInt16, -response_time UInt16, -dom_building_time UInt16, -dom_content_loaded_event_time UInt16, -load_event_time UInt16, -min_fps UInt8, -avg_fps UInt8, -max_fps UInt8, -min_cpu UInt8, -avg_cpu UInt8, -max_cpu UInt8, -min_total_js_heap_size UInt64, -avg_total_js_heap_size UInt64, -max_total_js_heap_size UInt64, -min_used_js_heap_size UInt64, -avg_used_js_heap_size UInt64, -max_used_js_heap_size UInt64, -method Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8), -status UInt16, -success UInt8, -request_body String, -response_body String, -transfer_size UInt32, -selector String, -normalized_x Float32, -normalized_y Float32, -message_id UInt64 -) DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events', + "$properties" JSON DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events', description String DEFAULT '', group_id1 Array(String) DEFAULT [], group_id2 Array(String) DEFAULT [], @@ -868,3 +813,92 @@ FROM product_analytics.events WHERE randCanonical() < 0.5 -- This randomly skips inserts AND value != '' LIMIT 2 BY project_id,property_name; + +-- Autocomplete + +CREATE TABLE IF NOT EXISTS product_analytics.autocomplete_events +( + project_id UInt16, + value String COMMENT 'The $event_name', + _timestamp DateTime +) ENGINE = MergeTree() + ORDER BY (project_id, value, _timestamp) + TTL _timestamp + INTERVAL 1 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.autocomplete_events_mv + TO product_analytics.autocomplete_events AS +SELECT project_id, + `$event_name` AS value, + _timestamp +FROM product_analytics.events +WHERE _timestamp > now() - INTERVAL 1 MONTH; + +CREATE TABLE IF NOT EXISTS product_analytics.autocomplete_events_grouped +( + project_id UInt16, + value String COMMENT 'The $event_name', + data_count UInt16 COMMENT 'The number of appearance during the past month', + _timestamp DateTime +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, value) + TTL _timestamp + INTERVAL 1 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.autocomplete_events_grouped_mv + REFRESH EVERY 30 MINUTE TO product_analytics.autocomplete_events_grouped AS +SELECT project_id, + value, + count(1) AS data_count, + max(_timestamp) AS _timestamp +FROM product_analytics.autocomplete_events +WHERE autocomplete_events._timestamp > now() - INTERVAL 1 MONTH +GROUP BY project_id, value; + +CREATE TABLE IF NOT EXISTS product_analytics.autocomplete_event_properties +( + project_id UInt16, + event_name String COMMENT 'The $event_name', + property_name String, + value String COMMENT 'The property-value as a string', + _timestamp DateTime DEFAULT now() +) ENGINE = MergeTree() + ORDER BY (project_id, event_name, property_name, value, _timestamp) + TTL _timestamp + INTERVAL 1 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.autocomplete_event_properties_mv + TO product_analytics.autocomplete_event_properties AS +SELECT project_id, + `$event_name` AS event_name, + property_name, + JSONExtractString(toString(`$properties`), property_name) AS value, + _timestamp +FROM product_analytics.events + ARRAY JOIN JSONExtractKeys(toString(`$properties`)) as property_name +WHERE length(value) > 0 AND isNull(toFloat64OrNull(value)) + AND _timestamp > now() - INTERVAL 1 MONTH; + + +CREATE TABLE IF NOT EXISTS product_analytics.autocomplete_event_properties_grouped +( + project_id UInt16, + event_name String COMMENT 'The $event_name', + property_name String, + value String COMMENT 'The property-value as a string', + data_count UInt16 COMMENT 'The number of appearance during the past month', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, event_name, property_name, value) + TTL _timestamp + INTERVAL 1 MONTH; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_analytics.autocomplete_event_properties_grouped_mv + REFRESH EVERY 30 MINUTE TO product_analytics.autocomplete_event_properties_grouped AS +SELECT project_id, + event_name, + property_name, + value, + count(1) AS data_count, + max(_timestamp) AS _timestamp +FROM product_analytics.autocomplete_event_properties +WHERE length(value) > 0 + AND autocomplete_event_properties._timestamp > now() - INTERVAL 1 MONTH +GROUP BY project_id, event_name, property_name, value; + diff --git a/frontend/app/ThemeContext.tsx b/frontend/app/ThemeContext.tsx new file mode 100644 index 000000000..51e340231 --- /dev/null +++ b/frontend/app/ThemeContext.tsx @@ -0,0 +1,64 @@ +import React, { createContext, useContext, useState, useEffect, ReactNode } from 'react'; + +type ThemeType = 'light' | 'dark'; + +interface ThemeContextType { + theme: ThemeType; + toggleTheme: () => void; +} + +const ThemeContext = createContext(undefined); + +export const ThemeProvider: React.FC<{ children: ReactNode }> = ({ children }) => { + const getInitialTheme = (): ThemeType => { + const savedTheme = localStorage.getItem('theme'); + if (savedTheme && (savedTheme === 'light' || savedTheme === 'dark')) { + return savedTheme; + } + return window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light'; + }; + + const [theme, setTheme] = useState(getInitialTheme); + + useEffect(() => { + if (theme === 'dark') { + document.documentElement.classList.add('dark'); + } else { + document.documentElement.classList.remove('dark'); + } + + localStorage.setItem('theme', theme); + }, [theme]); + + useEffect(() => { + const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)'); + + const handleChange = (e: MediaQueryListEvent) => { + // Only apply system preference if user hasn't manually set a preference + if (!localStorage.getItem('theme')) { + setTheme(e.matches ? 'dark' : 'light'); + } + }; + + mediaQuery.addEventListener('change', handleChange); + return () => mediaQuery.removeEventListener('change', handleChange); + }, []); + + const toggleTheme = () => { + setTheme(prevTheme => (prevTheme === 'dark' ? 'light' : 'dark')); + }; + + return ( + + {children} + + ); +}; + +export const useTheme = (): ThemeContextType => { + const context = useContext(ThemeContext); + if (context === undefined) { + throw new Error('useTheme must be used within a ThemeProvider'); + } + return context; +}; diff --git a/frontend/app/components/Charts/SankeyChart.tsx b/frontend/app/components/Charts/SankeyChart.tsx index 4142e395e..6f51140e0 100644 --- a/frontend/app/components/Charts/SankeyChart.tsx +++ b/frontend/app/components/Charts/SankeyChart.tsx @@ -111,9 +111,13 @@ const EChartsSankey: React.FC = (props) => { if (echartNodes.length === 0) return; - const mainNodeLink = startPoint === 'end' ? echartNodes.findIndex(n => n.id === 0) : 0; + const startDepth = startPoint === 'end' ? Math.max(...echartNodes.map(n => n.depth ?? 0)) : 0; + const mainNodeLinks = echartNodes.filter(n => n.depth === startDepth).map(n => echartNodes.findIndex(node => node.id === n.id)) const startNodeValue = echartLinks - .filter((link) => startPoint === 'start' ? link.source === mainNodeLink : link.target === mainNodeLink) + .filter((link) => startPoint === 'start' + ? mainNodeLinks.includes(link.source) + : mainNodeLinks.includes(link.target) + ) .reduce((sum, link) => sum + link.value, 0); Object.keys(nodeValues).forEach((nodeId) => { diff --git a/frontend/app/components/Client/Projects/ProjectCodeSnippet.tsx b/frontend/app/components/Client/Projects/ProjectCodeSnippet.tsx index cbe27d481..6e040bba0 100644 --- a/frontend/app/components/Client/Projects/ProjectCodeSnippet.tsx +++ b/frontend/app/components/Client/Projects/ProjectCodeSnippet.tsx @@ -114,7 +114,7 @@ const ProjectCodeSnippet: React.FC = (props) => {
diff --git a/frontend/app/components/Client/Users/components/UserForm/UserForm.tsx b/frontend/app/components/Client/Users/components/UserForm/UserForm.tsx index 3fca31f02..71457d22f 100644 --- a/frontend/app/components/Client/Users/components/UserForm/UserForm.tsx +++ b/frontend/app/components/Client/Users/components/UserForm/UserForm.tsx @@ -87,7 +87,7 @@ function UserForm() { />
{!isSmtp && ( -
+
{t('SMTP is not configured (see')}  - {isSaas ? ( - <> - - - -
- {t('or')} -
- - ) : null}
); diff --git a/frontend/app/components/Dashboard/components/WidgetChart/WidgetChart.tsx b/frontend/app/components/Dashboard/components/WidgetChart/WidgetChart.tsx index 0460053a5..abdbc9066 100644 --- a/frontend/app/components/Dashboard/components/WidgetChart/WidgetChart.tsx +++ b/frontend/app/components/Dashboard/components/WidgetChart/WidgetChart.tsx @@ -181,9 +181,10 @@ function WidgetChart(props: Props) { } prevMetricRef.current = _metric; const timestmaps = drillDownPeriod.toTimestamps(); + const density = props.isPreview ? metric.density : dashboardStore.selectedDensity const payload = isSaved - ? { ...metricParams } - : { ...params, ...timestmaps, ..._metric.toJson() }; + ? { ...metricParams, density } + : { ...params, ...timestmaps, ..._metric.toJson(), density }; debounceRequest( _metric, payload, diff --git a/frontend/app/components/Dashboard/components/WidgetDateRange/RangeGranularity.tsx b/frontend/app/components/Dashboard/components/WidgetDateRange/RangeGranularity.tsx index 02d9a970c..18a2b7815 100644 --- a/frontend/app/components/Dashboard/components/WidgetDateRange/RangeGranularity.tsx +++ b/frontend/app/components/Dashboard/components/WidgetDateRange/RangeGranularity.tsx @@ -55,7 +55,7 @@ function RangeGranularity({ } const PAST_24_HR_MS = 24 * 60 * 60 * 1000; -function calculateGranularities(periodDurationMs: number) { +export function calculateGranularities(periodDurationMs: number) { const granularities = [ { label: 'Hourly', durationMs: 60 * 60 * 1000 }, { label: 'Daily', durationMs: 24 * 60 * 60 * 1000 }, diff --git a/frontend/app/components/Session/Player/ReplayPlayer/PlayerBlockHeader.tsx b/frontend/app/components/Session/Player/ReplayPlayer/PlayerBlockHeader.tsx index ef769ec5f..b7060595b 100644 --- a/frontend/app/components/Session/Player/ReplayPlayer/PlayerBlockHeader.tsx +++ b/frontend/app/components/Session/Player/ReplayPlayer/PlayerBlockHeader.tsx @@ -116,13 +116,11 @@ function PlayerBlockHeader(props: any) { )} {_metaList.length > 0 && ( -
- -
+ )}
{uiPlayerStore.showSearchEventsSwitchButton ? ( diff --git a/frontend/app/components/Session/Player/ReplayPlayer/playerBlockHeader.module.css b/frontend/app/components/Session/Player/ReplayPlayer/playerBlockHeader.module.css index 3d83c24be..446011877 100644 --- a/frontend/app/components/Session/Player/ReplayPlayer/playerBlockHeader.module.css +++ b/frontend/app/components/Session/Player/ReplayPlayer/playerBlockHeader.module.css @@ -2,7 +2,7 @@ height: 50px; border-bottom: solid thin $gray-lighter; padding-right: 0; - background-color: white; + background-color: $white; } .divider { diff --git a/frontend/app/components/Session_/EventsBlock/EventsBlock.tsx b/frontend/app/components/Session_/EventsBlock/EventsBlock.tsx index 4587fc5c8..33cdaa123 100644 --- a/frontend/app/components/Session_/EventsBlock/EventsBlock.tsx +++ b/frontend/app/components/Session_/EventsBlock/EventsBlock.tsx @@ -233,7 +233,7 @@ function EventsBlock(props: IProps) {
{uxtestingStore.isUxt() ? ( diff --git a/frontend/app/components/Session_/EventsBlock/event.module.css b/frontend/app/components/Session_/EventsBlock/event.module.css index 5b5cec4a5..f6685d02b 100644 --- a/frontend/app/components/Session_/EventsBlock/event.module.css +++ b/frontend/app/components/Session_/EventsBlock/event.module.css @@ -13,7 +13,7 @@ .event { position: relative; - background: #f6f6f6; + background: $gray-lightest; /* border-radius: 3px; */ user-select: none; transition: all 0.2s; @@ -147,5 +147,5 @@ } .lastInGroup { - background: white; + background: $white; } diff --git a/frontend/app/components/Session_/OverviewPanel/components/TimelineScale/TimelineScale.tsx b/frontend/app/components/Session_/OverviewPanel/components/TimelineScale/TimelineScale.tsx index 8788ec9d8..6484dd1de 100644 --- a/frontend/app/components/Session_/OverviewPanel/components/TimelineScale/TimelineScale.tsx +++ b/frontend/app/components/Session_/OverviewPanel/components/TimelineScale/TimelineScale.tsx @@ -22,7 +22,7 @@ function TimelineScale(props: Props) { el.style.opacity = '0.8'; el.innerHTML = `${txt}`; el.style.fontSize = '12px'; - el.style.color = 'white'; + el.classList.add('text-white') container.appendChild(el); } diff --git a/frontend/app/components/Session_/Player/Controls/components/KeyboardHelp.tsx b/frontend/app/components/Session_/Player/Controls/components/KeyboardHelp.tsx index 8dcfb77c4..16b8a4ad1 100644 --- a/frontend/app/components/Session_/Player/Controls/components/KeyboardHelp.tsx +++ b/frontend/app/components/Session_/Player/Controls/components/KeyboardHelp.tsx @@ -7,7 +7,7 @@ function Key({ label }: { label: string }) { return (
{label}
diff --git a/frontend/app/components/Session_/Player/Controls/components/ReadNote.tsx b/frontend/app/components/Session_/Player/Controls/components/ReadNote.tsx index dffd0c3e7..1a2095361 100644 --- a/frontend/app/components/Session_/Player/Controls/components/ReadNote.tsx +++ b/frontend/app/components/Session_/Player/Controls/components/ReadNote.tsx @@ -66,7 +66,7 @@ function ReadNote(props: Props) { className="flex items-center justify-center" >
diff --git a/frontend/app/components/Session_/Player/player.module.css b/frontend/app/components/Session_/Player/player.module.css index 44a519050..f5b86143c 100644 --- a/frontend/app/components/Session_/Player/player.module.css +++ b/frontend/app/components/Session_/Player/player.module.css @@ -13,7 +13,7 @@ } .checkers { - background: repeating-linear-gradient(135deg, #f3f3f3, #f3f3f3 1px, #f6f6f6 1px, #FFF 4px); + background: repeating-linear-gradient(135deg, var(--color-gray-lighter), var(--color-gray-lighter) 1px, var(--color-gray-lightest) 1px, var(--color-white) 4px); } .solidBg { background: $gray-lightest; diff --git a/frontend/app/components/Session_/playerBlockHeader.module.css b/frontend/app/components/Session_/playerBlockHeader.module.css index 29c6e1648..96a8cecad 100644 --- a/frontend/app/components/Session_/playerBlockHeader.module.css +++ b/frontend/app/components/Session_/playerBlockHeader.module.css @@ -3,7 +3,7 @@ border-bottom: solid thin $gray-light; padding-left: 15px; padding-right: 0; - background-color: white; + background-color: $white; } .divider { diff --git a/frontend/app/components/Spots/SpotPlayer/components/SpotPlayerControls.tsx b/frontend/app/components/Spots/SpotPlayer/components/SpotPlayerControls.tsx index e18adacb8..4010ea751 100644 --- a/frontend/app/components/Spots/SpotPlayer/components/SpotPlayerControls.tsx +++ b/frontend/app/components/Spots/SpotPlayer/components/SpotPlayerControls.tsx @@ -57,7 +57,7 @@ function SpotPlayerControls() {
-
+
/
{spotPlayerStore.durationString}
diff --git a/frontend/app/components/Spots/SpotPlayer/components/SpotPlayerHeader.tsx b/frontend/app/components/Spots/SpotPlayer/components/SpotPlayerHeader.tsx index 6bf8671ce..29023c4d7 100644 --- a/frontend/app/components/Spots/SpotPlayer/components/SpotPlayerHeader.tsx +++ b/frontend/app/components/Spots/SpotPlayer/components/SpotPlayerHeader.tsx @@ -32,6 +32,7 @@ import { Avatar, Icon } from 'UI'; import { TABS, Tab } from '../consts'; import AccessModal from './AccessModal'; import { useTranslation } from 'react-i18next'; +import { toast } from 'react-toastify' const spotLink = spotsList(); @@ -89,8 +90,12 @@ function SpotPlayerHeader({ const onMenuClick = async ({ key }: { key: string }) => { if (key === '1') { + const loader = toast.loading('Retrieving Spot video...') const { url } = await spotStore.getVideo(spotStore.currentSpot!.spotId); await downloadFile(url, `${spotStore.currentSpot!.title}.mp4`); + setTimeout(() => { + toast.dismiss(loader) + }, 0) } else if (key === '2') { spotStore.deleteSpot([spotStore.currentSpot!.spotId]).then(() => { history.push(spotsList()); @@ -245,12 +250,11 @@ function SpotPlayerHeader({ } async function downloadFile(url: string, fileName: string) { - const { t } = useTranslation(); try { const response = await fetch(url); if (!response.ok) { - throw new Error(t('Network response was not ok')); + throw new Error('Network response was not ok'); } const blob = await response.blob(); @@ -263,6 +267,7 @@ async function downloadFile(url: string, fileName: string) { document.body.removeChild(a); URL.revokeObjectURL(blobUrl); } catch (error) { + toast.error('Error downloading file.') console.error('Error downloading file:', error); } } diff --git a/frontend/app/components/Spots/SpotsList/SpotListItem.tsx b/frontend/app/components/Spots/SpotsList/SpotListItem.tsx index 573a2dd81..598706654 100644 --- a/frontend/app/components/Spots/SpotsList/SpotListItem.tsx +++ b/frontend/app/components/Spots/SpotsList/SpotListItem.tsx @@ -80,8 +80,12 @@ function SpotListItem({ case 'rename': return setIsEdit(true); case 'download': + const loader = toast.loading('Retrieving Spot video...') const { url } = await onVideo(spot.spotId); await downloadFile(url, `${spot.title}.mp4`); + setTimeout(() => { + toast.dismiss(loader) + }, 0) return; case 'copy': copy( diff --git a/frontend/app/components/ThemeToggle/index.tsx b/frontend/app/components/ThemeToggle/index.tsx new file mode 100644 index 000000000..1b5bde18f --- /dev/null +++ b/frontend/app/components/ThemeToggle/index.tsx @@ -0,0 +1,31 @@ +import React from 'react'; +import { Button } from 'antd'; +import { BulbOutlined, BulbFilled } from '@ant-design/icons'; +import { useTheme } from 'App/ThemeContext'; + +interface ThemeToggleProps { + className?: string; + style?: React.CSSProperties; + size?: 'large' | 'middle' | 'small'; +} + +const ThemeToggle: React.FC = ({ + className = '', + style = {}, + size = 'middle' +}) => { + const { theme, toggleTheme } = useTheme(); + + return ( +