From 97f9f59132bc169da2f877d6e25c56a01461ab6f Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Mon, 12 Aug 2024 19:07:20 +0200 Subject: [PATCH] Dev (#2483) * refactor(chalice): upgraded dependencies * refactor(chalice): upgraded dependencies feat(chalice): support heatmaps * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * refactor(chalice): upgraded dependencies * refactor(chalice): upgraded dependencies feat(chalice): support heatmaps * fix(chalice): fixed Math-operators validation refactor(chalice): search for sessions that have events for heatmaps * refactor(chalice): search for sessions that have at least 1 location event for heatmaps * refactor(chalice): upgraded dependencies refactor(crons): upgraded dependencies refactor(alerts): upgraded dependencies * feat(chalice): get top 10 values for autocomplete CH * refactor(chalice): cleaned code refactor(chalice): upgraded dependencies refactor(alerts): upgraded dependencies refactor(crons): upgraded dependencies * feat(chalice): autocomplete return top 10 with stats * fix(chalice): fixed autocomplete top 10 meta-filters * refactor(chalice): upgraded dependencies refactor(alerts): upgraded dependencies refactor(crons): upgraded dependencies * refactor(chalice): upgraded dependencies * feat(chalice): table of requests * feat(chalice): table of requests CH --- api/Pipfile | 11 +++++----- api/chalicelib/core/custom_metrics.py | 5 +++++ api/chalicelib/core/sessions.py | 21 +++++++++++++++++++ api/or_dependencies.py | 2 +- api/requirements-alerts.txt | 6 +++--- api/requirements.txt | 6 +++--- api/schemas/schemas.py | 1 + ee/api/Pipfile | 13 ++++++------ ee/api/chalicelib/core/custom_metrics.py | 5 +++++ ee/api/chalicelib/core/sessions_exp.py | 26 ++++++++++++++++++++++++ ee/api/or_dependencies.py | 2 +- ee/api/requirements-alerts.txt | 4 ++-- ee/api/requirements-crons.txt | 4 ++-- ee/api/requirements.txt | 4 ++-- 14 files changed, 85 insertions(+), 25 deletions(-) diff --git a/api/Pipfile b/api/Pipfile index 722afc199..b012f54bd 100644 --- a/api/Pipfile +++ b/api/Pipfile @@ -6,18 +6,19 @@ name = "pypi" [packages] urllib3 = "==1.26.16" requests = "==2.32.3" -boto3 = "==1.34.145" -pyjwt = "==2.8.0" +boto3 = "==1.34.158" +pyjwt = "==2.9.0" psycopg2-binary = "==2.9.9" elasticsearch = "==8.14.0" jira = "==3.8.0" -fastapi = "==0.111.1" +fastapi = "==0.112.0" python-decouple = "==3.8" apscheduler = "==3.10.4" redis = "==5.1.0b6" +cachetools = "==5.4.0" psycopg = {extras = ["pool", "binary"], version = "==3.2.1"} -uvicorn = {extras = ["standard"], version = "==0.30.1"} -pydantic = {extras = ["email"], version = "==2.3.0"} +uvicorn = {extras = ["standard"], version = "==0.30.5"} +pydantic = {extras = ["email"], version = "==2.8.2"} [dev-packages] diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 9e456a29f..0ea0aeca6 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -141,6 +141,10 @@ def __get_table_of_referrers(project_id: int, data: schemas.CardTable, user_id: return __get_table_of_series(project_id=project_id, data=data) +def __get_table_of_requests(project_id: int, data: schemas.CardTable, user_id: int = None): + return __get_table_of_series(project_id=project_id, data=data) + + def __get_table_chart(project_id: int, data: schemas.CardTable, user_id: int): supported = { schemas.MetricOfTable.SESSIONS: __get_table_of_sessions, @@ -152,6 +156,7 @@ def __get_table_chart(project_id: int, data: schemas.CardTable, user_id: int): schemas.MetricOfTable.USER_COUNTRY: __get_table_of_countries, schemas.MetricOfTable.VISITED_URL: __get_table_of_urls, schemas.MetricOfTable.REFERRER: __get_table_of_referrers, + schemas.MetricOfTable.FETCH: __get_table_of_requests } return supported.get(data.metric_of, not_supported)(project_id=project_id, data=data, user_id=user_id) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index f7cf63f1f..d683f7b6f 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -319,6 +319,23 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de if v not in extra_conditions[e.operator].value: extra_conditions[e.operator].value.append(v) extra_conditions = list(extra_conditions.values()) + elif metric_of == schemas.MetricOfTable.FETCH: + extra_event = "events_common.requests" + extra_conditions = {} + for e in data.events: + if e.type == schemas.EventType.REQUEST_DETAILS: + if e.operator not in extra_conditions: + extra_conditions[e.operator] = schemas.SessionSearchEventSchema2.model_validate({ + "type": e.type, + "isEvent": True, + "value": [], + "operator": e.operator, + "filters": [] + }) + for v in e.value: + if v not in extra_conditions[e.operator].value: + extra_conditions[e.operator].value.append(v) + extra_conditions = list(extra_conditions.values()) elif metric_of == schemas.MetricOfTable.ISSUES and len(metric_value) > 0: data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.ISSUE, @@ -360,6 +377,10 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de elif metric_of == schemas.MetricOfTable.REFERRER: main_col = "referrer" extra_col = ", referrer" + elif metric_of == schemas.MetricOfTable.FETCH: + main_col = "path" + extra_col = ", path" + distinct_on += ",path" if metric_format == schemas.MetricExtendedFormatType.SESSION_COUNT: main_query = f"""SELECT COUNT(*) AS count, diff --git a/api/or_dependencies.py b/api/or_dependencies.py index 16581a10a..828d0f1d5 100644 --- a/api/or_dependencies.py +++ b/api/or_dependencies.py @@ -34,7 +34,7 @@ class ORRoute(APIRoute): response: Response = await original_route_handler(request) except RequestValidationError as exc: # 422 validation exception - logger.warning(f"422 exception when calling: {request.method} {request.url}") + logger.warning(f"!!! 422 exception when calling: {request.method} {request.url}") logger.warning(exc.errors()) raise exc except HTTPException as e: diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index 1d3868d1c..dd20942ed 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -1,7 +1,7 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.32.3 -boto3==1.34.157 +boto3==1.34.158 pyjwt==2.9.0 psycopg2-binary==2.9.9 psycopg[pool,binary]==3.2.1 @@ -11,7 +11,7 @@ jira==3.8.0 fastapi==0.112.0 -uvicorn[standard]==0.30.3 +uvicorn[standard]==0.30.5 python-decouple==3.8 -pydantic[email]==2.3.0 +pydantic[email]==2.8.2 apscheduler==3.10.4 diff --git a/api/requirements.txt b/api/requirements.txt index 320e8034c..e3fbafd7a 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,7 +1,7 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.32.3 -boto3==1.34.157 +boto3==1.34.158 pyjwt==2.9.0 psycopg2-binary==2.9.9 psycopg[pool,binary]==3.2.1 @@ -12,9 +12,9 @@ cachetools==5.4.0 fastapi==0.112.0 -uvicorn[standard]==0.30.3 +uvicorn[standard]==0.30.5 python-decouple==3.8 -pydantic[email]==2.3.0 +pydantic[email]==2.8.2 apscheduler==3.10.4 redis==5.1.0b6 diff --git a/api/schemas/schemas.py b/api/schemas/schemas.py index 8154ede06..a7fedc537 100644 --- a/api/schemas/schemas.py +++ b/api/schemas/schemas.py @@ -1009,6 +1009,7 @@ class MetricOfTable(str, Enum): SESSIONS = "sessions" ERRORS = "jsException" REFERRER = "referrer" + FETCH = EventType.REQUEST_DETAILS.value class MetricOfTimeseries(str, Enum): diff --git a/ee/api/Pipfile b/ee/api/Pipfile index e9ae89447..901723f31 100644 --- a/ee/api/Pipfile +++ b/ee/api/Pipfile @@ -6,20 +6,21 @@ name = "pypi" [packages] urllib3 = "==1.26.16" requests = "==2.32.3" -boto3 = "==1.34.147" -pyjwt = "==2.8.0" +boto3 = "==1.34.158" +pyjwt = "==2.9.0" psycopg2-binary = "==2.9.9" elasticsearch = "==8.14.0" jira = "==3.8.0" -fastapi = "==0.111.1" +fastapi = "==0.112.0" gunicorn = "==22.0.0" python-decouple = "==3.8" apscheduler = "==3.10.4" python3-saml = "==1.16.0" redis = "==5.1.0b6" -azure-storage-blob = "==12.21.0" -psycopg = {extras = ["binary", "pool"], version = "==3.2.1"} -uvicorn = {extras = ["standard"], version = "==0.30.3"} +azure-storage-blob = "==12.22.0" +cachetools = "==5.4.0" +psycopg = {extras = ["pool", "binary"], version = "==3.2.1"} +uvicorn = {extras = ["standard"], version = "==0.30.5"} pydantic = {extras = ["email"], version = "==2.3.0"} clickhouse-driver = {extras = ["lz4"], version = "==0.2.8"} diff --git a/ee/api/chalicelib/core/custom_metrics.py b/ee/api/chalicelib/core/custom_metrics.py index f7d696427..a41d4fd9b 100644 --- a/ee/api/chalicelib/core/custom_metrics.py +++ b/ee/api/chalicelib/core/custom_metrics.py @@ -161,6 +161,10 @@ def __get_table_of_referrers(project_id: int, data: schemas.CardTable, user_id: return __get_table_of_series(project_id=project_id, data=data) +def __get_table_of_requests(project_id: int, data: schemas.CardTable, user_id: int = None): + return __get_table_of_series(project_id=project_id, data=data) + + def __get_table_chart(project_id: int, data: schemas.CardTable, user_id: int): supported = { schemas.MetricOfTable.SESSIONS: __get_table_of_sessions, @@ -172,6 +176,7 @@ def __get_table_chart(project_id: int, data: schemas.CardTable, user_id: int): schemas.MetricOfTable.USER_COUNTRY: __get_table_of_countries, schemas.MetricOfTable.VISITED_URL: __get_table_of_urls, schemas.MetricOfTable.REFERRER: __get_table_of_referrers, + schemas.MetricOfTable.FETCH: __get_table_of_requests } return supported.get(data.metric_of, not_supported)(project_id=project_id, data=data, user_id=user_id) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index dde34bdc4..2c870be49 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -382,6 +382,29 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de if v not in extra_conditions[e.operator].value: extra_conditions[e.operator].value.append(v) extra_conditions = list(extra_conditions.values()) + elif metric_of == schemas.MetricOfTable.FETCH: + extra_event = f"""SELECT DISTINCT ev.session_id, ev.url_path + FROM {exp_ch_helper.get_main_events_table(data.startTimestamp)} AS ev + WHERE ev.datetime >= toDateTime(%(startDate)s / 1000) + AND ev.datetime <= toDateTime(%(endDate)s / 1000) + AND ev.project_id = %(project_id)s + AND ev.event_type = 'REQUEST'""" + extra_deduplication.append("url_path") + extra_conditions = {} + for e in data.events: + if e.type == schemas.EventType.REQUEST_DETAILS: + if e.operator not in extra_conditions: + extra_conditions[e.operator] = schemas.SessionSearchEventSchema2.model_validate({ + "type": e.type, + "isEvent": True, + "value": [], + "operator": e.operator, + "filters": [] + }) + for v in e.value: + if v not in extra_conditions[e.operator].value: + extra_conditions[e.operator].value.append(v) + extra_conditions = list(extra_conditions.values()) elif metric_of == schemas.MetricOfTable.ISSUES and len(metric_value) > 0: data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.ISSUE, @@ -427,6 +450,9 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de elif metric_of == schemas.MetricOfTable.REFERRER: main_col = "referrer" extra_col = ", referrer" + elif metric_of == schemas.MetricOfTable.FETCH: + main_col = "url_path" + extra_col = ", s.url_path" if metric_format == schemas.MetricExtendedFormatType.SESSION_COUNT: main_query = f"""SELECT COUNT(DISTINCT {main_col}) OVER () AS main_count, diff --git a/ee/api/or_dependencies.py b/ee/api/or_dependencies.py index 081a74a21..3ee62aa02 100644 --- a/ee/api/or_dependencies.py +++ b/ee/api/or_dependencies.py @@ -36,7 +36,7 @@ class ORRoute(APIRoute): response: Response = await original_route_handler(request) except RequestValidationError as exc: # 422 validation exception - logger.warning(f"422 exception when calling: {request.method} {request.url}") + logger.warning(f"!!! 422 exception when calling: {request.method} {request.url}") logger.warning(exc.errors()) raise exc except HTTPException as e: diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index 7cb90fa03..d4283017e 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -1,7 +1,7 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.32.3 -boto3==1.34.157 +boto3==1.34.158 pyjwt==2.9.0 psycopg2-binary==2.9.9 psycopg[pool,binary]==3.2.1 @@ -13,7 +13,7 @@ jira==3.8.0 fastapi==0.112.0 uvicorn[standard]==0.30.5 python-decouple==3.8 -pydantic[email]==2.3.0 +pydantic[email]==2.8.2 apscheduler==3.10.4 clickhouse-driver[lz4]==0.2.8 diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 87dc3c381..7a5a9afed 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -1,7 +1,7 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.32.3 -boto3==1.34.157 +boto3==1.34.158 pyjwt==2.9.0 psycopg2-binary==2.9.9 psycopg[pool,binary]==3.2.1 @@ -12,7 +12,7 @@ jira==3.8.0 fastapi==0.112.0 python-decouple==3.8 -pydantic[email]==2.3.0 +pydantic[email]==2.8.2 apscheduler==3.10.4 clickhouse-driver[lz4]==0.2.8 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index dc1334755..e457eea90 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -1,7 +1,7 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.32.3 -boto3==1.34.157 +boto3==1.34.158 pyjwt==2.9.0 psycopg2-binary==2.9.9 psycopg[pool,binary]==3.2.1 @@ -15,7 +15,7 @@ fastapi==0.112.0 uvicorn[standard]==0.30.5 gunicorn==22.0.0 python-decouple==3.8 -pydantic[email]==2.3.0 +pydantic[email]==2.8.2 apscheduler==3.10.4 clickhouse-driver[lz4]==0.2.8