Dev (#2483)

* refactor(chalice): upgraded dependencies

* refactor(chalice): upgraded dependencies
feat(chalice): support heatmaps

* fix(chalice): fixed Math-operators validation
refactor(chalice): search for sessions that have events for heatmaps

* refactor(chalice): search for sessions that have at least 1 location event for heatmaps

* refactor(chalice): upgraded dependencies

* refactor(chalice): upgraded dependencies
feat(chalice): support heatmaps

* fix(chalice): fixed Math-operators validation
refactor(chalice): search for sessions that have events for heatmaps

* refactor(chalice): search for sessions that have at least 1 location event for heatmaps

* refactor(chalice): upgraded dependencies
refactor(crons): upgraded dependencies
refactor(alerts): upgraded dependencies

* feat(chalice): get top 10 values for autocomplete CH

* refactor(chalice): cleaned code
refactor(chalice): upgraded dependencies
refactor(alerts): upgraded dependencies
refactor(crons): upgraded dependencies

* feat(chalice): autocomplete return top 10 with stats

* fix(chalice): fixed autocomplete top 10 meta-filters

* refactor(chalice): upgraded dependencies
refactor(alerts): upgraded dependencies
refactor(crons): upgraded dependencies

* refactor(chalice): upgraded dependencies

* feat(chalice): table of requests

* feat(chalice): table of requests CH

api/Pipfile

@@ -6,18 +6,19 @@ name = "pypi"
 [packages]
 urllib3 = "==1.26.16"
 requests = "==2.32.3"
-boto3 = "==1.34.145"
+boto3 = "==1.34.158"
-pyjwt = "==2.8.0"
+pyjwt = "==2.9.0"
 psycopg2-binary = "==2.9.9"
 elasticsearch = "==8.14.0"
 jira = "==3.8.0"
-fastapi = "==0.111.1"
+fastapi = "==0.112.0"
 python-decouple = "==3.8"
 apscheduler = "==3.10.4"
 redis = "==5.1.0b6"
+cachetools = "==5.4.0"
 psycopg = {extras = ["pool", "binary"], version = "==3.2.1"}
-uvicorn = {extras = ["standard"], version = "==0.30.1"}
+uvicorn = {extras = ["standard"], version = "==0.30.5"}
-pydantic = {extras = ["email"], version = "==2.3.0"}
+pydantic = {extras = ["email"], version = "==2.8.2"}
 
 [dev-packages]
 

api/chalicelib/core/custom_metrics.py

@@ -141,6 +141,10 @@ def __get_table_of_referrers(project_id: int, data: schemas.CardTable, user_id:
     return __get_table_of_series(project_id=project_id, data=data)
 
 
+def __get_table_of_requests(project_id: int, data: schemas.CardTable, user_id: int = None):
+    return __get_table_of_series(project_id=project_id, data=data)
+
+
 def __get_table_chart(project_id: int, data: schemas.CardTable, user_id: int):
     supported = {
         schemas.MetricOfTable.SESSIONS: __get_table_of_sessions,
@@ -152,6 +156,7 @@ def __get_table_chart(project_id: int, data: schemas.CardTable, user_id: int):
         schemas.MetricOfTable.USER_COUNTRY: __get_table_of_countries,
         schemas.MetricOfTable.VISITED_URL: __get_table_of_urls,
         schemas.MetricOfTable.REFERRER: __get_table_of_referrers,
+        schemas.MetricOfTable.FETCH: __get_table_of_requests
     }
     return supported.get(data.metric_of, not_supported)(project_id=project_id, data=data, user_id=user_id)
 

api/chalicelib/core/sessions.py

@@ -319,6 +319,23 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de
                     if v not in extra_conditions[e.operator].value:
                         extra_conditions[e.operator].value.append(v)
         extra_conditions = list(extra_conditions.values())
+    elif metric_of == schemas.MetricOfTable.FETCH:
+        extra_event = "events_common.requests"
+        extra_conditions = {}
+        for e in data.events:
+            if e.type == schemas.EventType.REQUEST_DETAILS:
+                if e.operator not in extra_conditions:
+                    extra_conditions[e.operator] = schemas.SessionSearchEventSchema2.model_validate({
+                        "type": e.type,
+                        "isEvent": True,
+                        "value": [],
+                        "operator": e.operator,
+                        "filters": []
+                    })
+                for v in e.value:
+                    if v not in extra_conditions[e.operator].value:
+                        extra_conditions[e.operator].value.append(v)
+        extra_conditions = list(extra_conditions.values())
 
     elif metric_of == schemas.MetricOfTable.ISSUES and len(metric_value) > 0:
         data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.ISSUE,
@@ -360,6 +377,10 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de
             elif metric_of == schemas.MetricOfTable.REFERRER:
                 main_col = "referrer"
                 extra_col = ", referrer"
+            elif metric_of == schemas.MetricOfTable.FETCH:
+                main_col = "path"
+                extra_col = ", path"
+                distinct_on += ",path"
 
             if metric_format == schemas.MetricExtendedFormatType.SESSION_COUNT:
                 main_query = f"""SELECT COUNT(*) AS count,

api/or_dependencies.py

@@ -34,7 +34,7 @@ class ORRoute(APIRoute):
                 response: Response = await original_route_handler(request)
             except RequestValidationError as exc:
                 # 422 validation exception
-                logger.warning(f"422 exception when calling: {request.method} {request.url}")
+                logger.warning(f"!!! 422 exception when calling: {request.method} {request.url}")
                 logger.warning(exc.errors())
                 raise exc
             except HTTPException as e:

api/requirements-alerts.txt

@@ -1,7 +1,7 @@
 # Keep this version to not have conflicts between requests and boto3
 urllib3==1.26.16
 requests==2.32.3
-boto3==1.34.157
+boto3==1.34.158
 pyjwt==2.9.0
 psycopg2-binary==2.9.9
 psycopg[pool,binary]==3.2.1
@@ -11,7 +11,7 @@ jira==3.8.0
 
 
 fastapi==0.112.0
-uvicorn[standard]==0.30.3
+uvicorn[standard]==0.30.5
 python-decouple==3.8
-pydantic[email]==2.3.0
+pydantic[email]==2.8.2
 apscheduler==3.10.4

api/requirements.txt

@@ -1,7 +1,7 @@
 # Keep this version to not have conflicts between requests and boto3
 urllib3==1.26.16
 requests==2.32.3
-boto3==1.34.157
+boto3==1.34.158
 pyjwt==2.9.0
 psycopg2-binary==2.9.9
 psycopg[pool,binary]==3.2.1
@@ -12,9 +12,9 @@ cachetools==5.4.0
 
 
 fastapi==0.112.0
-uvicorn[standard]==0.30.3
+uvicorn[standard]==0.30.5
 python-decouple==3.8
-pydantic[email]==2.3.0
+pydantic[email]==2.8.2
 apscheduler==3.10.4
 
 redis==5.1.0b6

api/schemas/schemas.py

@@ -1009,6 +1009,7 @@ class MetricOfTable(str, Enum):
     SESSIONS = "sessions"
     ERRORS = "jsException"
     REFERRER = "referrer"
+    FETCH = EventType.REQUEST_DETAILS.value
 
 
 class MetricOfTimeseries(str, Enum):

ee/api/Pipfile

@@ -6,20 +6,21 @@ name = "pypi"
 [packages]
 urllib3 = "==1.26.16"
 requests = "==2.32.3"
-boto3 = "==1.34.147"
+boto3 = "==1.34.158"
-pyjwt = "==2.8.0"
+pyjwt = "==2.9.0"
 psycopg2-binary = "==2.9.9"
 elasticsearch = "==8.14.0"
 jira = "==3.8.0"
-fastapi = "==0.111.1"
+fastapi = "==0.112.0"
 gunicorn = "==22.0.0"
 python-decouple = "==3.8"
 apscheduler = "==3.10.4"
 python3-saml = "==1.16.0"
 redis = "==5.1.0b6"
-azure-storage-blob = "==12.21.0"
+azure-storage-blob = "==12.22.0"
-psycopg = {extras = ["binary", "pool"], version = "==3.2.1"}
+cachetools = "==5.4.0"
-uvicorn = {extras = ["standard"], version = "==0.30.3"}
+psycopg = {extras = ["pool", "binary"], version = "==3.2.1"}
+uvicorn = {extras = ["standard"], version = "==0.30.5"}
 pydantic = {extras = ["email"], version = "==2.3.0"}
 clickhouse-driver = {extras = ["lz4"], version = "==0.2.8"}
 

ee/api/chalicelib/core/custom_metrics.py

@@ -161,6 +161,10 @@ def __get_table_of_referrers(project_id: int, data: schemas.CardTable, user_id:
     return __get_table_of_series(project_id=project_id, data=data)
 
 
+def __get_table_of_requests(project_id: int, data: schemas.CardTable, user_id: int = None):
+    return __get_table_of_series(project_id=project_id, data=data)
+
+
 def __get_table_chart(project_id: int, data: schemas.CardTable, user_id: int):
     supported = {
         schemas.MetricOfTable.SESSIONS: __get_table_of_sessions,
@@ -172,6 +176,7 @@ def __get_table_chart(project_id: int, data: schemas.CardTable, user_id: int):
         schemas.MetricOfTable.USER_COUNTRY: __get_table_of_countries,
         schemas.MetricOfTable.VISITED_URL: __get_table_of_urls,
         schemas.MetricOfTable.REFERRER: __get_table_of_referrers,
+        schemas.MetricOfTable.FETCH: __get_table_of_requests
     }
     return supported.get(data.metric_of, not_supported)(project_id=project_id, data=data, user_id=user_id)
 

ee/api/chalicelib/core/sessions_exp.py

@@ -382,6 +382,29 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de
                     if v not in extra_conditions[e.operator].value:
                         extra_conditions[e.operator].value.append(v)
         extra_conditions = list(extra_conditions.values())
+    elif metric_of == schemas.MetricOfTable.FETCH:
+        extra_event = f"""SELECT DISTINCT ev.session_id, ev.url_path
+                            FROM {exp_ch_helper.get_main_events_table(data.startTimestamp)} AS ev
+                            WHERE ev.datetime >= toDateTime(%(startDate)s / 1000)
+                              AND ev.datetime <= toDateTime(%(endDate)s / 1000)
+                              AND ev.project_id = %(project_id)s
+                              AND ev.event_type = 'REQUEST'"""
+        extra_deduplication.append("url_path")
+        extra_conditions = {}
+        for e in data.events:
+            if e.type == schemas.EventType.REQUEST_DETAILS:
+                if e.operator not in extra_conditions:
+                    extra_conditions[e.operator] = schemas.SessionSearchEventSchema2.model_validate({
+                        "type": e.type,
+                        "isEvent": True,
+                        "value": [],
+                        "operator": e.operator,
+                        "filters": []
+                    })
+                for v in e.value:
+                    if v not in extra_conditions[e.operator].value:
+                        extra_conditions[e.operator].value.append(v)
+        extra_conditions = list(extra_conditions.values())
 
     elif metric_of == schemas.MetricOfTable.ISSUES and len(metric_value) > 0:
         data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.ISSUE,
@@ -427,6 +450,9 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de
             elif metric_of == schemas.MetricOfTable.REFERRER:
                 main_col = "referrer"
                 extra_col = ", referrer"
+            elif metric_of == schemas.MetricOfTable.FETCH:
+                main_col = "url_path"
+                extra_col = ", s.url_path"
 
             if metric_format == schemas.MetricExtendedFormatType.SESSION_COUNT:
                 main_query = f"""SELECT COUNT(DISTINCT {main_col}) OVER () AS main_count, 

ee/api/or_dependencies.py

@@ -36,7 +36,7 @@ class ORRoute(APIRoute):
                 response: Response = await original_route_handler(request)
             except RequestValidationError as exc:
                 # 422 validation exception
-                logger.warning(f"422 exception when calling: {request.method} {request.url}")
+                logger.warning(f"!!! 422 exception when calling: {request.method} {request.url}")
                 logger.warning(exc.errors())
                 raise exc
             except HTTPException as e:

ee/api/requirements-alerts.txt

@@ -1,7 +1,7 @@
 # Keep this version to not have conflicts between requests and boto3
 urllib3==1.26.16
 requests==2.32.3
-boto3==1.34.157
+boto3==1.34.158
 pyjwt==2.9.0
 psycopg2-binary==2.9.9
 psycopg[pool,binary]==3.2.1
@@ -13,7 +13,7 @@ jira==3.8.0
 fastapi==0.112.0
 uvicorn[standard]==0.30.5
 python-decouple==3.8
-pydantic[email]==2.3.0
+pydantic[email]==2.8.2
 apscheduler==3.10.4
 
 clickhouse-driver[lz4]==0.2.8

ee/api/requirements-crons.txt

@@ -1,7 +1,7 @@
 # Keep this version to not have conflicts between requests and boto3
 urllib3==1.26.16
 requests==2.32.3
-boto3==1.34.157
+boto3==1.34.158
 pyjwt==2.9.0
 psycopg2-binary==2.9.9
 psycopg[pool,binary]==3.2.1
@@ -12,7 +12,7 @@ jira==3.8.0
 
 fastapi==0.112.0
 python-decouple==3.8
-pydantic[email]==2.3.0
+pydantic[email]==2.8.2
 apscheduler==3.10.4
 
 clickhouse-driver[lz4]==0.2.8

ee/api/requirements.txt

@@ -1,7 +1,7 @@
 # Keep this version to not have conflicts between requests and boto3
 urllib3==1.26.16
 requests==2.32.3
-boto3==1.34.157
+boto3==1.34.158
 pyjwt==2.9.0
 psycopg2-binary==2.9.9
 psycopg[pool,binary]==3.2.1
@@ -15,7 +15,7 @@ fastapi==0.112.0
 uvicorn[standard]==0.30.5
 gunicorn==22.0.0
 python-decouple==3.8
-pydantic[email]==2.3.0
+pydantic[email]==2.8.2
 apscheduler==3.10.4
 
 clickhouse-driver[lz4]==0.2.8