From e455bcb688b2d4a2a94da8a238d4cb3e49ffb4fa Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Fri, 5 Apr 2024 18:32:27 +0200 Subject: [PATCH] Dev (#2046) * refactor(chalice): upgraded dependencies * fix(chalice): fixed table of URLs-values not filtered according to the specified sessions' filters --- api/Pipfile | 14 ++++++------- api/chalicelib/core/sessions.py | 35 +++++++++++++++++++++++++++++++-- api/requirements-alerts.txt | 12 +++++------ api/requirements.txt | 14 ++++++------- ee/api/Pipfile | 14 ++++++------- ee/api/requirements-alerts.txt | 12 +++++------ ee/api/requirements-crons.txt | 12 +++++------ ee/api/requirements.txt | 14 ++++++------- 8 files changed, 79 insertions(+), 48 deletions(-) diff --git a/api/Pipfile b/api/Pipfile index 62bf20b26..a7c013891 100644 --- a/api/Pipfile +++ b/api/Pipfile @@ -5,19 +5,19 @@ name = "pypi" [packages] requests = "==2.31.0" -boto3 = "==1.33.8" +boto3 = "==1.34.78" pyjwt = "==2.8.0" psycopg2-binary = "==2.9.9" -elasticsearch = "==8.11.0" -jira = "==3.5.2" -fastapi = "==0.104.1" +elasticsearch = "==8.13.0" +jira = "==3.8.0" +fastapi = "==0.110.1" python-decouple = "==3.8" apscheduler = "==3.10.4" -redis = "==5.0.1" +redis = "==5.0.3" urllib3 = "==1.26.16" -uvicorn = {extras = ["standard"], version = "==0.23.2"} +psycopg = {extras = ["pool", "binary"], version = "==3.1.18"} +uvicorn = {extras = ["standard"], version = "==0.29.0"} pydantic = {extras = ["email"], version = "==2.3.0"} -psycopg = {extras = ["binary", "pool"], version = "==3.1.14"} [dev-packages] diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 11278d506..d6326b051 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -282,14 +282,31 @@ def search2_table(data: schemas.SessionsSearchPayloadSchema, project_id: int, de step_size = int(metrics_helper.__get_step_size(endTimestamp=data.endTimestamp, startTimestamp=data.startTimestamp, density=density, factor=1, decimal=True)) extra_event = None + extra_conditions = None if metric_of == schemas.MetricOfTable.visited_url: extra_event = "events.pages" + extra_conditions = {} + for e in data.events: + if e.type == schemas.EventType.location: + if e.operator not in extra_conditions: + extra_conditions[e.operator] = schemas.SessionSearchEventSchema2.model_validate({ + "type": e.type, + "isEvent": True, + "value": [], + "operator": e.operator, + "filters": [] + }) + for v in e.value: + if v not in extra_conditions[e.operator].value: + extra_conditions[e.operator].value.append(v) + extra_conditions = list(extra_conditions.values()) + elif metric_of == schemas.MetricOfTable.issues and len(metric_value) > 0: data.filters.append(schemas.SessionSearchFilterSchema(value=metric_value, type=schemas.FilterType.issue, operator=schemas.SearchEventOperator._is)) full_args, query_part = search_query_parts(data=data, error_status=None, errors_only=False, favorite_only=False, issue=None, project_id=project_id, - user_id=None, extra_event=extra_event) + user_id=None, extra_event=extra_event, extra_conditions=extra_conditions) full_args["step_size"] = step_size with pg_client.PostgresClient() as cur: if isinstance(metric_of, schemas.MetricOfTable): @@ -400,7 +417,7 @@ def __is_valid_event(is_any: bool, event: schemas.SessionSearchEventSchema2): # this function generates the query and return the generated-query with the dict of query arguments def search_query_parts(data: schemas.SessionsSearchPayloadSchema, error_status, errors_only, favorite_only, issue, - project_id, user_id, platform="web", extra_event=None): + project_id, user_id, platform="web", extra_event=None, extra_conditions=None): ss_constraints = [] full_args = {"project_id": project_id, "startDate": data.startTimestamp, "endDate": data.endTimestamp, "projectId": project_id, "userId": user_id} @@ -1085,6 +1102,20 @@ def search_query_parts(data: schemas.SessionsSearchPayloadSchema, error_status, extra_join += f"""INNER JOIN {extra_event} AS ev USING(session_id)""" extra_constraints.append("ev.timestamp>=%(startDate)s") extra_constraints.append("ev.timestamp<=%(endDate)s") + if extra_conditions and len(extra_conditions) > 0: + _extra_or_condition = [] + for i, c in enumerate(extra_conditions): + if sh.isAny_opreator(c.operator): + continue + e_k = f"ec_value{i}" + op = sh.get_sql_operator(c.operator) + c.value = helper.values_for_operator(value=c.value, op=c.operator) + full_args = {**full_args, + **sh.multi_values(c.value, value_key=e_k)} + _extra_or_condition.append(sh.multi_conditions(f"ev.{events.EventType.LOCATION.column} {op} %({e_k})s", + c.value, value_key=e_k)) + if len(_extra_or_condition) > 0: + extra_constraints.append("(" + " OR ".join(_extra_or_condition) + ")") query_part = f"""\ FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} {extra_join} diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index 0dc1b6ce6..2a8fd9f2c 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -1,17 +1,17 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.31.0 -boto3==1.34.1 +boto3==1.34.78 pyjwt==2.8.0 psycopg2-binary==2.9.9 -psycopg[pool,binary]==3.1.15 -elasticsearch==8.11.1 -jira==3.5.2 +psycopg[pool,binary]==3.1.18 +elasticsearch==8.13.0 +jira==3.8.0 -fastapi==0.105.0 -uvicorn[standard]==0.23.2 +fastapi==0.110.1 +uvicorn[standard]==0.29.0 python-decouple==3.8 pydantic[email]==2.3.0 apscheduler==3.10.4 diff --git a/api/requirements.txt b/api/requirements.txt index 5d5c9d1ba..90a763785 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,19 +1,19 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.31.0 -boto3==1.34.1 +boto3==1.34.78 pyjwt==2.8.0 psycopg2-binary==2.9.9 -psycopg[pool,binary]==3.1.15 -elasticsearch==8.11.1 -jira==3.5.2 +psycopg[pool,binary]==3.1.18 +elasticsearch==8.13.0 +jira==3.8.0 -fastapi==0.105.0 -uvicorn[standard]==0.23.2 +fastapi==0.110.1 +uvicorn[standard]==0.29.0 python-decouple==3.8 pydantic[email]==2.3.0 apscheduler==3.10.4 -redis==5.0.1 +redis==5.0.3 diff --git a/ee/api/Pipfile b/ee/api/Pipfile index dd3f2ac95..b5de42ee4 100644 --- a/ee/api/Pipfile +++ b/ee/api/Pipfile @@ -6,21 +6,21 @@ name = "pypi" [packages] urllib3 = "==1.26.16" requests = "==2.31.0" -boto3 = "==1.29.7" +boto3 = "==1.34.78" pyjwt = "==2.8.0" psycopg2-binary = "==2.9.9" -elasticsearch = "==8.11.0" -jira = "==3.5.2" -fastapi = "==0.104.1" +elasticsearch = "==8.13.0" +jira = "==3.8.0" +fastapi = "==0.110.1" gunicorn = "==21.2.0" python-decouple = "==3.8" apscheduler = "==3.10.4" python3-saml = "==1.16.0" python-multipart = "==0.0.6" -redis = "==5.0.1" +redis = "==5.0.3" azure-storage-blob = "==12.19.0" -psycopg = {extras = ["binary", "pool"], version = "==3.1.14"} -uvicorn = {extras = ["standard"], version = "==0.23.2"} +psycopg = {extras = ["binary", "pool"], version = "==3.1.18"} +uvicorn = {extras = ["standard"], version = "==0.29.0"} pydantic = {extras = ["email"], version = "==2.3.0"} clickhouse-driver = {extras = ["lz4"], version = "==0.2.6"} diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index a92ea3dfa..9b576de66 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -1,17 +1,17 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.31.0 -boto3==1.34.1 +boto3==1.34.78 pyjwt==2.8.0 psycopg2-binary==2.9.9 -psycopg[pool,binary]==3.1.15 -elasticsearch==8.11.0 -jira==3.5.2 +psycopg[pool,binary]==3.1.18 +elasticsearch==8.13.0 +jira==3.8.0 -fastapi==0.105.0 -uvicorn[standard]==0.23.2 +fastapi==0.110.1 +uvicorn[standard]==0.29.0 python-decouple==3.8 pydantic[email]==2.3.0 apscheduler==3.10.4 diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 3298da78d..75fdf9625 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -1,20 +1,20 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.31.0 -boto3==1.34.1 +boto3==1.34.78 pyjwt==2.8.0 psycopg2-binary==2.9.9 -psycopg[pool,binary]==3.1.15 -elasticsearch==8.11.0 -jira==3.5.2 +psycopg[pool,binary]==3.1.18 +elasticsearch==8.13.0 +jira==3.8.0 -fastapi==0.105.0 +fastapi==0.110.1 python-decouple==3.8 pydantic[email]==2.3.0 apscheduler==3.10.4 clickhouse-driver[lz4]==0.2.6 -redis==5.0.1 +redis==5.0.3 azure-storage-blob==12.19.0 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 01ea144fd..348e9ff29 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -1,17 +1,17 @@ # Keep this version to not have conflicts between requests and boto3 urllib3==1.26.16 requests==2.31.0 -boto3==1.34.1 +boto3==1.34.78 pyjwt==2.8.0 psycopg2-binary==2.9.9 -psycopg[pool,binary]==3.1.15 -elasticsearch==8.11.1 -jira==3.5.2 +psycopg[pool,binary]==3.1.18 +elasticsearch==8.13.0 +jira==3.8.0 -fastapi==0.105.0 -uvicorn[standard]==0.23.2 +fastapi==0.110.1 +uvicorn[standard]==0.29.0 gunicorn==21.2.0 python-decouple==3.8 pydantic[email]==2.3.0 @@ -24,6 +24,6 @@ python3-saml==1.16.0 --no-binary=lxml #python3-saml==1.16.0 python-multipart==0.0.6 -redis==5.0.1 +redis==5.0.3 #confluent-kafka==2.1.0 azure-storage-blob==12.19.0