Dev (#2316)

* refactor(chalice): upgraded dependencies

* refactor(chalice): upgraded dependencies
feat(chalice): support heatmaps

* feat(chalice): support table-of-browsers showing user-count

* feat(chalice): support table-of-devices showing user-count

* feat(chalice): support table-of-URLs showing user-count

* fix(chalice): fixed saved card's sessions-drilldown

api/Pipfile

@@ -6,18 +6,18 @@ name = "pypi"
 [packages]
 urllib3 = "==1.26.16"
 requests = "==2.32.3"
-boto3 = "==1.34.134"
+boto3 = "==1.34.125"
 pyjwt = "==2.8.0"
 psycopg2-binary = "==2.9.9"
+psycopg = {extras = ["binary", "pool"], version = "==3.1.19"}
 elasticsearch = "==8.14.0"
 jira = "==3.8.0"
 fastapi = "==0.111.0"
+uvicorn = {extras = ["standard"], version = "==0.30.1"}
 python-decouple = "==3.8"
+pydantic = {extras = ["email"], version = "==2.3.0"}
 apscheduler = "==3.10.4"
 redis = "==5.1.0b6"
-psycopg = {extras = ["binary", "pool"], version = "==3.1.19"}
-uvicorn = {extras = ["standard"], version = "==0.30.1"}
-pydantic = {extras = ["email"], version = "==2.3.0"}
 
 [dev-packages]
 

api/chalicelib/core/custom_metrics.py

@@ -190,31 +190,34 @@ def get_chart(project_id: int, data: schemas.CardSchema, user_id: int):
     return supported.get(data.metric_type, not_supported)(project_id=project_id, data=data, user_id=user_id)
 
 
-def __merge_metric_with_data(metric: schemas.CardSchema,
-                             data: schemas.CardSessionsSchema) -> schemas.CardSchema:
-    metric.startTimestamp = data.startTimestamp
-    metric.endTimestamp = data.endTimestamp
-    metric.page = data.page
-    metric.limit = data.limit
-    metric.density = data.density
-    if data.series is not None and len(data.series) > 0:
-        metric.series = data.series
-
-    # if len(data.filters) > 0:
-    #     for s in metric.series:
-    #         s.filter.filters += data.filters
-    # metric = schemas.CardSchema(**metric.model_dump(by_alias=True))
-    return metric
+# def __merge_metric_with_data(metric: schemas.CardSchema,
+#                              data: schemas.CardSessionsSchema) -> schemas.CardSchema:
+#     metric.startTimestamp = data.startTimestamp
+#     metric.endTimestamp = data.endTimestamp
+#     metric.page = data.page
+#     metric.limit = data.limit
+#     metric.density = data.density
+#     if data.series is not None and len(data.series) > 0:
+#         metric.series = data.series
+#
+#     # if len(data.filters) > 0:
+#     #     for s in metric.series:
+#     #         s.filter.filters += data.filters
+#     # metric = schemas.CardSchema(**metric.model_dump(by_alias=True))
+#     return metric
 
 
 def get_sessions_by_card_id(project_id, user_id, metric_id, data: schemas.CardSessionsSchema):
-    card: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
-    if card is None:
+    # No need for this because UI is sending the full payload
+    # card: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
+    # if card is None:
+    #    return None
+    # metric: schemas.CardSchema = schemas.CardSchema(**card)
+    # metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
+    if not card_exists(metric_id=metric_id, project_id=project_id, user_id=user_id):
         return None
-    metric: schemas.CardSchema = schemas.CardSchema(**card)
-    metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
     results = []
-    for s in metric.series:
+    for s in data.series:
         results.append({"seriesId": s.series_id, "seriesName": s.name,
                         **sessions.search_sessions(data=s.filter, project_id=project_id, user_id=user_id)})
 
@@ -222,27 +225,33 @@ def get_sessions_by_card_id(project_id, user_id, metric_id, data: schemas.CardSe
 
 
 def get_funnel_issues(project_id, user_id, metric_id, data: schemas.CardSessionsSchema):
-    raw_metric: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
-    if raw_metric is None:
+    # No need for this because UI is sending the full payload
+    # raw_metric: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
+    # if raw_metric is None:
+    #     return None
+    # metric: schemas.CardSchema = schemas.CardSchema(**raw_metric)
+    # metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
+    # if metric is None:
+    #     return None
+    if not card_exists(metric_id=metric_id, project_id=project_id, user_id=user_id):
         return None
-    metric: schemas.CardSchema = schemas.CardSchema(**raw_metric)
-    metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
-    if metric is None:
-        return None
-    for s in metric.series:
+    for s in data.series:
         return {"seriesId": s.series_id, "seriesName": s.name,
                 **funnels.get_issues_on_the_fly_widget(project_id=project_id, data=s.filter)}
 
 
 def get_errors_list(project_id, user_id, metric_id, data: schemas.CardSessionsSchema):
-    raw_metric: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
-    if raw_metric is None:
+    # No need for this because UI is sending the full payload
+    # raw_metric: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
+    # if raw_metric is None:
+    #     return None
+    # metric: schemas.CardSchema = schemas.CardSchema(**raw_metric)
+    # metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
+    # if metric is None:
+    #     return None
+    if not card_exists(metric_id=metric_id, project_id=project_id, user_id=user_id):
         return None
-    metric: schemas.CardSchema = schemas.CardSchema(**raw_metric)
-    metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
-    if metric is None:
-        return None
-    for s in metric.series:
+    for s in data.series:
         return {"seriesId": s.series_id, "seriesName": s.name,
                 **errors.search(data=s.filter, project_id=project_id, user_id=user_id)}
 
@@ -626,14 +635,17 @@ def get_funnel_sessions_by_issue(user_id, project_id, metric_id, issue_id,
                                  data: schemas.CardSessionsSchema
                                  # , range_value=None, start_date=None, end_date=None
                                  ):
-    card: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
-    if card is None:
+    # No need for this because UI is sending the full payload
+    # card: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
+    # if card is None:
+    #     return None
+    # metric: schemas.CardSchema = schemas.CardSchema(**card)
+    # metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
+    # if metric is None:
+    #     return None
+    if not card_exists(metric_id=metric_id, project_id=project_id, user_id=user_id):
         return None
-    metric: schemas.CardSchema = schemas.CardSchema(**card)
-    metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
-    if metric is None:
-        return None
-    for s in metric.series:
+    for s in data.series:
         s.filter.startTimestamp = data.startTimestamp
         s.filter.endTimestamp = data.endTimestamp
         s.filter.limit = data.limit
@@ -693,3 +705,33 @@ def make_chart_from_card(project_id, user_id, metric_id, data: schemas.CardSessi
                 return raw_metric["data"]
 
     return get_chart(project_id=project_id, data=metric, user_id=user_id)
+
+
+def card_exists(metric_id, project_id, user_id) -> bool:
+    with pg_client.PostgresClient() as cur:
+        query = cur.mogrify(
+            f"""SELECT 1
+                FROM metrics
+                         LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards
+                                            FROM (SELECT dashboard_id, name, is_public
+                                                  FROM dashboards INNER JOIN dashboard_widgets USING (dashboard_id)
+                                                  WHERE deleted_at ISNULL
+                                                    AND project_id = %(project_id)s
+                                                    AND ((dashboards.user_id = %(user_id)s OR is_public))
+                                                    AND metric_id = %(metric_id)s) AS connected_dashboards
+                                            ) AS connected_dashboards ON (TRUE)
+                         LEFT JOIN LATERAL (SELECT email AS owner_email
+                                            FROM users
+                                            WHERE deleted_at ISNULL
+                                            AND users.user_id = metrics.user_id
+                                            ) AS owner ON (TRUE)
+                WHERE metrics.project_id = %(project_id)s
+                  AND metrics.deleted_at ISNULL
+                  AND (metrics.user_id = %(user_id)s OR metrics.is_public)
+                  AND metrics.metric_id = %(metric_id)s
+                ORDER BY created_at;""",
+            {"metric_id": metric_id, "project_id": project_id, "user_id": user_id}
+        )
+        cur.execute(query)
+        row = cur.fetchone()
+        return row is not None

api/schemas/schemas.py

@@ -1081,6 +1081,36 @@ class CardSessionsSchema(_TimedSchema, _PaginatedSchema):
 
         return values
 
+    @model_validator(mode="after")
+    def __merge_out_filters_with_series(cls, values):
+        if len(values.filters) > 0:
+            for f in values.filters:
+                for s in values.series:
+                    found = False
+
+                    if f.is_event:
+                        sub = s.filter.events
+                    else:
+                        sub = s.filter.filters
+
+                    for e in sub:
+                        if f.type == e.type and f.operator == e.operator:
+                            found = True
+                            if f.is_event:
+                                # If extra event: append value
+                                for v in f.value:
+                                    if v not in e.value:
+                                        e.value.append(v)
+                            else:
+                                # If extra filter: override value
+                                e.value = f.value
+                    if not found:
+                        sub.append(f)
+
+            values.filters = []
+
+        return values
+
 
 class CardConfigSchema(BaseModel):
     col: Optional[int] = Field(default=None)

ee/api/Pipfile

@@ -6,22 +6,22 @@ name = "pypi"
 [packages]
 urllib3 = "==1.26.16"
 requests = "==2.32.3"
-boto3 = "==1.34.134"
+boto3 = "==1.34.125"
 pyjwt = "==2.8.0"
 psycopg2-binary = "==2.9.9"
+psycopg = {extras = ["binary", "pool"], version = "==3.1.19"}
 elasticsearch = "==8.14.0"
 jira = "==3.8.0"
 fastapi = "==0.111.0"
+uvicorn = {extras = ["standard"], version = "==0.30.1"}
 gunicorn = "==22.0.0"
 python-decouple = "==3.8"
+pydantic = {extras = ["email"], version = "==2.3.0"}
 apscheduler = "==3.10.4"
+clickhouse-driver = {extras = ["lz4"], version = "==0.2.8"}
 python3-saml = "==1.16.0"
 redis = "==5.1.0b6"
 azure-storage-blob = "==12.21.0b1"
-psycopg = {extras = ["pool", "binary"], version = "==3.1.19"}
-uvicorn = {extras = ["standard"], version = "==0.30.1"}
-pydantic = {extras = ["email"], version = "==2.3.0"}
-clickhouse-driver = {extras = ["lz4"], version = "==0.2.8"}
 
 [dev-packages]
 

ee/api/chalicelib/core/custom_metrics.py

@@ -75,6 +75,7 @@ def __get_funnel_chart(project_id: int, data: schemas.CardFunnel, user_id: int =
             "stages": [],
             "totalDropDueToIssues": 0
         }
+
     return funnels.get_top_insights_on_the_fly_widget(project_id=project_id,
                                                       data=data.series[0].filter,
                                                       metric_of=data.metric_of)
@@ -209,31 +210,34 @@ def get_chart(project_id: int, data: schemas.CardSchema, user_id: int):
     return supported.get(data.metric_type, not_supported)(project_id=project_id, data=data, user_id=user_id)
 
 
-def __merge_metric_with_data(metric: schemas.CardSchema,
-                             data: schemas.CardSessionsSchema) -> schemas.CardSchema:
-    metric.startTimestamp = data.startTimestamp
-    metric.endTimestamp = data.endTimestamp
-    metric.page = data.page
-    metric.limit = data.limit
-    metric.density = data.density
-    if data.series is not None and len(data.series) > 0:
-        metric.series = data.series
-
-    # if len(data.filters) > 0:
-    #     for s in metric.series:
-    #         s.filter.filters += data.filters
-    # metric = schemas.CardSchema(**metric.model_dump(by_alias=True))
-    return metric
+# def __merge_metric_with_data(metric: schemas.CardSchema,
+#                              data: schemas.CardSessionsSchema) -> schemas.CardSchema:
+#     metric.startTimestamp = data.startTimestamp
+#     metric.endTimestamp = data.endTimestamp
+#     metric.page = data.page
+#     metric.limit = data.limit
+#     metric.density = data.density
+#     if data.series is not None and len(data.series) > 0:
+#         metric.series = data.series
+#
+#     # if len(data.filters) > 0:
+#     #     for s in metric.series:
+#     #         s.filter.filters += data.filters
+#     # metric = schemas.CardSchema(**metric.model_dump(by_alias=True))
+#     return metric
 
 
 def get_sessions_by_card_id(project_id, user_id, metric_id, data: schemas.CardSessionsSchema):
-    card: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
-    if card is None:
+    # No need for this because UI is sending the full payload
+    # card: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
+    # if card is None:
+    #     return None
+    # metric: schemas.CardSchema = schemas.CardSchema(**card)
+    # metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
+    if not card_exists(metric_id=metric_id, project_id=project_id, user_id=user_id):
         return None
-    metric: schemas.CardSchema = schemas.CardSchema(**card)
-    metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
     results = []
-    for s in metric.series:
+    for s in data.series:
         results.append({"seriesId": s.series_id, "seriesName": s.name,
                         **sessions.search_sessions(data=s.filter, project_id=project_id, user_id=user_id)})
 
@@ -241,27 +245,33 @@ def get_sessions_by_card_id(project_id, user_id, metric_id, data: schemas.CardSe
 
 
 def get_funnel_issues(project_id, user_id, metric_id, data: schemas.CardSessionsSchema):
-    raw_metric: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
-    if raw_metric is None:
+    # No need for this because UI is sending the full payload
+    # raw_metric: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
+    # if raw_metric is None:
+    #     return None
+    # metric: schemas.CardSchema = schemas.CardSchema(**raw_metric)
+    # metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
+    # if metric is None:
+    #     return None
+    if not card_exists(metric_id=metric_id, project_id=project_id, user_id=user_id):
         return None
-    metric: schemas.CardSchema = schemas.CardSchema(**raw_metric)
-    metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
-    if metric is None:
-        return None
-    for s in metric.series:
+    for s in data.series:
         return {"seriesId": s.series_id, "seriesName": s.name,
                 **funnels.get_issues_on_the_fly_widget(project_id=project_id, data=s.filter)}
 
 
 def get_errors_list(project_id, user_id, metric_id, data: schemas.CardSessionsSchema):
-    raw_metric: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
-    if raw_metric is None:
+    # No need for this because UI is sending the full payload
+    # raw_metric: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
+    # if raw_metric is None:
+    #     return None
+    # metric: schemas.CardSchema = schemas.CardSchema(**raw_metric)
+    # metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
+    # if metric is None:
+    #     return None
+    if not card_exists(metric_id=metric_id, project_id=project_id, user_id=user_id):
         return None
-    metric: schemas.CardSchema = schemas.CardSchema(**raw_metric)
-    metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
-    if metric is None:
-        return None
-    for s in metric.series:
+    for s in data.series:
         return {"seriesId": s.series_id, "seriesName": s.name,
                 **errors.search(data=s.filter, project_id=project_id, user_id=user_id)}
 
@@ -672,14 +682,17 @@ def get_funnel_sessions_by_issue(user_id, project_id, metric_id, issue_id,
                                  data: schemas.CardSessionsSchema
                                  # , range_value=None, start_date=None, end_date=None
                                  ):
-    card: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
-    if card is None:
+    # No need for this because UI is sending the full payload
+    # card: dict = get_card(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
+    # if card is None:
+    #     return None
+    # metric: schemas.CardSchema = schemas.CardSchema(**card)
+    # metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
+    # if metric is None:
+    #     return None
+    if not card_exists(metric_id=metric_id, project_id=project_id, user_id=user_id):
         return None
-    metric: schemas.CardSchema = schemas.CardSchema(**card)
-    metric: schemas.CardSchema = __merge_metric_with_data(metric=metric, data=data)
-    if metric is None:
-        return None
-    for s in metric.series:
+    for s in data.series:
         s.filter.startTimestamp = data.startTimestamp
         s.filter.endTimestamp = data.endTimestamp
         s.filter.limit = data.limit
@@ -739,3 +752,33 @@ def make_chart_from_card(project_id, user_id, metric_id, data: schemas.CardSessi
                 return raw_metric["data"]
 
     return get_chart(project_id=project_id, data=metric, user_id=user_id)
+
+
+def card_exists(metric_id, project_id, user_id) -> bool:
+    with pg_client.PostgresClient() as cur:
+        query = cur.mogrify(
+            f"""SELECT 1
+                FROM metrics
+                         LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards
+                                            FROM (SELECT dashboard_id, name, is_public
+                                                  FROM dashboards INNER JOIN dashboard_widgets USING (dashboard_id)
+                                                  WHERE deleted_at ISNULL
+                                                    AND project_id = %(project_id)s
+                                                    AND ((dashboards.user_id = %(user_id)s OR is_public))
+                                                    AND metric_id = %(metric_id)s) AS connected_dashboards
+                                            ) AS connected_dashboards ON (TRUE)
+                         LEFT JOIN LATERAL (SELECT email AS owner_email
+                                            FROM users
+                                            WHERE deleted_at ISNULL
+                                            AND users.user_id = metrics.user_id
+                                            ) AS owner ON (TRUE)
+                WHERE metrics.project_id = %(project_id)s
+                  AND metrics.deleted_at ISNULL
+                  AND (metrics.user_id = %(user_id)s OR metrics.is_public)
+                  AND metrics.metric_id = %(metric_id)s
+                ORDER BY created_at;""",
+            {"metric_id": metric_id, "project_id": project_id, "user_id": user_id}
+        )
+        cur.execute(query)
+        row = cur.fetchone()
+        return row is not None