From 2291980a89b1d9cfcd938f803e86fc01f890ce1a Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Wed, 8 Jan 2025 16:09:53 +0100 Subject: [PATCH 01/11] Dev (#2926) * fix(chalice): fixed path-finder first step-type issue * refactor(chalice): removed time between steps in path-finder --- .../core/metrics/product_analytics.py | 22 +++---------- .../core/metrics/product_analytics_ch.py | 32 +++++++++++-------- api/chalicelib/utils/ch_client_exp.py | 6 ++-- api/schemas/schemas.py | 4 +-- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/api/chalicelib/core/metrics/product_analytics.py b/api/chalicelib/core/metrics/product_analytics.py index f25d67564..ff9c83904 100644 --- a/api/chalicelib/core/metrics/product_analytics.py +++ b/api/chalicelib/core/metrics/product_analytics.py @@ -18,10 +18,7 @@ def __transform_journey(rows, reverse_path=False): break number_of_step1 += 1 total_100p += r["sessions_count"] - # for i in range(number_of_step1): - # rows[i]["value"] = 100 / number_of_step1 - # for i in range(number_of_step1, len(rows)): for i in range(len(rows)): rows[i]["value"] = rows[i]["sessions_count"] * 100 / total_100p @@ -32,22 +29,17 @@ def __transform_journey(rows, reverse_path=False): source = f"{r['event_number_in_session']}_{r['event_type']}_{r['e_value']}" if source not in nodes: nodes.append(source) - nodes_values.append({"name": r['e_value'], "eventType": r['event_type'], - "avgTimeFromPrevious": 0, "sessionsCount": 0}) + nodes_values.append({"name": r['e_value'], "eventType": r['event_type']}) if r['next_value']: target = f"{r['event_number_in_session'] + 1}_{r['next_type']}_{r['next_value']}" if target not in nodes: nodes.append(target) - nodes_values.append({"name": r['next_value'], "eventType": r['next_type'], - "avgTimeFromPrevious": 0, "sessionsCount": 0}) + nodes_values.append({"name": r['next_value'], "eventType": r['next_type']}) sr_idx = nodes.index(source) tg_idx = nodes.index(target) - if r["avg_time_from_previous"] is not None: - nodes_values[tg_idx]["avgTimeFromPrevious"] += r["avg_time_from_previous"] * r["sessions_count"] - nodes_values[tg_idx]["sessionsCount"] += r["sessions_count"] - link = {"eventType": r['event_type'], "sessionsCount": r["sessions_count"], - "value": r["value"], "avgTimeFromPrevious": r["avg_time_from_previous"]} + + link = {"eventType": r['event_type'], "sessionsCount": r["sessions_count"],"value": r["value"]} if not reverse_path: link["source"] = sr_idx link["target"] = tg_idx @@ -55,12 +47,6 @@ def __transform_journey(rows, reverse_path=False): link["source"] = tg_idx link["target"] = sr_idx links.append(link) - for n in nodes_values: - if n["sessionsCount"] > 0: - n["avgTimeFromPrevious"] = n["avgTimeFromPrevious"] / n["sessionsCount"] - else: - n["avgTimeFromPrevious"] = None - n.pop("sessionsCount") return {"nodes": nodes_values, "links": sorted(links, key=lambda x: (x["source"], x["target"]), reverse=False)} diff --git a/api/chalicelib/core/metrics/product_analytics_ch.py b/api/chalicelib/core/metrics/product_analytics_ch.py index a6082147d..d31adc8fb 100644 --- a/api/chalicelib/core/metrics/product_analytics_ch.py +++ b/api/chalicelib/core/metrics/product_analytics_ch.py @@ -27,7 +27,7 @@ JOURNEY_TYPES = { # query: Q5, the result is correct, # startPoints are computed before ranked_events to reduce the number of window functions over rows # replaced time_to_target by time_from_previous -# compute avg_time_from_previous at the same level as sessions_count +# compute avg_time_from_previous at the same level as sessions_count (this was removed in v1.22) # sort by top 5 according to sessions_count at the CTE level # final part project data without grouping # if start-point is selected, the selected event is ranked n°1 @@ -35,15 +35,29 @@ def path_analysis(project_id: int, data: schemas.CardPathAnalysis): sub_events = [] start_points_conditions = [] step_0_conditions = [] + step_1_post_conditions = ["event_number_in_session <= %(density)s"] + if len(data.metric_value) == 0: data.metric_value.append(schemas.ProductAnalyticsSelectedEventType.LOCATION) sub_events.append({"column": JOURNEY_TYPES[schemas.ProductAnalyticsSelectedEventType.LOCATION]["column"], "eventType": schemas.ProductAnalyticsSelectedEventType.LOCATION.value}) else: + if len(data.start_point) > 0: + extra_metric_values = [] + for s in data.start_point: + if s.type not in data.metric_value: + sub_events.append({"column": JOURNEY_TYPES[s.type]["column"], + "eventType": JOURNEY_TYPES[s.type]["eventType"]}) + step_1_post_conditions.append( + f"(event_type!='{JOURNEY_TYPES[s.type]["eventType"]}' OR event_number_in_session = 1)") + extra_metric_values.append(s.type) + data.metric_value += extra_metric_values + for v in data.metric_value: if JOURNEY_TYPES.get(v): sub_events.append({"column": JOURNEY_TYPES[v]["column"], "eventType": JOURNEY_TYPES[v]["eventType"]}) + if len(sub_events) == 1: main_column = sub_events[0]['column'] else: @@ -317,7 +331,6 @@ def path_analysis(project_id: int, data: schemas.CardPathAnalysis): e_value, next_type, next_value, - AVG(time_from_previous) AS avg_time_from_previous, COUNT(1) AS sessions_count FROM ranked_events WHERE event_number_in_session = 1 @@ -330,8 +343,7 @@ def path_analysis(project_id: int, data: schemas.CardPathAnalysis): e_value, next_type, next_value, - sessions_count, - avg_time_from_previous + sessions_count FROM n1"""] for i in range(2, data.density + 1): steps_query.append(f"""n{i} AS (SELECT * @@ -340,7 +352,6 @@ def path_analysis(project_id: int, data: schemas.CardPathAnalysis): re.e_value AS e_value, re.next_type AS next_type, re.next_value AS next_value, - AVG(re.time_from_previous) AS avg_time_from_previous, COUNT(1) AS sessions_count FROM n{i - 1} INNER JOIN ranked_events AS re ON (n{i - 1}.next_value = re.e_value AND n{i - 1}.next_type = re.event_type) @@ -353,8 +364,7 @@ def path_analysis(project_id: int, data: schemas.CardPathAnalysis): e_value, next_type, next_value, - sessions_count, - avg_time_from_previous + sessions_count FROM n{i}""") with ch_client.ClickHouseClient(database="experimental") as ch: @@ -382,7 +392,7 @@ WITH {initial_sessions_cte} FROM {main_events_table} {"INNER JOIN sub_sessions ON (sub_sessions.session_id = events.session_id)" if len(sessions_conditions) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} ) AS full_ranked_events - WHERE event_number_in_session <= %(density)s) + WHERE {" AND ".join(step_1_post_conditions)}) SELECT * FROM pre_ranked_events;""" logger.debug("---------Q1-----------") @@ -404,11 +414,7 @@ WITH pre_ranked_events AS (SELECT * ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS next_value, leadInFrame(toNullable(event_type)) OVER (PARTITION BY session_id ORDER BY datetime {path_direction} - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS next_type, - abs(lagInFrame(toNullable(datetime)) - OVER (PARTITION BY session_id ORDER BY datetime {path_direction} - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - - pre_ranked_events.datetime) AS time_from_previous + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS next_type FROM start_points INNER JOIN pre_ranked_events USING (session_id)) SELECT * FROM ranked_events;""" diff --git a/api/chalicelib/utils/ch_client_exp.py b/api/chalicelib/utils/ch_client_exp.py index 24cd11df8..f91b0361e 100644 --- a/api/chalicelib/utils/ch_client_exp.py +++ b/api/chalicelib/utils/ch_client_exp.py @@ -6,7 +6,6 @@ from queue import Queue, Empty import clickhouse_connect from clickhouse_connect.driver.query import QueryContext -from clickhouse_connect.driver.exceptions import DatabaseError from decouple import config logger = logging.getLogger(__name__) @@ -32,9 +31,10 @@ if config("CH_COMPRESSION", cast=bool, default=True): extra_args["compression"] = "lz4" -def transform_result(original_function): +def transform_result(self, original_function): @wraps(original_function) def wrapper(*args, **kwargs): + logger.debug(self.format(query=kwargs.get("query"), parameters=kwargs.get("parameters"))) result = original_function(*args, **kwargs) if isinstance(result, clickhouse_connect.driver.query.QueryResult): column_names = result.column_names @@ -140,7 +140,7 @@ class ClickHouseClient: else: self.__client = CH_pool.get_connection() - self.__client.execute = transform_result(self.__client.query) + self.__client.execute = transform_result(self, self.__client.query) self.__client.format = self.format def __enter__(self): diff --git a/api/schemas/schemas.py b/api/schemas/schemas.py index cbe331221..0e18e87fa 100644 --- a/api/schemas/schemas.py +++ b/api/schemas/schemas.py @@ -1209,10 +1209,10 @@ class CardPathAnalysis(__CardSchema): if len(s.value) == 0: continue start_point.append(s) - self.metric_value.append(s.type) + # self.metric_value.append(s.type) self.start_point = start_point - self.metric_value = remove_duplicate_values(self.metric_value) + # self.metric_value = remove_duplicate_values(self.metric_value) return self From bd9f95851c6d61c0cc0bac0eade8824ced592524 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Wed, 8 Jan 2025 16:28:07 +0100 Subject: [PATCH 02/11] Dev (#2927) * feat(chalice): autocomplete return top 10 with stats * fix(chalice): fixed autocomplete top 10 meta-filters * refactor(chalice): upgraded docker base image refactor(crons): upgraded docker base image refactor(alerts): upgraded docker base image --- api/Dockerfile | 2 +- api/Dockerfile_alerts | 2 +- ee/api/Dockerfile | 2 +- ee/api/Dockerfile_alerts | 2 +- ee/api/Dockerfile_crons | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 43e1df99a..232805144 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-alpine +FROM python:3.12-alpine LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" ARG GIT_SHA diff --git a/api/Dockerfile_alerts b/api/Dockerfile_alerts index 2435d31de..fb9c5f72f 100644 --- a/api/Dockerfile_alerts +++ b/api/Dockerfile_alerts @@ -1,4 +1,4 @@ -FROM python:3.11-alpine +FROM python:3.12-alpine LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" ARG GIT_SHA diff --git a/ee/api/Dockerfile b/ee/api/Dockerfile index 86071c52f..3d54c65dc 100644 --- a/ee/api/Dockerfile +++ b/ee/api/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-alpine +FROM python:3.12-alpine LABEL Maintainer="KRAIEM Taha Yassine" RUN apk add --no-cache build-base libressl libffi-dev libressl-dev libxslt-dev libxml2-dev xmlsec-dev xmlsec tini diff --git a/ee/api/Dockerfile_alerts b/ee/api/Dockerfile_alerts index 49f89c6ff..6ae93f936 100644 --- a/ee/api/Dockerfile_alerts +++ b/ee/api/Dockerfile_alerts @@ -1,4 +1,4 @@ -FROM python:3.11-alpine +FROM python:3.12-alpine LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" RUN apk add --no-cache build-base tini diff --git a/ee/api/Dockerfile_crons b/ee/api/Dockerfile_crons index a4d76751e..fc8f11515 100644 --- a/ee/api/Dockerfile_crons +++ b/ee/api/Dockerfile_crons @@ -1,4 +1,4 @@ -FROM python:3.11-alpine +FROM python:3.12-alpine LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" RUN apk add --no-cache build-base tini From 8c66fd412dbe42b1e7c13783059690a2d5d9b363 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Thu, 9 Jan 2025 12:42:42 +0100 Subject: [PATCH 03/11] Dev (#2928) * fix(chalice): changed ee dockerfile --- ee/api/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ee/api/Dockerfile b/ee/api/Dockerfile index 3d54c65dc..a90b6a5ba 100644 --- a/ee/api/Dockerfile +++ b/ee/api/Dockerfile @@ -13,10 +13,11 @@ ENV SOURCE_MAP_VERSION=0.7.4 \ WORKDIR /work COPY requirements.txt ./requirements.txt # Caching the source build -RUN pip install --no-cache-dir --upgrade uv -RUN uv pip install --no-cache-dir --upgrade pip setuptools wheel --system -RUN uv pip install --no-cache-dir --upgrade python3-saml==1.16.0 --no-binary=lxml --system -RUN uv pip install --no-cache-dir --upgrade -r requirements.txt --system +#RUN pip install --no-cache-dir --upgrade uv +#RUN uv pip install --no-cache-dir --upgrade pip setuptools wheel --system +#RUN uv pip install --no-cache-dir --upgrade python3-saml==1.16.0 --no-binary=lxml --system +#RUN uv pip install --no-cache-dir --upgrade -r requirements.txt --system +RUN pip install --no-cache-dir --upgrade -r requirements.txt --system COPY . . RUN mv env.default .env From 4c81b195a153f5b8b90aab87f1bfcc4371b10c15 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Thu, 9 Jan 2025 12:48:48 +0100 Subject: [PATCH 04/11] Dev (#2929) * Revert "fix(chalice): changed ee dockerfile" This reverts commit c6ba000c49c6c0eb1f3017f7c8fbc555136d3db1. From c74d1671a51a6ac389d834a88095807cf98359f5 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Thu, 9 Jan 2025 13:05:25 +0100 Subject: [PATCH 05/11] Dev (#2930) * fix(chalice): changed ee dockerfile to use pip instead of uv --- ee/api/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/api/Dockerfile b/ee/api/Dockerfile index a90b6a5ba..35e7149f5 100644 --- a/ee/api/Dockerfile +++ b/ee/api/Dockerfile @@ -17,7 +17,7 @@ COPY requirements.txt ./requirements.txt #RUN uv pip install --no-cache-dir --upgrade pip setuptools wheel --system #RUN uv pip install --no-cache-dir --upgrade python3-saml==1.16.0 --no-binary=lxml --system #RUN uv pip install --no-cache-dir --upgrade -r requirements.txt --system -RUN pip install --no-cache-dir --upgrade -r requirements.txt --system +RUN pip install --no-cache-dir --upgrade -r requirements.txt COPY . . RUN mv env.default .env From a9bbf31f736e144ef2458f4cac15ee2f9eadb158 Mon Sep 17 00:00:00 2001 From: Kraiem Taha Yassine Date: Thu, 9 Jan 2025 13:19:10 +0100 Subject: [PATCH 06/11] Dev (#2931) * fix(chalice): fixed errors package --- ee/api/chalicelib/core/errors/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ee/api/chalicelib/core/errors/__init__.py b/ee/api/chalicelib/core/errors/__init__.py index dfa66e2bd..f14b004b0 100644 --- a/ee/api/chalicelib/core/errors/__init__.py +++ b/ee/api/chalicelib/core/errors/__init__.py @@ -11,5 +11,3 @@ if config("EXP_ERRORS_SEARCH", cast=bool, default=False): from . import errors_details_exp as errors_details else: from . import errors - -from . import errors_viewed_ee as errors_viewed From b5bf70a8e060dcaf5ebc021883ec20417de6f3d0 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Wed, 8 Jan 2025 18:01:35 +0100 Subject: [PATCH 07/11] change(ui): projects revamp - install docs change --- .../Client/Projects/ProjectCodeSnippet.tsx | 175 ++++++++++++++++++ .../Client/Projects/ProjectTabTracking.tsx | 98 +++++++--- .../Projects/projectCodeSnippet.module.css | 101 ++++++++++ 3 files changed, 344 insertions(+), 30 deletions(-) create mode 100644 frontend/app/components/Client/Projects/ProjectCodeSnippet.tsx create mode 100644 frontend/app/components/Client/Projects/projectCodeSnippet.module.css diff --git a/frontend/app/components/Client/Projects/ProjectCodeSnippet.tsx b/frontend/app/components/Client/Projects/ProjectCodeSnippet.tsx new file mode 100644 index 000000000..e48764f39 --- /dev/null +++ b/frontend/app/components/Client/Projects/ProjectCodeSnippet.tsx @@ -0,0 +1,175 @@ +import React, { useEffect, useState } from 'react'; +import { useStore } from 'App/mstore'; +import { observer } from 'mobx-react-lite'; +import { Checkbox, Loader, Toggler } from 'UI'; +import GDPR from 'App/mstore/types/gdpr'; +import cn from 'classnames'; +import stl from './projectCodeSnippet.module.css'; +import Select from 'Shared/Select'; +import CodeSnippet from 'Shared/CodeSnippet'; +import CircleNumber from 'Components/Onboarding/components/CircleNumber'; +import Project from '@/mstore/types/project'; + +interface InputModeOption { + label: string; + value: string; +} + +const inputModeOptions: InputModeOption[] = [ + { label: 'Record all inputs', value: 'plain' }, + { label: 'Ignore all inputs', value: 'obscured' }, + { label: 'Obscure all inputs', value: 'hidden' } +]; + +const inputModeOptionsMap: Record = {}; +inputModeOptions.forEach((o, i) => (inputModeOptionsMap[o.value] = i)); + +interface Props { + project: Project; +} + +const ProjectCodeSnippet: React.FC = (props: Props) => { + const { projectsStore } = useStore(); + const siteId = projectsStore.siteId; + const site = props.project; + const gdpr = site.gdpr as GDPR; + const sites = projectsStore.list; + const editGDPR = projectsStore.editGDPR; + const onSaveGDPR = projectsStore.saveGDPR; + const init = projectsStore.initProject; + const [changed, setChanged] = useState(false); + const [isAssistEnabled, setAssistEnabled] = useState(false); + const [showLoader, setShowLoader] = useState(false); + + useEffect(() => { + const currentSite = sites.find((s) => s.id === siteId); + if (currentSite) { + init(currentSite); + } + }, [init, siteId, sites]); + + const saveGDPR = () => { + setChanged(true); + void onSaveGDPR(site.id); + }; + + const onChangeSelect = (data: { name: string; value: string }) => { + editGDPR({ [data.name]: data.value }); + saveGDPR(); + }; + + const onChangeOption = (event: React.ChangeEvent) => { + const { name, checked } = event.target; + editGDPR({ [name]: checked }); + saveGDPR(); + }; + + useEffect(() => { + setShowLoader(true); + const timer = setTimeout(() => { + setShowLoader(false); + }, 200); + return () => clearTimeout(timer); + }, [isAssistEnabled]); + + return ( +
+
+
+ Choose data recording options +
+ +
+