jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Dev (#2286)

Browse source 
* refactor(chalice): upgraded dependencies

* refactor(chalice): upgraded dependencies
feat(chalice): support heatmaps
This commit is contained in:
Kraiem Taha Yassine 2024-06-19 18:49:07 +02:00 • committed by GitHub
parent 60af1214f6
commit fca98c8c85
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 418 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
api/Pipfile
View file

@ -5,19 +5,19 @@ name = "pypi"
[packages]
urllib3 = "==1.26.16"
requests = "==2.32.2"
boto3 = "==1.34.122"
requests = "==2.32.3"
boto3 = "==1.34.125"
pyjwt = "==2.8.0"
psycopg2-binary = "==2.9.9"
psycopg = {extras = ["binary", "pool"], version = "==3.1.19"}
elasticsearch = "==8.14.0"
jira = "==3.8.0"
fastapi = "==0.111.0"
python-decouple = "==3.8"
apscheduler = "==3.10.4"
redis = "==5.0.4"
psycopg = {extras = ["pool", "binary"], version = "==3.1.19"}
uvicorn = {extras = ["standard"], version = "==0.30.1"}
python-decouple = "==3.8"
pydantic = {extras = ["email"], version = "==2.3.0"}
apscheduler = "==3.10.4"
redis = "==5.1.0b6"
[dev-packages]

1
api/chalicelib/core/__init__.py
View file

@ -1 +0,0 @@
from . import sessions as sessions_legacy

80
api/chalicelib/core/click_maps.py
View file

@ -1,80 +0,0 @@
import logging
import schemas
from chalicelib.core import sessions_mobs, sessions_legacy as sessions_search, events
from chalicelib.utils import pg_client, helper
logger = logging.getLogger(__name__)
SESSION_PROJECTION_COLS = """s.project_id,
s.session_id::text AS session_id,
s.user_uuid,
s.user_id,
s.user_os,
s.user_browser,
s.user_device,
s.user_device_type,
s.user_country,
s.start_ts,
s.duration,
s.events_count,
s.pages_count,
s.errors_count,
s.user_anonymous_id,
s.platform,
s.issue_score,
to_jsonb(s.issue_types) AS issue_types,
favorite_sessions.session_id NOTNULL AS favorite,
COALESCE((SELECT TRUE
FROM public.user_viewed_sessions AS fs
WHERE s.session_id = fs.session_id
AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """
def search_short_session(data: schemas.ClickMapSessionsSearch, project_id, user_id, include_mobs: bool = True):
no_platform = True
for f in data.filters:
if f.type == schemas.FilterType.platform:
no_platform = False
break
if no_platform:
data.filters.append(schemas.SessionSearchFilterSchema(type=schemas.FilterType.platform,
value=[schemas.PlatformType.desktop],
operator=schemas.SearchEventOperator._is))
full_args, query_part = sessions_search.search_query_parts(data=data, error_status=None, errors_only=False,
favorite_only=data.bookmarked, issue=None,
project_id=project_id, user_id=user_id)
with pg_client.PostgresClient() as cur:
data.order = schemas.SortOrderType.desc
data.sort = 'duration'
# meta_keys = metadata.get(project_id=project_id)
meta_keys = []
main_query = cur.mogrify(f"""SELECT {SESSION_PROJECTION_COLS}
{"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])}
{query_part}
ORDER BY {data.sort} {data.order.value}
LIMIT 1;""", full_args)
logger.debug("--------------------")
logger.debug(main_query)
logger.debug("--------------------")
try:
cur.execute(main_query)
except Exception as err:
logger.warning("--------- CLICK MAP SHORT SESSION SEARCH QUERY EXCEPTION -----------")
logger.warning(main_query.decode('UTF-8'))
logger.warning("--------- PAYLOAD -----------")
logger.warning(data.model_dump_json())
logger.warning("--------------------")
raise err
session = cur.fetchone()
if session:
if include_mobs:
session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id)
session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"])
session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"],
event_type=schemas.EventType.location)
return helper.dict_to_camel_case(session)

5
api/chalicelib/core/custom_metrics.py
View file

@ -5,7 +5,7 @@ from decouple import config
from fastapi import HTTPException, status
import schemas
from chalicelib.core import sessions, funnels, errors, issues, click_maps, sessions_mobs, product_analytics, \
from chalicelib.core import sessions, funnels, errors, issues, heatmaps, sessions_mobs, product_analytics, \
custom_metrics_predefined
from chalicelib.utils import helper, pg_client
from chalicelib.utils.TimeUTC import TimeUTC
@ -90,7 +90,7 @@ def __get_click_map_chart(project_id, user_id, data: schemas.CardClickMap, inclu
return None
data.series[0].filter.filters += data.series[0].filter.events
data.series[0].filter.events = []
return click_maps.search_short_session(project_id=project_id, user_id=user_id,
return heatmaps.search_short_session(project_id=project_id, user_id=user_id,
data=schemas.ClickMapSessionsSearch(
**data.series[0].filter.model_dump()),
include_mobs=include_mobs)
@ -178,6 +178,7 @@ def get_chart(project_id: int, data: schemas.CardSchema, user_id: int):
schemas.MetricType.timeseries: __get_timeseries_chart,
schemas.MetricType.table: __get_table_chart,
schemas.MetricType.click_map: __get_click_map_chart,
schemas.MetricType.heat_map: __get_click_map_chart,
schemas.MetricType.funnel: __get_funnel_chart,
schemas.MetricType.insights: not_supported,
schemas.MetricType.pathAnalysis: __get_path_analysis_chart

148
api/chalicelib/core/heatmaps.py
View file

@ -1,8 +1,10 @@
import logging
import schemas
from chalicelib.utils import helper, pg_client
from chalicelib.utils import sql_helper as sh
from chalicelib.core import sessions_mobs, sessions, events
from chalicelib.utils import pg_client, helper
# from chalicelib.utils import sql_helper as sh
logger = logging.getLogger(__name__)
@ -19,42 +21,41 @@ def get_by_url(project_id, data: schemas.GetHeatmapPayloadSchema):
"duration IS NOT NULL",
"normalized_x IS NOT NULL"]
query_from = "events.clicks INNER JOIN sessions USING (session_id)"
q_count = "count(1) AS count"
has_click_rage_filter = False
if len(data.filters) > 0:
for i, f in enumerate(data.filters):
if f.type == schemas.FilterType.issue and len(f.value) > 0:
has_click_rage_filter = True
q_count = "max(real_count) AS count,TRUE AS click_rage"
query_from += """INNER JOIN events_common.issues USING (timestamp, session_id)
INNER JOIN issues AS mis USING (issue_id)
INNER JOIN LATERAL (
SELECT COUNT(1) AS real_count
FROM events.clicks AS sc
INNER JOIN sessions as ss USING (session_id)
WHERE ss.project_id = 2
AND (sc.url = %(url)s OR sc.path = %(url)s)
AND sc.timestamp >= %(startDate)s
AND sc.timestamp <= %(endDate)s
AND ss.start_ts >= %(startDate)s
AND ss.start_ts <= %(endDate)s
AND sc.selector = clicks.selector) AS r_clicks ON (TRUE)"""
constraints += ["mis.project_id = %(project_id)s",
"issues.timestamp >= %(startDate)s",
"issues.timestamp <= %(endDate)s"]
f_k = f"issue_value{i}"
args = {**args, **sh.multi_values(f.value, value_key=f_k)}
constraints.append(sh.multi_conditions(f"%({f_k})s = ANY (issue_types)",
f.value, value_key=f_k))
constraints.append(sh.multi_conditions(f"mis.type = %({f_k})s",
f.value, value_key=f_k))
# TODO: is this used ?
# if len(data.filters) > 0:
# for i, f in enumerate(data.filters):
# if f.type == schemas.FilterType.issue and len(f.value) > 0:
# has_click_rage_filter = True
# query_from += """INNER JOIN events_common.issues USING (timestamp, session_id)
# INNER JOIN issues AS mis USING (issue_id)
# INNER JOIN LATERAL (
# SELECT COUNT(1) AS real_count
# FROM events.clicks AS sc
# INNER JOIN sessions as ss USING (session_id)
# WHERE ss.project_id = 2
# AND (sc.url = %(url)s OR sc.path = %(url)s)
# AND sc.timestamp >= %(startDate)s
# AND sc.timestamp <= %(endDate)s
# AND ss.start_ts >= %(startDate)s
# AND ss.start_ts <= %(endDate)s
# AND sc.selector = clicks.selector) AS r_clicks ON (TRUE)"""
# constraints += ["mis.project_id = %(project_id)s",
# "issues.timestamp >= %(startDate)s",
# "issues.timestamp <= %(endDate)s"]
# f_k = f"issue_value{i}"
# args = {**args, **sh.multi_values(f.value, value_key=f_k)}
# constraints.append(sh.multi_conditions(f"%({f_k})s = ANY (issue_types)",
# f.value, value_key=f_k))
# constraints.append(sh.multi_conditions(f"mis.type = %({f_k})s",
# f.value, value_key=f_k))
if data.click_rage and not has_click_rage_filter:
constraints.append("""(issues.session_id IS NULL
OR (issues.timestamp >= %(startDate)s
AND issues.timestamp <= %(endDate)s
AND mis.project_id = %(project_id)s))""")
q_count += ",COALESCE(bool_or(mis.type = 'click_rage'), FALSE) AS click_rage"
AND mis.project_id = %(project_id)s
AND mis.type='click_rage'))""")
query_from += """LEFT JOIN events_common.issues USING (timestamp, session_id)
LEFT JOIN issues AS mis USING (issue_id)"""
with pg_client.PostgresClient() as cur:
@ -77,3 +78,86 @@ def get_by_url(project_id, data: schemas.GetHeatmapPayloadSchema):
rows = cur.fetchall()
return helper.list_to_camel_case(rows)
SESSION_PROJECTION_COLS = """s.project_id,
s.session_id::text AS session_id,
s.user_uuid,
s.user_id,
s.user_os,
s.user_browser,
s.user_device,
s.user_device_type,
s.user_country,
s.start_ts,
s.duration,
s.events_count,
s.pages_count,
s.errors_count,
s.user_anonymous_id,
s.platform,
s.issue_score,
to_jsonb(s.issue_types) AS issue_types,
favorite_sessions.session_id NOTNULL AS favorite,
COALESCE((SELECT TRUE
FROM public.user_viewed_sessions AS fs
WHERE s.session_id = fs.session_id
AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """
def search_short_session(data: schemas.ClickMapSessionsSearch, project_id, user_id,
include_mobs: bool = True, exclude_sessions: list[str] = [],
_depth: int = 3):
no_platform = True
for f in data.filters:
if f.type == schemas.FilterType.platform:
no_platform = False
break
if no_platform:
data.filters.append(schemas.SessionSearchFilterSchema(type=schemas.FilterType.platform,
value=[schemas.PlatformType.desktop],
operator=schemas.SearchEventOperator._is))
full_args, query_part = sessions.search_query_parts(data=data, error_status=None, errors_only=False,
favorite_only=data.bookmarked, issue=None,
project_id=project_id, user_id=user_id)
full_args["exclude_sessions"] = tuple(exclude_sessions)
if len(exclude_sessions) > 0:
query_part += "\n AND session_id NOT IN (%(exclude_sessions)s)"
with pg_client.PostgresClient() as cur:
data.order = schemas.SortOrderType.desc
data.sort = 'duration'
main_query = cur.mogrify(f"""SELECT {SESSION_PROJECTION_COLS}
{query_part}
ORDER BY {data.sort} {data.order.value}
LIMIT 1;""", full_args)
logger.debug("--------------------")
logger.debug(main_query)
logger.debug("--------------------")
try:
cur.execute(main_query)
except Exception as err:
logger.warning("--------- CLICK MAP SHORT SESSION SEARCH QUERY EXCEPTION -----------")
logger.warning(main_query.decode('UTF-8'))
logger.warning("--------- PAYLOAD -----------")
logger.warning(data.model_dump_json())
logger.warning("--------------------")
raise err
session = cur.fetchone()
if session:
if include_mobs:
session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id)
session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"])
if _depth > 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0:
return search_short_session(data=data, project_id=project_id, user_id=user_id,
include_mobs=include_mobs,
exclude_sessions=exclude_sessions + [session["session_id"]],
_depth=_depth - 1)
elif _depth == 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0:
logger.info("couldn't find an existing replay after 3 iterations for heatmap")
session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"],
event_type=schemas.EventType.location)
return helper.dict_to_camel_case(session)

4
api/requirements-alerts.txt
View file

@ -1,7 +1,7 @@
# Keep this version to not have conflicts between requests and boto3
urllib3==1.26.16
requests==2.32.2
boto3==1.34.122
requests==2.32.3
boto3==1.34.125
pyjwt==2.8.0
psycopg2-binary==2.9.9
psycopg[pool,binary]==3.1.19

6
api/requirements.txt
View file

@ -1,7 +1,7 @@
# Keep this version to not have conflicts between requests and boto3
urllib3==1.26.16
requests==2.32.2
boto3==1.34.122
requests==2.32.3
boto3==1.34.125
pyjwt==2.8.0
psycopg2-binary==2.9.9
psycopg[pool,binary]==3.1.19
@ -16,4 +16,4 @@ python-decouple==3.8
pydantic[email]==2.3.0
apscheduler==3.10.4
redis==5.0.4
redis==5.1.0b6

2
api/routers/core_dynamic.py
View file

@ -7,7 +7,7 @@ from starlette.responses import RedirectResponse, FileResponse, JSONResponse, Re
import schemas
from chalicelib.core import sessions, errors, errors_viewed, errors_favorite, sessions_assignments, heatmaps, \
sessions_favorite, assist, sessions_notes, click_maps, sessions_replay, signup, feature_flags
sessions_favorite, assist, sessions_notes, sessions_replay, signup, feature_flags
from chalicelib.core import sessions_viewed
from chalicelib.core import tenants, users, projects, license
from chalicelib.core import webhook

2
api/schemas/schemas.py
View file

@ -934,6 +934,8 @@ class MetricType(str, Enum):
retention = "retention"
stickiness = "stickiness"
click_map = "clickMap"
# click_map and heat_map are the same
heat_map = "heatMap"
insights = "insights"

2
ee/api/.gitignore vendored
View file

@ -190,7 +190,6 @@ Pipfile.lock
/chalicelib/core/authorizers.py
/chalicelib/core/autocomplete.py
/chalicelib/core/canvas.py
/chalicelib/core/click_maps.py
/chalicelib/core/collaboration_base.py
/chalicelib/core/collaboration_msteams.py
/chalicelib/core/collaboration_slack.py
@ -201,7 +200,6 @@ Pipfile.lock
/chalicelib/core/events_mobile.py
/chalicelib/core/feature_flags.py
/chalicelib/core/funnels.py
/chalicelib/core/heatmaps.py
/chalicelib/core/integration_base.py
/chalicelib/core/integration_base_issue.py
/chalicelib/core/integration_github.py

14
ee/api/Pipfile
View file

@ -5,23 +5,23 @@ name = "pypi"
[packages]
urllib3 = "==1.26.16"
requests = "==2.32.2"
boto3 = "==1.34.113"
requests = "==2.32.3"
boto3 = "==1.34.125"
pyjwt = "==2.8.0"
psycopg2-binary = "==2.9.9"
psycopg = {extras = ["binary", "pool"], version = "==3.1.19"}
elasticsearch = "==8.13.2"
elasticsearch = "==8.14.0"
jira = "==3.8.0"
fastapi = "==0.111.0"
uvicorn = {extras = ["standard"], version = "==0.29.0"}
uvicorn = {extras = ["standard"], version = "==0.30.1"}
gunicorn = "==22.0.0"
python-decouple = "==3.8"
pydantic = {extras = ["email"], version = "==2.3.0"}
apscheduler = "==3.10.4"
clickhouse-driver = {extras = ["lz4"], version = "==0.2.7"}
clickhouse-driver = {extras = ["lz4"], version = "==0.2.8"}
python3-saml = "==1.16.0"
redis = "==5.0.4"
azure-storage-blob = "==12.20.0"
redis = "==5.1.0b6"
azure-storage-blob = "==12.21.0b1"
[dev-packages]

11
ee/api/chalicelib/core/custom_metrics.py
View file

@ -5,7 +5,7 @@ from decouple import config
from fastapi import HTTPException, status
import schemas
from chalicelib.core import funnels, issues, click_maps, sessions_insights, sessions_mobs, sessions_favorite, \
from chalicelib.core import funnels, issues, heatmaps, sessions_insights, sessions_mobs, sessions_favorite, \
product_analytics, custom_metrics_predefined
from chalicelib.utils import helper, pg_client
from chalicelib.utils.TimeUTC import TimeUTC
@ -101,10 +101,10 @@ def __get_click_map_chart(project_id, user_id, data: schemas.CardClickMap, inclu
return None
data.series[0].filter.filters += data.series[0].filter.events
data.series[0].filter.events = []
return click_maps.search_short_session(project_id=project_id, user_id=user_id,
data=schemas.ClickMapSessionsSearch(
**data.series[0].filter.model_dump()),
include_mobs=include_mobs)
return heatmaps.search_short_session(project_id=project_id, user_id=user_id,
data=schemas.ClickMapSessionsSearch(
**data.series[0].filter.model_dump()),
include_mobs=include_mobs)
# EE only
@ -198,6 +198,7 @@ def get_chart(project_id: int, data: schemas.CardSchema, user_id: int):
schemas.MetricType.timeseries: __get_timeseries_chart,
schemas.MetricType.table: __get_table_chart,
schemas.MetricType.click_map: __get_click_map_chart,
schemas.MetricType.heat_map: __get_click_map_chart,
schemas.MetricType.funnel: __get_funnel_chart,
schemas.MetricType.insights: __get_insights_chart,
schemas.MetricType.pathAnalysis: __get_path_analysis_chart

256
ee/api/chalicelib/core/heatmaps.py Normal file
View file

@ -0,0 +1,256 @@
import logging
from decouple import config
import schemas
from chalicelib.core import sessions_mobs, events
# from chalicelib.utils import sql_helper as sh
if config("EXP_SESSIONS_SEARCH", cast=bool, default=False):
from chalicelib.core import sessions_exp as sessions
else:
from chalicelib.core import sessions
from chalicelib.utils import pg_client, helper, ch_client, exp_ch_helper
logger = logging.getLogger(__name__)
def get_by_url(project_id, data: schemas.GetHeatmapPayloadSchema):
args = {"startDate": data.startTimestamp, "endDate": data.endTimestamp,
"project_id": project_id, "url": data.url}
constraints = ["main_events.project_id = toUInt16(%(project_id)s)",
"(main_events.url_hostpath = %(url)s OR main_events.url_path = %(url)s)",
"main_events.datetime >= toDateTime(%(startDate)s/1000)",
"main_events.datetime <= toDateTime(%(endDate)s/1000)",
"main_events.event_type='CLICK'",
"isNotNull(main_events.normalized_x)"]
query_from = f"{exp_ch_helper.get_main_events_table(data.startTimestamp)} AS main_events"
has_click_rage_filter = False
# TODO: is this used ?
# if len(data.filters) > 0:
# for i, f in enumerate(data.filters):
# if f.type == schemas.FilterType.issue and len(f.value) > 0:
# has_click_rage_filter = True
# query_from += """INNER JOIN events_common.issues USING (timestamp, session_id)
# INNER JOIN issues AS mis USING (issue_id)
# INNER JOIN LATERAL (
# SELECT COUNT(1) AS real_count
# FROM events.clicks AS sc
# INNER JOIN sessions as ss USING (session_id)
# WHERE ss.project_id = 2
# AND (sc.url = %(url)s OR sc.path = %(url)s)
# AND sc.timestamp >= %(startDate)s
# AND sc.timestamp <= %(endDate)s
# AND ss.start_ts >= %(startDate)s
# AND ss.start_ts <= %(endDate)s
# AND sc.selector = clicks.selector) AS r_clicks ON (TRUE)"""
# constraints += ["mis.project_id = %(project_id)s",
# "issues.timestamp >= %(startDate)s",
# "issues.timestamp <= %(endDate)s"]
# f_k = f"issue_value{i}"
# args = {**args, **sh.multi_values(f.value, value_key=f_k)}
# constraints.append(sh.multi_conditions(f"%({f_k})s = ANY (issue_types)",
# f.value, value_key=f_k))
# constraints.append(sh.multi_conditions(f"mis.type = %({f_k})s",
# f.value, value_key=f_k))
if data.click_rage and not has_click_rage_filter:
constraints.append("""(issues.session_id IS NULL
OR (issues.datetime >= toDateTime(%(startDate)s/1000)
AND issues.datetime <= toDateTime(%(endDate)s/1000)
AND issues.project_id = toUInt16(%(project_id)s)
AND issues.event_type = 'ISSUE'
AND issues.project_id = toUInt16(%(project_id)s
AND mis.project_id = toUInt16(%(project_id)s
AND mis.type='click_rage'))))""")
query_from += """ LEFT JOIN experimental.events AS issues ON (main_events.session_id=issues.session_id)
LEFT JOIN experimental.issues AS mis ON (issues.issue_id=mis.issue_id)"""
with ch_client.ClickHouseClient() as cur:
query = cur.format(f"""SELECT main_events.normalized_x AS normalized_x,
main_events.normalized_y AS normalized_y
FROM {query_from}
WHERE {" AND ".join(constraints)}
LIMIT 500;""", args)
logger.debug("---------")
logger.debug(query)
logger.debug("---------")
try:
rows = cur.execute(query)
except Exception as err:
logger.warning("--------- HEATMAP 2 SEARCH QUERY EXCEPTION CH -----------")
logger.warning(query)
logger.warning("--------- PAYLOAD -----------")
logger.warning(data)
logger.warning("--------------------")
raise err
return helper.list_to_camel_case(rows)
if not config("EXP_SESSIONS_SEARCH", cast=bool, default=False):
# this part is identical to FOSS
SESSION_PROJECTION_COLS = """s.project_id,
s.session_id::text AS session_id,
s.user_uuid,
s.user_id,
s.user_os,
s.user_browser,
s.user_device,
s.user_device_type,
s.user_country,
s.start_ts,
s.duration,
s.events_count,
s.pages_count,
s.errors_count,
s.user_anonymous_id,
s.platform,
s.issue_score,
to_jsonb(s.issue_types) AS issue_types,
favorite_sessions.session_id NOTNULL AS favorite,
COALESCE((SELECT TRUE
FROM public.user_viewed_sessions AS fs
WHERE s.session_id = fs.session_id
AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """
def search_short_session(data: schemas.ClickMapSessionsSearch, project_id, user_id,
include_mobs: bool = True, exclude_sessions: list[str] = [],
_depth: int = 3):
no_platform = True
for f in data.filters:
if f.type == schemas.FilterType.platform:
no_platform = False
break
if no_platform:
data.filters.append(schemas.SessionSearchFilterSchema(type=schemas.FilterType.platform,
value=[schemas.PlatformType.desktop],
operator=schemas.SearchEventOperator._is))
full_args, query_part = sessions.search_query_parts(data=data, error_status=None, errors_only=False,
favorite_only=data.bookmarked, issue=None,
project_id=project_id, user_id=user_id)
full_args["exclude_sessions"] = tuple(exclude_sessions)
if len(exclude_sessions) > 0:
query_part += "\n AND session_id NOT IN (%(exclude_sessions)s)"
with pg_client.PostgresClient() as cur:
data.order = schemas.SortOrderType.desc
data.sort = 'duration'
main_query = cur.mogrify(f"""SELECT {SESSION_PROJECTION_COLS}
{query_part}
ORDER BY {data.sort} {data.order.value}
LIMIT 1;""", full_args)
logger.debug("--------------------")
logger.debug(main_query)
logger.debug("--------------------")
try:
cur.execute(main_query)
except Exception as err:
logger.warning("--------- CLICK MAP SHORT SESSION SEARCH QUERY EXCEPTION -----------")
logger.warning(main_query.decode('UTF-8'))
logger.warning("--------- PAYLOAD -----------")
logger.warning(data.model_dump_json())
logger.warning("--------------------")
raise err
session = cur.fetchone()
if session:
if include_mobs:
session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id)
session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"])
if _depth > 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0:
return search_short_session(data=data, project_id=project_id, user_id=user_id,
include_mobs=include_mobs,
exclude_sessions=exclude_sessions + [session["session_id"]],
_depth=_depth - 1)
elif _depth == 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0:
logger.info("couldn't find an existing replay after 3 iterations for heatmap")
session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"],
event_type=schemas.EventType.location)
return helper.dict_to_camel_case(session)
else:
# use CH
SESSION_PROJECTION_COLS = """
s.project_id,
s.session_id AS session_id,
s.user_uuid AS user_uuid,
s.user_id AS user_id,
s.user_os AS user_os,
s.user_browser AS user_browser,
s.user_device AS user_device,
s.user_device_type AS user_device_type,
s.user_country AS user_country,
s.user_city AS user_city,
s.user_state AS user_state,
toUnixTimestamp(s.datetime)*1000 AS start_ts,
s.duration AS duration,
s.events_count AS events_count,
s.pages_count AS pages_count,
s.errors_count AS errors_count,
s.user_anonymous_id AS user_anonymous_id,
s.platform AS platform,
s.timezone AS timezone,
coalesce(issue_score,0) AS issue_score,
s.issue_types AS issue_types """
def search_short_session(data: schemas.ClickMapSessionsSearch, project_id, user_id,
include_mobs: bool = True, exclude_sessions: list[str] = [],
_depth: int = 3):
no_platform = True
for f in data.filters:
if f.type == schemas.FilterType.platform:
no_platform = False
break
if no_platform:
data.filters.append(schemas.SessionSearchFilterSchema(type=schemas.FilterType.platform,
value=[schemas.PlatformType.desktop],
operator=schemas.SearchEventOperator._is))
full_args, query_part = sessions.search_query_parts_ch(data=data, error_status=None, errors_only=False,
favorite_only=data.bookmarked, issue=None,
project_id=project_id, user_id=user_id)
full_args["exclude_sessions"] = tuple(exclude_sessions)
if len(exclude_sessions) > 0:
query_part += "\n AND session_id NOT IN (%(exclude_sessions)s)"
with ch_client.ClickHouseClient() as cur:
data.order = schemas.SortOrderType.desc
data.sort = 'duration'
main_query = cur.format(f"""SELECT {SESSION_PROJECTION_COLS}
{query_part}
ORDER BY {data.sort} {data.order.value}
LIMIT 1;""", full_args)
logger.debug("--------------------")
logger.debug(main_query)
logger.debug("--------------------")
try:
session = cur.execute(main_query)
except Exception as err:
logger.warning("--------- CLICK MAP SHORT SESSION SEARCH QUERY EXCEPTION CH -----------")
logger.warning(main_query)
logger.warning("--------- PAYLOAD -----------")
logger.warning(data.model_dump_json())
logger.warning("--------------------")
raise err
if session:
if include_mobs:
session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id)
session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"])
if _depth > 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0:
return search_short_session(data=data, project_id=project_id, user_id=user_id,
include_mobs=include_mobs,
exclude_sessions=exclude_sessions + [session["session_id"]],
_depth=_depth - 1)
elif _depth == 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0:
logger.info("couldn't find an existing replay after 3 iterations for heatmap")
session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"],
event_type=schemas.EventType.location)
return helper.dict_to_camel_case(session)

2
ee/api/clean-dev.sh
View file

@ -11,7 +11,6 @@ rm -rf ./chalicelib/core/announcements.py
rm -rf ./chalicelib/core/assist.py
rm -rf ./chalicelib/core/authorizers.py
rm -rf ./chalicelib/core/autocomplete.py
rm -rf ./chalicelib/core/click_maps.py
rm -rf ./chalicelib/core/collaboration_base.py
rm -rf ./chalicelib/core/collaboration_msteams.py
rm -rf ./chalicelib/core/collaboration_slack.py
@ -22,7 +21,6 @@ rm -rf ./chalicelib/core/errors_favorite.py
rm -rf ./chalicelib/core/events_mobile.py
rm -rf ./chalicelib/core/feature_flags.py
rm -rf ./chalicelib/core/funnels.py
rm -rf ./chalicelib/core/heatmaps.py
rm -rf ./chalicelib/core/integration_base.py
rm -rf ./chalicelib/core/integration_base_issue.py
rm -rf ./chalicelib/core/integration_github.py

8
ee/api/requirements-alerts.txt
View file

@ -1,7 +1,7 @@
# Keep this version to not have conflicts between requests and boto3
urllib3==1.26.16
requests==2.32.2
boto3==1.34.122
requests==2.32.3
boto3==1.34.125
pyjwt==2.8.0
psycopg2-binary==2.9.9
psycopg[pool,binary]==3.1.19
@ -16,5 +16,5 @@ python-decouple==3.8
pydantic[email]==2.3.0
apscheduler==3.10.4
clickhouse-driver[lz4]==0.2.7
azure-storage-blob==12.19.1
clickhouse-driver[lz4]==0.2.8
azure-storage-blob==12.21.0b1

10
ee/api/requirements-crons.txt
View file

@ -1,7 +1,7 @@
# Keep this version to not have conflicts between requests and boto3
urllib3==1.26.16
requests==2.32.2
boto3==1.34.122
requests==2.32.3
boto3==1.34.125
pyjwt==2.8.0
psycopg2-binary==2.9.9
psycopg[pool,binary]==3.1.19
@ -15,6 +15,6 @@ python-decouple==3.8
pydantic[email]==2.3.0
apscheduler==3.10.4
clickhouse-driver[lz4]==0.2.7
redis==5.0.4
azure-storage-blob==12.20.0
clickhouse-driver[lz4]==0.2.8
redis==5.1.0b6
azure-storage-blob==12.21.0b1

10
ee/api/requirements.txt
View file

@ -1,7 +1,7 @@
# Keep this version to not have conflicts between requests and boto3
urllib3==1.26.16
requests==2.32.2
boto3==1.34.122
requests==2.32.3
boto3==1.34.125
pyjwt==2.8.0
psycopg2-binary==2.9.9
psycopg[pool,binary]==3.1.19
@ -17,11 +17,11 @@ python-decouple==3.8
pydantic[email]==2.3.0
apscheduler==3.10.4
clickhouse-driver[lz4]==0.2.7
clickhouse-driver[lz4]==0.2.8
# TODO: enable after xmlsec fix https://github.com/xmlsec/python-xmlsec/issues/252
#--no-binary is used to avoid libxml2 library version incompatibilities between xmlsec and lxml
python3-saml==1.16.0 --no-binary=lxml
redis==5.0.4
redis==5.1.0b6
#confluent-kafka==2.1.0
azure-storage-blob==12.20.0
azure-storage-blob==12.21.0b1

4
ee/api/routers/core_dynamic.py
View file

@ -7,7 +7,7 @@ from starlette.responses import RedirectResponse, FileResponse, JSONResponse, Re
import schemas
from chalicelib.core import sessions, assist, heatmaps, sessions_favorite, sessions_assignments, errors, errors_viewed, \
errors_favorite, sessions_notes, click_maps, sessions_replay, signup, feature_flags
errors_favorite, sessions_notes, sessions_replay, signup, feature_flags
from chalicelib.core import sessions_viewed
from chalicelib.core import tenants, users, projects, license
from chalicelib.core import webhook
@ -569,7 +569,7 @@ def get_all_notes(projectId: int, data: schemas.SearchNoteSchema = Body(...),
@app.post('/{projectId}/click_maps/search', tags=["click maps"], dependencies=[OR_scope(Permissions.session_replay)])
def click_map_search(projectId: int, data: schemas.ClickMapSessionsSearch = Body(...),
context: schemas.CurrentContext = Depends(OR_context)):
return {"data": click_maps.search_short_session(user_id=context.user_id, data=data, project_id=projectId)}
return {"data": heatmaps.search_short_session(user_id=context.user_id, data=data, project_id=projectId)}
@app.post('/{project_id}/feature-flags/search', tags=["feature flags"],