jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(chalice): search sessions by click-rage-selector

Browse source 
feat(chalice): support empty cards
feat(DB): remove unused indexes
This commit is contained in:
Taha Yassine Kraiem 2022-12-28 19:45:47 +01:00
parent ed0065b541
commit 58de043193
14 changed files with 395 additions and 430 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
api/chalicelib/core/custom_metrics.py
View file

@ -125,7 +125,8 @@ def merged_live(project_id, data: schemas.CreateCardSchema, user_id=None):
return __get_sessions_list(project_id=project_id, user_id=user_id, data=data)
elif __is_click_map(data):
return __get_click_map_chat(project_id=project_id, user_id=user_id, data=data)
elif len(data.series) == 0:
return []
series_charts = __try_live(project_id=project_id, data=data)
if data.view_type == schemas.MetricTimeseriesViewType.progress or data.metric_type == schemas.MetricType.table:
return series_charts
@ -376,7 +377,9 @@ def search_all(project_id, user_id, data: schemas.SearchCardsSchema, include_ser
AND metric_series.deleted_at ISNULL
) AS metric_series ON (TRUE)"""
query = cur.mogrify(
f"""SELECT *
f"""SELECT metric_id, project_id, user_id, name, is_public, created_at, edited_at,
metric_type, metric_of, metric_format, metric_value, view_type, is_pinned,
predefined_key, dashboards, owner_email, default_config AS config, thumbnail
FROM metrics
{sub_join}
LEFT JOIN LATERAL (SELECT COALESCE(jsonb_agg(connected_dashboards.* ORDER BY is_public,name),'[]'::jsonb) AS dashboards

2
api/chalicelib/core/events.py
View file

@ -42,7 +42,7 @@ def __get_grouped_clickrage(rows, session_id, project_id):
for c in click_rage_issues:
merge_count = c.get("payload")
if merge_count is not None:
merge_count = merge_count.get("count", 3)
merge_count = merge_count.get("Count", 3)
else:
merge_count = 3
for i in range(len(rows)):

3
api/chalicelib/core/funnels.py
View file

@ -2,6 +2,7 @@ import json
from typing import List
import chalicelib.utils.helper
import chalicelib.utils.sql_helper
import schemas
from chalicelib.core import significance, sessions
from chalicelib.utils import dev
@ -39,7 +40,7 @@ def __fix_stages(f_events: List[schemas._SessionSearchEventSchema]):
if not isinstance(e.value, list):
e.value = [e.value]
is_any = sessions._isAny_opreator(e.operator)
is_any = chalicelib.utils.sessions_helper._isAny_opreator(e.operator)
if not is_any and isinstance(e.value, list) and len(e.value) == 0:
continue
events.append(e)

10
api/chalicelib/core/metadata.py
View file

@ -6,7 +6,7 @@ from chalicelib.utils import pg_client
MAX_INDEXES = 10
def _get_column_names():
def column_names():
return [f"metadata_{i}" for i in range(1, MAX_INDEXES + 1)]
@ -16,7 +16,7 @@ def get(project_id):
cur.mogrify(
f"""\
SELECT
{",".join(_get_column_names())}
{",".join(column_names())}
FROM public.projects
WHERE project_id = %(project_id)s AND deleted_at ISNULL
LIMIT 1;""", {"project_id": project_id})
@ -38,7 +38,7 @@ def get_batch(project_ids):
cur.mogrify(
f"""\
SELECT
project_id, {",".join(_get_column_names())}
project_id, {",".join(column_names())}
FROM public.projects
WHERE project_id IN %(project_ids)s
AND deleted_at ISNULL;""", {"project_ids": tuple(project_ids)})
@ -140,7 +140,7 @@ def add(tenant_id, project_id, new_name):
def search(tenant_id, project_id, key, value):
value = value + "%"
s_query = []
for f in _get_column_names():
for f in column_names():
s_query.append(f"CASE WHEN {f}=%(key)s THEN TRUE ELSE FALSE END AS {f}")
with pg_client.PostgresClient() as cur:
@ -215,7 +215,7 @@ def get_keys_by_projects(project_ids):
f"""\
SELECT
project_id,
{",".join(_get_column_names())}
{",".join(column_names())}
FROM public.projects
WHERE project_id IN %(project_ids)s AND deleted_at ISNULL;""",
{"project_ids": tuple(project_ids)})

358
api/chalicelib/core/sessions.py
View file

@ -1,10 +1,11 @@
from typing import List, Union
from typing import List
import schemas
from chalicelib.core import events, metadata, events_ios, \
sessions_mobs, issues, projects, errors, resources, assist, performance_event, sessions_viewed, sessions_favorite, \
sessions_mobs, issues, projects, errors, resources, assist, performance_event, sessions_favorite, \
sessions_devtool, sessions_notes
from chalicelib.utils import pg_client, helper, metrics_helper
from chalicelib.utils import sql_helper as sh
SESSION_PROJECTION_COLS = """s.project_id,
s.session_id::text AS session_id,
@ -60,7 +61,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_
s.session_id::text AS session_id,
(SELECT project_key FROM public.projects WHERE project_id = %(project_id)s LIMIT 1) AS project_key
{"," if len(extra_query) > 0 else ""}{",".join(extra_query)}
{(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata._get_column_names()]) + ") AS project_metadata") if group_metadata else ''}
{(",json_build_object(" + ",".join([f"'{m}',p.{m}" for m in metadata.column_names()]) + ") AS project_metadata") if group_metadata else ''}
FROM public.sessions AS s {"INNER JOIN public.projects AS p USING (project_id)" if group_metadata else ""}
WHERE s.project_id = %(project_id)s
AND s.session_id = %(session_id)s;""",
@ -114,67 +115,6 @@ def get_by_id2_pg(project_id, session_id, context: schemas.CurrentContext, full_
return None
def __get_sql_operator(op: Union[schemas.SearchEventOperator, schemas.ClickEventExtraOperator]):
return {
schemas.SearchEventOperator._is: "=",
schemas.SearchEventOperator._is_any: "IN",
schemas.SearchEventOperator._on: "=",
schemas.SearchEventOperator._on_any: "IN",
schemas.SearchEventOperator._is_not: "!=",
schemas.SearchEventOperator._not_on: "!=",
schemas.SearchEventOperator._contains: "ILIKE",
schemas.SearchEventOperator._not_contains: "NOT ILIKE",
schemas.SearchEventOperator._starts_with: "ILIKE",
schemas.SearchEventOperator._ends_with: "ILIKE",
}.get(op, "=")
def __is_negation_operator(op: schemas.SearchEventOperator):
return op in [schemas.SearchEventOperator._is_not,
schemas.SearchEventOperator._not_on,
schemas.SearchEventOperator._not_contains]
def __reverse_sql_operator(op):
return "=" if op == "!=" else "!=" if op == "=" else "ILIKE" if op == "NOT ILIKE" else "NOT ILIKE"
def __get_sql_operator_multiple(op: schemas.SearchEventOperator):
return " IN " if op not in [schemas.SearchEventOperator._is_not, schemas.SearchEventOperator._not_on,
schemas.SearchEventOperator._not_contains] else " NOT IN "
def __get_sql_value_multiple(values):
if isinstance(values, tuple):
return values
return tuple(values) if isinstance(values, list) else (values,)
def _multiple_conditions(condition, values, value_key="value", is_not=False):
query = []
for i in range(len(values)):
k = f"{value_key}_{i}"
query.append(condition.replace(value_key, k))
return "(" + (" AND " if is_not else " OR ").join(query) + ")"
def _multiple_values(values, value_key="value"):
query_values = {}
if values is not None and isinstance(values, list):
for i in range(len(values)):
k = f"{value_key}_{i}"
query_values[k] = values[i]
return query_values
def _isAny_opreator(op: schemas.SearchEventOperator):
return op in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any]
def _isUndefined_operator(op: schemas.SearchEventOperator):
return op in [schemas.SearchEventOperator._is_undefined]
# This function executes the query and return result
def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False,
error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False):
@ -261,9 +201,9 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
ORDER BY s.session_id desc) AS filtred_sessions
ORDER BY {sort} {data.order}, issue_score DESC) AS full_sessions;""",
full_args)
# print("--------------------")
# print(main_query)
# print("--------------------")
print("--------------------")
print(main_query)
print("--------------------")
try:
cur.execute(main_query)
except Exception as err:
@ -438,15 +378,15 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
filter_type = f.type
f.value = helper.values_for_operator(value=f.value, op=f.operator)
f_k = f"f_value{i}"
full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)}
op = __get_sql_operator(f.operator) \
full_args = {**full_args, **sh.multi_values(f.value, value_key=f_k)}
op = sh.get_sql_operator(f.operator) \
if filter_type not in [schemas.FilterType.events_count] else f.operator
is_any = _isAny_opreator(f.operator)
is_undefined = _isUndefined_operator(f.operator)
is_any = sh.isAny_opreator(f.operator)
is_undefined = sh.isUndefined_operator(f.operator)
if not is_any and not is_undefined and len(f.value) == 0:
continue
is_not = False
if __is_negation_operator(f.operator):
if sh.is_negation_operator(f.operator):
is_not = True
if filter_type == schemas.FilterType.user_browser:
if is_any:
@ -454,9 +394,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_browser IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]:
if is_any:
@ -464,9 +405,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_os IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]:
if is_any:
@ -474,9 +415,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_device IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]:
if is_any:
@ -484,9 +425,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_country IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.utm_source]:
if is_any:
@ -497,11 +439,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.utm_source IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.utm_medium]:
if is_any:
extra_constraints.append('s.utm_medium IS NOT NULL')
@ -511,11 +453,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.utm_medium IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.utm_campaign]:
if is_any:
extra_constraints.append('s.utm_campaign IS NOT NULL')
@ -525,11 +467,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.utm_campaign IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
elif filter_type == schemas.FilterType.duration:
if len(f.value) > 0 and f.value[0] is not None:
@ -546,7 +488,8 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
extra_constraints.append('s.base_referrer IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
elif filter_type == events.event_type.METADATA.ui_type:
# get metadata list only if you need it
if meta_keys is None:
@ -561,11 +504,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NULL")
else:
extra_constraints.append(
_multiple_conditions(
sh.multi_conditions(
f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text",
f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(
sh.multi_conditions(
f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text",
f.value, is_not=is_not, value_key=f_k))
elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]:
@ -577,9 +520,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_id IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.user_anonymous_id,
schemas.FilterType.user_anonymous_id_ios]:
if is_any:
@ -590,11 +535,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_anonymous_id IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]:
if is_any:
extra_constraints.append('s.rev_id IS NOT NULL')
@ -604,40 +549,58 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.rev_id IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
elif filter_type == schemas.FilterType.platform:
# op = __get_sql_operator(f.operator)
# op = __ sh.get_sql_operator(f.operator)
extra_constraints.append(
_multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
elif filter_type == schemas.FilterType.issue:
if is_any:
extra_constraints.append("array_length(s.issue_types, 1) > 0")
ss_constraints.append("array_length(ms.issue_types, 1) > 0")
else:
extra_constraints.append(
_multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not,
value_key=f_k))
# search sessions with click_rage on a specific selector
if len(f.filters) > 0 and schemas.IssueType.click_rage in f.value:
for j, sf in enumerate(f.filters):
if sf.operator == schemas.IssueFilterOperator._on_selector:
f_k = f"f_value{i}_{j}"
full_args = {**full_args, **sh.multi_values(sf.value, value_key=f_k)}
extra_constraints += ["mc.timestamp>=%(startDate)s",
"mc.timestamp<=%(endDate)s",
"mis.type='click_rage'",
sh.multi_conditions(f"mc.selector=%({f_k})s",
sf.value, is_not=is_not,
value_key=f_k)]
extra_from += """INNER JOIN events.clicks AS mc USING(session_id)
INNER JOIN events_common.issues USING (session_id,timestamp)
INNER JOIN public.issues AS mis USING (issue_id)\n"""
elif filter_type == schemas.FilterType.events_count:
extra_constraints.append(
_multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
# ---------------------------------------------------------------------------
if len(data.events) > 0:
valid_events_count = 0
for event in data.events:
is_any = _isAny_opreator(event.operator)
is_any = sh.isAny_opreator(event.operator)
if not isinstance(event.value, list):
event.value = [event.value]
if __is_valid_event(is_any=is_any, event=event):
@ -649,16 +612,16 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
events_joiner = " UNION " if or_events else " INNER JOIN LATERAL "
for i, event in enumerate(data.events):
event_type = event.type
is_any = _isAny_opreator(event.operator)
is_any = sh.isAny_opreator(event.operator)
if not isinstance(event.value, list):
event.value = [event.value]
if not __is_valid_event(is_any=is_any, event=event):
continue
op = __get_sql_operator(event.operator)
op = sh.get_sql_operator(event.operator)
is_not = False
if __is_negation_operator(event.operator):
if sh.is_negation_operator(event.operator):
is_not = True
op = __reverse_sql_operator(op)
op = sh.reverse_sql_operator(op)
if event_index == 0 or or_events:
event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)"
event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s",
@ -678,49 +641,49 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
if event.type != schemas.PerformanceEventType.time_between_events:
event.value = helper.values_for_operator(value=event.value, op=event.operator)
full_args = {**full_args,
**_multiple_values(event.value, value_key=e_k),
**_multiple_values(event.source, value_key=s_k)}
**sh.multi_values(event.value, value_key=e_k),
**sh.multi_values(event.source, value_key=s_k)}
if event_type == events.event_type.CLICK.ui_type:
event_from = event_from % f"{events.event_type.CLICK.table} AS main "
if not is_any:
if event.operator == schemas.ClickEventExtraOperator._on_selector:
event_where.append(
_multiple_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k))
sh.multi_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k))
else:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value,
value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value,
value_key=e_k))
elif event_type == events.event_type.INPUT.ui_type:
event_from = event_from % f"{events.event_type.INPUT.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value,
value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value,
value_key=e_k))
if event.source is not None and len(event.source) > 0:
event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source,
value_key=f"custom{i}"))
full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")}
event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source,
value_key=f"custom{i}"))
full_args = {**full_args, **sh.multi_values(event.source, value_key=f"custom{i}")}
elif event_type == events.event_type.LOCATION.ui_type:
event_from = event_from % f"{events.event_type.LOCATION.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.CUSTOM.ui_type:
event_from = event_from % f"{events.event_type.CUSTOM.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value,
value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value,
value_key=e_k))
elif event_type == events.event_type.REQUEST.ui_type:
event_from = event_from % f"{events.event_type.REQUEST.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value,
value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value,
value_key=e_k))
# elif event_type == events.event_type.GRAPHQL.ui_type:
# event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main "
# if not is_any:
@ -731,17 +694,17 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
event_from = event_from % f"{events.event_type.STATEACTION.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.ERROR.ui_type:
event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)"
event.source = list(set(event.source))
if not is_any and event.value not in [None, "*", ""]:
event_where.append(
_multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)",
event.value, value_key=e_k))
sh.multi_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)",
event.value, value_key=e_k))
if len(event.source) > 0 and event.source[0] not in [None, "*", ""]:
event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k))
event_where.append(sh.multi_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k))
# ----- IOS
@ -749,49 +712,49 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.INPUT_IOS.ui_type:
event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
if event.source is not None and len(event.source) > 0:
event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source,
value_key="custom{i}"))
full_args = {**full_args, **_multiple_values(event.source, f"custom{i}")}
event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source,
value_key="custom{i}"))
full_args = {**full_args, **sh.multi_values(event.source, f"custom{i}")}
elif event_type == events.event_type.VIEW_IOS.ui_type:
event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.CUSTOM_IOS.ui_type:
event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.REQUEST_IOS.ui_type:
event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.ERROR_IOS.ui_type:
event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)"
if not is_any and event.value not in [None, "*", ""]:
event_where.append(
_multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)",
event.value, value_key=e_k))
sh.multi_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)",
event.value, value_key=e_k))
elif event_type == schemas.PerformanceEventType.fetch_failed:
event_from = event_from % f"{events.event_type.REQUEST.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s",
event.value, value_key=e_k))
col = performance_event.get_col(event_type)
colname = col["column"]
event_where.append(f"main.{colname} = FALSE")
@ -805,7 +768,7 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
# colname = col["column"]
# tname = "main"
# e_k += "_custom"
# full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)}
# full_args = {**full_args, **_ sh.multiple_values(event.source, value_key=e_k)}
# event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " +
# _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s",
# event.source, value_key=e_k))
@ -826,14 +789,14 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
f"{tname}.timestamp <= %(endDate)s"]
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s",
event.value, value_key=e_k))
e_k += "_custom"
full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)}
full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)}
event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " +
_multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s",
event.source, value_key=e_k))
sh.multi_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s",
event.source, value_key=e_k))
elif event_type == schemas.PerformanceEventType.time_between_events:
event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) "
if not isinstance(event.value[0].value, list):
@ -847,70 +810,71 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
e_k1 = e_k + "_e1"
e_k2 = e_k + "_e2"
full_args = {**full_args,
**_multiple_values(event.value[0].value, value_key=e_k1),
**_multiple_values(event.value[1].value, value_key=e_k2)}
s_op = __get_sql_operator(event.value[0].operator)
**sh.multi_values(event.value[0].value, value_key=e_k1),
**sh.multi_values(event.value[1].value, value_key=e_k2)}
s_op = sh.get_sql_operator(event.value[0].operator)
event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"]
if event_index > 0 and not or_events:
event_where.append("main2.session_id=event_0.session_id")
is_any = _isAny_opreator(event.value[0].operator)
is_any = sh.isAny_opreator(event.value[0].operator)
if not is_any:
event_where.append(
_multiple_conditions(
sh.multi_conditions(
f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s",
event.value[0].value, value_key=e_k1))
s_op = __get_sql_operator(event.value[1].operator)
is_any = _isAny_opreator(event.value[1].operator)
s_op = sh.get_sql_operator(event.value[1].operator)
is_any = sh.isAny_opreator(event.value[1].operator)
if not is_any:
event_where.append(
_multiple_conditions(
sh.multi_conditions(
f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s",
event.value[1].value, value_key=e_k2))
e_k += "_custom"
full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)}
full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)}
event_where.append(
_multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s",
event.source, value_key=e_k))
sh.multi_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s",
event.source, value_key=e_k))
elif event_type == schemas.EventType.request_details:
event_from = event_from % f"{events.event_type.REQUEST.table} AS main "
apply = False
for j, f in enumerate(event.filters):
is_any = _isAny_opreator(f.operator)
is_any = sh.isAny_opreator(f.operator)
if is_any or len(f.value) == 0:
continue
f.value = helper.values_for_operator(value=f.value, op=f.operator)
op = __get_sql_operator(f.operator)
op = sh.get_sql_operator(f.operator)
e_k_f = e_k + f"_fetch{j}"
full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)}
full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)}
if f.type == schemas.FetchFilterType._url:
event_where.append(
_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text",
f.value, value_key=e_k_f))
sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text",
f.value, value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._status_code:
event_where.append(
_multiple_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value,
value_key=e_k_f))
sh.multi_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value,
value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._method:
event_where.append(
_multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._duration:
event_where.append(
_multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value,
value_key=e_k_f))
sh.multi_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value,
value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._request_body:
event_where.append(
_multiple_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value,
value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._response_body:
event_where.append(
_multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value,
value_key=e_k_f))
sh.multi_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value,
value_key=e_k_f))
apply = True
else:
print(f"undefined FETCH filter: {f.type}")
@ -919,26 +883,26 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
elif event_type == schemas.EventType.graphql:
event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main "
for j, f in enumerate(event.filters):
is_any = _isAny_opreator(f.operator)
is_any = sh.isAny_opreator(f.operator)
if is_any or len(f.value) == 0:
continue
f.value = helper.values_for_operator(value=f.value, op=f.operator)
op = __get_sql_operator(f.operator)
op = sh.get_sql_operator(f.operator)
e_k_f = e_k + f"_graphql{j}"
full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)}
full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)}
if f.type == schemas.GraphqlFilterType._name:
event_where.append(
_multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value,
value_key=e_k_f))
sh.multi_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value,
value_key=e_k_f))
elif f.type == schemas.GraphqlFilterType._method:
event_where.append(
_multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f))
elif f.type == schemas.GraphqlFilterType._request_body:
event_where.append(
_multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f))
elif f.type == schemas.GraphqlFilterType._response_body:
event_where.append(
_multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f))
else:
print(f"undefined GRAPHQL filter: {f.type}")
else:

5
api/chalicelib/core/sessions_notes.py
View file

@ -2,6 +2,7 @@ from urllib.parse import urljoin
from decouple import config
import chalicelib.utils.sql_helper
import schemas
from chalicelib.core import sessions
from chalicelib.core.collaboration_slack import Slack
@ -58,8 +59,8 @@ def get_all_notes_by_project_id(tenant_id, project_id, user_id, data: schemas.Se
if data.tags and len(data.tags) > 0:
k = "tag_value"
conditions.append(
sessions._multiple_conditions(f"%({k})s = sessions_notes.tag", data.tags, value_key=k))
extra_params = sessions._multiple_values(data.tags, value_key=k)
chalicelib.utils.sql_helper.multi_conditions(f"%({k})s = sessions_notes.tag", data.tags, value_key=k))
extra_params = chalicelib.utils.sql_helper.multi_values(data.tags, value_key=k)
if data.shared_only:
conditions.append("sessions_notes.is_public")
elif data.mine_only:

41
api/chalicelib/core/significance.py
View file

@ -1,6 +1,7 @@
__author__ = "AZNAUROV David"
__maintainer__ = "KRAIEM Taha Yassine"
import chalicelib.utils.sql_helper
import schemas
from chalicelib.core import events, metadata, sessions
@ -49,33 +50,33 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
continue
f["value"] = helper.values_for_operator(value=f["value"], op=f["operator"])
# filter_args = _multiple_values(f["value"])
op = sessions.__get_sql_operator(f["operator"])
op = chalicelib.utils.sql_helper.get_sql_operator(f["operator"])
filter_type = f["type"]
# values[f_k] = sessions.__get_sql_value_multiple(f["value"])
f_k = f"f_value{i}"
values = {**values,
**sessions._multiple_values(helper.values_for_operator(value=f["value"], op=f["operator"]),
value_key=f_k)}
**chalicelib.utils.sql_helper.multi_values(helper.values_for_operator(value=f["value"], op=f["operator"]),
value_key=f_k)}
if filter_type == schemas.FilterType.user_browser:
# op = sessions.__get_sql_operator_multiple(f["operator"])
first_stage_extra_constraints.append(
sessions._multiple_conditions(f's.user_browser {op} %({f_k})s', f["value"], value_key=f_k))
chalicelib.utils.sql_helper.multi_conditions(f's.user_browser {op} %({f_k})s', f["value"], value_key=f_k))
elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]:
# op = sessions.__get_sql_operator_multiple(f["operator"])
first_stage_extra_constraints.append(
sessions._multiple_conditions(f's.user_os {op} %({f_k})s', f["value"], value_key=f_k))
chalicelib.utils.sql_helper.multi_conditions(f's.user_os {op} %({f_k})s', f["value"], value_key=f_k))
elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]:
# op = sessions.__get_sql_operator_multiple(f["operator"])
first_stage_extra_constraints.append(
sessions._multiple_conditions(f's.user_device {op} %({f_k})s', f["value"], value_key=f_k))
chalicelib.utils.sql_helper.multi_conditions(f's.user_device {op} %({f_k})s', f["value"], value_key=f_k))
elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]:
# op = sessions.__get_sql_operator_multiple(f["operator"])
first_stage_extra_constraints.append(
sessions._multiple_conditions(f's.user_country {op} %({f_k})s', f["value"], value_key=f_k))
chalicelib.utils.sql_helper.multi_conditions(f's.user_country {op} %({f_k})s', f["value"], value_key=f_k))
elif filter_type == schemas.FilterType.duration:
if len(f["value"]) > 0 and f["value"][0] is not None:
first_stage_extra_constraints.append(f's.duration >= %(minDuration)s')
@ -88,7 +89,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
filter_extra_from = [f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)"]
# op = sessions.__get_sql_operator_multiple(f["operator"])
first_stage_extra_constraints.append(
sessions._multiple_conditions(f"p.base_referrer {op} %({f_k})s", f["value"], value_key=f_k))
chalicelib.utils.sql_helper.multi_conditions(f"p.base_referrer {op} %({f_k})s", f["value"], value_key=f_k))
elif filter_type == events.event_type.METADATA.ui_type:
if meta_keys is None:
meta_keys = metadata.get(project_id=project_id)
@ -96,25 +97,25 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
# op = sessions.__get_sql_operator(f["operator"])
if f.get("key") in meta_keys.keys():
first_stage_extra_constraints.append(
sessions._multiple_conditions(
chalicelib.utils.sql_helper.multi_conditions(
f's.{metadata.index_to_colname(meta_keys[f["key"]])} {op} %({f_k})s', f["value"],
value_key=f_k))
# values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op)
elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]:
# op = sessions.__get_sql_operator(f["operator"])
first_stage_extra_constraints.append(
sessions._multiple_conditions(f's.user_id {op} %({f_k})s', f["value"], value_key=f_k))
chalicelib.utils.sql_helper.multi_conditions(f's.user_id {op} %({f_k})s', f["value"], value_key=f_k))
# values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op)
elif filter_type in [schemas.FilterType.user_anonymous_id,
schemas.FilterType.user_anonymous_id_ios]:
# op = sessions.__get_sql_operator(f["operator"])
first_stage_extra_constraints.append(
sessions._multiple_conditions(f's.user_anonymous_id {op} %({f_k})s', f["value"], value_key=f_k))
chalicelib.utils.sql_helper.multi_conditions(f's.user_anonymous_id {op} %({f_k})s', f["value"], value_key=f_k))
# values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op)
elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]:
# op = sessions.__get_sql_operator(f["operator"])
first_stage_extra_constraints.append(
sessions._multiple_conditions(f's.rev_id {op} %({f_k})s', f["value"], value_key=f_k))
chalicelib.utils.sql_helper.multi_conditions(f's.rev_id {op} %({f_k})s', f["value"], value_key=f_k))
# values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op)
i = -1
for s in stages:
@ -124,7 +125,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
if not isinstance(s["value"], list):
s["value"] = [s["value"]]
is_any = sessions._isAny_opreator(s["operator"])
is_any = chalicelib.utils.sql_helper.isAny_opreator(s["operator"])
if not is_any and isinstance(s["value"], list) and len(s["value"]) == 0:
continue
i += 1
@ -132,7 +133,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
extra_from = filter_extra_from + ["INNER JOIN public.sessions AS s USING (session_id)"]
else:
extra_from = []
op = sessions.__get_sql_operator(s["operator"])
op = chalicelib.utils.sql_helper.get_sql_operator(s["operator"])
event_type = s["type"].upper()
if event_type == events.event_type.CLICK.ui_type:
next_table = events.event_type.CLICK.table
@ -163,10 +164,10 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
print("=================UNDEFINED")
continue
values = {**values, **sessions._multiple_values(helper.values_for_operator(value=s["value"], op=s["operator"]),
value_key=f"value{i + 1}")}
if sessions.__is_negation_operator(op) and i > 0:
op = sessions.__reverse_sql_operator(op)
values = {**values, **chalicelib.utils.sql_helper.multi_values(helper.values_for_operator(value=s["value"], op=s["operator"]),
value_key=f"value{i + 1}")}
if chalicelib.utils.sql_helper.is_negation_operator(op) and i > 0:
op = chalicelib.utils.sql_helper.reverse_sql_operator(op)
main_condition = "left_not.session_id ISNULL"
extra_from.append(f"""LEFT JOIN LATERAL (SELECT session_id
FROM {next_table} AS s_main
@ -177,8 +178,8 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
if is_any:
main_condition = "TRUE"
else:
main_condition = sessions._multiple_conditions(f"main.{next_col_name} {op} %(value{i + 1})s",
values=s["value"], value_key=f"value{i + 1}")
main_condition = chalicelib.utils.sql_helper.multi_conditions(f"main.{next_col_name} {op} %(value{i + 1})s",
values=s["value"], value_key=f"value{i + 1}")
n_stages_query.append(f"""
(SELECT main.session_id,
{"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp

33
api/schemas.py
View file

@ -445,6 +445,10 @@ class ClickEventExtraOperator(str, Enum):
_on_text = "onText"
class IssueFilterOperator(str, Enum):
_on_selector = ClickEventExtraOperator._on_selector.value
class PlatformType(str, Enum):
mobile = "mobile"
desktop = "desktop"
@ -520,20 +524,30 @@ class GraphqlFilterType(str, Enum):
_response_body = "GRAPHQL_RESPONSE_BODY"
class IssueFilterType(str, Enum):
_selector = "CLICK_SELECTOR"
class RequestGraphqlFilterSchema(BaseModel):
type: Union[FetchFilterType, GraphqlFilterType] = Field(...)
value: List[Union[int, str]] = Field(...)
operator: Union[SearchEventOperator, MathOperator] = Field(...)
class IssueFilterSchema(BaseModel):
type: IssueFilterType = Field(...)
value: List[str] = Field(...)
operator: IssueFilterOperator = Field(...)
class _SessionSearchEventRaw(__MixedSearchFilter):
is_event: bool = Field(default=True, const=True)
value: List[str] = Field(...)
type: Union[EventType, PerformanceEventType] = Field(...)
operator: Union[SearchEventOperator, ClickEventExtraOperator] = Field(...)
source: Optional[List[Union[ErrorSource, int, str]]] = Field(None)
sourceOperator: Optional[MathOperator] = Field(None)
filters: Optional[List[RequestGraphqlFilterSchema]] = Field(None)
source: Optional[List[Union[ErrorSource, int, str]]] = Field(default=None)
sourceOperator: Optional[MathOperator] = Field(default=None)
filters: Optional[List[Union[RequestGraphqlFilterSchema, IssueFilterSchema]]] = Field(default=None)
@root_validator
def event_validator(cls, values):
@ -581,11 +595,14 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw):
class SessionSearchFilterSchema(__MixedSearchFilter):
is_event: bool = Field(False, const=False)
value: Union[Optional[Union[IssueType, PlatformType, int, str]],
Optional[List[Union[IssueType, PlatformType, int, str]]]] = Field(...)
# TODO: remove this if there nothing broken from the UI
# value: Union[Optional[Union[IssueType, PlatformType, int, str]],
# Optional[List[Union[IssueType, PlatformType, int, str]]]] = Field(...)
value: List[Union[IssueType, PlatformType, int, str]] = Field(default=[])
type: FilterType = Field(...)
operator: Union[SearchEventOperator, MathOperator] = Field(...)
source: Optional[Union[ErrorSource, str]] = Field(default=None)
filters: List[IssueFilterSchema] = Field(default=[])
@root_validator
def filter_validator(cls, values):
@ -1215,12 +1232,8 @@ class FlatClickMapSessionsSearch(SessionsSearchPayloadSchema):
return values
class IssueFilterType(str, Enum):
_on_selector = ClickEventExtraOperator._on_selector.value
class IssueAdvancedFilter(BaseModel):
type: IssueFilterType = Field(default=IssueFilterType._on_selector)
type: IssueFilterType = Field(default=IssueFilterType._selector)
value: List[str] = Field(default=[])
operator: SearchEventOperator = Field(default=SearchEventOperator._is)

1
ee/api/.gitignore vendored
View file

@ -236,6 +236,7 @@ Pipfile
/chalicelib/utils/pg_client.py
/chalicelib/utils/s3.py
/chalicelib/utils/smtp.py
/chalicelib/utils/sql_helper.py
/chalicelib/utils/strings.py
/chalicelib/utils/TimeUTC.py
/routers/app/__init__.py

2
ee/api/chalicelib/core/autocomplete_exp.py
View file

@ -3,7 +3,7 @@ from chalicelib.utils import ch_client
from chalicelib.utils import helper
from chalicelib.utils.event_filter_definition import Event
TABLE = "final.autocomplete"
TABLE = "experimental.autocomplete"
def __get_autocomplete_table(value, project_id):

329
ee/api/chalicelib/core/sessions.py
View file

@ -6,6 +6,7 @@ from chalicelib.core import events, metadata, events_ios, \
sessions_mobs, issues, projects, errors, resources, assist, performance_event, sessions_viewed, sessions_favorite, \
sessions_devtool, sessions_notes
from chalicelib.utils import pg_client, helper, metrics_helper
from chalicelib.utils import sql_helper as sh
SESSION_PROJECTION_COLS = """s.project_id,
s.session_id::text AS session_id,
@ -117,67 +118,6 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu
return None
def __get_sql_operator(op: Union[schemas.SearchEventOperator, schemas.ClickEventExtraOperator]):
return {
schemas.SearchEventOperator._is: "=",
schemas.SearchEventOperator._is_any: "IN",
schemas.SearchEventOperator._on: "=",
schemas.SearchEventOperator._on_any: "IN",
schemas.SearchEventOperator._is_not: "!=",
schemas.SearchEventOperator._not_on: "!=",
schemas.SearchEventOperator._contains: "ILIKE",
schemas.SearchEventOperator._not_contains: "NOT ILIKE",
schemas.SearchEventOperator._starts_with: "ILIKE",
schemas.SearchEventOperator._ends_with: "ILIKE",
}.get(op, "=")
def __is_negation_operator(op: schemas.SearchEventOperator):
return op in [schemas.SearchEventOperator._is_not,
schemas.SearchEventOperator._not_on,
schemas.SearchEventOperator._not_contains]
def __reverse_sql_operator(op):
return "=" if op == "!=" else "!=" if op == "=" else "ILIKE" if op == "NOT ILIKE" else "NOT ILIKE"
def __get_sql_operator_multiple(op: schemas.SearchEventOperator):
return " IN " if op not in [schemas.SearchEventOperator._is_not, schemas.SearchEventOperator._not_on,
schemas.SearchEventOperator._not_contains] else " NOT IN "
def __get_sql_value_multiple(values):
if isinstance(values, tuple):
return values
return tuple(values) if isinstance(values, list) else (values,)
def _multiple_conditions(condition, values, value_key="value", is_not=False):
query = []
for i in range(len(values)):
k = f"{value_key}_{i}"
query.append(condition.replace(value_key, k))
return "(" + (" AND " if is_not else " OR ").join(query) + ")"
def _multiple_values(values, value_key="value"):
query_values = {}
if values is not None and isinstance(values, list):
for i in range(len(values)):
k = f"{value_key}_{i}"
query_values[k] = values[i]
return query_values
def _isAny_opreator(op: schemas.SearchEventOperator):
return op in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any]
def _isUndefined_operator(op: schemas.SearchEventOperator):
return op in [schemas.SearchEventOperator._is_undefined]
# This function executes the query and return result
def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False,
error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False):
@ -441,15 +381,15 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
filter_type = f.type
f.value = helper.values_for_operator(value=f.value, op=f.operator)
f_k = f"f_value{i}"
full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)}
op = __get_sql_operator(f.operator) \
full_args = {**full_args, **sh.multi_values(f.value, value_key=f_k)}
op = sh.get_sql_operator(f.operator) \
if filter_type not in [schemas.FilterType.events_count] else f.operator
is_any = _isAny_opreator(f.operator)
is_undefined = _isUndefined_operator(f.operator)
is_any = sh.isAny_opreator(f.operator)
is_undefined = sh.isUndefined_operator(f.operator)
if not is_any and not is_undefined and len(f.value) == 0:
continue
is_not = False
if __is_negation_operator(f.operator):
if sh.is_negation_operator(f.operator):
is_not = True
if filter_type == schemas.FilterType.user_browser:
if is_any:
@ -457,9 +397,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_browser IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]:
if is_any:
@ -467,9 +408,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_os IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]:
if is_any:
@ -477,9 +418,9 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_device IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]:
if is_any:
@ -487,9 +428,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_country IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.utm_source]:
if is_any:
@ -500,11 +442,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.utm_source IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f's.utm_source {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f'ms.utm_source {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.utm_medium]:
if is_any:
extra_constraints.append('s.utm_medium IS NOT NULL')
@ -514,11 +456,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.utm_medium IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f's.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f'ms.utm_medium {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.utm_campaign]:
if is_any:
extra_constraints.append('s.utm_campaign IS NOT NULL')
@ -528,11 +470,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.utm_campaign IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f's.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f'ms.utm_campaign {op} %({f_k})s::text', f.value, is_not=is_not,
value_key=f_k))
elif filter_type == schemas.FilterType.duration:
if len(f.value) > 0 and f.value[0] is not None:
@ -549,7 +491,8 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
extra_constraints.append('s.base_referrer IS NOT NULL')
else:
extra_constraints.append(
_multiple_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"s.base_referrer {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
elif filter_type == events.event_type.METADATA.ui_type:
# get metadata list only if you need it
if meta_keys is None:
@ -564,11 +507,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append(f"ms.{metadata.index_to_colname(meta_keys[f.source])} IS NULL")
else:
extra_constraints.append(
_multiple_conditions(
sh.multi_conditions(
f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text",
f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(
sh.multi_conditions(
f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s::text",
f.value, is_not=is_not, value_key=f_k))
elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]:
@ -580,9 +523,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_id IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"s.user_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"ms.user_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.user_anonymous_id,
schemas.FilterType.user_anonymous_id_ios]:
if is_any:
@ -593,11 +538,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.user_anonymous_id IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"s.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"ms.user_anonymous_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]:
if is_any:
extra_constraints.append('s.rev_id IS NOT NULL')
@ -607,40 +552,41 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
ss_constraints.append('ms.rev_id IS NULL')
else:
extra_constraints.append(
_multiple_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"s.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not, value_key=f_k))
sh.multi_conditions(f"ms.rev_id {op} %({f_k})s::text", f.value, is_not=is_not,
value_key=f_k))
elif filter_type == schemas.FilterType.platform:
# op = __get_sql_operator(f.operator)
# op = __ sh.get_sql_operator(f.operator)
extra_constraints.append(
_multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
elif filter_type == schemas.FilterType.issue:
if is_any:
extra_constraints.append("array_length(s.issue_types, 1) > 0")
ss_constraints.append("array_length(ms.issue_types, 1) > 0")
else:
extra_constraints.append(
_multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not,
value_key=f_k))
elif filter_type == schemas.FilterType.events_count:
extra_constraints.append(
_multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
ss_constraints.append(
_multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
sh.multi_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not,
value_key=f_k))
# ---------------------------------------------------------------------------
if len(data.events) > 0:
valid_events_count = 0
for event in data.events:
is_any = _isAny_opreator(event.operator)
is_any = sh.isAny_opreator(event.operator)
if not isinstance(event.value, list):
event.value = [event.value]
if __is_valid_event(is_any=is_any, event=event):
@ -652,16 +598,16 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
events_joiner = " UNION " if or_events else " INNER JOIN LATERAL "
for i, event in enumerate(data.events):
event_type = event.type
is_any = _isAny_opreator(event.operator)
is_any = sh.isAny_opreator(event.operator)
if not isinstance(event.value, list):
event.value = [event.value]
if not __is_valid_event(is_any=is_any, event=event):
continue
op = __get_sql_operator(event.operator)
op = sh.get_sql_operator(event.operator)
is_not = False
if __is_negation_operator(event.operator):
if sh.is_negation_operator(event.operator):
is_not = True
op = __reverse_sql_operator(op)
op = sh.reverse_sql_operator(op)
if event_index == 0 or or_events:
event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)"
event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s",
@ -681,49 +627,49 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
if event.type != schemas.PerformanceEventType.time_between_events:
event.value = helper.values_for_operator(value=event.value, op=event.operator)
full_args = {**full_args,
**_multiple_values(event.value, value_key=e_k),
**_multiple_values(event.source, value_key=s_k)}
**sh.multi_values(event.value, value_key=e_k),
**sh.multi_values(event.source, value_key=s_k)}
if event_type == events.event_type.CLICK.ui_type:
event_from = event_from % f"{events.event_type.CLICK.table} AS main "
if not is_any:
if event.operator == schemas.ClickEventExtraOperator._on_selector:
event_where.append(
_multiple_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k))
sh.multi_conditions(f"main.selector = %({e_k})s", event.value, value_key=e_k))
else:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value,
value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value,
value_key=e_k))
elif event_type == events.event_type.INPUT.ui_type:
event_from = event_from % f"{events.event_type.INPUT.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value,
value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value,
value_key=e_k))
if event.source is not None and len(event.source) > 0:
event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source,
value_key=f"custom{i}"))
full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")}
event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source,
value_key=f"custom{i}"))
full_args = {**full_args, **sh.multi_values(event.source, value_key=f"custom{i}")}
elif event_type == events.event_type.LOCATION.ui_type:
event_from = event_from % f"{events.event_type.LOCATION.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.CUSTOM.ui_type:
event_from = event_from % f"{events.event_type.CUSTOM.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value,
value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value,
value_key=e_k))
elif event_type == events.event_type.REQUEST.ui_type:
event_from = event_from % f"{events.event_type.REQUEST.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value,
value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value,
value_key=e_k))
# elif event_type == events.event_type.GRAPHQL.ui_type:
# event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main "
# if not is_any:
@ -734,17 +680,17 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
event_from = event_from % f"{events.event_type.STATEACTION.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.ERROR.ui_type:
event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)"
event.source = list(set(event.source))
if not is_any and event.value not in [None, "*", ""]:
event_where.append(
_multiple_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)",
event.value, value_key=e_k))
sh.multi_conditions(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)",
event.value, value_key=e_k))
if len(event.source) > 0 and event.source[0] not in [None, "*", ""]:
event_where.append(_multiple_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k))
event_where.append(sh.multi_conditions(f"main1.source = %({s_k})s", event.source, value_key=s_k))
# ----- IOS
@ -752,49 +698,49 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.INPUT_IOS.ui_type:
event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
if event.source is not None and len(event.source) > 0:
event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source,
value_key="custom{i}"))
full_args = {**full_args, **_multiple_values(event.source, f"custom{i}")}
event_where.append(sh.multi_conditions(f"main.value ILIKE %(custom{i})s", event.source,
value_key="custom{i}"))
full_args = {**full_args, **sh.multi_values(event.source, f"custom{i}")}
elif event_type == events.event_type.VIEW_IOS.ui_type:
event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.CUSTOM_IOS.ui_type:
event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.REQUEST_IOS.ui_type:
event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s",
event.value, value_key=e_k))
elif event_type == events.event_type.ERROR_IOS.ui_type:
event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)"
if not is_any and event.value not in [None, "*", ""]:
event_where.append(
_multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)",
event.value, value_key=e_k))
sh.multi_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)",
event.value, value_key=e_k))
elif event_type == schemas.PerformanceEventType.fetch_failed:
event_from = event_from % f"{events.event_type.REQUEST.table} AS main "
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s",
event.value, value_key=e_k))
col = performance_event.get_col(event_type)
colname = col["column"]
event_where.append(f"main.{colname} = FALSE")
@ -808,7 +754,7 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
# colname = col["column"]
# tname = "main"
# e_k += "_custom"
# full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)}
# full_args = {**full_args, **_ sh.multiple_values(event.source, value_key=e_k)}
# event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " +
# _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s",
# event.source, value_key=e_k))
@ -829,14 +775,14 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
f"{tname}.timestamp <= %(endDate)s"]
if not is_any:
event_where.append(
_multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s",
event.value, value_key=e_k))
sh.multi_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s",
event.value, value_key=e_k))
e_k += "_custom"
full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)}
full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)}
event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " +
_multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s",
event.source, value_key=e_k))
sh.multi_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s",
event.source, value_key=e_k))
elif event_type == schemas.PerformanceEventType.time_between_events:
event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) "
if not isinstance(event.value[0].value, list):
@ -850,70 +796,71 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
e_k1 = e_k + "_e1"
e_k2 = e_k + "_e2"
full_args = {**full_args,
**_multiple_values(event.value[0].value, value_key=e_k1),
**_multiple_values(event.value[1].value, value_key=e_k2)}
s_op = __get_sql_operator(event.value[0].operator)
**sh.multi_values(event.value[0].value, value_key=e_k1),
**sh.multi_values(event.value[1].value, value_key=e_k2)}
s_op = sh.get_sql_operator(event.value[0].operator)
event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"]
if event_index > 0 and not or_events:
event_where.append("main2.session_id=event_0.session_id")
is_any = _isAny_opreator(event.value[0].operator)
is_any = sh.isAny_opreator(event.value[0].operator)
if not is_any:
event_where.append(
_multiple_conditions(
sh.multi_conditions(
f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s",
event.value[0].value, value_key=e_k1))
s_op = __get_sql_operator(event.value[1].operator)
is_any = _isAny_opreator(event.value[1].operator)
s_op = sh.get_sql_operator(event.value[1].operator)
is_any = sh.isAny_opreator(event.value[1].operator)
if not is_any:
event_where.append(
_multiple_conditions(
sh.multi_conditions(
f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s",
event.value[1].value, value_key=e_k2))
e_k += "_custom"
full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)}
full_args = {**full_args, **sh.multi_values(event.source, value_key=e_k)}
event_where.append(
_multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s",
event.source, value_key=e_k))
sh.multi_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s",
event.source, value_key=e_k))
elif event_type == schemas.EventType.request_details:
event_from = event_from % f"{events.event_type.REQUEST.table} AS main "
apply = False
for j, f in enumerate(event.filters):
is_any = _isAny_opreator(f.operator)
is_any = sh.isAny_opreator(f.operator)
if is_any or len(f.value) == 0:
continue
f.value = helper.values_for_operator(value=f.value, op=f.operator)
op = __get_sql_operator(f.operator)
op = sh.get_sql_operator(f.operator)
e_k_f = e_k + f"_fetch{j}"
full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)}
full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)}
if f.type == schemas.FetchFilterType._url:
event_where.append(
_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text",
f.value, value_key=e_k_f))
sh.multi_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s::text",
f.value, value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._status_code:
event_where.append(
_multiple_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value,
value_key=e_k_f))
sh.multi_conditions(f"main.status_code {f.operator} %({e_k_f})s::integer", f.value,
value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._method:
event_where.append(
_multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._duration:
event_where.append(
_multiple_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value,
value_key=e_k_f))
sh.multi_conditions(f"main.duration {f.operator} %({e_k_f})s::integer", f.value,
value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._request_body:
event_where.append(
_multiple_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.request_body {op} %({e_k_f})s::text", f.value,
value_key=e_k_f))
apply = True
elif f.type == schemas.FetchFilterType._response_body:
event_where.append(
_multiple_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value,
value_key=e_k_f))
sh.multi_conditions(f"main.response_body {op} %({e_k_f})s::text", f.value,
value_key=e_k_f))
apply = True
else:
print(f"undefined FETCH filter: {f.type}")
@ -922,26 +869,26 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr
elif event_type == schemas.EventType.graphql:
event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main "
for j, f in enumerate(event.filters):
is_any = _isAny_opreator(f.operator)
is_any = sh.isAny_opreator(f.operator)
if is_any or len(f.value) == 0:
continue
f.value = helper.values_for_operator(value=f.value, op=f.operator)
op = __get_sql_operator(f.operator)
op = sh.get_sql_operator(f.operator)
e_k_f = e_k + f"_graphql{j}"
full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)}
full_args = {**full_args, **sh.multi_values(f.value, value_key=e_k_f)}
if f.type == schemas.GraphqlFilterType._name:
event_where.append(
_multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value,
value_key=e_k_f))
sh.multi_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k_f})s", f.value,
value_key=e_k_f))
elif f.type == schemas.GraphqlFilterType._method:
event_where.append(
_multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f))
elif f.type == schemas.GraphqlFilterType._request_body:
event_where.append(
_multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f))
elif f.type == schemas.GraphqlFilterType._response_body:
event_where.append(
_multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f))
sh.multi_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f))
else:
print(f"undefined GRAPHQL filter: {f.type}")
else:

1
ee/api/clean.sh
View file

@ -58,6 +58,7 @@ rm -rf ./chalicelib/utils/metrics_helper.py
rm -rf ./chalicelib/utils/pg_client.py
rm -rf ./chalicelib/utils/s3.py
rm -rf ./chalicelib/utils/smtp.py
rm -rf ./chalicelib/utils/sql_helper.py
rm -rf ./chalicelib/utils/strings.py
rm -rf ./chalicelib/utils/TimeUTC.py
rm -rf ./routers/app/__init__.py

16
ee/scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql
View file

@ -91,6 +91,22 @@ DROP TYPE IF EXISTS metric_view_type;
ALTER TABLE IF EXISTS events.clicks
ADD COLUMN IF NOT EXISTS path text;
DROP INDEX IF EXISTS events.inputs_label_value_idx;
DROP INDEX IF EXISTS events.inputs_label_idx;
DROP INDEX IF EXISTS events.pages_base_path_idx2;
DROP INDEX IF EXISTS events.pages_base_referrer_gin_idx2;
DROP INDEX IF EXISTS events.resources_url_gin_idx;
DROP INDEX IF EXISTS events.resources_url_idx;
DROP INDEX IF EXISTS events.resources_url_hostpath_idx;
DROP INDEX IF EXISTS events.resources_session_id_timestamp_idx;
DROP INDEX IF EXISTS events.resources_duration_durationgt0_idx;
DROP INDEX IF EXISTS events.state_actions_name_idx;
DROP INDEX IF EXISTS events_common.requests_query_nn_idx;
DROP INDEX IF EXISTS events_common.requests_host_nn_idx;
DROP INDEX IF EXISTS public.sessions_user_country_gin_idx;
DROP INDEX IF EXISTS public.sessions_user_browser_gin_idx;
DROP INDEX IF EXISTS public.sessions_user_os_gin_idx;
COMMIT;
CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector);

17
scripts/schema/db/init_dbs/postgresql/1.9.5/1.9.5.sql
View file

@ -78,6 +78,23 @@ DROP TYPE IF EXISTS metric_view_type;
ALTER TABLE IF EXISTS events.clicks
ADD COLUMN IF NOT EXISTS path text;
DROP INDEX IF EXISTS events.inputs_label_value_idx;
DROP INDEX IF EXISTS events.inputs_label_idx;
DROP INDEX IF EXISTS events.pages_base_path_idx2;
DROP INDEX IF EXISTS events.pages_base_referrer_gin_idx2;
DROP INDEX IF EXISTS events.resources_url_gin_idx;
DROP INDEX IF EXISTS events.resources_url_idx;
DROP INDEX IF EXISTS events.resources_url_hostpath_idx;
DROP INDEX IF EXISTS events.resources_session_id_timestamp_idx;
DROP INDEX IF EXISTS events.resources_duration_durationgt0_idx;
DROP INDEX IF EXISTS events.state_actions_name_idx;
DROP INDEX IF EXISTS events_common.requests_query_nn_idx;
DROP INDEX IF EXISTS events_common.requests_host_nn_idx;
DROP INDEX IF EXISTS public.sessions_user_country_gin_idx;
DROP INDEX IF EXISTS public.sessions_user_browser_gin_idx;
DROP INDEX IF EXISTS public.sessions_user_os_gin_idx;
COMMIT;
CREATE INDEX CONCURRENTLY IF NOT EXISTS clicks_selector_idx ON events.clicks (selector);