jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(api): search sessions support AND|THEN|OR between events

Browse source 
feat(api): search sessions support single-operator-multiple-values
This commit is contained in:
Taha Yassine Kraiem 2021-12-20 19:18:22 +01:00
parent c0183da5ff
commit 3db36ea688
4 changed files with 192 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
api/chalicelib/core/events.py
View file

@ -1,6 +1,7 @@
from chalicelib.utils import pg_client, helper
from chalicelib.core import sessions_metas, metadata
import schemas
from chalicelib.core import issues
from chalicelib.core import sessions_metas, metadata
from chalicelib.utils import pg_client, helper
from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.utils.event_filter_definition import SupportedFilter, Event
@ -235,23 +236,23 @@ def __generic_autocomplete(event: Event):
class event_type:
CLICK = Event(ui_type="CLICK", table="events.clicks", column="label")
INPUT = Event(ui_type="INPUT", table="events.inputs", column="label")
LOCATION = Event(ui_type="LOCATION", table="events.pages", column="base_path")
CUSTOM = Event(ui_type="CUSTOM", table="events_common.customs", column="name")
REQUEST = Event(ui_type="REQUEST", table="events_common.requests", column="url")
GRAPHQL = Event(ui_type="GRAPHQL", table="events.graphql", column="name")
STATEACTION = Event(ui_type="STATEACTION", table="events.state_actions", column="name")
ERROR = Event(ui_type="ERROR", table="events.errors",
CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label")
INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label")
LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="base_path")
CUSTOM = Event(ui_type=schemas.EventType.custom, table="events_common.customs", column="name")
REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="url")
GRAPHQL = Event(ui_type=schemas.EventType.graphql, table="events.graphql", column="name")
STATEACTION = Event(ui_type=schemas.EventType.state_action, table="events.state_actions", column="name")
ERROR = Event(ui_type=schemas.EventType.error, table="events.errors",
column=None) # column=None because errors are searched by name or message
METADATA = Event(ui_type="METADATA", table="public.sessions", column=None)
METADATA = Event(ui_type=schemas.EventType.metadata, table="public.sessions", column=None)
# IOS
CLICK_IOS = Event(ui_type="CLICK_IOS", table="events_ios.clicks", column="label")
INPUT_IOS = Event(ui_type="INPUT_IOS", table="events_ios.inputs", column="label")
VIEW_IOS = Event(ui_type="VIEW_IOS", table="events_ios.views", column="name")
CUSTOM_IOS = Event(ui_type="CUSTOM_IOS", table="events_common.customs", column="name")
REQUEST_IOS = Event(ui_type="REQUEST_IOS", table="events_common.requests", column="url")
ERROR_IOS = Event(ui_type="ERROR_IOS", table="events_ios.crashes",
CLICK_IOS = Event(ui_type=schemas.EventType.click_ios, table="events_ios.clicks", column="label")
INPUT_IOS = Event(ui_type=schemas.EventType.input_ios, table="events_ios.inputs", column="label")
VIEW_IOS = Event(ui_type=schemas.EventType.view_ios, table="events_ios.views", column="name")
CUSTOM_IOS = Event(ui_type=schemas.EventType.custom_ios, table="events_common.customs", column="name")
REQUEST_IOS = Event(ui_type=schemas.EventType.request_ios, table="events_common.requests", column="url")
ERROR_IOS = Event(ui_type=schemas.EventType.error_ios, table="events_ios.crashes",
column=None) # column=None because errors are searched by name or message

124
api/chalicelib/core/sessions.py
View file

@ -109,7 +109,6 @@ def __is_multivalue(op: schemas.SearchEventOperator):
def __get_sql_operator(op: schemas.SearchEventOperator):
op = op.lower()
return {
schemas.SearchEventOperator._is: "=",
schemas.SearchEventOperator._is_any: "IN",
@ -145,6 +144,22 @@ def __get_sql_value_multiple(values):
return tuple(values) if isinstance(values, list) else (values,)
def __multiple_conditions(condition, values, value_key="value"):
query = []
for i in range(len(values)):
k = f"{value_key}_{i}"
query.append(condition.replace("value", k))
return "(" + " OR ".join(query) + ")"
def __multiple_values(values, value_key="value"):
query_values = {}
for i in range(len(values)):
k = f"{value_key}_{i}"
query_values[k] = values[i]
return query_values
@dev.timed
def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, favorite_only=False, errors_only=False,
error_status="ALL",
@ -257,15 +272,17 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
ss_constraints = [s.decode('UTF-8') for s in ss_constraints]
events_query_from = []
event_index = 0
events_joiner = " FULL JOIN " if data.events_order == schemas.SearchEventOrder._or else " INNER JOIN LATERAL "
for event in data.events:
event_type = event.type.upper()
if not isinstance(event.value, list):
event.value = [event.value]
op = __get_sql_operator(event.operator)
is_not = False
if __is_negation_operator(event.operator):
is_not = True
op = __reverse_sql_operator(op)
if event_index == 0:
if event_index == 0 or data.events_order == schemas.SearchEventOrder._or:
event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)"
event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s",
"main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s",
@ -273,13 +290,12 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
else:
event_from = "%s"
event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s",
f"event_{event_index - 1}.timestamp <= main.timestamp",
"main.session_id=event_0.session_id"]
if __is_multivalue(event.operator):
event_args = {"value": __get_sql_value_multiple(event.value)}
else:
event.value = helper.string_to_op(value=event.value, op=event.operator)
event_args = {"value": helper.string_to_sql_like_with_op(event.value, op)}
if data.events_order == schemas.SearchEventOrder._then:
event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp")
event.value = helper.values_for_operator(value=event.value, op=event.operator)
event_args = __multiple_values(event.value)
if event_type not in list(events.SUPPORTED_TYPES.keys()) \
or event.value in [None, "", "*"] \
and (event_type != events.event_type.ERROR.ui_type \
@ -287,32 +303,50 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
continue
if event_type == events.event_type.CLICK.ui_type:
event_from = event_from % f"{events.event_type.CLICK.table} AS main "
event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s")
event_where.append(__multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s")
elif event_type == events.event_type.INPUT.ui_type:
event_from = event_from % f"{events.event_type.INPUT.table} AS main "
event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s")
event_where.append(__multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s")
if len(event.custom) > 0:
event_where.append("main.value ILIKE %(custom)s")
event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE")
event_where.append(__multiple_conditions(f"main.value ILIKE %(custom)s",
event.custom, value_key="custom"))
event_args = {**event_args, **__multiple_values(event.custom, value_key="custom")}
# event_where.append("main.value ILIKE %(custom)s")
# event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE")
elif event_type == events.event_type.LOCATION.ui_type:
event_from = event_from % f"{events.event_type.LOCATION.table} AS main "
event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s")
event_where.append(__multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s")
elif event_type == events.event_type.CUSTOM.ui_type:
event_from = event_from % f"{events.event_type.CUSTOM.table} AS main "
event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s")
event_where.append(__multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s")
elif event_type == events.event_type.REQUEST.ui_type:
event_from = event_from % f"{events.event_type.REQUEST.table} AS main "
event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s")
event_where.append(__multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s")
elif event_type == events.event_type.GRAPHQL.ui_type:
event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main "
event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s")
event_where.append(__multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s")
elif event_type == events.event_type.STATEACTION.ui_type:
event_from = event_from % f"{events.event_type.STATEACTION.table} AS main "
event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s")
event_where.append(
__multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s")
elif event_type == events.event_type.ERROR.ui_type:
if event.source in [None, "*", ""]:
event.source = "js_exception"
# if event.source in [None, "*", ""]:
# event.source = "js_exception"
event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)"
if event.value not in [None, "*", ""]:
event_where.append(f"(main1.message {op} %(value)s OR main1.name {op} %(value)s)")
@ -326,40 +360,55 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
# ----- IOS
elif event_type == events.event_type.CLICK_IOS.ui_type:
event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main "
event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s")
event_where.append(
__multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s", event.value))
# event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s")
elif event_type == events.event_type.INPUT_IOS.ui_type:
event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main "
event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s")
event_where.append(
__multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s", event.value))
# event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s")
if len(event.custom) > 0:
event_where.append("main.value ILIKE %(custom)s")
event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE")
event_where.append(__multiple_conditions("main.value ILIKE %(custom)s", event.custom))
event_args = {**event_args, **__multiple_values(event.custom, "custom")}
# event_where.append("main.value ILIKE %(custom)s")
# event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE")
elif event_type == events.event_type.VIEW_IOS.ui_type:
event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main "
event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s")
event_where.append(
__multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s", event.value))
# event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s")
elif event_type == events.event_type.CUSTOM_IOS.ui_type:
event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main "
event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s")
event_where.append(
__multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s")
elif event_type == events.event_type.REQUEST_IOS.ui_type:
event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main "
event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s")
event_where.append(
__multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s",
event.value))
# event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s")
elif event_type == events.event_type.ERROR_IOS.ui_type:
event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)"
if event.value not in [None, "*", ""]:
event_where.append(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)")
event_where.append(
__multiple_conditions(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)",
event.value))
# event_where.append(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)")
else:
continue
if event_index == 0:
if event_index == 0 or data.events_order == schemas.SearchEventOrder._or:
event_where += ss_constraints
if is_not:
if event_index == 0:
events_query_from.append(cur.mogrify(f"""\
(SELECT
session_id,
0 AS timestamp,
{event_index} AS funnel_step
0 AS timestamp
FROM sessions
WHERE EXISTS(SELECT session_id
FROM {event_from}
@ -375,14 +424,13 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
events_query_from.append(cur.mogrify(f"""\
(SELECT
event_0.session_id,
event_{event_index - 1}.timestamp AS timestamp,
{event_index} AS funnel_step
event_{event_index - 1}.timestamp AS timestamp
WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE
) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\
""", {**generic_args, **event_args}).decode('UTF-8'))
else:
events_query_from.append(cur.mogrify(f"""\
(SELECT main.session_id, MIN(timestamp) AS timestamp,{event_index} AS funnel_step
(SELECT main.session_id, MIN(timestamp) AS timestamp
FROM {event_from}
WHERE {" AND ".join(event_where)}
GROUP BY 1
@ -394,7 +442,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
event_0.session_id,
MIN(event_0.timestamp) AS first_event_ts,
MAX(event_{event_index - 1}.timestamp) AS last_event_ts
FROM {(" INNER JOIN LATERAL ").join(events_query_from)}
FROM {events_joiner.join(events_query_from)}
GROUP BY 1
{fav_only_join}"""
else:
@ -488,8 +536,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f
ORDER BY favorite DESC, issue_score DESC, {sort} {order};""",
generic_args)
# print("--------------------")
# print(main_query)
print("--------------------")
print(main_query)
cur.execute(main_query)

72
api/chalicelib/utils/helper.py
View file

@ -1,6 +1,7 @@
import random
import re
import string
from typing import Union
import math
import requests
@ -168,39 +169,56 @@ def string_to_sql_like(value):
def string_to_sql_like_with_op(value, op):
if isinstance(value, list) and len(value) > 0:
_value = value[0]
if isinstance(value, list):
r = []
for v in value:
r.append(string_to_sql_like_with_op(v, op))
return r
else:
_value = value
if _value is None:
return _value
if op.upper() != 'ILIKE':
if _value is None:
return _value
if op.upper() != 'ILIKE':
return _value.replace("%", "%%")
_value = _value.replace("*", "%")
if _value.startswith("^"):
_value = _value[1:]
elif not _value.startswith("%"):
_value = '%' + _value
if _value.endswith("$"):
_value = _value[:-1]
elif not _value.endswith("%"):
_value = _value + '%'
return _value.replace("%", "%%")
_value = _value.replace("*", "%")
if _value.startswith("^"):
_value = _value[1:]
elif not _value.startswith("%"):
_value = '%' + _value
if _value.endswith("$"):
_value = _value[:-1]
elif not _value.endswith("%"):
_value = _value + '%'
return _value.replace("%", "%%")
def string_to_op(value: str, op: schemas.SearchEventOperator):
if isinstance(value, list) and len(value) > 0:
_value = value[0]
likable_operators = [schemas.SearchEventOperator._starts_with, schemas.SearchEventOperator._ends_with,
schemas.SearchEventOperator._contains, schemas.SearchEventOperator._notcontains]
def is_likable(op: schemas.SearchEventOperator):
return op in likable_operators
def values_for_operator(value: Union[str, list], op: schemas.SearchEventOperator):
if not is_likable(op):
return value
if isinstance(value, list):
r = []
for v in value:
r.append(values_for_operator(v, op))
return r
else:
_value = value
if _value is None:
return _value
if op == schemas.SearchEventOperator._starts_with:
_value = '^' + _value
elif op == schemas.SearchEventOperator._ends_with:
_value = _value + '$'
return _value
if value is None:
return value
if op == schemas.SearchEventOperator._starts_with:
return value + '%%'
elif op == schemas.SearchEventOperator._ends_with:
return '%%' + value
elif op == schemas.SearchEventOperator._contains:
return '%%' + value + '%%'
return value
def is_valid_email(email):

46
api/schemas.py
View file

@ -330,6 +330,36 @@ class SourcemapUploadPayloadSchema(BaseModel):
urls: List[str] = Field(..., alias="URL")
class ErrorSource(str, Enum):
js_exception = "js_exception"
bugsnag = "bugsnag"
cloudwatch = "cloudwatch"
datadog = "datadog"
newrelic = "newrelic"
rollbar = "rollbar"
sentry = "sentry"
stackdriver = "stackdriver"
sumologic = "sumologic"
class EventType(str, Enum):
click = "CLICK"
input = "INPUT"
location = "LOCATION"
custom = "CUSTOM"
request = "REQUEST"
graphql = "GRAPHQL"
state_action = "STATEACTION"
error = "ERROR"
metadata = "METADATA"
click_ios = "CLICK_IOS"
input_ios = "INPUT_IOS"
view_ios = "VIEW_IOS"
custom_ios = "CUSTOM_IOS"
request_ios = "REQUEST_IOS"
error_ios = "ERROR_IOS"
class SearchEventOperator(str, Enum):
_is = "is"
_is_any = "isAny"
@ -348,13 +378,19 @@ class PlatformType(str, Enum):
desktop = "desktop"
class SearchEventOrder(str, Enum):
_then = "then"
_or = "or"
_and = "and"
class _SessionSearchEventSchema(BaseModel):
custom: Optional[str] = Field(None)
custom: Optional[List[str]] = Field(None)
key: Optional[str] = Field(None)
value: Union[Optional[str], Optional[List[str]]] = Field(...)
type: str = Field(...)
type: EventType = Field(...)
operator: SearchEventOperator = Field(...)
source: Optional[str] = Field(...)
source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception)
class _SessionSearchFilterSchema(_SessionSearchEventSchema):
@ -371,6 +407,10 @@ class SessionsSearchPayloadSchema(BaseModel):
sort: str = Field(...)
order: str = Field(default="DESC")
platform: Optional[PlatformType] = Field(None)
events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then)
class Config:
alias_generator = attribute_to_camel_case
class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema):