diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index f80c363ab..8e6bcf853 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -104,29 +104,25 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_ return None -def __is_multivalue(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._is_any, schemas.SearchEventOperator._on_any] - - def __get_sql_operator(op: schemas.SearchEventOperator): return { schemas.SearchEventOperator._is: "=", schemas.SearchEventOperator._is_any: "IN", schemas.SearchEventOperator._on: "=", schemas.SearchEventOperator._on_any: "IN", - schemas.SearchEventOperator._isnot: "!=", - schemas.SearchEventOperator._noton: "!=", + schemas.SearchEventOperator._is_not: "!=", + schemas.SearchEventOperator._not_on: "!=", schemas.SearchEventOperator._contains: "ILIKE", - schemas.SearchEventOperator._notcontains: "NOT ILIKE", + schemas.SearchEventOperator._not_contains: "NOT ILIKE", schemas.SearchEventOperator._starts_with: "ILIKE", schemas.SearchEventOperator._ends_with: "ILIKE", }.get(op, "=") def __is_negation_operator(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._isnot, - schemas.SearchEventOperator._noton, - schemas.SearchEventOperator._notcontains] + return op in [schemas.SearchEventOperator._is_not, + schemas.SearchEventOperator._not_on, + schemas.SearchEventOperator._not_contains] def __reverse_sql_operator(op): @@ -134,8 +130,8 @@ def __reverse_sql_operator(op): def __get_sql_operator_multiple(op: schemas.SearchEventOperator): - # op == schemas.SearchEventOperator._is is for filter support - return " IN " if __is_multivalue(op) or op == schemas.SearchEventOperator._is else " NOT IN " + return " IN " if op not in [schemas.SearchEventOperator._is_not, schemas.SearchEventOperator._not_on, + schemas.SearchEventOperator._not_contains] else " NOT IN " def __get_sql_value_multiple(values): @@ -144,12 +140,12 @@ def __get_sql_value_multiple(values): return tuple(values) if isinstance(values, list) else (values,) -def __multiple_conditions(condition, values, value_key="value"): +def __multiple_conditions(condition, values, value_key="value", is_not=False): query = [] for i in range(len(values)): k = f"{value_key}_{i}" query.append(condition.replace("value", k)) - return "(" + " OR ".join(query) + ")" + return "(" + (" AND " if is_not else " OR ").join(query) + ")" def __multiple_values(values, value_key="value"): @@ -181,41 +177,63 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f events_query_part = "" if len(data.filters) > 0: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} + meta_keys = None for f in data.filters: if not isinstance(f.value, list): f.value = [f.value] if len(f.value) == 0 or f.value[0] is None: continue - filter_type = f.type.upper() - f.value = __get_sql_value_multiple(f.value) + filter_type = f.type + # f.value = __get_sql_value_multiple(f.value) + f.value = helper.values_for_operator(value=f.value, op=f.operator) + filter_args = __multiple_values(f.value) + op = __get_sql_operator(f.operator) + is_not = False + if __is_negation_operator(f.operator): + is_not = True + # op = __reverse_sql_operator(op) if filter_type == sessions_metas.meta_type.USERBROWSER: - op = __get_sql_operator_multiple(f.operator) - extra_constraints.append(cur.mogrify(f's.user_browser {op} %(value)s', {"value": f.value})) - ss_constraints.append(cur.mogrify(f'ms.user_browser {op} %(value)s', {"value": f.value})) + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f's.user_browser {op} %(value)s', f.value, is_not=is_not), + filter_args)) + ss_constraints.append( + cur.mogrify(__multiple_conditions(f'ms.user_browser {op} %(value)s', f.value, is_not=is_not), + filter_args)) elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: - op = __get_sql_operator_multiple(f.operator) - extra_constraints.append(cur.mogrify(f's.user_os {op} %(value)s', {"value": f.value})) - ss_constraints.append(cur.mogrify(f'ms.user_os {op} %(value)s', {"value": f.value})) + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f's.user_os {op} %(value)s', f.value, is_not=is_not), + filter_args)) + ss_constraints.append( + cur.mogrify(__multiple_conditions(f'ms.user_os {op} %(value)s', f.value, is_not=is_not), + filter_args)) elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: - op = __get_sql_operator_multiple(f.operator) - extra_constraints.append(cur.mogrify(f's.user_device {op} %(value)s', {"value": f.value})) - ss_constraints.append(cur.mogrify(f'ms.user_device {op} %(value)s', {"value": f.value})) + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f's.user_device {op} %(value)s', f.value, is_not=is_not), + filter_args)) + ss_constraints.append( + cur.mogrify(__multiple_conditions(f'ms.user_device {op} %(value)s', f.value, is_not=is_not), + filter_args)) elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: - op = __get_sql_operator_multiple(f.operator) - extra_constraints.append(cur.mogrify(f's.user_country {op} %(value)s', {"value": f.value})) - ss_constraints.append(cur.mogrify(f'ms.user_country {op} %(value)s', {"value": f.value})) - elif filter_type == "duration".upper(): + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f's.user_country {op} %(value)s', f.value, is_not=is_not), + filter_args)) + ss_constraints.append( + cur.mogrify(__multiple_conditions(f'ms.user_country {op} %(value)s', f.value, is_not=is_not), + filter_args)) + elif filter_type == schemas.FilterType.duration: if len(f.value) > 0 and f.value[0] is not None: extra_constraints.append( cur.mogrify("s.duration >= %(minDuration)s", {"minDuration": f.value[0]})) ss_constraints.append( cur.mogrify("ms.duration >= %(minDuration)s", {"minDuration": f.value[0]})) - if len(f.value) > 1 and f.value[1] is not None and f.value[1] > 0: + if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: extra_constraints.append( cur.mogrify("s.duration <= %(maxDuration)s", {"maxDuration": f.value[1]})) ss_constraints.append( @@ -223,48 +241,69 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f elif filter_type == sessions_metas.meta_type.REFERRER: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" - op = __get_sql_operator_multiple(f.operator) + # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( - cur.mogrify(f"p.base_referrer {op} %(referrer)s", {"referrer": f.value})) + cur.mogrify(__multiple_conditions(f"p.base_referrer {op} %(value)s", f.value, is_not=is_not), + filter_args)) elif filter_type == events.event_type.METADATA.ui_type: - op = __get_sql_operator(f.operator) + # get metadata list only if you need it + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + # op = __get_sql_operator(f.operator) if f.key in meta_keys.keys(): extra_constraints.append( - cur.mogrify(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)})) + cur.mogrify( + __multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", + f.value, is_not=is_not), filter_args)) ss_constraints.append( - cur.mogrify(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)})) + cur.mogrify(__multiple_conditions( + f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", f.value, + is_not=is_not), + filter_args)) elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: - op = __get_sql_operator(f.operator) + # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(f"s.user_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify(__multiple_conditions(f"s.user_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) ss_constraints.append( - cur.mogrify(f"ms.user_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify(__multiple_conditions(f"ms.user_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, sessions_metas.meta_type.USERANONYMOUSID_IOS]: - op = __get_sql_operator(f.operator) + # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(f"s.user_anonymous_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify( + __multiple_conditions(f"s.user_anonymous_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) ss_constraints.append( - cur.mogrify(f"ms.user_anonymous_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify( + __multiple_conditions(f"ms.user_anonymous_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: - op = __get_sql_operator(f.operator) + # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(f"s.rev_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify(__multiple_conditions(f"s.rev_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) ss_constraints.append( - cur.mogrify(f"ms.rev_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify(__multiple_conditions(f"ms.rev_id {op} %(value)s", f.value, is_not=is_not), + filter_args) + ) + elif filter_type == schemas.FilterType.platform: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f"s.user_device_type {op} %(value)s", f.value, is_not=is_not), + filter_args) + ) + ss_constraints.append( + cur.mogrify( + __multiple_conditions(f"ms.user_device_type {op} %(value)s", f.value, is_not=is_not), + filter_args) ) # --------------------------------------------------------------------------- @@ -493,12 +532,12 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f else: data.endDate = None - if data.platform is not None: - if data.platform == schemas.PlatformType.mobile: - extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") - elif data.platform == schemas.PlatformType.desktop: - extra_constraints.append( - b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + # if data.platform is not None: + # if data.platform == schemas.PlatformType.mobile: + # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + # elif data.platform == schemas.PlatformType.desktop: + # extra_constraints.append( + # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") order = "DESC" if data.order is not None: diff --git a/api/chalicelib/core/sessions_metas.py b/api/chalicelib/core/sessions_metas.py index a21b78783..1e55720a9 100644 --- a/api/chalicelib/core/sessions_metas.py +++ b/api/chalicelib/core/sessions_metas.py @@ -1,3 +1,4 @@ +import schemas from chalicelib.utils import pg_client, helper from chalicelib.utils.event_filter_definition import SupportedFilter @@ -109,21 +110,21 @@ def __generic_autocomplete(typename): class meta_type: - USEROS = "USEROS" - USERBROWSER = "USERBROWSER" - USERDEVICE = "USERDEVICE" - USERCOUNTRY = "USERCOUNTRY" - USERID = "USERID" - USERANONYMOUSID = "USERANONYMOUSID" - REFERRER = "REFERRER" - REVID = "REVID" + USEROS = schemas.FilterType.user_os + USERBROWSER = schemas.FilterType.user_browser + USERDEVICE = schemas.FilterType.user_device + USERCOUNTRY = schemas.FilterType.user_country + USERID = schemas.FilterType.user_id + USERANONYMOUSID = schemas.FilterType.user_anonymous_id + REFERRER = schemas.FilterType.referrer + REVID = schemas.FilterType.rev_id # IOS - USEROS_IOS = "USEROS_IOS" - USERDEVICE_IOS = "USERDEVICE_IOS" - USERCOUNTRY_IOS = "USERCOUNTRY_IOS" - USERID_IOS = "USERID_IOS" - USERANONYMOUSID_IOS = "USERANONYMOUSID_IOS" - REVID_IOS = "REVID_IOS" + USEROS_IOS = schemas.FilterType.user_os_ios + USERDEVICE_IOS = schemas.FilterType.user_device_ios + USERCOUNTRY_IOS = schemas.FilterType.user_country_ios + USERID_IOS = schemas.FilterType.user_id_ios + USERANONYMOUSID_IOS = schemas.FilterType.user_anonymous_id_ios + REVID_IOS = schemas.FilterType.rev_id_ios SUPPORTED_TYPES = { diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index a9cd6afd3..868de0411 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -194,7 +194,7 @@ def string_to_sql_like_with_op(value, op): likable_operators = [schemas.SearchEventOperator._starts_with, schemas.SearchEventOperator._ends_with, - schemas.SearchEventOperator._contains, schemas.SearchEventOperator._notcontains] + schemas.SearchEventOperator._contains, schemas.SearchEventOperator._not_contains] def is_likable(op: schemas.SearchEventOperator): diff --git a/api/schemas.py b/api/schemas.py index b432dd0ac..69563b047 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -360,15 +360,37 @@ class EventType(str, Enum): error_ios = "ERROR_IOS" +class FilterType(str, Enum): + user_os = "USEROS" + user_browser = "USERBROWSER" + user_device = "USERDEVICE" + user_country = "USERCOUNTRY" + user_id = "USERID" + user_anonymous_id = "USERANONYMOUSID" + referrer = "REFERRER" + rev_id = "REVID" + # IOS + user_os_ios = "USEROS_IOS" + user_device_ios = "USERDEVICE_IOS" + user_country_ios = "USERCOUNTRY_IOS" + user_id_ios = "USERID_IOS" + user_anonymous_id_ios = "USERANONYMOUSID_IOS" + rev_id_ios = "REVID_IOS" + # + duration = "DURATION" + platform = "PLATFORM" + metadata = "METADATA" + + class SearchEventOperator(str, Enum): _is = "is" _is_any = "isAny" _on = "on" _on_any = "onAny" - _isnot = "isNot" - _noton = "notOn" + _is_not = "isNot" + _not_on = "notOn" _contains = "contains" - _notcontains = "notContains" + _not_contains = "notContains" _starts_with = "startsWith" _ends_with = "endsWith" @@ -393,8 +415,13 @@ class _SessionSearchEventSchema(BaseModel): source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) -class _SessionSearchFilterSchema(_SessionSearchEventSchema): - value: List[str] = Field(...) +class _SessionSearchFilterSchema(BaseModel): + custom: Optional[List[str]] = Field(None) + key: Optional[str] = Field(None) + value: Union[Optional[Union[str, int]], Optional[List[Union[str, int]]]] = Field(...) + type: FilterType = Field(...) + operator: SearchEventOperator = Field(...) + source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) class SessionsSearchPayloadSchema(BaseModel):