openreplay/api/chalicelib/core/events.py

import schemas
from chalicelib.core import issues
from chalicelib.core import sessions_metas, metadata
from chalicelib.utils import pg_client, helper
from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.utils.event_filter_definition import SupportedFilter, Event


def get_customs_by_sessionId2_pg(session_id, project_id):
    with pg_client.PostgresClient() as cur:
        cur.execute(cur.mogrify("""\
            SELECT 
                c.*,
                'CUSTOM' AS type
            FROM events_common.customs AS c
            WHERE 
              c.session_id = %(session_id)s
            ORDER BY c.timestamp;""",
                                {"project_id": project_id, "session_id": session_id})
                    )
        rows = cur.fetchall()
    return helper.dict_to_camel_case(rows)


def __merge_cells(rows, start, count, replacement):
    rows[start] = replacement
    rows = rows[:start + 1] + rows[start + count:]
    return rows


def __get_grouped_clickrage(rows, session_id, project_id):
    click_rage_issues = issues.get_by_session_id(session_id=session_id, issue_type="click_rage", project_id=project_id)
    if len(click_rage_issues) == 0:
        return rows

    for c in click_rage_issues:
        merge_count = c.get("payload")
        if merge_count is not None:
            merge_count = merge_count.get("count", 3)
        else:
            merge_count = 3
        for i in range(len(rows)):
            if rows[i]["timestamp"] == c["timestamp"]:
                rows = __merge_cells(rows=rows,
                                     start=i,
                                     count=merge_count,
                                     replacement={**rows[i], "type": "CLICKRAGE", "count": merge_count})
                break
    return rows


def get_by_sessionId2_pg(session_id, project_id, group_clickrage=False):
    with pg_client.PostgresClient() as cur:
        cur.execute(cur.mogrify("""\
            SELECT 
                c.*,
                'CLICK' AS type
            FROM events.clicks AS c
            WHERE 
              c.session_id = %(session_id)s
            ORDER BY c.timestamp;""",
                                {"project_id": project_id, "session_id": session_id})
                    )
        rows = cur.fetchall()
        if group_clickrage:
            rows = __get_grouped_clickrage(rows=rows, session_id=session_id, project_id=project_id)

        cur.execute(cur.mogrify("""
            SELECT 
                i.*,
                'INPUT' AS type
            FROM events.inputs AS i
            WHERE 
              i.session_id = %(session_id)s
            ORDER BY i.timestamp;""",
                                {"project_id": project_id, "session_id": session_id})
                    )
        rows += cur.fetchall()
        cur.execute(cur.mogrify("""\
            SELECT 
                l.*,
                l.path AS value,
                l.path AS url,
                'LOCATION' AS type
            FROM events.pages AS l
            WHERE 
              l.session_id = %(session_id)s
            ORDER BY l.timestamp;""", {"project_id": project_id, "session_id": session_id}))
        rows += cur.fetchall()
        rows = helper.list_to_camel_case(rows)
        rows = sorted(rows, key=lambda k: (k["timestamp"], k["messageId"]))
    return rows


def __get_data_for_extend(data):
    if "errors" not in data:
        return data["data"]


def __pg_errors_query(source=None, value_length=None):
    if value_length is None or value_length > 2:
        return f"""((SELECT DISTINCT ON(lg.message)
                        lg.message AS value,
                        source,
                        '{event_type.ERROR.ui_type}' AS type
                    FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id)
                    WHERE
                      s.project_id = %(project_id)s
                      AND lg.message ILIKE %(svalue)s
                      AND lg.project_id = %(project_id)s
                      {"AND source = %(source)s" if source is not None else ""}
                    LIMIT 5)
                    UNION ALL
                    (SELECT DISTINCT ON(lg.name)
                        lg.name AS value,
                        source,
                        '{event_type.ERROR.ui_type}' AS type
                    FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id)
                    WHERE
                      s.project_id = %(project_id)s
                      AND lg.name ILIKE %(svalue)s
                      AND lg.project_id = %(project_id)s
                      {"AND source = %(source)s" if source is not None else ""}
                    LIMIT 5)
                    UNION
                    (SELECT DISTINCT ON(lg.message)
                        lg.message AS value,
                        source,
                        '{event_type.ERROR.ui_type}' AS type
                    FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id)
                    WHERE
                      s.project_id = %(project_id)s
                      AND lg.message ILIKE %(value)s
                      AND lg.project_id = %(project_id)s
                      {"AND source = %(source)s" if source is not None else ""}
                    LIMIT 5)
                    UNION ALL
                    (SELECT DISTINCT ON(lg.name)
                        lg.name AS value,
                        source,
                        '{event_type.ERROR.ui_type}' AS type
                    FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id)
                    WHERE
                      s.project_id = %(project_id)s
                      AND lg.name ILIKE %(value)s
                      AND lg.project_id = %(project_id)s
                      {"AND source = %(source)s" if source is not None else ""}
                    LIMIT 5));"""
    return f"""((SELECT DISTINCT ON(lg.message)
                    lg.message AS value,
                    source,
                    '{event_type.ERROR.ui_type}' AS type
                FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id)
                WHERE
                  s.project_id = %(project_id)s
                  AND lg.message ILIKE %(svalue)s
                  AND lg.project_id = %(project_id)s
                  {"AND source = %(source)s" if source is not None else ""}
                LIMIT 5)
                UNION ALL
                (SELECT DISTINCT ON(lg.name)
                    lg.name AS value,
                    source,
                    '{event_type.ERROR.ui_type}' AS type
                FROM {event_type.ERROR.table} INNER JOIN public.errors AS lg USING (error_id) LEFT JOIN public.sessions AS s USING(session_id)
                WHERE
                  s.project_id = %(project_id)s
                  AND lg.name ILIKE %(svalue)s
                  AND lg.project_id = %(project_id)s
                  {"AND source = %(source)s" if source is not None else ""}
                LIMIT 5));"""


def __search_pg_errors(project_id, value, key=None, source=None):
    now = TimeUTC.now()

    with pg_client.PostgresClient() as cur:
        cur.execute(
            cur.mogrify(__pg_errors_query(source,
                                          value_length=len(value) \
                                              if SUPPORTED_TYPES[event_type.ERROR.ui_type].change_by_length else None),
                        {"project_id": project_id, "value": helper.string_to_sql_like(value),
                         "svalue": helper.string_to_sql_like("^" + value),
                         "source": source}))
        results = helper.list_to_camel_case(cur.fetchall())
    print(f"{TimeUTC.now() - now} : errors")
    return results


def __search_pg_errors_ios(project_id, value, key=None, source=None):
    now = TimeUTC.now()
    if SUPPORTED_TYPES[event_type.ERROR_IOS.ui_type].change_by_length is False or len(value) > 2:
        query = f"""(SELECT DISTINCT ON(lg.reason)
                        lg.reason AS value,
                        '{event_type.ERROR_IOS.ui_type}' AS type
                    FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id)
                    WHERE
                      s.project_id = %(project_id)s
                      AND lg.project_id = %(project_id)s
                      AND lg.reason ILIKE %(svalue)s
                    LIMIT 5)
                    UNION ALL
                    (SELECT DISTINCT ON(lg.name)
                        lg.name AS value,
                        '{event_type.ERROR_IOS.ui_type}' AS type
                    FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id)
                    WHERE
                      s.project_id = %(project_id)s
                      AND lg.project_id = %(project_id)s
                      AND lg.name ILIKE %(svalue)s
                    LIMIT 5)
                    UNION ALL
                    (SELECT DISTINCT ON(lg.reason)
                        lg.reason AS value,
                        '{event_type.ERROR_IOS.ui_type}' AS type
                    FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id)
                    WHERE
                      s.project_id = %(project_id)s
                      AND lg.project_id = %(project_id)s
                      AND lg.reason ILIKE %(value)s
                    LIMIT 5)
                    UNION ALL
                    (SELECT DISTINCT ON(lg.name)
                        lg.name AS value,
                        '{event_type.ERROR_IOS.ui_type}' AS type
                    FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id)
                    WHERE
                      s.project_id = %(project_id)s
                      AND lg.project_id = %(project_id)s
                      AND lg.name ILIKE %(value)s
                    LIMIT 5);"""
    else:
        query = f"""(SELECT DISTINCT ON(lg.reason)
                            lg.reason AS value,
                            '{event_type.ERROR_IOS.ui_type}' AS type
                        FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id)
                        WHERE
                          s.project_id = %(project_id)s
                          AND lg.project_id = %(project_id)s
                          AND lg.reason ILIKE %(svalue)s
                        LIMIT 5)
                        UNION ALL
                        (SELECT DISTINCT ON(lg.name)
                            lg.name AS value,
                            '{event_type.ERROR_IOS.ui_type}' AS type
                        FROM {event_type.ERROR_IOS.table} INNER JOIN public.crashes_ios AS lg USING (crash_id) LEFT JOIN public.sessions AS s USING(session_id)
                        WHERE
                          s.project_id = %(project_id)s
                          AND lg.project_id = %(project_id)s
                          AND lg.name ILIKE %(svalue)s
                        LIMIT 5);"""
    with pg_client.PostgresClient() as cur:
        cur.execute(cur.mogrify(query, {"project_id": project_id, "value": helper.string_to_sql_like(value),
                                        "svalue": helper.string_to_sql_like("^" + value)}))
        results = helper.list_to_camel_case(cur.fetchall())
    print(f"{TimeUTC.now() - now} : errors")
    return results


def __search_pg_metadata(project_id, value, key=None, source=None):
    meta_keys = metadata.get(project_id=project_id)
    meta_keys = {m["key"]: m["index"] for m in meta_keys}
    if len(meta_keys) == 0 or key is not None and key not in meta_keys.keys():
        return []
    sub_from = []
    if key is not None:
        meta_keys = {key: meta_keys[key]}

    for k in meta_keys.keys():
        colname = metadata.index_to_colname(meta_keys[k])
        if SUPPORTED_TYPES[event_type.METADATA.ui_type].change_by_length is False or len(value) > 2:
            sub_from.append(f"""((SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key 
                                FROM public.sessions 
                                WHERE project_id = %(project_id)s 
                                AND {colname} ILIKE %(svalue)s LIMIT 5)
                                UNION
                                (SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key 
                                FROM public.sessions 
                                WHERE project_id = %(project_id)s 
                                AND {colname} ILIKE %(value)s LIMIT 5))
                                """)
        else:
            sub_from.append(f"""(SELECT DISTINCT ON ({colname}) {colname} AS value, '{k}' AS key 
                                FROM public.sessions 
                                WHERE project_id = %(project_id)s 
                                AND {colname} ILIKE %(svalue)s LIMIT 5)""")
    with pg_client.PostgresClient() as cur:
        cur.execute(cur.mogrify(f"""\
                    SELECT key, value, 'METADATA' AS TYPE
                    FROM({" UNION ALL ".join(sub_from)}) AS all_metas
                    LIMIT 5;""", {"project_id": project_id, "value": helper.string_to_sql_like(value),
                                  "svalue": helper.string_to_sql_like("^" + value)}))
        results = helper.list_to_camel_case(cur.fetchall())
    return results


def __generic_query(typename, value_length=None):
    if value_length is None or value_length > 2:
        return f"""(SELECT DISTINCT value, type
                    FROM public.autocomplete
                    WHERE
                      project_id = %(project_id)s
                      AND type='{typename}'
                      AND value ILIKE %(svalue)s
                    LIMIT 5)
                    UNION
                    (SELECT DISTINCT value, type
                    FROM public.autocomplete
                    WHERE
                      project_id = %(project_id)s
                      AND type='{typename}'
                      AND value ILIKE %(value)s
                    LIMIT 5);"""
    return f"""SELECT DISTINCT value, type
                FROM public.autocomplete
                WHERE
                  project_id = %(project_id)s
                  AND type='{typename}'
                  AND value ILIKE %(svalue)s
                LIMIT 10;"""


def __generic_autocomplete(event: Event):
    def f(project_id, value, key=None, source=None):
        with pg_client.PostgresClient() as cur:
            cur.execute(
                cur.mogrify(
                    __generic_query(event.ui_type,
                                    value_length=len(value) \
                                        if SUPPORTED_TYPES[event.ui_type].change_by_length \
                                        else None),
                    {"project_id": project_id, "value": helper.string_to_sql_like(value),
                     "svalue": helper.string_to_sql_like("^" + value)}))
            return helper.list_to_camel_case(cur.fetchall())

    return f


class event_type:
    CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label")
    INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label")
    LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="path")
    CUSTOM = Event(ui_type=schemas.EventType.custom, table="events_common.customs", column="name")
    REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="path")
    GRAPHQL = Event(ui_type=schemas.EventType.graphql, table="events.graphql", column="name")
    STATEACTION = Event(ui_type=schemas.EventType.state_action, table="events.state_actions", column="name")
    ERROR = Event(ui_type=schemas.EventType.error, table="events.errors",
                  column=None)  # column=None because errors are searched by name or message
    METADATA = Event(ui_type=schemas.FilterType.metadata, table="public.sessions", column=None)
    #     IOS
    CLICK_IOS = Event(ui_type=schemas.EventType.click_ios, table="events_ios.clicks", column="label")
    INPUT_IOS = Event(ui_type=schemas.EventType.input_ios, table="events_ios.inputs", column="label")
    VIEW_IOS = Event(ui_type=schemas.EventType.view_ios, table="events_ios.views", column="name")
    CUSTOM_IOS = Event(ui_type=schemas.EventType.custom_ios, table="events_common.customs", column="name")
    REQUEST_IOS = Event(ui_type=schemas.EventType.request_ios, table="events_common.requests", column="url")
    ERROR_IOS = Event(ui_type=schemas.EventType.error_ios, table="events_ios.crashes",
                      column=None)  # column=None because errors are searched by name or message


SUPPORTED_TYPES = {
    event_type.CLICK.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CLICK),
                                              query=__generic_query(typename=event_type.CLICK.ui_type),
                                              change_by_length=True),
    event_type.INPUT.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.INPUT),
                                              query=__generic_query(typename=event_type.INPUT.ui_type),
                                              change_by_length=True),
    event_type.LOCATION.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.LOCATION),
                                                 query=__generic_query(typename=event_type.LOCATION.ui_type),
                                                 change_by_length=True),
    event_type.CUSTOM.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CUSTOM),
                                               query=__generic_query(typename=event_type.CUSTOM.ui_type),
                                               change_by_length=True),
    event_type.REQUEST.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.REQUEST),
                                                query=__generic_query(typename=event_type.REQUEST.ui_type),
                                                change_by_length=True),
    event_type.GRAPHQL.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.GRAPHQL),
                                                query=__generic_query(typename=event_type.GRAPHQL.ui_type),
                                                change_by_length=True),
    event_type.STATEACTION.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.STATEACTION),
                                                    query=__generic_query(typename=event_type.STATEACTION.ui_type),
                                                    change_by_length=True),
    event_type.ERROR.ui_type: SupportedFilter(get=__search_pg_errors,
                                              query=None, change_by_length=True),
    event_type.METADATA.ui_type: SupportedFilter(get=__search_pg_metadata,
                                                 query=None, change_by_length=True),
    #     IOS
    event_type.CLICK_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CLICK_IOS),
                                                  query=__generic_query(typename=event_type.CLICK_IOS.ui_type),
                                                  change_by_length=True),
    event_type.INPUT_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.INPUT_IOS),
                                                  query=__generic_query(typename=event_type.INPUT_IOS.ui_type),
                                                  change_by_length=True),
    event_type.VIEW_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.VIEW_IOS),
                                                 query=__generic_query(typename=event_type.VIEW_IOS.ui_type),
                                                 change_by_length=True),
    event_type.CUSTOM_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.CUSTOM_IOS),
                                                   query=__generic_query(typename=event_type.CUSTOM_IOS.ui_type),
                                                   change_by_length=True),
    event_type.REQUEST_IOS.ui_type: SupportedFilter(get=__generic_autocomplete(event_type.REQUEST_IOS),
                                                    query=__generic_query(typename=event_type.REQUEST_IOS.ui_type),
                                                    change_by_length=True),
    event_type.ERROR_IOS.ui_type: SupportedFilter(get=__search_pg_errors_ios,
                                                  query=None, change_by_length=True),
}


def __get_autocomplete_table(value, project_id):
    autocomplete_events = [schemas.FilterType.rev_id,
                           schemas.EventType.click,
                           schemas.FilterType.user_device,
                           schemas.FilterType.user_id,
                           schemas.FilterType.user_browser,
                           schemas.FilterType.user_os,
                           schemas.EventType.custom,
                           schemas.FilterType.user_country,
                           schemas.EventType.location,
                           schemas.EventType.input]
    autocomplete_events.sort()
    sub_queries = []
    for e in autocomplete_events:
        sub_queries.append(f"""(SELECT type, value
                                FROM public.autocomplete
                                WHERE project_id = %(project_id)s
                                    AND type= '{e}' 
                                    AND value ILIKE %(svalue)s
                                LIMIT 5)""")
        if len(value) > 2:
            sub_queries.append(f"""(SELECT type, value
                                    FROM public.autocomplete
                                    WHERE project_id = %(project_id)s
                                        AND type= '{e}' 
                                        AND value ILIKE %(value)s
                                    LIMIT 5)""")
    with pg_client.PostgresClient() as cur:
        query = cur.mogrify(" UNION ".join(sub_queries) + ";",
                            {"project_id": project_id, "value": helper.string_to_sql_like(value),
                             "svalue": helper.string_to_sql_like("^" + value)})
        try:
            cur.execute(query)
        except Exception as err:
            print("--------- AUTOCOMPLETE SEARCH QUERY EXCEPTION -----------")
            print(query.decode('UTF-8'))
            print("--------- VALUE -----------")
            print(value)
            print("--------------------")
            raise err
        results = helper.list_to_camel_case(cur.fetchall())
        return results


def search(text, event_type, project_id, source, key):
    if not event_type:
        return {"data": __get_autocomplete_table(text, project_id)}

    if event_type in SUPPORTED_TYPES.keys():
        rows = SUPPORTED_TYPES[event_type].get(project_id=project_id, value=text, key=key, source=source)
        # for IOS events autocomplete
        # if event_type + "_IOS" in SUPPORTED_TYPES.keys():
        #     rows += SUPPORTED_TYPES[event_type + "_IOS"].get(project_id=project_id, value=text, key=key,
        #                                                      source=source)
    elif event_type + "_IOS" in SUPPORTED_TYPES.keys():
        rows = SUPPORTED_TYPES[event_type + "_IOS"].get(project_id=project_id, value=text, key=key,
                                                        source=source)
    elif event_type in sessions_metas.SUPPORTED_TYPES.keys():
        return sessions_metas.search(text, event_type, project_id)
    elif event_type.endswith("_IOS") \
            and event_type[:-len("_IOS")] in sessions_metas.SUPPORTED_TYPES.keys():
        return sessions_metas.search(text, event_type, project_id)
    else:
        return {"errors": ["unsupported event"]}

    return {"data": rows}


def get_errors_by_session_id(session_id, project_id):
    with pg_client.PostgresClient() as cur:
        cur.execute(cur.mogrify(f"""\
                    SELECT er.*,ur.*, er.timestamp - s.start_ts AS time
                    FROM {event_type.ERROR.table} AS er INNER JOIN public.errors AS ur USING (error_id) INNER JOIN public.sessions AS s USING (session_id)
                    WHERE er.session_id = %(session_id)s AND s.project_id=%(project_id)s
                    ORDER BY timestamp;""", {"session_id": session_id, "project_id": project_id}))
        errors = cur.fetchall()
        for e in errors:
            e["stacktrace_parsed_at"] = TimeUTC.datetime_to_timestamp(e["stacktrace_parsed_at"])
        return helper.list_to_camel_case(errors)