From 5673b8d8edbf0ef336391a2602d2fb233f1ce863 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 28 Feb 2022 21:36:23 +0100 Subject: [PATCH] feat(api): search by request_details 1/2 --- api/chalicelib/core/sessions.py | 54 +++++++++++-------- api/schemas.py | 42 +++++++++++++-- .../db/init_dbs/postgresql/1.5.X/1.5.X.sql | 38 +++++++++++++ 3 files changed, 110 insertions(+), 24 deletions(-) create mode 100644 scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 0beec7aae..74b7f7cfa 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -223,11 +223,11 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", # full_args) - # print("--------------------") - # print(main_query) - + print("--------------------") + print(main_query) + print("--------------------") cur.execute(main_query) - # print("--------------------") + if count_only: return helper.dict_to_camel_case(cur.fetchone()) sessions = cur.fetchone() @@ -319,9 +319,6 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d elif metric_of == schemas.MetricOfType.visited_url: main_col = "base_path" extra_col = ", base_path" - # elif metric_of == schemas.MetricOfType.initial_page_load_time: - # main_col = "issue" - # extra_col = f", UNNEST(s.issue_types) AS {main_col}" main_query = cur.mogrify(f"""{pre_query} SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values FROM (SELECT {main_col} AS name, @@ -332,17 +329,15 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d s.user_id, s.user_os, s.user_browser, s.user_device, s.user_device_type, s.user_country, s.issue_types{extra_col} - ----------------- {query_part} - ----------------- ORDER BY s.session_id desc) AS filtred_sessions ) AS full_sessions GROUP BY {main_col} ORDER BY session_count DESC) AS users_sessions;""", full_args) - print("--------------------") - print(main_query) - print("--------------------") + # print("--------------------") + # print(main_query) + # print("--------------------") cur.execute(main_query) sessions = cur.fetchone() for s in sessions["values"]: @@ -788,15 +783,19 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] if event_index > 0 and not or_events: event_where.append("main2.session_id=event_0.session_id") - event_where.append( - _multiple_conditions( - f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", - event.value[0].value, value_key=e_k1)) + is_any = _isAny_opreator(event.value[0].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) s_op = __get_sql_operator(event.value[1].operator) - event_where.append( - _multiple_conditions( - f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", - event.value[1].value, value_key=e_k2)) + is_any = _isAny_opreator(event.value[1].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) e_k += "_custom" full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} @@ -804,7 +803,20 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", event.source, value_key=e_k)) - + elif event_type==schemas.EventType.request_details: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if len(event.value[0].url_value)>0 and not _isAny_opreator(event.value[0].url_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].url_value,value_key=e_k)) + if len(event.value[0].status_code_value)>0 and not _isAny_opreator(event.value[0].status_code_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].status_code_value,value_key=e_k)) + if len(event.value[0].method_value)>0 and not _isAny_opreator(event.value[0].method_operator): + event_where.append(_multiple_conditions(f"main.method {op} %({e_k})s", event.value[0].method_value,value_key=e_k)) + if len(event.value[0].duration_value)>0 and not _isAny_opreator(event.value[0].duration_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].duration_value,value_key=e_k)) + if len(event.value[0].request_value)>0 and not _isAny_opreator(event.value[0].request_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].request_value,value_key=e_k)) + if len(event.value[0].response_value)>0 and not _isAny_opreator(event.value[0].response_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].response_value,value_key=e_k)) else: continue if event_index == 0 or or_events: diff --git a/api/schemas.py b/api/schemas.py index b341a45fb..f35125ae5 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -384,6 +384,7 @@ class EventType(str, Enum): location = "LOCATION" custom = "CUSTOM" request = "REQUEST" + request_details = "REQUEST_DETAILS" graphql = "GRAPHQL" state_action = "STATEACTION" error = "ERROR" @@ -480,7 +481,8 @@ class __MixedSearchFilter(BaseModel): @root_validator(pre=True) def remove_duplicate_values(cls, values): if values.get("value") is not None: - if len(values["value"]) > 0 and isinstance(values["value"][0], int): + if len(values["value"]) > 0 \ + and (isinstance(values["value"][0], int) or isinstance(values["value"][0], dict)): return values values["value"] = list(set(values["value"])) return values @@ -489,6 +491,36 @@ class __MixedSearchFilter(BaseModel): alias_generator = attribute_to_camel_case +class HttpMethod(str, Enum): + _get = 'GET' + _head = 'HEAD' + _post = 'POST' + _put = 'PUT' + _delete = 'DELETE' + _connect = 'CONNECT' + _option = 'OPTIONS' + _trace = 'TRACE' + _patch = 'PATCH' + + +class RequestDetailsSchema(BaseModel): + url_value: List[str] = Field([]) + url_operator: SearchEventOperator = Field(SearchEventOperator._is_any) + status_code_value: List[int] = Field(None) + status_code_operator: MathOperator = Field(None) + method_value: List[HttpMethod] = Field([]) + method_operator: SearchEventOperator = Field(None) + duration_value: List[int] = Field(None) + duration_operator: MathOperator = Field(None) + request_value: List[str] = Field([]) + request_operator: SearchEventOperator = Field(None) + response_value: List[str] = Field([]) + response_operator: SearchEventOperator = Field(None) + + class Config: + alias_generator = attribute_to_camel_case + + class _SessionSearchEventRaw(__MixedSearchFilter): is_event: bool = Field(default=True, const=True) value: List[str] = Field(...) @@ -513,17 +545,22 @@ class _SessionSearchEventRaw(__MixedSearchFilter): assert isinstance(values["value"][0], _SessionSearchEventRaw) \ and isinstance(values["value"][1], _SessionSearchEventRaw), \ f"event should be of type _SessionSearchEventRaw for {PerformanceEventType.time_between_events}" + assert len(values["source"]) > 0 and isinstance(values["source"][0], int), \ + f"source of type int if required for {PerformanceEventType.time_between_events}" else: for c in values["source"]: assert isinstance(c, int), f"source value should be of type int for {values.get('type')}" elif values.get("type") == EventType.error and values.get("source") is None: values["source"] = [ErrorSource.js_exception] + elif values.get("type") == EventType.request_details: + assert len(values.get("value", [])) == 1 and isinstance(values["value"][0], RequestDetailsSchema), \ + f"event should be of type RequestDetailsSchema for {EventType.request_details}" return values class _SessionSearchEventSchema(_SessionSearchEventRaw): - value: Union[List[_SessionSearchEventRaw], str, List[str]] = Field(...) + value: Union[List[Union[RequestDetailsSchema, _SessionSearchEventRaw, str]], str] = Field(...) class _SessionSearchFilterSchema(__MixedSearchFilter): @@ -732,7 +769,6 @@ class MetricOfType(str, Enum): user_country = FilterType.user_country.value user_id = FilterType.user_id.value issues = FilterType.issue.value - initial_page_load_time = "INITIAL_PAGE_LOAD_TIME" visited_url = "VISITED_URL" diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql new file mode 100644 index 000000000..0b9a1d3b5 --- /dev/null +++ b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql @@ -0,0 +1,38 @@ +BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.5.X' +$$ LANGUAGE sql IMMUTABLE; + + +-- Split requests-URL: +UPDATE events_common.requests +SET schema=SUBSTRING(url, 1, POSITION('://' IN url) - 1), + host=CASE + WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( + SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, + POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) + ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, + base_path=CASE + WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN + CASE + WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN + SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), + POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1, + POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - + POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) + ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), + POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + + 1) END + ELSE '' END, + query_string=CASE + WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( + SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), + POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1) + END; + + + + +COMMIT; \ No newline at end of file