From 5673b8d8edbf0ef336391a2602d2fb233f1ce863 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 28 Feb 2022 21:36:23 +0100 Subject: [PATCH 1/3] feat(api): search by request_details 1/2 --- api/chalicelib/core/sessions.py | 54 +++++++++++-------- api/schemas.py | 42 +++++++++++++-- .../db/init_dbs/postgresql/1.5.X/1.5.X.sql | 38 +++++++++++++ 3 files changed, 110 insertions(+), 24 deletions(-) create mode 100644 scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 0beec7aae..74b7f7cfa 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -223,11 +223,11 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", # full_args) - # print("--------------------") - # print(main_query) - + print("--------------------") + print(main_query) + print("--------------------") cur.execute(main_query) - # print("--------------------") + if count_only: return helper.dict_to_camel_case(cur.fetchone()) sessions = cur.fetchone() @@ -319,9 +319,6 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d elif metric_of == schemas.MetricOfType.visited_url: main_col = "base_path" extra_col = ", base_path" - # elif metric_of == schemas.MetricOfType.initial_page_load_time: - # main_col = "issue" - # extra_col = f", UNNEST(s.issue_types) AS {main_col}" main_query = cur.mogrify(f"""{pre_query} SELECT COUNT(*) AS count, COALESCE(JSONB_AGG(users_sessions) FILTER ( WHERE rn <= 200 ), '[]'::JSONB) AS values FROM (SELECT {main_col} AS name, @@ -332,17 +329,15 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d s.user_id, s.user_os, s.user_browser, s.user_device, s.user_device_type, s.user_country, s.issue_types{extra_col} - ----------------- {query_part} - ----------------- ORDER BY s.session_id desc) AS filtred_sessions ) AS full_sessions GROUP BY {main_col} ORDER BY session_count DESC) AS users_sessions;""", full_args) - print("--------------------") - print(main_query) - print("--------------------") + # print("--------------------") + # print(main_query) + # print("--------------------") cur.execute(main_query) sessions = cur.fetchone() for s in sessions["values"]: @@ -788,15 +783,19 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] if event_index > 0 and not or_events: event_where.append("main2.session_id=event_0.session_id") - event_where.append( - _multiple_conditions( - f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", - event.value[0].value, value_key=e_k1)) + is_any = _isAny_opreator(event.value[0].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) s_op = __get_sql_operator(event.value[1].operator) - event_where.append( - _multiple_conditions( - f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", - event.value[1].value, value_key=e_k2)) + is_any = _isAny_opreator(event.value[1].operator) + if not is_any: + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) e_k += "_custom" full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} @@ -804,7 +803,20 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", event.source, value_key=e_k)) - + elif event_type==schemas.EventType.request_details: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if len(event.value[0].url_value)>0 and not _isAny_opreator(event.value[0].url_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].url_value,value_key=e_k)) + if len(event.value[0].status_code_value)>0 and not _isAny_opreator(event.value[0].status_code_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].status_code_value,value_key=e_k)) + if len(event.value[0].method_value)>0 and not _isAny_opreator(event.value[0].method_operator): + event_where.append(_multiple_conditions(f"main.method {op} %({e_k})s", event.value[0].method_value,value_key=e_k)) + if len(event.value[0].duration_value)>0 and not _isAny_opreator(event.value[0].duration_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].duration_value,value_key=e_k)) + if len(event.value[0].request_value)>0 and not _isAny_opreator(event.value[0].request_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].request_value,value_key=e_k)) + if len(event.value[0].response_value)>0 and not _isAny_opreator(event.value[0].response_operator): + event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].response_value,value_key=e_k)) else: continue if event_index == 0 or or_events: diff --git a/api/schemas.py b/api/schemas.py index b341a45fb..f35125ae5 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -384,6 +384,7 @@ class EventType(str, Enum): location = "LOCATION" custom = "CUSTOM" request = "REQUEST" + request_details = "REQUEST_DETAILS" graphql = "GRAPHQL" state_action = "STATEACTION" error = "ERROR" @@ -480,7 +481,8 @@ class __MixedSearchFilter(BaseModel): @root_validator(pre=True) def remove_duplicate_values(cls, values): if values.get("value") is not None: - if len(values["value"]) > 0 and isinstance(values["value"][0], int): + if len(values["value"]) > 0 \ + and (isinstance(values["value"][0], int) or isinstance(values["value"][0], dict)): return values values["value"] = list(set(values["value"])) return values @@ -489,6 +491,36 @@ class __MixedSearchFilter(BaseModel): alias_generator = attribute_to_camel_case +class HttpMethod(str, Enum): + _get = 'GET' + _head = 'HEAD' + _post = 'POST' + _put = 'PUT' + _delete = 'DELETE' + _connect = 'CONNECT' + _option = 'OPTIONS' + _trace = 'TRACE' + _patch = 'PATCH' + + +class RequestDetailsSchema(BaseModel): + url_value: List[str] = Field([]) + url_operator: SearchEventOperator = Field(SearchEventOperator._is_any) + status_code_value: List[int] = Field(None) + status_code_operator: MathOperator = Field(None) + method_value: List[HttpMethod] = Field([]) + method_operator: SearchEventOperator = Field(None) + duration_value: List[int] = Field(None) + duration_operator: MathOperator = Field(None) + request_value: List[str] = Field([]) + request_operator: SearchEventOperator = Field(None) + response_value: List[str] = Field([]) + response_operator: SearchEventOperator = Field(None) + + class Config: + alias_generator = attribute_to_camel_case + + class _SessionSearchEventRaw(__MixedSearchFilter): is_event: bool = Field(default=True, const=True) value: List[str] = Field(...) @@ -513,17 +545,22 @@ class _SessionSearchEventRaw(__MixedSearchFilter): assert isinstance(values["value"][0], _SessionSearchEventRaw) \ and isinstance(values["value"][1], _SessionSearchEventRaw), \ f"event should be of type _SessionSearchEventRaw for {PerformanceEventType.time_between_events}" + assert len(values["source"]) > 0 and isinstance(values["source"][0], int), \ + f"source of type int if required for {PerformanceEventType.time_between_events}" else: for c in values["source"]: assert isinstance(c, int), f"source value should be of type int for {values.get('type')}" elif values.get("type") == EventType.error and values.get("source") is None: values["source"] = [ErrorSource.js_exception] + elif values.get("type") == EventType.request_details: + assert len(values.get("value", [])) == 1 and isinstance(values["value"][0], RequestDetailsSchema), \ + f"event should be of type RequestDetailsSchema for {EventType.request_details}" return values class _SessionSearchEventSchema(_SessionSearchEventRaw): - value: Union[List[_SessionSearchEventRaw], str, List[str]] = Field(...) + value: Union[List[Union[RequestDetailsSchema, _SessionSearchEventRaw, str]], str] = Field(...) class _SessionSearchFilterSchema(__MixedSearchFilter): @@ -732,7 +769,6 @@ class MetricOfType(str, Enum): user_country = FilterType.user_country.value user_id = FilterType.user_id.value issues = FilterType.issue.value - initial_page_load_time = "INITIAL_PAGE_LOAD_TIME" visited_url = "VISITED_URL" diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql new file mode 100644 index 000000000..0b9a1d3b5 --- /dev/null +++ b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql @@ -0,0 +1,38 @@ +BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.5.X' +$$ LANGUAGE sql IMMUTABLE; + + +-- Split requests-URL: +UPDATE events_common.requests +SET schema=SUBSTRING(url, 1, POSITION('://' IN url) - 1), + host=CASE + WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( + SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, + POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) + ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, + base_path=CASE + WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN + CASE + WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN + SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), + POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1, + POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - + POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) + ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), + POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + + 1) END + ELSE '' END, + query_string=CASE + WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( + SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), + POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1) + END; + + + + +COMMIT; \ No newline at end of file From a9583db20f134a4c16e6c4c23fabedc026a4198e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 28 Feb 2022 22:07:35 +0100 Subject: [PATCH 2/3] feat(DB): changed update script --- .../helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql index 0b9a1d3b5..c8efda38c 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.X/1.5.X.sql @@ -1,3 +1,5 @@ +-- !!!!NOT IN CREATE DB YET + BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS @@ -8,13 +10,19 @@ $$ LANGUAGE sql IMMUTABLE; -- Split requests-URL: UPDATE events_common.requests -SET schema=SUBSTRING(url, 1, POSITION('://' IN url) - 1), +SET schema=CASE WHEN POSITION('://' IN url) > 0 THEN SUBSTRING(url, 1, POSITION('://' IN url) - 1) END, host=CASE + WHEN POSITION('://' IN url) = 0 THEN NULL WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), 1, POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) - 1) ELSE SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1) END, base_path=CASE + WHEN POSITION('://' IN url) = 0 THEN + CASE + WHEN POSITION('?' IN url) > 0 THEN + SUBSTRING(url, 1, POSITION('?' IN url) - 1) + ELSE url END WHEN POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN CASE WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN @@ -25,14 +33,11 @@ SET schema=SUBSTRING(url, 1, POSITION('://' IN url) - 1), ELSE SUBSTRING(SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), POSITION('/' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1) END - ELSE '' END, + END, query_string=CASE - WHEN POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) > 0 THEN SUBSTRING( - SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1), - POSITION('?' IN SUBSTRING(SUBSTRING(url, POSITION('://' IN url) + 3), 1)) + 1) + WHEN POSITION('?' IN url) > 0 THEN SUBSTRING(url, POSITION('?' IN url) + 1) END; - COMMIT; \ No newline at end of file From b587252bb54d905c9539c84d9907fd9da847a286 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 1 Mar 2022 16:31:07 +0100 Subject: [PATCH 3/3] feat(api): request changes --- api/chalicelib/core/sessions.py | 49 ++++++++++++++++++++++----------- api/schemas.py | 36 ++++++++++++------------ 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 74b7f7cfa..3c766b74c 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -567,7 +567,6 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr value_key=f_k)) # --------------------------------------------------------------------------- if len(data.events) > 0: - # ss_constraints = [s.decode('UTF-8') for s in ss_constraints] events_query_from = [] event_index = 0 or_events = data.events_order == schemas.SearchEventOrder._or @@ -578,13 +577,15 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr is_any = _isAny_opreator(event.operator) if not isinstance(event.value, list): event.value = [event.value] - if not is_any and len(event.value) == 0 \ + if not is_any and len(event.value) == 0 and event_type not in [schemas.EventType.request_details] \ or event_type in [schemas.PerformanceEventType.location_dom_complete, schemas.PerformanceEventType.location_largest_contentful_paint_time, schemas.PerformanceEventType.location_ttfb, schemas.PerformanceEventType.location_avg_cpu_load, schemas.PerformanceEventType.location_avg_memory_usage - ] and (event.source is None or len(event.source) == 0): + ] and (event.source is None or len(event.source) == 0) \ + or event_type in [schemas.EventType.request_details] and ( + event.filters is None or len(event.filters) == 0): continue op = __get_sql_operator(event.operator) is_not = False @@ -803,20 +804,36 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", event.source, value_key=e_k)) - elif event_type==schemas.EventType.request_details: + elif event_type == schemas.EventType.request_details: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - if len(event.value[0].url_value)>0 and not _isAny_opreator(event.value[0].url_operator): - event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].url_value,value_key=e_k)) - if len(event.value[0].status_code_value)>0 and not _isAny_opreator(event.value[0].status_code_operator): - event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].status_code_value,value_key=e_k)) - if len(event.value[0].method_value)>0 and not _isAny_opreator(event.value[0].method_operator): - event_where.append(_multiple_conditions(f"main.method {op} %({e_k})s", event.value[0].method_value,value_key=e_k)) - if len(event.value[0].duration_value)>0 and not _isAny_opreator(event.value[0].duration_operator): - event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].duration_value,value_key=e_k)) - if len(event.value[0].request_value)>0 and not _isAny_opreator(event.value[0].request_operator): - event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].request_value,value_key=e_k)) - if len(event.value[0].response_value)>0 and not _isAny_opreator(event.value[0].response_operator): - event_where.append(_multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value[0].response_value,value_key=e_k)) + for j, f in enumerate(event.filters): + is_any = _isAny_opreator(f.operator) + if is_any or len(f.value) == 0: + continue + op = __get_sql_operator(f.operator) + e_k_f = e_k + f"_fetch{j}" + full_args = {**full_args, **_multiple_values(f.value, value_key=e_k_f)} + if f.type == schemas.FetchFilterType._url: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k_f})s", f.value, + value_key=e_k_f)) + elif f.type == schemas.FetchFilterType._status_code: + event_where.append( + _multiple_conditions(f"main.status_code {op} %({e_k_f})s", f.value, value_key=e_k_f)) + elif f.type == schemas.FetchFilterType._method: + event_where.append( + _multiple_conditions(f"main.method {op} %({e_k_f})s", f.value, value_key=e_k_f)) + elif f.type == schemas.FetchFilterType._duration: + event_where.append( + _multiple_conditions(f"main.duration {op} %({e_k_f})s", f.value, value_key=e_k_f)) + elif f.type == schemas.FetchFilterType._request_body: + event_where.append( + _multiple_conditions(f"main.request_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + elif f.type == schemas.FetchFilterType._response_body: + event_where.append( + _multiple_conditions(f"main.response_body {op} %({e_k_f})s", f.value, value_key=e_k_f)) + else: + print(f"undefined fetch filter: {f.type}") else: continue if event_index == 0 or or_events: diff --git a/api/schemas.py b/api/schemas.py index f35125ae5..6e59bdafd 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -384,7 +384,7 @@ class EventType(str, Enum): location = "LOCATION" custom = "CUSTOM" request = "REQUEST" - request_details = "REQUEST_DETAILS" + request_details = "FETCH" graphql = "GRAPHQL" state_action = "STATEACTION" error = "ERROR" @@ -503,22 +503,19 @@ class HttpMethod(str, Enum): _patch = 'PATCH' -class RequestDetailsSchema(BaseModel): - url_value: List[str] = Field([]) - url_operator: SearchEventOperator = Field(SearchEventOperator._is_any) - status_code_value: List[int] = Field(None) - status_code_operator: MathOperator = Field(None) - method_value: List[HttpMethod] = Field([]) - method_operator: SearchEventOperator = Field(None) - duration_value: List[int] = Field(None) - duration_operator: MathOperator = Field(None) - request_value: List[str] = Field([]) - request_operator: SearchEventOperator = Field(None) - response_value: List[str] = Field([]) - response_operator: SearchEventOperator = Field(None) +class FetchFilterType(str, Enum): + _url = "FETCH_URL" + _status_code = "FETCH_STATUS_CODE" + _method = "FETCH_METHOD" + _duration = "FETCH_DURATION" + _request_body = "FETCH_REQUEST_BODY" + _response_body = "FETCH_RESPONSE_BODY" - class Config: - alias_generator = attribute_to_camel_case + +class RequestFilterSchema(BaseModel): + type: FetchFilterType = Field(...) + value: List[Union[int, str]] = Field(...) + operator: Union[SearchEventOperator, MathOperator] = Field(...) class _SessionSearchEventRaw(__MixedSearchFilter): @@ -528,6 +525,7 @@ class _SessionSearchEventRaw(__MixedSearchFilter): operator: SearchEventOperator = Field(...) source: Optional[List[Union[ErrorSource, int, str]]] = Field(None) sourceOperator: Optional[MathOperator] = Field(None) + filters: Optional[List[RequestFilterSchema]] = Field(None) @root_validator def event_validator(cls, values): @@ -553,14 +551,14 @@ class _SessionSearchEventRaw(__MixedSearchFilter): elif values.get("type") == EventType.error and values.get("source") is None: values["source"] = [ErrorSource.js_exception] elif values.get("type") == EventType.request_details: - assert len(values.get("value", [])) == 1 and isinstance(values["value"][0], RequestDetailsSchema), \ - f"event should be of type RequestDetailsSchema for {EventType.request_details}" + assert isinstance(values.get("filters"), List) and len(values.get("filters", [])) > 0, \ + f"filters should be defined for {EventType.request_details}" return values class _SessionSearchEventSchema(_SessionSearchEventRaw): - value: Union[List[Union[RequestDetailsSchema, _SessionSearchEventRaw, str]], str] = Field(...) + value: Union[List[Union[_SessionSearchEventRaw, str]], str] = Field(...) class _SessionSearchFilterSchema(__MixedSearchFilter):