From e7f061c439004956dc2f6a21f29fe793c43855b4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 23 Aug 2022 18:38:27 +0100 Subject: [PATCH] feat(chalice): CH max query execution time --- ee/api/chalicelib/core/metrics.py | 5 +---- ee/api/chalicelib/core/metrics_new.py | 9 +++------ ee/api/chalicelib/utils/ch_client.py | 7 ++++++- ee/api/env.default | 1 + 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index 0036ffc34..d0a0f6a42 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -1870,10 +1870,7 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e return __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"4xx": 0, - "5xx": 0, - "js": 0, - "integrations": 0}) + neutral={"4xx": 0, "5xx": 0, "js": 0, "integrations": 0}) def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_days=-1), diff --git a/ee/api/chalicelib/core/metrics_new.py b/ee/api/chalicelib/core/metrics_new.py index 7ef511d0c..6482dff39 100644 --- a/ee/api/chalicelib/core/metrics_new.py +++ b/ee/api/chalicelib/core/metrics_new.py @@ -1636,7 +1636,6 @@ def get_domains_errors_4xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), density=6, **args): - raise Exception("not supported widget") step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="resources", round_start=True, data=args) ch_sub_query.append("intDiv(resources.status, 100) == %(status_code)s") @@ -1648,7 +1647,7 @@ def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) groupArray([domain, toString(count)]) AS keys FROM (SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp, resources.url_host AS domain, COUNT(resources.session_id) AS count - FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} + FROM o_resources AS resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} GROUP BY timestamp,resources.url_host ORDER BY timestamp, count DESC @@ -1659,6 +1658,7 @@ def get_domains_errors_5xx(project_id, startTimestamp=TimeUTC.now(delta_days=-1) "endTimestamp": endTimestamp, "step_size": step_size, "status_code": 5, **__get_constraint_values(args)} + print(ch.execute(query=ch_query, params=params)) rows = ch.execute(query=ch_query, params=params) rows = __nested_array_to_dict_array(rows) neutral = __get_domains_errors_neutral(rows) @@ -1874,10 +1874,7 @@ def get_errors_per_type(project_id, startTimestamp=TimeUTC.now(delta_days=-1), e return __complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, density=density, - neutral={"4xx": 0, - "5xx": 0, - "js": 0, - "integrations": 0}) + neutral={"4xx": 0, "5xx": 0, "js": 0, "integrations": 0}) def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_days=-1), diff --git a/ee/api/chalicelib/utils/ch_client.py b/ee/api/chalicelib/utils/ch_client.py index a51230a19..75e1dc063 100644 --- a/ee/api/chalicelib/utils/ch_client.py +++ b/ee/api/chalicelib/utils/ch_client.py @@ -1,6 +1,10 @@ import clickhouse_driver from decouple import config +settings = None +if config('pg_timeout', cast=int, default=-1) <= 0: + settings = {"max_execution_time": config('pg_timeout', cast=int)} + class ClickHouseClient: __client = None @@ -8,7 +12,8 @@ class ClickHouseClient: def __init__(self): self.__client = clickhouse_driver.Client(host=config("ch_host"), database="default", - port=config("ch_port", cast=int)) \ + port=config("ch_port", cast=int), + settings=settings) \ if self.__client is None else self.__client def __enter__(self): diff --git a/ee/api/env.default b/ee/api/env.default index 0ce3ba237..a0d017f82 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -21,6 +21,7 @@ captcha_key= captcha_server= ch_host= ch_port= +pg_timeout=30 change_password_link=/reset-password?invitation=%s&&pass=%s email_basic=http://127.0.0.1:8000/async/basic/%s email_plans=http://127.0.0.1:8000/async/plans/%s