From 0734a3e46832d115fe672b976ff182e8b9a9248a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 25 Aug 2022 15:33:43 +0100 Subject: [PATCH] feat(chalice): CH client error handling feat(chalice): fastapi upgrade --- api/requirements-alerts.txt | 2 +- api/requirements.txt | 2 +- ee/api/chalicelib/core/metrics.py | 2 +- ee/api/chalicelib/core/metrics_new.py | 26 +++++++++++++------------- ee/api/chalicelib/utils/ch_client.py | 12 +++++++++--- ee/api/requirements-alerts.txt | 2 +- ee/api/requirements-crons.txt | 2 +- ee/api/requirements.txt | 2 +- 8 files changed, 28 insertions(+), 22 deletions(-) diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index 9a3762eac..fc141eb09 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -8,7 +8,7 @@ jira==3.3.1 -fastapi==0.79.0 +fastapi==0.80.0 uvicorn[standard]==0.18.2 python-decouple==3.6 pydantic[email]==1.9.2 diff --git a/api/requirements.txt b/api/requirements.txt index 9a3762eac..fc141eb09 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -8,7 +8,7 @@ jira==3.3.1 -fastapi==0.79.0 +fastapi==0.80.0 uvicorn[standard]==0.18.2 python-decouple==3.6 pydantic[email]==1.9.2 diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index 873813741..d9a81cc91 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -292,7 +292,7 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), "endTimestamp": endTimestamp, **__get_constraint_values(args)} rows = ch.execute(query=ch_query, params=params) - print(f"got {len(rows)} rows") + # print(f"got {len(rows)} rows") if len(rows) == 0: return [] error_ids = [r["error_id"] for r in rows] diff --git a/ee/api/chalicelib/core/metrics_new.py b/ee/api/chalicelib/core/metrics_new.py index 2a7e2654f..2a4e0c236 100644 --- a/ee/api/chalicelib/core/metrics_new.py +++ b/ee/api/chalicelib/core/metrics_new.py @@ -297,7 +297,7 @@ def get_errors_trend(project_id, startTimestamp=TimeUTC.now(delta_days=-1), "endTimestamp": endTimestamp, **__get_constraint_values(args)} rows = ch.execute(query=ch_query, params=params) - print(f"got {len(rows)} rows") + # print(f"got {len(rows)} rows") if len(rows) == 0: return [] error_ids = [r["error_id"] for r in rows] @@ -492,7 +492,7 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), GROUP BY url, timestamp ORDER BY url, timestamp;""" params["url"] = urls - print(ch.format(query=ch_query, params=params)) + # print(ch.format(query=ch_query, params=params)) u_rows = ch.execute(query=ch_query, params=params) for url in urls: sub_rows = [] @@ -771,8 +771,8 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, AND positionCaseInsensitiveUTF8(value, %(value)s) != 0 LIMIT 10 BY type""" - print(ch.format(query=ch_query, params={"project_id": project_id, "value": text, "key": key, - "platform_0": platform})) + # print(ch.format(query=ch_query, params={"project_id": project_id, "value": text, "key": key, + # "platform_0": platform})) rows = ch.execute(query=ch_query, params={"project_id": project_id, "value": text, "key": key, "platform_0": platform}) else: @@ -800,7 +800,7 @@ def get_missing_resources_trend(project_id, startTimestamp=TimeUTC.now(delta_day LIMIT 10;""" params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} - print(ch.format(query=ch_query, params=params)) + # print(ch.format(query=ch_query, params=params)) rows = ch.execute(query=ch_query, params=params) rows = [{"url": i["key"], "sessions": i["doc_count"]} for i in rows] @@ -1061,7 +1061,7 @@ def get_speed_index_location(project_id, startTimestamp=TimeUTC.now(delta_days=- params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} - print(ch.format(query=ch_query, params=params)) + # print(ch.format(query=ch_query, params=params)) rows = ch.execute(query=ch_query, params=params) ch_query = f"""SELECT COALESCE(avgOrNull(pages.speed_index),0) AS avg FROM {sessions_helper.get_main_events_table(startTimestamp)} AS pages @@ -1775,7 +1775,7 @@ def __get_calls_errors_4xx_or_5xx(status, project_id, startTimestamp=TimeUTC.now params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} - print(ch.format(query=ch_query, params=params)) + # print(ch.format(query=ch_query, params=params)) rows = ch.execute(query=ch_query, params=params) return helper.list_to_camel_case(rows) @@ -1852,7 +1852,7 @@ def resource_type_vs_response_end(project_id, startTimestamp=TimeUTC.now(delta_d WHERE {" AND ".join(ch_sub_query_chart)} GROUP BY timestamp ORDER BY timestamp;""" - print(ch.format(query=ch_query, params=params)) + # print(ch.format(query=ch_query, params=params)) actions = ch.execute(query=ch_query, params=params) actions = __complete_missing_steps(rows=actions, start_time=startTimestamp, end_time=endTimestamp, @@ -1947,9 +1947,9 @@ def get_resources_vs_visually_complete(project_id, startTimestamp=TimeUTC.now(de "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} - print(">>>>>>>>>>>>>>") - print(ch.format(query=ch_query, params=params)) - print(">>>>>>>>>>>>>>") + # print(">>>>>>>>>>>>>>") + # print(ch.format(query=ch_query, params=params)) + # print(">>>>>>>>>>>>>>") rows = ch.execute(query=ch_query, params=params) for r in rows: types = {} @@ -1996,7 +1996,7 @@ def get_resources_count_by_type(project_id, startTimestamp=TimeUTC.now(delta_day "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} - print(ch.format(query=ch_query, params=params)) + # print(ch.format(query=ch_query, params=params)) rows = ch.execute(query=ch_query, params=params) for r in rows: for t in r["types"]: @@ -2046,7 +2046,7 @@ def get_resources_by_party(project_id, startTimestamp=TimeUTC.now(delta_days=-1) "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} - print(ch.format(query=ch_query, params=params)) + # print(ch.format(query=ch_query, params=params)) rows = ch.execute(query=ch_query, params=params) return helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp, end_time=endTimestamp, diff --git a/ee/api/chalicelib/utils/ch_client.py b/ee/api/chalicelib/utils/ch_client.py index 86461a7b8..2e71d1d57 100644 --- a/ee/api/chalicelib/utils/ch_client.py +++ b/ee/api/chalicelib/utils/ch_client.py @@ -30,9 +30,15 @@ class ClickHouseClient: return self def execute(self, query, params=None, **args): - results = self.__client.execute(query=query, params=params, with_column_types=True, **args) - keys = tuple(x for x, y in results[1]) - return [dict(zip(keys, i)) for i in results[0]] + try: + results = self.__client.execute(query=query, params=params, with_column_types=True, **args) + keys = tuple(x for x, y in results[1]) + return [dict(zip(keys, i)) for i in results[0]] + except Exception as err: + logging.error("--------- CH QUERY EXCEPTION -----------") + logging.error(self.format(query=query, params=params)) + logging.error("--------------------") + raise err def insert(self, query, params=None, **args): return self.__client.execute(query=query, params=params, **args) diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index 6745149f4..475a39b5e 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -8,7 +8,7 @@ jira==3.3.1 -fastapi==0.79.0 +fastapi==0.80.0 uvicorn[standard]==0.18.2 python-decouple==3.6 pydantic[email]==1.9.2 diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 6745149f4..475a39b5e 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -8,7 +8,7 @@ jira==3.3.1 -fastapi==0.79.0 +fastapi==0.80.0 uvicorn[standard]==0.18.2 python-decouple==3.6 pydantic[email]==1.9.2 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 01c7be739..bdf363b7b 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -8,7 +8,7 @@ jira==3.3.1 -fastapi==0.79.0 +fastapi==0.80.0 uvicorn[standard]==0.18.2 python-decouple==3.6 pydantic[email]==1.9.2