From effe39d36d7a24bcdeafff1766676efed0abea7d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 5 Apr 2022 17:29:29 +0200 Subject: [PATCH] feat(api): pg helper support application name feat(api): removed widget name feat(db): removed widget name feat(api): changed edit dashboard to support list of metrics feat(api): get metrics/widgets cast created_at feat(api): get metrics/widgets cast edited_at feat(api): create dashboard with metrics, support default config feat(api): create dashboard with metrics, support widget position feat(api): edit dashboard with metrics, support default config feat(api): edit dashboard with metrics, support widget position --- api/Dockerfile | 1 + api/Dockerfile.alerts | 1 + api/chalicelib/core/custom_metrics.py | 7 +++- api/chalicelib/core/dashboards2.py | 39 +++++++++++++----- api/chalicelib/utils/pg_client.py | 5 ++- api/schemas.py | 14 +++---- .../db/init_dbs/postgresql/1.5.5/1.5.5.sql | 41 +++++++++---------- .../db/init_dbs/postgresql/init_schema.sql | 38 ++++++++--------- 8 files changed, 84 insertions(+), 62 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 780518ff3..4526c32bd 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /work COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env +ENV APP_NAME chalice # Add Tini # Startup daemon diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts index ed8f06eac..bdd1772ba 100644 --- a/api/Dockerfile.alerts +++ b/api/Dockerfile.alerts @@ -6,6 +6,7 @@ COPY . . RUN pip install -r requirements.txt RUN mv .env.default .env && mv app_alerts.py app.py ENV pg_minconn 2 +ENV APP_NAME alerts # Add Tini # Startup daemon diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 51f8655a4..6ca72f453 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -233,7 +233,7 @@ def get_all(project_id, user_id, include_series=False): WHERE metrics.project_id = %(project_id)s AND metrics.deleted_at ISNULL AND (user_id = %(user_id)s OR metrics.is_public) - ORDER BY created_at;""", + ORDER BY metrics.edited_at, metrics.created_at;""", {"project_id": project_id, "user_id": user_id} ) ) @@ -243,6 +243,10 @@ def get_all(project_id, user_id, include_series=False): # r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) for s in r["series"]: s["filter"] = helper.old_search_payload_to_flat(s["filter"]) + else: + for r in rows: + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["edited_at"] = TimeUTC.datetime_to_timestamp(r["edited_at"]) rows = helper.list_to_camel_case(rows) return rows @@ -297,6 +301,7 @@ def get(metric_id, project_id, user_id, flatten=True): if row is None: return None row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) + row["edited_at"] = TimeUTC.datetime_to_timestamp(row["edited_at"]) if flatten: for s in row["series"]: s["filter"] = helper.old_search_payload_to_flat(s["filter"]) diff --git a/api/chalicelib/core/dashboards2.py b/api/chalicelib/core/dashboards2.py index 7e06f52ac..c0bb3017f 100644 --- a/api/chalicelib/core/dashboards2.py +++ b/api/chalicelib/core/dashboards2.py @@ -32,11 +32,15 @@ def create_dashboard(project_id, user_id, data: schemas.CreateDashboardSchema): params = {"userId": user_id, "projectId": project_id, **data.dict()} if data.metrics is not None and len(data.metrics) > 0: pg_query = f"""WITH dash AS ({pg_query}) - INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id) - VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s)" for i in range(len(data.metrics))])} + INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) + VALUES {",".join([f"((SELECT dashboard_id FROM dash),%(metric_id_{i})s, %(userId)s, %(config_{i})s)" for i in range(len(data.metrics))])} RETURNING (SELECT dashboard_id FROM dash)""" - for i, m in enumerate(data.metrics): - params[f"metric_id_{i}"] = m + for i, m in enumerate(data.metrics): + params[f"metric_id_{i}"] = m + params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ + .get("properties", {}).get("config", {}).get("default", {}) + params[f"config_{i}"]["position"] = i + params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) cur.execute(cur.mogrify(pg_query, params)) row = cur.fetchone() if row is None: @@ -98,16 +102,29 @@ def delete_dashboard(project_id, user_id, dashboard_id): def update_dashboard(project_id, user_id, dashboard_id, data: schemas.EditDashboardSchema): with pg_client.PostgresClient() as cur: - pg_query = """UPDATE dashboards - SET name = %(name)s, is_public = %(is_public)s + pg_query = f"""UPDATE dashboards + SET name = %(name)s + {", is_public = %(is_public)s" if data.is_public is not None else ""} + {", is_pinned = %(is_pinned)s" if data.is_pinned is not None else ""} WHERE dashboards.project_id = %(projectId)s AND dashboard_id = %(dashboard_id)s - AND (dashboards.user_id = %(userId)s OR is_public) - RETURNING *;""" + AND (dashboards.user_id = %(userId)s OR is_public)""" params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, **data.dict()} + if data.metrics is not None and len(data.metrics) > 0: + pg_query = f"""WITH dash AS ({pg_query}) + INSERT INTO dashboard_widgets(dashboard_id, metric_id, user_id, config) + VALUES {",".join([f"(%(dashboard_id)s, %(metric_id_{i})s, %(userId)s, %(config_{i})s)" for i in range(len(data.metrics))])} + RETURNING (SELECT dashboard_id FROM dash)""" + for i, m in enumerate(data.metrics): + params[f"metric_id_{i}"] = m + params[f"config_{i}"] = schemas.AddWidgetToDashboardPayloadSchema.schema() \ + .get("properties", {}).get("config", {}).get("default", {}) + params[f"config_{i}"]["position"] = i + params[f"config_{i}"] = json.dumps(params[f"config_{i}"]) + cur.execute(cur.mogrify(pg_query, params)) - row = cur.fetchone() - return helper.dict_to_camel_case(row) + + return get_dashboard(project_id=project_id, user_id=user_id, dashboard_id=dashboard_id) def get_widget(project_id, user_id, dashboard_id, widget_id): @@ -154,7 +171,7 @@ def add_widget(project_id, user_id, dashboard_id, data: schemas.AddWidgetToDashb def update_widget(project_id, user_id, dashboard_id, widget_id, data: schemas.AddWidgetToDashboardPayloadSchema): with pg_client.PostgresClient() as cur: pg_query = """UPDATE dashboard_widgets - SET name= %(name)s, config= %(config)s + SET config= %(config)s WHERE dashboard_id=%(dashboard_id)s AND widget_id=%(widget_id)s RETURNINIG *;""" params = {"userId": user_id, "projectId": project_id, "dashboard_id": dashboard_id, diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index 6e4118689..6379bad1e 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -9,7 +9,8 @@ _PG_CONFIG = {"host": config("pg_host"), "database": config("pg_dbname"), "user": config("pg_user"), "password": config("pg_password"), - "port": config("pg_port", cast=int)} + "port": config("pg_port", cast=int), + "application_name": config("APP_NAME", default="PY")} PG_CONFIG = dict(_PG_CONFIG) if config("pg_timeout", cast=int, default=0) > 0: PG_CONFIG["options"] = f"-c statement_timeout={config('pg_timeout', cast=int) * 1000}" @@ -64,6 +65,8 @@ class PostgresClient: def __init__(self, long_query=False): self.long_query = long_query if long_query: + long_config = dict(_PG_CONFIG) + long_config["application_name"] += "-LONG" self.connection = psycopg2.connect(**_PG_CONFIG) else: self.connection = postgreSQL_pool.getconn() diff --git a/api/schemas.py b/api/schemas.py index a7fd003b7..1f1bec739 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -888,18 +888,15 @@ class CreateDashboardSchema(BaseModel): alias_generator = attribute_to_camel_case -class EditDashboardSchema(BaseModel): - name: str = Field(..., min_length=1) - is_public: bool = Field(default=False) - - class Config: - alias_generator = attribute_to_camel_case +class EditDashboardSchema(CreateDashboardSchema): + is_public: Optional[bool] = Field(default=None) + is_pinned: Optional[bool] = Field(default=None) class AddWidgetToDashboardPayloadSchema(BaseModel): metric_id: int = Field(default=None) - name: Optional[str] = Field(default=None) - config: dict = Field(default={}) + # if you change the config attribute name, please make sure to update it in dashboard2.py + config: dict = Field(default={"col": 1, "row": 1, "position": 0}) class Config: alias_generator = attribute_to_camel_case @@ -929,7 +926,6 @@ class TemplateKeys(str, Enum): avg_fps = "avg_fps" -# class CustomMetricAndTemplate(CreateCustomMetricsSchema): class CustomMetricAndTemplate(BaseModel): is_template: bool = Field(...) project_id: Optional[int] = Field(...) diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql index 6c563bfdc..1a3a918ed 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql @@ -45,31 +45,30 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets metric_id integer NOT NULL REFERENCES metrics (metric_id) ON DELETE CASCADE, user_id integer NOT NULL REFERENCES users (user_id) ON DELETE SET NULL, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - config jsonb NOT NULL DEFAULT '{}'::jsonb, - name text + config jsonb NOT NULL DEFAULT '{}'::jsonb ); INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('sessions count', 'overview', '{}', true, true, true, 'count_sessions'), - ('avg request load time', 'overview', '{}', true, true, true, 'avg_request_load_time'), - ('avg page load time', 'overview', '{}', true, true, true, 'avg_page_load_time'), - ('avg image load time', 'overview', '{}', true, true, true, 'avg_image_load_time'), - ('avg dom content load start', 'overview', '{}', true, true, true, 'avg_dom_content_load_start'), - ('avg first contentful pixel', 'overview', '{}', true, true, true, 'avg_first_contentful_pixel'), - ('avg visited pages count', 'overview', '{}', true, true, true, 'avg_visited_pages'), - ('avg session duration', 'overview', '{}', true, true, true, 'avg_session_duration'), - ('avg pages dom build time', 'overview', '{}', true, true, true, 'avg_pages_dom_buildtime'), - ('avg pages response time', 'overview', '{}', true, true, true, 'avg_pages_response_time'), - ('avg response time', 'overview', '{}', true, true, true, 'avg_response_time'), - ('avg first paint', 'overview', '{}', true, true, true, 'avg_first_paint'), - ('avg dom content loaded', 'overview', '{}', true, true, true, 'avg_dom_content_loaded'), - ('avg time till first bit', 'overview', '{}', true, true, true, 'avg_till_first_bit'), - ('avg time to interactive', 'overview', '{}', true, true, true, 'avg_time_to_interactive'), - ('requests count', 'overview', '{}', true, true, true, 'count_requests'), - ('avg time to render', 'overview', '{}', true, true, true, 'avg_time_to_render'), - ('avg used js heap size', 'overview', '{}', true, true, true, 'avg_used_js_heap_size'), - ('avg cpu', 'overview', '{}', true, true, true, 'avg_cpu') +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') ON CONFLICT (key) DO UPDATE SET name=excluded.name, category=excluded.category, config=excluded.config, diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 5b5bd7044..5114d2433 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1026,25 +1026,25 @@ $$ LANGUAGE plpgsql; INSERT INTO metrics (name, category, config, is_predefined, is_template, is_public, key) -VALUES ('sessions count', 'overview', '{}', true, true, true, 'count_sessions'), - ('avg request load time', 'overview', '{}', true, true, true, 'avg_request_load_time'), - ('avg page load time', 'overview', '{}', true, true, true, 'avg_page_load_time'), - ('avg image load time', 'overview', '{}', true, true, true, 'avg_image_load_time'), - ('avg dom content load start', 'overview', '{}', true, true, true, 'avg_dom_content_load_start'), - ('avg first contentful pixel', 'overview', '{}', true, true, true, 'avg_first_contentful_pixel'), - ('avg visited pages count', 'overview', '{}', true, true, true, 'avg_visited_pages'), - ('avg session duration', 'overview', '{}', true, true, true, 'avg_session_duration'), - ('avg pages dom build time', 'overview', '{}', true, true, true, 'avg_pages_dom_buildtime'), - ('avg pages response time', 'overview', '{}', true, true, true, 'avg_pages_response_time'), - ('avg response time', 'overview', '{}', true, true, true, 'avg_response_time'), - ('avg first paint', 'overview', '{}', true, true, true, 'avg_first_paint'), - ('avg dom content loaded', 'overview', '{}', true, true, true, 'avg_dom_content_loaded'), - ('avg time till first bit', 'overview', '{}', true, true, true, 'avg_till_first_bit'), - ('avg time to interactive', 'overview', '{}', true, true, true, 'avg_time_to_interactive'), - ('requests count', 'overview', '{}', true, true, true, 'count_requests'), - ('avg time to render', 'overview', '{}', true, true, true, 'avg_time_to_render'), - ('avg used js heap size', 'overview', '{}', true, true, true, 'avg_used_js_heap_size'), - ('avg cpu', 'overview', '{}', true, true, true, 'avg_cpu') +VALUES ('sessions count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions'), + ('avg request load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time'), + ('avg page load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time'), + ('avg image load time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time'), + ('avg dom content load start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start'), + ('avg first contentful pixel', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel'), + ('avg visited pages count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages'), + ('avg session duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration'), + ('avg pages dom build time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime'), + ('avg pages response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time'), + ('avg response time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time'), + ('avg first paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint'), + ('avg dom content loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded'), + ('avg time till first bit', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_bit'), + ('avg time to interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive'), + ('requests count', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests'), + ('avg time to render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render'), + ('avg used js heap size', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size'), + ('avg cpu', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu') ON CONFLICT (key) DO UPDATE SET name=excluded.name, category=excluded.category, config=excluded.config,