From 0886e3856a717cf18c514521cc4af61a47897df5 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 10 Jun 2022 12:33:36 +0200 Subject: [PATCH] feat(api): EE changed weekly report feat(api): changed login response --- api/routers/core_dynamic.py | 2 - ee/api/chalicelib/core/weekly_report.py | 245 ++++++++++++++++++++++++ ee/api/routers/core_dynamic.py | 2 - 3 files changed, 245 insertions(+), 4 deletions(-) create mode 100644 ee/api/chalicelib/core/weekly_report.py diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index 918d81541..a407e2833 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -52,9 +52,7 @@ def login(data: schemas.UserLoginSchema = Body(...)): c = tenants.get_by_tenant_id(tenant_id) c.pop("createdAt") c["smtp"] = helper.has_smtp() - c["iceServers"] = assist.get_ice_servers() r["smtp"] = c["smtp"] - r["iceServers"] = c["iceServers"] return { 'jwt': r.pop('jwt'), 'data': { diff --git a/ee/api/chalicelib/core/weekly_report.py b/ee/api/chalicelib/core/weekly_report.py new file mode 100644 index 000000000..e652010d4 --- /dev/null +++ b/ee/api/chalicelib/core/weekly_report.py @@ -0,0 +1,245 @@ +from chalicelib.utils import pg_client, helper, email_helper +from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.utils.helper import get_issue_title + +LOWEST_BAR_VALUE = 3 + + +def get_config(user_id): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""\ + SELECT users.weekly_report + FROM public.users + WHERE users.deleted_at ISNULL AND users.user_id=%(user_id)s + LIMIT 1;""", {"user_id": user_id})) + result = cur.fetchone() + return helper.dict_to_camel_case(result) + + +def edit_config(user_id, weekly_report): + with pg_client.PostgresClient() as cur: + cur.execute(cur.mogrify("""\ + UPDATE public.users + SET weekly_report= %(weekly_report)s + WHERE users.deleted_at ISNULL + AND users.user_id=%(user_id)s + RETURNING weekly_report;""", {"user_id": user_id, "weekly_report": weekly_report})) + result = cur.fetchone() + return helper.dict_to_camel_case(result) + + +def cron(): + if not helper.has_smtp(): + print("!!! No SMTP configuration found, ignoring weekly report") + return + with pg_client.PostgresClient(long_query=True) as cur: + params = {"tomorrow": TimeUTC.midnight(delta_days=1), + "3_days_ago": TimeUTC.midnight(delta_days=-3), + "1_week_ago": TimeUTC.midnight(delta_days=-7), + "2_week_ago": TimeUTC.midnight(delta_days=-14), + "5_week_ago": TimeUTC.midnight(delta_days=-35)} + cur.execute(cur.mogrify("""\ + SELECT project_id, + name AS project_name, + users.emails AS emails, + TO_CHAR(DATE_TRUNC('day', now()) - INTERVAL '1 week', 'Mon. DDth, YYYY') AS period_start, + TO_CHAR(DATE_TRUNC('day', now()), 'Mon. DDth, YYYY') AS period_end, + COALESCE(week_0_issues.count, 0) AS this_week_issues_count, + COALESCE(week_1_issues.count, 0) AS past_week_issues_count, + COALESCE(month_1_issues.count, 0) AS past_month_issues_count + FROM (SELECT tenant_id, project_id, name FROM public.projects WHERE projects.deleted_at ISNULL) AS projects + INNER JOIN LATERAL ( + SELECT sessions.project_id + FROM public.sessions + WHERE sessions.project_id = projects.project_id + AND start_ts >= %(3_days_ago)s + AND start_ts < %(tomorrow)s + LIMIT 1) AS recently_active USING (project_id) + INNER JOIN LATERAL ( + SELECT COALESCE(ARRAY_AGG(email), '{}') AS emails + FROM public.users + WHERE users.tenant_id = projects.tenant_id + AND users.deleted_at ISNULL + AND users.weekly_report + ) AS users ON (TRUE) + LEFT JOIN LATERAL ( + SELECT COUNT(1) AS count + FROM events_common.issues + INNER JOIN public.sessions USING (session_id) + WHERE sessions.project_id = projects.project_id + AND issues.timestamp >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND issues.timestamp < %(tomorrow)s + ) AS week_0_issues ON (TRUE) + LEFT JOIN LATERAL ( + SELECT COUNT(1) AS count + FROM events_common.issues + INNER JOIN public.sessions USING (session_id) + WHERE sessions.project_id = projects.project_id + AND issues.timestamp <= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND issues.timestamp >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '2 week') * 1000)::BIGINT + ) AS week_1_issues ON (TRUE) + LEFT JOIN LATERAL ( + SELECT COUNT(1) AS count + FROM events_common.issues + INNER JOIN public.sessions USING (session_id) + WHERE sessions.project_id = projects.project_id + AND issues.timestamp <= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND issues.timestamp >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '5 week') * 1000)::BIGINT + ) AS month_1_issues ON (TRUE);"""), params) + projects_data = cur.fetchall() + emails_to_send = [] + for p in projects_data: + params["project_id"] = p["project_id"] + print(f"checking {p['project_name']} : {p['project_id']}") + if len(p["emails"]) == 0 \ + or p["this_week_issues_count"] + p["past_week_issues_count"] + p["past_month_issues_count"] == 0: + print('ignore') + continue + print("valid") + p["past_week_issues_evolution"] = helper.__decimal_limit( + helper.__progress(p["this_week_issues_count"], p["past_week_issues_count"]), 1) + p["past_month_issues_evolution"] = helper.__decimal_limit( + helper.__progress(p["this_week_issues_count"], p["past_month_issues_count"]), 1) + cur.execute(cur.mogrify(""" + SELECT LEFT(TO_CHAR(timestamp_i, 'Dy'),1) AS day_short, + TO_CHAR(timestamp_i, 'Mon. DD, YYYY') AS day_long, + ( + SELECT COUNT(*) + FROM events_common.issues INNER JOIN public.issues USING (issue_id) + WHERE project_id = %(project_id)s + AND timestamp >= (EXTRACT(EPOCH FROM timestamp_i) * 1000)::BIGINT + AND timestamp <= (EXTRACT(EPOCH FROM timestamp_i + INTERVAL '1 day') * 1000)::BIGINT + ) AS issues_count + FROM generate_series( + DATE_TRUNC('day', now()) - INTERVAL '7 days', + DATE_TRUNC('day', now()) - INTERVAL '1 day', + '1 day'::INTERVAL + ) AS timestamp_i + ORDER BY timestamp_i;""", params)) + days_partition = cur.fetchall() + max_days_partition = max(x['issues_count'] for x in days_partition) + for d in days_partition: + if max_days_partition <= 0: + d["value"] = LOWEST_BAR_VALUE + else: + d["value"] = d["issues_count"] * 100 / max_days_partition + d["value"] = d["value"] if d["value"] > LOWEST_BAR_VALUE else LOWEST_BAR_VALUE + cur.execute(cur.mogrify("""\ + SELECT type, COUNT(*) AS count + FROM events_common.issues INNER JOIN public.issues USING (issue_id) + WHERE project_id = %(project_id)s + AND timestamp >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '7 days') * 1000)::BIGINT + GROUP BY type + ORDER BY count DESC, type + LIMIT 4;""", params)) + issues_by_type = cur.fetchall() + max_issues_by_type = sum(i["count"] for i in issues_by_type) + for i in issues_by_type: + i["type"] = get_issue_title(i["type"]) + if max_issues_by_type <= 0: + i["value"] = LOWEST_BAR_VALUE + else: + i["value"] = i["count"] * 100 / max_issues_by_type + cur.execute(cur.mogrify("""\ + SELECT TO_CHAR(timestamp_i, 'Dy') AS day_short, + TO_CHAR(timestamp_i, 'Mon. DD, YYYY') AS day_long, + COALESCE((SELECT JSONB_AGG(sub) + FROM ( + SELECT type, COUNT(*) AS count + FROM events_common.issues + INNER JOIN public.issues USING (issue_id) + WHERE project_id = %(project_id)s + AND timestamp >= (EXTRACT(EPOCH FROM timestamp_i) * 1000)::BIGINT + AND timestamp <= (EXTRACT(EPOCH FROM timestamp_i + INTERVAL '1 day') * 1000)::BIGINT + GROUP BY type + ORDER BY count + ) AS sub), '[]'::JSONB) AS partition + FROM generate_series( + DATE_TRUNC('day', now()) - INTERVAL '7 days', + DATE_TRUNC('day', now()) - INTERVAL '1 day', + '1 day'::INTERVAL + ) AS timestamp_i + GROUP BY timestamp_i + ORDER BY timestamp_i;""", params)) + issues_breakdown_by_day = cur.fetchall() + for i in issues_breakdown_by_day: + i["sum"] = sum(x["count"] for x in i["partition"]) + for j in i["partition"]: + j["type"] = get_issue_title(j["type"]) + max_days_partition = max(i["sum"] for i in issues_breakdown_by_day) + for i in issues_breakdown_by_day: + for j in i["partition"]: + if max_days_partition <= 0: + j["value"] = LOWEST_BAR_VALUE + else: + j["value"] = j["count"] * 100 / max_days_partition + j["value"] = j["value"] if j["value"] > LOWEST_BAR_VALUE else LOWEST_BAR_VALUE + cur.execute(cur.mogrify(""" + SELECT type, + COUNT(*) AS issue_count, + COUNT(DISTINCT session_id) AS sessions_count, + (SELECT COUNT(DISTINCT sessions.session_id) + FROM public.sessions + INNER JOIN events_common.issues AS sci USING (session_id) + INNER JOIN public.issues AS si USING (issue_id) + WHERE si.project_id = %(project_id)s + AND sessions.project_id = %(project_id)s + AND sessions.start_ts <= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND sessions.start_ts >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '2 weeks') * 1000)::BIGINT + AND si.type = mi.type + AND sessions.duration IS NOT NULL + ) AS last_week_sessions_count, + (SELECT COUNT(DISTINCT sci.session_id) + FROM public.sessions + INNER JOIN events_common.issues AS sci USING (session_id) + INNER JOIN public.issues AS si USING (issue_id) + WHERE si.project_id = %(project_id)s + AND sessions.project_id = %(project_id)s + AND sessions.start_ts <= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + AND sessions.start_ts >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '5 weeks') * 1000)::BIGINT + AND si.type = mi.type + AND sessions.duration IS NOT NULL + ) AS last_month_sessions_count + FROM events_common.issues + INNER JOIN public.issues AS mi USING (issue_id) + INNER JOIN public.sessions USING (session_id) + WHERE mi.project_id = %(project_id)s AND sessions.project_id = %(project_id)s AND sessions.duration IS NOT NULL + AND sessions.start_ts >= (EXTRACT(EPOCH FROM DATE_TRUNC('day', now()) - INTERVAL '1 week') * 1000)::BIGINT + GROUP BY type + ORDER BY issue_count DESC;""", params)) + issues_breakdown_list = cur.fetchall() + if len(issues_breakdown_list) > 4: + others = {"type": "Others", + "sessions_count": sum(i["sessions_count"] for i in issues_breakdown_list[4:]), + "issue_count": sum(i["issue_count"] for i in issues_breakdown_list[4:]), + "last_week_sessions_count": sum( + i["last_week_sessions_count"] for i in issues_breakdown_list[4:]), + "last_month_sessions_count": sum( + i["last_month_sessions_count"] for i in issues_breakdown_list[4:])} + issues_breakdown_list = issues_breakdown_list[:4] + issues_breakdown_list.append(others) + for i in issues_breakdown_list: + i["type"] = get_issue_title(i["type"]) + i["last_week_sessions_evolution"] = helper.__decimal_limit( + helper.__progress(i["sessions_count"], i["last_week_sessions_count"]), 1) + i["last_month_sessions_evolution"] = helper.__decimal_limit( + helper.__progress(i["sessions_count"], i["last_month_sessions_count"]), 1) + i["sessions_count"] = f'{i["sessions_count"]:,}' + keep_types = [i["type"] for i in issues_breakdown_list] + for i in issues_breakdown_by_day: + keep = [] + for j in i["partition"]: + if j["type"] in keep_types: + keep.append(j) + i["partition"] = keep + emails_to_send.append({"email": p.pop("emails"), + "data": { + **p, + "days_partition": days_partition, + "issues_by_type": issues_by_type, + "issues_breakdown_by_day": issues_breakdown_by_day, + "issues_breakdown_list": issues_breakdown_list + }}) + print(f">>> Sending weekly report to {len(emails_to_send)} email-group") + for e in emails_to_send: + email_helper.weekly_report2(recipients=e["email"], data=e["data"]) diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index 196764ad9..89f6a9bc9 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -57,9 +57,7 @@ def login(data: schemas.UserLoginSchema = Body(...)): c = tenants.get_by_tenant_id(tenant_id) c.pop("createdAt") c["smtp"] = helper.has_smtp() - c["iceServers"] = assist.get_ice_servers() r["smtp"] = c["smtp"] - r["iceServers"] = c["iceServers"] return { 'jwt': r.pop('jwt'), 'data': {