Api insights (#176)

* feat(db): added new foreign-key indexes

* feat(db): added new foreign-key indexes

* feat(api): pg_client fixed exception handler

* feat(api): changed change password response

* feat(api): insights retention

api/chalicelib/blueprints/subs/bp_insights.py

@@ -28,6 +28,17 @@ def get_insights_journey(projectId, context):
 
     return {"data": insights.get_journey(project_id=projectId, **{**data, **args})}
 
+
+@app.route('/{projectId}/insights/users_retention', methods=['GET', 'POST'])
+def get_users_retention(projectId, context):
+    data = app.current_request.json_body
+    if data is None:
+        data = {}
+    params = app.current_request.query_params
+    args = dashboard.dashboard_args(params)
+
+    return {"data": insights.get_retention(project_id=projectId, **{**data, **args})}
+
 #
 #
 # @app.route('/{projectId}/dashboard/{widget}/search', methods=['GET'])

api/chalicelib/core/insights.py

@@ -98,8 +98,61 @@ def get_journey(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimest
         params = {"project_id": project_id, "startTimestamp": startTimestamp,
                   "endTimestamp": endTimestamp, "event_start": event_start, "JOURNEY_DEPTH": JOURNEY_DEPTH,
                   **__get_constraint_values(args), **extra_values}
-        print(cur.mogrify(pg_query, params))
+        # print(cur.mogrify(pg_query, params))
         cur.execute(cur.mogrify(pg_query, params))
         rows = cur.fetchall()
 
     return __transform_journey(rows)
+
+
+def __compute_retention_percentage(rows):
+    if rows is None or len(rows) == 0:
+        return rows
+    t = -1
+    for r in rows:
+        if r["week"] == 0:
+            t = r["usersCount"]
+        r["percentage"] = r["usersCount"] / t
+    return rows
+
+
+@dev.timed
+def get_retention(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), filters=[],
+                  **args):
+    pg_sub_query = __get_constraints(project_id=project_id, data=args, duration=True, main_table="sessions",
+                                     time_constraint=True)
+
+    with pg_client.PostgresClient() as cur:
+        pg_query = f"""SELECT EXTRACT(EPOCH FROM first_connexion_week::date)::bigint*1000 AS first_connexion_week,
+                               FLOOR(DATE_PART('day', connexion_week - first_connexion_week) / 7)::integer AS week,
+                               COUNT(DISTINCT connexions_list.user_id)                            AS users_count,
+                               ARRAY_AGG(DISTINCT connexions_list.user_id)                        AS connected_users
+                        FROM (SELECT DISTINCT user_id, MIN(DATE_TRUNC('week', to_timestamp(start_ts / 1000))) AS first_connexion_week
+                              FROM sessions
+                              WHERE {" AND ".join(pg_sub_query)}
+                                AND NOT EXISTS((SELECT 1
+                                                FROM sessions AS bsess
+                                                WHERE bsess.start_ts<EXTRACT('EPOCH' FROM DATE_TRUNC('week', to_timestamp(%(startTimestamp)s / 1000))) * 1000
+                                                  AND project_id = %(project_id)s
+                                                  AND bsess.user_id = sessions.user_id
+                                                LIMIT 1))
+                              GROUP BY user_id) AS users_list
+                                 LEFT JOIN LATERAL (SELECT DATE_TRUNC('week', to_timestamp(start_ts / 1000)::timestamp) AS connexion_week,
+                                                           user_id
+                                                    FROM sessions
+                                                    WHERE users_list.user_id = sessions.user_id
+                                                      AND first_connexion_week <=
+                                                          DATE_TRUNC('week', to_timestamp(sessions.start_ts / 1000)::timestamp)
+                                                      AND sessions.project_id = 1
+                                                      AND sessions.start_ts < (%(startTimestamp)s + 10 * 7 * 24 * 60 * 60 * 1000::bigint - 1)
+                                                    GROUP BY connexion_week, user_id) AS connexions_list ON (TRUE)
+                        GROUP BY first_connexion_week, week
+                        ORDER BY first_connexion_week, week;"""
+
+        params = {"project_id": project_id, "startTimestamp": startTimestamp,
+                  "endTimestamp": endTimestamp, **__get_constraint_values(args)}
+        # print(cur.mogrify(pg_query, params))
+        cur.execute(cur.mogrify(pg_query, params))
+        rows = cur.fetchall()
+
+    return __compute_retention_percentage(helper.list_to_camel_case(rows))

api/chalicelib/core/users.py

@@ -426,8 +426,27 @@ def change_password(tenant_id, user_id, email, old_password, new_password):
     if auth is None:
         return {"errors": ["wrong password"]}
     changes = {"password": new_password, "generatedPassword": False}
-    return {"data": update(tenant_id=tenant_id, user_id=user_id, changes=changes),
-            "jwt": authenticate(email, new_password)["jwt"]}
+    user = update(tenant_id=tenant_id, user_id=user_id, changes=changes)
+    r = authenticate(user['email'], new_password)
+    tenant_id = r.pop("tenantId")
+
+    r["limits"] = {
+        "teamMember": -1,
+        "projects": -1,
+        "metadata": metadata.get_remaining_metadata_with_count(tenant_id)}
+
+    c = tenants.get_by_tenant_id(tenant_id)
+    c.pop("createdAt")
+    c["projects"] = projects.get_projects(tenant_id=tenant_id, recording_state=True, recorded=True,
+                                          stack_integrations=True)
+    c["smtp"] = helper.has_smtp()
+    return {
+        'jwt': r.pop('jwt'),
+        'data': {
+            "user": r,
+            "client": c
+        }
+    }
 
 
 def set_password_invitation(user_id, new_password):

api/chalicelib/utils/pg_client.py

@@ -51,7 +51,7 @@ class PostgresClient:
         try:
             self.connection.commit()
             self.cursor.close()
-        except:
+        except Exception as error:
             print("Error while committing/closing PG-connection", error)
             raise error
         finally:

ee/api/chalicelib/core/users.py

@@ -436,8 +436,27 @@ def change_password(tenant_id, user_id, email, old_password, new_password):
     if auth is None:
         return {"errors": ["wrong password"]}
     changes = {"password": new_password, "generatedPassword": False}
-    return {"data": update(tenant_id=tenant_id, user_id=user_id, changes=changes),
-            "jwt": authenticate(email, new_password)["jwt"]}
+    user = update(tenant_id=tenant_id, user_id=user_id, changes=changes)
+    r = authenticate(user['email'], new_password)
+
+    tenant_id = r.pop("tenantId")
+    r["limits"] = {
+        "teamMember": -1,
+        "projects": -1,
+        "metadata": metadata.get_remaining_metadata_with_count(tenant_id)}
+
+    c = tenants.get_by_tenant_id(tenant_id)
+    c.pop("createdAt")
+    c["projects"] = projects.get_projects(tenant_id=tenant_id, recording_state=True, recorded=True,
+                                          stack_integrations=True)
+    c["smtp"] = helper.has_smtp()
+    return {
+        'jwt': r.pop('jwt'),
+        'data': {
+            "user": r,
+            "client": c,
+        }
+    }
 
 
 def set_password_invitation(tenant_id, user_id, new_password):
@@ -457,6 +476,7 @@ def set_password_invitation(tenant_id, user_id, new_password):
     c.pop("createdAt")
     c["projects"] = projects.get_projects(tenant_id=tenant_id, recording_state=True, recorded=True,
                                           stack_integrations=True)
+    c["smtp"] = helper.has_smtp()
     return {
         'jwt': r.pop('jwt'),
         'data': {

scripts/helm/db/init_dbs/postgresql/1.4.0/1.4.0.sql

@@ -1,5 +1,10 @@
 BEGIN;
 CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp);
 
+CREATE INDEX projects_tenant_id_idx ON projects(tenant_id);
+CREATE INDEX webhooks_tenant_id_idx ON webhooks(tenant_id);
+CREATE INDEX issues_project_id_idx ON issues(project_id);
+CREATE INDEX jobs_project_id_idx ON jobs(project_id);
+
 
 COMMIT;

scripts/helm/db/init_dbs/postgresql/init_schema.sql

@@ -172,6 +172,7 @@ CREATE TABLE projects
       "defaultInputMode": "plain"
     }'::jsonb -- ??????
 );
+CREATE INDEX projects_tenant_id_idx ON projects(tenant_id);
 
 CREATE OR REPLACE FUNCTION notify_project() RETURNS trigger AS
 $$
@@ -247,7 +248,7 @@ create table webhooks
     index       integer   default 0                            not null,
     name        varchar(100)
 );
-
+CREATE INDEX webhooks_tenant_id_idx ON webhooks(tenant_id);
 
 -- --- notifications.sql ---
 
@@ -387,6 +388,7 @@ CREATE TABLE issues
 );
 CREATE INDEX ON issues (issue_id, type);
 CREATE INDEX issues_context_string_gin_idx ON public.issues USING GIN (context_string gin_trgm_ops);
+CREATE INDEX issues_project_id_idx ON issues(project_id);
 
 -- --- errors.sql ---
 
@@ -870,5 +872,6 @@ CREATE TABLE jobs
 );
 CREATE INDEX ON jobs (status);
 CREATE INDEX ON jobs (start_at);
+CREATE INDEX jobs_project_id_idx ON jobs(project_id);
 
 COMMIT;