feat(chalice): CH sessions viewed

feat(chalice): CH sessions favorite
feat(chalice): CH errors viewed

api/chalicelib/core/errors_favorite.py

@@ -0,0 +1,48 @@
+from chalicelib.utils import pg_client
+
+
+def add_favorite_error(project_id, user_id, error_id):
+    with pg_client.PostgresClient() as cur:
+        cur.execute(
+            cur.mogrify(f"""INSERT INTO public.user_favorite_errors(user_id, error_id) 
+                            VALUES (%(userId)s,%(error_id)s);""",
+                        {"userId": user_id, "error_id": error_id})
+        )
+    return {"errorId": error_id, "favorite": True}
+
+
+def remove_favorite_error(project_id, user_id, error_id):
+    with pg_client.PostgresClient() as cur:
+        cur.execute(
+            cur.mogrify(f"""DELETE FROM public.user_favorite_errors                          
+                            WHERE 
+                                user_id = %(userId)s
+                                AND error_id = %(error_id)s;""",
+                        {"userId": user_id, "error_id": error_id})
+        )
+    return {"errorId": error_id, "favorite": False}
+
+
+def favorite_error(project_id, user_id, error_id):
+    exists, favorite = error_exists_and_favorite(user_id=user_id, error_id=error_id)
+    if not exists:
+        return {"errors": ["cannot bookmark non-rehydrated errors"]}
+    if favorite:
+        return remove_favorite_error(project_id=project_id, user_id=user_id, error_id=error_id)
+    return add_favorite_error(project_id=project_id, user_id=user_id, error_id=error_id)
+
+
+def error_exists_and_favorite(user_id, error_id):
+    with pg_client.PostgresClient() as cur:
+        cur.execute(
+            cur.mogrify(
+                """SELECT errors.error_id AS exists, ufe.error_id AS favorite
+                    FROM public.errors
+                             LEFT JOIN (SELECT error_id FROM public.user_favorite_errors WHERE user_id = %(userId)s) AS ufe USING (error_id)
+                    WHERE error_id = %(error_id)s;""",
+                {"userId": user_id, "error_id": error_id})
+        )
+        r = cur.fetchone()
+        if r is None:
+            return False, False
+        return True, r.get("favorite") is not None

api/chalicelib/core/errors_favorite_viewed.py

@@ -1,91 +0,0 @@
-from chalicelib.utils import pg_client
-
-
-def add_favorite_error(project_id, user_id, error_id):
-    with pg_client.PostgresClient() as cur:
-        cur.execute(
-            cur.mogrify(f"""\
-                INSERT INTO public.user_favorite_errors 
-                    (user_id, error_id) 
-                VALUES 
-                    (%(userId)s,%(error_id)s);""",
-                        {"userId": user_id, "error_id": error_id})
-        )
-    return {"errorId": error_id, "favorite": True}
-
-
-def remove_favorite_error(project_id, user_id, error_id):
-    with pg_client.PostgresClient() as cur:
-        cur.execute(
-            cur.mogrify(f"""\
-                        DELETE FROM public.user_favorite_errors                          
-                        WHERE 
-                            user_id = %(userId)s
-                            AND error_id = %(error_id)s;""",
-                        {"userId": user_id, "error_id": error_id})
-        )
-    return {"errorId": error_id, "favorite": False}
-
-
-def favorite_error(project_id, user_id, error_id):
-    exists, favorite = error_exists_and_favorite(user_id=user_id, error_id=error_id)
-    if not exists:
-        return {"errors": ["cannot bookmark non-rehydrated errors"]}
-    if favorite:
-        return remove_favorite_error(project_id=project_id, user_id=user_id, error_id=error_id)
-    return add_favorite_error(project_id=project_id, user_id=user_id, error_id=error_id)
-
-
-def error_exists_and_favorite(user_id, error_id):
-    with pg_client.PostgresClient() as cur:
-        cur.execute(
-            cur.mogrify(
-                """SELECT errors.error_id AS exists, ufe.error_id AS favorite
-                    FROM public.errors
-                             LEFT JOIN (SELECT error_id FROM public.user_favorite_errors WHERE user_id = %(userId)s) AS ufe USING (error_id)
-                    WHERE error_id = %(error_id)s;""",
-                {"userId": user_id, "error_id": error_id})
-        )
-        r = cur.fetchone()
-        if r is None:
-            return False, False
-        return True, r.get("favorite") is not None
-
-
-def add_viewed_error(project_id, user_id, error_id):
-    with pg_client.PostgresClient() as cur:
-        cur.execute(
-            cur.mogrify("""\
-                INSERT INTO public.user_viewed_errors 
-                    (user_id, error_id) 
-                VALUES 
-                    (%(userId)s,%(error_id)s);""",
-                        {"userId": user_id, "error_id": error_id})
-        )
-
-
-def viewed_error_exists(user_id, error_id):
-    with pg_client.PostgresClient() as cur:
-        query = cur.mogrify(
-            """SELECT 
-                    errors.error_id AS hydrated,
-                    COALESCE((SELECT TRUE
-                                         FROM public.user_viewed_errors AS ve
-                                         WHERE ve.error_id = %(error_id)s
-                                           AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed                                                
-                FROM public.errors
-                WHERE error_id = %(error_id)s""",
-            {"userId": user_id, "error_id": error_id})
-        cur.execute(
-            query=query
-        )
-        r = cur.fetchone()
-        if r:
-            return r.get("viewed")
-    return True
-
-
-def viewed_error(project_id, user_id, error_id):
-    if viewed_error_exists(user_id=user_id, error_id=error_id):
-        return None
-    return add_viewed_error(project_id=project_id, user_id=user_id, error_id=error_id)

api/chalicelib/core/errors_viewed.py

@@ -0,0 +1,37 @@
+from chalicelib.utils import pg_client
+
+
+def add_viewed_error(project_id, user_id, error_id):
+    with pg_client.PostgresClient() as cur:
+        cur.execute(
+            cur.mogrify("""INSERT INTO public.user_viewed_errors(user_id, error_id) 
+                            VALUES (%(userId)s,%(error_id)s);""",
+                        {"userId": user_id, "error_id": error_id})
+        )
+
+
+def viewed_error_exists(user_id, error_id):
+    with pg_client.PostgresClient() as cur:
+        query = cur.mogrify(
+            """SELECT 
+                    errors.error_id AS hydrated,
+                    COALESCE((SELECT TRUE
+                                         FROM public.user_viewed_errors AS ve
+                                         WHERE ve.error_id = %(error_id)s
+                                           AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed                                                
+                FROM public.errors
+                WHERE error_id = %(error_id)s""",
+            {"userId": user_id, "error_id": error_id})
+        cur.execute(
+            query=query
+        )
+        r = cur.fetchone()
+        if r:
+            return r.get("viewed")
+    return True
+
+
+def viewed_error(project_id, user_id, error_id):
+    if viewed_error_exists(user_id=user_id, error_id=error_id):
+        return None
+    return add_viewed_error(project_id=project_id, user_id=user_id, error_id=error_id)

api/chalicelib/core/sessions_favorite.py

@@ -6,10 +6,8 @@ def add_favorite_session(project_id, user_id, session_id):
     with pg_client.PostgresClient() as cur:
         cur.execute(
             cur.mogrify(f"""\
-                INSERT INTO public.user_favorite_sessions 
+                INSERT INTO public.user_favorite_sessions(user_id, session_id) 
-                    (user_id, session_id) 
+                VALUES (%(userId)s,%(sessionId)s);""",
-                VALUES 
-                    (%(userId)s,%(sessionId)s);""",
                         {"userId": user_id, "sessionId": session_id})
         )
     return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False,
@@ -21,8 +19,7 @@ def remove_favorite_session(project_id, user_id, session_id):
         cur.execute(
             cur.mogrify(f"""\
                         DELETE FROM public.user_favorite_sessions                          
-                        WHERE 
+                        WHERE user_id = %(userId)s
-                            user_id = %(userId)s
                             AND session_id = %(sessionId)s;""",
                         {"userId": user_id, "sessionId": session_id})
         )
@@ -41,8 +38,7 @@ def favorite_session_exists(user_id, session_id):
     with pg_client.PostgresClient() as cur:
         cur.execute(
             cur.mogrify(
-                """SELECT 
+                """SELECT session_id                                                
-                        session_id                                                
                     FROM public.user_favorite_sessions 
                     WHERE
                      user_id = %(userId)s

api/chalicelib/core/sessions_viewed.py

@@ -4,11 +4,8 @@ from chalicelib.utils import pg_client
 def view_session(project_id, user_id, session_id):
     with pg_client.PostgresClient() as cur:
         cur.execute(
-            cur.mogrify("""\
+            cur.mogrify("""INSERT INTO public.user_viewed_sessions(user_id, session_id) 
-                INSERT INTO public.user_viewed_sessions 
+                            VALUES (%(userId)s,%(sessionId)s)
-                    (user_id, session_id) 
+                            ON CONFLICT DO NOTHING;""",
-                VALUES 
-                    (%(userId)s,%(sessionId)s)
-                ON CONFLICT DO NOTHING;""",
                         {"userId": user_id, "sessionId": session_id})
         )

api/routers/core.py

@@ -12,8 +12,8 @@ from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assig
     log_tool_stackdriver, reset_password, sessions_favorite, \
     log_tool_cloudwatch, log_tool_sentry, log_tool_sumologic, log_tools, errors, sessions, \
     log_tool_newrelic, announcements, log_tool_bugsnag, weekly_report, integration_jira_cloud, integration_github, \
-    assist, heatmaps, mobile, signup, tenants, errors_favorite_viewed, boarding, notifications, webhook, users, \
+    assist, heatmaps, mobile, signup, tenants, errors_viewed, boarding, notifications, webhook, users, \
-    custom_metrics, saved_search, integrations_global, sessions_viewed
+    custom_metrics, saved_search, integrations_global, sessions_viewed, errors_favorite
 from chalicelib.core.collaboration_slack import Slack
 from chalicelib.utils import email_helper, helper, captcha
 from chalicelib.utils.TimeUTC import TimeUTC
@@ -83,7 +83,7 @@ def add_remove_favorite_session2(projectId: int, sessionId: int,
                                  context: schemas.CurrentContext = Depends(OR_context)):
     return {
         "data": sessions_favorite.favorite_session(project_id=projectId, user_id=context.user_id,
-                                                          session_id=sessionId)}
+                                                   session_id=sessionId)}
 
 
 @app.get('/{projectId}/sessions/{sessionId}/assign', tags=["sessions"])
@@ -986,7 +986,7 @@ def errors_get_details(projectId: int, errorId: str, background_tasks: Backgroun
     data = errors.get_details(project_id=projectId, user_id=context.user_id, error_id=errorId,
                               **{"density24": density24, "density30": density30})
     if data.get("data") is not None:
-        background_tasks.add_task(errors_favorite_viewed.viewed_error, project_id=projectId, user_id=context.user_id,
+        background_tasks.add_task(errors_viewed.viewed_error, project_id=projectId, user_id=context.user_id,
                                   error_id=errorId)
     return data
 
@@ -1015,7 +1015,7 @@ def errors_get_details_sourcemaps(projectId: int, errorId: str,
 def add_remove_favorite_error(projectId: int, errorId: str, action: str, startDate: int = TimeUTC.now(-7),
                               endDate: int = TimeUTC.now(), context: schemas.CurrentContext = Depends(OR_context)):
     if action == "favorite":
-        return errors_favorite_viewed.favorite_error(project_id=projectId, user_id=context.user_id, error_id=errorId)
+        return errors_favorite.favorite_error(project_id=projectId, user_id=context.user_id, error_id=errorId)
     elif action == "sessions":
         start_date = startDate
         end_date = endDate

ee/api/.gitignore

@@ -185,7 +185,7 @@ Pipfile
 /chalicelib/core/autocomplete.py
 /chalicelib/core/collaboration_slack.py
 /chalicelib/core/errors.py
-/chalicelib/core/errors_favorite_viewed.py
+/chalicelib/core/errors_favorite.py
 /chalicelib/core/events.py
 /chalicelib/core/events_ios.py
 /chalicelib/core/funnels.py
@@ -213,7 +213,6 @@ Pipfile
 /chalicelib/core/sessions_assignments.py
 /chalicelib/core/sessions_metas.py
 /chalicelib/core/sessions_mobs.py
-/chalicelib/core/sessions_viewed.py
 /chalicelib/core/significance.py
 /chalicelib/core/slack.py
 /chalicelib/core/socket_ios.py

ee/api/chalicelib/core/errors_viewed.py

@@ -0,0 +1,39 @@
+from chalicelib.utils import pg_client
+from chalicelib.core import errors_viewed_exp
+
+
+def add_viewed_error(project_id, user_id, error_id):
+    with pg_client.PostgresClient() as cur:
+        cur.execute(
+            cur.mogrify("""INSERT INTO public.user_viewed_errors(user_id, error_id) 
+                            VALUES (%(userId)s,%(error_id)s);""",
+                        {"userId": user_id, "error_id": error_id})
+        )
+    errors_viewed_exp.add_viewed_error(project_id=project_id, user_id=user_id, error_id=error_id)
+
+
+def viewed_error_exists(user_id, error_id):
+    with pg_client.PostgresClient() as cur:
+        query = cur.mogrify(
+            """SELECT 
+                    errors.error_id AS hydrated,
+                    COALESCE((SELECT TRUE
+                                         FROM public.user_viewed_errors AS ve
+                                         WHERE ve.error_id = %(error_id)s
+                                           AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed                                                
+                FROM public.errors
+                WHERE error_id = %(error_id)s""",
+            {"userId": user_id, "error_id": error_id})
+        cur.execute(
+            query=query
+        )
+        r = cur.fetchone()
+        if r:
+            return r.get("viewed")
+    return True
+
+
+def viewed_error(project_id, user_id, error_id):
+    if viewed_error_exists(user_id=user_id, error_id=error_id):
+        return None
+    return add_viewed_error(project_id=project_id, user_id=user_id, error_id=error_id)

ee/api/chalicelib/core/errors_viewed_exp.py

@@ -0,0 +1,15 @@
+import logging
+
+from decouple import config
+
+from chalicelib.utils import ch_client, exp_ch_helper
+
+logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO))
+
+
+def add_viewed_error(project_id, user_id, error_id):
+    with ch_client.ClickHouseClient() as cur:
+        query = f"""INSERT INTO {exp_ch_helper.get_user_viewed_errors_table()}(project_id,user_id, error_id) 
+                    VALUES (%(project_id)s,%(userId)s,%(error_id)s);"""
+        params = {"userId": user_id, "error_id": error_id, "project_id": project_id}
+        cur.execute(query=query, params=params)

ee/api/chalicelib/core/sessions_favorite.py

@@ -1,18 +1,19 @@
-from chalicelib.core import sessions
-from chalicelib.utils import pg_client, s3_extra
 from decouple import config
 
+from chalicelib.core import sessions, sessions_favorite_exp
+from chalicelib.utils import pg_client, s3_extra
+
 
 def add_favorite_session(project_id, user_id, session_id):
     with pg_client.PostgresClient() as cur:
         cur.execute(
             cur.mogrify(f"""\
-                INSERT INTO public.user_favorite_sessions 
+                INSERT INTO public.user_favorite_sessions(user_id, session_id) 
-                    (user_id, session_id) 
+                VALUES (%(userId)s,%(sessionId)s);""",
-                VALUES 
-                    (%(userId)s,%(sessionId)s);""",
                         {"userId": user_id, "sessionId": session_id})
         )
+
+    sessions_favorite_exp.add_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id)
     return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False,
                                   include_fav_viewed=True)
 
@@ -22,11 +23,11 @@ def remove_favorite_session(project_id, user_id, session_id):
         cur.execute(
             cur.mogrify(f"""\
                         DELETE FROM public.user_favorite_sessions                          
-                        WHERE 
+                        WHERE user_id = %(userId)s
-                            user_id = %(userId)s
                             AND session_id = %(sessionId)s;""",
                         {"userId": user_id, "sessionId": session_id})
         )
+    sessions_favorite_exp.remove_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id)
     return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False,
                                   include_fav_viewed=True)
 
@@ -65,8 +66,7 @@ def favorite_session_exists(user_id, session_id):
     with pg_client.PostgresClient() as cur:
         cur.execute(
             cur.mogrify(
-                """SELECT 
+                """SELECT session_id                                                
-                        session_id                                                
                     FROM public.user_favorite_sessions 
                     WHERE
                      user_id = %(userId)s

ee/api/chalicelib/core/sessions_favorite_exp.py

@@ -0,0 +1,24 @@
+import logging
+
+from decouple import config
+
+from chalicelib.utils import ch_client, exp_ch_helper
+
+logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO))
+
+
+def add_favorite_session(project_id, user_id, session_id, sign=1):
+    try:
+        with ch_client.ClickHouseClient() as cur:
+            query = f"""INSERT INTO {exp_ch_helper.get_user_favorite_sessions_table()}(project_id,user_id, session_id, sign) 
+                        VALUES (%(project_id)s,%(userId)s,%(sessionId)s,%(sign)s);"""
+            params = {"userId": user_id, "sessionId": session_id, "project_id": project_id, "sign": sign}
+            cur.execute(query=query, params=params)
+
+    except Exception as err:
+        logging.error("------- Exception while adding favorite session to CH")
+        logging.error(err)
+
+
+def remove_favorite_session(project_id, user_id, session_id):
+    add_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id, sign=-1)

ee/api/chalicelib/core/sessions_viewed.py

@@ -0,0 +1,13 @@
+from chalicelib.core import sessions_viewed_exp
+from chalicelib.utils import pg_client
+
+
+def view_session(project_id, user_id, session_id):
+    with pg_client.PostgresClient() as cur:
+        cur.execute(
+            cur.mogrify("""INSERT INTO public.user_viewed_sessions (user_id, session_id) 
+                            VALUES (%(userId)s,%(sessionId)s)
+                            ON CONFLICT DO NOTHING;""",
+                        {"userId": user_id, "sessionId": session_id})
+        )
+    sessions_viewed_exp.view_session(project_id=project_id, user_id=user_id, session_id=session_id)

ee/api/chalicelib/core/sessions_viewed_exp.py

@@ -0,0 +1,17 @@
+from chalicelib.utils import ch_client, exp_ch_helper
+import logging
+from decouple import config
+
+logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO))
+
+
+def view_session(project_id, user_id, session_id):
+    try:
+        with ch_client.ClickHouseClient() as cur:
+            query = f"""INSERT INTO {exp_ch_helper.get_user_viewed_sessions_table()}(project_id, user_id, session_id) 
+                        VALUES (%(project_id)s,%(userId)s,%(sessionId)s);"""
+            params = {"userId": user_id, "sessionId": session_id, "project_id": project_id}
+            cur.execute(query=query, params=params)
+    except Exception as err:
+        logging.error("------- Exception while adding viewed session to CH")
+        logging.error(err)

ee/api/chalicelib/utils/exp_ch_helper.py

@@ -26,5 +26,17 @@ def get_main_resources_table(timestamp):
            and timestamp >= TimeUTC.now(delta_days=-7) else "experimental.resources"
 
 
-def get_autocomplete_table(timestamp):
+def get_autocomplete_table(timestamp=0):
     return "experimental.autocomplete"
+
+
+def get_user_favorite_sessions_table(timestamp=0):
+    return "experimental.user_favorite_sessions"
+
+
+def get_user_viewed_sessions_table(timestamp=0):
+    return "experimental.user_viewed_sessions"
+
+
+def get_user_viewed_errors_table(timestamp=0):
+    return "experimental.user_viewed_errors"

ee/api/clean.sh

@@ -6,7 +6,7 @@ rm -rf ./chalicelib/core/announcements.py
 rm -rf ./chalicelib/core/autocomplete.py
 rm -rf ./chalicelib/core/collaboration_slack.py
 rm -rf ./chalicelib/core/errors.py
-rm -rf ./chalicelib/core/errors_favorite_viewed.py
+rm -rf ./chalicelib/core/errors_favorite.py
 rm -rf ./chalicelib/core/events.py
 rm -rf ./chalicelib/core/events_ios.py
 rm -rf ./chalicelib/core/dashboards.py
@@ -35,7 +35,6 @@ rm -rf ./chalicelib/core/sessions.py
 rm -rf ./chalicelib/core/sessions_assignments.py
 rm -rf ./chalicelib/core/sessions_metas.py
 rm -rf ./chalicelib/core/sessions_mobs.py
-rm -rf ./chalicelib/core/sessions_viewed.py
 rm -rf ./chalicelib/core/significance.py
 rm -rf ./chalicelib/core/slack.py
 rm -rf ./chalicelib/core/socket_ios.py

ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/exp_2_user_favorite_sessions.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS experimental.user_favorite_sessions
+(
+    project_id UInt16,
+    user_id    UInt32,
+    session_id UInt64,
+    _timestamp DateTime DEFAULT now(),
+    sign Int8
+) ENGINE = CollapsingMergeTree(sign)
+      PARTITION BY toYYYYMM(_timestamp)
+      ORDER BY (project_id, user_id, session_id)
+      TTL _timestamp + INTERVAL 3 MONTH;

ee/scripts/helm/db/init_dbs/clickhouse/create/exp_2_user_favorite_sessions.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS experimental.user_favorite_sessions
+(
+    project_id UInt16,
+    user_id    UInt32,
+    session_id UInt64,
+    _timestamp DateTime DEFAULT now(),
+    sign Int8
+) ENGINE = CollapsingMergeTree(sign)
+      PARTITION BY toYYYYMM(_timestamp)
+      ORDER BY (project_id, user_id, session_id)
+      TTL _timestamp + INTERVAL 3 MONTH;