pulled dev

api/chalicelib/core/product_analytics/events.py

@@ -1,6 +1,12 @@
+import logging
+
+import schemas
 from chalicelib.utils import helper
+from chalicelib.utils import sql_helper as sh
 from chalicelib.utils.ch_client import ClickHouseClient
 
+logger = logging.getLogger(__name__)
+
 
 def get_events(project_id: int):
     with ClickHouseClient() as ch_client:
@@ -15,14 +21,88 @@ def get_events(project_id: int):
         return helper.list_to_camel_case(x)
 
 
-def search_events(project_id: int, data: dict):
+def search_events(project_id: int, data: schemas.EventsSearchPayloadSchema):
     with ClickHouseClient() as ch_client:
-        r = ch_client.format(
-            """SELECT * 
-                      FROM product_analytics.events 
-                      WHERE project_id=%(project_id)s
-                      ORDER BY created_at;""",
-            parameters={"project_id": project_id})
-        x = ch_client.execute(r)
+        full_args = {"project_id": project_id, "startDate": data.startTimestamp, "endDate": data.endTimestamp,
+                     "projectId": project_id, "limit": data.limit, "offset": (data.page - 1) * data.limit}
 
-        return helper.list_to_camel_case(x)
+        constraints = ["project_id = %(projectId)s",
+                       "created_at >= toDateTime(%(startDate)s/1000)",
+                       "created_at <= toDateTime(%(endDate)s/1000)"]
+        for i, f in enumerate(data.filters):
+            f.value = helper.values_for_operator(value=f.value, op=f.operator)
+            f_k = f"f_value{i}"
+            full_args = {**full_args, f_k: sh.single_value(f.value), **sh.multi_values(f.value, value_key=f_k)}
+            op = sh.get_sql_operator(f.operator)
+            is_any = sh.isAny_opreator(f.operator)
+            is_undefined = sh.isUndefined_operator(f.operator)
+            full_args = {**full_args, f_k: sh.single_value(f.value), **sh.multi_values(f.value, value_key=f_k)}
+            if f.is_predefined:
+                column = f.name
+            else:
+                column = f"properties.{f.name}"
+
+            if is_any:
+                condition = f"isNotNull({column})"
+            elif is_undefined:
+                condition = f"isNull({column})"
+            else:
+                condition = sh.multi_conditions(f"{column} {op} %({f_k})s", f.value, value_key=f_k)
+            constraints.append(condition)
+
+        ev_constraints = []
+        for i, e in enumerate(data.events):
+            e_k = f"e_value{i}"
+            full_args = {**full_args, e_k: e.event_name}
+            condition = f"`$event_name` = %({e_k})s"
+            sub_conditions = []
+            if len(e.properties.filters) > 0:
+                for j, f in enumerate(e.properties.filters):
+                    p_k = f"e_{i}_p_{j}"
+                    full_args = {**full_args, **sh.multi_values(f.value, value_key=p_k)}
+                    if f.is_predefined:
+                        sub_condition = f"{f.name} {op} %({p_k})s"
+                    else:
+                        sub_condition = f"properties.{f.name} {op} %({p_k})s"
+                    sub_conditions.append(sh.multi_conditions(sub_condition, f.value, value_key=p_k))
+                if len(sub_conditions) > 0:
+                    condition += " AND ("
+                    for j, c in enumerate(sub_conditions):
+                        if j > 0:
+                            condition += " " + e.properties.operators[j - 1] + " " + c
+                        else:
+                            condition += c
+                    condition += ")"
+
+            ev_constraints.append(condition)
+
+        constraints.append("(" + " OR ".join(ev_constraints) + ")")
+        query = ch_client.format(
+            f"""SELECT COUNT(1) OVER () AS total, 
+                            event_id,
+                           `$event_name`,
+                           created_at,
+                           `distinct_id`,
+                           `$browser`,
+                           `$import`,
+                           `$os`,
+                           `$country`,
+                           `$state`,
+                           `$city`,
+                           `$screen_height`,
+                           `$screen_width`,
+                           `$source`,
+                           `$user_id`,
+                           `$device` 
+                      FROM product_analytics.events 
+                      WHERE {" AND ".join(constraints)} 
+                      ORDER BY created_at
+                      LIMIT %(limit)s OFFSET %(offset)s;""",
+            parameters=full_args)
+        rows = ch_client.execute(query)
+        if len(rows) == 0:
+            return {"total": 0, "rows": [], "src": 2}
+        total = rows[0]["total"]
+        for r in rows:
+            r.pop("total")
+        return {"total": total, "rows": rows, "src": 2}

api/routers/subs/product_analytics.py

@@ -3,6 +3,7 @@ from chalicelib.core.product_analytics import events, properties
 from fastapi import Depends
 from or_dependencies import OR_context
 from routers.base import get_routers
+from fastapi import Body, Depends, BackgroundTasks
 
 public_app, app, app_apikey = get_routers()
 
@@ -15,14 +16,13 @@ def get_event_properties(projectId: int, event_name: str = None,
     return {"data": properties.get_properties(project_id=projectId, event_name=event_name)}
 
 
-@app.get('/{projectId}/events/names', tags=["dashboard"])
+@app.get('/{projectId}/events/names', tags=["product_analytics"])
 def get_all_events(projectId: int,
                    context: schemas.CurrentContext = Depends(OR_context)):
     return {"data": events.get_events(project_id=projectId)}
 
 
-@app.post('/{projectId}/events/search', tags=["dashboard"])
-def search_events(projectId: int,
-                  # data: schemas.CreateDashboardSchema = Body(...),
+@app.post('/{projectId}/events/search', tags=["product_analytics"])
+def search_events(projectId: int, data: schemas.EventsSearchPayloadSchema = Body(...),
                   context: schemas.CurrentContext = Depends(OR_context)):
-    return {"data": events.search_events(project_id=projectId, data={})}
+    return {"data": events.search_events(project_id=projectId, data=data)}

api/schemas/__init__.py

@@ -1,2 +1,3 @@
 from .schemas import *
+from .product_analytics import *
 from . import overrides as _overrides

api/schemas/product_analytics.py

@@ -0,0 +1,19 @@
+from typing import Optional, List
+
+from pydantic import Field
+
+from .overrides import BaseModel
+from .schemas import EventPropertiesSchema, SortOrderType, _TimedSchema, \
+    _PaginatedSchema, PropertyFilterSchema
+
+
+class EventSearchSchema(BaseModel):
+    event_name: str = Field(...)
+    properties: Optional[EventPropertiesSchema] = Field(default=None)
+
+
+class EventsSearchPayloadSchema(_TimedSchema, _PaginatedSchema):
+    events: List[EventSearchSchema] = Field(default_factory=list, description="operator between events is OR")
+    filters: List[PropertyFilterSchema] = Field(default_factory=list, description="operator between filters is AND")
+    sort: str = Field(default="startTs")
+    order: SortOrderType = Field(default=SortOrderType.DESC)

ee/api/.gitignore

@@ -292,4 +292,6 @@ Pipfile.lock
 /chalicelib/core/errors/errors_pg.py
 /chalicelib/core/errors/errors_ch.py
 /chalicelib/core/errors/errors_details.py
-/chalicelib/utils/contextual_validators.py
+/chalicelib/utils/contextual_validators.py
+/routers/subs/product_analytics.py
+/schemas/product_analytics.py

ee/api/app.py

@@ -21,7 +21,7 @@ from chalicelib.utils import pg_client, ch_client
 from crons import core_crons, ee_crons, core_dynamic_crons
 from routers import core, core_dynamic
 from routers import ee
-from routers.subs import insights, metrics, v1_api, health, usability_tests, spot, product_anaytics
+from routers.subs import insights, metrics, v1_api, health, usability_tests, spot, product_analytics
 from routers.subs import v1_api_ee
 
 if config("ENABLE_SSO", cast=bool, default=True):
@@ -150,9 +150,9 @@ app.include_router(spot.public_app)
 app.include_router(spot.app)
 app.include_router(spot.app_apikey)
 
-app.include_router(product_anaytics.public_app, prefix="/ap")
-app.include_router(product_anaytics.app, prefix="/ap")
-app.include_router(product_anaytics.app_apikey, prefix="/ap")
+app.include_router(product_analytics.public_app, prefix="/ap")
+app.include_router(product_analytics.app, prefix="/ap")
+app.include_router(product_analytics.app_apikey, prefix="/ap")
 
 if config("ENABLE_SSO", cast=bool, default=True):
     app.include_router(saml.public_app)

ee/api/clean-dev.sh

@@ -113,3 +113,5 @@ rm -rf ./chalicelib/core/errors/errors_pg.py
 rm -rf ./chalicelib/core/errors/errors_ch.py
 rm -rf ./chalicelib/core/errors/errors_details.py
 rm -rf ./chalicelib/utils/contextual_validators.py
+rm -rf ./routers/subs/product_analytics.py
+rm -rf ./schemas/product_analytics.py

ee/api/schemas/__init__.py

@@ -1,4 +1,5 @@
 from .schemas import *
 from .schemas_ee import *
 from .assist_stats_schema import *
+from .product_analytics import *
 from . import overrides as _overrides

ee/backend/pkg/sessions/storage.go

@@ -121,7 +121,16 @@ func (s *storageImpl) Get(sessionID uint64) (*Session, error) {
 
 // For the ender service only
 func (s *storageImpl) GetMany(sessionIDs []uint64) ([]*Session, error) {
-	rows, err := s.db.Query("SELECT session_id, COALESCE( duration, 0 ), start_ts FROM sessions WHERE session_id = ANY($1)", pq.Array(sessionIDs))
+	rows, err := s.db.Query(`
+	SELECT 
+		session_id, 
+		CASE 
+			WHEN duration IS NULL OR duration < 0 THEN 0 
+			ELSE duration 
+		END, 
+		start_ts 
+	FROM sessions 
+	WHERE session_id = ANY($1)`, pq.Array(sessionIDs))
 	if err != nil {
 		return nil, err
 	}

frontend/app/player/web/assist/AssistManager.ts

@@ -185,7 +185,7 @@ export default class AssistManager {
       const socket: Socket = (this.socket = io(urlObject.origin, {
         withCredentials: true,
         multiplex: true,
-        transports: ['polling', 'websocket'],
+        transports: ['websocket'],
         path: '/ws-assist/socket',
         auth: {
           token: agentToken,

scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml

@@ -11,12 +11,24 @@ metadata:
   annotations:
     nginx.ingress.kubernetes.io/rewrite-target: /$1
     nginx.ingress.kubernetes.io/configuration-snippet: |
-      # Extract sessionID from peerId using regex
-      if ($arg_peerId ~ ".*-(?<extracted_sid>[^-]+)-.*") {
-        set $session_id $extracted_sid;
-      }
-      add_header X-Debug-Session-ID $session_id;
-      add_header X-Debug-Session-Type "wss";
+      #set $sticky_used "no";
+      #if ($sessionid != "") {
+      #  set $sticky_used "yes";
+      #}
+
+      #add_header X-Debug-Session-ID $sessionid;
+      #add_header X-Debug-Session-Type "wss";
+      #add_header X-Sticky-Session-Used $sticky_used;
+      #add_header X-Upstream-Server $upstream_addr;
+
+      proxy_hide_header access-control-allow-headers;
+      proxy_hide_header Access-Control-Allow-Origin;
+      add_header 'Access-Control-Allow-Origin' $http_origin always;
+      add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always;
+      add_header 'Access-Control-Allow-Headers' 'sessionid, Content-Type, Authorization' always;
+      add_header 'Access-Control-Max-Age' 1728000;
+      add_header 'Content-Type' 'text/plain charset=UTF-8';
+
     nginx.ingress.kubernetes.io/upstream-hash-by: $session_id
 
   {{- with .Values.ingress.annotations }}

scripts/helmcharts/openreplay/charts/assist/values.yaml

@@ -77,16 +77,11 @@ ingress:
       # CORS configuration
       # We don't need the upstream header
       proxy_hide_header Access-Control-Allow-Origin;
-      add_header 'Access-Control-Allow-Credentials' 'true';
       add_header 'Access-Control-Allow-Origin' $http_origin always;
       add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always;
       add_header 'Access-Control-Allow-Headers' 'sessionid, Content-Type, Authorization' always;
       add_header 'Access-Control-Max-Age' 1728000;
       add_header 'Content-Type' 'text/plain charset=UTF-8';
-      add_header 'Content-Length' 0;
-      if ($request_method = 'OPTIONS') {
-        return 204;
-      }
     nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
     # kubernetes.io/ingress.class: nginx

scripts/helmcharts/openreplay/values.yaml

@@ -17,6 +17,13 @@ redis: &redis
 ingress-nginx:
   enabled: true
   controller:
+    config:
+      http-snippet: |-
+        # Extract sessionid from peerId, it'll be used for sticky session.
+        map $arg_peerId $sessionid {
+          default "";
+          "~.*-(\d+)(?:-.*|$)" $1;
+        }
     admissionWebhooks:
       patch:
         podAnnotations:

tracker/tracker-assist/src/Assist.ts

@@ -241,7 +241,7 @@ export default class Assist {
       extraHeaders: {
         sessionId,
       },
-      transports: ['polling', 'websocket',],
+      transports: ['websocket',],
       withCredentials: true,
       reconnection: true,
       reconnectionAttempts: 30,
@@ -852,4 +852,4 @@ export default class Assist {
  *               // })
  *               // slPeer.on('error', console.error)
  *               // this.emit('canvas_stream', { canvasId, })
- * */
+ * */