diff --git a/api/chalicelib/core/product_analytics/events.py b/api/chalicelib/core/product_analytics/events.py index 6e1e2e2cc..4b626d4aa 100644 --- a/api/chalicelib/core/product_analytics/events.py +++ b/api/chalicelib/core/product_analytics/events.py @@ -1,6 +1,12 @@ +import logging + +import schemas from chalicelib.utils import helper +from chalicelib.utils import sql_helper as sh from chalicelib.utils.ch_client import ClickHouseClient +logger = logging.getLogger(__name__) + def get_events(project_id: int): with ClickHouseClient() as ch_client: @@ -15,14 +21,88 @@ def get_events(project_id: int): return helper.list_to_camel_case(x) -def search_events(project_id: int, data: dict): +def search_events(project_id: int, data: schemas.EventsSearchPayloadSchema): with ClickHouseClient() as ch_client: - r = ch_client.format( - """SELECT * - FROM product_analytics.events - WHERE project_id=%(project_id)s - ORDER BY created_at;""", - parameters={"project_id": project_id}) - x = ch_client.execute(r) + full_args = {"project_id": project_id, "startDate": data.startTimestamp, "endDate": data.endTimestamp, + "projectId": project_id, "limit": data.limit, "offset": (data.page - 1) * data.limit} - return helper.list_to_camel_case(x) + constraints = ["project_id = %(projectId)s", + "created_at >= toDateTime(%(startDate)s/1000)", + "created_at <= toDateTime(%(endDate)s/1000)"] + for i, f in enumerate(data.filters): + f.value = helper.values_for_operator(value=f.value, op=f.operator) + f_k = f"f_value{i}" + full_args = {**full_args, f_k: sh.single_value(f.value), **sh.multi_values(f.value, value_key=f_k)} + op = sh.get_sql_operator(f.operator) + is_any = sh.isAny_opreator(f.operator) + is_undefined = sh.isUndefined_operator(f.operator) + full_args = {**full_args, f_k: sh.single_value(f.value), **sh.multi_values(f.value, value_key=f_k)} + if f.is_predefined: + column = f.name + else: + column = f"properties.{f.name}" + + if is_any: + condition = f"isNotNull({column})" + elif is_undefined: + condition = f"isNull({column})" + else: + condition = sh.multi_conditions(f"{column} {op} %({f_k})s", f.value, value_key=f_k) + constraints.append(condition) + + ev_constraints = [] + for i, e in enumerate(data.events): + e_k = f"e_value{i}" + full_args = {**full_args, e_k: e.event_name} + condition = f"`$event_name` = %({e_k})s" + sub_conditions = [] + if len(e.properties.filters) > 0: + for j, f in enumerate(e.properties.filters): + p_k = f"e_{i}_p_{j}" + full_args = {**full_args, **sh.multi_values(f.value, value_key=p_k)} + if f.is_predefined: + sub_condition = f"{f.name} {op} %({p_k})s" + else: + sub_condition = f"properties.{f.name} {op} %({p_k})s" + sub_conditions.append(sh.multi_conditions(sub_condition, f.value, value_key=p_k)) + if len(sub_conditions) > 0: + condition += " AND (" + for j, c in enumerate(sub_conditions): + if j > 0: + condition += " " + e.properties.operators[j - 1] + " " + c + else: + condition += c + condition += ")" + + ev_constraints.append(condition) + + constraints.append("(" + " OR ".join(ev_constraints) + ")") + query = ch_client.format( + f"""SELECT COUNT(1) OVER () AS total, + event_id, + `$event_name`, + created_at, + `distinct_id`, + `$browser`, + `$import`, + `$os`, + `$country`, + `$state`, + `$city`, + `$screen_height`, + `$screen_width`, + `$source`, + `$user_id`, + `$device` + FROM product_analytics.events + WHERE {" AND ".join(constraints)} + ORDER BY created_at + LIMIT %(limit)s OFFSET %(offset)s;""", + parameters=full_args) + rows = ch_client.execute(query) + if len(rows) == 0: + return {"total": 0, "rows": [], "src": 2} + total = rows[0]["total"] + for r in rows: + r.pop("total") + return {"total": total, "rows": rows, "src": 2} diff --git a/api/routers/subs/product_analytics.py b/api/routers/subs/product_analytics.py index 5bfe42de7..d8591b30d 100644 --- a/api/routers/subs/product_analytics.py +++ b/api/routers/subs/product_analytics.py @@ -3,6 +3,7 @@ from chalicelib.core.product_analytics import events, properties from fastapi import Depends from or_dependencies import OR_context from routers.base import get_routers +from fastapi import Body, Depends, BackgroundTasks public_app, app, app_apikey = get_routers() @@ -15,14 +16,13 @@ def get_event_properties(projectId: int, event_name: str = None, return {"data": properties.get_properties(project_id=projectId, event_name=event_name)} -@app.get('/{projectId}/events/names', tags=["dashboard"]) +@app.get('/{projectId}/events/names', tags=["product_analytics"]) def get_all_events(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": events.get_events(project_id=projectId)} -@app.post('/{projectId}/events/search', tags=["dashboard"]) -def search_events(projectId: int, - # data: schemas.CreateDashboardSchema = Body(...), +@app.post('/{projectId}/events/search', tags=["product_analytics"]) +def search_events(projectId: int, data: schemas.EventsSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": events.search_events(project_id=projectId, data={})} + return {"data": events.search_events(project_id=projectId, data=data)} diff --git a/api/schemas/__init__.py b/api/schemas/__init__.py index 6013d7c2b..4bcc5be80 100644 --- a/api/schemas/__init__.py +++ b/api/schemas/__init__.py @@ -1,2 +1,3 @@ from .schemas import * +from .product_analytics import * from . import overrides as _overrides diff --git a/api/schemas/product_analytics.py b/api/schemas/product_analytics.py new file mode 100644 index 000000000..bd4647b4e --- /dev/null +++ b/api/schemas/product_analytics.py @@ -0,0 +1,19 @@ +from typing import Optional, List + +from pydantic import Field + +from .overrides import BaseModel +from .schemas import EventPropertiesSchema, SortOrderType, _TimedSchema, \ + _PaginatedSchema, PropertyFilterSchema + + +class EventSearchSchema(BaseModel): + event_name: str = Field(...) + properties: Optional[EventPropertiesSchema] = Field(default=None) + + +class EventsSearchPayloadSchema(_TimedSchema, _PaginatedSchema): + events: List[EventSearchSchema] = Field(default_factory=list, description="operator between events is OR") + filters: List[PropertyFilterSchema] = Field(default_factory=list, description="operator between filters is AND") + sort: str = Field(default="startTs") + order: SortOrderType = Field(default=SortOrderType.DESC) diff --git a/ee/api/.gitignore b/ee/api/.gitignore index d46a28ff0..de55d4df4 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -292,4 +292,6 @@ Pipfile.lock /chalicelib/core/errors/errors_pg.py /chalicelib/core/errors/errors_ch.py /chalicelib/core/errors/errors_details.py -/chalicelib/utils/contextual_validators.py \ No newline at end of file +/chalicelib/utils/contextual_validators.py +/routers/subs/product_analytics.py +/schemas/product_analytics.py diff --git a/ee/api/app.py b/ee/api/app.py index e1dece2dd..5b3af9d80 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -21,7 +21,7 @@ from chalicelib.utils import pg_client, ch_client from crons import core_crons, ee_crons, core_dynamic_crons from routers import core, core_dynamic from routers import ee -from routers.subs import insights, metrics, v1_api, health, usability_tests, spot, product_anaytics +from routers.subs import insights, metrics, v1_api, health, usability_tests, spot, product_analytics from routers.subs import v1_api_ee if config("ENABLE_SSO", cast=bool, default=True): @@ -150,9 +150,9 @@ app.include_router(spot.public_app) app.include_router(spot.app) app.include_router(spot.app_apikey) -app.include_router(product_anaytics.public_app, prefix="/ap") -app.include_router(product_anaytics.app, prefix="/ap") -app.include_router(product_anaytics.app_apikey, prefix="/ap") +app.include_router(product_analytics.public_app, prefix="/ap") +app.include_router(product_analytics.app, prefix="/ap") +app.include_router(product_analytics.app_apikey, prefix="/ap") if config("ENABLE_SSO", cast=bool, default=True): app.include_router(saml.public_app) diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index abfa59369..b64c7ac97 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -113,3 +113,5 @@ rm -rf ./chalicelib/core/errors/errors_pg.py rm -rf ./chalicelib/core/errors/errors_ch.py rm -rf ./chalicelib/core/errors/errors_details.py rm -rf ./chalicelib/utils/contextual_validators.py +rm -rf ./routers/subs/product_analytics.py +rm -rf ./schemas/product_analytics.py \ No newline at end of file diff --git a/ee/api/schemas/__init__.py b/ee/api/schemas/__init__.py index 37ecaab44..4089e6867 100644 --- a/ee/api/schemas/__init__.py +++ b/ee/api/schemas/__init__.py @@ -1,4 +1,5 @@ from .schemas import * from .schemas_ee import * from .assist_stats_schema import * +from .product_analytics import * from . import overrides as _overrides diff --git a/ee/backend/pkg/sessions/storage.go b/ee/backend/pkg/sessions/storage.go index 41602c42a..e56dfbad8 100644 --- a/ee/backend/pkg/sessions/storage.go +++ b/ee/backend/pkg/sessions/storage.go @@ -121,7 +121,16 @@ func (s *storageImpl) Get(sessionID uint64) (*Session, error) { // For the ender service only func (s *storageImpl) GetMany(sessionIDs []uint64) ([]*Session, error) { - rows, err := s.db.Query("SELECT session_id, COALESCE( duration, 0 ), start_ts FROM sessions WHERE session_id = ANY($1)", pq.Array(sessionIDs)) + rows, err := s.db.Query(` + SELECT + session_id, + CASE + WHEN duration IS NULL OR duration < 0 THEN 0 + ELSE duration + END, + start_ts + FROM sessions + WHERE session_id = ANY($1)`, pq.Array(sessionIDs)) if err != nil { return nil, err } diff --git a/frontend/app/player/web/assist/AssistManager.ts b/frontend/app/player/web/assist/AssistManager.ts index 944b19b92..1a204cbea 100644 --- a/frontend/app/player/web/assist/AssistManager.ts +++ b/frontend/app/player/web/assist/AssistManager.ts @@ -185,7 +185,7 @@ export default class AssistManager { const socket: Socket = (this.socket = io(urlObject.origin, { withCredentials: true, multiplex: true, - transports: ['polling', 'websocket'], + transports: ['websocket'], path: '/ws-assist/socket', auth: { token: agentToken, diff --git a/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml index e553d8d70..3094af0a9 100644 --- a/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/templates/ingress.yaml @@ -11,12 +11,24 @@ metadata: annotations: nginx.ingress.kubernetes.io/rewrite-target: /$1 nginx.ingress.kubernetes.io/configuration-snippet: | - # Extract sessionID from peerId using regex - if ($arg_peerId ~ ".*-(?[^-]+)-.*") { - set $session_id $extracted_sid; - } - add_header X-Debug-Session-ID $session_id; - add_header X-Debug-Session-Type "wss"; + #set $sticky_used "no"; + #if ($sessionid != "") { + # set $sticky_used "yes"; + #} + + #add_header X-Debug-Session-ID $sessionid; + #add_header X-Debug-Session-Type "wss"; + #add_header X-Sticky-Session-Used $sticky_used; + #add_header X-Upstream-Server $upstream_addr; + + proxy_hide_header access-control-allow-headers; + proxy_hide_header Access-Control-Allow-Origin; + add_header 'Access-Control-Allow-Origin' $http_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'sessionid, Content-Type, Authorization' always; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + nginx.ingress.kubernetes.io/upstream-hash-by: $session_id {{- with .Values.ingress.annotations }} diff --git a/scripts/helmcharts/openreplay/charts/assist/values.yaml b/scripts/helmcharts/openreplay/charts/assist/values.yaml index c4b7622bc..a3a14eff5 100644 --- a/scripts/helmcharts/openreplay/charts/assist/values.yaml +++ b/scripts/helmcharts/openreplay/charts/assist/values.yaml @@ -77,16 +77,11 @@ ingress: # CORS configuration # We don't need the upstream header proxy_hide_header Access-Control-Allow-Origin; - add_header 'Access-Control-Allow-Credentials' 'true'; add_header 'Access-Control-Allow-Origin' $http_origin always; add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always; add_header 'Access-Control-Allow-Headers' 'sessionid, Content-Type, Authorization' always; add_header 'Access-Control-Max-Age' 1728000; add_header 'Content-Type' 'text/plain charset=UTF-8'; - add_header 'Content-Length' 0; - if ($request_method = 'OPTIONS') { - return 204; - } nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" # kubernetes.io/ingress.class: nginx diff --git a/scripts/helmcharts/openreplay/values.yaml b/scripts/helmcharts/openreplay/values.yaml index 9d90dfef2..8be97a2dc 100644 --- a/scripts/helmcharts/openreplay/values.yaml +++ b/scripts/helmcharts/openreplay/values.yaml @@ -17,6 +17,13 @@ redis: &redis ingress-nginx: enabled: true controller: + config: + http-snippet: |- + # Extract sessionid from peerId, it'll be used for sticky session. + map $arg_peerId $sessionid { + default ""; + "~.*-(\d+)(?:-.*|$)" $1; + } admissionWebhooks: patch: podAnnotations: diff --git a/tracker/tracker-assist/src/Assist.ts b/tracker/tracker-assist/src/Assist.ts index 21b72781e..c06d7ab76 100644 --- a/tracker/tracker-assist/src/Assist.ts +++ b/tracker/tracker-assist/src/Assist.ts @@ -241,7 +241,7 @@ export default class Assist { extraHeaders: { sessionId, }, - transports: ['polling', 'websocket',], + transports: ['websocket',], withCredentials: true, reconnection: true, reconnectionAttempts: 30, @@ -852,4 +852,4 @@ export default class Assist { * // }) * // slPeer.on('error', console.error) * // this.emit('canvas_stream', { canvasId, }) - * */ \ No newline at end of file + * */