diff --git a/api/chalicelib/core/sessions/__init__.py b/api/chalicelib/core/sessions/__init__.py index 5d9ffc497..75797c824 100644 --- a/api/chalicelib/core/sessions/__init__.py +++ b/api/chalicelib/core/sessions/__init__.py @@ -6,8 +6,18 @@ logger = logging.getLogger(__name__) from . import sessions_pg from . import sessions_pg as sessions_legacy from . import sessions_ch +from . import sessions_search_pg +from . import sessions_search_pg as sessions_search_legacy -if config("EXP_METRICS", cast=bool, default=False): +if config("EXP_SESSIONS_SEARCH", cast=bool, default=False): + logger.info(">>> Using experimental sessions search") from . import sessions_ch as sessions + from . import sessions_search_ch as sessions_search else: from . import sessions_pg as sessions + from . import sessions_search_pg as sessions_search + +# if config("EXP_METRICS", cast=bool, default=False): +# from . import sessions_ch as sessions +# else: +# from . import sessions_pg as sessions diff --git a/ee/api/chalicelib/core/sessions/sessions_legacy_mobil.py b/api/chalicelib/core/sessions/sessions_legacy_mobil.py similarity index 99% rename from ee/api/chalicelib/core/sessions/sessions_legacy_mobil.py rename to api/chalicelib/core/sessions/sessions_legacy_mobil.py index 84b3187ed..3c5263b37 100644 --- a/ee/api/chalicelib/core/sessions/sessions_legacy_mobil.py +++ b/api/chalicelib/core/sessions/sessions_legacy_mobil.py @@ -1,6 +1,5 @@ import ast import logging -from typing import List, Union import schemas from chalicelib.core import events, metadata, projects diff --git a/ee/api/chalicelib/core/sessions/sessions_search_exp.py b/api/chalicelib/core/sessions/sessions_search_ch.py similarity index 100% rename from ee/api/chalicelib/core/sessions/sessions_search_exp.py rename to api/chalicelib/core/sessions/sessions_search_ch.py diff --git a/api/chalicelib/core/sessions/sessions_search.py b/api/chalicelib/core/sessions/sessions_search_pg.py similarity index 100% rename from api/chalicelib/core/sessions/sessions_search.py rename to api/chalicelib/core/sessions/sessions_search_pg.py diff --git a/api/env.default b/api/env.default index 947e3ad12..383e74273 100644 --- a/api/env.default +++ b/api/env.default @@ -74,4 +74,5 @@ EXP_CH_DRIVER=true EXP_AUTOCOMPLETE=true EXP_ALERTS=true EXP_ERRORS_SEARCH=true -EXP_METRICS=true \ No newline at end of file +EXP_METRICS=true +EXP_SESSIONS_SEARCH=true \ No newline at end of file diff --git a/api/schemas/schemas.py b/api/schemas/schemas.py index 218b61d2a..067a00c74 100644 --- a/api/schemas/schemas.py +++ b/api/schemas/schemas.py @@ -1529,3 +1529,30 @@ class TagCreate(TagUpdate): class ScopeSchema(BaseModel): scope: int = Field(default=1, ge=1, le=2) + + +class SessionModel(BaseModel): + duration: int + errorsCount: int + eventsCount: int + favorite: bool = Field(default=False) + issueScore: int + issueTypes: List[IssueType] = Field(default=[]) + metadata: dict = Field(default={}) + pagesCount: int + platform: str + projectId: int + sessionId: str + startTs: int + timezone: Optional[str] + userAnonymousId: Optional[str] + userBrowser: str + userCity: str + userCountry: str + userDevice: Optional[str] + userDeviceType: str + userId: Optional[str] + userOs: str + userState: str + userUuid: str + viewed: bool = Field(default=False) diff --git a/ee/api/.gitignore b/ee/api/.gitignore index ebb54e743..9317c3d12 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -223,6 +223,9 @@ Pipfile.lock /chalicelib/core/sessions/performance_event.py /chalicelib/core/sessions/sessions_viewed/sessions_viewed.py /chalicelib/core/sessions/unprocessed_sessions.py +/chalicelib/core/sessions/__init__.py +/chalicelib/core/sessions/sessions_legacy_mobil.py +/chalicelib/core/sessions/sessions_search_exp.py /chalicelib/core/metrics/modules /chalicelib/core/socket_ios.py /chalicelib/core/sourcemaps @@ -288,4 +291,4 @@ Pipfile.lock /chalicelib/core/errors/errors_pg.py /chalicelib/core/errors/errors_ch.py /chalicelib/core/errors/errors_details.py -/chalicelib/utils/contextual_validators.py +/chalicelib/utils/contextual_validators.py \ No newline at end of file diff --git a/ee/api/chalicelib/core/sessions/__init__.py b/ee/api/chalicelib/core/sessions/__init__.py deleted file mode 100644 index ebcb11644..000000000 --- a/ee/api/chalicelib/core/sessions/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging - -from decouple import config - -logger = logging.getLogger(__name__) -from . import sessions_pg -from . import sessions_pg as sessions_legacy -from . import sessions_ch -from . import sessions_search as sessions_search_legacy - -if config("EXP_SESSIONS_SEARCH", cast=bool, default=False): - logger.info(">>> Using experimental sessions search") - from . import sessions_ch as sessions - from . import sessions_search_exp as sessions_search -else: - from . import sessions_pg as sessions - from . import sessions_search as sessions_search diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index 357421c10..bfd7a7a90 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -44,6 +44,9 @@ rm -rf ./chalicelib/core/sessions/sessions_search.py rm -rf ./chalicelib/core/sessions/performance_event.py rm -rf ./chalicelib/core/sessions/sessions_viewed/sessions_viewed.py rm -rf ./chalicelib/core/sessions/unprocessed_sessions.py +rm -rf ./chalicelib/core/sessions/__init__.py +rm -rf ./chalicelib/core/sessions/sessions_legacy_mobil.py +rm -rf ./chalicelib/core/sessions/sessions_search_exp.py rm -rf ./chalicelib/core/metrics/modules rm -rf ./chalicelib/core/socket_ios.py rm -rf ./chalicelib/core/sourcemaps diff --git a/ee/api/schemas/schemas_ee.py b/ee/api/schemas/schemas_ee.py index a5cc5c78f..394f88859 100644 --- a/ee/api/schemas/schemas_ee.py +++ b/ee/api/schemas/schemas_ee.py @@ -4,7 +4,7 @@ from pydantic import Field, EmailStr, field_validator, model_validator from chalicelib.utils.TimeUTC import TimeUTC from . import schemas -from .overrides import BaseModel, Enum, ORUnion +from .overrides import BaseModel, Enum from .transformers_validators import remove_whitespace @@ -91,33 +91,6 @@ class TrailSearchPayloadSchema(schemas._PaginatedSchema): return values -class SessionModel(BaseModel): - duration: int - errorsCount: int - eventsCount: int - favorite: bool = Field(default=False) - issueScore: int - issueTypes: List[schemas.IssueType] = Field(default=[]) - metadata: dict = Field(default={}) - pagesCount: int - platform: str - projectId: int - sessionId: str - startTs: int - timezone: Optional[str] - userAnonymousId: Optional[str] - userBrowser: str - userCity: str - userCountry: str - userDevice: Optional[str] - userDeviceType: str - userId: Optional[str] - userOs: str - userState: str - userUuid: str - viewed: bool = Field(default=False) - - class AssistRecordUpdatePayloadSchema(BaseModel): name: str = Field(..., min_length=1) _transform_name = field_validator('name', mode="before")(remove_whitespace) diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql new file mode 100644 index 000000000..39c0822da --- /dev/null +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql @@ -0,0 +1 @@ +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.23.0-ee'; diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index b3d156134..12ef017f2 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.22.0-ee'; +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.23.0-ee'; CREATE DATABASE IF NOT EXISTS experimental; CREATE TABLE IF NOT EXISTS experimental.autocomplete diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.23.0/1.23.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.23.0/1.23.0.sql new file mode 100644 index 000000000..957be6dfd --- /dev/null +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.23.0/1.23.0.sql @@ -0,0 +1,30 @@ +\set previous_version 'v1.22.0-ee' +\set next_version 'v1.23.0-ee' +SELECT openreplay_version() AS current_version, + openreplay_version() = :'previous_version' AS valid_previous, + openreplay_version() = :'next_version' AS is_next +\gset + +\if :valid_previous +\echo valid previous DB version :'previous_version', starting DB upgrade to :'next_version' +BEGIN; +SELECT format($fn_def$ +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT '%1$s' +$$ LANGUAGE sql IMMUTABLE; +$fn_def$, :'next_version') +\gexec + +-- + + + +COMMIT; + +\elif :is_next +\echo new version detected :'next_version', nothing to do +\else +\warn skipping DB upgrade of :'next_version', expected previous version :'previous_version', found :'current_version' +\endif diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index af64c00d4..0fc74fb1c 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -1,4 +1,4 @@ -\set or_version 'v1.22.0-ee' +\set or_version 'v1.23.0-ee' SET client_min_messages TO NOTICE; \set ON_ERROR_STOP true SELECT EXISTS (SELECT 1 diff --git a/ee/scripts/schema/db/rollback_dbs/clickhouse/1.23.0/1.23.0.sql b/ee/scripts/schema/db/rollback_dbs/clickhouse/1.23.0/1.23.0.sql new file mode 100644 index 000000000..caac93e06 --- /dev/null +++ b/ee/scripts/schema/db/rollback_dbs/clickhouse/1.23.0/1.23.0.sql @@ -0,0 +1 @@ +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.22.0-ee'; diff --git a/ee/scripts/schema/db/rollback_dbs/postgresql/1.23.0/1.23.0.sql b/ee/scripts/schema/db/rollback_dbs/postgresql/1.23.0/1.23.0.sql new file mode 100644 index 000000000..ba1cecc07 --- /dev/null +++ b/ee/scripts/schema/db/rollback_dbs/postgresql/1.23.0/1.23.0.sql @@ -0,0 +1,27 @@ +\set previous_version 'v1.23.0-ee' +\set next_version 'v1.22.0-ee' +SELECT openreplay_version() AS current_version, + openreplay_version() = :'previous_version' AS valid_previous, + openreplay_version() = :'next_version' AS is_next +\gset + +\if :valid_previous +\echo valid previous DB version :'previous_version', starting DB downgrade to :'next_version' +BEGIN; +SELECT format($fn_def$ +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT '%1$s' +$$ LANGUAGE sql IMMUTABLE; +$fn_def$, :'next_version') +\gexec + + +COMMIT; + +\elif :is_next +\echo new version detected :'next_version', nothing to do +\else +\warn skipping DB downgrade of :'next_version', expected previous version :'previous_version', found :'current_version' +\endif \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql b/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql new file mode 100644 index 000000000..7b5e3ad91 --- /dev/null +++ b/scripts/schema/db/init_dbs/clickhouse/1.23.0/1.23.0.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.23.0'; + + +CREATE TABLE IF NOT EXISTS experimental.user_viewed_sessions +( + project_id UInt16, + user_id UInt32, + session_id UInt64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(_timestamp) + ORDER BY (project_id, user_id, session_id) + TTL _timestamp + INTERVAL 3 MONTH; diff --git a/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 150b7661e..cee2400d2 100644 --- a/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.22.0'; +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.23.0'; CREATE DATABASE IF NOT EXISTS experimental; CREATE TABLE IF NOT EXISTS experimental.autocomplete @@ -532,3 +532,15 @@ CREATE TABLE IF NOT EXISTS product_analytics.all_properties _timestamp DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_timestamp) ORDER BY (project_id, property_name, is_event_property); + + +CREATE TABLE IF NOT EXISTS experimental.user_viewed_sessions +( + project_id UInt16, + user_id UInt32, + session_id UInt64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + PARTITION BY toYYYYMM(_timestamp) + ORDER BY (project_id, user_id, session_id) + TTL _timestamp + INTERVAL 3 MONTH; diff --git a/scripts/schema/db/init_dbs/postgresql/1.23.0/1.23.0.sql b/scripts/schema/db/init_dbs/postgresql/1.23.0/1.23.0.sql new file mode 100644 index 000000000..541f5f808 --- /dev/null +++ b/scripts/schema/db/init_dbs/postgresql/1.23.0/1.23.0.sql @@ -0,0 +1,30 @@ +\set previous_version 'v1.22.0' +\set next_version 'v1.23.0' +SELECT openreplay_version() AS current_version, + openreplay_version() = :'previous_version' AS valid_previous, + openreplay_version() = :'next_version' AS is_next +\gset + +\if :valid_previous +\echo valid previous DB version :'previous_version', starting DB upgrade to :'next_version' +BEGIN; +SELECT format($fn_def$ +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT '%1$s' +$$ LANGUAGE sql IMMUTABLE; +$fn_def$, :'next_version') +\gexec + +-- + + + +COMMIT; + +\elif :is_next +\echo new version detected :'next_version', nothing to do +\else +\warn skipping DB upgrade of :'next_version', expected previous version :'previous_version', found :'current_version' +\endif diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index b10d2cc3d..2a5a79473 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -1,4 +1,4 @@ -\set or_version 'v1.22.0' +\set or_version 'v1.23.0' SET client_min_messages TO NOTICE; \set ON_ERROR_STOP true SELECT EXISTS (SELECT 1 diff --git a/scripts/schema/db/rollback_dbs/clickhouse/1.23.0/1.23.0.sql b/scripts/schema/db/rollback_dbs/clickhouse/1.23.0/1.23.0.sql new file mode 100644 index 000000000..dee235cd0 --- /dev/null +++ b/scripts/schema/db/rollback_dbs/clickhouse/1.23.0/1.23.0.sql @@ -0,0 +1,4 @@ +CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.22.0'; + + +DROP TABLE IF EXISTS experimental.user_viewed_sessions; \ No newline at end of file diff --git a/scripts/schema/db/rollback_dbs/postgresql/1.23.0/1.23.0.sql b/scripts/schema/db/rollback_dbs/postgresql/1.23.0/1.23.0.sql new file mode 100644 index 000000000..481eb3200 --- /dev/null +++ b/scripts/schema/db/rollback_dbs/postgresql/1.23.0/1.23.0.sql @@ -0,0 +1,27 @@ +\set previous_version 'v1.23.0' +\set next_version 'v1.22.0' +SELECT openreplay_version() AS current_version, + openreplay_version() = :'previous_version' AS valid_previous, + openreplay_version() = :'next_version' AS is_next +\gset + +\if :valid_previous +\echo valid previous DB version :'previous_version', starting DB downgrade to :'next_version' +BEGIN; +SELECT format($fn_def$ +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT '%1$s' +$$ LANGUAGE sql IMMUTABLE; +$fn_def$, :'next_version') +\gexec + + +COMMIT; + +\elif :is_next +\echo new version detected :'next_version', nothing to do +\else +\warn skipping DB downgrade of :'next_version', expected previous version :'previous_version', found :'current_version' +\endif \ No newline at end of file