diff --git a/api/Pipfile b/api/Pipfile index b70bb321d..dd8d6f5cf 100644 --- a/api/Pipfile +++ b/api/Pipfile @@ -4,25 +4,26 @@ verify_ssl = true name = "pypi" [packages] -urllib3 = "==2.2.3" +urllib3 = "==2.3.0" requests = "==2.32.3" -boto3 = "==1.35.86" +boto3 = "==1.36.9" pyjwt = "==2.10.1" psycopg2-binary = "==2.9.10" -clickhouse-connect = "==0.8.11" -elasticsearch = "==8.17.0" +psycopg = {extras = ["pool", "binary"], version = "==3.2.4"} +clickhouse-driver = {extras = ["lz4"], version = "==0.2.9"} +clickhouse-connect = "==0.8.15" +elasticsearch = "==8.17.1" jira = "==3.8.0" cachetools = "==5.5.0" -fastapi = "==0.115.6" +fastapi = "==0.115.7" +uvicorn = {extras = ["standard"], version = "==0.34.0"} python-decouple = "==3.8" +pydantic = {extras = ["email"], version = "==2.10.6"} apscheduler = "==3.11.0" redis = "==5.2.1" -psycopg = {extras = ["binary", "pool"], version = "==3.2.3"} -clickhouse-driver = {extras = ["lz4"], version = "==0.2.9"} -uvicorn = {extras = ["standard"], version = "==0.34.0"} -pydantic = {extras = ["email"], version = "==2.10.4"} [dev-packages] [requires] python_version = "3.12" +python_full_version = "3.12.8" diff --git a/api/chalicelib/core/boarding.py b/api/chalicelib/core/boarding.py index 26f2e3dcb..50e2ca713 100644 --- a/api/chalicelib/core/boarding.py +++ b/api/chalicelib/core/boarding.py @@ -21,7 +21,7 @@ def get_state(tenant_id): recorded = cur.fetchone()["exists"] meta = False if recorded: - query = cur.mogrify("""SELECT EXISTS((SELECT 1 + query = cur.mogrify(f"""SELECT EXISTS((SELECT 1 FROM public.projects AS p LEFT JOIN LATERAL ( SELECT 1 FROM public.sessions diff --git a/api/chalicelib/core/metrics/product_analytics_ch.py b/api/chalicelib/core/metrics/product_analytics_ch.py index 5f0521b78..273c08042 100644 --- a/api/chalicelib/core/metrics/product_analytics_ch.py +++ b/api/chalicelib/core/metrics/product_analytics_ch.py @@ -36,8 +36,8 @@ def path_analysis(project_id: int, data: schemas.CardPathAnalysis): start_points_conditions = [] step_0_conditions = [] step_1_post_conditions = ["event_number_in_session <= %(density)s"] - q2_extra_col = "" - q2_extra_condition = "" + q2_extra_col = None + q2_extra_condition = None if len(data.metric_value) == 0: data.metric_value.append(schemas.ProductAnalyticsSelectedEventType.LOCATION) sub_events.append({"column": JOURNEY_TYPES[schemas.ProductAnalyticsSelectedEventType.LOCATION]["column"], @@ -53,7 +53,7 @@ def path_analysis(project_id: int, data: schemas.CardPathAnalysis): f"(`$event_name`='{JOURNEY_TYPES[s.type]["eventType"]}' AND event_number_in_session = 1 \ OR `$event_name`!='{JOURNEY_TYPES[s.type]["eventType"]}' AND event_number_in_session > 1)") extra_metric_values.append(s.type) - if q2_extra_col == "": + if not q2_extra_col: # This is used in case start event has different type of the visible event, # because it causes intermediary events to be removed, so you find a jump from step-0 to step-3 # because step-2 is not of a visible event @@ -449,11 +449,11 @@ WITH pre_ranked_events AS (SELECT * leadInFrame(toNullable(`$event_name`)) OVER (PARTITION BY session_id ORDER BY created_at {path_direction} ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS next_type - {q2_extra_col%path_direction} + {q2_extra_col % path_direction if q2_extra_col else ""} FROM start_points INNER JOIN pre_ranked_events USING (session_id)) SELECT * FROM ranked_events -{q2_extra_condition}""" +{q2_extra_condition if q2_extra_condition else ""}""" logger.debug("---------Q2-----------") ch.execute(query=ch_query2, parameters=params) if time() - _now > 2: @@ -557,7 +557,12 @@ FROM ranked_events {",\n".join(steps_query)}, drop_n AS ({"\nUNION ALL\n".join(drop_query)}) {sub_cte} - SELECT * + SELECT event_number_in_session, + `$event_name` AS event_type, + e_value, + next_type, + next_value, + sessions_count FROM ( {projection_query} ) AS chart_steps diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index c085444dc..6a61f0bf4 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -1,19 +1,17 @@ -urllib3==2.2.3 +urllib3==2.3.0 requests==2.32.3 -boto3==1.35.86 +boto3==1.36.9 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.3 +psycopg[pool,binary]==3.2.4 clickhouse-driver[lz4]==0.2.9 -clickhouse-connect==0.8.11 -elasticsearch==8.17.0 +clickhouse-connect==0.8.15 +elasticsearch==8.17.1 jira==3.8.0 cachetools==5.5.0 - - -fastapi==0.115.6 +fastapi==0.115.7 uvicorn[standard]==0.34.0 python-decouple==3.8 -pydantic[email]==2.10.4 +pydantic[email]==2.10.6 apscheduler==3.11.0 diff --git a/api/requirements.txt b/api/requirements.txt index e9fd3b025..f7c6aec76 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,21 +1,19 @@ -urllib3==2.2.3 +urllib3==2.3.0 requests==2.32.3 -boto3==1.35.86 +boto3==1.36.9 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.3 +psycopg[pool,binary]==3.2.4 clickhouse-driver[lz4]==0.2.9 -clickhouse-connect==0.8.11 -elasticsearch==8.17.0 +clickhouse-connect==0.8.15 +elasticsearch==8.17.1 jira==3.8.0 cachetools==5.5.0 - - -fastapi==0.115.6 +fastapi==0.115.7 uvicorn[standard]==0.34.0 python-decouple==3.8 -pydantic[email]==2.10.4 +pydantic[email]==2.10.6 apscheduler==3.11.0 redis==5.2.1 diff --git a/ee/api/Pipfile b/ee/api/Pipfile index df07b14ca..3fe808998 100644 --- a/ee/api/Pipfile +++ b/ee/api/Pipfile @@ -4,29 +4,31 @@ verify_ssl = true name = "pypi" [packages] -urllib3 = "==1.26.16" +urllib3 = "==2.3.0" requests = "==2.32.3" -boto3 = "==1.35.60" -pyjwt = "==2.9.0" +boto3 = "==1.36.9" +pyjwt = "==2.10.1" psycopg2-binary = "==2.9.10" -psycopg = {extras = ["binary", "pool"], version = "==3.2.3"} -elasticsearch = "==8.16.0" +psycopg = {extras = ["binary", "pool"], version = "==3.2.4"} +clickhouse-driver = {extras = ["lz4"], version = "==0.2.9"} +clickhouse-connect = "==0.8.15" +elasticsearch = "==8.17.1" jira = "==3.8.0" cachetools = "==5.5.0" -fastapi = "==0.115.5" -uvicorn = {extras = ["standard"], version = "==0.32.0"} +fastapi = "==0.115.7" +uvicorn = {extras = ["standard"], version = "==0.34.0"} gunicorn = "==23.0.0" python-decouple = "==3.8" -pydantic = {extras = ["email"], version = "==2.9.2"} -apscheduler = "==3.10.4" -clickhouse-driver = {extras = ["lz4"], version = "==0.2.9"} +pydantic = {extras = ["email"], version = "==2.10.6"} +apscheduler = "==3.11.0" lxml = "!=4.7.0,<=5.2.1,>=4.6.5" python3-saml = "==1.16.0" -python-multipart = "==0.0.17" -redis = "==5.2.0" -azure-storage-blob = "==12.23.1" +python-multipart = "==0.0.20" +redis = "==5.2.1" +azure-storage-blob = "==12.24.1" [dev-packages] [requires] python_version = "3.12" +python_full_version = "3.12.8" diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index adfc42bba..08baf5917 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -1,21 +1,19 @@ -urllib3==2.2.3 +urllib3==2.3.0 requests==2.32.3 -boto3==1.35.76 +boto3==1.36.9 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.3 +psycopg[pool,binary]==3.2.4 clickhouse-driver[lz4]==0.2.9 -clickhouse-connect==0.8.9 -elasticsearch==8.16.0 +clickhouse-connect==0.8.15 +elasticsearch==8.17.1 jira==3.8.0 cachetools==5.5.0 - - -fastapi==0.115.6 -uvicorn[standard]==0.32.1 +fastapi==0.115.7 +uvicorn[standard]==0.34.0 python-decouple==3.8 -pydantic[email]==2.10.3 +pydantic[email]==2.10.6 apscheduler==3.11.0 -azure-storage-blob==12.24.0 \ No newline at end of file +azure-storage-blob==12.24.1 diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 68c115f1c..b4d09298a 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -1,21 +1,19 @@ -urllib3==2.2.3 +urllib3==2.3.0 requests==2.32.3 -boto3==1.35.76 +boto3==1.36.9 pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.3 +psycopg[pool,binary]==3.2.4 clickhouse-driver[lz4]==0.2.9 -clickhouse-connect==0.8.9 -elasticsearch==8.16.0 +clickhouse-connect==0.8.15 +elasticsearch==8.17.1 jira==3.8.0 cachetools==5.5.0 - - -fastapi==0.115.6 +fastapi==0.115.7 python-decouple==3.8 -pydantic[email]==2.10.3 +pydantic[email]==2.10.6 apscheduler==3.11.0 -redis==5.2.0 -azure-storage-blob==12.24.0 +redis==5.2.1 +azure-storage-blob==12.24.1 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 9d5fd8ec2..e7ba78137 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -1,29 +1,28 @@ -# Keep this version to not have conflicts between requests and boto3 urllib3==2.3.0 requests==2.32.3 -boto3==1.35.60 -pyjwt==2.9.0 +boto3==1.36.9 +pyjwt==2.10.1 psycopg2-binary==2.9.10 -psycopg[pool,binary]==3.2.3 -elasticsearch==8.16.0 +psycopg[pool,binary]==3.2.4 +clickhouse-driver[lz4]==0.2.9 +clickhouse-connect==0.8.15 +elasticsearch==8.17.1 jira==3.8.0 cachetools==5.5.0 -fastapi==0.115.5 -uvicorn[standard]==0.32.0 +fastapi==0.115.7 +uvicorn[standard]==0.34.0 gunicorn==23.0.0 python-decouple==3.8 -pydantic[email]==2.9.2 -apscheduler==3.10.4 +pydantic[email]==2.10.6 +apscheduler==3.11.0 -clickhouse-driver[lz4]==0.2.9 -clickhouse-connect==0.8.11 # TODO: enable after xmlsec fix https://github.com/xmlsec/python-xmlsec/issues/252 #--no-binary is used to avoid libxml2 library version incompatibilities between xmlsec and lxml lxml >= 4.6.5, !=4.7.0, <=5.2.1 python3-saml==1.16.0 --no-binary=lxml python-multipart==0.0.20 -redis==5.2.0 +redis==5.2.1 #confluent-kafka==2.1.0 -azure-storage-blob==12.23.1 +azure-storage-blob==12.24.1 diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql b/ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql index caac93e06..67231a79f 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql @@ -1 +1,336 @@ CREATE OR REPLACE FUNCTION openreplay_version AS() -> 'v1.22.0-ee'; + +SET allow_experimental_json_type = 1; + +CREATE DATABASE IF NOT EXISTS product_analytics; + +-- The table of identified users +CREATE TABLE IF NOT EXISTS product_analytics.users +( + project_id UInt16, + "$distinct_id" UInt16, + "$email" String DEFAULT '', + "$name" String DEFAULT '', + "$first_name" String DEFAULT '', + "$last_name" String DEFAULT '', + "$phone" String DEFAULT '', + "$avatar" String DEFAULT '', + "$created_at" DateTime DEFAULT now(), + properties JSON DEFAULT '{}', + group_id1 Array(String) DEFAULT [], + group_id2 Array(String) DEFAULT [], + group_id3 Array(String) DEFAULT [], + group_id4 Array(String) DEFAULT [], + group_id5 Array(String) DEFAULT [], + group_id6 Array(String) DEFAULT [], + + "$sdk_edition" LowCardinality(String), + "$sdk_version" LowCardinality(String), + "$current_url" String DEFAULT '', + "$initial_referrer" String DEFAULT '', + "$referring_domain" String DEFAULT '', + initial_utm_source String DEFAULT '', + initial_utm_medium String DEFAULT '', + initial_utm_campaign String DEFAULT '', + "$country" Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, ''=127) DEFAULT '', + "$state" LowCardinality(String) DEFAULT '', + "$city" LowCardinality(String) DEFAULT '', + "$or_api_endpoint" LowCardinality(String), + "$timezone" Int8 DEFAULT 0 COMMENT 'timezone will be x10 in order to take into consideration countries with tz=N,5H', + + "$first_event_at" DateTime DEFAULT '1970-01-01 00:00:00', + "$last_seen" DateTime DEFAULT now() COMMENT 'the last time the person was identified', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$distinct_id") + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + + +CREATE TABLE IF NOT EXISTS product_analytics.devices +( + project_id UInt16, + "$device_id" String, + "$device" String DEFAULT '', + "$screen_height" UInt16 DEFAULT 0, + "$screen_width" UInt16 DEFAULT 0, + "$os" LowCardinality(String) DEFAULT '', + "$browser" LowCardinality(String) DEFAULT '', + "$browser_version" String DEFAULT '', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$device_id") + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + +-- This table is used in order to identify all devices used by a specific user +CREATE TABLE IF NOT EXISTS product_analytics.user_devices +( + project_id UInt16, + "$device_id" String, + user_id UInt16 COMMENT 'if 0: the person has been deleted, and should set user_id to 0 in the events table', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$device_id", user_id) + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + + +-- This table is used in order to relate a distinct_id to an identified user. +-- The data in this table will be used to propagate changes of user_id in events table +CREATE TABLE IF NOT EXISTS product_analytics.users_distinct_id +( + project_id UInt16, + distinct_id String COMMENT 'this is the event\'s distinct_id', + user_id UInt16 COMMENT 'if 0: the person has been deleted, and should set user_id to 0 in the events table', + + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, distinct_id); + + +CREATE TABLE IF NOT EXISTS product_analytics.events +( + project_id UInt16, + event_id UUID, + "$event_name" String, + created_at DateTime64, + distinct_id String, + "$user_id" UInt16 DEFAULT 0, + session_id UInt64 DEFAULT 0, + "$time" UInt32 DEFAULT 0 COMMENT 'the time of the event in EPOCH, if not provided, the time of arrival to the server', + "$source" LowCardinality(String) DEFAULT '' COMMENT 'the name of the integration that sent the event', + "$duration_s" UInt16 DEFAULT 0 COMMENT 'the duration from session-start in seconds', + properties JSON DEFAULT '{}', + "$properties" JSON DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events', + description String DEFAULT '', + group_id1 Array(String) DEFAULT [], + group_id2 Array(String) DEFAULT [], + group_id3 Array(String) DEFAULT [], + group_id4 Array(String) DEFAULT [], + group_id5 Array(String) DEFAULT [], + group_id6 Array(String) DEFAULT [], + + "$auto_captured" BOOL DEFAULT FALSE, + "$sdk_edition" LowCardinality(String), + "$sdk_version" LowCardinality(String), + "$device_id" String, + "$os" LowCardinality(String) DEFAULT '', + "$browser" LowCardinality(String) DEFAULT '', + "$browser_version" String DEFAULT '', + "$device" String DEFAULT '' COMMENT 'web/mobile', + "$screen_height" UInt16 DEFAULT 0, + "$screen_width" UInt16 DEFAULT 0, + "$current_url" String DEFAULT '', + "$initial_referrer" String DEFAULT '', + "$referring_domain" String DEFAULT '', + "$referrer" String DEFAULT '', + "$initial_referring_domain" String DEFAULT '', + "$search_engine" LowCardinality(String) DEFAULT '', + "$search_engine_keyword" String DEFAULT '', + "utm_source" String DEFAULT '', + "utm_medium" String DEFAULT '', + "utm_campaign" String DEFAULT '', + "$country" Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, ''=127) DEFAULT '', + "$state" LowCardinality(String) DEFAULT '', + "$city" LowCardinality(String) DEFAULT '', + "$or_api_endpoint" LowCardinality(String), + "$timezone" Int8 DEFAULT 0 COMMENT 'timezone will be x10 in order to take into consideration countries with tz=N,5H', + issue_type Enum8(''=0,'click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20,'app_crash'=21) DEFAULT '', + issue_id String DEFAULT '', + -- Created by the backend + "$tags" Array(String) DEFAULT [] COMMENT 'tags are used to filter events', + "$import" BOOL DEFAULT FALSE, + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$event_name", created_at, session_id) + TTL _timestamp + INTERVAL 1 MONTH , + _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + +-- The list of events that should not be ingested, +-- according to a specific event_name and optional properties +CREATE TABLE IF NOT EXISTS product_analytics.dropped_events +( + project_id UInt16, + event_name String, + created_at DateTime64, + -- conditions = {prop_name:{"operator":"less","value":"XYZ"}} + -- example: {"person_id":{"operator":"equal","value":"taha"},"_country":{"operator":"equal","value":"FR"}} + conditions JSON DEFAULT '{}' COMMENT 'properties will have all constraints', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, event_name); + +-- The list of properties that should not be ingested in ALL events, +-- according to a specific rule +CREATE TABLE IF NOT EXISTS product_analytics.dropped_properties +( + project_id UInt16, + property_name String, + created_at DateTime64, + -- example: {"operator":"equal","value":"taha"} + conditions JSON DEFAULT '{}' COMMENT 'in the form {"operator":"less","value":"XYZ"}', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, property_name); + + +-- The list of events that should be hidden in the UI +CREATE TABLE IF NOT EXISTS product_analytics.hidden_events +( + project_id UInt16, + event_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, event_name); + +-- The list of properties that should be hidden in the UI +CREATE TABLE IF NOT EXISTS product_analytics.hidden_properties +( + project_id UInt16, + property_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, property_name); + +-- The list created event's tags +CREATE TABLE IF NOT EXISTS product_analytics.tags +( + project_id UInt16, + tag_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, tag_name, created_at); + + +-- A group of events related with an OR condition +CREATE TABLE IF NOT EXISTS product_analytics.actions +( + project_id UInt16, + action_id UUID DEFAULT generateUUIDv4(), + name String, + created_at DateTime DEFAULT now(), + created_by UInt16 COMMENT 'the OpenReplay user who created this action', + visibility String DEFAULT 'no' COMMENT 'no/read-only/read-write', + last_queried_by UInt16 COMMENT 'the OpenReplay user who last queried this action', + -- definition is the list of filter to use in this action, should have the form: + -- {event_name String, filter JSON COMMENT 'the filter to apply to the selected event_name'} + definition Array(JSON), + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, action_id); + +-- A cohort is a group of events-properties during a specific time period, +-- related with an AND condition to identify users +CREATE TABLE IF NOT EXISTS product_analytics.cohorts +( + project_id UInt16, + cohort_id UUID DEFAULT generateUUIDv4(), + name String, + description String DEFAULT '', + created_at DateTime64, + created_by UInt16 COMMENT 'the OpenReplay user who created this custom event', + visibility String DEFAULT 'no' COMMENT 'if this custom event is public to the team: no/read-only/read-write', + -- definition is the list of filter to use in this cohort during a specific time period, should have the form: + -- {filter JSON COMMENT 'the filter to apply to the selected event_name',time_range LowCardinality(String)} + definition Array(JSON), + count UInt32 DEFAULT 0 COMMENT 'the number of users in this cohort', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, cohort_id); + +-- Mapping between group_id and group_key +CREATE TABLE IF NOT EXISTS product_analytics.groups +( + project_id UInt16, + group_key1 String DEFAULT '', + group_key1_display_name String DEFAULT '', + group_key1_properties Array(String) DEFAULT [], + group_key2 String DEFAULT '', + group_key2_display_name String DEFAULT '', + group_key2_properties Array(String) DEFAULT [], + group_key3 String DEFAULT '', + group_key3_display_name String DEFAULT '', + group_key3_properties Array(String) DEFAULT [], + group_key4 String DEFAULT '', + group_key4_display_name String DEFAULT '', + group_key4_properties Array(String) DEFAULT [], + group_key5 String DEFAULT '', + group_key5_display_name String DEFAULT '', + group_key5_properties Array(String) DEFAULT [], + group_key6 String DEFAULT '', + group_key6_display_name String DEFAULT '', + group_key6_properties Array(String) DEFAULT [], + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id); + +-- The list of property-values of a specific group key-id +CREATE TABLE IF NOT EXISTS product_analytics.group_properties +( + project_id UInt16, + group_key String DEFAULT '', + group_id String DEFAULT '', + -- example: group_key: color, group_id: red properties: {"hex":"#123","name":"magenta"} + properties JSON DEFAULT '{}', + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, group_key, group_id); + + +-- The full list of events +CREATE TABLE IF NOT EXISTS product_analytics.all_events +( + project_id UInt16, + event_name String, + display_name String DEFAULT '', + description String DEFAULT '', + event_count_l30days UInt32 DEFAULT 0, + query_count_l30days UInt32 DEFAULT 0, + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, event_name); + + +-- The full list of properties (events and users) +CREATE TABLE IF NOT EXISTS product_analytics.all_properties +( + project_id UInt16, + property_name String, + is_event_property BOOL, + display_name String DEFAULT '', + description String DEFAULT '', + status String DEFAULT 'visible' COMMENT 'visible/hidden/dropped', + data_count UInt32 DEFAULT 1, + query_count UInt32 DEFAULT 0, + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, property_name, is_event_property); diff --git a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 1b7ca4dbb..413595d79 100644 --- a/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -402,4 +402,340 @@ CREATE TABLE IF NOT EXISTS experimental.ios_events ) ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMM(datetime) ORDER BY (project_id, datetime, event_type, session_id, message_id) - TTL datetime + INTERVAL 3 MONTH; \ No newline at end of file + TTL datetime + INTERVAL 3 MONTH; + + +SET allow_experimental_json_type = 1; + +CREATE DATABASE IF NOT EXISTS product_analytics; + +-- The table of identified users +CREATE TABLE IF NOT EXISTS product_analytics.users +( + project_id UInt16, + "$distinct_id" UInt16, + "$email" String DEFAULT '', + "$name" String DEFAULT '', + "$first_name" String DEFAULT '', + "$last_name" String DEFAULT '', + "$phone" String DEFAULT '', + "$avatar" String DEFAULT '', + "$created_at" DateTime DEFAULT now(), + properties JSON DEFAULT '{}', + group_id1 Array(String) DEFAULT [], + group_id2 Array(String) DEFAULT [], + group_id3 Array(String) DEFAULT [], + group_id4 Array(String) DEFAULT [], + group_id5 Array(String) DEFAULT [], + group_id6 Array(String) DEFAULT [], + + "$sdk_edition" LowCardinality(String), + "$sdk_version" LowCardinality(String), + "$current_url" String DEFAULT '', + "$initial_referrer" String DEFAULT '', + "$referring_domain" String DEFAULT '', + initial_utm_source String DEFAULT '', + initial_utm_medium String DEFAULT '', + initial_utm_campaign String DEFAULT '', + "$country" Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, ''=127) DEFAULT '', + "$state" LowCardinality(String) DEFAULT '', + "$city" LowCardinality(String) DEFAULT '', + "$or_api_endpoint" LowCardinality(String), + "$timezone" Int8 DEFAULT 0 COMMENT 'timezone will be x10 in order to take into consideration countries with tz=N,5H', + + "$first_event_at" DateTime DEFAULT '1970-01-01 00:00:00', + "$last_seen" DateTime DEFAULT now() COMMENT 'the last time the person was identified', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$distinct_id") + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + + +CREATE TABLE IF NOT EXISTS product_analytics.devices +( + project_id UInt16, + "$device_id" String, + "$device" String DEFAULT '', + "$screen_height" UInt16 DEFAULT 0, + "$screen_width" UInt16 DEFAULT 0, + "$os" LowCardinality(String) DEFAULT '', + "$browser" LowCardinality(String) DEFAULT '', + "$browser_version" String DEFAULT '', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$device_id") + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + +-- This table is used in order to identify all devices used by a specific user +CREATE TABLE IF NOT EXISTS product_analytics.user_devices +( + project_id UInt16, + "$device_id" String, + user_id UInt16 COMMENT 'if 0: the person has been deleted, and should set user_id to 0 in the events table', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$device_id", user_id) + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + + +-- This table is used in order to relate a distinct_id to an identified user. +-- The data in this table will be used to propagate changes of user_id in events table +CREATE TABLE IF NOT EXISTS product_analytics.users_distinct_id +( + project_id UInt16, + distinct_id String COMMENT 'this is the event\'s distinct_id', + user_id UInt16 COMMENT 'if 0: the person has been deleted, and should set user_id to 0 in the events table', + + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, distinct_id); + + +CREATE TABLE IF NOT EXISTS product_analytics.events +( + project_id UInt16, + event_id UUID, + "$event_name" String, + created_at DateTime64, + distinct_id String, + "$user_id" UInt16 DEFAULT 0, + session_id UInt64 DEFAULT 0, + "$time" UInt32 DEFAULT 0 COMMENT 'the time of the event in EPOCH, if not provided, the time of arrival to the server', + "$source" LowCardinality(String) DEFAULT '' COMMENT 'the name of the integration that sent the event', + "$duration_s" UInt16 DEFAULT 0 COMMENT 'the duration from session-start in seconds', + properties JSON DEFAULT '{}', + "$properties" JSON DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events', + description String DEFAULT '', + group_id1 Array(String) DEFAULT [], + group_id2 Array(String) DEFAULT [], + group_id3 Array(String) DEFAULT [], + group_id4 Array(String) DEFAULT [], + group_id5 Array(String) DEFAULT [], + group_id6 Array(String) DEFAULT [], + + "$auto_captured" BOOL DEFAULT FALSE, + "$sdk_edition" LowCardinality(String), + "$sdk_version" LowCardinality(String), + "$device_id" String, + "$os" LowCardinality(String) DEFAULT '', + "$browser" LowCardinality(String) DEFAULT '', + "$browser_version" String DEFAULT '', + "$device" String DEFAULT '' COMMENT 'web/mobile', + "$screen_height" UInt16 DEFAULT 0, + "$screen_width" UInt16 DEFAULT 0, + "$current_url" String DEFAULT '', + "$initial_referrer" String DEFAULT '', + "$referring_domain" String DEFAULT '', + "$referrer" String DEFAULT '', + "$initial_referring_domain" String DEFAULT '', + "$search_engine" LowCardinality(String) DEFAULT '', + "$search_engine_keyword" String DEFAULT '', + "utm_source" String DEFAULT '', + "utm_medium" String DEFAULT '', + "utm_campaign" String DEFAULT '', + "$country" Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, ''=127) DEFAULT '', + "$state" LowCardinality(String) DEFAULT '', + "$city" LowCardinality(String) DEFAULT '', + "$or_api_endpoint" LowCardinality(String), + "$timezone" Int8 DEFAULT 0 COMMENT 'timezone will be x10 in order to take into consideration countries with tz=N,5H', + issue_type Enum8(''=0,'click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20,'app_crash'=21) DEFAULT '', + issue_id String DEFAULT '', + -- Created by the backend + "$tags" Array(String) DEFAULT [] COMMENT 'tags are used to filter events', + "$import" BOOL DEFAULT FALSE, + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$event_name", created_at, session_id) + TTL _timestamp + INTERVAL 1 MONTH , + _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + +-- The list of events that should not be ingested, +-- according to a specific event_name and optional properties +CREATE TABLE IF NOT EXISTS product_analytics.dropped_events +( + project_id UInt16, + event_name String, + created_at DateTime64, + -- conditions = {prop_name:{"operator":"less","value":"XYZ"}} + -- example: {"person_id":{"operator":"equal","value":"taha"},"_country":{"operator":"equal","value":"FR"}} + conditions JSON DEFAULT '{}' COMMENT 'properties will have all constraints', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, event_name); + +-- The list of properties that should not be ingested in ALL events, +-- according to a specific rule +CREATE TABLE IF NOT EXISTS product_analytics.dropped_properties +( + project_id UInt16, + property_name String, + created_at DateTime64, + -- example: {"operator":"equal","value":"taha"} + conditions JSON DEFAULT '{}' COMMENT 'in the form {"operator":"less","value":"XYZ"}', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, property_name); + + +-- The list of events that should be hidden in the UI +CREATE TABLE IF NOT EXISTS product_analytics.hidden_events +( + project_id UInt16, + event_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, event_name); + +-- The list of properties that should be hidden in the UI +CREATE TABLE IF NOT EXISTS product_analytics.hidden_properties +( + project_id UInt16, + property_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, property_name); + +-- The list created event's tags +CREATE TABLE IF NOT EXISTS product_analytics.tags +( + project_id UInt16, + tag_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, tag_name, created_at); + + +-- A group of events related with an OR condition +CREATE TABLE IF NOT EXISTS product_analytics.actions +( + project_id UInt16, + action_id UUID DEFAULT generateUUIDv4(), + name String, + created_at DateTime DEFAULT now(), + created_by UInt16 COMMENT 'the OpenReplay user who created this action', + visibility String DEFAULT 'no' COMMENT 'no/read-only/read-write', + last_queried_by UInt16 COMMENT 'the OpenReplay user who last queried this action', + -- definition is the list of filter to use in this action, should have the form: + -- {event_name String, filter JSON COMMENT 'the filter to apply to the selected event_name'} + definition Array(JSON), + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, action_id); + +-- A cohort is a group of events-properties during a specific time period, +-- related with an AND condition to identify users +CREATE TABLE IF NOT EXISTS product_analytics.cohorts +( + project_id UInt16, + cohort_id UUID DEFAULT generateUUIDv4(), + name String, + description String DEFAULT '', + created_at DateTime64, + created_by UInt16 COMMENT 'the OpenReplay user who created this custom event', + visibility String DEFAULT 'no' COMMENT 'if this custom event is public to the team: no/read-only/read-write', + -- definition is the list of filter to use in this cohort during a specific time period, should have the form: + -- {filter JSON COMMENT 'the filter to apply to the selected event_name',time_range LowCardinality(String)} + definition Array(JSON), + count UInt32 DEFAULT 0 COMMENT 'the number of users in this cohort', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, cohort_id); + +-- Mapping between group_id and group_key +CREATE TABLE IF NOT EXISTS product_analytics.groups +( + project_id UInt16, + group_key1 String DEFAULT '', + group_key1_display_name String DEFAULT '', + group_key1_properties Array(String) DEFAULT [], + group_key2 String DEFAULT '', + group_key2_display_name String DEFAULT '', + group_key2_properties Array(String) DEFAULT [], + group_key3 String DEFAULT '', + group_key3_display_name String DEFAULT '', + group_key3_properties Array(String) DEFAULT [], + group_key4 String DEFAULT '', + group_key4_display_name String DEFAULT '', + group_key4_properties Array(String) DEFAULT [], + group_key5 String DEFAULT '', + group_key5_display_name String DEFAULT '', + group_key5_properties Array(String) DEFAULT [], + group_key6 String DEFAULT '', + group_key6_display_name String DEFAULT '', + group_key6_properties Array(String) DEFAULT [], + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id); + +-- The list of property-values of a specific group key-id +CREATE TABLE IF NOT EXISTS product_analytics.group_properties +( + project_id UInt16, + group_key String DEFAULT '', + group_id String DEFAULT '', + -- example: group_key: color, group_id: red properties: {"hex":"#123","name":"magenta"} + properties JSON DEFAULT '{}', + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, group_key, group_id); + + +-- The full list of events +CREATE TABLE IF NOT EXISTS product_analytics.all_events +( + project_id UInt16, + event_name String, + display_name String DEFAULT '', + description String DEFAULT '', + event_count_l30days UInt32 DEFAULT 0, + query_count_l30days UInt32 DEFAULT 0, + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, event_name); + + +-- The full list of properties (events and users) +CREATE TABLE IF NOT EXISTS product_analytics.all_properties +( + project_id UInt16, + property_name String, + is_event_property BOOL, + display_name String DEFAULT '', + description String DEFAULT '', + status String DEFAULT 'visible' COMMENT 'visible/hidden/dropped', + data_count UInt32 DEFAULT 1, + query_count UInt32 DEFAULT 0, + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, property_name, is_event_property); diff --git a/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql b/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql index 6eeb07eb3..2b9d89a8c 100644 --- a/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql +++ b/scripts/schema/db/init_dbs/clickhouse/1.22.0/1.22.0.sql @@ -193,3 +193,338 @@ CREATE TABLE IF NOT EXISTS experimental.ios_events PARTITION BY toYYYYMM(datetime) ORDER BY (project_id, datetime, event_type, session_id, message_id) TTL datetime + INTERVAL 1 MONTH; + +SET allow_experimental_json_type = 1; + +CREATE DATABASE IF NOT EXISTS product_analytics; + +-- The table of identified users +CREATE TABLE IF NOT EXISTS product_analytics.users +( + project_id UInt16, + "$distinct_id" UInt16, + "$email" String DEFAULT '', + "$name" String DEFAULT '', + "$first_name" String DEFAULT '', + "$last_name" String DEFAULT '', + "$phone" String DEFAULT '', + "$avatar" String DEFAULT '', + "$created_at" DateTime DEFAULT now(), + properties JSON DEFAULT '{}', + group_id1 Array(String) DEFAULT [], + group_id2 Array(String) DEFAULT [], + group_id3 Array(String) DEFAULT [], + group_id4 Array(String) DEFAULT [], + group_id5 Array(String) DEFAULT [], + group_id6 Array(String) DEFAULT [], + + "$sdk_edition" LowCardinality(String), + "$sdk_version" LowCardinality(String), + "$current_url" String DEFAULT '', + "$initial_referrer" String DEFAULT '', + "$referring_domain" String DEFAULT '', + initial_utm_source String DEFAULT '', + initial_utm_medium String DEFAULT '', + initial_utm_campaign String DEFAULT '', + "$country" Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, ''=127) DEFAULT '', + "$state" LowCardinality(String) DEFAULT '', + "$city" LowCardinality(String) DEFAULT '', + "$or_api_endpoint" LowCardinality(String), + "$timezone" Int8 DEFAULT 0 COMMENT 'timezone will be x10 in order to take into consideration countries with tz=N,5H', + + "$first_event_at" DateTime DEFAULT '1970-01-01 00:00:00', + "$last_seen" DateTime DEFAULT now() COMMENT 'the last time the person was identified', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$distinct_id") + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + + +CREATE TABLE IF NOT EXISTS product_analytics.devices +( + project_id UInt16, + "$device_id" String, + "$device" String DEFAULT '', + "$screen_height" UInt16 DEFAULT 0, + "$screen_width" UInt16 DEFAULT 0, + "$os" LowCardinality(String) DEFAULT '', + "$browser" LowCardinality(String) DEFAULT '', + "$browser_version" String DEFAULT '', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$device_id") + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + +-- This table is used in order to identify all devices used by a specific user +CREATE TABLE IF NOT EXISTS product_analytics.user_devices +( + project_id UInt16, + "$device_id" String, + user_id UInt16 COMMENT 'if 0: the person has been deleted, and should set user_id to 0 in the events table', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$device_id", user_id) + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + + +-- This table is used in order to relate a distinct_id to an identified user. +-- The data in this table will be used to propagate changes of user_id in events table +CREATE TABLE IF NOT EXISTS product_analytics.users_distinct_id +( + project_id UInt16, + distinct_id String COMMENT 'this is the event\'s distinct_id', + user_id UInt16 COMMENT 'if 0: the person has been deleted, and should set user_id to 0 in the events table', + + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, distinct_id); + + +CREATE TABLE IF NOT EXISTS product_analytics.events +( + project_id UInt16, + event_id UUID, + "$event_name" String, + created_at DateTime64, + distinct_id String, + "$user_id" UInt16 DEFAULT 0, + session_id UInt64 DEFAULT 0, + "$time" UInt32 DEFAULT 0 COMMENT 'the time of the event in EPOCH, if not provided, the time of arrival to the server', + "$source" LowCardinality(String) DEFAULT '' COMMENT 'the name of the integration that sent the event', + "$duration_s" UInt16 DEFAULT 0 COMMENT 'the duration from session-start in seconds', + properties JSON DEFAULT '{}', + "$properties" JSON DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events', + description String DEFAULT '', + group_id1 Array(String) DEFAULT [], + group_id2 Array(String) DEFAULT [], + group_id3 Array(String) DEFAULT [], + group_id4 Array(String) DEFAULT [], + group_id5 Array(String) DEFAULT [], + group_id6 Array(String) DEFAULT [], + + "$auto_captured" BOOL DEFAULT FALSE, + "$sdk_edition" LowCardinality(String), + "$sdk_version" LowCardinality(String), + "$device_id" String, + "$os" LowCardinality(String) DEFAULT '', + "$browser" LowCardinality(String) DEFAULT '', + "$browser_version" String DEFAULT '', + "$device" String DEFAULT '' COMMENT 'web/mobile', + "$screen_height" UInt16 DEFAULT 0, + "$screen_width" UInt16 DEFAULT 0, + "$current_url" String DEFAULT '', + "$initial_referrer" String DEFAULT '', + "$referring_domain" String DEFAULT '', + "$referrer" String DEFAULT '', + "$initial_referring_domain" String DEFAULT '', + "$search_engine" LowCardinality(String) DEFAULT '', + "$search_engine_keyword" String DEFAULT '', + "utm_source" String DEFAULT '', + "utm_medium" String DEFAULT '', + "utm_campaign" String DEFAULT '', + "$country" Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, ''=127) DEFAULT '', + "$state" LowCardinality(String) DEFAULT '', + "$city" LowCardinality(String) DEFAULT '', + "$or_api_endpoint" LowCardinality(String), + "$timezone" Int8 DEFAULT 0 COMMENT 'timezone will be x10 in order to take into consideration countries with tz=N,5H', + issue_type Enum8(''=0,'click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20,'app_crash'=21) DEFAULT '', + issue_id String DEFAULT '', + -- Created by the backend + "$tags" Array(String) DEFAULT [] COMMENT 'tags are used to filter events', + "$import" BOOL DEFAULT FALSE, + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$event_name", created_at, session_id) + TTL _timestamp + INTERVAL 1 MONTH , + _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + +-- The list of events that should not be ingested, +-- according to a specific event_name and optional properties +CREATE TABLE IF NOT EXISTS product_analytics.dropped_events +( + project_id UInt16, + event_name String, + created_at DateTime64, + -- conditions = {prop_name:{"operator":"less","value":"XYZ"}} + -- example: {"person_id":{"operator":"equal","value":"taha"},"_country":{"operator":"equal","value":"FR"}} + conditions JSON DEFAULT '{}' COMMENT 'properties will have all constraints', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, event_name); + +-- The list of properties that should not be ingested in ALL events, +-- according to a specific rule +CREATE TABLE IF NOT EXISTS product_analytics.dropped_properties +( + project_id UInt16, + property_name String, + created_at DateTime64, + -- example: {"operator":"equal","value":"taha"} + conditions JSON DEFAULT '{}' COMMENT 'in the form {"operator":"less","value":"XYZ"}', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, property_name); + + +-- The list of events that should be hidden in the UI +CREATE TABLE IF NOT EXISTS product_analytics.hidden_events +( + project_id UInt16, + event_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, event_name); + +-- The list of properties that should be hidden in the UI +CREATE TABLE IF NOT EXISTS product_analytics.hidden_properties +( + project_id UInt16, + property_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, property_name); + +-- The list created event's tags +CREATE TABLE IF NOT EXISTS product_analytics.tags +( + project_id UInt16, + tag_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, tag_name, created_at); + + +-- A group of events related with an OR condition +CREATE TABLE IF NOT EXISTS product_analytics.actions +( + project_id UInt16, + action_id UUID DEFAULT generateUUIDv4(), + name String, + created_at DateTime DEFAULT now(), + created_by UInt16 COMMENT 'the OpenReplay user who created this action', + visibility String DEFAULT 'no' COMMENT 'no/read-only/read-write', + last_queried_by UInt16 COMMENT 'the OpenReplay user who last queried this action', + -- definition is the list of filter to use in this action, should have the form: + -- {event_name String, filter JSON COMMENT 'the filter to apply to the selected event_name'} + definition Array(JSON), + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, action_id); + +-- A cohort is a group of events-properties during a specific time period, +-- related with an AND condition to identify users +CREATE TABLE IF NOT EXISTS product_analytics.cohorts +( + project_id UInt16, + cohort_id UUID DEFAULT generateUUIDv4(), + name String, + description String DEFAULT '', + created_at DateTime64, + created_by UInt16 COMMENT 'the OpenReplay user who created this custom event', + visibility String DEFAULT 'no' COMMENT 'if this custom event is public to the team: no/read-only/read-write', + -- definition is the list of filter to use in this cohort during a specific time period, should have the form: + -- {filter JSON COMMENT 'the filter to apply to the selected event_name',time_range LowCardinality(String)} + definition Array(JSON), + count UInt32 DEFAULT 0 COMMENT 'the number of users in this cohort', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, cohort_id); + +-- Mapping between group_id and group_key +CREATE TABLE IF NOT EXISTS product_analytics.groups +( + project_id UInt16, + group_key1 String DEFAULT '', + group_key1_display_name String DEFAULT '', + group_key1_properties Array(String) DEFAULT [], + group_key2 String DEFAULT '', + group_key2_display_name String DEFAULT '', + group_key2_properties Array(String) DEFAULT [], + group_key3 String DEFAULT '', + group_key3_display_name String DEFAULT '', + group_key3_properties Array(String) DEFAULT [], + group_key4 String DEFAULT '', + group_key4_display_name String DEFAULT '', + group_key4_properties Array(String) DEFAULT [], + group_key5 String DEFAULT '', + group_key5_display_name String DEFAULT '', + group_key5_properties Array(String) DEFAULT [], + group_key6 String DEFAULT '', + group_key6_display_name String DEFAULT '', + group_key6_properties Array(String) DEFAULT [], + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id); + +-- The list of property-values of a specific group key-id +CREATE TABLE IF NOT EXISTS product_analytics.group_properties +( + project_id UInt16, + group_key String DEFAULT '', + group_id String DEFAULT '', + -- example: group_key: color, group_id: red properties: {"hex":"#123","name":"magenta"} + properties JSON DEFAULT '{}', + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, group_key, group_id); + + +-- The full list of events +CREATE TABLE IF NOT EXISTS product_analytics.all_events +( + project_id UInt16, + event_name String, + display_name String DEFAULT '', + description String DEFAULT '', + event_count_l30days UInt32 DEFAULT 0, + query_count_l30days UInt32 DEFAULT 0, + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, event_name); + + +-- The full list of properties (events and users) +CREATE TABLE IF NOT EXISTS product_analytics.all_properties +( + project_id UInt16, + property_name String, + is_event_property BOOL, + display_name String DEFAULT '', + description String DEFAULT '', + status String DEFAULT 'visible' COMMENT 'visible/hidden/dropped', + data_count UInt32 DEFAULT 1, + query_count UInt32 DEFAULT 0, + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, property_name, is_event_property); diff --git a/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql b/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql index 6eeb07eb3..400321741 100644 --- a/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql +++ b/scripts/schema/db/init_dbs/clickhouse/create/init_schema.sql @@ -193,3 +193,339 @@ CREATE TABLE IF NOT EXISTS experimental.ios_events PARTITION BY toYYYYMM(datetime) ORDER BY (project_id, datetime, event_type, session_id, message_id) TTL datetime + INTERVAL 1 MONTH; + + +SET allow_experimental_json_type = 1; + +CREATE DATABASE IF NOT EXISTS product_analytics; + +-- The table of identified users +CREATE TABLE IF NOT EXISTS product_analytics.users +( + project_id UInt16, + "$distinct_id" UInt16, + "$email" String DEFAULT '', + "$name" String DEFAULT '', + "$first_name" String DEFAULT '', + "$last_name" String DEFAULT '', + "$phone" String DEFAULT '', + "$avatar" String DEFAULT '', + "$created_at" DateTime DEFAULT now(), + properties JSON DEFAULT '{}', + group_id1 Array(String) DEFAULT [], + group_id2 Array(String) DEFAULT [], + group_id3 Array(String) DEFAULT [], + group_id4 Array(String) DEFAULT [], + group_id5 Array(String) DEFAULT [], + group_id6 Array(String) DEFAULT [], + + "$sdk_edition" LowCardinality(String), + "$sdk_version" LowCardinality(String), + "$current_url" String DEFAULT '', + "$initial_referrer" String DEFAULT '', + "$referring_domain" String DEFAULT '', + initial_utm_source String DEFAULT '', + initial_utm_medium String DEFAULT '', + initial_utm_campaign String DEFAULT '', + "$country" Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, ''=127) DEFAULT '', + "$state" LowCardinality(String) DEFAULT '', + "$city" LowCardinality(String) DEFAULT '', + "$or_api_endpoint" LowCardinality(String), + "$timezone" Int8 DEFAULT 0 COMMENT 'timezone will be x10 in order to take into consideration countries with tz=N,5H', + + "$first_event_at" DateTime DEFAULT '1970-01-01 00:00:00', + "$last_seen" DateTime DEFAULT now() COMMENT 'the last time the person was identified', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$distinct_id") + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + + +CREATE TABLE IF NOT EXISTS product_analytics.devices +( + project_id UInt16, + "$device_id" String, + "$device" String DEFAULT '', + "$screen_height" UInt16 DEFAULT 0, + "$screen_width" UInt16 DEFAULT 0, + "$os" LowCardinality(String) DEFAULT '', + "$browser" LowCardinality(String) DEFAULT '', + "$browser_version" String DEFAULT '', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$device_id") + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + +-- This table is used in order to identify all devices used by a specific user +CREATE TABLE IF NOT EXISTS product_analytics.user_devices +( + project_id UInt16, + "$device_id" String, + user_id UInt16 COMMENT 'if 0: the person has been deleted, and should set user_id to 0 in the events table', + + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$device_id", user_id) + TTL _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + + +-- This table is used in order to relate a distinct_id to an identified user. +-- The data in this table will be used to propagate changes of user_id in events table +CREATE TABLE IF NOT EXISTS product_analytics.users_distinct_id +( + project_id UInt16, + distinct_id String COMMENT 'this is the event\'s distinct_id', + user_id UInt16 COMMENT 'if 0: the person has been deleted, and should set user_id to 0 in the events table', + + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, distinct_id); + + +CREATE TABLE IF NOT EXISTS product_analytics.events +( + project_id UInt16, + event_id UUID, + "$event_name" String, + created_at DateTime64, + distinct_id String, + "$user_id" UInt16 DEFAULT 0, + session_id UInt64 DEFAULT 0, + "$time" UInt32 DEFAULT 0 COMMENT 'the time of the event in EPOCH, if not provided, the time of arrival to the server', + "$source" LowCardinality(String) DEFAULT '' COMMENT 'the name of the integration that sent the event', + "$duration_s" UInt16 DEFAULT 0 COMMENT 'the duration from session-start in seconds', + properties JSON DEFAULT '{}', + "$properties" JSON DEFAULT '{}' COMMENT 'these properties belongs to the auto-captured events', + description String DEFAULT '', + group_id1 Array(String) DEFAULT [], + group_id2 Array(String) DEFAULT [], + group_id3 Array(String) DEFAULT [], + group_id4 Array(String) DEFAULT [], + group_id5 Array(String) DEFAULT [], + group_id6 Array(String) DEFAULT [], + + "$auto_captured" BOOL DEFAULT FALSE, + "$sdk_edition" LowCardinality(String), + "$sdk_version" LowCardinality(String), + "$device_id" String, + "$os" LowCardinality(String) DEFAULT '', + "$browser" LowCardinality(String) DEFAULT '', + "$browser_version" String DEFAULT '', + "$device" String DEFAULT '' COMMENT 'web/mobile', + "$screen_height" UInt16 DEFAULT 0, + "$screen_width" UInt16 DEFAULT 0, + "$current_url" String DEFAULT '', + "$initial_referrer" String DEFAULT '', + "$referring_domain" String DEFAULT '', + "$referrer" String DEFAULT '', + "$initial_referring_domain" String DEFAULT '', + "$search_engine" LowCardinality(String) DEFAULT '', + "$search_engine_keyword" String DEFAULT '', + "utm_source" String DEFAULT '', + "utm_medium" String DEFAULT '', + "utm_campaign" String DEFAULT '', + "$country" Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122,'BU'=123, 'VD'=124, 'YD'=125, 'DD'=126, ''=127) DEFAULT '', + "$state" LowCardinality(String) DEFAULT '', + "$city" LowCardinality(String) DEFAULT '', + "$or_api_endpoint" LowCardinality(String), + "$timezone" Int8 DEFAULT 0 COMMENT 'timezone will be x10 in order to take into consideration countries with tz=N,5H', + issue_type Enum8(''=0,'click_rage'=1,'dead_click'=2,'excessive_scrolling'=3,'bad_request'=4,'missing_resource'=5,'memory'=6,'cpu'=7,'slow_resource'=8,'slow_page_load'=9,'crash'=10,'ml_cpu'=11,'ml_memory'=12,'ml_dead_click'=13,'ml_click_rage'=14,'ml_mouse_thrashing'=15,'ml_excessive_scrolling'=16,'ml_slow_resources'=17,'custom'=18,'js_exception'=19,'mouse_thrashing'=20,'app_crash'=21) DEFAULT '', + issue_id String DEFAULT '', + -- Created by the backend + "$tags" Array(String) DEFAULT [] COMMENT 'tags are used to filter events', + "$import" BOOL DEFAULT FALSE, + _deleted_at DateTime DEFAULT '1970-01-01 00:00:00', + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, "$event_name", created_at, session_id) + TTL _timestamp + INTERVAL 1 MONTH , + _deleted_at + INTERVAL 1 DAY DELETE WHERE _deleted_at != '1970-01-01 00:00:00'; + +-- The list of events that should not be ingested, +-- according to a specific event_name and optional properties +CREATE TABLE IF NOT EXISTS product_analytics.dropped_events +( + project_id UInt16, + event_name String, + created_at DateTime64, + -- conditions = {prop_name:{"operator":"less","value":"XYZ"}} + -- example: {"person_id":{"operator":"equal","value":"taha"},"_country":{"operator":"equal","value":"FR"}} + conditions JSON DEFAULT '{}' COMMENT 'properties will have all constraints', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, event_name); + +-- The list of properties that should not be ingested in ALL events, +-- according to a specific rule +CREATE TABLE IF NOT EXISTS product_analytics.dropped_properties +( + project_id UInt16, + property_name String, + created_at DateTime64, + -- example: {"operator":"equal","value":"taha"} + conditions JSON DEFAULT '{}' COMMENT 'in the form {"operator":"less","value":"XYZ"}', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, property_name); + + +-- The list of events that should be hidden in the UI +CREATE TABLE IF NOT EXISTS product_analytics.hidden_events +( + project_id UInt16, + event_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, event_name); + +-- The list of properties that should be hidden in the UI +CREATE TABLE IF NOT EXISTS product_analytics.hidden_properties +( + project_id UInt16, + property_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, property_name); + +-- The list created event's tags +CREATE TABLE IF NOT EXISTS product_analytics.tags +( + project_id UInt16, + tag_name String, + created_at DateTime64, + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, tag_name, created_at); + + +-- A group of events related with an OR condition +CREATE TABLE IF NOT EXISTS product_analytics.actions +( + project_id UInt16, + action_id UUID DEFAULT generateUUIDv4(), + name String, + created_at DateTime DEFAULT now(), + created_by UInt16 COMMENT 'the OpenReplay user who created this action', + visibility String DEFAULT 'no' COMMENT 'no/read-only/read-write', + last_queried_by UInt16 COMMENT 'the OpenReplay user who last queried this action', + -- definition is the list of filter to use in this action, should have the form: + -- {event_name String, filter JSON COMMENT 'the filter to apply to the selected event_name'} + definition Array(JSON), + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, action_id); + +-- A cohort is a group of events-properties during a specific time period, +-- related with an AND condition to identify users +CREATE TABLE IF NOT EXISTS product_analytics.cohorts +( + project_id UInt16, + cohort_id UUID DEFAULT generateUUIDv4(), + name String, + description String DEFAULT '', + created_at DateTime64, + created_by UInt16 COMMENT 'the OpenReplay user who created this custom event', + visibility String DEFAULT 'no' COMMENT 'if this custom event is public to the team: no/read-only/read-write', + -- definition is the list of filter to use in this cohort during a specific time period, should have the form: + -- {filter JSON COMMENT 'the filter to apply to the selected event_name',time_range LowCardinality(String)} + definition Array(JSON), + count UInt32 DEFAULT 0 COMMENT 'the number of users in this cohort', + + _sign Int8 DEFAULT 1, + _timestamp DateTime DEFAULT now() +) ENGINE = CollapsingMergeTree(_sign) + ORDER BY (project_id, cohort_id); + +-- Mapping between group_id and group_key +CREATE TABLE IF NOT EXISTS product_analytics.groups +( + project_id UInt16, + group_key1 String DEFAULT '', + group_key1_display_name String DEFAULT '', + group_key1_properties Array(String) DEFAULT [], + group_key2 String DEFAULT '', + group_key2_display_name String DEFAULT '', + group_key2_properties Array(String) DEFAULT [], + group_key3 String DEFAULT '', + group_key3_display_name String DEFAULT '', + group_key3_properties Array(String) DEFAULT [], + group_key4 String DEFAULT '', + group_key4_display_name String DEFAULT '', + group_key4_properties Array(String) DEFAULT [], + group_key5 String DEFAULT '', + group_key5_display_name String DEFAULT '', + group_key5_properties Array(String) DEFAULT [], + group_key6 String DEFAULT '', + group_key6_display_name String DEFAULT '', + group_key6_properties Array(String) DEFAULT [], + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id); + +-- The list of property-values of a specific group key-id +CREATE TABLE IF NOT EXISTS product_analytics.group_properties +( + project_id UInt16, + group_key String DEFAULT '', + group_id String DEFAULT '', + -- example: group_key: color, group_id: red properties: {"hex":"#123","name":"magenta"} + properties JSON DEFAULT '{}', + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, group_key, group_id); + + +-- The full list of events +CREATE TABLE IF NOT EXISTS product_analytics.all_events +( + project_id UInt16, + event_name String, + display_name String DEFAULT '', + description String DEFAULT '', + event_count_l30days UInt32 DEFAULT 0, + query_count_l30days UInt32 DEFAULT 0, + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, event_name); + + +-- The full list of properties (events and users) +CREATE TABLE IF NOT EXISTS product_analytics.all_properties +( + project_id UInt16, + property_name String, + is_event_property BOOL, + display_name String DEFAULT '', + description String DEFAULT '', + status String DEFAULT 'visible' COMMENT 'visible/hidden/dropped', + data_count UInt32 DEFAULT 1, + query_count UInt32 DEFAULT 0, + + created_at DateTime64, + _timestamp DateTime DEFAULT now() +) ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (project_id, property_name, is_event_property);