From db0c33a6ecdf41796e9731296c44602cb68ad4a9 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 30 Aug 2022 12:16:15 +0100 Subject: [PATCH 1/8] feat(chalice): fixed sessions search --- ee/api/chalicelib/core/sessions_exp.py | 121 +------------------------ 1 file changed, 5 insertions(+), 116 deletions(-) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index 2b78f46f2..b5cd005d6 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -226,119 +226,9 @@ def _isUndefined_operator(op: schemas.SearchEventOperator): return op in [schemas.SearchEventOperator._is_undefined] -# This function executes the query and return result -def search_sessions_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, - error_status=schemas.ErrorStatus.all, count_only=False, issue=None): - full_args, query_part = search_query_parts(data=data, error_status=error_status, errors_only=errors_only, - favorite_only=data.bookmarked, issue=issue, project_id=project_id, - user_id=user_id) - if data.limit is not None and data.page is not None: - full_args["sessions_limit_s"] = (data.page - 1) * data.limit - full_args["sessions_limit_e"] = data.page * data.limit - else: - full_args["sessions_limit_s"] = 1 - full_args["sessions_limit_e"] = 200 - - meta_keys = [] - with pg_client.PostgresClient() as cur: - if count_only: - main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, - COUNT(DISTINCT s.user_uuid) AS count_users - {query_part};""", full_args) - elif data.group_by_user: - g_sort = "count(full_sessions)" - if data.order is None: - data.order = schemas.SortOrderType.desc - else: - data.order = data.order.upper() - if data.sort is not None and data.sort != 'sessionsCount': - sort = helper.key_to_snake_case(data.sort) - g_sort = f"{'MIN' if data.order == schemas.SortOrderType.desc else 'MAX'}({sort})" - else: - sort = 'start_ts' - - meta_keys = metadata.get(project_id=project_id) - main_query = cur.mogrify(f"""SELECT COUNT(*) AS count, - COALESCE(JSONB_AGG(users_sessions) - FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions - FROM (SELECT user_id, - count(full_sessions) AS user_sessions_count, - jsonb_agg(full_sessions) FILTER (WHERE rn <= 1) AS last_session, - MIN(full_sessions.start_ts) AS first_session_ts, - ROW_NUMBER() OVER (ORDER BY {g_sort} {data.order}) AS rn - FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY {sort} {data.order}) AS rn - FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} - {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} - {query_part} - ) AS filtred_sessions - ) AS full_sessions - GROUP BY user_id - ) AS users_sessions;""", - full_args) - else: - if data.order is None: - data.order = schemas.SortOrderType.desc - sort = 'session_id' - if data.sort is not None and data.sort != "session_id": - # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) - sort = helper.key_to_snake_case(data.sort) - - meta_keys = metadata.get(project_id=project_id) - main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, - COALESCE(JSONB_AGG(full_sessions) - FILTER (WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s), '[]'::JSONB) AS sessions - FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY {sort} {data.order}, issue_score DESC) AS rn - FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} - {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} - {query_part} - ORDER BY s.session_id desc) AS filtred_sessions - ORDER BY {sort} {data.order}, issue_score DESC) AS full_sessions;""", - full_args) - print("--------------------") - print(main_query) - print("--------------------") - try: - cur.execute(main_query) - except Exception as err: - print("--------- SESSIONS SEARCH QUERY EXCEPTION -----------") - print(main_query.decode('UTF-8')) - print("--------- PAYLOAD -----------") - print(data.json()) - print("--------------------") - raise err - if errors_only: - return helper.list_to_camel_case(cur.fetchall()) - - sessions = cur.fetchone() - if count_only: - return helper.dict_to_camel_case(sessions) - - total = sessions["count"] - sessions = sessions["sessions"] - - if data.group_by_user: - for i, s in enumerate(sessions): - sessions[i] = {**s.pop("last_session")[0], **s} - sessions[i].pop("rn") - sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ - if sessions[i][f'metadata_{k["index"]}'] is not None} - else: - for i, s in enumerate(sessions): - sessions[i]["metadata"] = {k["key"]: sessions[i][f'metadata_{k["index"]}'] for k in meta_keys \ - if sessions[i][f'metadata_{k["index"]}'] is not None} - # if not data.group_by_user and data.sort is not None and data.sort != "session_id": - # sessions = sorted(sessions, key=lambda s: s[helper.key_to_snake_case(data.sort)], - # reverse=data.order.upper() == "DESC") - return { - 'total': total, - 'sessions': helper.list_to_camel_case(sessions) - } - - # This function executes the query and return result def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, error_status=schemas.ErrorStatus.all, count_only=False, issue=None): - print("------ search2_ch") full_args, query_part = search_query_parts_ch(data=data, error_status=error_status, errors_only=errors_only, favorite_only=data.bookmarked, issue=issue, project_id=project_id, user_id=user_id) @@ -417,13 +307,13 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ -- ORDER BY {sort} {data.order} ) AS raw WHERE rn>%(sessions_limit_s)s AND rn<=%(sessions_limit_e)s;""", full_args) - print("--------------------") - print(main_query) - print("--------------------") + # print("--------------------") + # print(main_query) + # print("--------------------") try: sessions = cur.execute(main_query) except Exception as err: - print("--------- SESSIONS SEARCH QUERY EXCEPTION -----------") + print("--------- SESSIONS-CH SEARCH QUERY EXCEPTION -----------") print(main_query) print("--------- PAYLOAD -----------") print(data.json()) @@ -1651,7 +1541,7 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, # TODO: isNot for ERROR elif event_type == events.event_type.ERROR.ui_type: event_from = event_from % f"{MAIN_EVENTS_TABLE} AS main" - events_extra_join = "SELECT * FROM final.errors AS main1 WHERE main1.project_id=%(project_id)s" + events_extra_join = f"SELECT * FROM {MAIN_EVENTS_TABLE} AS main1 WHERE main1.project_id=%(project_id)s" event_where.append(f"main.event_type='{__get_event_type(event_type)}'") events_conditions.append({"type": event_where[-1]}) event.source = tuple(event.source) @@ -2001,7 +1891,6 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, GROUP BY session_id {having}""" else: - print(">>>>> OR EVENTS") type_conditions = [] sequence_conditions = [] has_values = False From 36f59f97e663c938cc751f885230ab8ccc8d033d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 30 Aug 2022 12:42:48 +0100 Subject: [PATCH 2/8] feat(DB): added missing columns --- ee/api/chalicelib/core/sessions_exp.py | 27 ------------------- .../db/init_dbs/clickhouse/1.8.0/1.8.0.sql | 6 ++++- .../clickhouse/create/init_schema.sql | 6 ++++- 3 files changed, 10 insertions(+), 29 deletions(-) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index b5cd005d6..e9e6b0c21 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -5,33 +5,6 @@ import schemas_ee from chalicelib.core import events, metadata, events_ios, \ sessions_mobs, issues, projects, errors, resources, assist, performance_event, metrics from chalicelib.utils import pg_client, helper, metrics_helper, ch_client, exp_ch_helper -from chalicelib.utils.TimeUTC import TimeUTC - -SESSION_PROJECTION_COLS = """\ -s.project_id, -s.session_id::text AS session_id, -s.user_uuid, -s.user_id, -s.user_os, -s.user_browser, -s.user_device, -s.user_device_type, -s.user_country, -s.start_ts, -s.duration, -s.events_count, -s.pages_count, -s.errors_count, -s.user_anonymous_id, -s.platform, -s.issue_score, -to_jsonb(s.issue_types) AS issue_types, -favorite_sessions.session_id NOTNULL AS favorite, -COALESCE((SELECT TRUE - FROM public.user_viewed_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed - """ SESSION_PROJECTION_COLS_CH = """\ s.project_id, diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql index 8afef67b7..c1687164c 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.8.0/1.8.0.sql @@ -124,6 +124,7 @@ CREATE TABLE IF NOT EXISTS experimental.sessions user_device Nullable(String), user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + platform Enum8('web'=1,'ios'=2,'android'=3) DEFAULT 'web', datetime DateTime, duration UInt32, pages_count UInt16, @@ -133,6 +134,7 @@ CREATE TABLE IF NOT EXISTS experimental.sessions utm_medium Nullable(String), utm_campaign Nullable(String), user_id Nullable(String), + user_anonymous_id Nullable(String), metadata_1 Nullable(String), metadata_2 Nullable(String), metadata_3 Nullable(String), @@ -146,7 +148,7 @@ CREATE TABLE IF NOT EXISTS experimental.sessions issue_types Array(LowCardinality(String)), referrer Nullable(String), base_referrer Nullable(String) MATERIALIZED lower(concat(domain(referrer), path(referrer))), - _timestamp DateTime DEFAULT now() + _timestamp DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMMDD(datetime) ORDER BY (project_id, datetime, session_id) @@ -302,6 +304,7 @@ SELECT session_id, user_device, user_device_type, user_country, + platform, datetime, duration, pages_count, @@ -311,6 +314,7 @@ SELECT session_id, utm_medium, utm_campaign, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql index 8afef67b7..c1687164c 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/init_schema.sql @@ -124,6 +124,7 @@ CREATE TABLE IF NOT EXISTS experimental.sessions user_device Nullable(String), user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + platform Enum8('web'=1,'ios'=2,'android'=3) DEFAULT 'web', datetime DateTime, duration UInt32, pages_count UInt16, @@ -133,6 +134,7 @@ CREATE TABLE IF NOT EXISTS experimental.sessions utm_medium Nullable(String), utm_campaign Nullable(String), user_id Nullable(String), + user_anonymous_id Nullable(String), metadata_1 Nullable(String), metadata_2 Nullable(String), metadata_3 Nullable(String), @@ -146,7 +148,7 @@ CREATE TABLE IF NOT EXISTS experimental.sessions issue_types Array(LowCardinality(String)), referrer Nullable(String), base_referrer Nullable(String) MATERIALIZED lower(concat(domain(referrer), path(referrer))), - _timestamp DateTime DEFAULT now() + _timestamp DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_timestamp) PARTITION BY toYYYYMMDD(datetime) ORDER BY (project_id, datetime, session_id) @@ -302,6 +304,7 @@ SELECT session_id, user_device, user_device_type, user_country, + platform, datetime, duration, pages_count, @@ -311,6 +314,7 @@ SELECT session_id, utm_medium, utm_campaign, user_id, + user_anonymous_id, metadata_1, metadata_2, metadata_3, From bedf526891b89e8e412bf98b9284335dab6a1828 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 30 Aug 2022 14:34:39 +0100 Subject: [PATCH 3/8] feat(backend): changed build script --- backend/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/build.sh b/backend/build.sh index ef57b0887..d2a919d9a 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -29,6 +29,8 @@ function build_service() { } function build_api(){ + cp -R ../backend ../_backend + cd ../_backend # Copy enterprise code [[ $1 == "ee" ]] && { cp -r ../ee/backend/* ./ @@ -43,6 +45,8 @@ function build_api(){ build_service $image echo "::set-output name=image::${DOCKER_REPO:-'local'}/$image:${git_sha1}" done + cd ../backend + rm -rf ../_backend echo "backend build completed" } From 764fba5e8a2cbcad68fdbd0965b7f441fcc3edeb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 30 Aug 2022 14:36:48 +0100 Subject: [PATCH 4/8] feat(api): ignore crons build script --- api/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/.gitignore b/api/.gitignore index 68797b56a..773f59916 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -175,4 +175,5 @@ SUBNETS.json ./chalicelib/.configs README/* -.local \ No newline at end of file +.local +build_crons.sh \ No newline at end of file From a1cd4496d4d3387ae2818db66a14cfb6e137c072 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 30 Aug 2022 15:00:24 +0100 Subject: [PATCH 5/8] feat(chalice): fixed CH pagination --- ee/api/chalicelib/core/sessions_exp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index e9e6b0c21..b2bc1186e 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -274,7 +274,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ meta_keys = metadata.get(project_id=project_id) main_query = cur.format(f"""SELECT any(total) AS count, groupArray(%(sessions_limit)s)(details) AS sessions FROM (SELECT COUNT() OVER () AS total, - rowNumberInAllBlocks() AS rn, + rowNumberInAllBlocks()+1 AS rn, map({SESSION_PROJECTION_COLS_CH_MAP}) AS details {query_part} -- ORDER BY {sort} {data.order} From 0f425a346f06292900e4b96272a474d2470f6694 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 30 Aug 2022 17:24:14 +0100 Subject: [PATCH 6/8] feat(chalice): CH search with pre-ReplacingMergeTree --- ee/api/chalicelib/core/sessions_exp.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index b2bc1186e..e6aff9f32 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -219,9 +219,9 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ meta_keys = [] with ch_client.ClickHouseClient() as cur: if errors_only: - print("--------------------QP") - print(cur.format(query_part, full_args)) - print("--------------------") + # print("--------------------QP") + # print(cur.format(query_part, full_args)) + # print("--------------------") main_query = cur.format(f"""SELECT DISTINCT er.error_id, COALESCE((SELECT TRUE FROM {exp_ch_helper.get_user_viewed_errors_table()} AS ve @@ -1963,7 +1963,9 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, else: extra_join += f"""(SELECT * FROM {MAIN_SESSIONS_TABLE} AS s {extra_event} - WHERE {" AND ".join(extra_constraints)}) AS s""" + WHERE {" AND ".join(extra_constraints)} + ORDER BY _timestamp DESC + LIMIT 1 BY session_id) AS s""" query_part = f"""\ FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else ""} {extra_join} From 4a17e5ba75ac5e334ef36f572af12b0d1fff62c2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 30 Aug 2022 20:07:21 +0100 Subject: [PATCH 7/8] feat(chalice): CH fixed search favorite sessions --- ee/api/chalicelib/core/sessions_exp.py | 30 ++++++++++---------------- ee/api/chalicelib/utils/ch_client.py | 2 ++ 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/ee/api/chalicelib/core/sessions_exp.py b/ee/api/chalicelib/core/sessions_exp.py index e6aff9f32..32ca723b7 100644 --- a/ee/api/chalicelib/core/sessions_exp.py +++ b/ee/api/chalicelib/core/sessions_exp.py @@ -24,16 +24,8 @@ s.errors_count AS errors_count, s.user_anonymous_id AS user_anonymous_id, s.platform AS platform, 0 AS issue_score, -s.issue_types AS issue_types, --- , --- to_jsonb(s.issue_types) AS issue_types, -isNotNull(favorite_sessions.session_id) AS favorite, --- COALESCE((SELECT TRUE --- FROM public.user_viewed_sessions AS fs --- WHERE s.session_id = fs.session_id --- AND fs.user_id = %(userId)s - AND fs.project_id = %(project_id)s LIMIT 1), FALSE) AS viewed - """ +s.issue_types AS issue_types +""" SESSION_PROJECTION_COLS_CH_MAP = """\ 'project_id', toString(%(project_id)s), @@ -52,8 +44,7 @@ SESSION_PROJECTION_COLS_CH_MAP = """\ 'errors_count', toString(s.errors_count), 'user_anonymous_id', toString(s.user_anonymous_id), 'platform', toString(s.platform), -'issue_score', '0', -'favorite', toString(isNotNull(favorite_sessions.session_id)) +'issue_score', '0' """ @@ -1926,13 +1917,14 @@ def search_query_parts_ch(data, error_status, errors_only, favorite_only, issue, # extra_constraints.append("ufe.user_id = %(userId)s") if favorite_only and not errors_only and user_id is not None: - extra_from += """INNER JOIN (SELECT 1 AS session_id) AS favorite_sessions - ON (TRUE)""" - elif not favorite_only and not errors_only and user_id is not None: - extra_from += f"""LEFT JOIN (SELECT session_id - FROM {exp_ch_helper.get_user_favorite_sessions_table()} AS user_favorite_sessions - WHERE user_id = %(userId)s) AS favorite_sessions - ON (s.session_id=favorite_sessions.session_id)""" + extra_from += f"""INNER JOIN (SELECT session_id + FROM {exp_ch_helper.get_user_favorite_sessions_table()} + WHERE user_id=%(userId)s) AS favorite_sessions USING (session_id)""" + # elif not favorite_only and not errors_only and user_id is not None: + # extra_from += f"""LEFT JOIN (SELECT session_id + # FROM {exp_ch_helper.get_user_favorite_sessions_table()} AS user_favorite_sessions + # WHERE user_id = %(userId)s) AS favorite_sessions + # ON (s.session_id=favorite_sessions.session_id)""" extra_join = "" if issue is not None: extra_join = """ diff --git a/ee/api/chalicelib/utils/ch_client.py b/ee/api/chalicelib/utils/ch_client.py index 1c9d0b01d..514820212 100644 --- a/ee/api/chalicelib/utils/ch_client.py +++ b/ee/api/chalicelib/utils/ch_client.py @@ -46,6 +46,8 @@ class ClickHouseClient: return self.__client def format(self, query, params): + if params is None: + return query return self.__client.substitute_params(query, params, self.__client.connection.context) def __exit__(self, *args): From 0c7101a82db12c606ed1ad8f928e703482d65fdd Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 30 Aug 2022 20:11:21 +0100 Subject: [PATCH 8/8] feat(chalice): fixed get favorite sessions without time limit --- api/chalicelib/core/sessions_favorite.py | 2 +- ee/api/chalicelib/core/sessions_favorite.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/sessions_favorite.py b/api/chalicelib/core/sessions_favorite.py index 728ba649d..98d7f18ce 100644 --- a/api/chalicelib/core/sessions_favorite.py +++ b/api/chalicelib/core/sessions_favorite.py @@ -61,4 +61,4 @@ def get_start_end_timestamp(project_id, user_id): {"userId": user_id, "project_id": project_id}) ) r = cur.fetchone() - return 0, 0 if r is None else r["max_start_ts"], r["min_start_ts"] + return (0, 0) if r is None else (r["max_start_ts"], r["min_start_ts"]) diff --git a/ee/api/chalicelib/core/sessions_favorite.py b/ee/api/chalicelib/core/sessions_favorite.py index c5487376d..bcb79cfb7 100644 --- a/ee/api/chalicelib/core/sessions_favorite.py +++ b/ee/api/chalicelib/core/sessions_favorite.py @@ -89,4 +89,4 @@ def get_start_end_timestamp(project_id, user_id): {"userId": user_id, "project_id": project_id}) ) r = cur.fetchone() - return 0, 0 if r is None else r["max_start_ts"], r["min_start_ts"] + return (0, 0) if r is None else (r["max_start_ts"], r["min_start_ts"])