diff --git a/.github/workflows/workers-ee.yaml b/.github/workflows/workers-ee.yaml index 35580b5a9..e434d2716 100644 --- a/.github/workflows/workers-ee.yaml +++ b/.github/workflows/workers-ee.yaml @@ -86,7 +86,11 @@ jobs: ;; esac - [[ $(cat /tmp/images_to_build.txt) != "" ]] || (echo "Nothing to build here"; exit 1) + if [[ $(cat /tmp/images_to_build.txt) == "" ]]; then + echo "Nothing to build here" + touch /tmp/nothing-to-build-here + exit 0 + fi # # Pushing image to registry # @@ -94,7 +98,7 @@ jobs: for image in $(cat /tmp/images_to_build.txt); do echo "Bulding $image" - PUSH_IMAGE=0 bash -x ./build.sh skip $image + PUSH_IMAGE=0 bash -x ./build.sh ee $image [[ "x$skip_security_checks" == "xtrue" ]] || { curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ ./trivy image --exit-code 1 --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG @@ -105,7 +109,7 @@ jobs: } && { echo "Skipping Security Checks" } - PUSH_IMAGE=1 bash -x ./build.sh skip $image + PUSH_IMAGE=1 bash -x ./build.sh ee $image echo "::set-output name=image::$DOCKER_REPO/$image:$IMAGE_TAG" done @@ -118,6 +122,7 @@ jobs: # Deploying image to environment. # set -x + [[ -f /tmp/nothing-to-build-here ]] && exit 0 cd scripts/helmcharts/ ## Update secerts diff --git a/.github/workflows/workers.yaml b/.github/workflows/workers.yaml index 341a196ad..e222e00fb 100644 --- a/.github/workflows/workers.yaml +++ b/.github/workflows/workers.yaml @@ -86,7 +86,11 @@ jobs: ;; esac - [[ $(cat /tmp/images_to_build.txt) != "" ]] || (echo "Nothing to build here"; exit 1) + if [[ $(cat /tmp/images_to_build.txt) == "" ]]; then + echo "Nothing to build here" + touch /tmp/nothing-to-build-here + exit 0 + fi # # Pushing image to registry # @@ -116,6 +120,8 @@ jobs: # # Deploying image to environment. # + set -x + [[ -f /tmp/nothing-to-build-here ]] && exit 0 cd scripts/helmcharts/ ## Update secerts diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 2ed9105b2..76ae5c615 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -199,7 +199,8 @@ def process(): logging.info(f"Valid alert, notifying users, alertId:{alert['alertId']} name: {alert['name']}") notifications.append(generate_notification(alert, result)) except Exception as e: - logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}") + logging.error( + f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}") logging.error(query) logging.error(e) cur = cur.recreate(rollback=True) @@ -212,12 +213,22 @@ def process(): alerts.process_notifications(notifications) +def __format_value(x): + if x % 1 == 0: + x = int(x) + else: + x = round(x, 2) + return f"{x:,}" + + def generate_notification(alert, result): + left = __format_value(result['value']) + right = __format_value(alert['query']['right']) return { "alertId": alert["alertId"], "tenantId": alert["tenantId"], "title": alert["name"], - "description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).", + "description": f"has been triggered, {alert['query']['left']} = {left} ({alert['query']['operator']} {right}).", "buttonText": "Check metrics for more details", "buttonUrl": f"/{alert['projectId']}/metrics", "imageUrl": None, diff --git a/api/chalicelib/core/metrics.py b/api/chalicelib/core/metrics.py index bf388c093..b25b441ed 100644 --- a/api/chalicelib/core/metrics.py +++ b/api/chalicelib/core/metrics.py @@ -419,7 +419,7 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True, chart=True, data=args) pg_sub_query_chart.append("resources.type = 'img'") - pg_sub_query_chart.append("resources.url = top_img.url") + pg_sub_query_chart.append("resources.url_hostpath = top_img.url_hostpath") pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True, chart=False, data=args) @@ -431,13 +431,13 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), with pg_client.PostgresClient() as cur: pg_query = f"""SELECT * - FROM (SELECT resources.url, + FROM (SELECT resources.url_hostpath, COALESCE(AVG(resources.duration), 0) AS avg_duration, COUNT(resources.session_id) AS sessions_count FROM events.resources INNER JOIN sessions USING (session_id) WHERE {" AND ".join(pg_sub_query_subset)} - GROUP BY resources.url + GROUP BY resources.url_hostpath ORDER BY avg_duration DESC LIMIT 10) AS top_img LEFT JOIN LATERAL ( @@ -485,13 +485,13 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi if resources and len(resources) > 0: for r in resources: if r["type"] == "IMG": - img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s") + img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s") img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] elif r["type"] == "LOCATION": location_constraints.append(f"pages.path = %(val_{len(location_constraints)})s") location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value'] else: - request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s") + request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s") request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} @@ -627,12 +627,12 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, pg_sub_query.append("url_hostpath ILIKE %(value)s") with pg_client.PostgresClient() as cur: pg_query = f"""SELECT key, value - FROM ( SELECT DISTINCT ON (url) ROW_NUMBER() OVER (PARTITION BY type ORDER BY url) AS r, - url AS value, + FROM ( SELECT DISTINCT ON (url_hostpath) ROW_NUMBER() OVER (PARTITION BY type ORDER BY url_hostpath) AS r, + url_hostpath AS value, type AS key FROM events.resources INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} - ORDER BY url, type ASC) AS ranked_values + ORDER BY url_hostpath, type ASC) AS ranked_values WHERE ranked_values.r<=5;""" cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text)})) rows = cur.fetchall() @@ -893,7 +893,7 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days if type is not None: pg_sub_query_subset.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") if url is not None: - pg_sub_query_subset.append(f"resources.url = %(value)s") + pg_sub_query_subset.append(f"resources.url_hostpath = %(value)s") with pg_client.PostgresClient() as cur: pg_query = f"""WITH resources AS (SELECT resources.duration, timestamp @@ -1009,7 +1009,7 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ORDER BY avg DESC LIMIT 10) AS main_list INNER JOIN LATERAL ( - SELECT url, type + SELECT url_hostpath AS url, type FROM events.resources INNER JOIN public.sessions USING (session_id) WHERE {" AND ".join(pg_sub_query)} diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index fcea8621d..91efb967f 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -177,7 +177,7 @@ def _isUndefined_operator(op: schemas.SearchEventOperator): # This function executes the query and return result def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, - error_status=schemas.ErrorStatus.all, count_only=False, issue=None): + error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False): if data.bookmarked: data.startDate, data.endDate = sessions_favorite.get_start_end_timestamp(project_id, user_id) @@ -185,9 +185,11 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ favorite_only=data.bookmarked, issue=issue, project_id=project_id, user_id=user_id) if data.limit is not None and data.page is not None: + full_args["sessions_limit"] = data.limit full_args["sessions_limit_s"] = (data.page - 1) * data.limit full_args["sessions_limit_e"] = data.page * data.limit else: + full_args["sessions_limit"] = 200 full_args["sessions_limit_s"] = 1 full_args["sessions_limit_e"] = 200 @@ -235,6 +237,12 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ GROUP BY user_id ) AS users_sessions;""", full_args) + elif ids_only: + main_query = cur.mogrify(f"""SELECT DISTINCT ON(s.session_id) s.session_id + {query_part} + ORDER BY s.session_id desc + LIMIT %(sessions_limit)s OFFSET %(sessions_limit_s)s;""", + full_args) else: if data.order is None: data.order = schemas.SortOrderType.desc @@ -242,7 +250,6 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ if data.sort is not None and data.sort != "session_id": # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) sort = helper.key_to_snake_case(data.sort) - meta_keys = metadata.get(project_id=project_id) main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) @@ -266,7 +273,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ print(data.json()) print("--------------------") raise err - if errors_only: + if errors_only or ids_only: return helper.list_to_camel_case(cur.fetchall()) sessions = cur.fetchone() diff --git a/api/chalicelib/core/sessions_devtool.py b/api/chalicelib/core/sessions_devtool.py index eef7b8e6b..2afc3c366 100644 --- a/api/chalicelib/core/sessions_devtool.py +++ b/api/chalicelib/core/sessions_devtool.py @@ -1,6 +1,6 @@ from decouple import config -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 def __get_devtools_keys(project_id, session_id): @@ -16,7 +16,7 @@ def __get_devtools_keys(project_id, session_id): def get_urls(session_id, project_id): results = [] for k in __get_devtools_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) diff --git a/api/chalicelib/core/sessions_mobs.py b/api/chalicelib/core/sessions_mobs.py index 3d966a47c..9a9237be8 100644 --- a/api/chalicelib/core/sessions_mobs.py +++ b/api/chalicelib/core/sessions_mobs.py @@ -1,7 +1,6 @@ from decouple import config from chalicelib.utils import s3 -from chalicelib.utils.s3 import client def __get_mob_keys(project_id, session_id): @@ -15,10 +14,14 @@ def __get_mob_keys(project_id, session_id): ] +def __get_mob_keys_deprecated(session_id): + return [str(session_id), str(session_id) + "e"] + + def get_urls(project_id, session_id): results = [] for k in __get_mob_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) @@ -27,27 +30,18 @@ def get_urls(project_id, session_id): def get_urls_depercated(session_id): - return [ - client.generate_presigned_url( + results = [] + for k in __get_mob_keys_deprecated(session_id=session_id): + results.append(s3.client.generate_presigned_url( 'get_object', - Params={ - 'Bucket': config("sessions_bucket"), - 'Key': str(session_id) - }, + Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=100000 - ), - client.generate_presigned_url( - 'get_object', - Params={ - 'Bucket': config("sessions_bucket"), - 'Key': str(session_id) + "e" - }, - ExpiresIn=100000 - )] + )) + return results def get_ios(session_id): - return client.generate_presigned_url( + return s3.client.generate_presigned_url( 'get_object', Params={ 'Bucket': config("ios_bucket"), diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 2abd87cf7..a38dc82d1 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -181,9 +181,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: values=s["value"], value_key=f"value{i + 1}") n_stages_query.append(f""" (SELECT main.session_id, - {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp, - '{event_type}' AS type, - '{s["operator"]}' AS operator + {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp FROM {next_table} AS main {" ".join(extra_from)} WHERE main.timestamp >= {f"T{i}.stage{i}_timestamp" if i > 0 else "%(startTimestamp)s"} {f"AND main.session_id=T1.session_id" if i > 0 else ""} @@ -191,30 +189,33 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: {(" AND " + " AND ".join(stage_constraints)) if len(stage_constraints) > 0 else ""} {(" AND " + " AND ".join(first_stage_extra_constraints)) if len(first_stage_extra_constraints) > 0 and i == 0 else ""} GROUP BY main.session_id) - AS T{i + 1} {"USING (session_id)" if i > 0 else ""} + AS T{i + 1} {"ON (TRUE)" if i > 0 else ""} """) - if len(n_stages_query) == 0: + n_stages=len(n_stages_query) + if n_stages == 0: return [] n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query) n_stages_query += ") AS stages_t" n_stages_query = f""" - SELECT stages_and_issues_t.*, sessions.user_uuid FROM ( + SELECT stages_and_issues_t.*, sessions.user_uuid + FROM ( SELECT * FROM ( - SELECT * FROM - {n_stages_query} + SELECT T1.session_id, {",".join([f"stage{i + 1}_timestamp" for i in range(n_stages)])} + FROM {n_stages_query} LEFT JOIN LATERAL - ( SELECT ISE.session_id, - ISS.type as issue_type, + ( SELECT ISS.type as issue_type, ISE.timestamp AS issue_timestamp, - ISS.context_string as issue_context, + COALESCE(ISS.context_string,'') as issue_context, ISS.issue_id as issue_id FROM events_common.issues AS ISE INNER JOIN issues AS ISS USING (issue_id) WHERE ISE.timestamp >= stages_t.stage1_timestamp AND ISE.timestamp <= stages_t.stage{i + 1}_timestamp AND ISS.project_id=%(project_id)s + AND ISE.session_id = stages_t.session_id {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""} - ) AS issues_t USING (session_id) + LIMIT 20 -- remove the limit to get exact stats + ) AS issues_t ON (TRUE) ) AS stages_and_issues_t INNER JOIN sessions USING(session_id); """ @@ -297,7 +298,21 @@ def pearson_corr(x: list, y: list): return r, confidence, False -def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_with_context, first_stage, last_stage): +# def tuple_or(t: tuple): +# x = 0 +# for el in t: +# x |= el # | is for bitwise OR +# return x +# +# The following function is correct optimization of the previous function because t is a list of 0,1 +def tuple_or(t: tuple): + for el in t: + if el > 0: + return 1 + return 0 + + +def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues, first_stage, last_stage): """ Returns two lists with binary values 0/1: @@ -316,12 +331,6 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_ transitions = [] n_sess_affected = 0 errors = {} - for issue in all_issues_with_context: - split = issue.split('__^__') - errors[issue] = { - "errors": [], - "issue_type": split[0], - "context": split[1]} for row in rows: t = 0 @@ -329,38 +338,26 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_ last_ts = row[f'stage{last_stage}_timestamp'] if first_ts is None: continue - elif first_ts is not None and last_ts is not None: + elif last_ts is not None: t = 1 transitions.append(t) ic_present = False - for issue_type_with_context in errors: + for error_id in all_issues: + if error_id not in errors: + errors[error_id] = [] ic = 0 - issue_type = errors[issue_type_with_context]["issue_type"] - context = errors[issue_type_with_context]["context"] - if row['issue_type'] is not None: + row_issue_id=row['issue_id'] + if row_issue_id is not None: if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts): - context_in_row = row['issue_context'] if row['issue_context'] is not None else '' - if issue_type == row['issue_type'] and context == context_in_row: + if error_id == row_issue_id: ic = 1 ic_present = True - errors[issue_type_with_context]["errors"].append(ic) + errors[error_id].append(ic) if ic_present and t: n_sess_affected += 1 - # def tuple_or(t: tuple): - # x = 0 - # for el in t: - # x |= el - # return x - def tuple_or(t: tuple): - for el in t: - if el > 0: - return 1 - return 0 - - errors = {key: errors[key]["errors"] for key in errors} all_errors = [tuple_or(t) for t in zip(*errors.values())] return transitions, errors, all_errors, n_sess_affected @@ -376,10 +373,9 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage): """ affected_users = defaultdict(lambda: set()) affected_sessions = defaultdict(lambda: set()) - contexts = defaultdict(lambda: None) + all_issues = {} n_affected_users_dict = defaultdict(lambda: None) n_affected_sessions_dict = defaultdict(lambda: None) - all_issues_with_context = set() n_issues_dict = defaultdict(lambda: 0) issues_by_session = defaultdict(lambda: 0) @@ -395,15 +391,13 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage): # check that the issue exists and belongs to subfunnel: if iss is not None and (row[f'stage{last_stage}_timestamp'] is None or (row[f'stage{first_stage}_timestamp'] < iss_ts < row[f'stage{last_stage}_timestamp'])): - context_string = row['issue_context'] if row['issue_context'] is not None else '' - issue_with_context = iss + '__^__' + context_string - contexts[issue_with_context] = {"context": context_string, "id": row["issue_id"]} - all_issues_with_context.add(issue_with_context) - n_issues_dict[issue_with_context] += 1 + if row["issue_id"] not in all_issues: + all_issues[row["issue_id"]] = {"context": row['issue_context'], "issue_type": row["issue_type"]} + n_issues_dict[row["issue_id"]] += 1 if row['user_uuid'] is not None: - affected_users[issue_with_context].add(row['user_uuid']) + affected_users[row["issue_id"]].add(row['user_uuid']) - affected_sessions[issue_with_context].add(row['session_id']) + affected_sessions[row["issue_id"]].add(row['session_id']) issues_by_session[row[f'session_id']] += 1 if len(affected_users) > 0: @@ -414,29 +408,28 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage): n_affected_sessions_dict.update({ iss: len(affected_sessions[iss]) for iss in affected_sessions }) - return all_issues_with_context, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict, contexts + return all_issues, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict def count_sessions(rows, n_stages): session_counts = {i: set() for i in range(1, n_stages + 1)} - for ind, row in enumerate(rows): + for row in rows: for i in range(1, n_stages + 1): if row[f"stage{i}_timestamp"] is not None: session_counts[i].add(row[f"session_id"]) + session_counts = {i: len(session_counts[i]) for i in session_counts} return session_counts def count_users(rows, n_stages): - users_in_stages = defaultdict(lambda: set()) - - for ind, row in enumerate(rows): + users_in_stages = {i: set() for i in range(1, n_stages + 1)} + for row in rows: for i in range(1, n_stages + 1): if row[f"stage{i}_timestamp"] is not None: users_in_stages[i].add(row["user_uuid"]) users_count = {i: len(users_in_stages[i]) for i in range(1, n_stages + 1)} - return users_count @@ -489,18 +482,18 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) last_stage = n_stages n_critical_issues = 0 - issues_dict = dict({"significant": [], - "insignificant": []}) + issues_dict = {"significant": [], + "insignificant": []} session_counts = count_sessions(rows, n_stages) drop = session_counts[first_stage] - session_counts[last_stage] - all_issues_with_context, n_issues_dict, affected_users_dict, affected_sessions, contexts = get_affected_users_for_all_issues( + all_issues, n_issues_dict, affected_users_dict, affected_sessions = get_affected_users_for_all_issues( rows, first_stage, last_stage) transitions, errors, all_errors, n_sess_affected = get_transitions_and_issues_of_each_type(rows, - all_issues_with_context, + all_issues, first_stage, last_stage) - # print("len(transitions) =", len(transitions)) + del rows if any(all_errors): total_drop_corr, conf, is_sign = pearson_corr(transitions, all_errors) @@ -513,33 +506,32 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) if drop_only: return total_drop_due_to_issues - for issue in all_issues_with_context: + for issue_id in all_issues: - if not any(errors[issue]): + if not any(errors[issue_id]): continue - r, confidence, is_sign = pearson_corr(transitions, errors[issue]) + r, confidence, is_sign = pearson_corr(transitions, errors[issue_id]) if r is not None and drop is not None and is_sign: - lost_conversions = int(r * affected_sessions[issue]) + lost_conversions = int(r * affected_sessions[issue_id]) else: lost_conversions = None if r is None: r = 0 - split = issue.split('__^__') issues_dict['significant' if is_sign else 'insignificant'].append({ - "type": split[0], - "title": helper.get_issue_title(split[0]), - "affected_sessions": affected_sessions[issue], - "unaffected_sessions": session_counts[1] - affected_sessions[issue], + "type": all_issues[issue_id]["issue_type"], + "title": helper.get_issue_title(all_issues[issue_id]["issue_type"]), + "affected_sessions": affected_sessions[issue_id], + "unaffected_sessions": session_counts[1] - affected_sessions[issue_id], "lost_conversions": lost_conversions, - "affected_users": affected_users_dict[issue], + "affected_users": affected_users_dict[issue_id], "conversion_impact": round(r * 100), - "context_string": contexts[issue]["context"], - "issue_id": contexts[issue]["id"] + "context_string": all_issues[issue_id]["context"], + "issue_id": issue_id }) if is_sign: - n_critical_issues += n_issues_dict[issue] + n_critical_issues += n_issues_dict[issue_id] return n_critical_issues, issues_dict, total_drop_due_to_issues diff --git a/api/chalicelib/core/sourcemaps.py b/api/chalicelib/core/sourcemaps.py index 921649d97..89df77926 100644 --- a/api/chalicelib/core/sourcemaps.py +++ b/api/chalicelib/core/sourcemaps.py @@ -77,7 +77,7 @@ def format_payload(p, truncate_to_first=False): def url_exists(url): try: r = requests.head(url, allow_redirects=False) - return r.status_code == 200 and r.headers.get("Content-Type") != "text/html" + return r.status_code == 200 and "text/html" not in r.headers.get("Content-Type", "") except Exception as e: print(f"!! Issue checking if URL exists: {url}") print(e) @@ -100,7 +100,6 @@ def get_traces_group(project_id, payload): and not (file_url[:params_idx] if params_idx > -1 else file_url).endswith(".js"): print(f"{u['absPath']} sourcemap is not a JS file") payloads[key] = None - continue if key not in payloads: file_exists_in_bucket = len(file_url) > 0 and s3.exists(config('sourcemaps_bucket'), key) diff --git a/api/env.default b/api/env.default index 82419328c..3ee65e89c 100644 --- a/api/env.default +++ b/api/env.default @@ -42,11 +42,11 @@ sourcemaps_reader=http://sourcemaps-reader-openreplay.app.svc.cluster.local:9000 STAGE=default-foss version_number=1.4.0 FS_DIR=/mnt/efs -EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob -EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob +EFS_SESSION_MOB_PATTERN=%(sessionId)s +EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe -DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs +DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index b30e65988..ff36f3099 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -1,15 +1,15 @@ requests==2.28.1 urllib3==1.26.12 -boto3==1.26.4 +boto3==1.26.14 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.5.0 +elasticsearch==8.5.1 jira==3.4.1 -fastapi==0.86.0 -uvicorn[standard]==0.19.0 +fastapi==0.87.0 +uvicorn[standard]==0.20.0 python-decouple==3.6 pydantic[email]==1.10.2 -apscheduler==3.9.1 \ No newline at end of file +apscheduler==3.9.1.post1 \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt index b30e65988..ff36f3099 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,15 +1,15 @@ requests==2.28.1 urllib3==1.26.12 -boto3==1.26.4 +boto3==1.26.14 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.5.0 +elasticsearch==8.5.1 jira==3.4.1 -fastapi==0.86.0 -uvicorn[standard]==0.19.0 +fastapi==0.87.0 +uvicorn[standard]==0.20.0 python-decouple==3.6 pydantic[email]==1.10.2 -apscheduler==3.9.1 \ No newline at end of file +apscheduler==3.9.1.post1 \ No newline at end of file diff --git a/api/routers/core.py b/api/routers/core.py index 80f2b6296..7ee8364e7 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -56,6 +56,14 @@ def sessions_search(projectId: int, data: schemas.FlatSessionsSearchPayloadSchem return {'data': data} +@app.post('/{projectId}/sessions/search/ids', tags=["sessions"]) +@app.post('/{projectId}/sessions/search2/ids', tags=["sessions"]) +def session_ids_search(projectId: int, data: schemas.FlatSessionsSearchPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + data = sessions.search_sessions(data=data, project_id=projectId, user_id=context.user_id, ids_only=True) + return {'data': data} + + @app.get('/{projectId}/events/search', tags=["events"]) def events_search(projectId: int, q: str, type: Union[schemas.FilterType, schemas.EventType, diff --git a/backend/Dockerfile b/backend/Dockerfile index 4e0064e9d..0d7cad075 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.18-alpine3.15 AS prepare -RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash +RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash librdkafka-dev cyrus-sasl cyrus-sasl-gssapiv2 krb5 WORKDIR /root @@ -15,11 +15,11 @@ COPY pkg pkg COPY internal internal ARG SERVICE_NAME -RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o service -tags musl openreplay/backend/cmd/$SERVICE_NAME +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o service -tags dynamic openreplay/backend/cmd/$SERVICE_NAME FROM alpine AS entrypoint -RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates librdkafka-dev cyrus-sasl cyrus-sasl-gssapiv2 krb5 RUN adduser -u 1001 openreplay -D ENV TZ=UTC \ @@ -29,6 +29,18 @@ ENV TZ=UTC \ UAPARSER_FILE=/home/openreplay/regexes.yaml \ HTTP_PORT=8080 \ KAFKA_USE_SSL=true \ + # KAFKA_USE_KERBEROS should be set true if you wish to use Kerberos auth for Kafka + KAFKA_USE_KERBEROS=false \ + # KERBEROS_SERVICE_NAME is the primary name of the Brokers configured in the Broker JAAS file + KERBEROS_SERVICE_NAME="" \ + # KERBEROS_PRINCIPAL is this client's principal name + KERBEROS_PRINCIPAL="" \ + # KERBEROS_PRINCIPAL is the absolute path to the keytab to be used for authentication + KERBEROS_KEYTAB_LOCATION="" \ + # KAFKA_SSL_KEY is the absolute path to the CA cert for verifying the broker's key + KAFKA_SSL_KEY="" \ + # KAFKA_SSL_CERT is a CA cert string (PEM format) for verifying the broker's key + KAFKA_SSL_CERT="" \ KAFKA_MAX_POLL_INTERVAL_MS=400000 \ REDIS_STREAMS_MAX_LEN=10000 \ TOPIC_RAW_WEB=raw \ diff --git a/backend/Dockerfile.bundle b/backend/Dockerfile.bundle index 407a7b9d8..19c3b325c 100644 --- a/backend/Dockerfile.bundle +++ b/backend/Dockerfile.bundle @@ -1,6 +1,6 @@ FROM golang:1.18-alpine3.15 AS prepare -RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash +RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash librdkafka-dev cyrus-sasl-gssapi cyrus-sasl-devel WORKDIR /root @@ -14,11 +14,11 @@ COPY cmd cmd COPY pkg pkg COPY internal internal -RUN for name in assets db ender http integrations sink storage;do CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o bin/$name -tags musl openreplay/backend/cmd/$name; done +RUN for name in assets db ender http integrations sink storage;do CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o bin/$name -tags dynamic openreplay/backend/cmd/$name; done FROM alpine AS entrypoint #FROM pygmy/alpine-tini:latest -RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates librdkafka-dev cyrus-sasl-gssapi cyrus-sasl-devel pkgconf ENV TZ=UTC \ FS_ULIMIT=1000 \ @@ -28,6 +28,18 @@ ENV TZ=UTC \ HTTP_PORT=80 \ BEACON_SIZE_LIMIT=7000000 \ KAFKA_USE_SSL=true \ + # KAFKA_USE_KERBEROS should be set true if you wish to use Kerberos auth for Kafka + KAFKA_USE_KERBEROS=false \ + # KERBEROS_SERVICE_NAME is the primary name of the Brokers configured in the Broker JAAS file + KERBEROS_SERVICE_NAME="" \ + # KERBEROS_PRINCIPAL is this client's principal name + KERBEROS_PRINCIPAL="" \ + # KERBEROS_PRINCIPAL is the absolute path to the keytab to be used for authentication + KERBEROS_KEYTAB_LOCATION="" \ + # KAFKA_SSL_KEY is the absolute path to the CA cert for verifying the broker's key + KAFKA_SSL_KEY="" \ + # KAFKA_SSL_CERT is a CA cert string (PEM format) for verifying the broker's key + KAFKA_SSL_CERT="" \ KAFKA_MAX_POLL_INTERVAL_MS=400000 \ REDIS_STREAMS_MAX_LEN=3000 \ TOPIC_RAW_WEB=raw \ diff --git a/backend/cmd/sink/main.go b/backend/cmd/sink/main.go index d3cc99e40..84520dd33 100644 --- a/backend/cmd/sink/main.go +++ b/backend/cmd/sink/main.go @@ -3,19 +3,18 @@ package main import ( "context" "log" - "openreplay/backend/pkg/pprof" "os" "os/signal" - "strings" "syscall" "time" "openreplay/backend/internal/config/sink" "openreplay/backend/internal/sink/assetscache" - "openreplay/backend/internal/sink/oswriter" + "openreplay/backend/internal/sink/sessionwriter" "openreplay/backend/internal/storage" "openreplay/backend/pkg/messages" "openreplay/backend/pkg/monitoring" + "openreplay/backend/pkg/pprof" "openreplay/backend/pkg/queue" "openreplay/backend/pkg/url/assets" ) @@ -33,7 +32,7 @@ func main() { log.Fatalf("%v doesn't exist. %v", cfg.FsDir, err) } - writer := oswriter.NewWriter(cfg.FsUlimit, cfg.FsDir) + writer := sessionwriter.NewWriter(cfg.FsUlimit, cfg.FsDir, cfg.DeadSessionTimeout) producer := queue.NewProducer(cfg.MessageSizeLimit, true) defer producer.Close(cfg.ProducerCloseTimeout) @@ -64,6 +63,7 @@ func main() { if err := producer.Produce(cfg.TopicTrigger, msg.SessionID(), msg.Encode()); err != nil { log.Printf("can't send SessionEnd to trigger topic: %s; sessID: %d", err, msg.SessionID()) } + writer.Close(msg.SessionID()) return } @@ -98,39 +98,18 @@ func main() { // Write encoded message with index to session file data := msg.EncodeWithIndex() if data == nil { - log.Printf("can't encode with index, err: %s", err) return } - wasWritten := false // To avoid timestamp duplicates in original mob file + + // Write message to file if messages.IsDOMType(msg.TypeID()) { if err := writer.WriteDOM(msg.SessionID(), data); err != nil { - if strings.Contains(err.Error(), "not a directory") { - // Trying to write data to mob file by original path - oldErr := writer.WriteMOB(msg.SessionID(), data) - if oldErr != nil { - log.Printf("MOB Writeer error: %s, prev DOM error: %s, info: %s", oldErr, err, msg.Meta().Batch().Info()) - } else { - wasWritten = true - } - } else { - log.Printf("DOM Writer error: %s, info: %s", err, msg.Meta().Batch().Info()) - } + log.Printf("Writer error: %v\n", err) } } if !messages.IsDOMType(msg.TypeID()) || msg.TypeID() == messages.MsgTimestamp { - // TODO: write only necessary timestamps if err := writer.WriteDEV(msg.SessionID(), data); err != nil { - if strings.Contains(err.Error(), "not a directory") { - if !wasWritten { - // Trying to write data to mob file by original path - oldErr := writer.WriteMOB(msg.SessionID(), data) - if oldErr != nil { - log.Printf("MOB Writeer error: %s, prev DEV error: %s, info: %s", oldErr, err, msg.Meta().Batch().Info()) - } - } - } else { - log.Printf("Devtools Writer error: %s, info: %s", err, msg.Meta().Batch().Info()) - } + log.Printf("Writer error: %v\n", err) } } @@ -158,22 +137,20 @@ func main() { select { case sig := <-sigchan: log.Printf("Caught signal %v: terminating\n", sig) - if err := writer.CloseAll(); err != nil { - log.Printf("closeAll error: %v\n", err) - } + // Sync and stop writer + writer.Stop() + // Commit and stop consumer if err := consumer.Commit(); err != nil { log.Printf("can't commit messages: %s", err) } consumer.Close() os.Exit(0) case <-tick: - if err := writer.SyncAll(); err != nil { - log.Fatalf("sync error: %v\n", err) - } counter.Print() if err := consumer.Commit(); err != nil { log.Printf("can't commit messages: %s", err) } + log.Printf("writer: %s", writer.Info()) default: err := consumer.ConsumeNext() if err != nil { diff --git a/backend/go.mod b/backend/go.mod index 0eead389c..b1046b08e 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -69,8 +69,8 @@ require ( golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + golang.org/x/text v0.4.0 // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect diff --git a/backend/go.sum b/backend/go.sum index dbaee7216..fea2aa1a3 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -678,8 +678,9 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -690,8 +691,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/backend/internal/config/sink/config.go b/backend/internal/config/sink/config.go index a7481f93a..a8703a596 100644 --- a/backend/internal/config/sink/config.go +++ b/backend/internal/config/sink/config.go @@ -9,6 +9,7 @@ type Config struct { common.Config FsDir string `env:"FS_DIR,required"` FsUlimit uint16 `env:"FS_ULIMIT,required"` + DeadSessionTimeout int64 `env:"DEAD_SESSION_TIMEOUT,default=600"` GroupSink string `env:"GROUP_SINK,required"` TopicRawWeb string `env:"TOPIC_RAW_WEB,required"` TopicRawIOS string `env:"TOPIC_RAW_IOS,required"` @@ -17,7 +18,7 @@ type Config struct { CacheAssets bool `env:"CACHE_ASSETS,required"` AssetsOrigin string `env:"ASSETS_ORIGIN,required"` ProducerCloseTimeout int `env:"PRODUCER_CLOSE_TIMEOUT,default=15000"` - CacheThreshold int64 `env:"CACHE_THRESHOLD,default=75"` + CacheThreshold int64 `env:"CACHE_THRESHOLD,default=5"` CacheExpiration int64 `env:"CACHE_EXPIRATION,default=120"` } diff --git a/backend/internal/config/storage/config.go b/backend/internal/config/storage/config.go index fdf29b7db..6083f0249 100644 --- a/backend/internal/config/storage/config.go +++ b/backend/internal/config/storage/config.go @@ -11,7 +11,6 @@ type Config struct { S3Region string `env:"AWS_REGION_WEB,required"` S3Bucket string `env:"S3_BUCKET_WEB,required"` FSDir string `env:"FS_DIR,required"` - FSCleanHRS int `env:"FS_CLEAN_HRS,required"` FileSplitSize int `env:"FILE_SPLIT_SIZE,required"` RetryTimeout time.Duration `env:"RETRY_TIMEOUT,default=2m"` GroupStorage string `env:"GROUP_STORAGE,required"` @@ -21,6 +20,7 @@ type Config struct { DeleteTimeout time.Duration `env:"DELETE_TIMEOUT,default=48h"` ProducerCloseTimeout int `env:"PRODUCER_CLOSE_TIMEOUT,default=15000"` UseFailover bool `env:"USE_FAILOVER,default=false"` + MaxFileSize int64 `env:"MAX_FILE_SIZE,default=524288000"` } func New() *Config { diff --git a/backend/internal/sink/oswriter/oswriter.go b/backend/internal/sink/oswriter/oswriter.go deleted file mode 100644 index 070540b1d..000000000 --- a/backend/internal/sink/oswriter/oswriter.go +++ /dev/null @@ -1,166 +0,0 @@ -package oswriter - -import ( - "errors" - "log" - "math" - "os" - "path/filepath" - "strconv" - "time" -) - -type Writer struct { - ulimit int - dir string - files map[string]*os.File - atimes map[string]int64 -} - -func NewWriter(ulimit uint16, dir string) *Writer { - return &Writer{ - ulimit: int(ulimit), - dir: dir + "/", - files: make(map[string]*os.File), - atimes: make(map[string]int64), - } -} - -func (w *Writer) open(fname string) (*os.File, error) { - file, ok := w.files[fname] - if ok { - return file, nil - } - if len(w.atimes) == w.ulimit { - var m_k string - var m_t int64 = math.MaxInt64 - for k, t := range w.atimes { - if t < m_t { - m_k = k - m_t = t - } - } - if err := w.close(m_k); err != nil { - return nil, err - } - } - - // mkdir if not exist - pathTo := w.dir + filepath.Dir(fname) - if info, err := os.Stat(pathTo); os.IsNotExist(err) { - if err := os.MkdirAll(pathTo, 0755); err != nil { - log.Printf("os.MkdirAll error: %s", err) - } - } else { - if err != nil { - return nil, err - } - if !info.IsDir() { - return nil, errors.New("not a directory") - } - } - - file, err := os.OpenFile(w.dir+fname, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - log.Printf("os.OpenFile error: %s", err) - return nil, err - } - w.files[fname] = file - w.atimes[fname] = time.Now().Unix() - return file, nil -} - -func (w *Writer) close(fname string) error { - file := w.files[fname] - if file == nil { - return nil - } - if err := file.Sync(); err != nil { - return err - } - if err := file.Close(); err != nil { - return err - } - delete(w.files, fname) - delete(w.atimes, fname) - return nil -} - -func (w *Writer) WriteDOM(sid uint64, data []byte) error { - return w.write(strconv.FormatUint(sid, 10)+"/dom.mob", data) -} - -func (w *Writer) WriteDEV(sid uint64, data []byte) error { - return w.write(strconv.FormatUint(sid, 10)+"/devtools.mob", data) -} - -func (w *Writer) WriteMOB(sid uint64, data []byte) error { - // Use session id as a file name without directory - fname := strconv.FormatUint(sid, 10) - file, err := w.openWithoutDir(fname) - if err != nil { - return err - } - _, err = file.Write(data) - return err -} - -func (w *Writer) write(fname string, data []byte) error { - file, err := w.open(fname) - if err != nil { - return err - } - _, err = file.Write(data) - return err -} - -func (w *Writer) openWithoutDir(fname string) (*os.File, error) { - file, ok := w.files[fname] - if ok { - return file, nil - } - if len(w.atimes) == w.ulimit { - var m_k string - var m_t int64 = math.MaxInt64 - for k, t := range w.atimes { - if t < m_t { - m_k = k - m_t = t - } - } - if err := w.close(m_k); err != nil { - return nil, err - } - } - - file, err := os.OpenFile(w.dir+fname, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - return nil, err - } - w.files[fname] = file - w.atimes[fname] = time.Now().Unix() - return file, nil -} - -func (w *Writer) SyncAll() error { - for _, file := range w.files { - if err := file.Sync(); err != nil { - return err - } - } - return nil -} - -func (w *Writer) CloseAll() error { - for _, file := range w.files { - if err := file.Sync(); err != nil { - return err - } - if err := file.Close(); err != nil { - return err - } - } - w.files = nil - w.atimes = nil - return nil -} diff --git a/backend/internal/sink/sessionwriter/session.go b/backend/internal/sink/sessionwriter/session.go new file mode 100644 index 000000000..f107c387b --- /dev/null +++ b/backend/internal/sink/sessionwriter/session.go @@ -0,0 +1,81 @@ +package sessionwriter + +import ( + "fmt" + "os" + "strconv" + "sync" + "time" +) + +type Session struct { + lock *sync.Mutex + dom *os.File + dev *os.File + lastUpdate time.Time +} + +func NewSession(dir string, id uint64) (*Session, error) { + if id == 0 { + return nil, fmt.Errorf("wrong session id") + } + + filePath := dir + strconv.FormatUint(id, 10) + domFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return nil, err + } + filePath += "devtools" + devFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + domFile.Close() // should close first file descriptor + return nil, err + } + + return &Session{ + lock: &sync.Mutex{}, + dom: domFile, + dev: devFile, + lastUpdate: time.Now(), + }, nil +} + +func (s *Session) Lock() { + s.lock.Lock() +} + +func (s *Session) Unlock() { + s.lock.Unlock() +} + +func (s *Session) Write(mode FileType, data []byte) (err error) { + if mode == DOM { + _, err = s.dom.Write(data) + } else { + _, err = s.dev.Write(data) + } + s.lastUpdate = time.Now() + return err +} + +func (s *Session) LastUpdate() time.Time { + return s.lastUpdate +} + +func (s *Session) Sync() error { + domErr := s.dom.Sync() + devErr := s.dev.Sync() + if domErr == nil && devErr == nil { + return nil + } + return fmt.Errorf("dom: %s, dev: %s", domErr, devErr) +} + +func (s *Session) Close() error { + domErr := s.dom.Close() + devErr := s.dev.Close() + if domErr == nil && devErr == nil { + return nil + } + return fmt.Errorf("dom: %s, dev: %s", domErr, devErr) +} diff --git a/backend/internal/sink/sessionwriter/types.go b/backend/internal/sink/sessionwriter/types.go new file mode 100644 index 000000000..a20f61375 --- /dev/null +++ b/backend/internal/sink/sessionwriter/types.go @@ -0,0 +1,8 @@ +package sessionwriter + +type FileType int + +const ( + DOM FileType = 1 + DEV FileType = 2 +) diff --git a/backend/internal/sink/sessionwriter/writer.go b/backend/internal/sink/sessionwriter/writer.go new file mode 100644 index 000000000..94ff5dd66 --- /dev/null +++ b/backend/internal/sink/sessionwriter/writer.go @@ -0,0 +1,179 @@ +package sessionwriter + +import ( + "fmt" + "log" + "math" + "sync" + "time" +) + +type SessionWriter struct { + ulimit int + dir string + zombieSessionTimeout float64 + lock *sync.Mutex + sessions *sync.Map + meta map[uint64]int64 + done chan struct{} + stopped chan struct{} +} + +func NewWriter(ulimit uint16, dir string, zombieSessionTimeout int64) *SessionWriter { + w := &SessionWriter{ + ulimit: int(ulimit) / 2, // should divide by 2 because each session has 2 files + dir: dir + "/", + zombieSessionTimeout: float64(zombieSessionTimeout), + lock: &sync.Mutex{}, + sessions: &sync.Map{}, + meta: make(map[uint64]int64, ulimit), + done: make(chan struct{}), + stopped: make(chan struct{}), + } + go w.synchronizer() + return w +} + +func (w *SessionWriter) WriteDOM(sid uint64, data []byte) error { + return w.write(sid, DOM, data) +} + +func (w *SessionWriter) WriteDEV(sid uint64, data []byte) error { + return w.write(sid, DEV, data) +} + +func (w *SessionWriter) Close(sid uint64) { + w.close(sid) +} + +func (w *SessionWriter) Stop() { + w.done <- struct{}{} + <-w.stopped +} + +func (w *SessionWriter) Info() string { + return fmt.Sprintf("%d sessions", w.numberOfSessions()) +} + +func (w *SessionWriter) addSession(sid uint64) { + w.lock.Lock() + w.meta[sid] = time.Now().Unix() + w.lock.Unlock() +} + +func (w *SessionWriter) deleteSession(sid uint64) { + w.lock.Lock() + delete(w.meta, sid) + w.lock.Unlock() +} + +func (w *SessionWriter) numberOfSessions() int { + w.lock.Lock() + defer w.lock.Unlock() + return len(w.meta) +} + +func (w *SessionWriter) write(sid uint64, mode FileType, data []byte) error { + var ( + sess *Session + err error + ) + + sessObj, ok := w.sessions.Load(sid) + if !ok { + sess, err = NewSession(w.dir, sid) + if err != nil { + return fmt.Errorf("can't write to session: %d, err: %s", sid, err) + } + sess.Lock() + defer sess.Unlock() + + // Check opened files limit + if len(w.meta) >= w.ulimit { + var oldSessID uint64 + var minTimestamp int64 = math.MaxInt64 + for sessID, timestamp := range w.meta { + if timestamp < minTimestamp { + oldSessID = sessID + minTimestamp = timestamp + } + } + if err := w.close(oldSessID); err != nil { + log.Printf("can't close session: %s", err) + } + } + + // Add new session to manager + w.sessions.Store(sid, sess) + w.addSession(sid) + } else { + sess = sessObj.(*Session) + sess.Lock() + defer sess.Unlock() + } + + // Write data to session + return sess.Write(mode, data) +} + +func (w *SessionWriter) sync(sid uint64) error { + sessObj, ok := w.sessions.Load(sid) + if !ok { + return fmt.Errorf("can't sync, session: %d not found", sid) + } + sess := sessObj.(*Session) + sess.Lock() + defer sess.Unlock() + + err := sess.Sync() + if time.Now().Sub(sess.LastUpdate()).Seconds() > w.zombieSessionTimeout { + if err != nil { + log.Printf("can't sync session: %d, err: %s", sid, err) + } + // Close "zombie" session + err = sess.Close() + w.deleteSession(sid) + } + return err +} + +func (w *SessionWriter) close(sid uint64) error { + sessObj, ok := w.sessions.LoadAndDelete(sid) + if !ok { + return fmt.Errorf("can't close, session: %d not found", sid) + } + sess := sessObj.(*Session) + sess.Lock() + defer sess.Unlock() + + if err := sess.Sync(); err != nil { + log.Printf("can't sync session: %d, err: %s", sid, err) + } + err := sess.Close() + w.deleteSession(sid) + return err +} + +func (w *SessionWriter) synchronizer() { + tick := time.Tick(2 * time.Second) + for { + select { + case <-tick: + w.sessions.Range(func(sid, lockObj any) bool { + if err := w.sync(sid.(uint64)); err != nil { + log.Printf("can't sync file descriptor: %s", err) + } + return true + }) + case <-w.done: + w.sessions.Range(func(sid, lockObj any) bool { + if err := w.close(sid.(uint64)); err != nil { + log.Printf("can't close file descriptor: %s", err) + } + return true + }) + w.stopped <- struct{}{} + return + } + } +} diff --git a/backend/internal/storage/storage.go b/backend/internal/storage/storage.go index 7fdc06c4f..12a37183f 100644 --- a/backend/internal/storage/storage.go +++ b/backend/internal/storage/storage.go @@ -13,7 +13,6 @@ import ( "openreplay/backend/pkg/storage" "os" "strconv" - "strings" "time" ) @@ -71,43 +70,46 @@ func New(cfg *config.Config, s3 *storage.S3, metrics *monitoring.Metrics) (*Stor } func (s *Storage) UploadSessionFiles(msg *messages.SessionEnd) error { - sessionDir := strconv.FormatUint(msg.SessionID(), 10) - if err := s.uploadKey(msg.SessionID(), sessionDir+"/dom.mob", true, 5, msg.EncryptionKey); err != nil { - oldErr := s.uploadKey(msg.SessionID(), sessionDir, true, 5, msg.EncryptionKey) - if oldErr != nil { - return fmt.Errorf("upload file error: %s. failed checking mob file using old path: %s", err, oldErr) - } - // Exit method anyway because we don't have dev tools separation in prev version - return nil - } - if err := s.uploadKey(msg.SessionID(), sessionDir+"/devtools.mob", false, 4, msg.EncryptionKey); err != nil { + if err := s.uploadKey(msg.SessionID(), "/dom.mob", true, 5, msg.EncryptionKey); err != nil { return err } + if err := s.uploadKey(msg.SessionID(), "/devtools.mob", false, 4, msg.EncryptionKey); err != nil { + log.Printf("can't find devtools for session: %d, err: %s", msg.SessionID(), err) + } return nil } -// TODO: make a bit cleaner -func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCount int, encryptionKey string) error { +// TODO: make a bit cleaner. +// TODO: Of course, I'll do! +func (s *Storage) uploadKey(sessID uint64, suffix string, shouldSplit bool, retryCount int, encryptionKey string) error { if retryCount <= 0 { return nil } - start := time.Now() - file, err := os.Open(s.cfg.FSDir + "/" + key) + fileName := strconv.FormatUint(sessID, 10) + mobFileName := fileName + if suffix == "/devtools.mob" { + mobFileName += "devtools" + } + filePath := s.cfg.FSDir + "/" + mobFileName + + // Check file size before download into memory + info, err := os.Stat(filePath) + if err == nil { + if info.Size() > s.cfg.MaxFileSize { + log.Printf("big file, size: %d, session: %d", info.Size(), sessID) + return nil + } + } + file, err := os.Open(filePath) if err != nil { return fmt.Errorf("File open error: %v; sessID: %s, part: %d, sessStart: %s\n", - err, key, sessID%16, + err, fileName, sessID%16, time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))), ) } defer file.Close() - // Ignore "s" at the end of mob file name for "old" sessions - newVers := false - if strings.Contains(key, "/") { - newVers = true - } - var fileSize int64 = 0 fileInfo, err := file.Stat() if err != nil { @@ -117,17 +119,18 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo } var encryptedData []byte + fileName += suffix if shouldSplit { nRead, err := file.Read(s.startBytes) if err != nil { log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s", err, - key, + fileName, sessID%16, time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))), ) time.AfterFunc(s.cfg.RetryTimeout, func() { - s.uploadKey(sessID, key, shouldSplit, retryCount-1, encryptionKey) + s.uploadKey(sessID, suffix, shouldSplit, retryCount-1, encryptionKey) }) return nil } @@ -146,11 +149,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo } // Compress and save to s3 startReader := bytes.NewBuffer(encryptedData) - startKey := key - if newVers { - startKey += "s" - } - if err := s.s3.Upload(s.gzipFile(startReader), startKey, "application/octet-stream", true); err != nil { + if err := s.s3.Upload(s.gzipFile(startReader), fileName+"s", "application/octet-stream", true); err != nil { log.Fatalf("Storage: start upload failed. %v\n", err) } // TODO: fix possible error (if we read less then FileSplitSize) @@ -161,7 +160,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo if err != nil { log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s", err, - key, + fileName, sessID%16, time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))), ) @@ -183,7 +182,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo } // Compress and save to s3 endReader := bytes.NewBuffer(encryptedData) - if err := s.s3.Upload(s.gzipFile(endReader), key+"e", "application/octet-stream", true); err != nil { + if err := s.s3.Upload(s.gzipFile(endReader), fileName+"e", "application/octet-stream", true); err != nil { log.Fatalf("Storage: end upload failed. %v\n", err) } } @@ -195,7 +194,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo if err != nil { log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s", err, - key, + fileName, sessID%16, time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))), ) @@ -216,7 +215,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo encryptedData = fileData } endReader := bytes.NewBuffer(encryptedData) - if err := s.s3.Upload(s.gzipFile(endReader), key+"s", "application/octet-stream", true); err != nil { + if err := s.s3.Upload(s.gzipFile(endReader), fileName, "application/octet-stream", true); err != nil { log.Fatalf("Storage: end upload failed. %v\n", err) } s.archivingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds())) diff --git a/backend/pkg/db/types/error-event.go b/backend/pkg/db/types/error-event.go index 826cbba9e..bef9abd99 100644 --- a/backend/pkg/db/types/error-event.go +++ b/backend/pkg/db/types/error-event.go @@ -11,6 +11,8 @@ import ( . "openreplay/backend/pkg/messages" ) +const SOURCE_JS = "js_exception" + type ErrorEvent struct { MessageID uint64 Timestamp uint64 @@ -64,7 +66,7 @@ func WrapJSException(m *JSException) *ErrorEvent { return &ErrorEvent{ MessageID: m.Meta().Index, Timestamp: uint64(m.Meta().Timestamp), - Source: "js_exception", + Source: SOURCE_JS, Name: m.Name, Message: m.Message, Payload: m.Payload, @@ -105,14 +107,16 @@ func (e *ErrorEvent) ID(projectID uint32) string { hash.Write([]byte(e.Source)) hash.Write([]byte(e.Name)) hash.Write([]byte(e.Message)) - frame, err := parseFirstFrame(e.Payload) - if err != nil { - log.Printf("Can't parse stackframe ((( %v ))): %v", e.Payload, err) - } - if frame != nil { - hash.Write([]byte(frame.FileName)) - hash.Write([]byte(strconv.Itoa(frame.LineNo))) - hash.Write([]byte(strconv.Itoa(frame.ColNo))) + if e.Source == SOURCE_JS { + frame, err := parseFirstFrame(e.Payload) + if err != nil { + log.Printf("Can't parse stackframe ((( %v ))): %v", e.Payload, err) + } + if frame != nil { + hash.Write([]byte(frame.FileName)) + hash.Write([]byte(strconv.Itoa(frame.LineNo))) + hash.Write([]byte(strconv.Itoa(frame.ColNo))) + } } return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil)) } diff --git a/ee/api/chalicelib/core/alerts_processor.py b/ee/api/chalicelib/core/alerts_processor.py index 087f23a05..326d17ffc 100644 --- a/ee/api/chalicelib/core/alerts_processor.py +++ b/ee/api/chalicelib/core/alerts_processor.py @@ -204,7 +204,8 @@ def process(): logging.info(f"Valid alert, notifying users, alertId:{alert['alertId']} name: {alert['name']}") notifications.append(generate_notification(alert, result)) except Exception as e: - logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}") + logging.error( + f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}") logging.error(query) logging.error(e) cur = cur.recreate(rollback=True) @@ -217,12 +218,22 @@ def process(): alerts.process_notifications(notifications) +def __format_value(x): + if x % 1 == 0: + x = int(x) + else: + x = round(x, 2) + return f"{x:,}" + + def generate_notification(alert, result): + left = __format_value(result['value']) + right = __format_value(alert['query']['right']) return { "alertId": alert["alertId"], "tenantId": alert["tenantId"], "title": alert["name"], - "description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).", + "description": f"has been triggered, {alert['query']['left']} = {left} ({alert['query']['operator']} {right}).", "buttonText": "Check metrics for more details", "buttonUrl": f"/{alert['projectId']}/metrics", "imageUrl": None, diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index 452566194..2a2f6ee20 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -452,18 +452,18 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query.append("resources.type = 'img'") ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) ch_sub_query_chart.append("resources.type = 'img'") - ch_sub_query_chart.append("resources.url IN %(url)s") + ch_sub_query_chart.append("resources.url_hostpath IN %(url)s") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT resources.url, + ch_query = f"""SELECT resources.url_hostpath AS url, COALESCE(avgOrNull(resources.duration),0) AS avg, COUNT(1) AS count FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query)} AND resources.duration>0 - GROUP BY resources.url ORDER BY avg DESC LIMIT 10;""" + GROUP BY resources.url_hostpath ORDER BY avg DESC LIMIT 10;""" params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} rows = ch.execute(query=ch_query, params=params) @@ -474,13 +474,13 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), urls = [row["url"] for row in rows] charts = {} - ch_query = f"""SELECT url, + ch_query = f"""SELECT url_hostpath AS url, toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, COALESCE(avgOrNull(resources.duration),0) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} WHERE {" AND ".join(ch_sub_query_chart)} AND resources.duration>0 - GROUP BY url, timestamp - ORDER BY url, timestamp;""" + GROUP BY url_hostpath, timestamp + ORDER BY url_hostpath, timestamp;""" params["url"] = urls u_rows = ch.execute(query=ch_query, params=params) for url in urls: @@ -526,13 +526,13 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi if resources and len(resources) > 0: for r in resources: if r["type"] == "IMG": - img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s") + img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s") img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] elif r["type"] == "LOCATION": location_constraints.append(f"pages.url_path = %(val_{len(location_constraints)})s") location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value'] else: - request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s") + request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s") request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} @@ -638,7 +638,7 @@ def search(text, resource_type, project_id, performance=False, pages_only=False, if resource_type == "ALL" and not pages_only and not events_only: ch_sub_query.append("positionUTF8(url_hostpath,%(value)s)!=0") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url))), 1, 5)) AS value, + ch_query = f"""SELECT arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url_hostpath))), 1, 5)) AS value, type AS key FROM resources WHERE {" AND ".join(ch_sub_query)} @@ -884,7 +884,7 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days if type is not None: ch_sub_query_chart.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") if url is not None: - ch_sub_query_chart.append(f"resources.url = %(value)s") + ch_sub_query_chart.append(f"resources.url_hostpath = %(value)s") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition ch_sub_query_chart.append("resources.duration>0") @@ -966,7 +966,7 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query_chart.append("isNotNull(resources.duration)") ch_sub_query_chart.append("resources.duration>0") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT any(url) AS url, any(type) AS type, + ch_query = f"""SELECT any(url_hostpath) AS url, any(type) AS type, splitByChar('/', resources.url_hostpath)[-1] AS name, COALESCE(avgOrNull(NULLIF(resources.duration,0)),0) AS avg FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""} @@ -2179,7 +2179,7 @@ def get_performance_avg_image_load_time(ch, project_id, startTimestamp=TimeUTC.n if resources and len(resources) > 0: for r in resources: if r["type"] == "IMG": - img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s") + img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s") img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, @@ -2254,7 +2254,7 @@ def get_performance_avg_request_load_time(ch, project_id, startTimestamp=TimeUTC if resources and len(resources) > 0: for r in resources: if r["type"] != "IMG" and r["type"] == "LOCATION": - request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s") + request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s") request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} diff --git a/ee/api/chalicelib/core/metrics_exp.py b/ee/api/chalicelib/core/metrics_exp.py index 9a8af012b..c41676d4a 100644 --- a/ee/api/chalicelib/core/metrics_exp.py +++ b/ee/api/chalicelib/core/metrics_exp.py @@ -462,18 +462,18 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args) # ch_sub_query_chart.append("events.event_type='RESOURCE'") ch_sub_query_chart.append("resources.type = 'img'") - ch_sub_query_chart.append("resources.url IN %(url)s") + ch_sub_query_chart.append("resources.url_hostpath IN %(url)s") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition ch_sub_query_chart += meta_condition with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT resources.url, + ch_query = f"""SELECT resources.url_hostpath AS url, COALESCE(avgOrNull(resources.duration),0) AS avg, COUNT(1) AS count FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources WHERE {" AND ".join(ch_sub_query)} AND resources.duration>0 - GROUP BY resources.url ORDER BY avg DESC LIMIT 10;""" + GROUP BY resources.url_hostpath ORDER BY avg DESC LIMIT 10;""" params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp, **__get_constraint_values(args)} rows = ch.execute(query=ch_query, params=params) @@ -484,13 +484,13 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1), urls = [row["url"] for row in rows] charts = {} - ch_query = f"""SELECT url, + ch_query = f"""SELECT url_hostpath AS url, toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp, COALESCE(avgOrNull(resources.duration),0) AS avg FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources WHERE {" AND ".join(ch_sub_query_chart)} AND resources.duration>0 - GROUP BY url, timestamp - ORDER BY url, timestamp;""" + GROUP BY url_hostpath, timestamp + ORDER BY url_hostpath, timestamp;""" params["url"] = urls # print(ch.format(query=ch_query, params=params)) u_rows = ch.execute(query=ch_query, params=params) @@ -538,13 +538,13 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi if resources and len(resources) > 0: for r in resources: if r["type"] == "IMG": - img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s") + img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s") img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] elif r["type"] == "LOCATION": location_constraints.append(f"pages.url_path = %(val_{len(location_constraints)})s") location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value'] else: - request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s") + request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s") request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} @@ -891,7 +891,7 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days if type is not None: ch_sub_query_chart.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'") if url is not None: - ch_sub_query_chart.append(f"resources.url = %(value)s") + ch_sub_query_chart.append(f"resources.url_hostpath = %(value)s") meta_condition = __get_meta_constraint(args) ch_sub_query_chart += meta_condition ch_sub_query_chart.append("resources.duration>0") @@ -974,7 +974,7 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1), ch_sub_query_chart.append("isNotNull(resources.duration)") ch_sub_query_chart.append("resources.duration>0") with ch_client.ClickHouseClient() as ch: - ch_query = f"""SELECT any(url) AS url, any(type) AS type, name, + ch_query = f"""SELECT any(url_hostpath) AS url, any(type) AS type, name, COALESCE(avgOrNull(NULLIF(resources.duration,0)),0) AS avg FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources WHERE {" AND ".join(ch_sub_query)} @@ -2185,7 +2185,7 @@ def get_performance_avg_image_load_time(ch, project_id, startTimestamp=TimeUTC.n if resources and len(resources) > 0: for r in resources: if r["type"] == "IMG": - img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s") + img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s") img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value'] params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, @@ -2260,7 +2260,7 @@ def get_performance_avg_request_load_time(ch, project_id, startTimestamp=TimeUTC if resources and len(resources) > 0: for r in resources: if r["type"] != "IMG" and r["type"] == "LOCATION": - request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s") + request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s") request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value'] params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp} diff --git a/ee/api/chalicelib/core/sessions.py b/ee/api/chalicelib/core/sessions.py index 8c9eaf006..7d999fe6c 100644 --- a/ee/api/chalicelib/core/sessions.py +++ b/ee/api/chalicelib/core/sessions.py @@ -107,8 +107,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu session_id=session_id, user_id=context.user_id) data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data) data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id) - data['live'] = live and assist.is_live(project_id=project_id, - session_id=session_id, + data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id, project_key=data["projectKey"]) data["inDB"] = True return data @@ -181,7 +180,7 @@ def _isUndefined_operator(op: schemas.SearchEventOperator): # This function executes the query and return result def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False, - error_status=schemas.ErrorStatus.all, count_only=False, issue=None): + error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False): if data.bookmarked: data.startDate, data.endDate = sessions_favorite.get_start_end_timestamp(project_id, user_id) @@ -189,9 +188,11 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ favorite_only=data.bookmarked, issue=issue, project_id=project_id, user_id=user_id) if data.limit is not None and data.page is not None: + full_args["sessions_limit"] = data.limit full_args["sessions_limit_s"] = (data.page - 1) * data.limit full_args["sessions_limit_e"] = data.page * data.limit else: + full_args["sessions_limit"] = 200 full_args["sessions_limit_s"] = 1 full_args["sessions_limit_e"] = 200 @@ -239,6 +240,12 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ GROUP BY user_id ) AS users_sessions;""", full_args) + elif ids_only: + main_query = cur.mogrify(f"""SELECT DISTINCT ON(s.session_id) s.session_id + {query_part} + ORDER BY s.session_id desc + LIMIT %(sessions_limit)s OFFSET %(sessions_limit_s)s;""", + full_args) else: if data.order is None: data.order = schemas.SortOrderType.desc @@ -246,7 +253,6 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ if data.sort is not None and data.sort != "session_id": # sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) sort = helper.key_to_snake_case(data.sort) - meta_keys = metadata.get(project_id=project_id) main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) @@ -270,7 +276,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_ print(data.json()) print("--------------------") raise err - if errors_only: + if errors_only or ids_only: return helper.list_to_camel_case(cur.fetchall()) sessions = cur.fetchone() diff --git a/ee/api/chalicelib/core/sessions_devtool.py b/ee/api/chalicelib/core/sessions_devtool.py index ed6ecf694..9435c2e24 100644 --- a/ee/api/chalicelib/core/sessions_devtool.py +++ b/ee/api/chalicelib/core/sessions_devtool.py @@ -3,7 +3,7 @@ from fastapi.security import SecurityScopes import schemas_ee from chalicelib.core import permissions -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 SCOPES = SecurityScopes([schemas_ee.Permissions.dev_tools]) @@ -23,7 +23,7 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext): return [] results = [] for k in __get_devtools_keys(project_id=project_id, session_id=session_id): - results.append(client.generate_presigned_url( + results.append(s3.client.generate_presigned_url( 'get_object', Params={'Bucket': config("sessions_bucket"), 'Key': k}, ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900) diff --git a/ee/api/chalicelib/core/sessions_favorite.py b/ee/api/chalicelib/core/sessions_favorite.py index 3d6496424..7af995bad 100644 --- a/ee/api/chalicelib/core/sessions_favorite.py +++ b/ee/api/chalicelib/core/sessions_favorite.py @@ -1,7 +1,7 @@ from decouple import config import schemas_ee -from chalicelib.core import sessions, sessions_favorite_exp +from chalicelib.core import sessions, sessions_favorite_exp, sessions_mobs, sessions_devtool from chalicelib.utils import pg_client, s3_extra @@ -34,32 +34,31 @@ def remove_favorite_session(context: schemas_ee.CurrentContext, project_id, sess def favorite_session(context: schemas_ee.CurrentContext, project_id, session_id): + keys = sessions_mobs.__get_mob_keys(project_id=project_id, session_id=session_id) + keys += sessions_mobs.__get_mob_keys_deprecated(session_id=session_id) # To support old sessions + keys += sessions_devtool.__get_devtools_keys(project_id=project_id, session_id=session_id) + if favorite_session_exists(user_id=context.user_id, session_id=session_id): - key = str(session_id) - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_D_VALUE', default='default')) - except Exception as e: - print(f"!!!Error while tagging: {key} to default") - print(str(e)) - key = str(session_id) + "e" - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_D_VALUE', default='default')) - except Exception as e: - print(f"!!!Error while tagging: {key} to default") - print(str(e)) + tag = config('RETENTION_D_VALUE', default='default') + + for k in keys: + try: + s3_extra.tag_session(file_key=k, tag_value=tag) + except Exception as e: + print(f"!!!Error while tagging: {k} to {tag} for removal") + print(str(e)) + return remove_favorite_session(context=context, project_id=project_id, session_id=session_id) - key = str(session_id) - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_L_VALUE', default='vault')) - except Exception as e: - print(f"!!!Error while tagging: {key} to vault") - print(str(e)) - key = str(session_id) + "e" - try: - s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_L_VALUE', default='vault')) - except Exception as e: - print(f"!!!Error while tagging: {key} to vault") - print(str(e)) + + tag = config('RETENTION_L_VALUE', default='vault') + + for k in keys: + try: + s3_extra.tag_session(file_key=k, tag_value=tag) + except Exception as e: + print(f"!!!Error while tagging: {k} to {tag} for vault") + print(str(e)) + return add_favorite_session(context=context, project_id=project_id, session_id=session_id) diff --git a/ee/api/chalicelib/core/significance.py b/ee/api/chalicelib/core/significance.py index 59f773c9e..75df1cd94 100644 --- a/ee/api/chalicelib/core/significance.py +++ b/ee/api/chalicelib/core/significance.py @@ -188,9 +188,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: values=s["value"], value_key=f"value{i + 1}") n_stages_query.append(f""" (SELECT main.session_id, - {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp, - '{event_type}' AS type, - '{s["operator"]}' AS operator + {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp FROM {next_table} AS main {" ".join(extra_from)} WHERE main.timestamp >= {f"T{i}.stage{i}_timestamp" if i > 0 else "%(startTimestamp)s"} {f"AND main.session_id=T1.session_id" if i > 0 else ""} @@ -198,45 +196,54 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: {(" AND " + " AND ".join(stage_constraints)) if len(stage_constraints) > 0 else ""} {(" AND " + " AND ".join(first_stage_extra_constraints)) if len(first_stage_extra_constraints) > 0 and i == 0 else ""} GROUP BY main.session_id) - AS T{i + 1} {"USING (session_id)" if i > 0 else ""} + AS T{i + 1} {"ON (TRUE)" if i > 0 else ""} """) - if len(n_stages_query) == 0: + n_stages=len(n_stages_query) + if n_stages == 0: return [] n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query) n_stages_query += ") AS stages_t" n_stages_query = f""" - SELECT stages_and_issues_t.*,sessions.session_id, sessions.user_uuid FROM ( + SELECT stages_and_issues_t.*, sessions.user_uuid + FROM ( SELECT * FROM ( - SELECT * FROM - {n_stages_query} + SELECT T1.session_id, {",".join([f"stage{i + 1}_timestamp" for i in range(n_stages)])} + FROM {n_stages_query} LEFT JOIN LATERAL - ( - SELECT * FROM - (SELECT ISE.session_id, - ISS.type as issue_type, + ( SELECT ISS.type as issue_type, ISE.timestamp AS issue_timestamp, - ISS.context_string as issue_context, + COALESCE(ISS.context_string,'') as issue_context, ISS.issue_id as issue_id FROM events_common.issues AS ISE INNER JOIN issues AS ISS USING (issue_id) WHERE ISE.timestamp >= stages_t.stage1_timestamp AND ISE.timestamp <= stages_t.stage{i + 1}_timestamp AND ISS.project_id=%(project_id)s - {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""}) AS base_t - ) AS issues_t - USING (session_id)) AS stages_and_issues_t - inner join sessions USING(session_id); + AND ISE.session_id = stages_t.session_id + {"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""} + LIMIT 20 -- remove the limit to get exact stats + ) AS issues_t ON (TRUE) + ) AS stages_and_issues_t INNER JOIN sessions USING(session_id); """ # LIMIT 10000 params = {"project_id": project_id, "startTimestamp": filter_d["startDate"], "endTimestamp": filter_d["endDate"], "issueTypes": tuple(filter_issues), **values} with pg_client.PostgresClient() as cur: + query = cur.mogrify(n_stages_query, params) # print("---------------------------------------------------") - # print(cur.mogrify(n_stages_query, params)) + # print(query) # print("---------------------------------------------------") - cur.execute(cur.mogrify(n_stages_query, params)) - rows = cur.fetchall() + try: + cur.execute(query) + rows = cur.fetchall() + except Exception as err: + print("--------- FUNNEL SEARCH QUERY EXCEPTION -----------") + print(query.decode('UTF-8')) + print("--------- PAYLOAD -----------") + print(filter_d) + print("--------------------") + raise err return rows @@ -298,7 +305,21 @@ def pearson_corr(x: list, y: list): return r, confidence, False -def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_with_context, first_stage, last_stage): +# def tuple_or(t: tuple): +# x = 0 +# for el in t: +# x |= el # | is for bitwise OR +# return x +# +# The following function is correct optimization of the previous function because t is a list of 0,1 +def tuple_or(t: tuple): + for el in t: + if el > 0: + return 1 + return 0 + + +def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues, first_stage, last_stage): """ Returns two lists with binary values 0/1: @@ -317,12 +338,6 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_ transitions = [] n_sess_affected = 0 errors = {} - for issue in all_issues_with_context: - split = issue.split('__^__') - errors[issue] = { - "errors": [], - "issue_type": split[0], - "context": split[1]} for row in rows: t = 0 @@ -330,38 +345,26 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_ last_ts = row[f'stage{last_stage}_timestamp'] if first_ts is None: continue - elif first_ts is not None and last_ts is not None: + elif last_ts is not None: t = 1 transitions.append(t) ic_present = False - for issue_type_with_context in errors: + for error_id in all_issues: + if error_id not in errors: + errors[error_id] = [] ic = 0 - issue_type = errors[issue_type_with_context]["issue_type"] - context = errors[issue_type_with_context]["context"] - if row['issue_type'] is not None: + row_issue_id=row['issue_id'] + if row_issue_id is not None: if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts): - context_in_row = row['issue_context'] if row['issue_context'] is not None else '' - if issue_type == row['issue_type'] and context == context_in_row: + if error_id == row_issue_id: ic = 1 ic_present = True - errors[issue_type_with_context]["errors"].append(ic) + errors[error_id].append(ic) if ic_present and t: n_sess_affected += 1 - # def tuple_or(t: tuple): - # x = 0 - # for el in t: - # x |= el - # return x - def tuple_or(t: tuple): - for el in t: - if el > 0: - return 1 - return 0 - - errors = {key: errors[key]["errors"] for key in errors} all_errors = [tuple_or(t) for t in zip(*errors.values())] return transitions, errors, all_errors, n_sess_affected @@ -377,10 +380,9 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage): """ affected_users = defaultdict(lambda: set()) affected_sessions = defaultdict(lambda: set()) - contexts = defaultdict(lambda: None) + all_issues = {} n_affected_users_dict = defaultdict(lambda: None) n_affected_sessions_dict = defaultdict(lambda: None) - all_issues_with_context = set() n_issues_dict = defaultdict(lambda: 0) issues_by_session = defaultdict(lambda: 0) @@ -396,15 +398,13 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage): # check that the issue exists and belongs to subfunnel: if iss is not None and (row[f'stage{last_stage}_timestamp'] is None or (row[f'stage{first_stage}_timestamp'] < iss_ts < row[f'stage{last_stage}_timestamp'])): - context_string = row['issue_context'] if row['issue_context'] is not None else '' - issue_with_context = iss + '__^__' + context_string - contexts[issue_with_context] = {"context": context_string, "id": row["issue_id"]} - all_issues_with_context.add(issue_with_context) - n_issues_dict[issue_with_context] += 1 + if row["issue_id"] not in all_issues: + all_issues[row["issue_id"]] = {"context": row['issue_context'], "issue_type": row["issue_type"]} + n_issues_dict[row["issue_id"]] += 1 if row['user_uuid'] is not None: - affected_users[issue_with_context].add(row['user_uuid']) + affected_users[row["issue_id"]].add(row['user_uuid']) - affected_sessions[issue_with_context].add(row['session_id']) + affected_sessions[row["issue_id"]].add(row['session_id']) issues_by_session[row[f'session_id']] += 1 if len(affected_users) > 0: @@ -415,29 +415,28 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage): n_affected_sessions_dict.update({ iss: len(affected_sessions[iss]) for iss in affected_sessions }) - return all_issues_with_context, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict, contexts + return all_issues, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict def count_sessions(rows, n_stages): session_counts = {i: set() for i in range(1, n_stages + 1)} - for ind, row in enumerate(rows): + for row in rows: for i in range(1, n_stages + 1): if row[f"stage{i}_timestamp"] is not None: session_counts[i].add(row[f"session_id"]) + session_counts = {i: len(session_counts[i]) for i in session_counts} return session_counts def count_users(rows, n_stages): - users_in_stages = defaultdict(lambda: set()) - - for ind, row in enumerate(rows): + users_in_stages = {i: set() for i in range(1, n_stages + 1)} + for row in rows: for i in range(1, n_stages + 1): if row[f"stage{i}_timestamp"] is not None: users_in_stages[i].add(row["user_uuid"]) users_count = {i: len(users_in_stages[i]) for i in range(1, n_stages + 1)} - return users_count @@ -490,18 +489,18 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) last_stage = n_stages n_critical_issues = 0 - issues_dict = dict({"significant": [], - "insignificant": []}) + issues_dict = {"significant": [], + "insignificant": []} session_counts = count_sessions(rows, n_stages) drop = session_counts[first_stage] - session_counts[last_stage] - all_issues_with_context, n_issues_dict, affected_users_dict, affected_sessions, contexts = get_affected_users_for_all_issues( + all_issues, n_issues_dict, affected_users_dict, affected_sessions = get_affected_users_for_all_issues( rows, first_stage, last_stage) transitions, errors, all_errors, n_sess_affected = get_transitions_and_issues_of_each_type(rows, - all_issues_with_context, + all_issues, first_stage, last_stage) - # print("len(transitions) =", len(transitions)) + del rows if any(all_errors): total_drop_corr, conf, is_sign = pearson_corr(transitions, all_errors) @@ -514,33 +513,32 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False) if drop_only: return total_drop_due_to_issues - for issue in all_issues_with_context: + for issue_id in all_issues: - if not any(errors[issue]): + if not any(errors[issue_id]): continue - r, confidence, is_sign = pearson_corr(transitions, errors[issue]) + r, confidence, is_sign = pearson_corr(transitions, errors[issue_id]) if r is not None and drop is not None and is_sign: - lost_conversions = int(r * affected_sessions[issue]) + lost_conversions = int(r * affected_sessions[issue_id]) else: lost_conversions = None if r is None: r = 0 - split = issue.split('__^__') issues_dict['significant' if is_sign else 'insignificant'].append({ - "type": split[0], - "title": helper.get_issue_title(split[0]), - "affected_sessions": affected_sessions[issue], - "unaffected_sessions": session_counts[1] - affected_sessions[issue], + "type": all_issues[issue_id]["issue_type"], + "title": helper.get_issue_title(all_issues[issue_id]["issue_type"]), + "affected_sessions": affected_sessions[issue_id], + "unaffected_sessions": session_counts[1] - affected_sessions[issue_id], "lost_conversions": lost_conversions, - "affected_users": affected_users_dict[issue], + "affected_users": affected_users_dict[issue_id], "conversion_impact": round(r * 100), - "context_string": contexts[issue]["context"], - "issue_id": contexts[issue]["id"] + "context_string": all_issues[issue_id]["context"], + "issue_id": issue_id }) if is_sign: - n_critical_issues += n_issues_dict[issue] + n_critical_issues += n_issues_dict[issue_id] return n_critical_issues, issues_dict, total_drop_due_to_issues diff --git a/ee/api/chalicelib/utils/s3_extra.py b/ee/api/chalicelib/utils/s3_extra.py index f2a538dcc..0e594c890 100644 --- a/ee/api/chalicelib/utils/s3_extra.py +++ b/ee/api/chalicelib/utils/s3_extra.py @@ -1,12 +1,16 @@ from decouple import config -from chalicelib.utils.s3 import client +from chalicelib.utils import s3 -def tag_file(session_id, tag_key='retention', tag_value='vault'): - return client.put_object_tagging( - Bucket=config("sessions_bucket"), - Key=session_id, +def tag_session(file_key, tag_key='retention', tag_value='vault'): + return tag_file(file_key=file_key, bucket=config("sessions_bucket"), tag_key=tag_key, tag_value=tag_value) + + +def tag_file(file_key, bucket, tag_key, tag_value): + return s3.client.put_object_tagging( + Bucket=bucket, + Key=file_key, Tagging={ 'TagSet': [ { diff --git a/ee/api/env.default b/ee/api/env.default index e707bec57..f5574a8a1 100644 --- a/ee/api/env.default +++ b/ee/api/env.default @@ -61,11 +61,11 @@ EXP_ALERTS=false EXP_FUNNELS=false EXP_RESOURCES=true TRACE_PERIOD=300 -EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob -EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob +EFS_SESSION_MOB_PATTERN=%(sessionId)s +EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe -DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs +DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob PRESIGNED_URL_EXPIRATION=3600 ASSIST_JWT_EXPIRATION=144000 ASSIST_JWT_SECRET= \ No newline at end of file diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index fce0ba6cc..02042a778 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -1,18 +1,18 @@ requests==2.28.1 urllib3==1.26.12 -boto3==1.26.4 +boto3==1.26.14 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.5.0 +elasticsearch==8.5.1 jira==3.4.1 -fastapi==0.86.0 -uvicorn[standard]==0.19.0 +fastapi==0.87.0 +uvicorn[standard]==0.20.0 python-decouple==3.6 pydantic[email]==1.10.2 -apscheduler==3.9.1 +apscheduler==3.9.1.post1 clickhouse-driver==0.2.4 python-multipart==0.0.5 \ No newline at end of file diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index fce0ba6cc..02042a778 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -1,18 +1,18 @@ requests==2.28.1 urllib3==1.26.12 -boto3==1.26.4 +boto3==1.26.14 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.5.0 +elasticsearch==8.5.1 jira==3.4.1 -fastapi==0.86.0 -uvicorn[standard]==0.19.0 +fastapi==0.87.0 +uvicorn[standard]==0.20.0 python-decouple==3.6 pydantic[email]==1.10.2 -apscheduler==3.9.1 +apscheduler==3.9.1.post1 clickhouse-driver==0.2.4 python-multipart==0.0.5 \ No newline at end of file diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 23fc32fe7..ac4f27a9d 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -1,18 +1,18 @@ requests==2.28.1 urllib3==1.26.12 -boto3==1.26.4 +boto3==1.26.14 pyjwt==2.6.0 psycopg2-binary==2.9.5 -elasticsearch==8.5.0 +elasticsearch==8.5.1 jira==3.4.1 -fastapi==0.86.0 -uvicorn[standard]==0.19.0 +fastapi==0.87.0 +uvicorn[standard]==0.20.0 python-decouple==3.6 pydantic[email]==1.10.2 -apscheduler==3.9.1 +apscheduler==3.9.1.post1 clickhouse-driver==0.2.4 python3-saml==1.14.0 diff --git a/ee/backend/pkg/kafka/consumer.go b/ee/backend/pkg/kafka/consumer.go index b951fcd9c..14f8d5a68 100644 --- a/ee/backend/pkg/kafka/consumer.go +++ b/ee/backend/pkg/kafka/consumer.go @@ -47,6 +47,16 @@ func NewConsumer( kafkaConfig.SetKey("ssl.key.location", os.Getenv("KAFKA_SSL_KEY")) kafkaConfig.SetKey("ssl.certificate.location", os.Getenv("KAFKA_SSL_CERT")) } + + // Apply Kerberos configuration + if env.Bool("KAFKA_USE_KERBEROS") { + kafkaConfig.SetKey("security.protocol", "sasl_plaintext") + kafkaConfig.SetKey("sasl.mechanisms", "GSSAPI") + kafkaConfig.SetKey("sasl.kerberos.service.name", os.Getenv("KERBEROS_SERVICE_NAME")) + kafkaConfig.SetKey("sasl.kerberos.principal", os.Getenv("KERBEROS_PRINCIPAL")) + kafkaConfig.SetKey("sasl.kerberos.keytab", os.Getenv("KERBEROS_KEYTAB_LOCATION")) + } + c, err := kafka.NewConsumer(kafkaConfig) if err != nil { log.Fatalln(err) diff --git a/ee/backend/pkg/kafka/producer.go b/ee/backend/pkg/kafka/producer.go index 6fb893b7a..f895241a7 100644 --- a/ee/backend/pkg/kafka/producer.go +++ b/ee/backend/pkg/kafka/producer.go @@ -30,6 +30,15 @@ func NewProducer(messageSizeLimit int, useBatch bool) *Producer { kafkaConfig.SetKey("ssl.key.location", os.Getenv("KAFKA_SSL_KEY")) kafkaConfig.SetKey("ssl.certificate.location", os.Getenv("KAFKA_SSL_CERT")) } + // Apply Kerberos configuration + if env.Bool("KAFKA_USE_KERBEROS") { + kafkaConfig.SetKey("security.protocol", "sasl_plaintext") + kafkaConfig.SetKey("sasl.mechanisms", "GSSAPI") + kafkaConfig.SetKey("sasl.kerberos.service.name", os.Getenv("KERBEROS_SERVICE_NAME")) + kafkaConfig.SetKey("sasl.kerberos.principal", os.Getenv("KERBEROS_PRINCIPAL")) + kafkaConfig.SetKey("sasl.kerberos.keytab", os.Getenv("KERBEROS_KEYTAB_LOCATION")) + } + producer, err := kafka.NewProducer(kafkaConfig) if err != nil { log.Fatalln(err) diff --git a/ee/connectors/deploy/requirements_snowflake.txt b/ee/connectors/deploy/requirements_snowflake.txt index 983a313d6..895326b32 100644 --- a/ee/connectors/deploy/requirements_snowflake.txt +++ b/ee/connectors/deploy/requirements_snowflake.txt @@ -1,8 +1,8 @@ pandas==1.5.1 kafka-python==2.0.2 SQLAlchemy==1.4.43 -snowflake-connector-python==2.8.1 -snowflake-sqlalchemy==1.4.3 +snowflake-connector-python==2.8.2 +snowflake-sqlalchemy==1.4.4 PyYAML asn1crypto==1.5.1 azure-common==1.1.28 diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql index 65db23b07..6da0eebed 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql @@ -70,4 +70,11 @@ WHERE deleted_at IS NOT NULL; UPDATE roles SET permissions=array_remove(permissions, 'ERRORS'); +DROP INDEX IF EXISTS events_common.requests_url_idx; +DROP INDEX IF EXISTS events_common.requests_url_gin_idx; +DROP INDEX IF EXISTS events_common.requests_url_gin_idx2; + +DROP INDEX IF EXISTS events.resources_url_gin_idx; +DROP INDEX IF EXISTS events.resources_url_idx; + COMMIT; \ No newline at end of file diff --git a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql index f486c731e..78026e245 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -1221,19 +1221,9 @@ $$ query text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); - CREATE INDEX IF NOT EXISTS requests_url_idx ON events_common.requests (url); + CREATE INDEX IF NOT EXISTS requests_duration_idx ON events_common.requests (duration); - CREATE INDEX IF NOT EXISTS requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops); CREATE INDEX IF NOT EXISTS requests_timestamp_idx ON events_common.requests (timestamp); - CREATE INDEX IF NOT EXISTS requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, - length(url) - - (CASE - WHEN url LIKE 'http://%' - THEN 7 - WHEN url LIKE 'https://%' - THEN 8 - ELSE 0 END)) - gin_trgm_ops); CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; CREATE INDEX IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; CREATE INDEX IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; diff --git a/frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx b/frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx index aaecc0b14..3e8a68f11 100644 --- a/frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx +++ b/frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { Icon } from 'UI'; import { checkForRecent } from 'App/date'; import { withSiteId, alertEdit } from 'App/routes'; +import { numberWithCommas } from 'App/utils'; // @ts-ignore import { DateTime } from 'luxon'; import { withRouter, RouteComponentProps } from 'react-router-dom'; @@ -108,7 +109,7 @@ function AlertListItem(props: Props) { {' is '} {alert.query.operator} - {alert.query.right} {alert.metric.unit} + {numberWithCommas(alert.query.right)} {alert.metric.unit} {' over the past '} {getThreshold(alert.currentPeriod)} diff --git a/frontend/app/components/Dashboard/components/Alerts/NewAlert.tsx b/frontend/app/components/Dashboard/components/Alerts/NewAlert.tsx index 0ea012e71..6027646f7 100644 --- a/frontend/app/components/Dashboard/components/Alerts/NewAlert.tsx +++ b/frontend/app/components/Dashboard/components/Alerts/NewAlert.tsx @@ -122,6 +122,9 @@ const NewAlert = (props: IProps) => { ) { remove(instance.alertId).then(() => { props.history.push(withSiteId(alerts(), siteId)); + toast.success('Alert deleted'); + }).catch(() => { + toast.error('Failed to delete an alert'); }); } }; @@ -135,6 +138,8 @@ const NewAlert = (props: IProps) => { } else { toast.success('Alert updated'); } + }).catch(() => { + toast.error('Failed to create an alert'); }); }; diff --git a/frontend/app/components/Dashboard/components/DashboardList/DashboardsView.tsx b/frontend/app/components/Dashboard/components/DashboardList/DashboardsView.tsx index 5341c3487..7378e88f8 100644 --- a/frontend/app/components/Dashboard/components/DashboardList/DashboardsView.tsx +++ b/frontend/app/components/Dashboard/components/DashboardList/DashboardsView.tsx @@ -35,7 +35,7 @@ function DashboardsView({ history, siteId }: { history: any, siteId: string }) {
- A dashboard is a custom visualization using your OpenReplay data. + A Dashboard is a collection of Metrics that can be shared across teams.
diff --git a/frontend/app/components/Dashboard/components/MetricsView/MetricsView.tsx b/frontend/app/components/Dashboard/components/MetricsView/MetricsView.tsx index dd87b2fef..6c39114cd 100644 --- a/frontend/app/components/Dashboard/components/MetricsView/MetricsView.tsx +++ b/frontend/app/components/Dashboard/components/MetricsView/MetricsView.tsx @@ -30,7 +30,7 @@ function MetricsView({ siteId }: Props) {
- Create custom Metrics to capture key interactions and track KPIs. + Create custom Metrics to capture user frustrations, monitor your app's performance and track other KPIs.
diff --git a/frontend/app/components/Errors/Error/ErrorInfo.js b/frontend/app/components/Errors/Error/ErrorInfo.js index 91c06b617..abfa4e76b 100644 --- a/frontend/app/components/Errors/Error/ErrorInfo.js +++ b/frontend/app/components/Errors/Error/ErrorInfo.js @@ -35,8 +35,10 @@ export default class ErrorInfo extends React.PureComponent { componentDidMount() { this.ensureInstance(); } - componentDidUpdate() { - this.ensureInstance(); + componentDidUpdate(prevProps) { + if (prevProps.errorId !== this.props.errorId || prevProps.errorIdInStore !== this.props.errorIdInStore) { + this.ensureInstance(); + } } next = () => { const { list, errorId } = this.props; diff --git a/frontend/app/components/Session/LivePlayer.js b/frontend/app/components/Session/LivePlayer.js index 2168d468f..f06de8972 100644 --- a/frontend/app/components/Session/LivePlayer.js +++ b/frontend/app/components/Session/LivePlayer.js @@ -2,13 +2,9 @@ import React from 'react'; import { useEffect, useState } from 'react'; import { connect } from 'react-redux'; import { toggleFullscreen, closeBottomBlock } from 'Duck/components/player'; -import { withRequest } from 'HOCs' -import { - PlayerProvider, -} from 'Player'; +import { withRequest } from 'HOCs'; import withPermissions from 'HOCs/withPermissions'; import { PlayerContext, defaultContextValue } from './playerContext'; -import { createLiveWebPlayer } from 'Player'; import { makeAutoObservable } from 'mobx'; import PlayerBlockHeader from '../Session_/PlayerBlockHeader'; @@ -25,13 +21,12 @@ function LivePlayer ({ request, isEnterprise, userEmail, - userName + userName, }) { const [contextValue, setContextValue] = useState(defaultContextValue); - + const [fullView, setFullView] = useState(false); useEffect(() => { if (!loadingCredentials) { - const sessionWithAgentData = { ...session.toJS(), agentInfo: { @@ -53,19 +48,24 @@ function LivePlayer ({ // LAYOUT (TODO: local layout state - useContext or something..) useEffect(() => { + const queryParams = new URLSearchParams(window.location.search); + if (queryParams.has('fullScreen') && queryParams.get('fullScreen') === 'true') { + setFullView(true); + } + if (isEnterprise) { request(); } return () => { toggleFullscreen(false); closeBottomBlock(); - } - }, []) + }; + }, []); const TABS = { EVENTS: 'User Steps', HEATMAPS: 'Click Map', - } + }; const [activeTab, setActiveTab] = useState(''); if (!contextValue.player) return null; @@ -73,31 +73,39 @@ function LivePlayer ({ return ( - + {!fullView && ()}
); -}; +} export default withRequest({ initialData: null, endpoint: '/assist/credentials', - dataWrapper: data => data, + dataWrapper: (data) => data, dataName: 'assistCredendials', loadingName: 'loadingCredentials', -})(withPermissions(['ASSIST_LIVE'], '', true)(connect( - state => { - return { - session: state.getIn([ 'sessions', 'current' ]), - showAssist: state.getIn([ 'sessions', 'showChatWindow' ]), - fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]), - isEnterprise: state.getIn([ 'user', 'account', 'edition' ]) === 'ee', - userEmail: state.getIn(['user', 'account', 'email']), - userName: state.getIn(['user', 'account', 'name']), - } - }, - { toggleFullscreen, closeBottomBlock }, -)(LivePlayer))); +})( + withPermissions( + ['ASSIST_LIVE'], + '', + true + )( + connect( + (state) => { + return { + session: state.getIn(['sessions', 'current']), + showAssist: state.getIn(['sessions', 'showChatWindow']), + fullscreen: state.getIn(['components', 'player', 'fullscreen']), + isEnterprise: state.getIn(['user', 'account', 'edition']) === 'ee', + userEmail: state.getIn(['user', 'account', 'email']), + userName: state.getIn(['user', 'account', 'name']), + }; + }, + { toggleFullscreen, closeBottomBlock } + )(LivePlayer) + ) +); diff --git a/frontend/app/components/Session_/Console/ConsoleRow/ConsoleRow.tsx b/frontend/app/components/Session_/Console/ConsoleRow/ConsoleRow.tsx new file mode 100644 index 000000000..85457d6b1 --- /dev/null +++ b/frontend/app/components/Session_/Console/ConsoleRow/ConsoleRow.tsx @@ -0,0 +1,51 @@ +import React, { useState } from 'react'; +import cn from 'classnames'; +import stl from '../console.module.css'; +import { Icon } from 'UI'; +import JumpButton from 'Shared/DevTools/JumpButton'; + +interface Props { + log: any; + iconProps: any; + jump?: any; + renderWithNL?: any; + style?: any; +} +function ConsoleRow(props: Props) { + const { log, iconProps, jump, renderWithNL, style } = props; + const [expanded, setExpanded] = useState(false); + const lines = log.value.split('\n').filter((l: any) => !!l); + const canExpand = lines.length > 1; + + return ( +
setExpanded(!expanded)} + > +
+ +
+ {/*
+ {Duration.fromMillis(log.time).toFormat('mm:ss.SSS')} +
*/} +
+
+ {canExpand && ( + + )} + {renderWithNL(lines.pop())} +
+ {canExpand && expanded && lines.map((l: any, i: number) =>
{l}
)} +
+ jump(log.time)} /> +
+ ); +} + +export default ConsoleRow; diff --git a/frontend/app/components/Session_/OverviewPanel/OverviewPanel.tsx b/frontend/app/components/Session_/OverviewPanel/OverviewPanel.tsx index 667f37d5b..0b542dce6 100644 --- a/frontend/app/components/Session_/OverviewPanel/OverviewPanel.tsx +++ b/frontend/app/components/Session_/OverviewPanel/OverviewPanel.tsx @@ -17,6 +17,7 @@ import { PlayerContext } from 'App/components/Session/playerContext'; function OverviewPanel({ issuesList }: { issuesList: any[] }) { const { store } = React.useContext(PlayerContext) +function OverviewPanel() { const [dataLoaded, setDataLoaded] = React.useState(false); const [selectedFeatures, setSelectedFeatures] = React.useState([ 'PERFORMANCE', @@ -35,6 +36,8 @@ function OverviewPanel({ issuesList }: { issuesList: any[] }) { graphqlList, } = store.get() + const fetchPresented = fetchList.length > 0; + const resourceList = resourceListUnmap .filter((r: any) => r.isRed() || r.isYellow()) .concat(fetchList.filter((i: any) => parseInt(i.status) >= 400)) @@ -84,7 +87,11 @@ function OverviewPanel({ issuesList }: { issuesList: any[] }) { -
+
( - + )} endTime={endTime} message={HELP_MESSAGE[feature]} diff --git a/frontend/app/components/Session_/OverviewPanel/components/EventRow/EventRow.tsx b/frontend/app/components/Session_/OverviewPanel/components/EventRow/EventRow.tsx index c5545fbb6..cbbdbbddc 100644 --- a/frontend/app/components/Session_/OverviewPanel/components/EventRow/EventRow.tsx +++ b/frontend/app/components/Session_/OverviewPanel/components/EventRow/EventRow.tsx @@ -37,7 +37,7 @@ const EventRow = React.memo((props: Props) => {
{ > {title}
- {message ? : null} + {message ? : null}
{isGraph ? ( @@ -78,9 +78,9 @@ const EventRow = React.memo((props: Props) => { export default EventRow; -function RowInfo({ message, zIndex }: any) { +function RowInfo({ message }: any) { return ( - + ); diff --git a/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx b/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx index c91e2b362..bf4599e10 100644 --- a/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx +++ b/frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx @@ -30,7 +30,7 @@ function FeatureSelection(props: Props) { const checked = list.includes(feature); const _disabled = disabled && !checked; return ( - + { const { player } = React.useContext(PlayerContext) @@ -37,7 +38,7 @@ const TimelinePointer = React.memo((props: Props) => { if (pointer.tp === 'graph_ql') { showModal(, { right: true }); } else { - showModal(, { right: true }); + showModal(, { right: true }); } } // props.toggleBottomBlock(type); @@ -49,7 +50,7 @@ const TimelinePointer = React.memo((props: Props) => { - {item.success ? 'Slow resource: ' : 'Missing resource:'} + {item.success ? 'Slow resource: ' : '4xx/5xx Error:'}
{name.length > 200 ? name.slice(0, 100) + ' ... ' + name.slice(-50) diff --git a/frontend/app/components/Session_/Player/Controls/Controls.js b/frontend/app/components/Session_/Player/Controls/Controls.js new file mode 100644 index 000000000..005403731 --- /dev/null +++ b/frontend/app/components/Session_/Player/Controls/Controls.js @@ -0,0 +1,435 @@ +import React from 'react'; +import cn from 'classnames'; +import { connect } from 'react-redux'; +import { + connectPlayer, + STORAGE_TYPES, + selectStorageType, + selectStorageListNow, +} from 'Player'; +import LiveTag from 'Shared/LiveTag'; +import { jumpToLive } from 'Player'; + +import { Icon, Tooltip } from 'UI'; +import { toggleInspectorMode } from 'Player'; +import { + fullscreenOn, + fullscreenOff, + toggleBottomBlock, + changeSkipInterval, + OVERVIEW, + CONSOLE, + NETWORK, + STACKEVENTS, + STORAGE, + PROFILER, + PERFORMANCE, + GRAPHQL, + INSPECTOR, +} from 'Duck/components/player'; +import { AssistDuration } from './Time'; +import Timeline from './Timeline'; +import ControlButton from './ControlButton'; +import PlayerControls from './components/PlayerControls'; + +import styles from './controls.module.css'; +import XRayButton from 'Shared/XRayButton'; + +const SKIP_INTERVALS = { + 2: 2e3, + 5: 5e3, + 10: 1e4, + 15: 15e3, + 20: 2e4, + 30: 3e4, + 60: 6e4, +}; + +function getStorageName(type) { + switch (type) { + case STORAGE_TYPES.REDUX: + return 'REDUX'; + case STORAGE_TYPES.MOBX: + return 'MOBX'; + case STORAGE_TYPES.VUEX: + return 'VUEX'; + case STORAGE_TYPES.NGRX: + return 'NGRX'; + case STORAGE_TYPES.ZUSTAND: + return 'ZUSTAND'; + case STORAGE_TYPES.NONE: + return 'STATE'; + } +} + +@connectPlayer((state) => ({ + time: state.time, + endTime: state.endTime, + live: state.live, + livePlay: state.livePlay, + playing: state.playing, + completed: state.completed, + skip: state.skip, + skipToIssue: state.skipToIssue, + speed: state.speed, + disabled: state.cssLoading || state.messagesLoading || state.inspectorMode || state.markedTargets, + inspectorMode: state.inspectorMode, + fullscreenDisabled: state.messagesLoading, + // logCount: state.logList.length, + logRedCount: state.logMarkedCount, + showExceptions: state.exceptionsList.length > 0, + resourceRedCount: state.resourceMarkedCount, + fetchRedCount: state.fetchMarkedCount, + showStack: state.stackList.length > 0, + stackCount: state.stackList.length, + stackRedCount: state.stackMarkedCount, + profilesCount: state.profilesList.length, + storageCount: selectStorageListNow(state).length, + storageType: selectStorageType(state), + showStorage: selectStorageType(state) !== STORAGE_TYPES.NONE, + showProfiler: state.profilesList.length > 0, + showGraphql: state.graphqlList.length > 0, + showFetch: state.fetchCount > 0, + fetchCount: state.fetchCount, + graphqlCount: state.graphqlList.length, + liveTimeTravel: state.liveTimeTravel, +})) +@connect( + (state, props) => { + const permissions = state.getIn(['user', 'account', 'permissions']) || []; + const isEnterprise = state.getIn(['user', 'account', 'edition']) === 'ee'; + return { + disabled: props.disabled || (isEnterprise && !permissions.includes('DEV_TOOLS')), + fullscreen: state.getIn(['components', 'player', 'fullscreen']), + bottomBlock: state.getIn(['components', 'player', 'bottomBlock']), + showStorage: + props.showStorage || !state.getIn(['components', 'player', 'hiddenHints', 'storage']), + showStack: props.showStack || !state.getIn(['components', 'player', 'hiddenHints', 'stack']), + closedLive: + !!state.getIn(['sessions', 'errors']) || !state.getIn(['sessions', 'current', 'live']), + skipInterval: state.getIn(['components', 'player', 'skipInterval']), + }; + }, + { + fullscreenOn, + fullscreenOff, + toggleBottomBlock, + changeSkipInterval, + } +) +export default class Controls extends React.Component { + componentDidMount() { + document.addEventListener('keydown', this.onKeyDown); + } + + componentWillUnmount() { + document.removeEventListener('keydown', this.onKeyDown); + //this.props.toggleInspectorMode(false); + } + + shouldComponentUpdate(nextProps) { + if ( + nextProps.fullscreen !== this.props.fullscreen || + nextProps.bottomBlock !== this.props.bottomBlock || + nextProps.live !== this.props.live || + nextProps.livePlay !== this.props.livePlay || + nextProps.playing !== this.props.playing || + nextProps.completed !== this.props.completed || + nextProps.skip !== this.props.skip || + nextProps.skipToIssue !== this.props.skipToIssue || + nextProps.speed !== this.props.speed || + nextProps.disabled !== this.props.disabled || + nextProps.fullscreenDisabled !== this.props.fullscreenDisabled || + // nextProps.inspectorMode !== this.props.inspectorMode || + // nextProps.logCount !== this.props.logCount || + nextProps.logRedCount !== this.props.logRedCount || + nextProps.showExceptions !== this.props.showExceptions || + nextProps.resourceRedCount !== this.props.resourceRedCount || + nextProps.fetchRedCount !== this.props.fetchRedCount || + nextProps.showStack !== this.props.showStack || + nextProps.stackCount !== this.props.stackCount || + nextProps.stackRedCount !== this.props.stackRedCount || + nextProps.profilesCount !== this.props.profilesCount || + nextProps.storageCount !== this.props.storageCount || + nextProps.storageType !== this.props.storageType || + nextProps.showStorage !== this.props.showStorage || + nextProps.showProfiler !== this.props.showProfiler || + nextProps.showGraphql !== this.props.showGraphql || + nextProps.showFetch !== this.props.showFetch || + nextProps.fetchCount !== this.props.fetchCount || + nextProps.graphqlCount !== this.props.graphqlCount || + nextProps.liveTimeTravel !== this.props.liveTimeTravel || + nextProps.skipInterval !== this.props.skipInterval + ) + return true; + return false; + } + + onKeyDown = (e) => { + if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) { + return; + } + if (this.props.inspectorMode) { + if (e.key === 'Esc' || e.key === 'Escape') { + toggleInspectorMode(false); + } + } + // if (e.key === ' ') { + // document.activeElement.blur(); + // this.props.togglePlay(); + // } + if (e.key === 'Esc' || e.key === 'Escape') { + this.props.fullscreenOff(); + } + if (e.key === 'ArrowRight') { + this.forthTenSeconds(); + } + if (e.key === 'ArrowLeft') { + this.backTenSeconds(); + } + if (e.key === 'ArrowDown') { + this.props.speedDown(); + } + if (e.key === 'ArrowUp') { + this.props.speedUp(); + } + }; + + forthTenSeconds = () => { + const { time, endTime, jump, skipInterval } = this.props; + jump(Math.min(endTime, time + SKIP_INTERVALS[skipInterval])); + }; + + backTenSeconds = () => { + //shouldComponentUpdate + const { time, jump, skipInterval } = this.props; + jump(Math.max(1, time - SKIP_INTERVALS[skipInterval])); + }; + + goLive = () => this.props.jump(this.props.endTime); + + renderPlayBtn = () => { + const { completed, playing } = this.props; + let label; + let icon; + if (completed) { + icon = 'arrow-clockwise'; + label = 'Replay this session'; + } else if (playing) { + icon = 'pause-fill'; + label = 'Pause'; + } else { + icon = 'play-fill-new'; + label = 'Pause'; + label = 'Play'; + } + + return ( + +
+ +
+
+ ); + }; + + controlIcon = (icon, size, action, isBackwards, additionalClasses) => ( +
+ +
+ ); + + render() { + const { + bottomBlock, + toggleBottomBlock, + live, + livePlay, + skip, + speed, + disabled, + logRedCount, + showExceptions, + resourceRedCount, + fetchRedCount, + showStack, + stackRedCount, + showStorage, + storageType, + showProfiler, + showGraphql, + fullscreen, + inspectorMode, + closedLive, + toggleSpeed, + toggleSkip, + liveTimeTravel, + changeSkipInterval, + skipInterval, + } = this.props; + + const toggleBottomTools = (blockName) => { + if (blockName === INSPECTOR) { + toggleInspectorMode(); + bottomBlock && toggleBottomBlock(); + } else { + toggleInspectorMode(false); + toggleBottomBlock(blockName); + } + }; + + return ( +
+ + {!fullscreen && ( +
+
+ {!live && ( + <> + +
+ toggleBottomTools(OVERVIEW)} + /> + + )} + + {live && !closedLive && ( +
+ (livePlay ? null : jumpToLive())} /> +
+ +
+
+ )} +
+ +
+ toggleBottomTools(CONSOLE)} + active={bottomBlock === CONSOLE && !inspectorMode} + label="CONSOLE" + noIcon + labelClassName="!text-base font-semibold" + hasErrors={logRedCount > 0 || showExceptions} + containerClassName="mx-2" + /> + {!live && ( + toggleBottomTools(NETWORK)} + active={bottomBlock === NETWORK && !inspectorMode} + label="NETWORK" + hasErrors={resourceRedCount > 0 || fetchRedCount > 0} + noIcon + labelClassName="!text-base font-semibold" + containerClassName="mx-2" + /> + )} + {!live && ( + toggleBottomTools(PERFORMANCE)} + active={bottomBlock === PERFORMANCE && !inspectorMode} + label="PERFORMANCE" + noIcon + labelClassName="!text-base font-semibold" + containerClassName="mx-2" + /> + )} + {!live && showGraphql && ( + toggleBottomTools(GRAPHQL)} + active={bottomBlock === GRAPHQL && !inspectorMode} + label="GRAPHQL" + noIcon + labelClassName="!text-base font-semibold" + containerClassName="mx-2" + /> + )} + {!live && showStorage && ( + toggleBottomTools(STORAGE)} + active={bottomBlock === STORAGE && !inspectorMode} + label={getStorageName(storageType)} + noIcon + labelClassName="!text-base font-semibold" + containerClassName="mx-2" + /> + )} + {!live && ( + toggleBottomTools(STACKEVENTS)} + active={bottomBlock === STACKEVENTS && !inspectorMode} + label="EVENTS" + noIcon + labelClassName="!text-base font-semibold" + containerClassName="mx-2" + hasErrors={stackRedCount > 0} + /> + )} + {!live && showProfiler && ( + toggleBottomTools(PROFILER)} + active={bottomBlock === PROFILER && !inspectorMode} + label="PROFILER" + noIcon + labelClassName="!text-base font-semibold" + containerClassName="mx-2" + /> + )} + {!live && ( + + {this.controlIcon( + 'arrows-angle-extend', + 16, + this.props.fullscreenOn, + false, + 'rounded hover:bg-gray-light-shade color-gray-medium' + )} + + )} +
+
+ )} +
+ ); + } +} diff --git a/frontend/app/components/Session_/Player/Controls/components/PlayerControls.tsx b/frontend/app/components/Session_/Player/Controls/components/PlayerControls.tsx index a5fda31fb..d152fad0f 100644 --- a/frontend/app/components/Session_/Player/Controls/components/PlayerControls.tsx +++ b/frontend/app/components/Session_/Player/Controls/components/PlayerControls.tsx @@ -94,11 +94,11 @@ function PlayerControls(props: Props) {
{/* @ts-ignore */} - - - -
- - ( -
-
- Jump (Secs) -
- {Object.keys(skipIntervals).map((interval) => ( -
{ - toggleTooltip(); - setSkipInterval(parseInt(interval, 10)); - }} - className={cn( - 'py-2 px-4 cursor-pointer w-full text-left font-semibold', - 'hover:bg-active-blue border-t border-borderColor-gray-light-shade' - )} - > - {interval} - s -
- ))} + + + +
+ ( +
+
+ Jump (Secs)
- )} - > -
- {/* @ts-ignore */} - - {currentInterval}s - + {Object.keys(skipIntervals).map((interval) => ( +
{ + close(); + setSkipInterval(parseInt(interval, 10)); + }} + className={cn( + 'py-2 px-4 cursor-pointer w-full text-left font-semibold', + 'hover:bg-active-blue border-t border-borderColor-gray-light-shade' + )} + > + {interval} + s +
+ ))}
- - -
- {/* @ts-ignore */} - -
+ + - + +
{!live && ( diff --git a/frontend/app/components/Session_/Player/Player.js b/frontend/app/components/Session_/Player/Player.js index a37fca62b..9312589f8 100644 --- a/frontend/app/components/Session_/Player/Player.js +++ b/frontend/app/components/Session_/Player/Player.js @@ -35,6 +35,7 @@ import OverviewPanel from '../OverviewPanel'; import ConsolePanel from 'Shared/DevTools/ConsolePanel'; import ProfilerPanel from 'Shared/DevTools/ProfilerPanel'; import { PlayerContext } from 'App/components/Session/playerContext'; +import StackEventPanel from 'Shared/DevTools/StackEventPanel'; function Player(props) { const { @@ -84,7 +85,8 @@ function Player(props) { // )} - {bottomBlock === STACKEVENTS && } + {/* {bottomBlock === STACKEVENTS && } */} + {bottomBlock === STACKEVENTS && } {bottomBlock === STORAGE && } {bottomBlock === PROFILER && } {bottomBlock === PERFORMANCE && } @@ -94,11 +96,11 @@ function Player(props) { {bottomBlock === INSPECTOR && }
)} - + />}
) } diff --git a/frontend/app/components/Session_/PlayerBlock.js b/frontend/app/components/Session_/PlayerBlock.js index 68d2c51c8..54130adf5 100644 --- a/frontend/app/components/Session_/PlayerBlock.js +++ b/frontend/app/components/Session_/PlayerBlock.js @@ -1,46 +1,36 @@ import React from 'react'; -import cn from "classnames"; +import cn from 'classnames'; import { connect } from 'react-redux'; -import { - NONE, -} from 'Duck/components/player'; +import { NONE } from 'Duck/components/player'; import Player from './Player'; import SubHeader from './Subheader'; import styles from './playerBlock.module.css'; -@connect(state => ({ - fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]), - bottomBlock: state.getIn([ 'components', 'player', 'bottomBlock' ]), - sessionId: state.getIn([ 'sessions', 'current', 'sessionId' ]), - disabled: state.getIn([ 'components', 'targetDefiner', 'inspectorMode' ]), - jiraConfig: state.getIn([ 'issues', 'list' ]).first(), +@connect((state) => ({ + fullscreen: state.getIn(['components', 'player', 'fullscreen']), + bottomBlock: state.getIn(['components', 'player', 'bottomBlock']), + sessionId: state.getIn(['sessions', 'current', 'sessionId']), + disabled: state.getIn(['components', 'targetDefiner', 'inspectorMode']), + jiraConfig: state.getIn(['issues', 'list']).first(), })) export default class PlayerBlock extends React.PureComponent { render() { - const { - fullscreen, - bottomBlock, - sessionId, - disabled, - activeTab, - jiraConfig, - } = this.props; + const { fullscreen, bottomBlock, sessionId, disabled, activeTab, jiraConfig, fullView = false } = this.props; return ( -
- {!fullscreen && } +
+ {!fullscreen && !fullView && ( + + )}
); diff --git a/frontend/app/components/Session_/Storage/DiffRow.tsx b/frontend/app/components/Session_/Storage/DiffRow.tsx index 21eca8adb..73e7c7d37 100644 --- a/frontend/app/components/Session_/Storage/DiffRow.tsx +++ b/frontend/app/components/Session_/Storage/DiffRow.tsx @@ -50,31 +50,41 @@ function DiffRow({ diff, path }: Props) { : newValue; return ( -
+
20 ? 'cursor-pointer' : ''} onClick={() => setShorten(!shorten)}> {pathStr} {': '} - setShortenOldVal(!shortenOldVal)} className={cn( - 'line-through text-disabled-text', + 'text-disabled-text', diffLengths[0] > 50 ? 'cursor-pointer' : '' )} > - {oldValueSafe || 'undefined'} - + {oldValueSafe || 'undefined'} + {diffLengths[0] > 50 + ? ( +
setShortenOldVal(!shortenOldVal)} className="cursor-pointer px-1 text-white bg-gray-light rounded text-sm w-fit"> + {!shortenOldVal ? 'collapse' : 'expand'} +
+ ) : null} +
{' -> '} - setShortenNewVal(!shortenNewVal)} +
50 ? 'cursor-pointer' : '' )} > {newValueSafe || 'undefined'} - + {diffLengths[1] > 50 + ? ( +
setShortenNewVal(!shortenNewVal)} className="cursor-pointer px-1 text-white bg-gray-light rounded text-sm w-fit"> + {!shortenNewVal ? 'collapse' : 'expand'} +
+ ) : null} +
); } diff --git a/frontend/app/components/Session_/Storage/Storage.js b/frontend/app/components/Session_/Storage/Storage.js new file mode 100644 index 000000000..e0fbb73f6 --- /dev/null +++ b/frontend/app/components/Session_/Storage/Storage.js @@ -0,0 +1,368 @@ +import React from 'react'; +import { connect } from 'react-redux'; +import { hideHint } from 'Duck/components/player'; +import { + connectPlayer, + selectStorageType, + STORAGE_TYPES, + selectStorageListNow, + selectStorageList, +} from 'Player'; +import { JSONTree, NoContent, Tooltip } from 'UI'; +import { formatMs } from 'App/date'; +import { diff } from 'deep-diff'; +import { jump } from 'Player'; +import BottomBlock from '../BottomBlock/index'; +import DiffRow from './DiffRow'; +import stl from './storage.module.css'; +import { List, CellMeasurer, CellMeasurerCache, AutoSizer } from 'react-virtualized'; + +const ROW_HEIGHT = 90; + +function getActionsName(type) { + switch (type) { + case STORAGE_TYPES.MOBX: + case STORAGE_TYPES.VUEX: + return 'MUTATIONS'; + default: + return 'ACTIONS'; + } +} + +@connectPlayer((state) => ({ + type: selectStorageType(state), + list: selectStorageList(state), + listNow: selectStorageListNow(state), +})) +@connect( + (state) => ({ + hintIsHidden: state.getIn(['components', 'player', 'hiddenHints', 'storage']), + }), + { + hideHint, + } +) +export default class Storage extends React.PureComponent { + constructor(props) { + super(props); + + this.lastBtnRef = React.createRef(); + this._list = React.createRef(); + this.cache = new CellMeasurerCache({ + fixedWidth: true, + keyMapper: (index) => this.props.listNow[index], + }); + this._rowRenderer = this._rowRenderer.bind(this); + } + + focusNextButton() { + if (this.lastBtnRef.current) { + this.lastBtnRef.current.focus(); + } + } + + componentDidMount() { + this.focusNextButton(); + } + + componentDidUpdate(prevProps, prevState) { + if (prevProps.listNow.length !== this.props.listNow.length) { + this.focusNextButton(); + /** possible performance gain, but does not work with dynamic list insertion for some reason + * getting NaN offsets, maybe I detect changed rows wrongly + */ + // const newRows = this.props.listNow.filter(evt => prevProps.listNow.indexOf(evt._index) < 0); + // if (newRows.length > 0) { + // const newRowsIndexes = newRows.map(r => this.props.listNow.indexOf(r)) + // newRowsIndexes.forEach(ind => this.cache.clear(ind)) + // this._list.recomputeRowHeights(newRowsIndexes) + // } + } + } + + renderDiff(item, prevItem) { + if (!prevItem) { + // we don't have state before first action + return
; + } + + const stateDiff = diff(prevItem.state, item.state); + + if (!stateDiff) { + return ( +
+ No diff +
+ ); + } + + return ( +
+ {stateDiff.map((d, i) => this.renderDiffs(d, i))} +
+ ); + } + + renderDiffs(diff, i) { + const path = this.createPath(diff); + + return ( + + + + ); + } + + createPath = (diff) => { + let path = []; + + if (diff.path) { + path = path.concat(diff.path); + } + if (typeof diff.index !== 'undefined') { + path.push(diff.index); + } + + const pathStr = path.length ? path.join('.') : ''; + return pathStr; + }; + + ensureString(actionType) { + if (typeof actionType === 'string') return actionType; + return 'UNKNOWN'; + } + + goNext = () => { + const { list, listNow } = this.props; + jump(list[listNow.length].time, list[listNow.length]._index); + }; + + renderTab() { + const { listNow } = this.props; + if (listNow.length === 0) { + return 'Not initialized'; //? + } + return ; + } + + renderItem(item, i, prevItem, style) { + const { type } = this.props; + let src; + let name; + + switch (type) { + case STORAGE_TYPES.REDUX: + case STORAGE_TYPES.NGRX: + src = item.action; + name = src && src.type; + break; + case STORAGE_TYPES.VUEX: + src = item.mutation; + name = src && src.type; + break; + case STORAGE_TYPES.MOBX: + src = item.payload; + name = `@${item.type} ${src && src.type}`; + break; + case STORAGE_TYPES.ZUSTAND: + src = null; + name = item.mutation.join(''); + } + + return ( +
+ {src === null ? ( +
+ {name} +
+ ) : ( + <> + {this.renderDiff(item, prevItem, i)} +
+ console.log('test')} + /> +
+ + )} +
+ {typeof item.duration === 'number' && ( +
{formatMs(item.duration)}
+ )} +
+ {i + 1 < this.props.listNow.length && ( + + )} + {i + 1 === this.props.listNow.length && i + 1 < this.props.list.length && ( + + )} +
+
+
+ ); + } + + _rowRenderer({ index, parent, key, style }) { + const { listNow } = this.props; + + if (!listNow[index]) return console.warn(index, listNow); + + return ( + + {this.renderItem(listNow[index], index, index > 0 ? listNow[index - 1] : undefined, style)} + + ); + } + + render() { + const { type, list, listNow, hintIsHidden } = this.props; + + const showStore = type !== STORAGE_TYPES.MOBX; + return ( + + + {list.length > 0 && ( +
+ {showStore && ( +

+ {'STATE'} +

+ )} + {type !== STORAGE_TYPES.ZUSTAND ? ( +

+ DIFFS +

+ ) : null} +

+ {getActionsName(type)} +

+

+ TTE +

+
+ )} +
+ + + { + 'Inspect your application state while you’re replaying your users sessions. OpenReplay supports ' + } + + Redux + + {', '} + + VueX + + {', '} + + Pinia + + {', '} + + Zustand + + {', '} + + MobX + + {' and '} + + NgRx + + . +
+
+ + + ) : null + } + size="small" + show={listNow.length === 0} + > + {showStore && ( +
+ {listNow.length === 0 ? ( +
+ {'Empty state.'} +
+ ) : ( + this.renderTab() + )} +
+ )} +
+ + {({ height, width }) => ( + { + this._list = element; + }} + deferredMeasurementCache={this.cache} + overscanRowCount={1} + rowCount={Math.ceil(parseInt(this.props.listNow.length) || 1)} + rowHeight={ROW_HEIGHT} + rowRenderer={this._rowRenderer} + width={width} + height={height} + /> + )} + +
+
+
+
+ ); + } +} diff --git a/frontend/app/components/shared/DevTools/BottomBlock/BottomBlock.js b/frontend/app/components/shared/DevTools/BottomBlock/BottomBlock.js index 069757e60..8b7826755 100644 --- a/frontend/app/components/shared/DevTools/BottomBlock/BottomBlock.js +++ b/frontend/app/components/shared/DevTools/BottomBlock/BottomBlock.js @@ -1,17 +1,29 @@ -import React from 'react'; +import React, { useEffect } from 'react'; import cn from 'classnames'; import stl from './bottomBlock.module.css'; +let timer = null; const BottomBlock = ({ children = null, className = '', additionalHeight = 0, + onMouseEnter = () => {}, + onMouseLeave = () => {}, ...props -}) => ( -
- { children } -
-); +}) => { + useEffect(() => {}, []); + + return ( +
+ {children} +
+ ); +}; BottomBlock.displayName = 'BottomBlock'; diff --git a/frontend/app/components/shared/DevTools/ConsolePanel/ConsolePanel.tsx b/frontend/app/components/shared/DevTools/ConsolePanel/ConsolePanel.tsx index ae275a37a..d22ff088b 100644 --- a/frontend/app/components/shared/DevTools/ConsolePanel/ConsolePanel.tsx +++ b/frontend/app/components/shared/DevTools/ConsolePanel/ConsolePanel.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import Log from 'Types/session/log'; import BottomBlock from '../BottomBlock'; import { LEVEL } from 'Types/session/log'; @@ -8,6 +8,11 @@ import ConsoleRow from '../ConsoleRow'; import { getRE } from 'App/utils'; import { PlayerContext } from 'App/components/Session/playerContext'; import { observer } from 'mobx-react-lite'; +import { List, CellMeasurer, CellMeasurerCache, AutoSizer } from 'react-virtualized'; +import { useObserver } from 'mobx-react-lite'; +import { useStore } from 'App/mstore'; +import ErrorDetailsModal from 'App/components/Dashboard/components/Errors/ErrorDetailsModal'; +import { useModal } from 'App/components/Modal'; const ALL = 'ALL'; const INFO = 'INFO'; @@ -52,11 +57,16 @@ const getIconProps = (level: any) => { return null; }; + +const INDEX_KEY = 'console'; +let timeOut: any = null; +const TIMEOUT_DURATION = 5000; + function ConsolePanel() { const { player, store } = React.useContext(PlayerContext) const jump = (t: number) => player.jump(t) - const { logList, exceptionsList } = store.get() + const { logList, exceptionsList, time } = store.get() const logExceptions = exceptionsList.map(({ time, errorId, name, projectId }: any) => Log({ @@ -68,12 +78,106 @@ function ConsolePanel() { ); // @ts-ignore const logs = logList.concat(logExceptions) - const additionalHeight = 0; - const [activeTab, setActiveTab] = useState(ALL); - const [filter, setFilter] = useState(''); + const { + sessionStore: { devTools }, + } = useStore(); - let filtered = React.useMemo(() => { + const [isDetailsModalActive, setIsDetailsModalActive] = useState(false); + const [filteredList, setFilteredList] = useState([]); + const filter = useObserver(() => devTools[INDEX_KEY].filter); + const activeTab = useObserver(() => devTools[INDEX_KEY].activeTab); + const activeIndex = useObserver(() => devTools[INDEX_KEY].index); + const [pauseSync, setPauseSync] = useState(activeIndex > 0); + const synRef: any = useRef({}); + const { showModal } = useModal(); + + const onTabClick = (activeTab: any) => devTools.update(INDEX_KEY, { activeTab }); + const onFilterChange = ({ target: { value } }: any) => { + devTools.update(INDEX_KEY, { filter: value }); + }; + + synRef.current = { + pauseSync, + activeIndex, + }; + + const removePause = () => { + setIsDetailsModalActive(false); + clearTimeout(timeOut); + timeOut = setTimeout(() => { + devTools.update(INDEX_KEY, { index: getCurrentIndex() }); + setPauseSync(false); + }, TIMEOUT_DURATION); + }; + + const onMouseLeave = () => { + if (isDetailsModalActive) return; + removePause(); + }; + + useEffect(() => { + if (pauseSync) { + removePause(); + } + + return () => { + clearTimeout(timeOut); + if (!synRef.current.pauseSync) { + devTools.update(INDEX_KEY, { index: 0 }); + } + }; + }, []); + + const getCurrentIndex = () => { + return filteredList.filter((item: any) => item.time <= time).length - 1; + }; + + useEffect(() => { + const currentIndex = getCurrentIndex(); + if (currentIndex !== activeIndex && !pauseSync) { + devTools.update(INDEX_KEY, { index: currentIndex }); + } + }, [time]); + + const cache = new CellMeasurerCache({ + fixedWidth: true, + keyMapper: (index: number) => filteredList[index], + }); + const _list = React.useRef(); + + const showDetails = (log: any) => { + setIsDetailsModalActive(true); + showModal(, { right: true, onClose: removePause }); + devTools.update(INDEX_KEY, { index: filteredList.indexOf(log) }); + setPauseSync(true); + }; + + const _rowRenderer = ({ index, key, parent, style }: any) => { + const item = filteredList[index]; + + return ( + // @ts-ignore + + {({ measure }: any) => ( + showDetails(item)} + recalcHeight={() => { + measure(); + (_list as any).current.recomputeRowHeights(index); + }} + /> + )} + + ); + }; + + React.useMemo(() => { const filterRE = getRE(filter, 'i'); let list = logs; @@ -82,14 +186,23 @@ function ConsolePanel() { (!!filter ? filterRE.test(value) : true) && (activeTab === ALL || activeTab === LEVEL_TAB[level]) ); - return list; - }, [filter, activeTab]); + setFilteredList(list); + }, [logs, filter, activeTab]); - const onTabClick = (activeTab: any) => setActiveTab(activeTab); - const onFilterChange = ({ target: { value } }: any) => setFilter(value); + useEffect(() => { + if (_list.current) { + // @ts-ignore + _list.current.scrollToRow(activeIndex); + } + }, [activeIndex]); return ( - + setPauseSync(true)} + onMouseLeave={onMouseLeave} + > + {/* @ts-ignore */}
Console @@ -103,8 +216,11 @@ function ConsolePanel() { name="filter" height={28} onChange={onFilterChange} + value={filter} /> + {/* @ts-ignore */} + {/* @ts-ignore */} } size="small" - show={filtered.length === 0} + show={filteredList.length === 0} > - {/* */} - {filtered.map((l: any, index: any) => ( - - ))} - {/* */} + {/* @ts-ignore */} + + {({ height, width }: any) => ( + // @ts-ignore + + )} + + {/* @ts-ignore */} ); diff --git a/frontend/app/components/shared/DevTools/ConsoleRow/ConsoleRow.tsx b/frontend/app/components/shared/DevTools/ConsoleRow/ConsoleRow.tsx index 145f14640..83929cbed 100644 --- a/frontend/app/components/shared/DevTools/ConsoleRow/ConsoleRow.tsx +++ b/frontend/app/components/shared/DevTools/ConsoleRow/ConsoleRow.tsx @@ -1,43 +1,43 @@ import React, { useState } from 'react'; import cn from 'classnames'; -// import stl from '../console.module.css'; import { Icon } from 'UI'; import JumpButton from 'Shared/DevTools/JumpButton'; -import { useModal } from 'App/components/Modal'; -import ErrorDetailsModal from 'App/components/Dashboard/components/Errors/ErrorDetailsModal'; interface Props { log: any; iconProps: any; jump?: any; renderWithNL?: any; + style?: any; + recalcHeight?: () => void; + onClick: () => void; } function ConsoleRow(props: Props) { - const { log, iconProps, jump, renderWithNL } = props; - const { showModal } = useModal(); + const { log, iconProps, jump, renderWithNL, style, recalcHeight } = props; const [expanded, setExpanded] = useState(false); const lines = log.value.split('\n').filter((l: any) => !!l); const canExpand = lines.length > 1; const clickable = canExpand || !!log.errorId; - const onErrorClick = () => { - showModal(, { right: true }); + const toggleExpand = () => { + setExpanded(!expanded); + setTimeout(() => recalcHeight(), 0); }; return (
(!!log.errorId ? onErrorClick() : setExpanded(!expanded)) : () => {} - } + onClick={clickable ? () => (!!log.errorId ? props.onClick() : toggleExpand()) : () => {}} >
@@ -49,7 +49,13 @@ function ConsoleRow(props: Props) { )} {renderWithNL(lines.pop())}
- {canExpand && expanded && lines.map((l: any) =>
{l}
)} + {canExpand && + expanded && + lines.map((l: string, i: number) => ( +
+ {l} +
+ ))}
jump(log.time)} />
diff --git a/frontend/app/components/shared/DevTools/JumpButton/JumpButton.tsx b/frontend/app/components/shared/DevTools/JumpButton/JumpButton.tsx index c52b0cffd..31307fd9b 100644 --- a/frontend/app/components/shared/DevTools/JumpButton/JumpButton.tsx +++ b/frontend/app/components/shared/DevTools/JumpButton/JumpButton.tsx @@ -6,10 +6,10 @@ interface Props { tooltip?: string; } function JumpButton(props: Props) { - const { tooltip = '' } = props; + const { tooltip } = props; return (
- +
{ diff --git a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx index 444d448fc..5301066cb 100644 --- a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx +++ b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; import { Tooltip, Tabs, Input, NoContent, Icon, Toggler } from 'UI'; import { getRE } from 'App/utils'; import Resource, { TYPES } from 'Types/session/resource'; @@ -13,6 +13,9 @@ import { useModal } from 'App/components/Modal'; import FetchDetailsModal from 'Shared/FetchDetailsModal'; import { PlayerContext } from 'App/components/Session/playerContext'; import { observer } from 'mobx-react-lite'; +import { useStore } from 'App/mstore'; + +const INDEX_KEY = 'network'; const ALL = 'ALL'; const XHR = 'xhr'; @@ -72,7 +75,7 @@ function renderSize(r: any) { if (r.responseBodySize) return formatBytes(r.responseBodySize); let triggerText; let content; - if (r.decodedBodySize == null) { + if (r.decodedBodySize == null || r.decodedBodySize === 0) { triggerText = 'x'; content = 'Not captured'; } else { @@ -120,6 +123,19 @@ export function renderDuration(r: any) { ); } +interface Props { + location: any; + resources: any; + fetchList: any; + domContentLoadedTime: any; + loadTime: any; + playing: boolean; + domBuildingTime: any; + time: any; +} +let timeOut: any = null; +const TIMEOUT_DURATION = 5000; + function NetworkPanel() { const { player, store } = React.useContext(PlayerContext) @@ -127,48 +143,104 @@ function NetworkPanel() { resourceList: resources, domContentLoadedTime, loadTime, + time, domBuildingTime, fetchList: fetchUnmap, } = store.get(); - const fetchList = fetchUnmap.map((i: any) => Resource({ ...i.toJS(), type: TYPES.XHR })) + const fetchList = fetchUnmap.map((i: any) => Resource({ ...i.toJS(), type: TYPES.XHR, time: i.time < 0 ? 0 : i.time })) - const { showModal, hideModal } = useModal(); - const [activeTab, setActiveTab] = useState(ALL); + const { showModal } = useModal(); const [sortBy, setSortBy] = useState('time'); const [sortAscending, setSortAscending] = useState(true); - const [filter, setFilter] = useState(''); const [showOnlyErrors, setShowOnlyErrors] = useState(false); - const onTabClick = (activeTab: any) => setActiveTab(activeTab); - const onFilterChange = ({ target: { value } }: any) => setFilter(value); + + const [filteredList, setFilteredList] = useState([]); + const [isDetailsModalActive, setIsDetailsModalActive] = useState(false); const additionalHeight = 0; const fetchPresented = fetchList.length > 0; + const { + sessionStore: { devTools }, + } = useStore(); + const filter = devTools[INDEX_KEY].filter; + const activeTab = devTools[INDEX_KEY].activeTab; + const activeIndex = devTools[INDEX_KEY].index; + const [pauseSync, setPauseSync] = useState(activeIndex > 0); + const synRef: any = useRef({}); - const resourcesSize = resources.reduce( - (sum: any, { decodedBodySize }: any) => sum + (decodedBodySize || 0), - 0 - ); + const onTabClick = (activeTab: any) => devTools.update(INDEX_KEY, { activeTab }); + const onFilterChange = ({ target: { value } }: any) => { + devTools.update(INDEX_KEY, { filter: value }); + }; - const transferredSize = resources.reduce( - (sum: any, { headerSize, encodedBodySize }: any) => - sum + (headerSize || 0) + (encodedBodySize || 0), - 0 - ); + synRef.current = { + pauseSync, + activeIndex, + }; - const filterRE = getRE(filter, 'i'); - let filtered = React.useMemo(() => { + const removePause = () => { + setIsDetailsModalActive(false); + clearTimeout(timeOut); + timeOut = setTimeout(() => { + devTools.update(INDEX_KEY, { index: getCurrentIndex() }); + setPauseSync(false); + }, TIMEOUT_DURATION); + }; + + const onMouseLeave = () => { + if (isDetailsModalActive) return; + removePause(); + }; + + useEffect(() => { + if (pauseSync) { + removePause(); + } + + return () => { + clearTimeout(timeOut); + if (!synRef.current.pauseSync) { + devTools.update(INDEX_KEY, { index: 0 }); + } + }; + }, []); + + const getCurrentIndex = () => { + return filteredList.filter((item: any) => item.time <= time).length - 1; + }; + + useEffect(() => { + const currentIndex = getCurrentIndex(); + if (currentIndex !== activeIndex && !pauseSync) { + devTools.update(INDEX_KEY, { index: currentIndex }); + } + }, [time]); + + const { resourcesSize, transferredSize } = useMemo(() => { + const resourcesSize = resources.reduce( + (sum: any, { decodedBodySize }: any) => sum + (decodedBodySize || 0), + 0 + ); + + const transferredSize = resources.reduce( + (sum: any, { headerSize, encodedBodySize }: any) => + sum + (headerSize || 0) + (encodedBodySize || 0), + 0 + ); + return { + resourcesSize, + transferredSize, + }; + }, [resources]); + + useEffect(() => { + const filterRE = getRE(filter, 'i'); let list = resources; fetchList.forEach( (fetchCall: any) => (list = list.filter((networkCall: any) => networkCall.url !== fetchCall.url)) ); + // @ts-ignore list = list.concat(fetchList); - list = list.sort((a: any, b: any) => { - return compare(a, b, sortBy); - }); - - if (!sortAscending) { - list = list.reverse(); - } list = list.filter( ({ type, name, status, success }: any) => @@ -176,41 +248,53 @@ function NetworkPanel() { (activeTab === ALL || type === TAB_TO_TYPE_MAP[activeTab]) && (showOnlyErrors ? parseInt(status) >= 400 || !success : true) ); - return list; - }, [filter, sortBy, sortAscending, showOnlyErrors, activeTab]); + setFilteredList(list); + }, [resources, filter, showOnlyErrors, activeTab]); - // const lastIndex = currentIndex || filtered.filter((item: any) => item.time <= time).length - 1; - const referenceLines = []; - if (domContentLoadedTime != null) { - referenceLines.push({ - time: domContentLoadedTime.time, - color: DOM_LOADED_TIME_COLOR, - }); - } - if (loadTime != null) { - referenceLines.push({ - time: loadTime.time, - color: LOAD_TIME_COLOR, - }); - } + const referenceLines = useMemo(() => { + const arr = []; - const onRowClick = (row: any) => { - showModal(, { - right: true, - }); - }; - - const handleSort = (sortKey: string) => { - if (sortKey === sortBy) { - setSortAscending(!sortAscending); - // setSortBy('time'); + if (domContentLoadedTime != null) { + arr.push({ + time: domContentLoadedTime.time, + color: DOM_LOADED_TIME_COLOR, + }); } - setSortBy(sortKey); + if (loadTime != null) { + arr.push({ + time: loadTime.time, + color: LOAD_TIME_COLOR, + }); + } + + return arr; + }, []); + + const showDetailsModal = (row: any) => { + setIsDetailsModalActive(true); + showModal( + , + { + right: true, + onClose: removePause, + } + ); + devTools.update(INDEX_KEY, { index: filteredList.indexOf(row) }); + setPauseSync(true); }; + useEffect(() => { + devTools.update(INDEX_KEY, { filter, activeTab }); + }, [filter, activeTab]); + return ( - + setPauseSync(true)} + onMouseLeave={onMouseLeave} + >
Network @@ -231,6 +315,7 @@ function NetworkPanel() { onChange={onFilterChange} height={28} width={230} + value={filter} /> @@ -244,7 +329,7 @@ function NetworkPanel() { />
- + } size="small" - show={filtered.length === 0} + show={filteredList.length === 0} > player.jump(t)} sortBy={sortBy} sortAscending={sortAscending} - // activeIndex={lastIndex} + onJump={(row: any) => { + setPauseSync(true); + devTools.update(INDEX_KEY, { index: filteredList.indexOf(row) }); + player.jump(row.time); + }} + activeIndex={activeIndex} > {[ // { @@ -305,28 +394,24 @@ function NetworkPanel() { label: 'Status', dataKey: 'status', width: 70, - onClick: handleSort, }, { label: 'Type', dataKey: 'type', width: 90, render: renderType, - onClick: handleSort, }, { label: 'Name', width: 240, dataKey: 'name', render: renderName, - onClick: handleSort, }, { label: 'Size', width: 80, dataKey: 'decodedBodySize', render: renderSize, - onClick: handleSort, hidden: activeTab === XHR, }, { @@ -334,7 +419,6 @@ function NetworkPanel() { width: 80, dataKey: 'duration', render: renderDuration, - onClick: handleSort, }, ]} diff --git a/frontend/app/components/shared/DevTools/StackEventPanel/StackEventPanel.tsx b/frontend/app/components/shared/DevTools/StackEventPanel/StackEventPanel.tsx new file mode 100644 index 000000000..557c72172 --- /dev/null +++ b/frontend/app/components/shared/DevTools/StackEventPanel/StackEventPanel.tsx @@ -0,0 +1,203 @@ +import React, { useEffect, useMemo, useRef, useState } from 'react'; +import { hideHint } from 'Duck/components/player'; +import { Tooltip, Tabs, Input, NoContent, Icon, Toggler } from 'UI'; +import { getRE } from 'App/utils'; +import { List, CellMeasurer, CellMeasurerCache, AutoSizer } from 'react-virtualized'; + +import BottomBlock from '../BottomBlock'; +import { connectPlayer, jump } from 'Player'; +import { useModal } from 'App/components/Modal'; +import { useStore } from 'App/mstore'; +import { useObserver } from 'mobx-react-lite'; +import { DATADOG, SENTRY, STACKDRIVER, typeList } from 'Types/session/stackEvent'; +import { connect } from 'react-redux'; +import StackEventRow from 'Shared/DevTools/StackEventRow'; +import StackEventModal from '../StackEventModal'; + +let timeOut: any = null; +const TIMEOUT_DURATION = 5000; +const INDEX_KEY = 'stackEvent'; +const ALL = 'ALL'; +const TABS = [ALL, ...typeList].map((tab) => ({ text: tab, key: tab })); + +interface Props { + list: any; + hideHint: any; + time: any; +} +function StackEventPanel(props: Props) { + const { list, time } = props; + const additionalHeight = 0; + const { + sessionStore: { devTools }, + } = useStore(); + const { showModal } = useModal(); + const [isDetailsModalActive, setIsDetailsModalActive] = useState(false); + const [filteredList, setFilteredList] = useState([]); + const filter = useObserver(() => devTools[INDEX_KEY].filter); + const activeTab = useObserver(() => devTools[INDEX_KEY].activeTab); + const activeIndex = useObserver(() => devTools[INDEX_KEY].index); + const [pauseSync, setPauseSync] = useState(activeIndex > 0); + const synRef: any = useRef({}); + synRef.current = { + pauseSync, + activeIndex, + }; + const _list = React.useRef(); + + const onTabClick = (activeTab: any) => devTools.update(INDEX_KEY, { activeTab }); + const onFilterChange = ({ target: { value } }: any) => { + devTools.update(INDEX_KEY, { filter: value }); + }; + + const getCurrentIndex = () => { + return filteredList.filter((item: any) => item.time <= time).length - 1; + }; + + const removePause = () => { + clearTimeout(timeOut); + setIsDetailsModalActive(false); + timeOut = setTimeout(() => { + devTools.update(INDEX_KEY, { index: getCurrentIndex() }); + setPauseSync(false); + }, TIMEOUT_DURATION); + }; + + useEffect(() => { + const currentIndex = getCurrentIndex(); + if (currentIndex !== activeIndex && !pauseSync) { + devTools.update(INDEX_KEY, { index: currentIndex }); + } + }, [time]); + + const onMouseLeave = () => { + if (isDetailsModalActive) return; + removePause(); + }; + + React.useMemo(() => { + const filterRE = getRE(filter, 'i'); + let list = props.list; + + list = list.filter( + ({ name, source }: any) => + (!!filter ? filterRE.test(name) : true) && (activeTab === ALL || activeTab === source) + ); + + setFilteredList(list); + }, [filter, activeTab]); + + const tabs = useMemo(() => { + return TABS.filter(({ key }) => key === ALL || list.some(({ source }: any) => key === source)); + }, []); + + const cache = new CellMeasurerCache({ + fixedWidth: true, + keyMapper: (index: number) => filteredList[index], + }); + + const showDetails = (item: any) => { + setIsDetailsModalActive(true); + showModal(, { right: true, onClose: removePause }); + devTools.update(INDEX_KEY, { index: filteredList.indexOf(item) }); + setPauseSync(true); + }; + + const _rowRenderer = ({ index, key, parent, style }: any) => { + const item = filteredList[index]; + + return ( + // @ts-ignore + + {() => ( + { + setPauseSync(true); + devTools.update(INDEX_KEY, { index: filteredList.indexOf(item) }); + jump(item.time); + }} + onClick={() => showDetails(item)} + /> + )} + + ); + }; + + useEffect(() => { + if (_list.current) { + // @ts-ignore + _list.current.scrollToRow(activeIndex); + } + }, [activeIndex]); + + return ( + setPauseSync(true)} + onMouseLeave={onMouseLeave} + > + +
+ Stack Events + +
+ +
+ + + + No Data +
+ } + size="small" + show={filteredList.length === 0} + > + + {({ height, width }: any) => ( + + )} + + + + + ); +} + +export default connect( + (state: any) => ({ + hintIsHidden: + state.getIn(['components', 'player', 'hiddenHints', 'stack']) || + !state.getIn(['site', 'list']).some((s: any) => s.stackIntegrations), + }), + { hideHint } +)( + connectPlayer((state: any) => ({ + list: state.stackList, + time: state.time, + }))(StackEventPanel) +); diff --git a/frontend/app/components/shared/DevTools/StackEventPanel/index.ts b/frontend/app/components/shared/DevTools/StackEventPanel/index.ts new file mode 100644 index 000000000..bb0ca8cb6 --- /dev/null +++ b/frontend/app/components/shared/DevTools/StackEventPanel/index.ts @@ -0,0 +1 @@ +export { default } from './StackEventPanel'; diff --git a/frontend/app/components/shared/DevTools/StackEventRow/StackEventRow.tsx b/frontend/app/components/shared/DevTools/StackEventRow/StackEventRow.tsx index b6b1a8a6f..0d2eeb554 100644 --- a/frontend/app/components/shared/DevTools/StackEventRow/StackEventRow.tsx +++ b/frontend/app/components/shared/DevTools/StackEventRow/StackEventRow.tsx @@ -3,21 +3,18 @@ import JumpButton from '../JumpButton'; import { Icon } from 'UI'; import cn from 'classnames'; import { OPENREPLAY, SENTRY, DATADOG, STACKDRIVER } from 'Types/session/stackEvent'; -import { useModal } from 'App/components/Modal'; -import StackEventModal from '../StackEventModal'; interface Props { event: any; onJump: any; + style?: any; + isActive?: boolean; + onClick?: any; } function StackEventRow(props: Props) { - const { event, onJump } = props; + const { event, onJump, style, isActive } = props; let message = event.payload[0] || ''; message = typeof message === 'string' ? message : JSON.stringify(message); - const onClickDetails = () => { - showModal(, { right: true }); - }; - const { showModal } = useModal(); const iconProps: any = React.useMemo(() => { const { source } = event; @@ -30,11 +27,13 @@ function StackEventRow(props: Props) { return (
diff --git a/frontend/app/components/shared/DevTools/TimeTable/BarRow.tsx b/frontend/app/components/shared/DevTools/TimeTable/BarRow.tsx index 2b151cf14..f283eb7ac 100644 --- a/frontend/app/components/shared/DevTools/TimeTable/BarRow.tsx +++ b/frontend/app/components/shared/DevTools/TimeTable/BarRow.tsx @@ -27,6 +27,40 @@ const BarRow = ({ }: Props) => { const timeOffset = time - timestart; ttfb = ttfb || 0; + // TODO fix the tooltip + + const content = ( + + {ttfb != null && ( +
+
{'Waiting (TTFB)'}
+
+
+
+
{formatTime(ttfb)}
+
+ )} +
+
{'Content Download'}
+
+
+
+
{formatTime(duration - ttfb)}
+
+ + ); const trigger = (
- - {ttfb != null && ( -
-
{'Waiting (TTFB)'}
-
-
-
-
{formatTime(ttfb)}
-
- )} -
-
{'Content Download'}
-
-
-
-
{formatTime(duration - ttfb)}
-
- - } - placement="top" - > - {trigger} - +
+ {trigger}
); }; diff --git a/frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx b/frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx index 2b242f331..8271a6561 100644 --- a/frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx +++ b/frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx @@ -72,8 +72,6 @@ type Props = { hoverable?: boolean; onRowClick?: (row: any, index: number) => void; onJump?: (time: any) => void; - sortBy?: string; - sortAscending?: boolean; }; type TimeLineInfo = { @@ -145,8 +143,19 @@ export default class TimeTable extends React.PureComponent { scroller = React.createRef(); autoScroll = true; - componentDidMount() { - if (this.scroller.current) { + // componentDidMount() { + // if (this.scroller.current) { + // this.scroller.current.scrollToRow(this.props.activeIndex); + // } + // } + + adjustScroll(prevActiveIndex: number) { + if ( + this.props.activeIndex && + this.props.activeIndex >= 0 && + prevActiveIndex !== this.props.activeIndex && + this.scroller.current + ) { this.scroller.current.scrollToRow(this.props.activeIndex); } } @@ -161,14 +170,8 @@ export default class TimeTable extends React.PureComponent { ...computeTimeLine(this.props.rows, this.state.firstVisibleRowIndex, this.visibleCount), }); } - if ( - this.props.activeIndex && - this.props.activeIndex >= 0 && - prevProps.activeIndex !== this.props.activeIndex && - this.scroller.current - ) { - this.scroller.current.scrollToRow(this.props.activeIndex); - } + + // this.adjustScroll(prevProps.activeIndex); } onScroll = ({ @@ -190,7 +193,7 @@ export default class TimeTable extends React.PureComponent { onJump = (index: any) => { if (this.props.onJump) { - this.props.onJump(this.props.rows[index].time); + this.props.onJump(this.props.rows[index]); } }; @@ -203,23 +206,29 @@ export default class TimeTable extends React.PureComponent {
activeIndex, - })} + className={cn( + 'dev-row border-b border-color-gray-light-shade group items-center', + stl.row, + { + [stl.hoverable]: hoverable, + 'error color-red': !!row.isRed && row.isRed(), + 'cursor-pointer': typeof onRowClick === 'function', + [stl.activeRow]: activeIndex === index, + // [stl.inactiveRow]: !activeIndex || index > activeIndex, + } + )} onClick={typeof onRowClick === 'function' ? () => onRowClick(row, index) : undefined} id="table-row" > - {columns.filter((i: any) => !i.hidden).map(({ dataKey, render, width }) => ( -
- {render - ? render(row) - : row[dataKey || ''] || {'empty'}} -
- ))} + {columns + .filter((i: any) => !i.hidden) + .map(({ dataKey, render, width }) => ( +
+ {render + ? render(row) + : row[dataKey || ''] || {'empty'}} +
+ ))}
@@ -270,8 +279,6 @@ export default class TimeTable extends React.PureComponent { referenceLines = [], additionalHeight = 0, activeIndex, - sortBy = '', - sortAscending = true, } = this.props; const columns = this.props.children.filter((i: any) => !i.hidden); const { timewidth, timestart } = this.state; @@ -324,10 +331,9 @@ export default class TimeTable extends React.PureComponent { 'cursor-pointer': typeof onClick === 'function', })} style={{ width: `${width}px` }} - onClick={() => this.onColumnClick(dataKey, onClick)} + // onClick={() => this.onColumnClick(dataKey, onClick)} > {label} - {!!sortBy && sortBy === dataKey && }
))}
@@ -360,6 +366,7 @@ export default class TimeTable extends React.PureComponent { {({ width }: { width: number }) => ( { rowHeight={ROW_HEIGHT} rowRenderer={this.renderRow} onScroll={this.onScroll} - scrollToAlignment="start" + scrollToAlignment="center" forceUpdateProp={timestart | timewidth | (activeIndex || 0)} /> )} diff --git a/frontend/app/components/shared/FetchDetailsModal/FetchDetailsModal.tsx b/frontend/app/components/shared/FetchDetailsModal/FetchDetailsModal.tsx index 2121c9aa1..bcee5f5b9 100644 --- a/frontend/app/components/shared/FetchDetailsModal/FetchDetailsModal.tsx +++ b/frontend/app/components/shared/FetchDetailsModal/FetchDetailsModal.tsx @@ -4,6 +4,7 @@ import { Button } from 'UI'; import FetchPluginMessage from './components/FetchPluginMessage'; import { TYPES } from 'Types/session/resource'; import FetchTabs from './components/FetchTabs/FetchTabs'; +import { useStore } from 'App/mstore'; interface Props { resource: any; @@ -15,6 +16,10 @@ function FetchDetailsModal(props: Props) { const [resource, setResource] = useState(props.resource); const [first, setFirst] = useState(false); const [last, setLast] = useState(false); + const isXHR = resource.type === TYPES.XHR || resource.type === TYPES.FETCH; + const { + sessionStore: { devTools }, + } = useStore(); useEffect(() => { const index = rows.indexOf(resource); @@ -27,6 +32,7 @@ function FetchDetailsModal(props: Props) { const index = rows.indexOf(resource); if (index > 0) { setResource(rows[index - 1]); + devTools.update('network', { index: index - 1 }) } }; @@ -34,6 +40,7 @@ function FetchDetailsModal(props: Props) { const index = rows.indexOf(resource); if (index < rows.length - 1) { setResource(rows[index + 1]); + devTools.update('network', { index: index + 1 }) } }; @@ -42,9 +49,8 @@ function FetchDetailsModal(props: Props) {
Network Request
- {resource.type === TYPES.XHR && !fetchPresented && } - - {resource.type === TYPES.XHR && fetchPresented && } + {isXHR && !fetchPresented && } + {isXHR && } {rows && rows.length > 0 && (
diff --git a/frontend/app/components/shared/FetchDetailsModal/components/FetchBasicDetails/FetchBasicDetails.tsx b/frontend/app/components/shared/FetchDetailsModal/components/FetchBasicDetails/FetchBasicDetails.tsx index 6c81a744c..49e16c00f 100644 --- a/frontend/app/components/shared/FetchDetailsModal/components/FetchBasicDetails/FetchBasicDetails.tsx +++ b/frontend/app/components/shared/FetchDetailsModal/components/FetchBasicDetails/FetchBasicDetails.tsx @@ -1,20 +1,27 @@ -import React from 'react'; -import { Icon } from 'UI'; +import React, { useMemo } from 'react'; import { formatBytes } from 'App/utils'; import CopyText from 'Shared/CopyText'; -import { TYPES } from 'Types/session/resource'; +import cn from 'classnames'; interface Props { resource: any; } function FetchBasicDetails({ resource }: Props) { const _duration = parseInt(resource.duration); + const text = useMemo(() => { + if (resource.url.length > 50) { + const endText = resource.url.split('/').pop(); + return resource.url.substring(0, 50 - endText.length) + '.../' + endText; + } + return resource.url; + }, [resource]); + return (
Name
- {resource.name} + {text}
@@ -46,7 +53,12 @@ function FetchBasicDetails({ resource }: Props) { {resource.status && (
Status
-
+
{resource.status === '200' && (
)} diff --git a/frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx b/frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx index 647ee68bd..f2624b460 100644 --- a/frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx +++ b/frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx @@ -49,10 +49,12 @@ const SessionSearchQueryParamHandler = React.memo((props: Props) => { } else { const _filters: any = { ...filtersMap }; const _filter = _filters[key]; - _filter.value = valueArr; - _filter.operator = operator; - _filter.source = sourceArr; - props.addFilter(_filter); + if (!!_filter) { + _filter.value = valueArr; + _filter.operator = operator; + _filter.source = sourceArr; + props.addFilter(_filter); + } } } }; diff --git a/frontend/app/components/ui/JSONTree/JSONTree.js b/frontend/app/components/ui/JSONTree/JSONTree.js index dc6ab786c..b94324ebd 100644 --- a/frontend/app/components/ui/JSONTree/JSONTree.js +++ b/frontend/app/components/ui/JSONTree/JSONTree.js @@ -8,7 +8,7 @@ function updateObjectLink(obj) { } export default ({ src, ...props }) => ( - ( iconStle="triangle" { ...props } /> -); \ No newline at end of file +); diff --git a/frontend/app/components/ui/SVG.tsx b/frontend/app/components/ui/SVG.tsx index 58e73ad61..50a451917 100644 --- a/frontend/app/components/ui/SVG.tsx +++ b/frontend/app/components/ui/SVG.tsx @@ -256,7 +256,7 @@ const SVG = (props: Props) => { case 'id-card': return ; case 'image': return ; case 'info-circle-fill': return ; - case 'info-circle': return ; + case 'info-circle': return ; case 'info-square': return ; case 'info': return ; case 'inspect': return ; diff --git a/frontend/app/mstore/sessionStore.ts b/frontend/app/mstore/sessionStore.ts index 98a7061e6..d055a9aa8 100644 --- a/frontend/app/mstore/sessionStore.ts +++ b/frontend/app/mstore/sessionStore.ts @@ -5,75 +5,105 @@ import Session from './types/session'; import Record, { LAST_7_DAYS } from 'Types/app/period'; class UserFilter { - endDate: number = new Date().getTime(); - startDate: number = new Date().getTime() - 24 * 60 * 60 * 1000; - rangeName: string = LAST_7_DAYS; - filters: any = []; - page: number = 1; - limit: number = 10; - period: any = Record({ rangeName: LAST_7_DAYS }); + endDate: number = new Date().getTime(); + startDate: number = new Date().getTime() - 24 * 60 * 60 * 1000; + rangeName: string = LAST_7_DAYS; + filters: any = []; + page: number = 1; + limit: number = 10; + period: any = Record({ rangeName: LAST_7_DAYS }); - constructor() { - makeAutoObservable(this, { - page: observable, - update: action, - }); + constructor() { + makeAutoObservable(this, { + page: observable, + update: action, + }); + } + + update(key: string, value: any) { + // @ts-ignore + this[key] = value; + + if (key === 'period') { + this.startDate = this.period.start; + this.endDate = this.period.end; } + } - update(key: string, value: any) { - this[key] = value; + setFilters(filters: any[]) { + this.filters = filters; + } - if (key === 'period') { - this.startDate = this.period.start; - this.endDate = this.period.end; - } - } + setPage(page: number) { + this.page = page; + } - setFilters(filters: any[]) { - this.filters = filters; - } + toJson() { + return { + endDate: this.period.end, + startDate: this.period.start, + filters: this.filters.map(filterMap), + page: this.page, + limit: this.limit, + }; + } +} - setPage(page: number) { - this.page = page; - } +interface BaseDevState { + index: number; + filter: string; + activeTab: string; + isError: boolean; +} - toJson() { - return { - endDate: this.period.end, - startDate: this.period.start, - filters: this.filters.map(filterMap), - page: this.page, - limit: this.limit, - }; - } +class DevTools { + network: BaseDevState; + stackEvent: BaseDevState; + console: BaseDevState; + + constructor() { + this.network = { index: 0, filter: '', activeTab: 'ALL', isError: false }; + this.stackEvent = { index: 0, filter: '', activeTab: 'ALL', isError: false }; + this.console = { index: 0, filter: '', activeTab: 'ALL', isError: false }; + makeAutoObservable(this, { + update: action, + }); + } + + update(key: string, value: any) { + // @ts-ignore + this[key] = Object.assign(this[key], value); + } } export default class SessionStore { - userFilter: UserFilter = new UserFilter(); + userFilter: UserFilter = new UserFilter(); + devTools: DevTools = new DevTools(); - constructor() { - makeAutoObservable(this, { - userFilter: observable, + constructor() { + makeAutoObservable(this, { + userFilter: observable, + devTools: observable, + }); + } + + resetUserFilter() { + this.userFilter = new UserFilter(); + } + + getSessions(filter: any): Promise { + return new Promise((resolve, reject) => { + sessionService + .getSessions(filter.toJson()) + .then((response: any) => { + resolve({ + sessions: response.sessions.map((session: any) => new Session().fromJson(session)), + total: response.total, + }); + }) + .catch((error: any) => { + reject(error); }); - } - - resetUserFilter() { - this.userFilter = new UserFilter(); - } - - getSessions(filter: any): Promise { - return new Promise((resolve, reject) => { - sessionService - .getSessions(filter.toJson()) - .then((response: any) => { - resolve({ - sessions: response.sessions.map((session: any) => new Session().fromJson(session)), - total: response.total, - }); - }) - .catch((error: any) => { - reject(error); - }); - }); - } + }); + } } diff --git a/frontend/app/mstore/types/funnelStage.ts b/frontend/app/mstore/types/funnelStage.ts index 85cbb7a64..7d79bb402 100644 --- a/frontend/app/mstore/types/funnelStage.ts +++ b/frontend/app/mstore/types/funnelStage.ts @@ -24,10 +24,11 @@ export default class FunnelStage { } fromJSON(json: any, total: number = 0, previousSessionCount: number = 0) { - this.dropDueToIssues = json.dropDueToIssues; + previousSessionCount = previousSessionCount || 0; + this.dropDueToIssues = json.dropDueToIssues || 0; this.dropPct = json.dropPct; this.operator = json.operator; - this.sessionsCount = json.sessionsCount; + this.sessionsCount = json.sessionsCount || 0; this.usersCount = json.usersCount; this.value = json.value; this.type = json.type; diff --git a/frontend/app/mstore/types/widget.ts b/frontend/app/mstore/types/widget.ts index c43cf76e2..68e2813f4 100644 --- a/frontend/app/mstore/types/widget.ts +++ b/frontend/app/mstore/types/widget.ts @@ -6,7 +6,7 @@ import Session from "App/mstore/types/session"; import Funnelissue from 'App/mstore/types/funnelIssue'; import { issueOptions } from 'App/constants/filterOptions'; import { FilterKey } from 'Types/filter/filterType'; -import Period, { LAST_24_HOURS, LAST_30_DAYS } from 'Types/app/period'; +import Period, { LAST_24_HOURS } from 'Types/app/period'; export default class Widget { public static get ID_KEY():string { return "metricId" } diff --git a/frontend/app/player/_singletone.ts b/frontend/app/player/_singletone.ts index 2449542b6..63d545edd 100644 --- a/frontend/app/player/_singletone.ts +++ b/frontend/app/player/_singletone.ts @@ -24,7 +24,7 @@ export function init(session, config, live = false) { export function clean() { if (instance === null) return; instance.clean(); - cleanStore() + cleanStore(); instance = null; } @@ -88,4 +88,4 @@ export const Controls = { speedUp, speedDown, callPeer -} \ No newline at end of file +} diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index deaf09f11..92c67fd7b 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -24,7 +24,7 @@ import { decryptSessionBytes } from './network/crypto'; import Lists, { INITIAL_STATE as LISTS_INITIAL_STATE, State as ListsState } from './Lists'; -import Screen, { +import Screen, { INITIAL_STATE as SCREEN_INITIAL_STATE, State as ScreenState, } from './Screen/Screen'; @@ -117,7 +117,7 @@ export default class MessageManager { private lastMessageInFileTime: number = 0; constructor( - private readonly session: any /*Session*/, + private readonly session: any /*Session*/, private readonly state: Store, private readonly screen: Screen, initialLists?: Partial @@ -293,7 +293,7 @@ export default class MessageManager { /* == REFACTOR_ME == */ const lastLoadedLocationMsg = this.loadedLocationManager.moveGetLast(t, index); if (!!lastLoadedLocationMsg) { - // TODO: page-wise resources list // setListsStartTime(lastLoadedLocationMsg.time) + // TODO: page-wise resources list // setListsStartTime(lastLoadedLocationMsg.time) this.navigationStartOffset = lastLoadedLocationMsg.navigationStart - this.sessionStart; } const llEvent = this.locationEventManager.moveGetLast(t, index); @@ -525,7 +525,7 @@ export default class MessageManager { this.state.update({ cssLoading }); } - private setSize({ height, width }: { height: number, width: number }) { + private setSize({ height, width }: { height: number, width: number }) { this.screen.scale({ height, width }); this.state.update({ width, height }); diff --git a/frontend/app/player/web/assist/ListWalkerWithMarks.ts b/frontend/app/player/web/assist/ListWalkerWithMarks.ts new file mode 100644 index 000000000..43c05382b --- /dev/null +++ b/frontend/app/player/web/assist/ListWalkerWithMarks.ts @@ -0,0 +1,42 @@ +import type { Timed } from './types'; +import ListWalker from './ListWalker' + + +type CheckFn = (t: T) => boolean + + +export default class ListWalkerWithMarks extends ListWalker { + private _markCountNow: number = 0 + private _markCount: number = 0 + constructor(private isMarked: CheckFn, initialList: T[] = []) { + super(initialList) + this._markCount = initialList.reduce((n, item) => isMarked(item) ? n+1 : n, 0) + } + + append(item: T) { + if (this.isMarked(item)) { this._markCount++ } + super.append(item) + } + + protected moveNext() { + const val = super.moveNext() + if (val && this.isMarked(val)) { + this._markCountNow++ + } + return val + } + protected movePrev() { + const val = super.movePrev() + if (val && this.isMarked(val)) { + this._markCountNow-- + } + return val + } + get markedCountNow(): number { + return this._markCountNow + } + get markedCount(): number { + return this._markCount + } + +} diff --git a/frontend/app/styles/general.css b/frontend/app/styles/general.css index cce982514..a21cfe239 100644 --- a/frontend/app/styles/general.css +++ b/frontend/app/styles/general.css @@ -355,4 +355,8 @@ p { width: 80px; height: 80px; transform: rotate(45deg); +} + +.dev-row { + transition: all 0.5s; } \ No newline at end of file diff --git a/frontend/app/svg/icons/info-circle.svg b/frontend/app/svg/icons/info-circle.svg index 035661835..42dc99c8f 100644 --- a/frontend/app/svg/icons/info-circle.svg +++ b/frontend/app/svg/icons/info-circle.svg @@ -1,11 +1,4 @@ - - - - - - - - - - - + + + + \ No newline at end of file diff --git a/frontend/app/types/dashboard/helper.js b/frontend/app/types/dashboard/helper.js index 05e50d757..f6e819da8 100644 --- a/frontend/app/types/dashboard/helper.js +++ b/frontend/app/types/dashboard/helper.js @@ -27,7 +27,7 @@ const weekdays = [ 'Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat' ]; // const months = [ "January", "February" ]; export const getTimeString = (ts, period) => { const date = new Date(ts); - const diff = period.end - period.start; + const diff = period.endTimestamp - period.startTimestamp; if (diff <= DAY) { var isPM = date.getHours() >= 12; return `${ isPM ? date.getHours() - 12 : date.getHours() }:${ startWithZero(date.getMinutes()) } ${isPM? 'pm' : 'am'}`; diff --git a/frontend/package.json b/frontend/package.json index af78fbdaf..c4f0a68de 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -91,6 +91,7 @@ "@types/react-dom": "^18.0.4", "@types/react-redux": "^7.1.24", "@types/react-router-dom": "^5.3.3", + "@types/react-virtualized": "^9.21.21", "@typescript-eslint/eslint-plugin": "^5.24.0", "@typescript-eslint/parser": "^5.24.0", "autoprefixer": "^10.4.7", diff --git a/peers/package-lock.json b/peers/package-lock.json index a903cfd08..ce7c3c1c4 100644 --- a/peers/package-lock.json +++ b/peers/package-lock.json @@ -10,87 +10,7 @@ "license": "Elastic License 2.0 (ELv2)", "dependencies": { "express": "^4.18.1", - "peer": "^0.6.1" - } - }, - "node_modules/@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "node_modules/@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/cors": { - "version": "2.8.12", - "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", - "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" - }, - "node_modules/@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.30", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.30.tgz", - "integrity": "sha512-gstzbTWro2/nFed1WXtf+TtrpwxH7Ggs4RLYTLbeVgIkUQOI3WG/JKjgeOU1zXDvezllupjrf8OPIdvTbIaVOQ==", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/mime": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", - "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" - }, - "node_modules/@types/node": { - "version": "18.7.16", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz", - "integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg==" - }, - "node_modules/@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "node_modules/@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "node_modules/@types/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==", - "dependencies": { - "@types/mime": "*", - "@types/node": "*" - } - }, - "node_modules/@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "dependencies": { - "@types/node": "*" + "peer": "^v1.0.0-rc.4" } }, "node_modules/accepts": { @@ -655,17 +575,12 @@ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "node_modules/peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", + "version": "1.0.0-rc.4", + "resolved": "https://registry.npmjs.org/peer/-/peer-1.0.0-rc.4.tgz", + "integrity": "sha512-xaNIDm3yWR5m8cuijK7jEFAMOWqNJDGSVJ0+Y3qKW5XTNYsNWEdqtg/Btq9eznGxTTeqQZGNw/SxwyrCVdmmDg==", "dependencies": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", "cors": "^2.8.5", "express": "^4.17.1", - "uuid": "^3.4.0", "ws": "^7.2.3", "yargs": "^15.3.1" }, @@ -673,7 +588,7 @@ "peerjs": "bin/peerjs" }, "engines": { - "node": ">=10" + "node": ">=14" } }, "node_modules/proxy-addr": { @@ -894,15 +809,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -989,86 +895,6 @@ } }, "dependencies": { - "@types/body-parser": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", - "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", - "requires": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "@types/connect": { - "version": "3.4.35", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", - "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", - "requires": { - "@types/node": "*" - } - }, - "@types/cors": { - "version": "2.8.12", - "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", - "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==" - }, - "@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", - "requires": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.18", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "@types/express-serve-static-core": { - "version": "4.17.30", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.30.tgz", - "integrity": "sha512-gstzbTWro2/nFed1WXtf+TtrpwxH7Ggs4RLYTLbeVgIkUQOI3WG/JKjgeOU1zXDvezllupjrf8OPIdvTbIaVOQ==", - "requires": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "@types/mime": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", - "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" - }, - "@types/node": { - "version": "18.7.16", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz", - "integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg==" - }, - "@types/qs": { - "version": "6.9.7", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" - }, - "@types/range-parser": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "@types/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==", - "requires": { - "@types/mime": "*", - "@types/node": "*" - } - }, - "@types/ws": { - "version": "7.4.7", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", - "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", - "requires": { - "@types/node": "*" - } - }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1482,17 +1308,12 @@ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "peer": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz", - "integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==", + "version": "1.0.0-rc.4", + "resolved": "https://registry.npmjs.org/peer/-/peer-1.0.0-rc.4.tgz", + "integrity": "sha512-xaNIDm3yWR5m8cuijK7jEFAMOWqNJDGSVJ0+Y3qKW5XTNYsNWEdqtg/Btq9eznGxTTeqQZGNw/SxwyrCVdmmDg==", "requires": { - "@types/cors": "^2.8.6", - "@types/express": "^4.17.3", - "@types/ws": "^7.2.3", - "body-parser": "^1.19.0", "cors": "^2.8.5", "express": "^4.17.1", - "uuid": "^3.4.0", "ws": "^7.2.3", "yargs": "^15.3.1" } @@ -1655,11 +1476,6 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", diff --git a/peers/package.json b/peers/package.json index 51f37b5fa..a38ad3343 100644 --- a/peers/package.json +++ b/peers/package.json @@ -19,6 +19,6 @@ "homepage": "https://github.com/openreplay/openreplay#readme", "dependencies": { "express": "^4.18.1", - "peer": "^0.6.1" + "peer": "^v1.0.0-rc.4" } } diff --git a/scripts/helmcharts/openreplay/files/minio.sh b/scripts/helmcharts/openreplay/files/minio.sh index 0e7b4c506..fc0a7238f 100644 --- a/scripts/helmcharts/openreplay/files/minio.sh +++ b/scripts/helmcharts/openreplay/files/minio.sh @@ -5,7 +5,7 @@ set -e cd /tmp -buckets=("mobs" "sessions-assets" "static" "sourcemaps" "sessions-mobile-assets" "quickwit" "vault-data") +buckets=("mobs" "sessions-assets" "sourcemaps" "sessions-mobile-assets" "quickwit" "vault-data") mc alias set minio http://minio.db.svc.cluster.local:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY @@ -31,11 +31,14 @@ mc mb minio/${bucket} || true done mc ilm import minio/mobs < /tmp/lifecycle.json || true -# Creating frontend bucket +##################################################### +# Creating frontend bucket; Do not change this block! +# !! PUBLIC BUCKETS !! +##################################################### mc mb minio/frontend || true mc policy set download minio/frontend || true mc policy set download minio/sessions-assets || true -mc policy set download minio/static || true + } # /bin/bash kafka.sh migrate $migration_versions diff --git a/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql b/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql index 638d0774b..c4c146d9b 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql @@ -60,4 +60,11 @@ BEGIN END; $$ LANGUAGE plpgsql; +DROP INDEX IF EXISTS events_common.requests_url_idx; +DROP INDEX IF EXISTS events_common.requests_url_gin_idx; +DROP INDEX IF EXISTS events_common.requests_url_gin_idx2; + +DROP INDEX IF EXISTS events.resources_url_gin_idx; +DROP INDEX IF EXISTS events.resources_url_idx; + COMMIT; \ No newline at end of file diff --git a/scripts/schema/db/init_dbs/postgresql/init_schema.sql b/scripts/schema/db/init_dbs/postgresql/init_schema.sql index c58e65d4c..a57978965 100644 --- a/scripts/schema/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/schema/db/init_dbs/postgresql/init_schema.sql @@ -596,17 +596,9 @@ $$ query text NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); - CREATE INDEX requests_url_idx ON events_common.requests (url); + CREATE INDEX requests_duration_idx ON events_common.requests (duration); - CREATE INDEX requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops); CREATE INDEX requests_timestamp_idx ON events_common.requests (timestamp); - CREATE INDEX requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, length(url) - (CASE - WHEN url LIKE 'http://%' - THEN 7 - WHEN url LIKE 'https://%' - THEN 8 - ELSE 0 END)) - gin_trgm_ops); CREATE INDEX requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; CREATE INDEX requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL; CREATE INDEX requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL; diff --git a/sourcemap-uploader/cli.js b/sourcemap-uploader/cli.js index 7085f1345..28c167ea3 100755 --- a/sourcemap-uploader/cli.js +++ b/sourcemap-uploader/cli.js @@ -55,10 +55,10 @@ dir.addArgument(['-u', '--js-dir-url'], { // TODO: exclude in dir -const { command, api_key, project_key, server, verbose, ...args } = +const { command, api_key, project_key, server, logs, ...args } = parser.parseArgs(); -global._VERBOSE = !!verbose; +global._VERBOSE = !!logs; (command === 'file' ? uploadFile( diff --git a/sourcemap-uploader/package.json b/sourcemap-uploader/package.json index e8da522a1..ee495605e 100644 --- a/sourcemap-uploader/package.json +++ b/sourcemap-uploader/package.json @@ -1,6 +1,6 @@ { "name": "@openreplay/sourcemap-uploader", - "version": "3.0.6", + "version": "3.0.7", "description": "NPM module to upload your JS sourcemaps files to OpenReplay", "bin": "cli.js", "main": "index.js", diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 9c640af29..ce2473ef5 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -164,7 +164,7 @@ export default class App { this.worker.onmessage = ({ data }: MessageEvent) => { if (data === 'restart') { this.stop(false) - this.start({ forceNew: true }) // TODO: keep userID & metadata (draw scenarios) + this.start({}, true) } else if (data.type === 'failure') { this.stop(false) this._debug('worker_failed', data.reason) @@ -201,7 +201,6 @@ export default class App { send(message: Message, urgent = false): void { if (this.activityState === ActivityState.NotActive) { - // this.debug.log('SendiTrying to send when not active', message) <- crashing the app return } this.messages.push(message) @@ -370,7 +369,7 @@ export default class App { this.sessionStorage.removeItem(this.options.session_reset_key) } } - private _start(startOpts: StartOptions): Promise { + private _start(startOpts: StartOptions = {}, resetByWorker = false): Promise { if (!this.worker) { return Promise.resolve(UnsuccessfulStart('No worker found: perhaps, CSP is not set.')) } @@ -382,9 +381,19 @@ export default class App { ) } this.activityState = ActivityState.Starting + if (startOpts.sessionHash) { this.session.applySessionHash(startOpts.sessionHash) } + if (startOpts.forceNew) { + // Reset session metadata only if requested directly + this.session.reset() + } + this.session.assign({ + // MBTODO: maybe it would make sense to `forceNew` if the `userID` was changed + userID: startOpts.userID, + metadata: startOpts.metadata, + }) const timestamp = now() this.worker.postMessage({ @@ -397,17 +406,9 @@ export default class App { connAttemptGap: this.options.connAttemptGap, }) - this.session.update({ - // TODO: transparent "session" module logic AND explicit internal api for plugins. - // "updating" with old metadata in order to trigger session's UpdateCallbacks. - // (for the case of internal .start() calls, like on "restart" webworker signal or assistent connection in tracker-assist ) - metadata: startOpts.metadata || this.session.getInfo().metadata, - userID: startOpts.userID, - }) - - const sReset = this.sessionStorage.getItem(this.options.session_reset_key) + const lsReset = this.sessionStorage.getItem(this.options.session_reset_key) !== null this.sessionStorage.removeItem(this.options.session_reset_key) - const shouldReset = startOpts.forceNew || sReset !== null + const needNewSessionID = startOpts.forceNew || lsReset || resetByWorker return window .fetch(this.options.ingestPoint + '/v1/web/start', { @@ -419,7 +420,7 @@ export default class App { ...this.getTrackerInfo(), timestamp, userID: this.session.getInfo().userID, - token: shouldReset ? undefined : this.session.getSessionToken(), + token: needNewSessionID ? undefined : this.session.getSessionToken(), deviceMemory, jsHeapSizeLimit, }), @@ -447,29 +448,33 @@ export default class App { const { token, userUUID, - sessionID, projectID, beaconSizeLimit, - startTimestamp, // real startTS, derived from sessionID - delay, + delay, // derived from token + sessionID, // derived from token + startTimestamp, // real startTS (server time), derived from sessionID } = r if ( typeof token !== 'string' || typeof userUUID !== 'string' || - //typeof startTimestamp !== 'number' || - //typeof sessionID !== 'string' || + (typeof startTimestamp !== 'number' && typeof startTimestamp !== 'undefined') || + typeof sessionID !== 'string' || typeof delay !== 'number' || (typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined') ) { return Promise.reject(`Incorrect server response: ${JSON.stringify(r)}`) } this.delay = delay - const prevSessionID = this.session.getInfo().sessionID - if (prevSessionID && prevSessionID !== sessionID) { - this.session.reset() - } this.session.setSessionToken(token) - this.session.update({ sessionID, timestamp: startTimestamp || timestamp, projectID }) // TODO: no no-explicit 'any' + this.session.assign({ + sessionID, + timestamp: startTimestamp || timestamp, + projectID, + }) + // (Re)send Metadata for the case of a new session + Object.entries(this.session.getInfo().metadata).forEach(([key, value]) => + this.send(Metadata(key, value)), + ) this.localStorage.setItem(this.options.local_uuid_key, userUUID) this.worker.postMessage({ @@ -506,15 +511,15 @@ export default class App { }) } - start(options: StartOptions = {}): Promise { + start(...args: Parameters): Promise { if (!document.hidden) { - return this._start(options) + return this._start(...args) } else { return new Promise((resolve) => { const onVisibilityChange = () => { if (!document.hidden) { document.removeEventListener('visibilitychange', onVisibilityChange) - resolve(this._start(options)) + resolve(this._start(...args)) } } document.addEventListener('visibilitychange', onVisibilityChange) @@ -538,8 +543,4 @@ export default class App { } } } - restart() { - this.stop(false) - this.start({ forceNew: false }) - } } diff --git a/tracker/tracker/src/main/app/session.ts b/tracker/tracker/src/main/app/session.ts index 5c3db5ac5..4682bcc43 100644 --- a/tracker/tracker/src/main/app/session.ts +++ b/tracker/tracker/src/main/app/session.ts @@ -37,7 +37,7 @@ export default class Session { this.callbacks.forEach((cb) => cb(newInfo)) } - update(newInfo: Partial): void { + assign(newInfo: Partial): void { if (newInfo.userID !== undefined) { // TODO clear nullable/undefinable types this.userID = newInfo.userID