jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' into player-refactoring-phase-1

Browse source
This commit is contained in:
sylenien 2022-11-25 11:42:11 +01:00
commit b756fee165
94 changed files with 2604 additions and 1210 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.github/workflows/workers-ee.yaml vendored
View file

@ -86,7 +86,11 @@ jobs:
;;
esac
[[ $(cat /tmp/images_to_build.txt) != "" ]] || (echo "Nothing to build here"; exit 1)
if [[ $(cat /tmp/images_to_build.txt) == "" ]]; then
echo "Nothing to build here"
touch /tmp/nothing-to-build-here
exit 0
fi
#
# Pushing image to registry
#
@ -94,7 +98,7 @@ jobs:
for image in $(cat /tmp/images_to_build.txt);
do
echo "Bulding $image"
PUSH_IMAGE=0 bash -x ./build.sh skip $image
PUSH_IMAGE=0 bash -x ./build.sh ee $image
[[ "x$skip_security_checks" == "xtrue" ]] || {
curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./
./trivy image --exit-code 1 --vuln-type os,library --severity "HIGH,CRITICAL" --ignore-unfixed $DOCKER_REPO/$image:$IMAGE_TAG
@ -105,7 +109,7 @@ jobs:
} && {
echo "Skipping Security Checks"
}
PUSH_IMAGE=1 bash -x ./build.sh skip $image
PUSH_IMAGE=1 bash -x ./build.sh ee $image
echo "::set-output name=image::$DOCKER_REPO/$image:$IMAGE_TAG"
done
@ -118,6 +122,7 @@ jobs:
# Deploying image to environment.
#
set -x
[[ -f /tmp/nothing-to-build-here ]] && exit 0
cd scripts/helmcharts/
## Update secerts

8
.github/workflows/workers.yaml vendored
View file

@ -86,7 +86,11 @@ jobs:
;;
esac
[[ $(cat /tmp/images_to_build.txt) != "" ]] || (echo "Nothing to build here"; exit 1)
if [[ $(cat /tmp/images_to_build.txt) == "" ]]; then
echo "Nothing to build here"
touch /tmp/nothing-to-build-here
exit 0
fi
#
# Pushing image to registry
#
@ -116,6 +120,8 @@ jobs:
#
# Deploying image to environment.
#
set -x
[[ -f /tmp/nothing-to-build-here ]] && exit 0
cd scripts/helmcharts/
## Update secerts

15
api/chalicelib/core/alerts_processor.py
View file

@ -199,7 +199,8 @@ def process():
logging.info(f"Valid alert, notifying users, alertId:{alert['alertId']} name: {alert['name']}")
notifications.append(generate_notification(alert, result))
except Exception as e:
logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}")
logging.error(
f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}")
logging.error(query)
logging.error(e)
cur = cur.recreate(rollback=True)
@ -212,12 +213,22 @@ def process():
alerts.process_notifications(notifications)
def __format_value(x):
if x % 1 == 0:
x = int(x)
else:
x = round(x, 2)
return f"{x:,}"
def generate_notification(alert, result):
left = __format_value(result['value'])
right = __format_value(alert['query']['right'])
return {
"alertId": alert["alertId"],
"tenantId": alert["tenantId"],
"title": alert["name"],
"description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).",
"description": f"has been triggered, {alert['query']['left']} = {left} ({alert['query']['operator']} {right}).",
"buttonText": "Check metrics for more details",
"buttonUrl": f"/{alert['projectId']}/metrics",
"imageUrl": None,

20
api/chalicelib/core/metrics.py
View file

@ -419,7 +419,7 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True,
chart=True, data=args)
pg_sub_query_chart.append("resources.type = 'img'")
pg_sub_query_chart.append("resources.url = top_img.url")
pg_sub_query_chart.append("resources.url_hostpath = top_img.url_hostpath")
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
chart=False, data=args)
@ -431,13 +431,13 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
with pg_client.PostgresClient() as cur:
pg_query = f"""SELECT *
FROM (SELECT resources.url,
FROM (SELECT resources.url_hostpath,
COALESCE(AVG(resources.duration), 0) AS avg_duration,
COUNT(resources.session_id) AS sessions_count
FROM events.resources
INNER JOIN sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_subset)}
GROUP BY resources.url
GROUP BY resources.url_hostpath
ORDER BY avg_duration DESC
LIMIT 10) AS top_img
LEFT JOIN LATERAL (
@ -485,13 +485,13 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi
if resources and len(resources) > 0:
for r in resources:
if r["type"] == "IMG":
img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s")
img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s")
img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value']
elif r["type"] == "LOCATION":
location_constraints.append(f"pages.path = %(val_{len(location_constraints)})s")
location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value']
else:
request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s")
request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s")
request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value']
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
@ -627,12 +627,12 @@ def search(text, resource_type, project_id, performance=False, pages_only=False,
pg_sub_query.append("url_hostpath ILIKE %(value)s")
with pg_client.PostgresClient() as cur:
pg_query = f"""SELECT key, value
FROM ( SELECT DISTINCT ON (url) ROW_NUMBER() OVER (PARTITION BY type ORDER BY url) AS r,
url AS value,
FROM ( SELECT DISTINCT ON (url_hostpath) ROW_NUMBER() OVER (PARTITION BY type ORDER BY url_hostpath) AS r,
url_hostpath AS value,
type AS key
FROM events.resources INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query)}
ORDER BY url, type ASC) AS ranked_values
ORDER BY url_hostpath, type ASC) AS ranked_values
WHERE ranked_values.r<=5;"""
cur.execute(cur.mogrify(pg_query, {"project_id": project_id, "value": helper.string_to_sql_like(text)}))
rows = cur.fetchall()
@ -893,7 +893,7 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days
if type is not None:
pg_sub_query_subset.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'")
if url is not None:
pg_sub_query_subset.append(f"resources.url = %(value)s")
pg_sub_query_subset.append(f"resources.url_hostpath = %(value)s")
with pg_client.PostgresClient() as cur:
pg_query = f"""WITH resources AS (SELECT resources.duration, timestamp
@ -1009,7 +1009,7 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
ORDER BY avg DESC
LIMIT 10) AS main_list
INNER JOIN LATERAL (
SELECT url, type
SELECT url_hostpath AS url, type
FROM events.resources
INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query)}

13
api/chalicelib/core/sessions.py
View file

@ -177,7 +177,7 @@ def _isUndefined_operator(op: schemas.SearchEventOperator):
# This function executes the query and return result
def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False,
error_status=schemas.ErrorStatus.all, count_only=False, issue=None):
error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False):
if data.bookmarked:
data.startDate, data.endDate = sessions_favorite.get_start_end_timestamp(project_id, user_id)
@ -185,9 +185,11 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
favorite_only=data.bookmarked, issue=issue, project_id=project_id,
user_id=user_id)
if data.limit is not None and data.page is not None:
full_args["sessions_limit"] = data.limit
full_args["sessions_limit_s"] = (data.page - 1) * data.limit
full_args["sessions_limit_e"] = data.page * data.limit
else:
full_args["sessions_limit"] = 200
full_args["sessions_limit_s"] = 1
full_args["sessions_limit_e"] = 200
@ -235,6 +237,12 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
GROUP BY user_id
) AS users_sessions;""",
full_args)
elif ids_only:
main_query = cur.mogrify(f"""SELECT DISTINCT ON(s.session_id) s.session_id
{query_part}
ORDER BY s.session_id desc
LIMIT %(sessions_limit)s OFFSET %(sessions_limit_s)s;""",
full_args)
else:
if data.order is None:
data.order = schemas.SortOrderType.desc
@ -242,7 +250,6 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
if data.sort is not None and data.sort != "session_id":
# sort += " " + data.order + "," + helper.key_to_snake_case(data.sort)
sort = helper.key_to_snake_case(data.sort)
meta_keys = metadata.get(project_id=project_id)
main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count,
COALESCE(JSONB_AGG(full_sessions)
@ -266,7 +273,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
print(data.json())
print("--------------------")
raise err
if errors_only:
if errors_only or ids_only:
return helper.list_to_camel_case(cur.fetchall())
sessions = cur.fetchone()

4
api/chalicelib/core/sessions_devtool.py
View file

@ -1,6 +1,6 @@
from decouple import config
from chalicelib.utils.s3 import client
from chalicelib.utils import s3
def __get_devtools_keys(project_id, session_id):
@ -16,7 +16,7 @@ def __get_devtools_keys(project_id, session_id):
def get_urls(session_id, project_id):
results = []
for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
results.append(client.generate_presigned_url(
results.append(s3.client.generate_presigned_url(
'get_object',
Params={'Bucket': config("sessions_bucket"), 'Key': k},
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)

30
api/chalicelib/core/sessions_mobs.py
View file

@ -1,7 +1,6 @@
from decouple import config
from chalicelib.utils import s3
from chalicelib.utils.s3 import client
def __get_mob_keys(project_id, session_id):
@ -15,10 +14,14 @@ def __get_mob_keys(project_id, session_id):
]
def __get_mob_keys_deprecated(session_id):
return [str(session_id), str(session_id) + "e"]
def get_urls(project_id, session_id):
results = []
for k in __get_mob_keys(project_id=project_id, session_id=session_id):
results.append(client.generate_presigned_url(
results.append(s3.client.generate_presigned_url(
'get_object',
Params={'Bucket': config("sessions_bucket"), 'Key': k},
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
@ -27,27 +30,18 @@ def get_urls(project_id, session_id):
def get_urls_depercated(session_id):
return [
client.generate_presigned_url(
results = []
for k in __get_mob_keys_deprecated(session_id=session_id):
results.append(s3.client.generate_presigned_url(
'get_object',
Params={
'Bucket': config("sessions_bucket"),
'Key': str(session_id)
},
Params={'Bucket': config("sessions_bucket"), 'Key': k},
ExpiresIn=100000
),
client.generate_presigned_url(
'get_object',
Params={
'Bucket': config("sessions_bucket"),
'Key': str(session_id) + "e"
},
ExpiresIn=100000
)]
))
return results
def get_ios(session_id):
return client.generate_presigned_url(
return s3.client.generate_presigned_url(
'get_object',
Params={
'Bucket': config("ios_bucket"),

136
api/chalicelib/core/significance.py
View file

@ -181,9 +181,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
values=s["value"], value_key=f"value{i + 1}")
n_stages_query.append(f"""
(SELECT main.session_id,
{"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp,
'{event_type}' AS type,
'{s["operator"]}' AS operator
{"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp
FROM {next_table} AS main {" ".join(extra_from)}
WHERE main.timestamp >= {f"T{i}.stage{i}_timestamp" if i > 0 else "%(startTimestamp)s"}
{f"AND main.session_id=T1.session_id" if i > 0 else ""}
@ -191,30 +189,33 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
{(" AND " + " AND ".join(stage_constraints)) if len(stage_constraints) > 0 else ""}
{(" AND " + " AND ".join(first_stage_extra_constraints)) if len(first_stage_extra_constraints) > 0 and i == 0 else ""}
GROUP BY main.session_id)
AS T{i + 1} {"USING (session_id)" if i > 0 else ""}
AS T{i + 1} {"ON (TRUE)" if i > 0 else ""}
""")
if len(n_stages_query) == 0:
n_stages=len(n_stages_query)
if n_stages == 0:
return []
n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query)
n_stages_query += ") AS stages_t"
n_stages_query = f"""
SELECT stages_and_issues_t.*, sessions.user_uuid FROM (
SELECT stages_and_issues_t.*, sessions.user_uuid
FROM (
SELECT * FROM (
SELECT * FROM
{n_stages_query}
SELECT T1.session_id, {",".join([f"stage{i + 1}_timestamp" for i in range(n_stages)])}
FROM {n_stages_query}
LEFT JOIN LATERAL
( SELECT ISE.session_id,
ISS.type as issue_type,
( SELECT ISS.type as issue_type,
ISE.timestamp AS issue_timestamp,
ISS.context_string as issue_context,
COALESCE(ISS.context_string,'') as issue_context,
ISS.issue_id as issue_id
FROM events_common.issues AS ISE INNER JOIN issues AS ISS USING (issue_id)
WHERE ISE.timestamp >= stages_t.stage1_timestamp
AND ISE.timestamp <= stages_t.stage{i + 1}_timestamp
AND ISS.project_id=%(project_id)s
AND ISE.session_id = stages_t.session_id
{"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""}
) AS issues_t USING (session_id)
LIMIT 20 -- remove the limit to get exact stats
) AS issues_t ON (TRUE)
) AS stages_and_issues_t INNER JOIN sessions USING(session_id);
"""
@ -297,7 +298,21 @@ def pearson_corr(x: list, y: list):
return r, confidence, False
def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_with_context, first_stage, last_stage):
# def tuple_or(t: tuple):
# x = 0
# for el in t:
# x |= el # | is for bitwise OR
# return x
#
# The following function is correct optimization of the previous function because t is a list of 0,1
def tuple_or(t: tuple):
for el in t:
if el > 0:
return 1
return 0
def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues, first_stage, last_stage):
"""
Returns two lists with binary values 0/1:
@ -316,12 +331,6 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_
transitions = []
n_sess_affected = 0
errors = {}
for issue in all_issues_with_context:
split = issue.split('__^__')
errors[issue] = {
"errors": [],
"issue_type": split[0],
"context": split[1]}
for row in rows:
t = 0
@ -329,38 +338,26 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_
last_ts = row[f'stage{last_stage}_timestamp']
if first_ts is None:
continue
elif first_ts is not None and last_ts is not None:
elif last_ts is not None:
t = 1
transitions.append(t)
ic_present = False
for issue_type_with_context in errors:
for error_id in all_issues:
if error_id not in errors:
errors[error_id] = []
ic = 0
issue_type = errors[issue_type_with_context]["issue_type"]
context = errors[issue_type_with_context]["context"]
if row['issue_type'] is not None:
row_issue_id=row['issue_id']
if row_issue_id is not None:
if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts):
context_in_row = row['issue_context'] if row['issue_context'] is not None else ''
if issue_type == row['issue_type'] and context == context_in_row:
if error_id == row_issue_id:
ic = 1
ic_present = True
errors[issue_type_with_context]["errors"].append(ic)
errors[error_id].append(ic)
if ic_present and t:
n_sess_affected += 1
# def tuple_or(t: tuple):
# x = 0
# for el in t:
# x |= el
# return x
def tuple_or(t: tuple):
for el in t:
if el > 0:
return 1
return 0
errors = {key: errors[key]["errors"] for key in errors}
all_errors = [tuple_or(t) for t in zip(*errors.values())]
return transitions, errors, all_errors, n_sess_affected
@ -376,10 +373,9 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage):
"""
affected_users = defaultdict(lambda: set())
affected_sessions = defaultdict(lambda: set())
contexts = defaultdict(lambda: None)
all_issues = {}
n_affected_users_dict = defaultdict(lambda: None)
n_affected_sessions_dict = defaultdict(lambda: None)
all_issues_with_context = set()
n_issues_dict = defaultdict(lambda: 0)
issues_by_session = defaultdict(lambda: 0)
@ -395,15 +391,13 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage):
# check that the issue exists and belongs to subfunnel:
if iss is not None and (row[f'stage{last_stage}_timestamp'] is None or
(row[f'stage{first_stage}_timestamp'] < iss_ts < row[f'stage{last_stage}_timestamp'])):
context_string = row['issue_context'] if row['issue_context'] is not None else ''
issue_with_context = iss + '__^__' + context_string
contexts[issue_with_context] = {"context": context_string, "id": row["issue_id"]}
all_issues_with_context.add(issue_with_context)
n_issues_dict[issue_with_context] += 1
if row["issue_id"] not in all_issues:
all_issues[row["issue_id"]] = {"context": row['issue_context'], "issue_type": row["issue_type"]}
n_issues_dict[row["issue_id"]] += 1
if row['user_uuid'] is not None:
affected_users[issue_with_context].add(row['user_uuid'])
affected_users[row["issue_id"]].add(row['user_uuid'])
affected_sessions[issue_with_context].add(row['session_id'])
affected_sessions[row["issue_id"]].add(row['session_id'])
issues_by_session[row[f'session_id']] += 1
if len(affected_users) > 0:
@ -414,29 +408,28 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage):
n_affected_sessions_dict.update({
iss: len(affected_sessions[iss]) for iss in affected_sessions
})
return all_issues_with_context, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict, contexts
return all_issues, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict
def count_sessions(rows, n_stages):
session_counts = {i: set() for i in range(1, n_stages + 1)}
for ind, row in enumerate(rows):
for row in rows:
for i in range(1, n_stages + 1):
if row[f"stage{i}_timestamp"] is not None:
session_counts[i].add(row[f"session_id"])
session_counts = {i: len(session_counts[i]) for i in session_counts}
return session_counts
def count_users(rows, n_stages):
users_in_stages = defaultdict(lambda: set())
for ind, row in enumerate(rows):
users_in_stages = {i: set() for i in range(1, n_stages + 1)}
for row in rows:
for i in range(1, n_stages + 1):
if row[f"stage{i}_timestamp"] is not None:
users_in_stages[i].add(row["user_uuid"])
users_count = {i: len(users_in_stages[i]) for i in range(1, n_stages + 1)}
return users_count
@ -489,18 +482,18 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False)
last_stage = n_stages
n_critical_issues = 0
issues_dict = dict({"significant": [],
"insignificant": []})
issues_dict = {"significant": [],
"insignificant": []}
session_counts = count_sessions(rows, n_stages)
drop = session_counts[first_stage] - session_counts[last_stage]
all_issues_with_context, n_issues_dict, affected_users_dict, affected_sessions, contexts = get_affected_users_for_all_issues(
all_issues, n_issues_dict, affected_users_dict, affected_sessions = get_affected_users_for_all_issues(
rows, first_stage, last_stage)
transitions, errors, all_errors, n_sess_affected = get_transitions_and_issues_of_each_type(rows,
all_issues_with_context,
all_issues,
first_stage, last_stage)
# print("len(transitions) =", len(transitions))
del rows
if any(all_errors):
total_drop_corr, conf, is_sign = pearson_corr(transitions, all_errors)
@ -513,33 +506,32 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False)
if drop_only:
return total_drop_due_to_issues
for issue in all_issues_with_context:
for issue_id in all_issues:
if not any(errors[issue]):
if not any(errors[issue_id]):
continue
r, confidence, is_sign = pearson_corr(transitions, errors[issue])
r, confidence, is_sign = pearson_corr(transitions, errors[issue_id])
if r is not None and drop is not None and is_sign:
lost_conversions = int(r * affected_sessions[issue])
lost_conversions = int(r * affected_sessions[issue_id])
else:
lost_conversions = None
if r is None:
r = 0
split = issue.split('__^__')
issues_dict['significant' if is_sign else 'insignificant'].append({
"type": split[0],
"title": helper.get_issue_title(split[0]),
"affected_sessions": affected_sessions[issue],
"unaffected_sessions": session_counts[1] - affected_sessions[issue],
"type": all_issues[issue_id]["issue_type"],
"title": helper.get_issue_title(all_issues[issue_id]["issue_type"]),
"affected_sessions": affected_sessions[issue_id],
"unaffected_sessions": session_counts[1] - affected_sessions[issue_id],
"lost_conversions": lost_conversions,
"affected_users": affected_users_dict[issue],
"affected_users": affected_users_dict[issue_id],
"conversion_impact": round(r * 100),
"context_string": contexts[issue]["context"],
"issue_id": contexts[issue]["id"]
"context_string": all_issues[issue_id]["context"],
"issue_id": issue_id
})
if is_sign:
n_critical_issues += n_issues_dict[issue]
n_critical_issues += n_issues_dict[issue_id]
return n_critical_issues, issues_dict, total_drop_due_to_issues

3
api/chalicelib/core/sourcemaps.py
View file

@ -77,7 +77,7 @@ def format_payload(p, truncate_to_first=False):
def url_exists(url):
try:
r = requests.head(url, allow_redirects=False)
return r.status_code == 200 and r.headers.get("Content-Type") != "text/html"
return r.status_code == 200 and "text/html" not in r.headers.get("Content-Type", "")
except Exception as e:
print(f"!! Issue checking if URL exists: {url}")
print(e)
@ -100,7 +100,6 @@ def get_traces_group(project_id, payload):
and not (file_url[:params_idx] if params_idx > -1 else file_url).endswith(".js"):
print(f"{u['absPath']} sourcemap is not a JS file")
payloads[key] = None
continue
if key not in payloads:
file_exists_in_bucket = len(file_url) > 0 and s3.exists(config('sourcemaps_bucket'), key)

6
api/env.default
View file

@ -42,11 +42,11 @@ sourcemaps_reader=http://sourcemaps-reader-openreplay.app.svc.cluster.local:9000
STAGE=default-foss
version_number=1.4.0
FS_DIR=/mnt/efs
EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob
EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
EFS_SESSION_MOB_PATTERN=%(sessionId)s
EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools
SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs
SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe
DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs
DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
PRESIGNED_URL_EXPIRATION=3600
ASSIST_JWT_EXPIRATION=144000
ASSIST_JWT_SECRET=

10
api/requirements-alerts.txt
View file

@ -1,15 +1,15 @@
requests==2.28.1
urllib3==1.26.12
boto3==1.26.4
boto3==1.26.14
pyjwt==2.6.0
psycopg2-binary==2.9.5
elasticsearch==8.5.0
elasticsearch==8.5.1
jira==3.4.1
fastapi==0.86.0
uvicorn[standard]==0.19.0
fastapi==0.87.0
uvicorn[standard]==0.20.0
python-decouple==3.6
pydantic[email]==1.10.2
apscheduler==3.9.1
apscheduler==3.9.1.post1

10
api/requirements.txt
View file

@ -1,15 +1,15 @@
requests==2.28.1
urllib3==1.26.12
boto3==1.26.4
boto3==1.26.14
pyjwt==2.6.0
psycopg2-binary==2.9.5
elasticsearch==8.5.0
elasticsearch==8.5.1
jira==3.4.1
fastapi==0.86.0
uvicorn[standard]==0.19.0
fastapi==0.87.0
uvicorn[standard]==0.20.0
python-decouple==3.6
pydantic[email]==1.10.2
apscheduler==3.9.1
apscheduler==3.9.1.post1

8
api/routers/core.py
View file

@ -56,6 +56,14 @@ def sessions_search(projectId: int, data: schemas.FlatSessionsSearchPayloadSchem
return {'data': data}
@app.post('/{projectId}/sessions/search/ids', tags=["sessions"])
@app.post('/{projectId}/sessions/search2/ids', tags=["sessions"])
def session_ids_search(projectId: int, data: schemas.FlatSessionsSearchPayloadSchema = Body(...),
context: schemas.CurrentContext = Depends(OR_context)):
data = sessions.search_sessions(data=data, project_id=projectId, user_id=context.user_id, ids_only=True)
return {'data': data}
@app.get('/{projectId}/events/search', tags=["events"])
def events_search(projectId: int, q: str,
type: Union[schemas.FilterType, schemas.EventType,

18
backend/Dockerfile
View file

@ -1,6 +1,6 @@
FROM golang:1.18-alpine3.15 AS prepare
RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash
RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash librdkafka-dev cyrus-sasl cyrus-sasl-gssapiv2 krb5
WORKDIR /root
@ -15,11 +15,11 @@ COPY pkg pkg
COPY internal internal
ARG SERVICE_NAME
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o service -tags musl openreplay/backend/cmd/$SERVICE_NAME
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o service -tags dynamic openreplay/backend/cmd/$SERVICE_NAME
FROM alpine AS entrypoint
RUN apk add --no-cache ca-certificates
RUN apk add --no-cache ca-certificates librdkafka-dev cyrus-sasl cyrus-sasl-gssapiv2 krb5
RUN adduser -u 1001 openreplay -D
ENV TZ=UTC \
@ -29,6 +29,18 @@ ENV TZ=UTC \
UAPARSER_FILE=/home/openreplay/regexes.yaml \
HTTP_PORT=8080 \
KAFKA_USE_SSL=true \
# KAFKA_USE_KERBEROS should be set true if you wish to use Kerberos auth for Kafka
KAFKA_USE_KERBEROS=false \
# KERBEROS_SERVICE_NAME is the primary name of the Brokers configured in the Broker JAAS file
KERBEROS_SERVICE_NAME="" \
# KERBEROS_PRINCIPAL is this client's principal name
KERBEROS_PRINCIPAL="" \
# KERBEROS_PRINCIPAL is the absolute path to the keytab to be used for authentication
KERBEROS_KEYTAB_LOCATION="" \
# KAFKA_SSL_KEY is the absolute path to the CA cert for verifying the broker's key
KAFKA_SSL_KEY="" \
# KAFKA_SSL_CERT is a CA cert string (PEM format) for verifying the broker's key
KAFKA_SSL_CERT="" \
KAFKA_MAX_POLL_INTERVAL_MS=400000 \
REDIS_STREAMS_MAX_LEN=10000 \
TOPIC_RAW_WEB=raw \

18
backend/Dockerfile.bundle
View file

@ -1,6 +1,6 @@
FROM golang:1.18-alpine3.15 AS prepare
RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash
RUN apk add --no-cache git openssh openssl-dev pkgconf gcc g++ make libc-dev bash librdkafka-dev cyrus-sasl-gssapi cyrus-sasl-devel
WORKDIR /root
@ -14,11 +14,11 @@ COPY cmd cmd
COPY pkg pkg
COPY internal internal
RUN for name in assets db ender http integrations sink storage;do CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o bin/$name -tags musl openreplay/backend/cmd/$name; done
RUN for name in assets db ender http integrations sink storage;do CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o bin/$name -tags dynamic openreplay/backend/cmd/$name; done
FROM alpine AS entrypoint
#FROM pygmy/alpine-tini:latest
RUN apk add --no-cache ca-certificates
RUN apk add --no-cache ca-certificates librdkafka-dev cyrus-sasl-gssapi cyrus-sasl-devel pkgconf
ENV TZ=UTC \
FS_ULIMIT=1000 \
@ -28,6 +28,18 @@ ENV TZ=UTC \
HTTP_PORT=80 \
BEACON_SIZE_LIMIT=7000000 \
KAFKA_USE_SSL=true \
# KAFKA_USE_KERBEROS should be set true if you wish to use Kerberos auth for Kafka
KAFKA_USE_KERBEROS=false \
# KERBEROS_SERVICE_NAME is the primary name of the Brokers configured in the Broker JAAS file
KERBEROS_SERVICE_NAME="" \
# KERBEROS_PRINCIPAL is this client's principal name
KERBEROS_PRINCIPAL="" \
# KERBEROS_PRINCIPAL is the absolute path to the keytab to be used for authentication
KERBEROS_KEYTAB_LOCATION="" \
# KAFKA_SSL_KEY is the absolute path to the CA cert for verifying the broker's key
KAFKA_SSL_KEY="" \
# KAFKA_SSL_CERT is a CA cert string (PEM format) for verifying the broker's key
KAFKA_SSL_CERT="" \
KAFKA_MAX_POLL_INTERVAL_MS=400000 \
REDIS_STREAMS_MAX_LEN=3000 \
TOPIC_RAW_WEB=raw \

47
backend/cmd/sink/main.go
View file

@ -3,19 +3,18 @@ package main
import (
"context"
"log"
"openreplay/backend/pkg/pprof"
"os"
"os/signal"
"strings"
"syscall"
"time"
"openreplay/backend/internal/config/sink"
"openreplay/backend/internal/sink/assetscache"
"openreplay/backend/internal/sink/oswriter"
"openreplay/backend/internal/sink/sessionwriter"
"openreplay/backend/internal/storage"
"openreplay/backend/pkg/messages"
"openreplay/backend/pkg/monitoring"
"openreplay/backend/pkg/pprof"
"openreplay/backend/pkg/queue"
"openreplay/backend/pkg/url/assets"
)
@ -33,7 +32,7 @@ func main() {
log.Fatalf("%v doesn't exist. %v", cfg.FsDir, err)
}
writer := oswriter.NewWriter(cfg.FsUlimit, cfg.FsDir)
writer := sessionwriter.NewWriter(cfg.FsUlimit, cfg.FsDir, cfg.DeadSessionTimeout)
producer := queue.NewProducer(cfg.MessageSizeLimit, true)
defer producer.Close(cfg.ProducerCloseTimeout)
@ -64,6 +63,7 @@ func main() {
if err := producer.Produce(cfg.TopicTrigger, msg.SessionID(), msg.Encode()); err != nil {
log.Printf("can't send SessionEnd to trigger topic: %s; sessID: %d", err, msg.SessionID())
}
writer.Close(msg.SessionID())
return
}
@ -98,39 +98,18 @@ func main() {
// Write encoded message with index to session file
data := msg.EncodeWithIndex()
if data == nil {
log.Printf("can't encode with index, err: %s", err)
return
}
wasWritten := false // To avoid timestamp duplicates in original mob file
// Write message to file
if messages.IsDOMType(msg.TypeID()) {
if err := writer.WriteDOM(msg.SessionID(), data); err != nil {
if strings.Contains(err.Error(), "not a directory") {
// Trying to write data to mob file by original path
oldErr := writer.WriteMOB(msg.SessionID(), data)
if oldErr != nil {
log.Printf("MOB Writeer error: %s, prev DOM error: %s, info: %s", oldErr, err, msg.Meta().Batch().Info())
} else {
wasWritten = true
}
} else {
log.Printf("DOM Writer error: %s, info: %s", err, msg.Meta().Batch().Info())
}
log.Printf("Writer error: %v\n", err)
}
}
if !messages.IsDOMType(msg.TypeID()) || msg.TypeID() == messages.MsgTimestamp {
// TODO: write only necessary timestamps
if err := writer.WriteDEV(msg.SessionID(), data); err != nil {
if strings.Contains(err.Error(), "not a directory") {
if !wasWritten {
// Trying to write data to mob file by original path
oldErr := writer.WriteMOB(msg.SessionID(), data)
if oldErr != nil {
log.Printf("MOB Writeer error: %s, prev DEV error: %s, info: %s", oldErr, err, msg.Meta().Batch().Info())
}
}
} else {
log.Printf("Devtools Writer error: %s, info: %s", err, msg.Meta().Batch().Info())
}
log.Printf("Writer error: %v\n", err)
}
}
@ -158,22 +137,20 @@ func main() {
select {
case sig := <-sigchan:
log.Printf("Caught signal %v: terminating\n", sig)
if err := writer.CloseAll(); err != nil {
log.Printf("closeAll error: %v\n", err)
}
// Sync and stop writer
writer.Stop()
// Commit and stop consumer
if err := consumer.Commit(); err != nil {
log.Printf("can't commit messages: %s", err)
}
consumer.Close()
os.Exit(0)
case <-tick:
if err := writer.SyncAll(); err != nil {
log.Fatalf("sync error: %v\n", err)
}
counter.Print()
if err := consumer.Commit(); err != nil {
log.Printf("can't commit messages: %s", err)
}
log.Printf("writer: %s", writer.Info())
default:
err := consumer.ConsumeNext()
if err != nil {

4
backend/go.mod
View file

@ -69,8 +69,8 @@ require (
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 // indirect
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect

6
backend/go.sum
View file

@ -678,8 +678,9 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -690,8 +691,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

3
backend/internal/config/sink/config.go
View file

@ -9,6 +9,7 @@ type Config struct {
common.Config
FsDir string `env:"FS_DIR,required"`
FsUlimit uint16 `env:"FS_ULIMIT,required"`
DeadSessionTimeout int64 `env:"DEAD_SESSION_TIMEOUT,default=600"`
GroupSink string `env:"GROUP_SINK,required"`
TopicRawWeb string `env:"TOPIC_RAW_WEB,required"`
TopicRawIOS string `env:"TOPIC_RAW_IOS,required"`
@ -17,7 +18,7 @@ type Config struct {
CacheAssets bool `env:"CACHE_ASSETS,required"`
AssetsOrigin string `env:"ASSETS_ORIGIN,required"`
ProducerCloseTimeout int `env:"PRODUCER_CLOSE_TIMEOUT,default=15000"`
CacheThreshold int64 `env:"CACHE_THRESHOLD,default=75"`
CacheThreshold int64 `env:"CACHE_THRESHOLD,default=5"`
CacheExpiration int64 `env:"CACHE_EXPIRATION,default=120"`
}

2
backend/internal/config/storage/config.go
View file

@ -11,7 +11,6 @@ type Config struct {
S3Region string `env:"AWS_REGION_WEB,required"`
S3Bucket string `env:"S3_BUCKET_WEB,required"`
FSDir string `env:"FS_DIR,required"`
FSCleanHRS int `env:"FS_CLEAN_HRS,required"`
FileSplitSize int `env:"FILE_SPLIT_SIZE,required"`
RetryTimeout time.Duration `env:"RETRY_TIMEOUT,default=2m"`
GroupStorage string `env:"GROUP_STORAGE,required"`
@ -21,6 +20,7 @@ type Config struct {
DeleteTimeout time.Duration `env:"DELETE_TIMEOUT,default=48h"`
ProducerCloseTimeout int `env:"PRODUCER_CLOSE_TIMEOUT,default=15000"`
UseFailover bool `env:"USE_FAILOVER,default=false"`
MaxFileSize int64 `env:"MAX_FILE_SIZE,default=524288000"`
}
func New() *Config {

166
backend/internal/sink/oswriter/oswriter.go
View file

@ -1,166 +0,0 @@
package oswriter
import (
"errors"
"log"
"math"
"os"
"path/filepath"
"strconv"
"time"
)
type Writer struct {
ulimit int
dir string
files map[string]*os.File
atimes map[string]int64
}
func NewWriter(ulimit uint16, dir string) *Writer {
return &Writer{
ulimit: int(ulimit),
dir: dir + "/",
files: make(map[string]*os.File),
atimes: make(map[string]int64),
}
}
func (w *Writer) open(fname string) (*os.File, error) {
file, ok := w.files[fname]
if ok {
return file, nil
}
if len(w.atimes) == w.ulimit {
var m_k string
var m_t int64 = math.MaxInt64
for k, t := range w.atimes {
if t < m_t {
m_k = k
m_t = t
}
}
if err := w.close(m_k); err != nil {
return nil, err
}
}
// mkdir if not exist
pathTo := w.dir + filepath.Dir(fname)
if info, err := os.Stat(pathTo); os.IsNotExist(err) {
if err := os.MkdirAll(pathTo, 0755); err != nil {
log.Printf("os.MkdirAll error: %s", err)
}
} else {
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, errors.New("not a directory")
}
}
file, err := os.OpenFile(w.dir+fname, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Printf("os.OpenFile error: %s", err)
return nil, err
}
w.files[fname] = file
w.atimes[fname] = time.Now().Unix()
return file, nil
}
func (w *Writer) close(fname string) error {
file := w.files[fname]
if file == nil {
return nil
}
if err := file.Sync(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
delete(w.files, fname)
delete(w.atimes, fname)
return nil
}
func (w *Writer) WriteDOM(sid uint64, data []byte) error {
return w.write(strconv.FormatUint(sid, 10)+"/dom.mob", data)
}
func (w *Writer) WriteDEV(sid uint64, data []byte) error {
return w.write(strconv.FormatUint(sid, 10)+"/devtools.mob", data)
}
func (w *Writer) WriteMOB(sid uint64, data []byte) error {
// Use session id as a file name without directory
fname := strconv.FormatUint(sid, 10)
file, err := w.openWithoutDir(fname)
if err != nil {
return err
}
_, err = file.Write(data)
return err
}
func (w *Writer) write(fname string, data []byte) error {
file, err := w.open(fname)
if err != nil {
return err
}
_, err = file.Write(data)
return err
}
func (w *Writer) openWithoutDir(fname string) (*os.File, error) {
file, ok := w.files[fname]
if ok {
return file, nil
}
if len(w.atimes) == w.ulimit {
var m_k string
var m_t int64 = math.MaxInt64
for k, t := range w.atimes {
if t < m_t {
m_k = k
m_t = t
}
}
if err := w.close(m_k); err != nil {
return nil, err
}
}
file, err := os.OpenFile(w.dir+fname, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
return nil, err
}
w.files[fname] = file
w.atimes[fname] = time.Now().Unix()
return file, nil
}
func (w *Writer) SyncAll() error {
for _, file := range w.files {
if err := file.Sync(); err != nil {
return err
}
}
return nil
}
func (w *Writer) CloseAll() error {
for _, file := range w.files {
if err := file.Sync(); err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
}
w.files = nil
w.atimes = nil
return nil
}

81
backend/internal/sink/sessionwriter/session.go Normal file
View file

@ -0,0 +1,81 @@
package sessionwriter
import (
"fmt"
"os"
"strconv"
"sync"
"time"
)
type Session struct {
lock *sync.Mutex
dom *os.File
dev *os.File
lastUpdate time.Time
}
func NewSession(dir string, id uint64) (*Session, error) {
if id == 0 {
return nil, fmt.Errorf("wrong session id")
}
filePath := dir + strconv.FormatUint(id, 10)
domFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
return nil, err
}
filePath += "devtools"
devFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
domFile.Close() // should close first file descriptor
return nil, err
}
return &Session{
lock: &sync.Mutex{},
dom: domFile,
dev: devFile,
lastUpdate: time.Now(),
}, nil
}
func (s *Session) Lock() {
s.lock.Lock()
}
func (s *Session) Unlock() {
s.lock.Unlock()
}
func (s *Session) Write(mode FileType, data []byte) (err error) {
if mode == DOM {
_, err = s.dom.Write(data)
} else {
_, err = s.dev.Write(data)
}
s.lastUpdate = time.Now()
return err
}
func (s *Session) LastUpdate() time.Time {
return s.lastUpdate
}
func (s *Session) Sync() error {
domErr := s.dom.Sync()
devErr := s.dev.Sync()
if domErr == nil && devErr == nil {
return nil
}
return fmt.Errorf("dom: %s, dev: %s", domErr, devErr)
}
func (s *Session) Close() error {
domErr := s.dom.Close()
devErr := s.dev.Close()
if domErr == nil && devErr == nil {
return nil
}
return fmt.Errorf("dom: %s, dev: %s", domErr, devErr)
}

8
backend/internal/sink/sessionwriter/types.go Normal file
View file

@ -0,0 +1,8 @@
package sessionwriter
type FileType int
const (
DOM FileType = 1
DEV FileType = 2
)

179
backend/internal/sink/sessionwriter/writer.go Normal file
View file

@ -0,0 +1,179 @@
package sessionwriter
import (
"fmt"
"log"
"math"
"sync"
"time"
)
type SessionWriter struct {
ulimit int
dir string
zombieSessionTimeout float64
lock *sync.Mutex
sessions *sync.Map
meta map[uint64]int64
done chan struct{}
stopped chan struct{}
}
func NewWriter(ulimit uint16, dir string, zombieSessionTimeout int64) *SessionWriter {
w := &SessionWriter{
ulimit: int(ulimit) / 2, // should divide by 2 because each session has 2 files
dir: dir + "/",
zombieSessionTimeout: float64(zombieSessionTimeout),
lock: &sync.Mutex{},
sessions: &sync.Map{},
meta: make(map[uint64]int64, ulimit),
done: make(chan struct{}),
stopped: make(chan struct{}),
}
go w.synchronizer()
return w
}
func (w *SessionWriter) WriteDOM(sid uint64, data []byte) error {
return w.write(sid, DOM, data)
}
func (w *SessionWriter) WriteDEV(sid uint64, data []byte) error {
return w.write(sid, DEV, data)
}
func (w *SessionWriter) Close(sid uint64) {
w.close(sid)
}
func (w *SessionWriter) Stop() {
w.done <- struct{}{}
<-w.stopped
}
func (w *SessionWriter) Info() string {
return fmt.Sprintf("%d sessions", w.numberOfSessions())
}
func (w *SessionWriter) addSession(sid uint64) {
w.lock.Lock()
w.meta[sid] = time.Now().Unix()
w.lock.Unlock()
}
func (w *SessionWriter) deleteSession(sid uint64) {
w.lock.Lock()
delete(w.meta, sid)
w.lock.Unlock()
}
func (w *SessionWriter) numberOfSessions() int {
w.lock.Lock()
defer w.lock.Unlock()
return len(w.meta)
}
func (w *SessionWriter) write(sid uint64, mode FileType, data []byte) error {
var (
sess *Session
err error
)
sessObj, ok := w.sessions.Load(sid)
if !ok {
sess, err = NewSession(w.dir, sid)
if err != nil {
return fmt.Errorf("can't write to session: %d, err: %s", sid, err)
}
sess.Lock()
defer sess.Unlock()
// Check opened files limit
if len(w.meta) >= w.ulimit {
var oldSessID uint64
var minTimestamp int64 = math.MaxInt64
for sessID, timestamp := range w.meta {
if timestamp < minTimestamp {
oldSessID = sessID
minTimestamp = timestamp
}
}
if err := w.close(oldSessID); err != nil {
log.Printf("can't close session: %s", err)
}
}
// Add new session to manager
w.sessions.Store(sid, sess)
w.addSession(sid)
} else {
sess = sessObj.(*Session)
sess.Lock()
defer sess.Unlock()
}
// Write data to session
return sess.Write(mode, data)
}
func (w *SessionWriter) sync(sid uint64) error {
sessObj, ok := w.sessions.Load(sid)
if !ok {
return fmt.Errorf("can't sync, session: %d not found", sid)
}
sess := sessObj.(*Session)
sess.Lock()
defer sess.Unlock()
err := sess.Sync()
if time.Now().Sub(sess.LastUpdate()).Seconds() > w.zombieSessionTimeout {
if err != nil {
log.Printf("can't sync session: %d, err: %s", sid, err)
}
// Close "zombie" session
err = sess.Close()
w.deleteSession(sid)
}
return err
}
func (w *SessionWriter) close(sid uint64) error {
sessObj, ok := w.sessions.LoadAndDelete(sid)
if !ok {
return fmt.Errorf("can't close, session: %d not found", sid)
}
sess := sessObj.(*Session)
sess.Lock()
defer sess.Unlock()
if err := sess.Sync(); err != nil {
log.Printf("can't sync session: %d, err: %s", sid, err)
}
err := sess.Close()
w.deleteSession(sid)
return err
}
func (w *SessionWriter) synchronizer() {
tick := time.Tick(2 * time.Second)
for {
select {
case <-tick:
w.sessions.Range(func(sid, lockObj any) bool {
if err := w.sync(sid.(uint64)); err != nil {
log.Printf("can't sync file descriptor: %s", err)
}
return true
})
case <-w.done:
w.sessions.Range(func(sid, lockObj any) bool {
if err := w.close(sid.(uint64)); err != nil {
log.Printf("can't close file descriptor: %s", err)
}
return true
})
w.stopped <- struct{}{}
return
}
}
}

65
backend/internal/storage/storage.go
View file

@ -13,7 +13,6 @@ import (
"openreplay/backend/pkg/storage"
"os"
"strconv"
"strings"
"time"
)
@ -71,43 +70,46 @@ func New(cfg *config.Config, s3 *storage.S3, metrics *monitoring.Metrics) (*Stor
}
func (s *Storage) UploadSessionFiles(msg *messages.SessionEnd) error {
sessionDir := strconv.FormatUint(msg.SessionID(), 10)
if err := s.uploadKey(msg.SessionID(), sessionDir+"/dom.mob", true, 5, msg.EncryptionKey); err != nil {
oldErr := s.uploadKey(msg.SessionID(), sessionDir, true, 5, msg.EncryptionKey)
if oldErr != nil {
return fmt.Errorf("upload file error: %s. failed checking mob file using old path: %s", err, oldErr)
}
// Exit method anyway because we don't have dev tools separation in prev version
return nil
}
if err := s.uploadKey(msg.SessionID(), sessionDir+"/devtools.mob", false, 4, msg.EncryptionKey); err != nil {
if err := s.uploadKey(msg.SessionID(), "/dom.mob", true, 5, msg.EncryptionKey); err != nil {
return err
}
if err := s.uploadKey(msg.SessionID(), "/devtools.mob", false, 4, msg.EncryptionKey); err != nil {
log.Printf("can't find devtools for session: %d, err: %s", msg.SessionID(), err)
}
return nil
}
// TODO: make a bit cleaner
func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCount int, encryptionKey string) error {
// TODO: make a bit cleaner.
// TODO: Of course, I'll do!
func (s *Storage) uploadKey(sessID uint64, suffix string, shouldSplit bool, retryCount int, encryptionKey string) error {
if retryCount <= 0 {
return nil
}
start := time.Now()
file, err := os.Open(s.cfg.FSDir + "/" + key)
fileName := strconv.FormatUint(sessID, 10)
mobFileName := fileName
if suffix == "/devtools.mob" {
mobFileName += "devtools"
}
filePath := s.cfg.FSDir + "/" + mobFileName
// Check file size before download into memory
info, err := os.Stat(filePath)
if err == nil {
if info.Size() > s.cfg.MaxFileSize {
log.Printf("big file, size: %d, session: %d", info.Size(), sessID)
return nil
}
}
file, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("File open error: %v; sessID: %s, part: %d, sessStart: %s\n",
err, key, sessID%16,
err, fileName, sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
}
defer file.Close()
// Ignore "s" at the end of mob file name for "old" sessions
newVers := false
if strings.Contains(key, "/") {
newVers = true
}
var fileSize int64 = 0
fileInfo, err := file.Stat()
if err != nil {
@ -117,17 +119,18 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo
}
var encryptedData []byte
fileName += suffix
if shouldSplit {
nRead, err := file.Read(s.startBytes)
if err != nil {
log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s",
err,
key,
fileName,
sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
time.AfterFunc(s.cfg.RetryTimeout, func() {
s.uploadKey(sessID, key, shouldSplit, retryCount-1, encryptionKey)
s.uploadKey(sessID, suffix, shouldSplit, retryCount-1, encryptionKey)
})
return nil
}
@ -146,11 +149,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo
}
// Compress and save to s3
startReader := bytes.NewBuffer(encryptedData)
startKey := key
if newVers {
startKey += "s"
}
if err := s.s3.Upload(s.gzipFile(startReader), startKey, "application/octet-stream", true); err != nil {
if err := s.s3.Upload(s.gzipFile(startReader), fileName+"s", "application/octet-stream", true); err != nil {
log.Fatalf("Storage: start upload failed. %v\n", err)
}
// TODO: fix possible error (if we read less then FileSplitSize)
@ -161,7 +160,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo
if err != nil {
log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s",
err,
key,
fileName,
sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
@ -183,7 +182,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo
}
// Compress and save to s3
endReader := bytes.NewBuffer(encryptedData)
if err := s.s3.Upload(s.gzipFile(endReader), key+"e", "application/octet-stream", true); err != nil {
if err := s.s3.Upload(s.gzipFile(endReader), fileName+"e", "application/octet-stream", true); err != nil {
log.Fatalf("Storage: end upload failed. %v\n", err)
}
}
@ -195,7 +194,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo
if err != nil {
log.Printf("File read error: %s; sessID: %s, part: %d, sessStart: %s",
err,
key,
fileName,
sessID%16,
time.UnixMilli(int64(flakeid.ExtractTimestamp(sessID))),
)
@ -216,7 +215,7 @@ func (s *Storage) uploadKey(sessID uint64, key string, shouldSplit bool, retryCo
encryptedData = fileData
}
endReader := bytes.NewBuffer(encryptedData)
if err := s.s3.Upload(s.gzipFile(endReader), key+"s", "application/octet-stream", true); err != nil {
if err := s.s3.Upload(s.gzipFile(endReader), fileName, "application/octet-stream", true); err != nil {
log.Fatalf("Storage: end upload failed. %v\n", err)
}
s.archivingTime.Record(context.Background(), float64(time.Now().Sub(start).Milliseconds()))

22
backend/pkg/db/types/error-event.go
View file

@ -11,6 +11,8 @@ import (
. "openreplay/backend/pkg/messages"
)
const SOURCE_JS = "js_exception"
type ErrorEvent struct {
MessageID uint64
Timestamp uint64
@ -64,7 +66,7 @@ func WrapJSException(m *JSException) *ErrorEvent {
return &ErrorEvent{
MessageID: m.Meta().Index,
Timestamp: uint64(m.Meta().Timestamp),
Source: "js_exception",
Source: SOURCE_JS,
Name: m.Name,
Message: m.Message,
Payload: m.Payload,
@ -105,14 +107,16 @@ func (e *ErrorEvent) ID(projectID uint32) string {
hash.Write([]byte(e.Source))
hash.Write([]byte(e.Name))
hash.Write([]byte(e.Message))
frame, err := parseFirstFrame(e.Payload)
if err != nil {
log.Printf("Can't parse stackframe ((( %v ))): %v", e.Payload, err)
}
if frame != nil {
hash.Write([]byte(frame.FileName))
hash.Write([]byte(strconv.Itoa(frame.LineNo)))
hash.Write([]byte(strconv.Itoa(frame.ColNo)))
if e.Source == SOURCE_JS {
frame, err := parseFirstFrame(e.Payload)
if err != nil {
log.Printf("Can't parse stackframe ((( %v ))): %v", e.Payload, err)
}
if frame != nil {
hash.Write([]byte(frame.FileName))
hash.Write([]byte(strconv.Itoa(frame.LineNo)))
hash.Write([]byte(strconv.Itoa(frame.ColNo)))
}
}
return strconv.FormatUint(uint64(projectID), 16) + hex.EncodeToString(hash.Sum(nil))
}

15
ee/api/chalicelib/core/alerts_processor.py
View file

@ -204,7 +204,8 @@ def process():
logging.info(f"Valid alert, notifying users, alertId:{alert['alertId']} name: {alert['name']}")
notifications.append(generate_notification(alert, result))
except Exception as e:
logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}")
logging.error(
f"!!!Error while running alert query for alertId:{alert['alertId']} name: {alert['name']}")
logging.error(query)
logging.error(e)
cur = cur.recreate(rollback=True)
@ -217,12 +218,22 @@ def process():
alerts.process_notifications(notifications)
def __format_value(x):
if x % 1 == 0:
x = int(x)
else:
x = round(x, 2)
return f"{x:,}"
def generate_notification(alert, result):
left = __format_value(result['value'])
right = __format_value(alert['query']['right'])
return {
"alertId": alert["alertId"],
"tenantId": alert["tenantId"],
"title": alert["name"],
"description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).",
"description": f"has been triggered, {alert['query']['left']} = {left} ({alert['query']['operator']} {right}).",
"buttonText": "Check metrics for more details",
"buttonUrl": f"/{alert['projectId']}/metrics",
"imageUrl": None,

26
ee/api/chalicelib/core/metrics.py
View file

@ -452,18 +452,18 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
ch_sub_query.append("resources.type = 'img'")
ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args)
ch_sub_query_chart.append("resources.type = 'img'")
ch_sub_query_chart.append("resources.url IN %(url)s")
ch_sub_query_chart.append("resources.url_hostpath IN %(url)s")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
ch_sub_query_chart += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT resources.url,
ch_query = f"""SELECT resources.url_hostpath AS url,
COALESCE(avgOrNull(resources.duration),0) AS avg,
COUNT(1) AS count
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query)} AND resources.duration>0
GROUP BY resources.url ORDER BY avg DESC LIMIT 10;"""
GROUP BY resources.url_hostpath ORDER BY avg DESC LIMIT 10;"""
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)}
rows = ch.execute(query=ch_query, params=params)
@ -474,13 +474,13 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
urls = [row["url"] for row in rows]
charts = {}
ch_query = f"""SELECT url,
ch_query = f"""SELECT url_hostpath AS url,
toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
COALESCE(avgOrNull(resources.duration),0) AS avg
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)} AND resources.duration>0
GROUP BY url, timestamp
ORDER BY url, timestamp;"""
GROUP BY url_hostpath, timestamp
ORDER BY url_hostpath, timestamp;"""
params["url"] = urls
u_rows = ch.execute(query=ch_query, params=params)
for url in urls:
@ -526,13 +526,13 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi
if resources and len(resources) > 0:
for r in resources:
if r["type"] == "IMG":
img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s")
img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s")
img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value']
elif r["type"] == "LOCATION":
location_constraints.append(f"pages.url_path = %(val_{len(location_constraints)})s")
location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value']
else:
request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s")
request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s")
request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value']
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
@ -638,7 +638,7 @@ def search(text, resource_type, project_id, performance=False, pages_only=False,
if resource_type == "ALL" and not pages_only and not events_only:
ch_sub_query.append("positionUTF8(url_hostpath,%(value)s)!=0")
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url))), 1, 5)) AS value,
ch_query = f"""SELECT arrayJoin(arraySlice(arrayReverseSort(arrayDistinct(groupArray(url_hostpath))), 1, 5)) AS value,
type AS key
FROM resources
WHERE {" AND ".join(ch_sub_query)}
@ -884,7 +884,7 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days
if type is not None:
ch_sub_query_chart.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'")
if url is not None:
ch_sub_query_chart.append(f"resources.url = %(value)s")
ch_sub_query_chart.append(f"resources.url_hostpath = %(value)s")
meta_condition = __get_meta_constraint(args)
ch_sub_query_chart += meta_condition
ch_sub_query_chart.append("resources.duration>0")
@ -966,7 +966,7 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
ch_sub_query_chart.append("isNotNull(resources.duration)")
ch_sub_query_chart.append("resources.duration>0")
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT any(url) AS url, any(type) AS type,
ch_query = f"""SELECT any(url_hostpath) AS url, any(type) AS type,
splitByChar('/', resources.url_hostpath)[-1] AS name,
COALESCE(avgOrNull(NULLIF(resources.duration,0)),0) AS avg
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
@ -2179,7 +2179,7 @@ def get_performance_avg_image_load_time(ch, project_id, startTimestamp=TimeUTC.n
if resources and len(resources) > 0:
for r in resources:
if r["type"] == "IMG":
img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s")
img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s")
img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value']
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
@ -2254,7 +2254,7 @@ def get_performance_avg_request_load_time(ch, project_id, startTimestamp=TimeUTC
if resources and len(resources) > 0:
for r in resources:
if r["type"] != "IMG" and r["type"] == "LOCATION":
request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s")
request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s")
request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value']
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}

24
ee/api/chalicelib/core/metrics_exp.py
View file

@ -462,18 +462,18 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
ch_sub_query_chart = __get_basic_constraints(table_name="resources", round_start=True, data=args)
# ch_sub_query_chart.append("events.event_type='RESOURCE'")
ch_sub_query_chart.append("resources.type = 'img'")
ch_sub_query_chart.append("resources.url IN %(url)s")
ch_sub_query_chart.append("resources.url_hostpath IN %(url)s")
meta_condition = __get_meta_constraint(args)
ch_sub_query += meta_condition
ch_sub_query_chart += meta_condition
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT resources.url,
ch_query = f"""SELECT resources.url_hostpath AS url,
COALESCE(avgOrNull(resources.duration),0) AS avg,
COUNT(1) AS count
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
WHERE {" AND ".join(ch_sub_query)} AND resources.duration>0
GROUP BY resources.url ORDER BY avg DESC LIMIT 10;"""
GROUP BY resources.url_hostpath ORDER BY avg DESC LIMIT 10;"""
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)}
rows = ch.execute(query=ch_query, params=params)
@ -484,13 +484,13 @@ def get_slowest_images(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
urls = [row["url"] for row in rows]
charts = {}
ch_query = f"""SELECT url,
ch_query = f"""SELECT url_hostpath AS url,
toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
COALESCE(avgOrNull(resources.duration),0) AS avg
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
WHERE {" AND ".join(ch_sub_query_chart)} AND resources.duration>0
GROUP BY url, timestamp
ORDER BY url, timestamp;"""
GROUP BY url_hostpath, timestamp
ORDER BY url_hostpath, timestamp;"""
params["url"] = urls
# print(ch.format(query=ch_query, params=params))
u_rows = ch.execute(query=ch_query, params=params)
@ -538,13 +538,13 @@ def get_performance(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTi
if resources and len(resources) > 0:
for r in resources:
if r["type"] == "IMG":
img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s")
img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s")
img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value']
elif r["type"] == "LOCATION":
location_constraints.append(f"pages.url_path = %(val_{len(location_constraints)})s")
location_constraints_vals["val_" + str(len(location_constraints) - 1)] = r['value']
else:
request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s")
request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s")
request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value']
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
@ -891,7 +891,7 @@ def get_resources_loading_time(project_id, startTimestamp=TimeUTC.now(delta_days
if type is not None:
ch_sub_query_chart.append(f"resources.type = '{__get_resource_db_type_from_type(type)}'")
if url is not None:
ch_sub_query_chart.append(f"resources.url = %(value)s")
ch_sub_query_chart.append(f"resources.url_hostpath = %(value)s")
meta_condition = __get_meta_constraint(args)
ch_sub_query_chart += meta_condition
ch_sub_query_chart.append("resources.duration>0")
@ -974,7 +974,7 @@ def get_slowest_resources(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
ch_sub_query_chart.append("isNotNull(resources.duration)")
ch_sub_query_chart.append("resources.duration>0")
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT any(url) AS url, any(type) AS type, name,
ch_query = f"""SELECT any(url_hostpath) AS url, any(type) AS type, name,
COALESCE(avgOrNull(NULLIF(resources.duration,0)),0) AS avg
FROM {exp_ch_helper.get_main_resources_table(startTimestamp)} AS resources
WHERE {" AND ".join(ch_sub_query)}
@ -2185,7 +2185,7 @@ def get_performance_avg_image_load_time(ch, project_id, startTimestamp=TimeUTC.n
if resources and len(resources) > 0:
for r in resources:
if r["type"] == "IMG":
img_constraints.append(f"resources.url = %(val_{len(img_constraints)})s")
img_constraints.append(f"resources.url_hostpath = %(val_{len(img_constraints)})s")
img_constraints_vals["val_" + str(len(img_constraints) - 1)] = r['value']
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
@ -2260,7 +2260,7 @@ def get_performance_avg_request_load_time(ch, project_id, startTimestamp=TimeUTC
if resources and len(resources) > 0:
for r in resources:
if r["type"] != "IMG" and r["type"] == "LOCATION":
request_constraints.append(f"resources.url = %(val_{len(request_constraints)})s")
request_constraints.append(f"resources.url_hostpath = %(val_{len(request_constraints)})s")
request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value']
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}

16
ee/api/chalicelib/core/sessions.py
View file

@ -107,8 +107,7 @@ def get_by_id2_pg(project_id, session_id, context: schemas_ee.CurrentContext, fu
session_id=session_id, user_id=context.user_id)
data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data)
data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id)
data['live'] = live and assist.is_live(project_id=project_id,
session_id=session_id,
data['live'] = live and assist.is_live(project_id=project_id, session_id=session_id,
project_key=data["projectKey"])
data["inDB"] = True
return data
@ -181,7 +180,7 @@ def _isUndefined_operator(op: schemas.SearchEventOperator):
# This function executes the query and return result
def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, errors_only=False,
error_status=schemas.ErrorStatus.all, count_only=False, issue=None):
error_status=schemas.ErrorStatus.all, count_only=False, issue=None, ids_only=False):
if data.bookmarked:
data.startDate, data.endDate = sessions_favorite.get_start_end_timestamp(project_id, user_id)
@ -189,9 +188,11 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
favorite_only=data.bookmarked, issue=issue, project_id=project_id,
user_id=user_id)
if data.limit is not None and data.page is not None:
full_args["sessions_limit"] = data.limit
full_args["sessions_limit_s"] = (data.page - 1) * data.limit
full_args["sessions_limit_e"] = data.page * data.limit
else:
full_args["sessions_limit"] = 200
full_args["sessions_limit_s"] = 1
full_args["sessions_limit_e"] = 200
@ -239,6 +240,12 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
GROUP BY user_id
) AS users_sessions;""",
full_args)
elif ids_only:
main_query = cur.mogrify(f"""SELECT DISTINCT ON(s.session_id) s.session_id
{query_part}
ORDER BY s.session_id desc
LIMIT %(sessions_limit)s OFFSET %(sessions_limit_s)s;""",
full_args)
else:
if data.order is None:
data.order = schemas.SortOrderType.desc
@ -246,7 +253,6 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
if data.sort is not None and data.sort != "session_id":
# sort += " " + data.order + "," + helper.key_to_snake_case(data.sort)
sort = helper.key_to_snake_case(data.sort)
meta_keys = metadata.get(project_id=project_id)
main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count,
COALESCE(JSONB_AGG(full_sessions)
@ -270,7 +276,7 @@ def search_sessions(data: schemas.SessionsSearchPayloadSchema, project_id, user_
print(data.json())
print("--------------------")
raise err
if errors_only:
if errors_only or ids_only:
return helper.list_to_camel_case(cur.fetchall())
sessions = cur.fetchone()

4
ee/api/chalicelib/core/sessions_devtool.py
View file

@ -3,7 +3,7 @@ from fastapi.security import SecurityScopes
import schemas_ee
from chalicelib.core import permissions
from chalicelib.utils.s3 import client
from chalicelib.utils import s3
SCOPES = SecurityScopes([schemas_ee.Permissions.dev_tools])
@ -23,7 +23,7 @@ def get_urls(session_id, project_id, context: schemas_ee.CurrentContext):
return []
results = []
for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
results.append(client.generate_presigned_url(
results.append(s3.client.generate_presigned_url(
'get_object',
Params={'Bucket': config("sessions_bucket"), 'Key': k},
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)

49
ee/api/chalicelib/core/sessions_favorite.py
View file

@ -1,7 +1,7 @@
from decouple import config
import schemas_ee
from chalicelib.core import sessions, sessions_favorite_exp
from chalicelib.core import sessions, sessions_favorite_exp, sessions_mobs, sessions_devtool
from chalicelib.utils import pg_client, s3_extra
@ -34,32 +34,31 @@ def remove_favorite_session(context: schemas_ee.CurrentContext, project_id, sess
def favorite_session(context: schemas_ee.CurrentContext, project_id, session_id):
keys = sessions_mobs.__get_mob_keys(project_id=project_id, session_id=session_id)
keys += sessions_mobs.__get_mob_keys_deprecated(session_id=session_id) # To support old sessions
keys += sessions_devtool.__get_devtools_keys(project_id=project_id, session_id=session_id)
if favorite_session_exists(user_id=context.user_id, session_id=session_id):
key = str(session_id)
try:
s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_D_VALUE', default='default'))
except Exception as e:
print(f"!!!Error while tagging: {key} to default")
print(str(e))
key = str(session_id) + "e"
try:
s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_D_VALUE', default='default'))
except Exception as e:
print(f"!!!Error while tagging: {key} to default")
print(str(e))
tag = config('RETENTION_D_VALUE', default='default')
for k in keys:
try:
s3_extra.tag_session(file_key=k, tag_value=tag)
except Exception as e:
print(f"!!!Error while tagging: {k} to {tag} for removal")
print(str(e))
return remove_favorite_session(context=context, project_id=project_id, session_id=session_id)
key = str(session_id)
try:
s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_L_VALUE', default='vault'))
except Exception as e:
print(f"!!!Error while tagging: {key} to vault")
print(str(e))
key = str(session_id) + "e"
try:
s3_extra.tag_file(session_id=key, tag_value=config('RETENTION_L_VALUE', default='vault'))
except Exception as e:
print(f"!!!Error while tagging: {key} to vault")
print(str(e))
tag = config('RETENTION_L_VALUE', default='vault')
for k in keys:
try:
s3_extra.tag_session(file_key=k, tag_value=tag)
except Exception as e:
print(f"!!!Error while tagging: {k} to {tag} for vault")
print(str(e))
return add_favorite_session(context=context, project_id=project_id, session_id=session_id)

158
ee/api/chalicelib/core/significance.py
View file

@ -188,9 +188,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
values=s["value"], value_key=f"value{i + 1}")
n_stages_query.append(f"""
(SELECT main.session_id,
{"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp,
'{event_type}' AS type,
'{s["operator"]}' AS operator
{"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp
FROM {next_table} AS main {" ".join(extra_from)}
WHERE main.timestamp >= {f"T{i}.stage{i}_timestamp" if i > 0 else "%(startTimestamp)s"}
{f"AND main.session_id=T1.session_id" if i > 0 else ""}
@ -198,45 +196,54 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]:
{(" AND " + " AND ".join(stage_constraints)) if len(stage_constraints) > 0 else ""}
{(" AND " + " AND ".join(first_stage_extra_constraints)) if len(first_stage_extra_constraints) > 0 and i == 0 else ""}
GROUP BY main.session_id)
AS T{i + 1} {"USING (session_id)" if i > 0 else ""}
AS T{i + 1} {"ON (TRUE)" if i > 0 else ""}
""")
if len(n_stages_query) == 0:
n_stages=len(n_stages_query)
if n_stages == 0:
return []
n_stages_query = " LEFT JOIN LATERAL ".join(n_stages_query)
n_stages_query += ") AS stages_t"
n_stages_query = f"""
SELECT stages_and_issues_t.*,sessions.session_id, sessions.user_uuid FROM (
SELECT stages_and_issues_t.*, sessions.user_uuid
FROM (
SELECT * FROM (
SELECT * FROM
{n_stages_query}
SELECT T1.session_id, {",".join([f"stage{i + 1}_timestamp" for i in range(n_stages)])}
FROM {n_stages_query}
LEFT JOIN LATERAL
(
SELECT * FROM
(SELECT ISE.session_id,
ISS.type as issue_type,
( SELECT ISS.type as issue_type,
ISE.timestamp AS issue_timestamp,
ISS.context_string as issue_context,
COALESCE(ISS.context_string,'') as issue_context,
ISS.issue_id as issue_id
FROM events_common.issues AS ISE INNER JOIN issues AS ISS USING (issue_id)
WHERE ISE.timestamp >= stages_t.stage1_timestamp
AND ISE.timestamp <= stages_t.stage{i + 1}_timestamp
AND ISS.project_id=%(project_id)s
{"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""}) AS base_t
) AS issues_t
USING (session_id)) AS stages_and_issues_t
inner join sessions USING(session_id);
AND ISE.session_id = stages_t.session_id
{"AND ISS.type IN %(issueTypes)s" if len(filter_issues) > 0 else ""}
LIMIT 20 -- remove the limit to get exact stats
) AS issues_t ON (TRUE)
) AS stages_and_issues_t INNER JOIN sessions USING(session_id);
"""
# LIMIT 10000
params = {"project_id": project_id, "startTimestamp": filter_d["startDate"], "endTimestamp": filter_d["endDate"],
"issueTypes": tuple(filter_issues), **values}
with pg_client.PostgresClient() as cur:
query = cur.mogrify(n_stages_query, params)
# print("---------------------------------------------------")
# print(cur.mogrify(n_stages_query, params))
# print(query)
# print("---------------------------------------------------")
cur.execute(cur.mogrify(n_stages_query, params))
rows = cur.fetchall()
try:
cur.execute(query)
rows = cur.fetchall()
except Exception as err:
print("--------- FUNNEL SEARCH QUERY EXCEPTION -----------")
print(query.decode('UTF-8'))
print("--------- PAYLOAD -----------")
print(filter_d)
print("--------------------")
raise err
return rows
@ -298,7 +305,21 @@ def pearson_corr(x: list, y: list):
return r, confidence, False
def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_with_context, first_stage, last_stage):
# def tuple_or(t: tuple):
# x = 0
# for el in t:
# x |= el # | is for bitwise OR
# return x
#
# The following function is correct optimization of the previous function because t is a list of 0,1
def tuple_or(t: tuple):
for el in t:
if el > 0:
return 1
return 0
def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues, first_stage, last_stage):
"""
Returns two lists with binary values 0/1:
@ -317,12 +338,6 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_
transitions = []
n_sess_affected = 0
errors = {}
for issue in all_issues_with_context:
split = issue.split('__^__')
errors[issue] = {
"errors": [],
"issue_type": split[0],
"context": split[1]}
for row in rows:
t = 0
@ -330,38 +345,26 @@ def get_transitions_and_issues_of_each_type(rows: List[RealDictRow], all_issues_
last_ts = row[f'stage{last_stage}_timestamp']
if first_ts is None:
continue
elif first_ts is not None and last_ts is not None:
elif last_ts is not None:
t = 1
transitions.append(t)
ic_present = False
for issue_type_with_context in errors:
for error_id in all_issues:
if error_id not in errors:
errors[error_id] = []
ic = 0
issue_type = errors[issue_type_with_context]["issue_type"]
context = errors[issue_type_with_context]["context"]
if row['issue_type'] is not None:
row_issue_id=row['issue_id']
if row_issue_id is not None:
if last_ts is None or (first_ts < row['issue_timestamp'] < last_ts):
context_in_row = row['issue_context'] if row['issue_context'] is not None else ''
if issue_type == row['issue_type'] and context == context_in_row:
if error_id == row_issue_id:
ic = 1
ic_present = True
errors[issue_type_with_context]["errors"].append(ic)
errors[error_id].append(ic)
if ic_present and t:
n_sess_affected += 1
# def tuple_or(t: tuple):
# x = 0
# for el in t:
# x |= el
# return x
def tuple_or(t: tuple):
for el in t:
if el > 0:
return 1
return 0
errors = {key: errors[key]["errors"] for key in errors}
all_errors = [tuple_or(t) for t in zip(*errors.values())]
return transitions, errors, all_errors, n_sess_affected
@ -377,10 +380,9 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage):
"""
affected_users = defaultdict(lambda: set())
affected_sessions = defaultdict(lambda: set())
contexts = defaultdict(lambda: None)
all_issues = {}
n_affected_users_dict = defaultdict(lambda: None)
n_affected_sessions_dict = defaultdict(lambda: None)
all_issues_with_context = set()
n_issues_dict = defaultdict(lambda: 0)
issues_by_session = defaultdict(lambda: 0)
@ -396,15 +398,13 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage):
# check that the issue exists and belongs to subfunnel:
if iss is not None and (row[f'stage{last_stage}_timestamp'] is None or
(row[f'stage{first_stage}_timestamp'] < iss_ts < row[f'stage{last_stage}_timestamp'])):
context_string = row['issue_context'] if row['issue_context'] is not None else ''
issue_with_context = iss + '__^__' + context_string
contexts[issue_with_context] = {"context": context_string, "id": row["issue_id"]}
all_issues_with_context.add(issue_with_context)
n_issues_dict[issue_with_context] += 1
if row["issue_id"] not in all_issues:
all_issues[row["issue_id"]] = {"context": row['issue_context'], "issue_type": row["issue_type"]}
n_issues_dict[row["issue_id"]] += 1
if row['user_uuid'] is not None:
affected_users[issue_with_context].add(row['user_uuid'])
affected_users[row["issue_id"]].add(row['user_uuid'])
affected_sessions[issue_with_context].add(row['session_id'])
affected_sessions[row["issue_id"]].add(row['session_id'])
issues_by_session[row[f'session_id']] += 1
if len(affected_users) > 0:
@ -415,29 +415,28 @@ def get_affected_users_for_all_issues(rows, first_stage, last_stage):
n_affected_sessions_dict.update({
iss: len(affected_sessions[iss]) for iss in affected_sessions
})
return all_issues_with_context, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict, contexts
return all_issues, n_issues_dict, n_affected_users_dict, n_affected_sessions_dict
def count_sessions(rows, n_stages):
session_counts = {i: set() for i in range(1, n_stages + 1)}
for ind, row in enumerate(rows):
for row in rows:
for i in range(1, n_stages + 1):
if row[f"stage{i}_timestamp"] is not None:
session_counts[i].add(row[f"session_id"])
session_counts = {i: len(session_counts[i]) for i in session_counts}
return session_counts
def count_users(rows, n_stages):
users_in_stages = defaultdict(lambda: set())
for ind, row in enumerate(rows):
users_in_stages = {i: set() for i in range(1, n_stages + 1)}
for row in rows:
for i in range(1, n_stages + 1):
if row[f"stage{i}_timestamp"] is not None:
users_in_stages[i].add(row["user_uuid"])
users_count = {i: len(users_in_stages[i]) for i in range(1, n_stages + 1)}
return users_count
@ -490,18 +489,18 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False)
last_stage = n_stages
n_critical_issues = 0
issues_dict = dict({"significant": [],
"insignificant": []})
issues_dict = {"significant": [],
"insignificant": []}
session_counts = count_sessions(rows, n_stages)
drop = session_counts[first_stage] - session_counts[last_stage]
all_issues_with_context, n_issues_dict, affected_users_dict, affected_sessions, contexts = get_affected_users_for_all_issues(
all_issues, n_issues_dict, affected_users_dict, affected_sessions = get_affected_users_for_all_issues(
rows, first_stage, last_stage)
transitions, errors, all_errors, n_sess_affected = get_transitions_and_issues_of_each_type(rows,
all_issues_with_context,
all_issues,
first_stage, last_stage)
# print("len(transitions) =", len(transitions))
del rows
if any(all_errors):
total_drop_corr, conf, is_sign = pearson_corr(transitions, all_errors)
@ -514,33 +513,32 @@ def get_issues(stages, rows, first_stage=None, last_stage=None, drop_only=False)
if drop_only:
return total_drop_due_to_issues
for issue in all_issues_with_context:
for issue_id in all_issues:
if not any(errors[issue]):
if not any(errors[issue_id]):
continue
r, confidence, is_sign = pearson_corr(transitions, errors[issue])
r, confidence, is_sign = pearson_corr(transitions, errors[issue_id])
if r is not None and drop is not None and is_sign:
lost_conversions = int(r * affected_sessions[issue])
lost_conversions = int(r * affected_sessions[issue_id])
else:
lost_conversions = None
if r is None:
r = 0
split = issue.split('__^__')
issues_dict['significant' if is_sign else 'insignificant'].append({
"type": split[0],
"title": helper.get_issue_title(split[0]),
"affected_sessions": affected_sessions[issue],
"unaffected_sessions": session_counts[1] - affected_sessions[issue],
"type": all_issues[issue_id]["issue_type"],
"title": helper.get_issue_title(all_issues[issue_id]["issue_type"]),
"affected_sessions": affected_sessions[issue_id],
"unaffected_sessions": session_counts[1] - affected_sessions[issue_id],
"lost_conversions": lost_conversions,
"affected_users": affected_users_dict[issue],
"affected_users": affected_users_dict[issue_id],
"conversion_impact": round(r * 100),
"context_string": contexts[issue]["context"],
"issue_id": contexts[issue]["id"]
"context_string": all_issues[issue_id]["context"],
"issue_id": issue_id
})
if is_sign:
n_critical_issues += n_issues_dict[issue]
n_critical_issues += n_issues_dict[issue_id]
return n_critical_issues, issues_dict, total_drop_due_to_issues

14
ee/api/chalicelib/utils/s3_extra.py
View file

@ -1,12 +1,16 @@
from decouple import config
from chalicelib.utils.s3 import client
from chalicelib.utils import s3
def tag_file(session_id, tag_key='retention', tag_value='vault'):
return client.put_object_tagging(
Bucket=config("sessions_bucket"),
Key=session_id,
def tag_session(file_key, tag_key='retention', tag_value='vault'):
return tag_file(file_key=file_key, bucket=config("sessions_bucket"), tag_key=tag_key, tag_value=tag_value)
def tag_file(file_key, bucket, tag_key, tag_value):
return s3.client.put_object_tagging(
Bucket=bucket,
Key=file_key,
Tagging={
'TagSet': [
{

6
ee/api/env.default
View file

@ -61,11 +61,11 @@ EXP_ALERTS=false
EXP_FUNNELS=false
EXP_RESOURCES=true
TRACE_PERIOD=300
EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob
EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
EFS_SESSION_MOB_PATTERN=%(sessionId)s
EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)sdevtools
SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs
SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe
DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mobs
DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
PRESIGNED_URL_EXPIRATION=3600
ASSIST_JWT_EXPIRATION=144000
ASSIST_JWT_SECRET=

10
ee/api/requirements-alerts.txt
View file

@ -1,18 +1,18 @@
requests==2.28.1
urllib3==1.26.12
boto3==1.26.4
boto3==1.26.14
pyjwt==2.6.0
psycopg2-binary==2.9.5
elasticsearch==8.5.0
elasticsearch==8.5.1
jira==3.4.1
fastapi==0.86.0
uvicorn[standard]==0.19.0
fastapi==0.87.0
uvicorn[standard]==0.20.0
python-decouple==3.6
pydantic[email]==1.10.2
apscheduler==3.9.1
apscheduler==3.9.1.post1
clickhouse-driver==0.2.4
python-multipart==0.0.5

10
ee/api/requirements-crons.txt
View file

@ -1,18 +1,18 @@
requests==2.28.1
urllib3==1.26.12
boto3==1.26.4
boto3==1.26.14
pyjwt==2.6.0
psycopg2-binary==2.9.5
elasticsearch==8.5.0
elasticsearch==8.5.1
jira==3.4.1
fastapi==0.86.0
uvicorn[standard]==0.19.0
fastapi==0.87.0
uvicorn[standard]==0.20.0
python-decouple==3.6
pydantic[email]==1.10.2
apscheduler==3.9.1
apscheduler==3.9.1.post1
clickhouse-driver==0.2.4
python-multipart==0.0.5

10
ee/api/requirements.txt
View file

@ -1,18 +1,18 @@
requests==2.28.1
urllib3==1.26.12
boto3==1.26.4
boto3==1.26.14
pyjwt==2.6.0
psycopg2-binary==2.9.5
elasticsearch==8.5.0
elasticsearch==8.5.1
jira==3.4.1
fastapi==0.86.0
uvicorn[standard]==0.19.0
fastapi==0.87.0
uvicorn[standard]==0.20.0
python-decouple==3.6
pydantic[email]==1.10.2
apscheduler==3.9.1
apscheduler==3.9.1.post1
clickhouse-driver==0.2.4
python3-saml==1.14.0

10
ee/backend/pkg/kafka/consumer.go
View file

@ -47,6 +47,16 @@ func NewConsumer(
kafkaConfig.SetKey("ssl.key.location", os.Getenv("KAFKA_SSL_KEY"))
kafkaConfig.SetKey("ssl.certificate.location", os.Getenv("KAFKA_SSL_CERT"))
}
// Apply Kerberos configuration
if env.Bool("KAFKA_USE_KERBEROS") {
kafkaConfig.SetKey("security.protocol", "sasl_plaintext")
kafkaConfig.SetKey("sasl.mechanisms", "GSSAPI")
kafkaConfig.SetKey("sasl.kerberos.service.name", os.Getenv("KERBEROS_SERVICE_NAME"))
kafkaConfig.SetKey("sasl.kerberos.principal", os.Getenv("KERBEROS_PRINCIPAL"))
kafkaConfig.SetKey("sasl.kerberos.keytab", os.Getenv("KERBEROS_KEYTAB_LOCATION"))
}
c, err := kafka.NewConsumer(kafkaConfig)
if err != nil {
log.Fatalln(err)

9
ee/backend/pkg/kafka/producer.go
View file

@ -30,6 +30,15 @@ func NewProducer(messageSizeLimit int, useBatch bool) *Producer {
kafkaConfig.SetKey("ssl.key.location", os.Getenv("KAFKA_SSL_KEY"))
kafkaConfig.SetKey("ssl.certificate.location", os.Getenv("KAFKA_SSL_CERT"))
}
// Apply Kerberos configuration
if env.Bool("KAFKA_USE_KERBEROS") {
kafkaConfig.SetKey("security.protocol", "sasl_plaintext")
kafkaConfig.SetKey("sasl.mechanisms", "GSSAPI")
kafkaConfig.SetKey("sasl.kerberos.service.name", os.Getenv("KERBEROS_SERVICE_NAME"))
kafkaConfig.SetKey("sasl.kerberos.principal", os.Getenv("KERBEROS_PRINCIPAL"))
kafkaConfig.SetKey("sasl.kerberos.keytab", os.Getenv("KERBEROS_KEYTAB_LOCATION"))
}
producer, err := kafka.NewProducer(kafkaConfig)
if err != nil {
log.Fatalln(err)

4
ee/connectors/deploy/requirements_snowflake.txt
View file

@ -1,8 +1,8 @@
pandas==1.5.1
kafka-python==2.0.2
SQLAlchemy==1.4.43
snowflake-connector-python==2.8.1
snowflake-sqlalchemy==1.4.3
snowflake-connector-python==2.8.2
snowflake-sqlalchemy==1.4.4
PyYAML
asn1crypto==1.5.1
azure-common==1.1.28

7
ee/scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql
View file

@ -70,4 +70,11 @@ WHERE deleted_at IS NOT NULL;
UPDATE roles
SET permissions=array_remove(permissions, 'ERRORS');
DROP INDEX IF EXISTS events_common.requests_url_idx;
DROP INDEX IF EXISTS events_common.requests_url_gin_idx;
DROP INDEX IF EXISTS events_common.requests_url_gin_idx2;
DROP INDEX IF EXISTS events.resources_url_gin_idx;
DROP INDEX IF EXISTS events.resources_url_idx;
COMMIT;

12
ee/scripts/schema/db/init_dbs/postgresql/init_schema.sql
View file

@ -1221,19 +1221,9 @@ $$
query text NULL,
PRIMARY KEY (session_id, timestamp, seq_index)
);
CREATE INDEX IF NOT EXISTS requests_url_idx ON events_common.requests (url);
CREATE INDEX IF NOT EXISTS requests_duration_idx ON events_common.requests (duration);
CREATE INDEX IF NOT EXISTS requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops);
CREATE INDEX IF NOT EXISTS requests_timestamp_idx ON events_common.requests (timestamp);
CREATE INDEX IF NOT EXISTS requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url,
length(url) -
(CASE
WHEN url LIKE 'http://%'
THEN 7
WHEN url LIKE 'https://%'
THEN 8
ELSE 0 END))
gin_trgm_ops);
CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE;
CREATE INDEX IF NOT EXISTS requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL;

3
frontend/app/components/Dashboard/components/Alerts/AlertListItem.tsx
View file

@ -2,6 +2,7 @@ import React from 'react';
import { Icon } from 'UI';
import { checkForRecent } from 'App/date';
import { withSiteId, alertEdit } from 'App/routes';
import { numberWithCommas } from 'App/utils';
// @ts-ignore
import { DateTime } from 'luxon';
import { withRouter, RouteComponentProps } from 'react-router-dom';
@ -108,7 +109,7 @@ function AlertListItem(props: Props) {
{' is '}
<span className="font-semibold" style={{ fontFamily: 'Menlo, Monaco, Consolas' }}>
{alert.query.operator}
{alert.query.right} {alert.metric.unit}
{numberWithCommas(alert.query.right)} {alert.metric.unit}
</span>
{' over the past '}
<span className="font-semibold" style={{ fontFamily: 'Menlo, Monaco, Consolas' }}>{getThreshold(alert.currentPeriod)}</span>

5
frontend/app/components/Dashboard/components/Alerts/NewAlert.tsx
View file

@ -122,6 +122,9 @@ const NewAlert = (props: IProps) => {
) {
remove(instance.alertId).then(() => {
props.history.push(withSiteId(alerts(), siteId));
toast.success('Alert deleted');
}).catch(() => {
toast.error('Failed to delete an alert');
});
}
};
@ -135,6 +138,8 @@ const NewAlert = (props: IProps) => {
} else {
toast.success('Alert updated');
}
}).catch(() => {
toast.error('Failed to create an alert');
});
};

2
frontend/app/components/Dashboard/components/DashboardList/DashboardsView.tsx
View file

@ -35,7 +35,7 @@ function DashboardsView({ history, siteId }: { history: any, siteId: string }) {
</div>
<div className="text-base text-disabled-text flex items-center px-6">
<Icon name="info-circle-fill" className="mr-2" size={16} />
A dashboard is a custom visualization using your OpenReplay data.
A Dashboard is a collection of Metrics that can be shared across teams.
</div>
<DashboardList />
</div>

2
frontend/app/components/Dashboard/components/MetricsView/MetricsView.tsx
View file

@ -30,7 +30,7 @@ function MetricsView({ siteId }: Props) {
</div>
<div className="text-base text-disabled-text flex items-center px-6">
<Icon name="info-circle-fill" className="mr-2" size={16} />
Create custom Metrics to capture key interactions and track KPIs.
Create custom Metrics to capture user frustrations, monitor your app's performance and track other KPIs.
</div>
<MetricsList siteId={siteId} />
</div>

6
frontend/app/components/Errors/Error/ErrorInfo.js
View file

@ -35,8 +35,10 @@ export default class ErrorInfo extends React.PureComponent {
componentDidMount() {
this.ensureInstance();
}
componentDidUpdate() {
this.ensureInstance();
componentDidUpdate(prevProps) {
if (prevProps.errorId !== this.props.errorId || prevProps.errorIdInStore !== this.props.errorIdInStore) {
this.ensureInstance();
}
}
next = () => {
const { list, errorId } = this.props;

62
frontend/app/components/Session/LivePlayer.js
View file

@ -2,13 +2,9 @@ import React from 'react';
import { useEffect, useState } from 'react';
import { connect } from 'react-redux';
import { toggleFullscreen, closeBottomBlock } from 'Duck/components/player';
import { withRequest } from 'HOCs'
import {
PlayerProvider,
} from 'Player';
import { withRequest } from 'HOCs';
import withPermissions from 'HOCs/withPermissions';
import { PlayerContext, defaultContextValue } from './playerContext';
import { createLiveWebPlayer } from 'Player';
import { makeAutoObservable } from 'mobx';
import PlayerBlockHeader from '../Session_/PlayerBlockHeader';
@ -25,13 +21,12 @@ function LivePlayer ({
request,
isEnterprise,
userEmail,
userName
userName,
}) {
const [contextValue, setContextValue] = useState<IPlayerContext>(defaultContextValue);
const [fullView, setFullView] = useState(false);
useEffect(() => {
if (!loadingCredentials) {
const sessionWithAgentData = {
...session.toJS(),
agentInfo: {
@ -53,19 +48,24 @@ function LivePlayer ({
// LAYOUT (TODO: local layout state - useContext or something..)
useEffect(() => {
const queryParams = new URLSearchParams(window.location.search);
if (queryParams.has('fullScreen') && queryParams.get('fullScreen') === 'true') {
setFullView(true);
}
if (isEnterprise) {
request();
}
return () => {
toggleFullscreen(false);
closeBottomBlock();
}
}, [])
};
}, []);
const TABS = {
EVENTS: 'User Steps',
HEATMAPS: 'Click Map',
}
};
const [activeTab, setActiveTab] = useState('');
if (!contextValue.player) return null;
@ -73,31 +73,39 @@ function LivePlayer ({
return (
<PlayerContext.Provider value={contextValue}>
<PlayerProvider>
<PlayerBlockHeader activeTab={activeTab} setActiveTab={setActiveTab} tabs={TABS} fullscreen={fullscreen}/>
{!fullView && (<PlayerBlockHeader activeTab={activeTab} setActiveTab={setActiveTab} tabs={TABS} fullscreen={fullscreen}/>)}
<div className={ styles.session } data-fullscreen={fullscreen}>
<PlayerBlock />
</div>
</PlayerProvider>
</PlayerContext.Provider>
);
};
}
export default withRequest({
initialData: null,
endpoint: '/assist/credentials',
dataWrapper: data => data,
dataWrapper: (data) => data,
dataName: 'assistCredendials',
loadingName: 'loadingCredentials',
})(withPermissions(['ASSIST_LIVE'], '', true)(connect(
state => {
return {
session: state.getIn([ 'sessions', 'current' ]),
showAssist: state.getIn([ 'sessions', 'showChatWindow' ]),
fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]),
isEnterprise: state.getIn([ 'user', 'account', 'edition' ]) === 'ee',
userEmail: state.getIn(['user', 'account', 'email']),
userName: state.getIn(['user', 'account', 'name']),
}
},
{ toggleFullscreen, closeBottomBlock },
)(LivePlayer)));
})(
withPermissions(
['ASSIST_LIVE'],
'',
true
)(
connect(
(state) => {
return {
session: state.getIn(['sessions', 'current']),
showAssist: state.getIn(['sessions', 'showChatWindow']),
fullscreen: state.getIn(['components', 'player', 'fullscreen']),
isEnterprise: state.getIn(['user', 'account', 'edition']) === 'ee',
userEmail: state.getIn(['user', 'account', 'email']),
userName: state.getIn(['user', 'account', 'name']),
};
},
{ toggleFullscreen, closeBottomBlock }
)(LivePlayer)
)
);

51
frontend/app/components/Session_/Console/ConsoleRow/ConsoleRow.tsx Normal file
View file

@ -0,0 +1,51 @@
import React, { useState } from 'react';
import cn from 'classnames';
import stl from '../console.module.css';
import { Icon } from 'UI';
import JumpButton from 'Shared/DevTools/JumpButton';
interface Props {
log: any;
iconProps: any;
jump?: any;
renderWithNL?: any;
style?: any;
}
function ConsoleRow(props: Props) {
const { log, iconProps, jump, renderWithNL, style } = props;
const [expanded, setExpanded] = useState(false);
const lines = log.value.split('\n').filter((l: any) => !!l);
const canExpand = lines.length > 1;
return (
<div
className={cn(stl.line, 'flex py-2 px-4 overflow-hidden group relative select-none', {
info: !log.isYellow() && !log.isRed(),
warn: log.isYellow(),
error: log.isRed(),
'cursor-pointer': canExpand,
})}
style={style}
onClick={() => setExpanded(!expanded)}
>
<div className={cn(stl.timestamp)}>
<Icon size="14" className={stl.icon} {...iconProps} />
</div>
{/* <div className={cn(stl.timestamp, {})}>
{Duration.fromMillis(log.time).toFormat('mm:ss.SSS')}
</div> */}
<div key={log.key} className={cn('')} data-scroll-item={log.isRed()}>
<div className={cn(stl.message, 'flex items-center')}>
{canExpand && (
<Icon name={expanded ? 'caret-down-fill' : 'caret-right-fill'} className="mr-2" />
)}
<span>{renderWithNL(lines.pop())}</span>
</div>
{canExpand && expanded && lines.map((l: any, i: number) => <div key={l.slice(0,3)+i} className="ml-4 mb-1">{l}</div>)}
</div>
<JumpButton onClick={() => jump(log.time)} />
</div>
);
}
export default ConsoleRow;

15
frontend/app/components/Session_/OverviewPanel/OverviewPanel.tsx
View file

@ -17,6 +17,7 @@ import { PlayerContext } from 'App/components/Session/playerContext';
function OverviewPanel({ issuesList }: { issuesList: any[] }) {
const { store } = React.useContext(PlayerContext)
function OverviewPanel() {
const [dataLoaded, setDataLoaded] = React.useState(false);
const [selectedFeatures, setSelectedFeatures] = React.useState([
'PERFORMANCE',
@ -35,6 +36,8 @@ function OverviewPanel({ issuesList }: { issuesList: any[] }) {
graphqlList,
} = store.get()
const fetchPresented = fetchList.length > 0;
const resourceList = resourceListUnmap
.filter((r: any) => r.isRed() || r.isYellow())
.concat(fetchList.filter((i: any) => parseInt(i.status) >= 400))
@ -84,7 +87,11 @@ function OverviewPanel({ issuesList }: { issuesList: any[] }) {
<BottomBlock.Content>
<OverviewPanelContainer endTime={endTime}>
<TimelineScale endTime={endTime} />
<div style={{ width: '100%', height: '187px', overflow: 'hidden' }} className="transition relative">
<div
// style={{ width: '100%', height: '187px', overflow: 'hidden' }}
style={{ width: 'calc(100vw - 1rem)', margin: '0 auto', height: '187px' }}
className="transition relative"
>
<NoContent
show={selectedFeatures.length === 0}
title={
@ -105,7 +112,11 @@ function OverviewPanel({ issuesList }: { issuesList: any[] }) {
title={feature}
list={resources[feature]}
renderElement={(pointer: any) => (
<TimelinePointer pointer={pointer} type={feature} />
<TimelinePointer
pointer={pointer}
type={feature}
fetchPresented={fetchPresented}
/>
)}
endTime={endTime}
message={HELP_MESSAGE[feature]}

8
frontend/app/components/Session_/OverviewPanel/components/EventRow/EventRow.tsx
View file

@ -37,7 +37,7 @@ const EventRow = React.memo((props: Props) => {
<div
className={cn(
'uppercase color-gray-medium text-sm flex items-center py-1',
props.noMargin ? '' : 'ml-4'
props.noMargin ? '' : 'ml-2'
)}
>
<div
@ -46,7 +46,7 @@ const EventRow = React.memo((props: Props) => {
>
{title}
</div>
{message ? <RowInfo zIndex={props.zIndex} message={message} /> : null}
{message ? <RowInfo message={message} /> : null}
</div>
<div className="relative w-full" style={{ zIndex: props.zIndex ? props.zIndex : undefined }}>
{isGraph ? (
@ -78,9 +78,9 @@ const EventRow = React.memo((props: Props) => {
export default EventRow;
function RowInfo({ message, zIndex }: any) {
function RowInfo({ message }: any) {
return (
<Tooltip title={message} delay={0} style={{ zIndex: zIndex ? zIndex : undefined }}>
<Tooltip title={message} delay={0}>
<Icon name="info-circle" color="gray-medium" />
</Tooltip>
);

2
frontend/app/components/Session_/OverviewPanel/components/FeatureSelection/FeatureSelection.tsx
View file

@ -30,7 +30,7 @@ function FeatureSelection(props: Props) {
const checked = list.includes(feature);
const _disabled = disabled && !checked;
return (
<Tooltip title="X-RAY supports up to 3 views" disabled={!_disabled} delay={0}>
<Tooltip key={index} title="X-RAY supports up to 3 views" disabled={!_disabled} delay={0}>
<Checkbox
key={index}
label={feature}

5
frontend/app/components/Session_/OverviewPanel/components/TimelinePointer/TimelinePointer.tsx
View file

@ -12,6 +12,7 @@ interface Props {
pointer: any;
type: any;
noClick?: boolean;
fetchPresented?: boolean;
}
const TimelinePointer = React.memo((props: Props) => {
const { player } = React.useContext(PlayerContext)
@ -37,7 +38,7 @@ const TimelinePointer = React.memo((props: Props) => {
if (pointer.tp === 'graph_ql') {
showModal(<GraphQLDetailsModal resource={pointer} />, { right: true });
} else {
showModal(<FetchDetails resource={pointer} />, { right: true });
showModal(<FetchDetails resource={pointer} fetchPresented={props.fetchPresented} />, { right: true });
}
}
// props.toggleBottomBlock(type);
@ -49,7 +50,7 @@ const TimelinePointer = React.memo((props: Props) => {
<Tooltip
title={
<div className="">
<b>{item.success ? 'Slow resource: ' : 'Missing resource:'}</b>
<b>{item.success ? 'Slow resource: ' : '4xx/5xx Error:'}</b>
<br />
{name.length > 200
? name.slice(0, 100) + ' ... ' + name.slice(-50)

435
frontend/app/components/Session_/Player/Controls/Controls.js Normal file
View file

@ -0,0 +1,435 @@
import React from 'react';
import cn from 'classnames';
import { connect } from 'react-redux';
import {
connectPlayer,
STORAGE_TYPES,
selectStorageType,
selectStorageListNow,
} from 'Player';
import LiveTag from 'Shared/LiveTag';
import { jumpToLive } from 'Player';
import { Icon, Tooltip } from 'UI';
import { toggleInspectorMode } from 'Player';
import {
fullscreenOn,
fullscreenOff,
toggleBottomBlock,
changeSkipInterval,
OVERVIEW,
CONSOLE,
NETWORK,
STACKEVENTS,
STORAGE,
PROFILER,
PERFORMANCE,
GRAPHQL,
INSPECTOR,
} from 'Duck/components/player';
import { AssistDuration } from './Time';
import Timeline from './Timeline';
import ControlButton from './ControlButton';
import PlayerControls from './components/PlayerControls';
import styles from './controls.module.css';
import XRayButton from 'Shared/XRayButton';
const SKIP_INTERVALS = {
2: 2e3,
5: 5e3,
10: 1e4,
15: 15e3,
20: 2e4,
30: 3e4,
60: 6e4,
};
function getStorageName(type) {
switch (type) {
case STORAGE_TYPES.REDUX:
return 'REDUX';
case STORAGE_TYPES.MOBX:
return 'MOBX';
case STORAGE_TYPES.VUEX:
return 'VUEX';
case STORAGE_TYPES.NGRX:
return 'NGRX';
case STORAGE_TYPES.ZUSTAND:
return 'ZUSTAND';
case STORAGE_TYPES.NONE:
return 'STATE';
}
}
@connectPlayer((state) => ({
time: state.time,
endTime: state.endTime,
live: state.live,
livePlay: state.livePlay,
playing: state.playing,
completed: state.completed,
skip: state.skip,
skipToIssue: state.skipToIssue,
speed: state.speed,
disabled: state.cssLoading || state.messagesLoading || state.inspectorMode || state.markedTargets,
inspectorMode: state.inspectorMode,
fullscreenDisabled: state.messagesLoading,
// logCount: state.logList.length,
logRedCount: state.logMarkedCount,
showExceptions: state.exceptionsList.length > 0,
resourceRedCount: state.resourceMarkedCount,
fetchRedCount: state.fetchMarkedCount,
showStack: state.stackList.length > 0,
stackCount: state.stackList.length,
stackRedCount: state.stackMarkedCount,
profilesCount: state.profilesList.length,
storageCount: selectStorageListNow(state).length,
storageType: selectStorageType(state),
showStorage: selectStorageType(state) !== STORAGE_TYPES.NONE,
showProfiler: state.profilesList.length > 0,
showGraphql: state.graphqlList.length > 0,
showFetch: state.fetchCount > 0,
fetchCount: state.fetchCount,
graphqlCount: state.graphqlList.length,
liveTimeTravel: state.liveTimeTravel,
}))
@connect(
(state, props) => {
const permissions = state.getIn(['user', 'account', 'permissions']) || [];
const isEnterprise = state.getIn(['user', 'account', 'edition']) === 'ee';
return {
disabled: props.disabled || (isEnterprise && !permissions.includes('DEV_TOOLS')),
fullscreen: state.getIn(['components', 'player', 'fullscreen']),
bottomBlock: state.getIn(['components', 'player', 'bottomBlock']),
showStorage:
props.showStorage || !state.getIn(['components', 'player', 'hiddenHints', 'storage']),
showStack: props.showStack || !state.getIn(['components', 'player', 'hiddenHints', 'stack']),
closedLive:
!!state.getIn(['sessions', 'errors']) || !state.getIn(['sessions', 'current', 'live']),
skipInterval: state.getIn(['components', 'player', 'skipInterval']),
};
},
{
fullscreenOn,
fullscreenOff,
toggleBottomBlock,
changeSkipInterval,
}
)
export default class Controls extends React.Component {
componentDidMount() {
document.addEventListener('keydown', this.onKeyDown);
}
componentWillUnmount() {
document.removeEventListener('keydown', this.onKeyDown);
//this.props.toggleInspectorMode(false);
}
shouldComponentUpdate(nextProps) {
if (
nextProps.fullscreen !== this.props.fullscreen ||
nextProps.bottomBlock !== this.props.bottomBlock ||
nextProps.live !== this.props.live ||
nextProps.livePlay !== this.props.livePlay ||
nextProps.playing !== this.props.playing ||
nextProps.completed !== this.props.completed ||
nextProps.skip !== this.props.skip ||
nextProps.skipToIssue !== this.props.skipToIssue ||
nextProps.speed !== this.props.speed ||
nextProps.disabled !== this.props.disabled ||
nextProps.fullscreenDisabled !== this.props.fullscreenDisabled ||
// nextProps.inspectorMode !== this.props.inspectorMode ||
// nextProps.logCount !== this.props.logCount ||
nextProps.logRedCount !== this.props.logRedCount ||
nextProps.showExceptions !== this.props.showExceptions ||
nextProps.resourceRedCount !== this.props.resourceRedCount ||
nextProps.fetchRedCount !== this.props.fetchRedCount ||
nextProps.showStack !== this.props.showStack ||
nextProps.stackCount !== this.props.stackCount ||
nextProps.stackRedCount !== this.props.stackRedCount ||
nextProps.profilesCount !== this.props.profilesCount ||
nextProps.storageCount !== this.props.storageCount ||
nextProps.storageType !== this.props.storageType ||
nextProps.showStorage !== this.props.showStorage ||
nextProps.showProfiler !== this.props.showProfiler ||
nextProps.showGraphql !== this.props.showGraphql ||
nextProps.showFetch !== this.props.showFetch ||
nextProps.fetchCount !== this.props.fetchCount ||
nextProps.graphqlCount !== this.props.graphqlCount ||
nextProps.liveTimeTravel !== this.props.liveTimeTravel ||
nextProps.skipInterval !== this.props.skipInterval
)
return true;
return false;
}
onKeyDown = (e) => {
if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) {
return;
}
if (this.props.inspectorMode) {
if (e.key === 'Esc' || e.key === 'Escape') {
toggleInspectorMode(false);
}
}
// if (e.key === ' ') {
// document.activeElement.blur();
// this.props.togglePlay();
// }
if (e.key === 'Esc' || e.key === 'Escape') {
this.props.fullscreenOff();
}
if (e.key === 'ArrowRight') {
this.forthTenSeconds();
}
if (e.key === 'ArrowLeft') {
this.backTenSeconds();
}
if (e.key === 'ArrowDown') {
this.props.speedDown();
}
if (e.key === 'ArrowUp') {
this.props.speedUp();
}
};
forthTenSeconds = () => {
const { time, endTime, jump, skipInterval } = this.props;
jump(Math.min(endTime, time + SKIP_INTERVALS[skipInterval]));
};
backTenSeconds = () => {
//shouldComponentUpdate
const { time, jump, skipInterval } = this.props;
jump(Math.max(1, time - SKIP_INTERVALS[skipInterval]));
};
goLive = () => this.props.jump(this.props.endTime);
renderPlayBtn = () => {
const { completed, playing } = this.props;
let label;
let icon;
if (completed) {
icon = 'arrow-clockwise';
label = 'Replay this session';
} else if (playing) {
icon = 'pause-fill';
label = 'Pause';
} else {
icon = 'play-fill-new';
label = 'Pause';
label = 'Play';
}
return (
<Tooltip
title={label}
className="mr-4"
>
<div
onClick={this.props.togglePlay}
className="hover-main color-main cursor-pointer rounded hover:bg-gray-light-shade"
>
<Icon name={icon} size="36" color="inherit" />
</div>
</Tooltip>
);
};
controlIcon = (icon, size, action, isBackwards, additionalClasses) => (
<div
onClick={action}
className={cn('py-2 px-2 hover-main cursor-pointer bg-gray-lightest', additionalClasses)}
style={{ transform: isBackwards ? 'rotate(180deg)' : '' }}
>
<Icon name={icon} size={size} color="inherit" />
</div>
);
render() {
const {
bottomBlock,
toggleBottomBlock,
live,
livePlay,
skip,
speed,
disabled,
logRedCount,
showExceptions,
resourceRedCount,
fetchRedCount,
showStack,
stackRedCount,
showStorage,
storageType,
showProfiler,
showGraphql,
fullscreen,
inspectorMode,
closedLive,
toggleSpeed,
toggleSkip,
liveTimeTravel,
changeSkipInterval,
skipInterval,
} = this.props;
const toggleBottomTools = (blockName) => {
if (blockName === INSPECTOR) {
toggleInspectorMode();
bottomBlock && toggleBottomBlock();
} else {
toggleInspectorMode(false);
toggleBottomBlock(blockName);
}
};
return (
<div className={styles.controls}>
<Timeline
live={live}
jump={this.props.jump}
liveTimeTravel={liveTimeTravel}
pause={this.props.pause}
togglePlay={this.props.togglePlay}
/>
{!fullscreen && (
<div className={cn(styles.buttons, { '!px-5 !pt-0': live })} data-is-live={live}>
<div className="flex items-center">
{!live && (
<>
<PlayerControls
live={live}
skip={skip}
speed={speed}
disabled={disabled}
backTenSeconds={this.backTenSeconds}
forthTenSeconds={this.forthTenSeconds}
toggleSpeed={toggleSpeed}
toggleSkip={toggleSkip}
playButton={this.renderPlayBtn()}
controlIcon={this.controlIcon}
ref={this.speedRef}
skipIntervals={SKIP_INTERVALS}
setSkipInterval={changeSkipInterval}
currentInterval={skipInterval}
/>
<div className={cn('mx-2')} />
<XRayButton
isActive={bottomBlock === OVERVIEW && !inspectorMode}
onClick={() => toggleBottomTools(OVERVIEW)}
/>
</>
)}
{live && !closedLive && (
<div className={styles.buttonsLeft}>
<LiveTag isLive={livePlay} onClick={() => (livePlay ? null : jumpToLive())} />
<div className="font-semibold px-2">
<AssistDuration isLivePlay={livePlay} />
</div>
</div>
)}
</div>
<div className="flex items-center h-full">
<ControlButton
disabled={disabled && !inspectorMode}
onClick={() => toggleBottomTools(CONSOLE)}
active={bottomBlock === CONSOLE && !inspectorMode}
label="CONSOLE"
noIcon
labelClassName="!text-base font-semibold"
hasErrors={logRedCount > 0 || showExceptions}
containerClassName="mx-2"
/>
{!live && (
<ControlButton
disabled={disabled && !inspectorMode}
onClick={() => toggleBottomTools(NETWORK)}
active={bottomBlock === NETWORK && !inspectorMode}
label="NETWORK"
hasErrors={resourceRedCount > 0 || fetchRedCount > 0}
noIcon
labelClassName="!text-base font-semibold"
containerClassName="mx-2"
/>
)}
{!live && (
<ControlButton
disabled={disabled && !inspectorMode}
onClick={() => toggleBottomTools(PERFORMANCE)}
active={bottomBlock === PERFORMANCE && !inspectorMode}
label="PERFORMANCE"
noIcon
labelClassName="!text-base font-semibold"
containerClassName="mx-2"
/>
)}
{!live && showGraphql && (
<ControlButton
disabled={disabled && !inspectorMode}
onClick={() => toggleBottomTools(GRAPHQL)}
active={bottomBlock === GRAPHQL && !inspectorMode}
label="GRAPHQL"
noIcon
labelClassName="!text-base font-semibold"
containerClassName="mx-2"
/>
)}
{!live && showStorage && (
<ControlButton
disabled={disabled && !inspectorMode}
onClick={() => toggleBottomTools(STORAGE)}
active={bottomBlock === STORAGE && !inspectorMode}
label={getStorageName(storageType)}
noIcon
labelClassName="!text-base font-semibold"
containerClassName="mx-2"
/>
)}
{!live && (
<ControlButton
disabled={disabled && !inspectorMode}
onClick={() => toggleBottomTools(STACKEVENTS)}
active={bottomBlock === STACKEVENTS && !inspectorMode}
label="EVENTS"
noIcon
labelClassName="!text-base font-semibold"
containerClassName="mx-2"
hasErrors={stackRedCount > 0}
/>
)}
{!live && showProfiler && (
<ControlButton
disabled={disabled && !inspectorMode}
onClick={() => toggleBottomTools(PROFILER)}
active={bottomBlock === PROFILER && !inspectorMode}
label="PROFILER"
noIcon
labelClassName="!text-base font-semibold"
containerClassName="mx-2"
/>
)}
{!live && (
<Tooltip title="Fullscreen" delay={0} position="top-end" className="mx-4">
{this.controlIcon(
'arrows-angle-extend',
16,
this.props.fullscreenOn,
false,
'rounded hover:bg-gray-light-shade color-gray-medium'
)}
</Tooltip>
)}
</div>
</div>
)}
</div>
);
}
}

109
frontend/app/components/Session_/Player/Controls/components/PlayerControls.tsx
View file

@ -94,11 +94,11 @@ function PlayerControls(props: Props) {
<div className="rounded ml-4 bg-active-blue border border-active-blue-border flex items-stretch">
{/* @ts-ignore */}
<Tooltip anchorClassName='h-full' title={`Rewind ${currentInterval}s`} position="top">
<button
ref={arrowBackRef}
className="h-full hover:border-active-blue-border focus:border focus:border-blue border-borderColor-transparent"
>
<button
ref={arrowBackRef}
className="h-full hover:border-active-blue-border focus:border focus:border-blue border-borderColor-transparent"
>
<Tooltip anchorClassName="h-full" title={`Rewind ${currentInterval}s`} placement="top">
{controlIcon(
'skip-forward-fill',
18,
@ -106,56 +106,57 @@ function PlayerControls(props: Props) {
true,
'hover:bg-active-blue-border color-main h-full flex items-center'
)}
</button>
</Tooltip>
<div className="p-1 border-l border-r bg-active-blue-border border-active-blue-border">
<OutsideClickDetectingDiv onClickOutside={handleClickOutside}>
<Popover
// @ts-ignore
theme="nopadding"
animation="none"
duration={0}
className="cursor-pointer select-none"
distance={20}
render={() => (
<div className="flex flex-col bg-white border border-borderColor-gray-light-shade text-figmaColors-text-primary rounded">
<div className="font-semibold py-2 px-4 w-full text-left">
Jump <span className="text-disabled-text">(Secs)</span>
</div>
{Object.keys(skipIntervals).map((interval) => (
<div
key={interval}
onClick={() => {
toggleTooltip();
setSkipInterval(parseInt(interval, 10));
}}
className={cn(
'py-2 px-4 cursor-pointer w-full text-left font-semibold',
'hover:bg-active-blue border-t border-borderColor-gray-light-shade'
)}
>
{interval}
<span className="text-disabled-text">s</span>
</div>
))}
</Tooltip>
</button>
<div className="p-1 border-l border-r bg-active-blue-border border-active-blue-border flex items-center">
<Popover
// open={showTooltip}
// interactive
// @ts-ignore
theme="nopadding"
animation="none"
duration={0}
className="cursor-pointer select-none"
distance={20}
render={({ close }: any) => (
<div className="flex flex-col bg-white border border-borderColor-gray-light-shade text-figmaColors-text-primary rounded">
<div className="font-semibold py-2 px-4 w-full text-left">
Jump <span className="text-disabled-text">(Secs)</span>
</div>
)}
>
<div onClick={toggleTooltip} ref={skipRef}>
{/* @ts-ignore */}
<Tooltip anchorClassName='cursor-pointer' disabled={showTooltip} title="Set default skip duration">
{currentInterval}s
</Tooltip>
{Object.keys(skipIntervals).map((interval) => (
<div
key={interval}
onClick={() => {
close();
setSkipInterval(parseInt(interval, 10));
}}
className={cn(
'py-2 px-4 cursor-pointer w-full text-left font-semibold',
'hover:bg-active-blue border-t border-borderColor-gray-light-shade'
)}
>
{interval}
<span className="text-disabled-text">s</span>
</div>
))}
</div>
</Popover>
</OutsideClickDetectingDiv>
</div>
{/* @ts-ignore */}
<Tooltip anchorClassName='h-full' title={`Forward ${currentInterval}s`} position="top">
<button
ref={arrowForwardRef}
className="h-full hover:border-active-blue-border focus:border focus:border-blue border-borderColor-transparent"
)}
>
<div onClick={toggleTooltip} ref={skipRef} className="cursor-pointer select-none">
{/* @ts-ignore */}
<Tooltip disabled={showTooltip} title="Set default skip duration">
{currentInterval}s
</Tooltip>
</div>
</Popover>
</div>
<button
ref={arrowForwardRef}
className="h-full hover:border-active-blue-border focus:border focus:border-blue border-borderColor-transparent"
>
<Tooltip anchorClassName="h-full" title={`Rewind ${currentInterval}s`} placement="top">
{controlIcon(
'skip-forward-fill',
18,
@ -163,8 +164,8 @@ function PlayerControls(props: Props) {
false,
'hover:bg-active-blue-border color-main h-full flex items-center'
)}
</button>
</Tooltip>
</Tooltip>
</button>
</div>
{!live && (

8
frontend/app/components/Session_/Player/Player.js
View file

@ -35,6 +35,7 @@ import OverviewPanel from '../OverviewPanel';
import ConsolePanel from 'Shared/DevTools/ConsolePanel';
import ProfilerPanel from 'Shared/DevTools/ProfilerPanel';
import { PlayerContext } from 'App/components/Session/playerContext';
import StackEventPanel from 'Shared/DevTools/StackEventPanel';
function Player(props) {
const {
@ -84,7 +85,8 @@ function Player(props) {
// <Network />
<NetworkPanel />
)}
{bottomBlock === STACKEVENTS && <StackEvents />}
{/* {bottomBlock === STACKEVENTS && <StackEvents />} */}
{bottomBlock === STACKEVENTS && <StackEventPanel />}
{bottomBlock === STORAGE && <Storage />}
{bottomBlock === PROFILER && <ProfilerPanel />}
{bottomBlock === PERFORMANCE && <ConnectedPerformance />}
@ -94,11 +96,11 @@ function Player(props) {
{bottomBlock === INSPECTOR && <Inspector />}
</div>
)}
<Controls
{!fullView && <Controls
speedDown={playerContext.player.speedDown}
speedUp={playerContext.player.speedUp}
jump={playerContext.player.jump}
/>
/>}
</div>
)
}

40
frontend/app/components/Session_/PlayerBlock.js
View file

@ -1,46 +1,36 @@
import React from 'react';
import cn from "classnames";
import cn from 'classnames';
import { connect } from 'react-redux';
import {
NONE,
} from 'Duck/components/player';
import { NONE } from 'Duck/components/player';
import Player from './Player';
import SubHeader from './Subheader';
import styles from './playerBlock.module.css';
@connect(state => ({
fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]),
bottomBlock: state.getIn([ 'components', 'player', 'bottomBlock' ]),
sessionId: state.getIn([ 'sessions', 'current', 'sessionId' ]),
disabled: state.getIn([ 'components', 'targetDefiner', 'inspectorMode' ]),
jiraConfig: state.getIn([ 'issues', 'list' ]).first(),
@connect((state) => ({
fullscreen: state.getIn(['components', 'player', 'fullscreen']),
bottomBlock: state.getIn(['components', 'player', 'bottomBlock']),
sessionId: state.getIn(['sessions', 'current', 'sessionId']),
disabled: state.getIn(['components', 'targetDefiner', 'inspectorMode']),
jiraConfig: state.getIn(['issues', 'list']).first(),
}))
export default class PlayerBlock extends React.PureComponent {
render() {
const {
fullscreen,
bottomBlock,
sessionId,
disabled,
activeTab,
jiraConfig,
} = this.props;
const { fullscreen, bottomBlock, sessionId, disabled, activeTab, jiraConfig, fullView = false } = this.props;
return (
<div className={ cn(styles.playerBlock, "flex flex-col overflow-x-hidden") }>
{!fullscreen && <SubHeader
sessionId={sessionId}
disabled={disabled}
jiraConfig={jiraConfig}
/>}
<div className={cn(styles.playerBlock, 'flex flex-col overflow-x-hidden')}>
{!fullscreen && !fullView && (
<SubHeader sessionId={sessionId} disabled={disabled} jiraConfig={jiraConfig} />
)}
<Player
className="flex-1"
bottomBlockIsActive={ !fullscreen && bottomBlock !== NONE }
bottomBlockIsActive={!fullscreen && bottomBlock !== NONE}
// bottomBlockIsActive={ true }
bottomBlock={bottomBlock}
fullscreen={fullscreen}
activeTab={activeTab}
fullView={fullView}
/>
</div>
);

28
frontend/app/components/Session_/Storage/DiffRow.tsx
View file

@ -50,31 +50,41 @@ function DiffRow({ diff, path }: Props) {
: newValue;
return (
<div className="p-1 rounded">
<div className="p-1 rounded flex flex-wrap gap-2">
<span className={length > 20 ? 'cursor-pointer' : ''} onClick={() => setShorten(!shorten)}>
{pathStr}
{': '}
</span>
<span
<div
onClick={() => setShortenOldVal(!shortenOldVal)}
className={cn(
'line-through text-disabled-text',
'text-disabled-text',
diffLengths[0] > 50 ? 'cursor-pointer' : ''
)}
>
{oldValueSafe || 'undefined'}
</span>
<span className="line-through">{oldValueSafe || 'undefined'}</span>
{diffLengths[0] > 50
? (
<div onClick={() => setShortenOldVal(!shortenOldVal)} className="cursor-pointer px-1 text-white bg-gray-light rounded text-sm w-fit">
{!shortenOldVal ? 'collapse' : 'expand'}
</div>
) : null}
</div>
{' -> '}
<span
onClick={() => setShortenNewVal(!shortenNewVal)}
<div
className={cn(
'whitespace-pre',
newValue ? 'text-red' : 'text-green',
diffLengths[1] > 50 ? 'cursor-pointer' : ''
)}
>
{newValueSafe || 'undefined'}
</span>
{diffLengths[1] > 50
? (
<div onClick={() => setShortenNewVal(!shortenNewVal)} className="cursor-pointer px-1 text-white bg-gray-light rounded text-sm w-fit">
{!shortenNewVal ? 'collapse' : 'expand'}
</div>
) : null}
</div>
</div>
);
}

368
frontend/app/components/Session_/Storage/Storage.js Normal file
View file

@ -0,0 +1,368 @@
import React from 'react';
import { connect } from 'react-redux';
import { hideHint } from 'Duck/components/player';
import {
connectPlayer,
selectStorageType,
STORAGE_TYPES,
selectStorageListNow,
selectStorageList,
} from 'Player';
import { JSONTree, NoContent, Tooltip } from 'UI';
import { formatMs } from 'App/date';
import { diff } from 'deep-diff';
import { jump } from 'Player';
import BottomBlock from '../BottomBlock/index';
import DiffRow from './DiffRow';
import stl from './storage.module.css';
import { List, CellMeasurer, CellMeasurerCache, AutoSizer } from 'react-virtualized';
const ROW_HEIGHT = 90;
function getActionsName(type) {
switch (type) {
case STORAGE_TYPES.MOBX:
case STORAGE_TYPES.VUEX:
return 'MUTATIONS';
default:
return 'ACTIONS';
}
}
@connectPlayer((state) => ({
type: selectStorageType(state),
list: selectStorageList(state),
listNow: selectStorageListNow(state),
}))
@connect(
(state) => ({
hintIsHidden: state.getIn(['components', 'player', 'hiddenHints', 'storage']),
}),
{
hideHint,
}
)
export default class Storage extends React.PureComponent {
constructor(props) {
super(props);
this.lastBtnRef = React.createRef();
this._list = React.createRef();
this.cache = new CellMeasurerCache({
fixedWidth: true,
keyMapper: (index) => this.props.listNow[index],
});
this._rowRenderer = this._rowRenderer.bind(this);
}
focusNextButton() {
if (this.lastBtnRef.current) {
this.lastBtnRef.current.focus();
}
}
componentDidMount() {
this.focusNextButton();
}
componentDidUpdate(prevProps, prevState) {
if (prevProps.listNow.length !== this.props.listNow.length) {
this.focusNextButton();
/** possible performance gain, but does not work with dynamic list insertion for some reason
* getting NaN offsets, maybe I detect changed rows wrongly
*/
// const newRows = this.props.listNow.filter(evt => prevProps.listNow.indexOf(evt._index) < 0);
// if (newRows.length > 0) {
// const newRowsIndexes = newRows.map(r => this.props.listNow.indexOf(r))
// newRowsIndexes.forEach(ind => this.cache.clear(ind))
// this._list.recomputeRowHeights(newRowsIndexes)
// }
}
}
renderDiff(item, prevItem) {
if (!prevItem) {
// we don't have state before first action
return <div style={{ flex: 3 }} className="p-1" />;
}
const stateDiff = diff(prevItem.state, item.state);
if (!stateDiff) {
return (
<div style={{ flex: 3 }} className="flex flex-col p-2 pr-0 font-mono text-disabled-text">
No diff
</div>
);
}
return (
<div
style={{ flex: 3, maxHeight: ROW_HEIGHT, overflowY: 'scroll' }}
className="flex flex-col p-1 font-mono"
>
{stateDiff.map((d, i) => this.renderDiffs(d, i))}
</div>
);
}
renderDiffs(diff, i) {
const path = this.createPath(diff);
return (
<React.Fragment key={i}>
<DiffRow shades={this.pathShades} path={path} diff={diff} />
</React.Fragment>
);
}
createPath = (diff) => {
let path = [];
if (diff.path) {
path = path.concat(diff.path);
}
if (typeof diff.index !== 'undefined') {
path.push(diff.index);
}
const pathStr = path.length ? path.join('.') : '';
return pathStr;
};
ensureString(actionType) {
if (typeof actionType === 'string') return actionType;
return 'UNKNOWN';
}
goNext = () => {
const { list, listNow } = this.props;
jump(list[listNow.length].time, list[listNow.length]._index);
};
renderTab() {
const { listNow } = this.props;
if (listNow.length === 0) {
return 'Not initialized'; //?
}
return <JSONTree collapsed={2} src={listNow[listNow.length - 1].state} />;
}
renderItem(item, i, prevItem, style) {
const { type } = this.props;
let src;
let name;
switch (type) {
case STORAGE_TYPES.REDUX:
case STORAGE_TYPES.NGRX:
src = item.action;
name = src && src.type;
break;
case STORAGE_TYPES.VUEX:
src = item.mutation;
name = src && src.type;
break;
case STORAGE_TYPES.MOBX:
src = item.payload;
name = `@${item.type} ${src && src.type}`;
break;
case STORAGE_TYPES.ZUSTAND:
src = null;
name = item.mutation.join('');
}
return (
<div
style={{ ...style, height: ROW_HEIGHT }}
className="flex justify-between items-start border-b"
key={`store-${i}`}
>
{src === null ? (
<div className="font-mono" style={{ flex: 2, marginLeft: '26.5%' }}>
{name}
</div>
) : (
<>
{this.renderDiff(item, prevItem, i)}
<div
style={{ flex: 2, maxHeight: ROW_HEIGHT, overflowY: 'scroll', overflowX: 'scroll' }}
className="flex pl-10 pt-2"
>
<JSONTree
name={this.ensureString(name)}
src={src}
collapsed
collapseStringsAfterLength={7}
onSelect={() => console.log('test')}
/>
</div>
</>
)}
<div
style={{ flex: 1 }}
className="flex-1 flex gap-2 pt-2 items-center justify-end self-start"
>
{typeof item.duration === 'number' && (
<div className="font-size-12 color-gray-medium">{formatMs(item.duration)}</div>
)}
<div className="w-12">
{i + 1 < this.props.listNow.length && (
<button className={stl.button} onClick={() => jump(item.time, item._index)}>
{'JUMP'}
</button>
)}
{i + 1 === this.props.listNow.length && i + 1 < this.props.list.length && (
<button className={stl.button} ref={this.lastBtnRef} onClick={this.goNext}>
{'NEXT'}
</button>
)}
</div>
</div>
</div>
);
}
_rowRenderer({ index, parent, key, style }) {
const { listNow } = this.props;
if (!listNow[index]) return console.warn(index, listNow);
return (
<CellMeasurer cache={this.cache} columnIndex={0} key={key} rowIndex={index} parent={parent}>
{this.renderItem(listNow[index], index, index > 0 ? listNow[index - 1] : undefined, style)}
</CellMeasurer>
);
}
render() {
const { type, list, listNow, hintIsHidden } = this.props;
const showStore = type !== STORAGE_TYPES.MOBX;
return (
<BottomBlock>
<BottomBlock.Header>
{list.length > 0 && (
<div className="flex w-full">
{showStore && (
<h3 style={{ width: '25%', marginRight: 20 }} className="font-semibold">
{'STATE'}
</h3>
)}
{type !== STORAGE_TYPES.ZUSTAND ? (
<h3 style={{ width: '39%' }} className="font-semibold">
DIFFS
</h3>
) : null}
<h3 style={{ width: '30%' }} className="font-semibold">
{getActionsName(type)}
</h3>
<h3 style={{ paddingRight: 30, marginLeft: 'auto' }} className="font-semibold">
<Tooltip title="Time to execute">TTE</Tooltip>
</h3>
</div>
)}
</BottomBlock.Header>
<BottomBlock.Content className="flex">
<NoContent
title="Nothing to display yet."
subtext={
!hintIsHidden ? (
<>
{
'Inspect your application state while you’re replaying your users sessions. OpenReplay supports '
}
<a
className="underline color-teal"
href="https://docs.openreplay.com/plugins/redux"
target="_blank"
>
Redux
</a>
{', '}
<a
className="underline color-teal"
href="https://docs.openreplay.com/plugins/vuex"
target="_blank"
>
VueX
</a>
{', '}
<a
className="underline color-teal"
href="https://docs.openreplay.com/plugins/pinia"
target="_blank"
>
Pinia
</a>
{', '}
<a
className="underline color-teal"
href="https://docs.openreplay.com/plugins/zustand"
target="_blank"
>
Zustand
</a>
{', '}
<a
className="underline color-teal"
href="https://docs.openreplay.com/plugins/mobx"
target="_blank"
>
MobX
</a>
{' and '}
<a
className="underline color-teal"
href="https://docs.openreplay.com/plugins/ngrx"
target="_blank"
>
NgRx
</a>
.
<br />
<br />
<button className="color-teal" onClick={() => this.props.hideHint('storage')}>
Got It!
</button>
</>
) : null
}
size="small"
show={listNow.length === 0}
>
{showStore && (
<div className="ph-10 scroll-y" style={{ width: '25%' }}>
{listNow.length === 0 ? (
<div className="color-gray-light font-size-16 mt-20 text-center">
{'Empty state.'}
</div>
) : (
this.renderTab()
)}
</div>
)}
<div className="flex" style={{ width: showStore ? '75%' : '100%' }}>
<AutoSizer>
{({ height, width }) => (
<List
ref={(element) => {
this._list = element;
}}
deferredMeasurementCache={this.cache}
overscanRowCount={1}
rowCount={Math.ceil(parseInt(this.props.listNow.length) || 1)}
rowHeight={ROW_HEIGHT}
rowRenderer={this._rowRenderer}
width={width}
height={height}
/>
)}
</AutoSizer>
</div>
</NoContent>
</BottomBlock.Content>
</BottomBlock>
);
}
}

24
frontend/app/components/shared/DevTools/BottomBlock/BottomBlock.js
View file

@ -1,17 +1,29 @@
import React from 'react';
import React, { useEffect } from 'react';
import cn from 'classnames';
import stl from './bottomBlock.module.css';
let timer = null;
const BottomBlock = ({
children = null,
className = '',
additionalHeight = 0,
onMouseEnter = () => {},
onMouseLeave = () => {},
...props
}) => (
<div className={ cn(stl.wrapper, "flex flex-col mb-2") } { ...props } >
{ children }
</div>
);
}) => {
useEffect(() => {}, []);
return (
<div
className={cn(stl.wrapper, 'flex flex-col mb-2')}
{...props}
onMouseEnter={onMouseEnter}
onMouseLeave={onMouseLeave}
>
{children}
</div>
);
};
BottomBlock.displayName = 'BottomBlock';

170
frontend/app/components/shared/DevTools/ConsolePanel/ConsolePanel.tsx
View file

@ -1,4 +1,4 @@
import React, { useState } from 'react';
import React, { useEffect, useRef, useState } from 'react';
import Log from 'Types/session/log';
import BottomBlock from '../BottomBlock';
import { LEVEL } from 'Types/session/log';
@ -8,6 +8,11 @@ import ConsoleRow from '../ConsoleRow';
import { getRE } from 'App/utils';
import { PlayerContext } from 'App/components/Session/playerContext';
import { observer } from 'mobx-react-lite';
import { List, CellMeasurer, CellMeasurerCache, AutoSizer } from 'react-virtualized';
import { useObserver } from 'mobx-react-lite';
import { useStore } from 'App/mstore';
import ErrorDetailsModal from 'App/components/Dashboard/components/Errors/ErrorDetailsModal';
import { useModal } from 'App/components/Modal';
const ALL = 'ALL';
const INFO = 'INFO';
@ -52,11 +57,16 @@ const getIconProps = (level: any) => {
return null;
};
const INDEX_KEY = 'console';
let timeOut: any = null;
const TIMEOUT_DURATION = 5000;
function ConsolePanel() {
const { player, store } = React.useContext(PlayerContext)
const jump = (t: number) => player.jump(t)
const { logList, exceptionsList } = store.get()
const { logList, exceptionsList, time } = store.get()
const logExceptions = exceptionsList.map(({ time, errorId, name, projectId }: any) =>
Log({
@ -68,12 +78,106 @@ function ConsolePanel() {
);
// @ts-ignore
const logs = logList.concat(logExceptions)
const additionalHeight = 0;
const [activeTab, setActiveTab] = useState(ALL);
const [filter, setFilter] = useState('');
const {
sessionStore: { devTools },
} = useStore();
let filtered = React.useMemo(() => {
const [isDetailsModalActive, setIsDetailsModalActive] = useState(false);
const [filteredList, setFilteredList] = useState([]);
const filter = useObserver(() => devTools[INDEX_KEY].filter);
const activeTab = useObserver(() => devTools[INDEX_KEY].activeTab);
const activeIndex = useObserver(() => devTools[INDEX_KEY].index);
const [pauseSync, setPauseSync] = useState(activeIndex > 0);
const synRef: any = useRef({});
const { showModal } = useModal();
const onTabClick = (activeTab: any) => devTools.update(INDEX_KEY, { activeTab });
const onFilterChange = ({ target: { value } }: any) => {
devTools.update(INDEX_KEY, { filter: value });
};
synRef.current = {
pauseSync,
activeIndex,
};
const removePause = () => {
setIsDetailsModalActive(false);
clearTimeout(timeOut);
timeOut = setTimeout(() => {
devTools.update(INDEX_KEY, { index: getCurrentIndex() });
setPauseSync(false);
}, TIMEOUT_DURATION);
};
const onMouseLeave = () => {
if (isDetailsModalActive) return;
removePause();
};
useEffect(() => {
if (pauseSync) {
removePause();
}
return () => {
clearTimeout(timeOut);
if (!synRef.current.pauseSync) {
devTools.update(INDEX_KEY, { index: 0 });
}
};
}, []);
const getCurrentIndex = () => {
return filteredList.filter((item: any) => item.time <= time).length - 1;
};
useEffect(() => {
const currentIndex = getCurrentIndex();
if (currentIndex !== activeIndex && !pauseSync) {
devTools.update(INDEX_KEY, { index: currentIndex });
}
}, [time]);
const cache = new CellMeasurerCache({
fixedWidth: true,
keyMapper: (index: number) => filteredList[index],
});
const _list = React.useRef();
const showDetails = (log: any) => {
setIsDetailsModalActive(true);
showModal(<ErrorDetailsModal errorId={log.errorId} />, { right: true, onClose: removePause });
devTools.update(INDEX_KEY, { index: filteredList.indexOf(log) });
setPauseSync(true);
};
const _rowRenderer = ({ index, key, parent, style }: any) => {
const item = filteredList[index];
return (
// @ts-ignore
<CellMeasurer cache={cache} columnIndex={0} key={key} rowIndex={index} parent={parent}>
{({ measure }: any) => (
<ConsoleRow
style={style}
log={item}
jump={jump}
iconProps={getIconProps(item.level)}
renderWithNL={renderWithNL}
onClick={() => showDetails(item)}
recalcHeight={() => {
measure();
(_list as any).current.recomputeRowHeights(index);
}}
/>
)}
</CellMeasurer>
);
};
React.useMemo(() => {
const filterRE = getRE(filter, 'i');
let list = logs;
@ -82,14 +186,23 @@ function ConsolePanel() {
(!!filter ? filterRE.test(value) : true) &&
(activeTab === ALL || activeTab === LEVEL_TAB[level])
);
return list;
}, [filter, activeTab]);
setFilteredList(list);
}, [logs, filter, activeTab]);
const onTabClick = (activeTab: any) => setActiveTab(activeTab);
const onFilterChange = ({ target: { value } }: any) => setFilter(value);
useEffect(() => {
if (_list.current) {
// @ts-ignore
_list.current.scrollToRow(activeIndex);
}
}, [activeIndex]);
return (
<BottomBlock style={{ height: 300 + additionalHeight + 'px' }}>
<BottomBlock
style={{ height: 300 + additionalHeight + 'px' }}
onMouseEnter={() => setPauseSync(true)}
onMouseLeave={onMouseLeave}
>
{/* @ts-ignore */}
<BottomBlock.Header>
<div className="flex items-center">
<span className="font-semibold color-gray-medium mr-4">Console</span>
@ -103,8 +216,11 @@ function ConsolePanel() {
name="filter"
height={28}
onChange={onFilterChange}
value={filter}
/>
{/* @ts-ignore */}
</BottomBlock.Header>
{/* @ts-ignore */}
<BottomBlock.Content className="overflow-y-auto">
<NoContent
title={
@ -114,20 +230,28 @@ function ConsolePanel() {
</div>
}
size="small"
show={filtered.length === 0}
show={filteredList.length === 0}
>
{/* <Autoscroll> */}
{filtered.map((l: any, index: any) => (
<ConsoleRow
key={index}
log={l}
jump={jump}
iconProps={getIconProps(l.level)}
renderWithNL={renderWithNL}
/>
))}
{/* </Autoscroll> */}
{/* @ts-ignore */}
<AutoSizer>
{({ height, width }: any) => (
// @ts-ignore
<List
ref={_list}
deferredMeasurementCache={cache}
overscanRowCount={5}
rowCount={Math.ceil(filteredList.length || 1)}
rowHeight={cache.rowHeight}
rowRenderer={_rowRenderer}
width={width}
height={height}
// scrollToIndex={activeIndex}
scrollToAlignment="center"
/>
)}
</AutoSizer>
</NoContent>
{/* @ts-ignore */}
</BottomBlock.Content>
</BottomBlock>
);

30
frontend/app/components/shared/DevTools/ConsoleRow/ConsoleRow.tsx
View file

@ -1,43 +1,43 @@
import React, { useState } from 'react';
import cn from 'classnames';
// import stl from '../console.module.css';
import { Icon } from 'UI';
import JumpButton from 'Shared/DevTools/JumpButton';
import { useModal } from 'App/components/Modal';
import ErrorDetailsModal from 'App/components/Dashboard/components/Errors/ErrorDetailsModal';
interface Props {
log: any;
iconProps: any;
jump?: any;
renderWithNL?: any;
style?: any;
recalcHeight?: () => void;
onClick: () => void;
}
function ConsoleRow(props: Props) {
const { log, iconProps, jump, renderWithNL } = props;
const { showModal } = useModal();
const { log, iconProps, jump, renderWithNL, style, recalcHeight } = props;
const [expanded, setExpanded] = useState(false);
const lines = log.value.split('\n').filter((l: any) => !!l);
const canExpand = lines.length > 1;
const clickable = canExpand || !!log.errorId;
const onErrorClick = () => {
showModal(<ErrorDetailsModal errorId={log.errorId} />, { right: true });
const toggleExpand = () => {
setExpanded(!expanded);
setTimeout(() => recalcHeight(), 0);
};
return (
<div
style={style}
className={cn(
'border-b flex items-center py-2 px-4 overflow-hidden group relative select-none',
{
info: !log.isYellow() && !log.isRed(),
warn: log.isYellow(),
error: log.isRed(),
'cursor-pointer underline decoration-dotted decoration-gray-200': clickable,
'cursor-pointer': clickable,
'cursor-pointer underline decoration-dotted decoration-gray-200': !!log.errorId,
}
)}
onClick={
clickable ? () => (!!log.errorId ? onErrorClick() : setExpanded(!expanded)) : () => {}
}
onClick={clickable ? () => (!!log.errorId ? props.onClick() : toggleExpand()) : () => {}}
>
<div className="mr-2">
<Icon size="14" {...iconProps} />
@ -49,7 +49,13 @@ function ConsoleRow(props: Props) {
)}
<span>{renderWithNL(lines.pop())}</span>
</div>
{canExpand && expanded && lines.map((l: any) => <div className="ml-4 mb-1">{l}</div>)}
{canExpand &&
expanded &&
lines.map((l: string, i: number) => (
<div key={l.slice(0, 4) + i} className="ml-4 mb-1">
{l}
</div>
))}
</div>
<JumpButton onClick={() => jump(log.time)} />
</div>

4
frontend/app/components/shared/DevTools/JumpButton/JumpButton.tsx
View file

@ -6,10 +6,10 @@ interface Props {
tooltip?: string;
}
function JumpButton(props: Props) {
const { tooltip = '' } = props;
const { tooltip } = props;
return (
<div className="absolute right-0 top-0 bottom-0 my-auto flex items-center">
<Tooltip title={tooltip} disabled={!!tooltip}>
<Tooltip title={tooltip} disabled={!tooltip}>
<div
className="mr-2 border cursor-pointer invisible group-hover:visible rounded-lg bg-active-blue text-xs flex items-center px-2 py-1 color-teal hover:shadow h-6"
onClick={(e: any) => {

214
frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx
View file

@ -1,4 +1,4 @@
import React, { useState } from 'react';
import React, { useEffect, useMemo, useRef, useState } from 'react';
import { Tooltip, Tabs, Input, NoContent, Icon, Toggler } from 'UI';
import { getRE } from 'App/utils';
import Resource, { TYPES } from 'Types/session/resource';
@ -13,6 +13,9 @@ import { useModal } from 'App/components/Modal';
import FetchDetailsModal from 'Shared/FetchDetailsModal';
import { PlayerContext } from 'App/components/Session/playerContext';
import { observer } from 'mobx-react-lite';
import { useStore } from 'App/mstore';
const INDEX_KEY = 'network';
const ALL = 'ALL';
const XHR = 'xhr';
@ -72,7 +75,7 @@ function renderSize(r: any) {
if (r.responseBodySize) return formatBytes(r.responseBodySize);
let triggerText;
let content;
if (r.decodedBodySize == null) {
if (r.decodedBodySize == null || r.decodedBodySize === 0) {
triggerText = 'x';
content = 'Not captured';
} else {
@ -120,6 +123,19 @@ export function renderDuration(r: any) {
);
}
interface Props {
location: any;
resources: any;
fetchList: any;
domContentLoadedTime: any;
loadTime: any;
playing: boolean;
domBuildingTime: any;
time: any;
}
let timeOut: any = null;
const TIMEOUT_DURATION = 5000;
function NetworkPanel() {
const { player, store } = React.useContext(PlayerContext)
@ -127,48 +143,104 @@ function NetworkPanel() {
resourceList: resources,
domContentLoadedTime,
loadTime,
time,
domBuildingTime,
fetchList: fetchUnmap,
} = store.get();
const fetchList = fetchUnmap.map((i: any) => Resource({ ...i.toJS(), type: TYPES.XHR }))
const fetchList = fetchUnmap.map((i: any) => Resource({ ...i.toJS(), type: TYPES.XHR, time: i.time < 0 ? 0 : i.time }))
const { showModal, hideModal } = useModal();
const [activeTab, setActiveTab] = useState(ALL);
const { showModal } = useModal();
const [sortBy, setSortBy] = useState('time');
const [sortAscending, setSortAscending] = useState(true);
const [filter, setFilter] = useState('');
const [showOnlyErrors, setShowOnlyErrors] = useState(false);
const onTabClick = (activeTab: any) => setActiveTab(activeTab);
const onFilterChange = ({ target: { value } }: any) => setFilter(value);
const [filteredList, setFilteredList] = useState([]);
const [isDetailsModalActive, setIsDetailsModalActive] = useState(false);
const additionalHeight = 0;
const fetchPresented = fetchList.length > 0;
const {
sessionStore: { devTools },
} = useStore();
const filter = devTools[INDEX_KEY].filter;
const activeTab = devTools[INDEX_KEY].activeTab;
const activeIndex = devTools[INDEX_KEY].index;
const [pauseSync, setPauseSync] = useState(activeIndex > 0);
const synRef: any = useRef({});
const resourcesSize = resources.reduce(
(sum: any, { decodedBodySize }: any) => sum + (decodedBodySize || 0),
0
);
const onTabClick = (activeTab: any) => devTools.update(INDEX_KEY, { activeTab });
const onFilterChange = ({ target: { value } }: any) => {
devTools.update(INDEX_KEY, { filter: value });
};
const transferredSize = resources.reduce(
(sum: any, { headerSize, encodedBodySize }: any) =>
sum + (headerSize || 0) + (encodedBodySize || 0),
0
);
synRef.current = {
pauseSync,
activeIndex,
};
const filterRE = getRE(filter, 'i');
let filtered = React.useMemo(() => {
const removePause = () => {
setIsDetailsModalActive(false);
clearTimeout(timeOut);
timeOut = setTimeout(() => {
devTools.update(INDEX_KEY, { index: getCurrentIndex() });
setPauseSync(false);
}, TIMEOUT_DURATION);
};
const onMouseLeave = () => {
if (isDetailsModalActive) return;
removePause();
};
useEffect(() => {
if (pauseSync) {
removePause();
}
return () => {
clearTimeout(timeOut);
if (!synRef.current.pauseSync) {
devTools.update(INDEX_KEY, { index: 0 });
}
};
}, []);
const getCurrentIndex = () => {
return filteredList.filter((item: any) => item.time <= time).length - 1;
};
useEffect(() => {
const currentIndex = getCurrentIndex();
if (currentIndex !== activeIndex && !pauseSync) {
devTools.update(INDEX_KEY, { index: currentIndex });
}
}, [time]);
const { resourcesSize, transferredSize } = useMemo(() => {
const resourcesSize = resources.reduce(
(sum: any, { decodedBodySize }: any) => sum + (decodedBodySize || 0),
0
);
const transferredSize = resources.reduce(
(sum: any, { headerSize, encodedBodySize }: any) =>
sum + (headerSize || 0) + (encodedBodySize || 0),
0
);
return {
resourcesSize,
transferredSize,
};
}, [resources]);
useEffect(() => {
const filterRE = getRE(filter, 'i');
let list = resources;
fetchList.forEach(
(fetchCall: any) =>
(list = list.filter((networkCall: any) => networkCall.url !== fetchCall.url))
);
// @ts-ignore
list = list.concat(fetchList);
list = list.sort((a: any, b: any) => {
return compare(a, b, sortBy);
});
if (!sortAscending) {
list = list.reverse();
}
list = list.filter(
({ type, name, status, success }: any) =>
@ -176,41 +248,53 @@ function NetworkPanel() {
(activeTab === ALL || type === TAB_TO_TYPE_MAP[activeTab]) &&
(showOnlyErrors ? parseInt(status) >= 400 || !success : true)
);
return list;
}, [filter, sortBy, sortAscending, showOnlyErrors, activeTab]);
setFilteredList(list);
}, [resources, filter, showOnlyErrors, activeTab]);
// const lastIndex = currentIndex || filtered.filter((item: any) => item.time <= time).length - 1;
const referenceLines = [];
if (domContentLoadedTime != null) {
referenceLines.push({
time: domContentLoadedTime.time,
color: DOM_LOADED_TIME_COLOR,
});
}
if (loadTime != null) {
referenceLines.push({
time: loadTime.time,
color: LOAD_TIME_COLOR,
});
}
const referenceLines = useMemo(() => {
const arr = [];
const onRowClick = (row: any) => {
showModal(<FetchDetailsModal resource={row} rows={filtered} fetchPresented={fetchPresented} />, {
right: true,
});
};
const handleSort = (sortKey: string) => {
if (sortKey === sortBy) {
setSortAscending(!sortAscending);
// setSortBy('time');
if (domContentLoadedTime != null) {
arr.push({
time: domContentLoadedTime.time,
color: DOM_LOADED_TIME_COLOR,
});
}
setSortBy(sortKey);
if (loadTime != null) {
arr.push({
time: loadTime.time,
color: LOAD_TIME_COLOR,
});
}
return arr;
}, []);
const showDetailsModal = (row: any) => {
setIsDetailsModalActive(true);
showModal(
<FetchDetailsModal resource={row} rows={filteredList} fetchPresented={fetchPresented} />,
{
right: true,
onClose: removePause,
}
);
devTools.update(INDEX_KEY, { index: filteredList.indexOf(row) });
setPauseSync(true);
};
useEffect(() => {
devTools.update(INDEX_KEY, { filter, activeTab });
}, [filter, activeTab]);
return (
<React.Fragment>
<BottomBlock style={{ height: 300 + additionalHeight + 'px' }} className="border">
<BottomBlock
style={{ height: 300 + additionalHeight + 'px' }}
className="border"
onMouseEnter={() => setPauseSync(true)}
onMouseLeave={onMouseLeave}
>
<BottomBlock.Header>
<div className="flex items-center">
<span className="font-semibold color-gray-medium mr-4">Network</span>
@ -231,6 +315,7 @@ function NetworkPanel() {
onChange={onFilterChange}
height={28}
width={230}
value={filter}
/>
</BottomBlock.Header>
<BottomBlock.Content>
@ -244,7 +329,7 @@ function NetworkPanel() {
/>
</div>
<InfoLine>
<InfoLine.Point label={filtered.length} value=" requests" />
<InfoLine.Point label={filteredList.length + ''} value=" requests" />
<InfoLine.Point
label={formatBytes(transferredSize)}
value="transferred"
@ -282,18 +367,22 @@ function NetworkPanel() {
</div>
}
size="small"
show={filtered.length === 0}
show={filteredList.length === 0}
>
<TimeTable
rows={filtered}
rows={filteredList}
referenceLines={referenceLines}
renderPopup
onRowClick={onRowClick}
onRowClick={showDetailsModal}
additionalHeight={additionalHeight}
onJump={(t: number) => player.jump(t)}
sortBy={sortBy}
sortAscending={sortAscending}
// activeIndex={lastIndex}
onJump={(row: any) => {
setPauseSync(true);
devTools.update(INDEX_KEY, { index: filteredList.indexOf(row) });
player.jump(row.time);
}}
activeIndex={activeIndex}
>
{[
// {
@ -305,28 +394,24 @@ function NetworkPanel() {
label: 'Status',
dataKey: 'status',
width: 70,
onClick: handleSort,
},
{
label: 'Type',
dataKey: 'type',
width: 90,
render: renderType,
onClick: handleSort,
},
{
label: 'Name',
width: 240,
dataKey: 'name',
render: renderName,
onClick: handleSort,
},
{
label: 'Size',
width: 80,
dataKey: 'decodedBodySize',
render: renderSize,
onClick: handleSort,
hidden: activeTab === XHR,
},
{
@ -334,7 +419,6 @@ function NetworkPanel() {
width: 80,
dataKey: 'duration',
render: renderDuration,
onClick: handleSort,
},
]}
</TimeTable>

203
frontend/app/components/shared/DevTools/StackEventPanel/StackEventPanel.tsx Normal file
View file

@ -0,0 +1,203 @@
import React, { useEffect, useMemo, useRef, useState } from 'react';
import { hideHint } from 'Duck/components/player';
import { Tooltip, Tabs, Input, NoContent, Icon, Toggler } from 'UI';
import { getRE } from 'App/utils';
import { List, CellMeasurer, CellMeasurerCache, AutoSizer } from 'react-virtualized';
import BottomBlock from '../BottomBlock';
import { connectPlayer, jump } from 'Player';
import { useModal } from 'App/components/Modal';
import { useStore } from 'App/mstore';
import { useObserver } from 'mobx-react-lite';
import { DATADOG, SENTRY, STACKDRIVER, typeList } from 'Types/session/stackEvent';
import { connect } from 'react-redux';
import StackEventRow from 'Shared/DevTools/StackEventRow';
import StackEventModal from '../StackEventModal';
let timeOut: any = null;
const TIMEOUT_DURATION = 5000;
const INDEX_KEY = 'stackEvent';
const ALL = 'ALL';
const TABS = [ALL, ...typeList].map((tab) => ({ text: tab, key: tab }));
interface Props {
list: any;
hideHint: any;
time: any;
}
function StackEventPanel(props: Props) {
const { list, time } = props;
const additionalHeight = 0;
const {
sessionStore: { devTools },
} = useStore();
const { showModal } = useModal();
const [isDetailsModalActive, setIsDetailsModalActive] = useState(false);
const [filteredList, setFilteredList] = useState([]);
const filter = useObserver(() => devTools[INDEX_KEY].filter);
const activeTab = useObserver(() => devTools[INDEX_KEY].activeTab);
const activeIndex = useObserver(() => devTools[INDEX_KEY].index);
const [pauseSync, setPauseSync] = useState(activeIndex > 0);
const synRef: any = useRef({});
synRef.current = {
pauseSync,
activeIndex,
};
const _list = React.useRef();
const onTabClick = (activeTab: any) => devTools.update(INDEX_KEY, { activeTab });
const onFilterChange = ({ target: { value } }: any) => {
devTools.update(INDEX_KEY, { filter: value });
};
const getCurrentIndex = () => {
return filteredList.filter((item: any) => item.time <= time).length - 1;
};
const removePause = () => {
clearTimeout(timeOut);
setIsDetailsModalActive(false);
timeOut = setTimeout(() => {
devTools.update(INDEX_KEY, { index: getCurrentIndex() });
setPauseSync(false);
}, TIMEOUT_DURATION);
};
useEffect(() => {
const currentIndex = getCurrentIndex();
if (currentIndex !== activeIndex && !pauseSync) {
devTools.update(INDEX_KEY, { index: currentIndex });
}
}, [time]);
const onMouseLeave = () => {
if (isDetailsModalActive) return;
removePause();
};
React.useMemo(() => {
const filterRE = getRE(filter, 'i');
let list = props.list;
list = list.filter(
({ name, source }: any) =>
(!!filter ? filterRE.test(name) : true) && (activeTab === ALL || activeTab === source)
);
setFilteredList(list);
}, [filter, activeTab]);
const tabs = useMemo(() => {
return TABS.filter(({ key }) => key === ALL || list.some(({ source }: any) => key === source));
}, []);
const cache = new CellMeasurerCache({
fixedWidth: true,
keyMapper: (index: number) => filteredList[index],
});
const showDetails = (item: any) => {
setIsDetailsModalActive(true);
showModal(<StackEventModal event={item} />, { right: true, onClose: removePause });
devTools.update(INDEX_KEY, { index: filteredList.indexOf(item) });
setPauseSync(true);
};
const _rowRenderer = ({ index, key, parent, style }: any) => {
const item = filteredList[index];
return (
// @ts-ignore
<CellMeasurer cache={cache} columnIndex={0} key={key} rowIndex={index} parent={parent}>
{() => (
<StackEventRow
isActive={activeIndex === index}
style={style}
key={item.key}
event={item}
onJump={() => {
setPauseSync(true);
devTools.update(INDEX_KEY, { index: filteredList.indexOf(item) });
jump(item.time);
}}
onClick={() => showDetails(item)}
/>
)}
</CellMeasurer>
);
};
useEffect(() => {
if (_list.current) {
// @ts-ignore
_list.current.scrollToRow(activeIndex);
}
}, [activeIndex]);
return (
<BottomBlock
style={{ height: 300 + additionalHeight + 'px' }}
onMouseEnter={() => setPauseSync(true)}
onMouseLeave={onMouseLeave}
>
<BottomBlock.Header>
<div className="flex items-center">
<span className="font-semibold color-gray-medium mr-4">Stack Events</span>
<Tabs tabs={tabs} active={activeTab} onClick={onTabClick} border={false} />
</div>
<Input
className="input-small h-8"
placeholder="Filter by keyword"
icon="search"
iconPosition="left"
name="filter"
height={28}
onChange={onFilterChange}
value={filter}
/>
</BottomBlock.Header>
<BottomBlock.Content className="overflow-y-auto">
<NoContent
title={
<div className="capitalize flex items-center mt-16">
<Icon name="info-circle" className="mr-2" size="18" />
No Data
</div>
}
size="small"
show={filteredList.length === 0}
>
<AutoSizer>
{({ height, width }: any) => (
<List
ref={_list}
deferredMeasurementCache={cache}
overscanRowCount={5}
rowCount={Math.ceil(filteredList.length || 1)}
rowHeight={cache.rowHeight}
rowRenderer={_rowRenderer}
width={width}
height={height}
scrollToAlignment="center"
/>
)}
</AutoSizer>
</NoContent>
</BottomBlock.Content>
</BottomBlock>
);
}
export default connect(
(state: any) => ({
hintIsHidden:
state.getIn(['components', 'player', 'hiddenHints', 'stack']) ||
!state.getIn(['site', 'list']).some((s: any) => s.stackIntegrations),
}),
{ hideHint }
)(
connectPlayer((state: any) => ({
list: state.stackList,
time: state.time,
}))(StackEventPanel)
);

1
frontend/app/components/shared/DevTools/StackEventPanel/index.ts Normal file
View file

@ -0,0 +1 @@
export { default } from './StackEventPanel';

17
frontend/app/components/shared/DevTools/StackEventRow/StackEventRow.tsx
View file

@ -3,21 +3,18 @@ import JumpButton from '../JumpButton';
import { Icon } from 'UI';
import cn from 'classnames';
import { OPENREPLAY, SENTRY, DATADOG, STACKDRIVER } from 'Types/session/stackEvent';
import { useModal } from 'App/components/Modal';
import StackEventModal from '../StackEventModal';
interface Props {
event: any;
onJump: any;
style?: any;
isActive?: boolean;
onClick?: any;
}
function StackEventRow(props: Props) {
const { event, onJump } = props;
const { event, onJump, style, isActive } = props;
let message = event.payload[0] || '';
message = typeof message === 'string' ? message : JSON.stringify(message);
const onClickDetails = () => {
showModal(<StackEventModal event={event} />, { right: true });
};
const { showModal } = useModal();
const iconProps: any = React.useMemo(() => {
const { source } = event;
@ -30,11 +27,13 @@ function StackEventRow(props: Props) {
return (
<div
style={style}
data-scroll-item={event.isRed()}
onClick={onClickDetails}
onClick={props.onClick}
className={cn(
'group flex items-center py-2 px-4 border-b cursor-pointer relative',
'hover:bg-active-blue'
'hover:bg-active-blue',
{ 'bg-teal-light': isActive }
)}
>
<div className={cn('mr-auto flex items-start')}>

74
frontend/app/components/shared/DevTools/TimeTable/BarRow.tsx
View file

@ -27,6 +27,40 @@ const BarRow = ({
}: Props) => {
const timeOffset = time - timestart;
ttfb = ttfb || 0;
// TODO fix the tooltip
const content = (
<React.Fragment>
{ttfb != null && (
<div className={styles.popupRow}>
<div className={styles.title}>{'Waiting (TTFB)'}</div>
<div className={styles.popupBarWrapper}>
<div
className={styles.ttfbBar}
style={{
left: 0,
width: `${percentOf(ttfb, duration)}%`,
}}
/>
</div>
<div className={styles.time}>{formatTime(ttfb)}</div>
</div>
)}
<div className={styles.popupRow}>
<div className={styles.title}>{'Content Download'}</div>
<div className={styles.popupBarWrapper}>
<div
className={styles.downloadBar}
style={{
left: `${percentOf(ttfb, duration)}%`,
width: `${percentOf(duration - ttfb, duration)}%`,
}}
/>
</div>
<div className={styles.time}>{formatTime(duration - ttfb)}</div>
</div>
</React.Fragment>
);
const trigger = (
<div
className={styles.barWrapper}
@ -60,44 +94,8 @@ const BarRow = ({
);
return (
<div key={key} className={tableStyles.row}>
<Tooltip
title={
<React.Fragment>
{ttfb != null && (
<div className={styles.popupRow}>
<div className={styles.title}>{'Waiting (TTFB)'}</div>
<div className={styles.popupBarWrapper}>
<div
className={styles.ttfbBar}
style={{
left: 0,
width: `${percentOf(ttfb, duration)}%`,
}}
/>
</div>
<div className={styles.time}>{formatTime(ttfb)}</div>
</div>
)}
<div className={styles.popupRow}>
<div className={styles.title}>{'Content Download'}</div>
<div className={styles.popupBarWrapper}>
<div
className={styles.downloadBar}
style={{
left: `${percentOf(ttfb, duration)}%`,
width: `${percentOf(duration - ttfb, duration)}%`,
}}
/>
</div>
<div className={styles.time}>{formatTime(duration - ttfb)}</div>
</div>
</React.Fragment>
}
placement="top"
>
{trigger}
</Tooltip>
<div key={key} className={tableStyles.row} style={{ height: '15px' }}>
{trigger}
</div>
);
};

71
frontend/app/components/shared/DevTools/TimeTable/TimeTable.tsx
View file

@ -72,8 +72,6 @@ type Props = {
hoverable?: boolean;
onRowClick?: (row: any, index: number) => void;
onJump?: (time: any) => void;
sortBy?: string;
sortAscending?: boolean;
};
type TimeLineInfo = {
@ -145,8 +143,19 @@ export default class TimeTable extends React.PureComponent<Props, State> {
scroller = React.createRef<List>();
autoScroll = true;
componentDidMount() {
if (this.scroller.current) {
// componentDidMount() {
// if (this.scroller.current) {
// this.scroller.current.scrollToRow(this.props.activeIndex);
// }
// }
adjustScroll(prevActiveIndex: number) {
if (
this.props.activeIndex &&
this.props.activeIndex >= 0 &&
prevActiveIndex !== this.props.activeIndex &&
this.scroller.current
) {
this.scroller.current.scrollToRow(this.props.activeIndex);
}
}
@ -161,14 +170,8 @@ export default class TimeTable extends React.PureComponent<Props, State> {
...computeTimeLine(this.props.rows, this.state.firstVisibleRowIndex, this.visibleCount),
});
}
if (
this.props.activeIndex &&
this.props.activeIndex >= 0 &&
prevProps.activeIndex !== this.props.activeIndex &&
this.scroller.current
) {
this.scroller.current.scrollToRow(this.props.activeIndex);
}
// this.adjustScroll(prevProps.activeIndex);
}
onScroll = ({
@ -190,7 +193,7 @@ export default class TimeTable extends React.PureComponent<Props, State> {
onJump = (index: any) => {
if (this.props.onJump) {
this.props.onJump(this.props.rows[index].time);
this.props.onJump(this.props.rows[index]);
}
};
@ -203,23 +206,29 @@ export default class TimeTable extends React.PureComponent<Props, State> {
<div
style={rowStyle}
key={key}
className={cn('border-b border-color-gray-light-shade group items-center', stl.row, {
[stl.hoverable]: hoverable,
'error color-red': !!row.isRed && row.isRed(),
'cursor-pointer': typeof onRowClick === 'function',
[stl.activeRow]: activeIndex === index,
// [stl.inactiveRow]: !activeIndex || index > activeIndex,
})}
className={cn(
'dev-row border-b border-color-gray-light-shade group items-center',
stl.row,
{
[stl.hoverable]: hoverable,
'error color-red': !!row.isRed && row.isRed(),
'cursor-pointer': typeof onRowClick === 'function',
[stl.activeRow]: activeIndex === index,
// [stl.inactiveRow]: !activeIndex || index > activeIndex,
}
)}
onClick={typeof onRowClick === 'function' ? () => onRowClick(row, index) : undefined}
id="table-row"
>
{columns.filter((i: any) => !i.hidden).map(({ dataKey, render, width }) => (
<div className={stl.cell} style={{ width: `${width}px` }}>
{render
? render(row)
: row[dataKey || ''] || <i className="color-gray-light">{'empty'}</i>}
</div>
))}
{columns
.filter((i: any) => !i.hidden)
.map(({ dataKey, render, width }) => (
<div className={stl.cell} style={{ width: `${width}px` }}>
{render
? render(row)
: row[dataKey || ''] || <i className="color-gray-light">{'empty'}</i>}
</div>
))}
<div className={cn('relative flex-1 flex', stl.timeBarWrapper)}>
<BarRow resource={row} timestart={timestart} timewidth={timewidth} popup={renderPopup} />
</div>
@ -270,8 +279,6 @@ export default class TimeTable extends React.PureComponent<Props, State> {
referenceLines = [],
additionalHeight = 0,
activeIndex,
sortBy = '',
sortAscending = true,
} = this.props;
const columns = this.props.children.filter((i: any) => !i.hidden);
const { timewidth, timestart } = this.state;
@ -324,10 +331,9 @@ export default class TimeTable extends React.PureComponent<Props, State> {
'cursor-pointer': typeof onClick === 'function',
})}
style={{ width: `${width}px` }}
onClick={() => this.onColumnClick(dataKey, onClick)}
// onClick={() => this.onColumnClick(dataKey, onClick)}
>
<span>{label}</span>
{!!sortBy && sortBy === dataKey && <Icon name={ sortAscending ? "caret-down-fill" : "caret-up-fill" } className="ml-1" />}
</div>
))}
</div>
@ -360,6 +366,7 @@ export default class TimeTable extends React.PureComponent<Props, State> {
<AutoSizer disableHeight>
{({ width }: { width: number }) => (
<List
scrollToIndex={this.props.activeIndex || 0}
ref={this.scroller}
className={stl.list}
height={this.tableHeight + additionalHeight}
@ -369,7 +376,7 @@ export default class TimeTable extends React.PureComponent<Props, State> {
rowHeight={ROW_HEIGHT}
rowRenderer={this.renderRow}
onScroll={this.onScroll}
scrollToAlignment="start"
scrollToAlignment="center"
forceUpdateProp={timestart | timewidth | (activeIndex || 0)}
/>
)}

12
frontend/app/components/shared/FetchDetailsModal/FetchDetailsModal.tsx
View file

@ -4,6 +4,7 @@ import { Button } from 'UI';
import FetchPluginMessage from './components/FetchPluginMessage';
import { TYPES } from 'Types/session/resource';
import FetchTabs from './components/FetchTabs/FetchTabs';
import { useStore } from 'App/mstore';
interface Props {
resource: any;
@ -15,6 +16,10 @@ function FetchDetailsModal(props: Props) {
const [resource, setResource] = useState(props.resource);
const [first, setFirst] = useState(false);
const [last, setLast] = useState(false);
const isXHR = resource.type === TYPES.XHR || resource.type === TYPES.FETCH;
const {
sessionStore: { devTools },
} = useStore();
useEffect(() => {
const index = rows.indexOf(resource);
@ -27,6 +32,7 @@ function FetchDetailsModal(props: Props) {
const index = rows.indexOf(resource);
if (index > 0) {
setResource(rows[index - 1]);
devTools.update('network', { index: index - 1 })
}
};
@ -34,6 +40,7 @@ function FetchDetailsModal(props: Props) {
const index = rows.indexOf(resource);
if (index < rows.length - 1) {
setResource(rows[index + 1]);
devTools.update('network', { index: index + 1 })
}
};
@ -42,9 +49,8 @@ function FetchDetailsModal(props: Props) {
<h5 className="mb-2 text-2xl">Network Request</h5>
<FetchBasicDetails resource={resource} />
{resource.type === TYPES.XHR && !fetchPresented && <FetchPluginMessage />}
{resource.type === TYPES.XHR && fetchPresented && <FetchTabs resource={resource} />}
{isXHR && !fetchPresented && <FetchPluginMessage />}
{isXHR && <FetchTabs resource={resource} />}
{rows && rows.length > 0 && (
<div className="flex justify-between absolute bottom-0 left-0 right-0 p-3 border-t bg-white">

22
frontend/app/components/shared/FetchDetailsModal/components/FetchBasicDetails/FetchBasicDetails.tsx
View file

@ -1,20 +1,27 @@
import React from 'react';
import { Icon } from 'UI';
import React, { useMemo } from 'react';
import { formatBytes } from 'App/utils';
import CopyText from 'Shared/CopyText';
import { TYPES } from 'Types/session/resource';
import cn from 'classnames';
interface Props {
resource: any;
}
function FetchBasicDetails({ resource }: Props) {
const _duration = parseInt(resource.duration);
const text = useMemo(() => {
if (resource.url.length > 50) {
const endText = resource.url.split('/').pop();
return resource.url.substring(0, 50 - endText.length) + '.../' + endText;
}
return resource.url;
}, [resource]);
return (
<div>
<div className="flex items-center py-1">
<div className="font-medium">Name</div>
<div className="rounded-lg bg-active-blue px-2 py-1 ml-2 whitespace-nowrap overflow-hidden text-clip cursor-pointer">
<CopyText content={resource.url}>{resource.name}</CopyText>
<CopyText content={resource.url}>{text}</CopyText>
</div>
</div>
@ -46,7 +53,12 @@ function FetchBasicDetails({ resource }: Props) {
{resource.status && (
<div className="flex items-center py-1">
<div className="font-medium">Status</div>
<div className="rounded bg-active-blue px-2 py-1 ml-2 whitespace-nowrap overflow-hidden text-clip flex items-center">
<div
className={cn(
'rounded bg-active-blue px-2 py-1 ml-2 whitespace-nowrap overflow-hidden text-clip flex items-center',
{ 'error color-red': !resource.success }
)}
>
{resource.status === '200' && (
<div className="w-4 h-4 bg-green rounded-full mr-2"></div>
)}

10
frontend/app/components/shared/SessionSearchQueryParamHandler/SessionSearchQueryParamHandler.tsx
View file

@ -49,10 +49,12 @@ const SessionSearchQueryParamHandler = React.memo((props: Props) => {
} else {
const _filters: any = { ...filtersMap };
const _filter = _filters[key];
_filter.value = valueArr;
_filter.operator = operator;
_filter.source = sourceArr;
props.addFilter(_filter);
if (!!_filter) {
_filter.value = valueArr;
_filter.operator = operator;
_filter.source = sourceArr;
props.addFilter(_filter);
}
}
}
};

4
frontend/app/components/ui/JSONTree/JSONTree.js
View file

@ -8,7 +8,7 @@ function updateObjectLink(obj) {
}
export default ({ src, ...props }) => (
<JSONTree
<JSONTree
name={ false }
collapsed={ 1 }
enableClipboard={ false }
@ -21,4 +21,4 @@ export default ({ src, ...props }) => (
iconStle="triangle"
{ ...props }
/>
);
);

2
frontend/app/components/ui/SVG.tsx
View file

@ -256,7 +256,7 @@ const SVG = (props: Props) => {
case 'id-card': return <svg viewBox="0 0 16 16" width={ `${ width }px` } height={ `${ height }px` } ><path d="M14.5 3a.5.5 0 0 1 .5.5v9a.5.5 0 0 1-.5.5h-13a.5.5 0 0 1-.5-.5v-9a.5.5 0 0 1 .5-.5h13zm-13-1A1.5 1.5 0 0 0 0 3.5v9A1.5 1.5 0 0 0 1.5 14h13a1.5 1.5 0 0 0 1.5-1.5v-9A1.5 1.5 0 0 0 14.5 2h-13z"/><path d="M3 8.5a.5.5 0 0 1 .5-.5h9a.5.5 0 0 1 0 1h-9a.5.5 0 0 1-.5-.5zm0 2a.5.5 0 0 1 .5-.5h6a.5.5 0 0 1 0 1h-6a.5.5 0 0 1-.5-.5zm0-5a.5.5 0 0 1 .5-.5h9a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-9a.5.5 0 0 1-.5-.5v-1z"/></svg>;
case 'image': return <svg viewBox="0 0 16 16" width={ `${ width }px` } height={ `${ height }px` } ><path d="M4.502 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3z"/><path d="M14.002 13a2 2 0 0 1-2 2h-10a2 2 0 0 1-2-2V5A2 2 0 0 1 2 3a2 2 0 0 1 2-2h10a2 2 0 0 1 2 2v8a2 2 0 0 1-1.998 2zM14 2H4a1 1 0 0 0-1 1h9.002a2 2 0 0 1 2 2v7A1 1 0 0 0 15 11V3a1 1 0 0 0-1-1zM2.002 4a1 1 0 0 0-1 1v8l2.646-2.354a.5.5 0 0 1 .63-.062l2.66 1.773 3.71-3.71a.5.5 0 0 1 .577-.094l1.777 1.947V5a1 1 0 0 0-1-1h-10z"/></svg>;
case 'info-circle-fill': return <svg viewBox="0 0 36 36" width={ `${ width }px` } height={ `${ height }px` } ><path d="M17.75 35.5a17.75 17.75 0 1 0 0-35.5 17.75 17.75 0 0 0 0 35.5Zm2.064-20.883-2.22 10.44c-.155.754.065 1.182.675 1.182.43 0 1.08-.155 1.522-.546l-.195.923c-.637.768-2.041 1.327-3.25 1.327-1.56 0-2.224-.937-1.793-2.927l1.637-7.694c.142-.65.014-.886-.637-1.043l-1-.18.182-.845 5.08-.637h-.002Zm-2.064-2.414a2.219 2.219 0 1 1 0-4.437 2.219 2.219 0 0 1 0 4.437Z"/></svg>;
case 'info-circle': return <svg viewBox="0 0 35 35" width={ `${ width }px` } height={ `${ height }px` } ><g clipPath="url(#a)"><path d="M17.5 32.813a15.313 15.313 0 1 1 0-30.626 15.313 15.313 0 0 1 0 30.625Zm0 2.187a17.5 17.5 0 1 0 0-35 17.5 17.5 0 0 0 0 35Z"/><path clipRule="evenodd" d="M17.5 13a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm1.5 2.877a1.5 1.5 0 1 0-3 0V24.5a1.5 1.5 0 0 0 3 0v-8.623Z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h35v35H0z"/></clipPath></defs></svg>;
case 'info-circle': return <svg viewBox="0 0 16 16" width={ `${ width }px` } height={ `${ height }px` } ><path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/><path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/></svg>;
case 'info-square': return <svg viewBox="0 0 16 16" width={ `${ width }px` } height={ `${ height }px` } ><path d="M14 1a1 1 0 0 1 1 1v12a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1h12zM2 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V2a2 2 0 0 0-2-2H2z"/><path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/></svg>;
case 'info': return <svg viewBox="0 0 16 16" width={ `${ width }px` } height={ `${ height }px` } ><path d="M14 1a1 1 0 0 1 1 1v12a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1h12zM2 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V2a2 2 0 0 0-2-2H2z"/><path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/></svg>;
case 'inspect': return <svg viewBox="0 0 512 512" width={ `${ width }px` } height={ `${ height }px` } ><path d="M506 240h-34.591C463.608 133.462 378.538 48.392 272 40.591V6a6 6 0 0 0-6-6h-20a6 6 0 0 0-6 6v34.591C133.462 48.392 48.392 133.462 40.591 240H6a6 6 0 0 0-6 6v20a6 6 0 0 0 6 6h34.591C48.392 378.538 133.462 463.608 240 471.409V506a6 6 0 0 0 6 6h20a6 6 0 0 0 6-6v-34.591C378.538 463.608 463.608 378.538 471.409 272H506a6 6 0 0 0 6-6v-20a6 6 0 0 0-6-6zM272 439.305V374a6 6 0 0 0-6-6h-20a6 6 0 0 0-6 6v65.305C151.282 431.711 80.315 361.031 72.695 272H138a6 6 0 0 0 6-6v-20a6 6 0 0 0-6-6H72.695C80.289 151.282 150.969 80.316 240 72.695V138a6 6 0 0 0 6 6h20a6 6 0 0 0 6-6V72.695C360.718 80.289 431.685 150.969 439.305 240H374a6 6 0 0 0-6 6v20a6 6 0 0 0 6 6h65.305C431.711 360.718 361.031 431.684 272 439.305zM280 256c0 13.255-10.745 24-24 24s-24-10.745-24-24 10.745-24 24-24 24 10.745 24 24z"/></svg>;

148
frontend/app/mstore/sessionStore.ts
View file

@ -5,75 +5,105 @@ import Session from './types/session';
import Record, { LAST_7_DAYS } from 'Types/app/period';
class UserFilter {
endDate: number = new Date().getTime();
startDate: number = new Date().getTime() - 24 * 60 * 60 * 1000;
rangeName: string = LAST_7_DAYS;
filters: any = [];
page: number = 1;
limit: number = 10;
period: any = Record({ rangeName: LAST_7_DAYS });
endDate: number = new Date().getTime();
startDate: number = new Date().getTime() - 24 * 60 * 60 * 1000;
rangeName: string = LAST_7_DAYS;
filters: any = [];
page: number = 1;
limit: number = 10;
period: any = Record({ rangeName: LAST_7_DAYS });
constructor() {
makeAutoObservable(this, {
page: observable,
update: action,
});
constructor() {
makeAutoObservable(this, {
page: observable,
update: action,
});
}
update(key: string, value: any) {
// @ts-ignore
this[key] = value;
if (key === 'period') {
this.startDate = this.period.start;
this.endDate = this.period.end;
}
}
update(key: string, value: any) {
this[key] = value;
setFilters(filters: any[]) {
this.filters = filters;
}
if (key === 'period') {
this.startDate = this.period.start;
this.endDate = this.period.end;
}
}
setPage(page: number) {
this.page = page;
}
setFilters(filters: any[]) {
this.filters = filters;
}
toJson() {
return {
endDate: this.period.end,
startDate: this.period.start,
filters: this.filters.map(filterMap),
page: this.page,
limit: this.limit,
};
}
}
setPage(page: number) {
this.page = page;
}
interface BaseDevState {
index: number;
filter: string;
activeTab: string;
isError: boolean;
}
toJson() {
return {
endDate: this.period.end,
startDate: this.period.start,
filters: this.filters.map(filterMap),
page: this.page,
limit: this.limit,
};
}
class DevTools {
network: BaseDevState;
stackEvent: BaseDevState;
console: BaseDevState;
constructor() {
this.network = { index: 0, filter: '', activeTab: 'ALL', isError: false };
this.stackEvent = { index: 0, filter: '', activeTab: 'ALL', isError: false };
this.console = { index: 0, filter: '', activeTab: 'ALL', isError: false };
makeAutoObservable(this, {
update: action,
});
}
update(key: string, value: any) {
// @ts-ignore
this[key] = Object.assign(this[key], value);
}
}
export default class SessionStore {
userFilter: UserFilter = new UserFilter();
userFilter: UserFilter = new UserFilter();
devTools: DevTools = new DevTools();
constructor() {
makeAutoObservable(this, {
userFilter: observable,
constructor() {
makeAutoObservable(this, {
userFilter: observable,
devTools: observable,
});
}
resetUserFilter() {
this.userFilter = new UserFilter();
}
getSessions(filter: any): Promise<any> {
return new Promise((resolve, reject) => {
sessionService
.getSessions(filter.toJson())
.then((response: any) => {
resolve({
sessions: response.sessions.map((session: any) => new Session().fromJson(session)),
total: response.total,
});
})
.catch((error: any) => {
reject(error);
});
}
resetUserFilter() {
this.userFilter = new UserFilter();
}
getSessions(filter: any): Promise<any> {
return new Promise((resolve, reject) => {
sessionService
.getSessions(filter.toJson())
.then((response: any) => {
resolve({
sessions: response.sessions.map((session: any) => new Session().fromJson(session)),
total: response.total,
});
})
.catch((error: any) => {
reject(error);
});
});
}
});
}
}

5
frontend/app/mstore/types/funnelStage.ts
View file

@ -24,10 +24,11 @@ export default class FunnelStage {
}
fromJSON(json: any, total: number = 0, previousSessionCount: number = 0) {
this.dropDueToIssues = json.dropDueToIssues;
previousSessionCount = previousSessionCount || 0;
this.dropDueToIssues = json.dropDueToIssues || 0;
this.dropPct = json.dropPct;
this.operator = json.operator;
this.sessionsCount = json.sessionsCount;
this.sessionsCount = json.sessionsCount || 0;
this.usersCount = json.usersCount;
this.value = json.value;
this.type = json.type;

2
frontend/app/mstore/types/widget.ts
View file

@ -6,7 +6,7 @@ import Session from "App/mstore/types/session";
import Funnelissue from 'App/mstore/types/funnelIssue';
import { issueOptions } from 'App/constants/filterOptions';
import { FilterKey } from 'Types/filter/filterType';
import Period, { LAST_24_HOURS, LAST_30_DAYS } from 'Types/app/period';
import Period, { LAST_24_HOURS } from 'Types/app/period';
export default class Widget {
public static get ID_KEY():string { return "metricId" }

4
frontend/app/player/_singletone.ts
View file

@ -24,7 +24,7 @@ export function init(session, config, live = false) {
export function clean() {
if (instance === null) return;
instance.clean();
cleanStore()
cleanStore();
instance = null;
}
@ -88,4 +88,4 @@ export const Controls = {
speedUp,
speedDown,
callPeer
}
}

8
frontend/app/player/web/MessageManager.ts
View file

@ -24,7 +24,7 @@ import { decryptSessionBytes } from './network/crypto';
import Lists, { INITIAL_STATE as LISTS_INITIAL_STATE, State as ListsState } from './Lists';
import Screen, {
import Screen, {
INITIAL_STATE as SCREEN_INITIAL_STATE,
State as ScreenState,
} from './Screen/Screen';
@ -117,7 +117,7 @@ export default class MessageManager {
private lastMessageInFileTime: number = 0;
constructor(
private readonly session: any /*Session*/,
private readonly session: any /*Session*/,
private readonly state: Store<State>,
private readonly screen: Screen,
initialLists?: Partial<InitialLists>
@ -293,7 +293,7 @@ export default class MessageManager {
/* == REFACTOR_ME == */
const lastLoadedLocationMsg = this.loadedLocationManager.moveGetLast(t, index);
if (!!lastLoadedLocationMsg) {
// TODO: page-wise resources list // setListsStartTime(lastLoadedLocationMsg.time)
// TODO: page-wise resources list // setListsStartTime(lastLoadedLocationMsg.time)
this.navigationStartOffset = lastLoadedLocationMsg.navigationStart - this.sessionStart;
}
const llEvent = this.locationEventManager.moveGetLast(t, index);
@ -525,7 +525,7 @@ export default class MessageManager {
this.state.update({ cssLoading });
}
private setSize({ height, width }: { height: number, width: number }) {
private setSize({ height, width }: { height: number, width: number }) {
this.screen.scale({ height, width });
this.state.update({ width, height });

42
frontend/app/player/web/assist/ListWalkerWithMarks.ts Normal file
View file

@ -0,0 +1,42 @@
import type { Timed } from './types';
import ListWalker from './ListWalker'
type CheckFn<T> = (t: T) => boolean
export default class ListWalkerWithMarks<T extends Timed> extends ListWalker<T> {
private _markCountNow: number = 0
private _markCount: number = 0
constructor(private isMarked: CheckFn<T>, initialList: T[] = []) {
super(initialList)
this._markCount = initialList.reduce((n, item) => isMarked(item) ? n+1 : n, 0)
}
append(item: T) {
if (this.isMarked(item)) { this._markCount++ }
super.append(item)
}
protected moveNext() {
const val = super.moveNext()
if (val && this.isMarked(val)) {
this._markCountNow++
}
return val
}
protected movePrev() {
const val = super.movePrev()
if (val && this.isMarked(val)) {
this._markCountNow--
}
return val
}
get markedCountNow(): number {
return this._markCountNow
}
get markedCount(): number {
return this._markCount
}
}

4
frontend/app/styles/general.css
View file

@ -355,4 +355,8 @@ p {
width: 80px;
height: 80px;
transform: rotate(45deg);
}
.dev-row {
transition: all 0.5s;
}

15
frontend/app/svg/icons/info-circle.svg
View file

@ -1,11 +1,4 @@
<svg viewBox="0 0 35 35" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_10_16)">
<path d="M17.5 32.8125C13.4389 32.8125 9.54408 31.1992 6.67243 28.3276C3.80078 25.4559 2.1875 21.5611 2.1875 17.5C2.1875 13.4389 3.80078 9.54408 6.67243 6.67243C9.54408 3.80078 13.4389 2.1875 17.5 2.1875C21.5611 2.1875 25.4559 3.80078 28.3276 6.67243C31.1992 9.54408 32.8125 13.4389 32.8125 17.5C32.8125 21.5611 31.1992 25.4559 28.3276 28.3276C25.4559 31.1992 21.5611 32.8125 17.5 32.8125ZM17.5 35C22.1413 35 26.5925 33.1563 29.8744 29.8744C33.1563 26.5925 35 22.1413 35 17.5C35 12.8587 33.1563 8.40752 29.8744 5.12563C26.5925 1.84375 22.1413 0 17.5 0C12.8587 0 8.40752 1.84375 5.12563 5.12563C1.84375 8.40752 0 12.8587 0 17.5C0 22.1413 1.84375 26.5925 5.12563 29.8744C8.40752 33.1563 12.8587 35 17.5 35V35Z" />
<path fill-rule="evenodd" clip-rule="evenodd" d="M17.5 13C18.3284 13 19 12.3284 19 11.5C19 10.6716 18.3284 10 17.5 10C16.6716 10 16 10.6716 16 11.5C16 12.3284 16.6716 13 17.5 13ZM19 15.877C19 15.0485 18.3284 14.377 17.5 14.377C16.6716 14.377 16 15.0485 16 15.877V24.5C16 25.3284 16.6716 26 17.5 26C18.3284 26 19 25.3284 19 24.5V15.877Z" />
</g>
<defs>
<clipPath id="clip0_10_16">
<rect width="35" height="35" fill="white"/>
</clipPath>
</defs>
</svg>
<svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="bi bi-info-circle" viewBox="0 0 16 16">
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
</svg>
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 451 B

2
frontend/app/types/dashboard/helper.js
View file

@ -27,7 +27,7 @@ const weekdays = [ 'Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat' ];
// const months = [ "January", "February" ];
export const getTimeString = (ts, period) => {
const date = new Date(ts);
const diff = period.end - period.start;
const diff = period.endTimestamp - period.startTimestamp;
if (diff <= DAY) {
var isPM = date.getHours() >= 12;
return `${ isPM ? date.getHours() - 12 : date.getHours() }:${ startWithZero(date.getMinutes()) } ${isPM? 'pm' : 'am'}`;

1
frontend/package.json
View file

@ -91,6 +91,7 @@
"@types/react-dom": "^18.0.4",
"@types/react-redux": "^7.1.24",
"@types/react-router-dom": "^5.3.3",
"@types/react-virtualized": "^9.21.21",
"@typescript-eslint/eslint-plugin": "^5.24.0",
"@typescript-eslint/parser": "^5.24.0",
"autoprefixer": "^10.4.7",

200
peers/package-lock.json generated
View file

@ -10,87 +10,7 @@
"license": "Elastic License 2.0 (ELv2)",
"dependencies": {
"express": "^4.18.1",
"peer": "^0.6.1"
}
},
"node_modules/@types/body-parser": {
"version": "1.19.2",
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz",
"integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==",
"dependencies": {
"@types/connect": "*",
"@types/node": "*"
}
},
"node_modules/@types/connect": {
"version": "3.4.35",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
"integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/cors": {
"version": "2.8.12",
"resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz",
"integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw=="
},
"node_modules/@types/express": {
"version": "4.17.13",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz",
"integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==",
"dependencies": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^4.17.18",
"@types/qs": "*",
"@types/serve-static": "*"
}
},
"node_modules/@types/express-serve-static-core": {
"version": "4.17.30",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.30.tgz",
"integrity": "sha512-gstzbTWro2/nFed1WXtf+TtrpwxH7Ggs4RLYTLbeVgIkUQOI3WG/JKjgeOU1zXDvezllupjrf8OPIdvTbIaVOQ==",
"dependencies": {
"@types/node": "*",
"@types/qs": "*",
"@types/range-parser": "*"
}
},
"node_modules/@types/mime": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz",
"integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA=="
},
"node_modules/@types/node": {
"version": "18.7.16",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz",
"integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg=="
},
"node_modules/@types/qs": {
"version": "6.9.7",
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz",
"integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw=="
},
"node_modules/@types/range-parser": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz",
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw=="
},
"node_modules/@types/serve-static": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz",
"integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==",
"dependencies": {
"@types/mime": "*",
"@types/node": "*"
}
},
"node_modules/@types/ws": {
"version": "7.4.7",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz",
"integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==",
"dependencies": {
"@types/node": "*"
"peer": "^v1.0.0-rc.4"
}
},
"node_modules/accepts": {
@ -655,17 +575,12 @@
"integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
},
"node_modules/peer": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz",
"integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==",
"version": "1.0.0-rc.4",
"resolved": "https://registry.npmjs.org/peer/-/peer-1.0.0-rc.4.tgz",
"integrity": "sha512-xaNIDm3yWR5m8cuijK7jEFAMOWqNJDGSVJ0+Y3qKW5XTNYsNWEdqtg/Btq9eznGxTTeqQZGNw/SxwyrCVdmmDg==",
"dependencies": {
"@types/cors": "^2.8.6",
"@types/express": "^4.17.3",
"@types/ws": "^7.2.3",
"body-parser": "^1.19.0",
"cors": "^2.8.5",
"express": "^4.17.1",
"uuid": "^3.4.0",
"ws": "^7.2.3",
"yargs": "^15.3.1"
},
@ -673,7 +588,7 @@
"peerjs": "bin/peerjs"
},
"engines": {
"node": ">=10"
"node": ">=14"
}
},
"node_modules/proxy-addr": {
@ -894,15 +809,6 @@
"node": ">= 0.4.0"
}
},
"node_modules/uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
"deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.",
"bin": {
"uuid": "bin/uuid"
}
},
"node_modules/vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
@ -989,86 +895,6 @@
}
},
"dependencies": {
"@types/body-parser": {
"version": "1.19.2",
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz",
"integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==",
"requires": {
"@types/connect": "*",
"@types/node": "*"
}
},
"@types/connect": {
"version": "3.4.35",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
"integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==",
"requires": {
"@types/node": "*"
}
},
"@types/cors": {
"version": "2.8.12",
"resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz",
"integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw=="
},
"@types/express": {
"version": "4.17.13",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz",
"integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==",
"requires": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^4.17.18",
"@types/qs": "*",
"@types/serve-static": "*"
}
},
"@types/express-serve-static-core": {
"version": "4.17.30",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.30.tgz",
"integrity": "sha512-gstzbTWro2/nFed1WXtf+TtrpwxH7Ggs4RLYTLbeVgIkUQOI3WG/JKjgeOU1zXDvezllupjrf8OPIdvTbIaVOQ==",
"requires": {
"@types/node": "*",
"@types/qs": "*",
"@types/range-parser": "*"
}
},
"@types/mime": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz",
"integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA=="
},
"@types/node": {
"version": "18.7.16",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz",
"integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg=="
},
"@types/qs": {
"version": "6.9.7",
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz",
"integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw=="
},
"@types/range-parser": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz",
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw=="
},
"@types/serve-static": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz",
"integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==",
"requires": {
"@types/mime": "*",
"@types/node": "*"
}
},
"@types/ws": {
"version": "7.4.7",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz",
"integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==",
"requires": {
"@types/node": "*"
}
},
"accepts": {
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@ -1482,17 +1308,12 @@
"integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
},
"peer": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/peer/-/peer-0.6.1.tgz",
"integrity": "sha512-zPJSPoZvo+83sPJNrW8o93QTktx7dKk67965RRDDNAIelWw1ZwE6ZmmhsvRrdNRlK0knQb3rR8GBdZlbWzCYJw==",
"version": "1.0.0-rc.4",
"resolved": "https://registry.npmjs.org/peer/-/peer-1.0.0-rc.4.tgz",
"integrity": "sha512-xaNIDm3yWR5m8cuijK7jEFAMOWqNJDGSVJ0+Y3qKW5XTNYsNWEdqtg/Btq9eznGxTTeqQZGNw/SxwyrCVdmmDg==",
"requires": {
"@types/cors": "^2.8.6",
"@types/express": "^4.17.3",
"@types/ws": "^7.2.3",
"body-parser": "^1.19.0",
"cors": "^2.8.5",
"express": "^4.17.1",
"uuid": "^3.4.0",
"ws": "^7.2.3",
"yargs": "^15.3.1"
}
@ -1655,11 +1476,6 @@
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
"integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="
},
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
},
"vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",

2
peers/package.json
View file

@ -19,6 +19,6 @@
"homepage": "https://github.com/openreplay/openreplay#readme",
"dependencies": {
"express": "^4.18.1",
"peer": "^0.6.1"
"peer": "^v1.0.0-rc.4"
}
}

9
scripts/helmcharts/openreplay/files/minio.sh
View file

@ -5,7 +5,7 @@ set -e
cd /tmp
buckets=("mobs" "sessions-assets" "static" "sourcemaps" "sessions-mobile-assets" "quickwit" "vault-data")
buckets=("mobs" "sessions-assets" "sourcemaps" "sessions-mobile-assets" "quickwit" "vault-data")
mc alias set minio http://minio.db.svc.cluster.local:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
@ -31,11 +31,14 @@ mc mb minio/${bucket} || true
done
mc ilm import minio/mobs < /tmp/lifecycle.json || true
# Creating frontend bucket
#####################################################
# Creating frontend bucket; Do not change this block!
# !! PUBLIC BUCKETS !!
#####################################################
mc mb minio/frontend || true
mc policy set download minio/frontend || true
mc policy set download minio/sessions-assets || true
mc policy set download minio/static || true
}
# /bin/bash kafka.sh migrate $migration_versions

7
scripts/schema/db/init_dbs/postgresql/1.9.0/1.9.0.sql
View file

@ -60,4 +60,11 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
DROP INDEX IF EXISTS events_common.requests_url_idx;
DROP INDEX IF EXISTS events_common.requests_url_gin_idx;
DROP INDEX IF EXISTS events_common.requests_url_gin_idx2;
DROP INDEX IF EXISTS events.resources_url_gin_idx;
DROP INDEX IF EXISTS events.resources_url_idx;
COMMIT;

10
scripts/schema/db/init_dbs/postgresql/init_schema.sql
View file

@ -596,17 +596,9 @@ $$
query text NULL,
PRIMARY KEY (session_id, timestamp, seq_index)
);
CREATE INDEX requests_url_idx ON events_common.requests (url);
CREATE INDEX requests_duration_idx ON events_common.requests (duration);
CREATE INDEX requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops);
CREATE INDEX requests_timestamp_idx ON events_common.requests (timestamp);
CREATE INDEX requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, length(url) - (CASE
WHEN url LIKE 'http://%'
THEN 7
WHEN url LIKE 'https://%'
THEN 8
ELSE 0 END))
gin_trgm_ops);
CREATE INDEX requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE;
CREATE INDEX requests_request_body_nn_gin_idx ON events_common.requests USING GIN (request_body gin_trgm_ops) WHERE request_body IS NOT NULL;
CREATE INDEX requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL;

4
sourcemap-uploader/cli.js
View file

@ -55,10 +55,10 @@ dir.addArgument(['-u', '--js-dir-url'], {
// TODO: exclude in dir
const { command, api_key, project_key, server, verbose, ...args } =
const { command, api_key, project_key, server, logs, ...args } =
parser.parseArgs();
global._VERBOSE = !!verbose;
global._VERBOSE = !!logs;
(command === 'file'
? uploadFile(

2
sourcemap-uploader/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@openreplay/sourcemap-uploader",
"version": "3.0.6",
"version": "3.0.7",
"description": "NPM module to upload your JS sourcemaps files to OpenReplay",
"bin": "cli.js",
"main": "index.js",

63
tracker/tracker/src/main/app/index.ts
View file

@ -164,7 +164,7 @@ export default class App {
this.worker.onmessage = ({ data }: MessageEvent<FromWorkerData>) => {
if (data === 'restart') {
this.stop(false)
this.start({ forceNew: true }) // TODO: keep userID & metadata (draw scenarios)
this.start({}, true)
} else if (data.type === 'failure') {
this.stop(false)
this._debug('worker_failed', data.reason)
@ -201,7 +201,6 @@ export default class App {
send(message: Message, urgent = false): void {
if (this.activityState === ActivityState.NotActive) {
// this.debug.log('SendiTrying to send when not active', message) <- crashing the app
return
}
this.messages.push(message)
@ -370,7 +369,7 @@ export default class App {
this.sessionStorage.removeItem(this.options.session_reset_key)
}
}
private _start(startOpts: StartOptions): Promise<StartPromiseReturn> {
private _start(startOpts: StartOptions = {}, resetByWorker = false): Promise<StartPromiseReturn> {
if (!this.worker) {
return Promise.resolve(UnsuccessfulStart('No worker found: perhaps, CSP is not set.'))
}
@ -382,9 +381,19 @@ export default class App {
)
}
this.activityState = ActivityState.Starting
if (startOpts.sessionHash) {
this.session.applySessionHash(startOpts.sessionHash)
}
if (startOpts.forceNew) {
// Reset session metadata only if requested directly
this.session.reset()
}
this.session.assign({
// MBTODO: maybe it would make sense to `forceNew` if the `userID` was changed
userID: startOpts.userID,
metadata: startOpts.metadata,
})
const timestamp = now()
this.worker.postMessage({
@ -397,17 +406,9 @@ export default class App {
connAttemptGap: this.options.connAttemptGap,
})
this.session.update({
// TODO: transparent "session" module logic AND explicit internal api for plugins.
// "updating" with old metadata in order to trigger session's UpdateCallbacks.
// (for the case of internal .start() calls, like on "restart" webworker signal or assistent connection in tracker-assist )
metadata: startOpts.metadata || this.session.getInfo().metadata,
userID: startOpts.userID,
})
const sReset = this.sessionStorage.getItem(this.options.session_reset_key)
const lsReset = this.sessionStorage.getItem(this.options.session_reset_key) !== null
this.sessionStorage.removeItem(this.options.session_reset_key)
const shouldReset = startOpts.forceNew || sReset !== null
const needNewSessionID = startOpts.forceNew || lsReset || resetByWorker
return window
.fetch(this.options.ingestPoint + '/v1/web/start', {
@ -419,7 +420,7 @@ export default class App {
...this.getTrackerInfo(),
timestamp,
userID: this.session.getInfo().userID,
token: shouldReset ? undefined : this.session.getSessionToken(),
token: needNewSessionID ? undefined : this.session.getSessionToken(),
deviceMemory,
jsHeapSizeLimit,
}),
@ -447,29 +448,33 @@ export default class App {
const {
token,
userUUID,
sessionID,
projectID,
beaconSizeLimit,
startTimestamp, // real startTS, derived from sessionID
delay,
delay, // derived from token
sessionID, // derived from token
startTimestamp, // real startTS (server time), derived from sessionID
} = r
if (
typeof token !== 'string' ||
typeof userUUID !== 'string' ||
//typeof startTimestamp !== 'number' ||
//typeof sessionID !== 'string' ||
(typeof startTimestamp !== 'number' && typeof startTimestamp !== 'undefined') ||
typeof sessionID !== 'string' ||
typeof delay !== 'number' ||
(typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')
) {
return Promise.reject(`Incorrect server response: ${JSON.stringify(r)}`)
}
this.delay = delay
const prevSessionID = this.session.getInfo().sessionID
if (prevSessionID && prevSessionID !== sessionID) {
this.session.reset()
}
this.session.setSessionToken(token)
this.session.update({ sessionID, timestamp: startTimestamp || timestamp, projectID }) // TODO: no no-explicit 'any'
this.session.assign({
sessionID,
timestamp: startTimestamp || timestamp,
projectID,
})
// (Re)send Metadata for the case of a new session
Object.entries(this.session.getInfo().metadata).forEach(([key, value]) =>
this.send(Metadata(key, value)),
)
this.localStorage.setItem(this.options.local_uuid_key, userUUID)
this.worker.postMessage({
@ -506,15 +511,15 @@ export default class App {
})
}
start(options: StartOptions = {}): Promise<StartPromiseReturn> {
start(...args: Parameters<App['_start']>): Promise<StartPromiseReturn> {
if (!document.hidden) {
return this._start(options)
return this._start(...args)
} else {
return new Promise((resolve) => {
const onVisibilityChange = () => {
if (!document.hidden) {
document.removeEventListener('visibilitychange', onVisibilityChange)
resolve(this._start(options))
resolve(this._start(...args))
}
}
document.addEventListener('visibilitychange', onVisibilityChange)
@ -538,8 +543,4 @@ export default class App {
}
}
}
restart() {
this.stop(false)
this.start({ forceNew: false })
}
}

2
tracker/tracker/src/main/app/session.ts
View file

@ -37,7 +37,7 @@ export default class Session {
this.callbacks.forEach((cb) => cb(newInfo))
}
update(newInfo: Partial<SessionInfo>): void {
assign(newInfo: Partial<SessionInfo>): void {
if (newInfo.userID !== undefined) {
// TODO clear nullable/undefinable types
this.userID = newInfo.userID