jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/api-v1.5.5' into dev

Browse source
This commit is contained in:
Taha Yassine Kraiem 2022-04-13 19:26:14 +02:00
commit 008c020f56
44 changed files with 4049 additions and 2299 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api/.env.default
View file

@ -44,6 +44,6 @@ sentryURL=
sessions_bucket=mobs
sessions_region=us-east-1
sourcemaps_bucket=sourcemaps
sourcemaps_reader=http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps
sourcemaps_reader=http://127.0.0.1:9000/
stage=default-foss
version_number=1.4.0

8
api/Dockerfile
View file

@ -6,6 +6,14 @@ COPY . .
RUN pip install -r requirements.txt
RUN mv .env.default .env
ENV APP_NAME chalice
# Installing Nodejs
RUN apt update && apt install -y curl && \
curl -fsSL https://deb.nodesource.com/setup_12.x | bash - && \
apt install -y nodejs && \
apt remove --purge -y curl && \
rm -rf /var/lib/apt/lists/* && \
cd sourcemap-reader && \
npm install
# Add Tini
# Startup daemon

2
api/build.sh
View file

@ -18,6 +18,8 @@ check_prereq() {
}
function build_api(){
cp -R ../utilities/utils ../sourcemap-reader/.
cp -R ../sourcemap-reader .
tag=""
# Copy enterprise code
[[ $1 == "ee" ]] && {

252
api/chalicelib/core/dashboard.py
View file

@ -1483,7 +1483,7 @@ def get_avg_cpu(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
cur.execute(cur.mogrify(pg_query, params))
avg = cur.fetchone()["avg"]
return {"value": avg, "chart": helper.list_to_camel_case(rows),
"unit": schemas.TemplatePredefinedUnits.percentage}
"unit": schemas.TemplatePredefinedUnits.percentage}
def get_avg_fps(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
@ -1784,7 +1784,7 @@ def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1)
WHERE {" AND ".join(pg_sub_query)}
GROUP BY resources.url_host
ORDER BY errors_count DESC
LIMIT 10;"""
LIMIT 5;"""
cur.execute(cur.mogrify(pg_query, {"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp, **__get_constraint_values(args)}))
@ -2265,7 +2265,7 @@ def get_application_activity_avg_image_load_time(project_id, startTimestamp=Time
with pg_client.PostgresClient() as cur:
row = __get_application_activity_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args)
results = row
results["chart"] = get_performance_avg_image_load_time(project_id, startTimestamp, endTimestamp, **args)
results["chart"] = get_performance_avg_image_load_time(cur, project_id, startTimestamp, endTimestamp, **args)
diff = endTimestamp - startTimestamp
endTimestamp = startTimestamp
startTimestamp = endTimestamp - diff
@ -2276,7 +2276,7 @@ def get_application_activity_avg_image_load_time(project_id, startTimestamp=Time
return results
def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
def get_performance_avg_image_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
density=19, **args):
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1)
@ -2286,34 +2286,33 @@ def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(d
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
with pg_client.PostgresClient() as cur:
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
chart=False, data=args)
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
chart=True, data=args, main_table="resources", time_column="timestamp",
duration=False)
pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s")
pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s")
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
chart=False, data=args)
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
chart=True, data=args, main_table="resources", time_column="timestamp",
duration=False)
pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s")
pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s")
pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp
FROM events.resources INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_subset)}
AND resources.type = 'img' AND resources.duration>0
{(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""}
)
SELECT generated_timestamp AS timestamp,
COALESCE(AVG(resources.duration),0) AS value
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
LEFT JOIN LATERAL (
SELECT resources.duration
FROM resources
WHERE {" AND ".join(pg_sub_query_chart)}
) AS resources ON (TRUE)
GROUP BY timestamp
ORDER BY timestamp;"""
cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)}))
rows = cur.fetchall()
rows = helper.list_to_camel_case(rows)
pg_query = f"""WITH resources AS (SELECT resources.duration, resources.timestamp
FROM events.resources INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_subset)}
AND resources.type = 'img' AND resources.duration>0
{(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""}
)
SELECT generated_timestamp AS timestamp,
COALESCE(AVG(resources.duration),0) AS value
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
LEFT JOIN LATERAL (
SELECT resources.duration
FROM resources
WHERE {" AND ".join(pg_sub_query_chart)}
) AS resources ON (TRUE)
GROUP BY timestamp
ORDER BY timestamp;"""
cur.execute(cur.mogrify(pg_query, {**params, **img_constraints_vals, **__get_constraint_values(args)}))
rows = cur.fetchall()
rows = helper.list_to_camel_case(rows)
return rows
@ -2341,7 +2340,7 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU
with pg_client.PostgresClient() as cur:
row = __get_application_activity_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args)
results = row
results["chart"] = get_performance_avg_page_load_time(project_id, startTimestamp, endTimestamp, **args)
results["chart"] = get_performance_avg_page_load_time(cur, project_id, startTimestamp, endTimestamp, **args)
diff = endTimestamp - startTimestamp
endTimestamp = startTimestamp
startTimestamp = endTimestamp - diff
@ -2352,7 +2351,7 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU
return results
def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
def get_performance_avg_page_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
density=19, **args):
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1)
@ -2360,31 +2359,30 @@ def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(de
location_constraints_vals = {}
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
with pg_client.PostgresClient() as cur:
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
chart=False, data=args)
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
chart=True, data=args, main_table="pages", time_column="timestamp",
duration=False)
pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s")
pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s")
pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp
FROM events.pages INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
)
SELECT generated_timestamp AS timestamp,
COALESCE(AVG(pages.load_time),0) AS value
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
LEFT JOIN LATERAL ( SELECT pages.load_time
FROM pages
WHERE {" AND ".join(pg_sub_query_chart)}
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
) AS pages ON (TRUE)
GROUP BY generated_timestamp
ORDER BY generated_timestamp;"""
cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)}))
rows = cur.fetchall()
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
chart=False, data=args)
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
chart=True, data=args, main_table="pages", time_column="timestamp",
duration=False)
pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s")
pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s")
pg_query = f"""WITH pages AS(SELECT pages.load_time, timestamp
FROM events.pages INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_subset)} AND pages.load_time>0 AND pages.load_time IS NOT NULL
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
)
SELECT generated_timestamp AS timestamp,
COALESCE(AVG(pages.load_time),0) AS value
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
LEFT JOIN LATERAL ( SELECT pages.load_time
FROM pages
WHERE {" AND ".join(pg_sub_query_chart)}
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
) AS pages ON (TRUE)
GROUP BY generated_timestamp
ORDER BY generated_timestamp;"""
cur.execute(cur.mogrify(pg_query, {**params, **location_constraints_vals, **__get_constraint_values(args)}))
rows = cur.fetchall()
return rows
@ -2411,7 +2409,7 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti
with pg_client.PostgresClient() as cur:
row = __get_application_activity_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args)
results = row
results["chart"] = get_performance_avg_request_load_time(project_id, startTimestamp, endTimestamp, **args)
results["chart"] = get_performance_avg_request_load_time(cur, project_id, startTimestamp, endTimestamp, **args)
diff = endTimestamp - startTimestamp
endTimestamp = startTimestamp
startTimestamp = endTimestamp - diff
@ -2422,47 +2420,42 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti
return results
def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
def get_performance_avg_request_load_time(cur, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
density=19, **args):
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1)
location_constraints = []
img_constraints = []
request_constraints = []
img_constraints_vals = {}
location_constraints_vals = {}
request_constraints_vals = {}
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
with pg_client.PostgresClient() as cur:
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
chart=False, data=args)
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
chart=True, data=args, main_table="resources", time_column="timestamp",
duration=False)
pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s")
pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s")
pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp
FROM events.resources INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_subset)}
AND resources.type = 'fetch' AND resources.duration>0
{(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""}
)
SELECT generated_timestamp AS timestamp,
COALESCE(AVG(resources.duration),0) AS value
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
LEFT JOIN LATERAL (
SELECT resources.duration
FROM resources
WHERE {" AND ".join(pg_sub_query_chart)}
) AS resources ON (TRUE)
GROUP BY generated_timestamp
ORDER BY generated_timestamp;"""
cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)}))
rows = cur.fetchall()
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
chart=False, data=args)
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
chart=True, data=args, main_table="resources", time_column="timestamp",
duration=False)
pg_sub_query_subset.append("resources.timestamp >= %(startTimestamp)s")
pg_sub_query_subset.append("resources.timestamp < %(endTimestamp)s")
pg_query = f"""WITH resources AS(SELECT resources.duration, resources.timestamp
FROM events.resources INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_subset)}
AND resources.type = 'fetch' AND resources.duration>0
{(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""}
)
SELECT generated_timestamp AS timestamp,
COALESCE(AVG(resources.duration),0) AS value
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
LEFT JOIN LATERAL (
SELECT resources.duration
FROM resources
WHERE {" AND ".join(pg_sub_query_chart)}
) AS resources ON (TRUE)
GROUP BY generated_timestamp
ORDER BY generated_timestamp;"""
cur.execute(cur.mogrify(pg_query, {**params, **request_constraints_vals, **__get_constraint_values(args)}))
rows = cur.fetchall()
return rows
@ -2470,16 +2463,16 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now
def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(), **args):
with pg_client.PostgresClient() as cur:
rows = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args)
if len(rows) > 0:
results = helper.dict_to_camel_case(rows[0])
row = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args)
results = helper.dict_to_camel_case(row)
results["chart"] = __get_page_metrics_avg_dom_content_load_start_chart(cur, project_id, startTimestamp,
endTimestamp, **args)
diff = endTimestamp - startTimestamp
endTimestamp = startTimestamp
startTimestamp = endTimestamp - diff
rows = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args)
if len(rows) > 0:
previous = helper.dict_to_camel_case(rows[0])
results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"])
row = __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestamp, endTimestamp, **args)
previous = helper.dict_to_camel_case(row)
results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"])
results["unit"] = schemas.TemplatePredefinedUnits.millisecond
return results
@ -2498,6 +2491,39 @@ def __get_page_metrics_avg_dom_content_load_start(cur, project_id, startTimestam
params = {"project_id": project_id, "startTimestamp": startTimestamp, "endTimestamp": endTimestamp,
**__get_constraint_values(args)}
cur.execute(cur.mogrify(pg_query, params))
row = cur.fetchone()
return row
def __get_page_metrics_avg_dom_content_load_start_chart(cur, project_id, startTimestamp, endTimestamp, density=19,
**args):
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density, factor=1)
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
pg_sub_query_subset = __get_constraints(project_id=project_id, time_constraint=True,
chart=False, data=args)
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=False, project=False,
chart=True, data=args, main_table="pages", time_column="timestamp",
duration=False)
pg_sub_query_subset.append("pages.timestamp >= %(startTimestamp)s")
pg_sub_query_subset.append("pages.timestamp < %(endTimestamp)s")
pg_sub_query_subset.append("pages.dom_content_loaded_time > 0")
pg_query = f"""WITH pages AS(SELECT pages.dom_content_loaded_time, pages.timestamp
FROM events.pages INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_subset)}
)
SELECT generated_timestamp AS timestamp,
COALESCE(AVG(pages.dom_content_loaded_time),0) AS value
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
LEFT JOIN LATERAL (
SELECT pages.dom_content_loaded_time
FROM pages
WHERE {" AND ".join(pg_sub_query_chart)}
) AS pages ON (TRUE)
GROUP BY generated_timestamp
ORDER BY generated_timestamp;"""
cur.execute(cur.mogrify(pg_query, {**params, **__get_constraint_values(args)}))
rows = cur.fetchall()
return rows
@ -2646,11 +2672,15 @@ def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta
def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(), value=None, **args):
endTimestamp=TimeUTC.now(), value=None, density=19, **args):
step_size = __get_step_size(startTimestamp, endTimestamp, density, factor=1)
pg_sub_query = __get_constraints(project_id=project_id, data=args)
pg_sub_query_chart = __get_constraints(project_id=project_id, time_constraint=True,
chart=True, data=args)
if value is not None:
pg_sub_query.append("pages.path = %(value)s")
pg_sub_query_chart.append("pages.path = %(value)s")
with pg_client.PostgresClient() as cur:
pg_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_time), 0) AS value
FROM events.pages
@ -2659,11 +2689,27 @@ def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.no
AND pages.timestamp >= %(startTimestamp)s
AND pages.timestamp < %(endTimestamp)s
AND pages.dom_content_loaded_time > 0;"""
cur.execute(cur.mogrify(pg_query, {"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp,
"value": value, **__get_constraint_values(args)}))
params = {"step_size": step_size,
"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp,
"value": value, **__get_constraint_values(args)}
cur.execute(cur.mogrify(pg_query, params))
row = cur.fetchone()
pg_query = f"""SELECT generated_timestamp AS timestamp,
COALESCE(AVG(NULLIF(pages.dom_content_loaded_time,0)),0) AS value
FROM generate_series(%(startTimestamp)s, %(endTimestamp)s, %(step_size)s) AS generated_timestamp
LEFT JOIN LATERAL (
SELECT dom_content_loaded_time
FROM events.pages INNER JOIN public.sessions USING (session_id)
WHERE {" AND ".join(pg_sub_query_chart)}
) AS pages ON (TRUE)
GROUP BY generated_timestamp
ORDER BY generated_timestamp ASC;"""
cur.execute(cur.mogrify(pg_query, params))
rows = cur.fetchall()
row["chart"] = helper.list_to_camel_case(rows),
row["unit"] = schemas.TemplatePredefinedUnits.millisecond
return helper.dict_to_camel_case(row)

10
api/chalicelib/core/resources.py
View file

@ -1,23 +1,23 @@
from chalicelib.utils import helper, pg_client
def get_by_session_id(session_id):
def get_by_session_id(session_id, project_id):
with pg_client.PostgresClient() as cur:
ch_query = """\
SELECT
timestamp AS datetime,
url,
type,
duration,
resources.duration AS duration,
ttfb,
header_size,
encoded_body_size,
decoded_body_size,
success,
COALESCE(status, CASE WHEN success THEN 200 END) AS status
FROM events.resources
WHERE session_id = %(session_id)s;"""
params = {"session_id": session_id}
FROM events.resources INNER JOIN sessions USING (session_id)
WHERE session_id = %(session_id)s AND project_id= %(project_id)s;"""
params = {"session_id": session_id, "project_id": project_id}
cur.execute(cur.mogrify(ch_query, params))
rows = cur.fetchall()
return helper.list_to_camel_case(rows)

2
api/chalicelib/core/sessions.py
View file

@ -94,7 +94,7 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_
data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id,
session_id=session_id)
data['mobsUrl'] = sessions_mobs.get_web(sessionId=session_id)
data['resources'] = resources.get_by_session_id(session_id=session_id)
data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id)
data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data)
data['issues'] = issues.get_by_session_id(session_id=session_id)

3
api/entrypoint.sh
View file

@ -1,2 +1,5 @@
#!/bin/bash
cd sourcemap-reader
nohup npm start &> /tmp/sourcemap-reader.log &
cd ..
uvicorn app:app --host 0.0.0.0 --reload

20
api/routers/core.py
View file

@ -1,7 +1,7 @@
from typing import Union
from decouple import config
from fastapi import Depends, Body
from fastapi import Depends, Body, BackgroundTasks
import schemas
from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assignments, projects, \
@ -23,7 +23,8 @@ public_app, app, app_apikey = get_routers()
@app.get('/{projectId}/sessions/{sessionId}', tags=["sessions"])
@app.get('/{projectId}/sessions2/{sessionId}', tags=["sessions"])
def get_session2(projectId: int, sessionId: Union[int, str], context: schemas.CurrentContext = Depends(OR_context)):
def get_session2(projectId: int, sessionId: Union[int, str], background_tasks: BackgroundTasks,
context: schemas.CurrentContext = Depends(OR_context)):
if isinstance(sessionId, str):
return {"errors": ["session not found"]}
data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, user_id=context.user_id,
@ -31,7 +32,8 @@ def get_session2(projectId: int, sessionId: Union[int, str], context: schemas.Cu
if data is None:
return {"errors": ["session not found"]}
if data.get("inDB"):
sessions_favorite_viewed.view_session(project_id=projectId, user_id=context.user_id, session_id=sessionId)
background_tasks.add_task(sessions_favorite_viewed.view_session, project_id=projectId, user_id=context.user_id,
session_id=sessionId)
return {
'data': data
}
@ -833,7 +835,8 @@ def sessions_live(projectId: int, userId: str = None, context: schemas.CurrentCo
@app.get('/{projectId}/assist/sessions/{sessionId}', tags=["assist"])
def get_live_session(projectId: int, sessionId: str, context: schemas.CurrentContext = Depends(OR_context)):
def get_live_session(projectId: int, sessionId: str, background_tasks: BackgroundTasks,
context: schemas.CurrentContext = Depends(OR_context)):
data = assist.get_live_session_by_id(project_id=projectId, session_id=sessionId)
if data is None:
data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True,
@ -841,7 +844,8 @@ def get_live_session(projectId: int, sessionId: str, context: schemas.CurrentCon
if data is None:
return {"errors": ["session not found"]}
if data.get("inDB"):
sessions_favorite_viewed.view_session(project_id=projectId, user_id=context.user_id, session_id=sessionId)
background_tasks.add_task(sessions_favorite_viewed.view_session, project_id=projectId,
user_id=context.user_id, session_id=sessionId)
return {'data': data}
@ -909,12 +913,14 @@ def errors_stats(projectId: int, startTimestamp: int, endTimestamp: int,
@app.get('/{projectId}/errors/{errorId}', tags=['errors'])
def errors_get_details(projectId: int, errorId: str, density24: int = 24, density30: int = 30,
def errors_get_details(projectId: int, errorId: str, background_tasks: BackgroundTasks, density24: int = 24,
density30: int = 30,
context: schemas.CurrentContext = Depends(OR_context)):
data = errors.get_details(project_id=projectId, user_id=context.user_id, error_id=errorId,
**{"density24": density24, "density30": density30})
if data.get("data") is not None:
errors_favorite_viewed.viewed_error(project_id=projectId, user_id=context.user_id, error_id=errorId)
background_tasks.add_task(errors_favorite_viewed.viewed_error, project_id=projectId, user_id=context.user_id,
error_id=errorId)
return data

2
ee/api/.env.default
View file

@ -53,6 +53,6 @@ sentryURL=
sessions_bucket=mobs
sessions_region=us-east-1
sourcemaps_bucket=sourcemaps
sourcemaps_reader=http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps
sourcemaps_reader=http://127.0.0.1:9000/
stage=default-ee
version_number=1.0.0

1
ee/api/.gitignore vendored
View file

@ -261,3 +261,4 @@ Pipfile
/routers/subs/metrics.py
/routers/subs/v1_api.py
/chalicelib/core/dashboards2.py
entrypoint.sh

8
ee/api/Dockerfile
View file

@ -7,6 +7,14 @@ COPY . .
RUN pip install -r requirements.txt
RUN mv .env.default .env
ENV APP_NAME chalice
# Installing Nodejs
RUN apt update && apt install -y curl && \
curl -fsSL https://deb.nodesource.com/setup_12.x | bash - && \
apt install -y nodejs && \
apt remove --purge -y curl && \
rm -rf /var/lib/apt/lists/* && \
cd sourcemap-reader && \
npm install
# Add Tini
# Startup daemon

196
ee/api/chalicelib/core/dashboard.py
View file

@ -1,6 +1,7 @@
import math
import random
import schemas
from chalicelib.utils import pg_client
from chalicelib.utils import args_transformer
from chalicelib.utils import helper
@ -1689,7 +1690,7 @@ def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1)
WHERE {" AND ".join(ch_sub_query)}
GROUP BY resources.url_host
ORDER BY errors_count DESC
LIMIT 10;"""
LIMIT 5;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
"startTimestamp": startTimestamp,
@ -2096,7 +2097,7 @@ def get_application_activity_avg_page_load_time(project_id, startTimestamp=TimeU
with ch_client.ClickHouseClient() as ch:
row = __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args)
results = helper.dict_to_camel_case(row)
results["chart"] = get_performance_avg_page_load_time(project_id, startTimestamp, endTimestamp, **args)
results["chart"] = get_performance_avg_page_load_time(ch, project_id, startTimestamp, endTimestamp, **args)
diff = endTimestamp - startTimestamp
endTimestamp = startTimestamp
startTimestamp = endTimestamp - diff
@ -2125,7 +2126,7 @@ def __get_application_activity_avg_page_load_time(ch, project_id, startTimestamp
return result
def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
def get_performance_avg_page_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
density=19, resources=None, **args):
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density)
@ -2142,30 +2143,30 @@ def get_performance_avg_page_load_time(project_id, startTimestamp=TimeUTC.now(de
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
with ch_client.ClickHouseClient() as ch:
ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True,
data=args)
ch_sub_query_chart += meta_condition
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
AVG(NULLIF(pages.load_event_end ,0)) AS value
FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)}
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
GROUP BY timestamp
ORDER BY timestamp;"""
ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True,
data=args)
ch_sub_query_chart += meta_condition
rows = ch.execute(query=ch_query,
params={**params, **location_constraints_vals, **__get_constraint_values(args)})
pages = [{"timestamp": i["timestamp"], "value": i["value"]} for i in
__complete_missing_steps(rows=rows, start_time=startTimestamp,
end_time=endTimestamp,
density=density, neutral={"value": 0})]
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
AVG(NULLIF(pages.load_event_end ,0)) AS value
FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)}
{(f' AND ({" OR ".join(location_constraints)})') if len(location_constraints) > 0 else ""}
GROUP BY timestamp
ORDER BY timestamp;"""
for s in pages:
for k in s:
if s[k] is None:
s[k] = 0
rows = ch.execute(query=ch_query,
params={**params, **location_constraints_vals, **__get_constraint_values(args)})
pages = [{"timestamp": i["timestamp"], "value": i["value"]} for i in
__complete_missing_steps(rows=rows, start_time=startTimestamp,
end_time=endTimestamp,
density=density, neutral={"value": 0})]
for s in pages:
for k in s:
if s[k] is None:
s[k] = 0
return pages
@ -2174,7 +2175,7 @@ def get_application_activity_avg_image_load_time(project_id, startTimestamp=Time
with ch_client.ClickHouseClient() as ch:
row = __get_application_activity_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args)
results = helper.dict_to_camel_case(row)
results["chart"] = get_performance_avg_image_load_time(project_id, startTimestamp, endTimestamp, **args)
results["chart"] = get_performance_avg_image_load_time(ch, project_id, startTimestamp, endTimestamp, **args)
diff = endTimestamp - startTimestamp
endTimestamp = startTimestamp
startTimestamp = endTimestamp - diff
@ -2203,7 +2204,7 @@ def __get_application_activity_avg_image_load_time(ch, project_id, startTimestam
return result
def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
def get_performance_avg_image_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
density=19, resources=None, **args):
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density)
@ -2222,25 +2223,24 @@ def get_performance_avg_image_load_time(project_id, startTimestamp=TimeUTC.now(d
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
AVG(NULLIF(resources.duration,0)) AS value
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)}
AND resources.type = 'img'
{(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""}
GROUP BY timestamp
ORDER BY timestamp;"""
rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)})
images = [{"timestamp": i["timestamp"], "value": i["value"]} for i in
__complete_missing_steps(rows=rows, start_time=startTimestamp,
end_time=endTimestamp,
density=density, neutral={"value": 0})]
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
AVG(NULLIF(resources.duration,0)) AS value
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)}
AND resources.type = 'img'
{(f' AND ({" OR ".join(img_constraints)})') if len(img_constraints) > 0 else ""}
GROUP BY timestamp
ORDER BY timestamp;"""
rows = ch.execute(query=ch_query, params={**params, **img_constraints_vals, **__get_constraint_values(args)})
images = [{"timestamp": i["timestamp"], "value": i["value"]} for i in
__complete_missing_steps(rows=rows, start_time=startTimestamp,
end_time=endTimestamp,
density=density, neutral={"value": 0})]
for s in images:
for k in s:
if s[k] is None:
s[k] = 0
for s in images:
for k in s:
if s[k] is None:
s[k] = 0
return images
@ -2249,7 +2249,7 @@ def get_application_activity_avg_request_load_time(project_id, startTimestamp=Ti
with ch_client.ClickHouseClient() as ch:
row = __get_application_activity_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args)
results = helper.dict_to_camel_case(row)
results["chart"] = get_performance_avg_request_load_time(project_id, startTimestamp, endTimestamp, **args)
results["chart"] = get_performance_avg_request_load_time(ch, project_id, startTimestamp, endTimestamp, **args)
diff = endTimestamp - startTimestamp
endTimestamp = startTimestamp
startTimestamp = endTimestamp - diff
@ -2278,7 +2278,7 @@ def __get_application_activity_avg_request_load_time(ch, project_id, startTimest
return result
def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
def get_performance_avg_request_load_time(ch, project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(),
density=19, resources=None, **args):
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density)
@ -2296,35 +2296,38 @@ def get_performance_avg_request_load_time(project_id, startTimestamp=TimeUTC.now
request_constraints_vals["val_" + str(len(request_constraints) - 1)] = r['value']
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
AVG(NULLIF(resources.duration,0)) AS value
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)}
AND resources.type = 'fetch'
{(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""}
GROUP BY timestamp
ORDER BY timestamp;"""
rows = ch.execute(query=ch_query,
params={**params, **request_constraints_vals, **__get_constraint_values(args)})
requests = [{"timestamp": i["timestamp"], "value": i["value"]} for i in
__complete_missing_steps(rows=rows, start_time=startTimestamp,
end_time=endTimestamp, density=density,
neutral={"value": 0})]
for s in requests:
for k in s:
if s[k] is None:
s[k] = 0
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(resources.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
AVG(NULLIF(resources.duration,0)) AS value
FROM resources {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)}
AND resources.type = 'fetch'
{(f' AND ({" OR ".join(request_constraints)})') if len(request_constraints) > 0 else ""}
GROUP BY timestamp
ORDER BY timestamp;"""
rows = ch.execute(query=ch_query,
params={**params, **request_constraints_vals, **__get_constraint_values(args)})
requests = [{"timestamp": i["timestamp"], "value": i["value"]} for i in
__complete_missing_steps(rows=rows, start_time=startTimestamp,
end_time=endTimestamp, density=density,
neutral={"value": 0})]
for s in requests:
for k in s:
if s[k] is None:
s[k] = 0
return requests
def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(), **args):
with ch_client.ClickHouseClient() as ch:
results = {}
rows = __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp, endTimestamp, **args)
if len(rows) > 0:
results = helper.dict_to_camel_case(rows[0])
results["chart"] = __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTimestamp,
endTimestamp, **args)
diff = endTimestamp - startTimestamp
endTimestamp = startTimestamp
startTimestamp = endTimestamp - diff
@ -2332,6 +2335,7 @@ def get_page_metrics_avg_dom_content_load_start(project_id, startTimestamp=TimeU
if len(rows) > 0:
previous = helper.dict_to_camel_case(rows[0])
results["progress"] = helper.__progress(old_val=previous["value"], new_val=results["value"])
results["unit"] = schemas.TemplatePredefinedUnits.millisecond
return results
@ -2349,6 +2353,35 @@ def __get_page_metrics_avg_dom_content_load_start(ch, project_id, startTimestamp
return rows
def __get_page_metrics_avg_dom_content_load_start_chart(ch, project_id, startTimestamp, endTimestamp, density=19,
**args):
step_size = __get_step_size(endTimestamp=endTimestamp, startTimestamp=startTimestamp, density=density)
ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args)
meta_condition = __get_meta_constraint(args)
ch_sub_query_chart += meta_condition
params = {"step_size": step_size, "project_id": project_id, "startTimestamp": startTimestamp,
"endTimestamp": endTimestamp}
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second ))*1000 AS timestamp,
AVG(NULLIF(pages.dom_content_loaded_event_end,0)) AS value
FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)}
GROUP BY timestamp
ORDER BY timestamp;"""
rows = ch.execute(query=ch_query, params={**params, **__get_constraint_values(args)})
rows = [{"timestamp": i["timestamp"], "value": i["value"]} for i in
__complete_missing_steps(rows=rows, start_time=startTimestamp,
end_time=endTimestamp,
density=density, neutral={"value": 0})]
for s in rows:
for k in s:
if s[k] is None:
s[k] = 0
return rows
def get_page_metrics_avg_first_contentful_pixel(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(), **args):
with ch_client.ClickHouseClient() as ch:
@ -2519,23 +2552,40 @@ def get_top_metrics_avg_first_paint(project_id, startTimestamp=TimeUTC.now(delta
def get_top_metrics_avg_dom_content_loaded(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
endTimestamp=TimeUTC.now(), value=None, **args):
ch_sub_query = __get_basic_constraints(table_name="pages", data=args)
endTimestamp=TimeUTC.now(), value=None, density=19, **args):
step_size = __get_step_size(startTimestamp, endTimestamp, density)
ch_sub_query_chart = __get_basic_constraints(table_name="pages", round_start=True, data=args)
meta_condition = __get_meta_constraint(args)
ch_sub_query_chart += meta_condition
ch_sub_query = __get_basic_constraints(table_name="pages", data=args)
ch_sub_query += meta_condition
if value is not None:
ch_sub_query.append("pages.url_path = %(value)s")
ch_sub_query_chart.append("pages.url_path = %(value)s")
with ch_client.ClickHouseClient() as ch:
ch_query = f"""SELECT COALESCE(AVG(pages.dom_content_loaded_event_time),0) AS value
FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query)} AND isNotNull(pages.dom_content_loaded_event_time) AND pages.dom_content_loaded_event_time>0;"""
rows = ch.execute(query=ch_query,
params={"project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp,
"value": value, **__get_constraint_values(args)})
return helper.dict_to_camel_case(rows[0])
params = {"step_size": step_size, "project_id": project_id,
"startTimestamp": startTimestamp,
"endTimestamp": endTimestamp,
"value": value, **__get_constraint_values(args)}
rows = ch.execute(query=ch_query, params=params)
results = helper.dict_to_camel_case(rows[0])
ch_query = f"""SELECT toUnixTimestamp(toStartOfInterval(pages.datetime, INTERVAL %(step_size)s second)) * 1000 AS timestamp,
COALESCE(AVG(NULLIF(pages.dom_content_loaded_event_time ,0)),0) AS value
FROM pages {"INNER JOIN sessions_metadata USING(session_id)" if len(meta_condition) > 0 else ""}
WHERE {" AND ".join(ch_sub_query_chart)}
GROUP BY timestamp
ORDER BY timestamp;;"""
rows = ch.execute(query=ch_query, params=params)
results["chart"] = helper.list_to_camel_case(__complete_missing_steps(rows=rows, start_time=startTimestamp,
end_time=endTimestamp,
density=density,
neutral={"value": 0}))
return results
def get_top_metrics_avg_till_first_bit(project_id, startTimestamp=TimeUTC.now(delta_days=-1),

6
ee/api/chalicelib/core/resources.py
View file

@ -3,14 +3,14 @@ from chalicelib.utils import ch_client
from chalicelib.utils.TimeUTC import TimeUTC
def get_by_session_id(session_id):
def get_by_session_id(session_id, project_id):
with ch_client.ClickHouseClient() as ch:
ch_query = """\
SELECT
datetime,url,type,duration,ttfb,header_size,encoded_body_size,decoded_body_size,success,coalesce(status,if(success, 200, status)) AS status
FROM resources
WHERE session_id = toUInt64(%(session_id)s);"""
params = {"session_id": session_id}
WHERE session_id = toUInt64(%(session_id)s) AND project_id=%(project_id)s;"""
params = {"session_id": session_id, "project_id": project_id}
rows = ch.execute(query=ch_query, params=params)
results = []
for r in rows:

2
ee/api/entrypoint.sh
View file

@ -1,2 +0,0 @@
#!/bin/bash
uvicorn app:app --host 0.0.0.0 --reload

305
ee/scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql
View file

@ -24,13 +24,17 @@ ALTER TABLE IF EXISTS metrics
DROP CONSTRAINT IF EXISTS unique_key;
ALTER TABLE IF EXISTS metrics
ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL,
ADD COLUMN IF NOT EXISTS edited_at timestamp NULL DEFAULT NULL,
ADD COLUMN IF NOT EXISTS is_pinned boolean NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom',
ADD COLUMN IF NOT EXISTS category text NULL DEFAULT 'custom',
ADD COLUMN IF NOT EXISTS is_predefined boolean NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS is_template boolean NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL,
ADD COLUMN IF NOT EXISTS default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb,
ADD COLUMN IF NOT EXISTS predefined_key text NULL DEFAULT NULL,
ADD COLUMN IF NOT EXISTS default_config jsonb NOT NULL DEFAULT '{
"col": 2,
"row": 2,
"position": 0
}'::jsonb,
ALTER COLUMN project_id DROP NOT NULL,
ADD CONSTRAINT null_project_id_for_template_only
CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ),
@ -48,7 +52,15 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets
config jsonb NOT NULL DEFAULT '{}'::jsonb
);
ALTER TABLE events_common.requests
ADD COLUMN IF NOT EXISTS host text NULL,
ADD COLUMN IF NOT EXISTS base_path text NULL,
ADD COLUMN IF NOT EXISTS query text NULL;
ALTER TABLE events.pages
ADD COLUMN IF NOT EXISTS query text NULL;
COMMIT;
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart';
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart';
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarChart';
@ -58,55 +70,236 @@ ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'map';
ALTER TYPE metric_type ADD VALUE IF NOT EXISTS 'predefined';
INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type)
VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'),
('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'),
('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'),
('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'),
('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'),
('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'),
('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'),
('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'),
('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'),
('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'),
('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'),
('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'),
('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'),
('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'),
('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'),
('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'),
('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'),
('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'),
('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'),
('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'),
INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type,
view_type)
VALUES ('Captured sessions', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'count_sessions', 'predefined', 'overview'),
('Request Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'),
('Page Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'),
('Image Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'),
('DOM Content Load Start', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'),
('First Meaningful paint', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'),
('No. of Visited Pages', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'),
('Session Duration', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_session_duration', 'predefined', 'overview'),
('DOM Build Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'),
('Pages Response Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'),
('Response Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_response_time', 'predefined', 'overview'),
('First Paint', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_first_paint', 'predefined', 'overview'),
('DOM Content Loaded', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'),
('Time Till First byte', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'),
('Time To Interactive', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'),
('Captured requests', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'count_requests', 'predefined', 'overview'),
('Time To Render', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'),
('Memory Consumption', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'),
('CPU Load', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_cpu', 'predefined', 'overview'),
('Frame rate', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_fps', 'predefined', 'overview'),
('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'),
('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'),
('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'),
('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'),
('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'),
('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'),
('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'),
('Sessions Affected by JS Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'),
('Top Domains with 4xx Fetch Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'),
('Top Domains with 5xx Fetch Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'),
('Errors per Domain', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'errors_per_domains', 'predefined', 'table'),
('Fetch Calls with Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'calls_errors', 'predefined', 'table'),
('Errors by Type', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'errors_per_type', 'predefined', 'barChart'),
('Errors by Origin', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'),
('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'),
('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'),
('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'),
('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'),
('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'),
('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'),
('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'),
('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'),
('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'),
('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'),
('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'),
('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'),
('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'),
('Speed Index by Location', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'speed_location', 'predefined', 'map'),
('Slowest Domains', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'slowest_domains', 'predefined', 'table'),
('Sessions per Browser', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'sessions_per_browser', 'predefined', 'table'),
('Time To Render', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'time_to_render', 'predefined', 'areaChart'),
('Sessions Impacted by Slow Pages', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'),
('Memory Consumption', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'),
('CPU Load', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'cpu', 'predefined', 'areaChart'),
('Frame Rate', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'fps', 'predefined', 'areaChart'),
('Crashes', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'crashes', 'predefined', 'areaChart'),
('Resources Loaded vs Visually Complete', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'),
('DOM Build Time', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'),
('Pages Response Time', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'),
('Pages Response Time Distribution', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'),
('Missing Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'),
('Slowest Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'),
('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'),
('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'),
('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart')
('Missing Resources', 'resources', '{
"col": 4,
"row": 2,
"position": 0
}', true, true, true, 'missing_resources', 'predefined', 'table'),
('Slowest Resources', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'slowest_resources', 'predefined', 'table'),
('Resources Fetch Time', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_loading_time', 'predefined', 'table'),
('Resource Loaded vs Response End', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'),
('Breakdown of Loaded Resources', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart')
ON CONFLICT (predefined_key) DO UPDATE
SET name=excluded.name,
category=excluded.category,
@ -115,4 +308,14 @@ ON CONFLICT (predefined_key) DO UPDATE
is_template=excluded.is_template,
is_public=excluded.is_public,
metric_type=excluded.metric_type,
view_type=excluded.view_type;
view_type=excluded.view_type;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;

292
ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql
View file

@ -811,7 +811,11 @@ $$
is_predefined boolean NOT NULL DEFAULT FALSE,
is_template boolean NOT NULL DEFAULT FALSE,
predefined_key text NULL DEFAULT NULL,
default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb,
default_config jsonb NOT NULL DEFAULT '{
"col": 2,
"row": 2,
"position": 0
}'::jsonb,
CONSTRAINT null_project_id_for_template_only
CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ),
CONSTRAINT unique_key UNIQUE (predefined_key)
@ -935,6 +939,7 @@ $$
host text NOT NULL,
path text NOT NULL,
base_path text NOT NULL,
query text NULL,
referrer text DEFAULT NULL,
base_referrer text DEFAULT NULL,
dom_building_time integer DEFAULT NULL,
@ -994,6 +999,9 @@ $$
CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp);
CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2;
CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE TABLE IF NOT EXISTS events.clicks
(
@ -1229,6 +1237,9 @@ $$
response_body text NULL,
status_code smallint NULL,
method http_method NULL,
host text NULL,
base_path text NULL,
query text NULL,
PRIMARY KEY (session_id, timestamp, seq_index)
);
CREATE INDEX IF NOT EXISTS requests_url_idx ON events_common.requests (url);
@ -1250,6 +1261,12 @@ $$
CREATE INDEX IF NOT EXISTS requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
END IF;
@ -1257,55 +1274,236 @@ $$
$$
LANGUAGE plpgsql;
INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type)
VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'),
('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'),
('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'),
('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'),
('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'),
('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'),
('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'),
('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'),
('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'),
('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'),
('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'),
('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'),
('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'),
('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'),
('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'),
('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'),
('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'),
('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'),
('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'),
('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'),
INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type,
view_type)
VALUES ('Captured sessions', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'count_sessions', 'predefined', 'overview'),
('Request Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'),
('Page Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'),
('Image Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'),
('DOM Content Load Start', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'),
('First Meaningful paint', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'),
('No. of Visited Pages', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'),
('Session Duration', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_session_duration', 'predefined', 'overview'),
('DOM Build Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'),
('Pages Response Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'),
('Response Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_response_time', 'predefined', 'overview'),
('First Paint', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_first_paint', 'predefined', 'overview'),
('DOM Content Loaded', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'),
('Time Till First byte', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'),
('Time To Interactive', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'),
('Captured requests', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'count_requests', 'predefined', 'overview'),
('Time To Render', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'),
('Memory Consumption', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'),
('CPU Load', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_cpu', 'predefined', 'overview'),
('Frame rate', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_fps', 'predefined', 'overview'),
('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'),
('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'),
('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'),
('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'),
('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'),
('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'),
('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'),
('Sessions Affected by JS Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'),
('Top Domains with 4xx Fetch Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'),
('Top Domains with 5xx Fetch Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'),
('Errors per Domain', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'errors_per_domains', 'predefined', 'table'),
('Fetch Calls with Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'calls_errors', 'predefined', 'table'),
('Errors by Type', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'errors_per_type', 'predefined', 'barChart'),
('Errors by Origin', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'),
('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'),
('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'),
('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'),
('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'),
('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'),
('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'),
('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'),
('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'),
('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'),
('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'),
('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'),
('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'),
('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'),
('Speed Index by Location', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'speed_location', 'predefined', 'map'),
('Slowest Domains', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'slowest_domains', 'predefined', 'table'),
('Sessions per Browser', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'sessions_per_browser', 'predefined', 'table'),
('Time To Render', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'time_to_render', 'predefined', 'areaChart'),
('Sessions Impacted by Slow Pages', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'),
('Memory Consumption', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'),
('CPU Load', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'cpu', 'predefined', 'areaChart'),
('Frame Rate', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'fps', 'predefined', 'areaChart'),
('Crashes', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'crashes', 'predefined', 'areaChart'),
('Resources Loaded vs Visually Complete', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'),
('DOM Build Time', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'),
('Pages Response Time', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'),
('Pages Response Time Distribution', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'),
('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'),
('Slowest Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'),
('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'),
('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'),
('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart')
('Missing Resources', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'missing_resources', 'predefined', 'table'),
('Slowest Resources', 'resources', '{
"col": 4,
"row": 2,
"position": 0
}', true, true, true, 'slowest_resources', 'predefined', 'table'),
('Resources Fetch Time', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_loading_time', 'predefined', 'table'),
('Resource Loaded vs Response End', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'),
('Breakdown of Loaded Resources', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart')
ON CONFLICT (predefined_key) DO UPDATE
SET name=excluded.name,
category=excluded.category,

3
ee/utilities/.gitignore vendored
View file

@ -11,4 +11,5 @@ servers/peerjs-server.js
servers/sourcemaps-handler.js
servers/sourcemaps-server.js
#servers/websocket.js
/utils/dump.js
/utils
/Dockerfile

891
ee/utilities/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

7
ee/utilities/package.json
View file

@ -1,5 +1,5 @@
{
"name": "utilities_server",
"name": "utilities-server",
"version": "1.0.0",
"description": "assist server to get live sessions & sourcemaps reader to get stack trace",
"main": "peerjs-server.js",
@ -12,7 +12,7 @@
"url": "git+https://github.com/openreplay/openreplay.git"
},
"author": "KRAIEM Taha Yassine <tahayk2@gmail.com>",
"license": "MIT",
"license": "Elastic License 2.0 (ELv2)",
"bugs": {
"url": "https://github.com/openreplay/openreplay/issues"
},
@ -20,12 +20,9 @@
"dependencies": {
"@maxmind/geoip2-node": "^3.4.0",
"@socket.io/redis-adapter": "^7.1.0",
"aws-sdk": "^2.992.0",
"express": "^4.17.1",
"peer": "^0.6.1",
"redis": "^4.0.3",
"socket.io": "^4.4.1",
"source-map": "^0.7.3",
"ua-parser-js": "^1.0.2",
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.6.0"
}

72
ee/utilities/server.js
View file

@ -1,8 +1,6 @@
const dumps = require('./utils/HeapSnapshot');
const sourcemapsReaderServer = require('./servers/sourcemaps-server');
const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server');
const {request_logger} = require('./utils/helper');
const express = require('express');
const {ExpressPeerServer} = require('peer');
let socket;
if (process.env.redis === "true") {
console.log("Using Redis");
@ -12,63 +10,29 @@ if (process.env.redis === "true") {
}
const HOST = '0.0.0.0';
const PORT = 9000;
const app = express();
const PORT = 9001;
let debug = process.env.debug === "1" || false;
const request_logger = (identity) => {
return (req, res, next) => {
debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl);
res.on('finish', function () {
if (this.statusCode !== 200 || debug) {
console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode);
}
})
next();
}
};
app.use(request_logger("[app]"));
app.use('/sourcemaps', sourcemapsReaderServer);
app.use('/assist', peerRouter);
const server = app.listen(PORT, HOST, () => {
console.log(`App listening on http://${HOST}:${PORT}`);
console.log('Press Ctrl+C to quit.');
});
const peerServer = ExpressPeerServer(server, {
debug: true,
path: '/',
proxied: true,
allow_discovery: false
});
peerServer.on('connection', peerConnection);
peerServer.on('disconnect', peerDisconnect);
peerServer.on('error', peerError);
app.use('/', peerServer);
app.enable('trust proxy');
app.use('/heapdump', dumps.router);
const PREFIX = process.env.prefix || `/assist`
if (process.env.uws !== "true") {
var wsapp = express();
let wsapp = express();
wsapp.use(request_logger("[wsapp]"));
wsapp.use('/assist', socket.wsRouter);
const wsserver = wsapp.listen(PORT + 1, HOST, () => {
console.log(`WS App listening on http://${HOST}:${PORT + 1}`);
wsapp.use(request_logger("[app]"));
wsapp.use(`/heapdump/${process.env.S3_KEY}`, dumps.router);
wsapp.use(`${PREFIX}/${process.env.S3_KEY}`, socket.wsRouter);
wsapp.enable('trust proxy');
const wsserver = wsapp.listen(PORT, HOST, () => {
console.log(`WS App listening on http://${HOST}:${PORT}`);
console.log('Press Ctrl+C to quit.');
});
wsapp.enable('trust proxy');
socket.start(wsserver);
module.exports = {wsserver, server};
module.exports = {wsserver};
} else {
console.log("Using uWebSocket");
const {App} = require("uWebSockets.js");
const PREFIX = process.env.prefix || '/assist'
const uapp = new App();
@ -77,6 +41,7 @@ if (process.env.uws !== "true") {
}
uapp.get(PREFIX, healthFn);
uapp.get(`${PREFIX}/`, healthFn);
uapp.get(`${PREFIX}/${process.env.S3_KEY}`, healthFn);
/* Either onAborted or simply finished request */
@ -112,11 +77,11 @@ if (process.env.uws !== "true") {
socket.start(uapp);
uapp.listen(HOST, PORT + 1, (token) => {
uapp.listen(HOST, PORT, (token) => {
if (!token) {
console.warn("port already in use");
}
console.log(`WS App listening on http://${HOST}:${PORT + 1}`);
console.log(`WS App listening on http://${HOST}:${PORT}`);
console.log('Press Ctrl+C to quit.');
});
@ -126,6 +91,5 @@ if (process.env.uws !== "true") {
debug && console.log(err.stack);
// process.exit(1);
});
module.exports = {uapp, server};
}
console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`);
module.exports = {uapp};
}

19
ee/utilities/servers/websocket-cluster.js
View file

@ -1,8 +1,8 @@
const _io = require('socket.io');
const express = require('express');
const uaParser = require('ua-parser-js');
const geoip2Reader = require('@maxmind/geoip2-node').Reader;
const {extractPeerId} = require('./peerjs-server');
const {extractPeerId} = require('../utils/helper');
const {geoip} = require('../utils/geoIP');
const {createAdapter} = require("@socket.io/redis-adapter");
const {createClient} = require("redis");
const wsRouter = express.Router();
@ -159,7 +159,7 @@ const socketsLive = async function (req, res) {
let liveSessions = {};
let rooms = await getAvailableRooms();
for (let peerId of rooms) {
let {projectKey, sessionId} = extractPeerId(peerId);
let {projectKey} = extractPeerId(peerId);
if (projectKey !== undefined) {
let connected_sockets = await io.in(peerId).fetchSockets();
for (let item of connected_sockets) {
@ -188,7 +188,7 @@ const socketsLiveByProject = async function (req, res) {
let liveSessions = {};
let rooms = await getAvailableRooms();
for (let peerId of rooms) {
let {projectKey, sessionId} = extractPeerId(peerId);
let {projectKey} = extractPeerId(peerId);
if (projectKey === _projectKey) {
let connected_sockets = await io.in(peerId).fetchSockets();
for (let item of connected_sockets) {
@ -254,15 +254,6 @@ async function get_all_agents_ids(io, socket) {
return agents;
}
let geoip = null;
geoip2Reader.open(process.env.MAXMINDDB_FILE, {})
.then(reader => {
geoip = reader;
})
.catch(error => {
console.log("Error while opening the MAXMINDDB_FILE.")
console.error(error);
});
function extractSessionInfo(socket) {
if (socket.handshake.query.sessionInfo !== undefined) {
@ -379,7 +370,7 @@ module.exports = {
let rooms = await io.of('/').adapter.allRooms();
let validRooms = [];
console.log(` ====== Rooms: ${rooms.size} ====== `);
const arr = Array.from(rooms)
// const arr = Array.from(rooms)
// const filtered = arr.filter(room => !room[1].has(room[0]))
for (let i of rooms) {
let {projectKey, sessionId} = extractPeerId(i);

25
ee/utilities/servers/websocket.js
View file

@ -1,8 +1,8 @@
const _io = require('socket.io');
const express = require('express');
const uaParser = require('ua-parser-js');
const geoip2Reader = require('@maxmind/geoip2-node').Reader;
const {extractPeerId} = require('./peerjs-server');
const {extractPeerId} = require('../utils/helper');
const {geoip} = require('../utils/geoIP');
const wsRouter = express.Router();
const UPDATE_EVENT = "UPDATE_SESSION";
const IDENTITIES = {agent: 'agent', session: 'session'};
@ -108,7 +108,7 @@ const socketsList = async function (req, res) {
}
respond(res, liveSessions);
}
wsRouter.get(`/${process.env.S3_KEY}/sockets-list`, socketsList);
wsRouter.get(`/sockets-list`, socketsList);
const socketsListByProject = async function (req, res) {
debug && console.log("[WS]looking for available sessions");
@ -134,7 +134,7 @@ const socketsListByProject = async function (req, res) {
}
respond(res, liveSessions[_projectKey] || []);
}
wsRouter.get(`/${process.env.S3_KEY}/sockets-list/:projectKey`, socketsListByProject);
wsRouter.get(`/sockets-list/:projectKey`, socketsListByProject);
const socketsLive = async function (req, res) {
debug && console.log("[WS]looking for all available LIVE sessions");
@ -142,7 +142,7 @@ const socketsLive = async function (req, res) {
let liveSessions = {};
let rooms = await getAvailableRooms();
for (let peerId of rooms) {
let {projectKey, sessionId} = extractPeerId(peerId);
let {projectKey} = extractPeerId(peerId);
if (projectKey !== undefined) {
let connected_sockets = await io.in(peerId).fetchSockets();
for (let item of connected_sockets) {
@ -161,7 +161,7 @@ const socketsLive = async function (req, res) {
}
respond(res, liveSessions);
}
wsRouter.get(`/${process.env.S3_KEY}/sockets-live`, socketsLive);
wsRouter.get(`/sockets-live`, socketsLive);
const socketsLiveByProject = async function (req, res) {
debug && console.log("[WS]looking for available LIVE sessions");
@ -170,7 +170,7 @@ const socketsLiveByProject = async function (req, res) {
let liveSessions = {};
let rooms = await getAvailableRooms();
for (let peerId of rooms) {
let {projectKey, sessionId} = extractPeerId(peerId);
let {projectKey} = extractPeerId(peerId);
if (projectKey === _projectKey) {
let connected_sockets = await io.in(peerId).fetchSockets();
for (let item of connected_sockets) {
@ -189,7 +189,7 @@ const socketsLiveByProject = async function (req, res) {
}
respond(res, liveSessions[_projectKey] || []);
}
wsRouter.get(`/${process.env.S3_KEY}/sockets-live/:projectKey`, socketsLiveByProject);
wsRouter.get(`/sockets-live/:projectKey`, socketsLiveByProject);
const findSessionSocketId = async (io, peerId) => {
const connected_sockets = await io.in(peerId).fetchSockets();
@ -233,15 +233,6 @@ async function get_all_agents_ids(io, socket) {
return agents;
}
let geoip = null;
geoip2Reader.open(process.env.MAXMINDDB_FILE, {})
.then(reader => {
geoip = reader;
})
.catch(error => {
console.log("Error while opening the MAXMINDDB_FILE.")
console.error(error);
});
function extractSessionInfo(socket) {
if (socket.handshake.query.sessionInfo !== undefined) {

6
peers/.gitignore vendored Normal file
View file

@ -0,0 +1,6 @@
.idea
node_modules
npm-debug.log
.cache
test.html
/utils/

14
peers/Dockerfile Normal file
View file

@ -0,0 +1,14 @@
FROM node:17-stretch
WORKDIR /work
COPY . .
RUN npm install
# Add Tini
# Startup daemon
ENV TINI_VERSION v0.19.0
ARG envarg
ENV ENTERPRISE_BUILD ${envarg}
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini
ENTRYPOINT ["/tini", "--"]
CMD npm start

33
peers/build.sh Normal file
View file

@ -0,0 +1,33 @@
#!/bin/bash
# Script to build api module
# flags to accept:
# Default will be OSS build.
# Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh <ee>
git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)}
check_prereq() {
which docker || {
echo "Docker not installed, please install docker."
exit=1
}
[[ exit -eq 1 ]] && exit 1
}
function build_api(){
cp -R ../utilities/utils .
# Copy enterprise code
[[ $1 == "ee" ]] && {
cp -rf ../ee/peers/* ./
}
docker build -f ./Dockerfile -t ${DOCKER_REPO:-'local'}/peers:${git_sha1} .
[[ $PUSH_IMAGE -eq 1 ]] && {
docker push ${DOCKER_REPO:-'local'}/peers:${git_sha1}
docker tag ${DOCKER_REPO:-'local'}/peers:${git_sha1} ${DOCKER_REPO:-'local'}/peers:latest
docker push ${DOCKER_REPO:-'local'}/peers:latest
}
}
check_prereq
build_api $1

1584
peers/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

24
peers/package.json Normal file
View file

@ -0,0 +1,24 @@
{
"name": "peers-server",
"version": "1.0.0",
"description": "assist server to get live sessions & sourcemaps reader to get stack trace",
"main": "peerjs-server.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node server.js"
},
"repository": {
"type": "git",
"url": "git+https://github.com/openreplay/openreplay.git"
},
"author": "KRAIEM Taha Yassine <tahayk2@gmail.com>",
"license": "Elastic License 2.0 (ELv2)",
"bugs": {
"url": "https://github.com/openreplay/openreplay/issues"
},
"homepage": "https://github.com/openreplay/openreplay#readme",
"dependencies": {
"express": "^4.17.1",
"peer": "^0.6.1"
}
}

33
peers/server.js Normal file
View file

@ -0,0 +1,33 @@
const dumps = require('./utils/HeapSnapshot');
const {request_logger} = require('./utils/helper');
const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server');
const express = require('express');
const {ExpressPeerServer} = require('peer');
const HOST = '0.0.0.0';
const PORT = 9000;
const app = express();
app.use(request_logger("[app]"));
app.use(`/${process.env.S3_KEY}/assist`, peerRouter);
app.use(`/${process.env.S3_KEY}/heapdump`, dumps.router);
const server = app.listen(PORT, HOST, () => {
console.log(`App listening on http://${HOST}:${PORT}`);
console.log('Press Ctrl+C to quit.');
});
const peerServer = ExpressPeerServer(server, {
debug: true,
path: '/',
proxied: true,
allow_discovery: false
});
peerServer.on('connection', peerConnection);
peerServer.on('disconnect', peerDisconnect);
peerServer.on('error', peerError);
app.use('/', peerServer);
app.enable('trust proxy');
module.exports = {server};

23
utilities/servers/peerjs-server.js → peers/servers/peerjs-server.js
View file

@ -1,20 +1,9 @@
var express = require('express');
var peerRouter = express.Router();
const express = require('express');
const peerRouter = express.Router();
const {extractPeerId} = require('../utils/helper');
let PROJECT_KEY_LENGTH = parseInt(process.env.PROJECT_KEY_LENGTH) || 20;
let debug = process.env.debug === "1" || false;
const extractPeerId = (peerId) => {
let splited = peerId.split("-");
if (splited.length !== 2) {
debug && console.error(`cannot split peerId: ${peerId}`);
return {};
}
if (PROJECT_KEY_LENGTH > 0 && splited[0].length !== PROJECT_KEY_LENGTH) {
debug && console.error(`wrong project key length for peerId: ${peerId}`);
return {};
}
return {projectKey: splited[0], sessionId: splited[1]};
};
const connectedPeers = {};
const peerConnection = (client) => {
@ -53,13 +42,13 @@ const peerError = (error) => {
}
peerRouter.get(`/${process.env.S3_KEY}/peers`, function (req, res) {
peerRouter.get(`/peers`, function (req, res) {
debug && console.log("looking for all available sessions");
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({"data": connectedPeers}));
});
peerRouter.get(`/${process.env.S3_KEY}/peers/:projectKey`, function (req, res) {
peerRouter.get(`/peers/:projectKey`, function (req, res) {
debug && console.log(`looking for available sessions for ${req.params.projectKey}`);
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');

21
scripts/helm/db/init_dbs/postgresql/1.5.5/1.5.5.sql
View file

@ -48,7 +48,15 @@ CREATE TABLE IF NOT EXISTS dashboard_widgets
config jsonb NOT NULL DEFAULT '{}'::jsonb
);
ALTER TABLE events_common.requests
ADD COLUMN IF NOT EXISTS host text NULL,
ADD COLUMN IF NOT EXISTS base_path text NULL,
ADD COLUMN IF NOT EXISTS query text NULL;
ALTER TABLE events.pages
ADD COLUMN IF NOT EXISTS query text NULL;
COMMIT;
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'areaChart';
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'barChart';
ALTER TYPE metric_view_type ADD VALUE IF NOT EXISTS 'stackedBarChart';
@ -114,4 +122,15 @@ ON CONFLICT (predefined_key) DO UPDATE
is_template=excluded.is_template,
is_public=excluded.is_public,
metric_type=excluded.metric_type,
view_type=excluded.view_type;
view_type=excluded.view_type;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;

291
scripts/helm/db/init_dbs/postgresql/init_schema.sql
View file

@ -645,6 +645,9 @@ $$
response_body text NULL,
status_code smallint NULL,
method http_method NULL,
host text NULL,
base_path text NULL,
query text NULL,
PRIMARY KEY (session_id, timestamp, seq_index)
);
CREATE INDEX requests_url_idx ON events_common.requests (url);
@ -664,6 +667,12 @@ $$
CREATE INDEX requests_response_body_nn_idx ON events_common.requests (response_body) WHERE response_body IS NOT NULL;
CREATE INDEX requests_response_body_nn_gin_idx ON events_common.requests USING GIN (response_body gin_trgm_ops) WHERE response_body IS NOT NULL;
CREATE INDEX requests_status_code_nn_idx ON events_common.requests (status_code) WHERE status_code IS NOT NULL;
CREATE INDEX requests_host_nn_idx ON events_common.requests (host) WHERE host IS NOT NULL;
CREATE INDEX requests_host_nn_gin_idx ON events_common.requests USING GIN (host gin_trgm_ops) WHERE host IS NOT NULL;
CREATE INDEX requests_base_path_nn_idx ON events_common.requests (base_path) WHERE base_path IS NOT NULL;
CREATE INDEX requests_base_path_nn_gin_idx ON events_common.requests USING GIN (base_path gin_trgm_ops) WHERE base_path IS NOT NULL;
CREATE INDEX requests_query_nn_idx ON events_common.requests (query) WHERE query IS NOT NULL;
CREATE INDEX requests_query_nn_gin_idx ON events_common.requests USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
-- --- events.sql ---
CREATE SCHEMA IF NOT EXISTS events;
@ -676,6 +685,7 @@ $$
host text NOT NULL,
path text NOT NULL,
base_path text NOT NULL,
query text NULL,
referrer text DEFAULT NULL,
base_referrer text DEFAULT NULL,
dom_building_time integer DEFAULT NULL,
@ -731,6 +741,8 @@ $$
CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL;
CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp);
CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2;
CREATE INDEX IF NOT EXISTS pages_query_nn_idx ON events.pages (query) WHERE query IS NOT NULL;
CREATE INDEX IF NOT EXISTS pages_query_nn_gin_idx ON events.pages USING GIN (query gin_trgm_ops) WHERE query IS NOT NULL;
CREATE TABLE events.clicks
@ -958,7 +970,11 @@ $$
is_predefined boolean NOT NULL DEFAULT FALSE,
is_template boolean NOT NULL DEFAULT FALSE,
predefined_key text NULL DEFAULT NULL,
default_config jsonb NOT NULL DEFAULT '{"col": 2,"row": 2,"position": 0}'::jsonb,
default_config jsonb NOT NULL DEFAULT '{
"col": 2,
"row": 2,
"position": 0
}'::jsonb,
CONSTRAINT null_project_id_for_template_only
CHECK ( (metrics.category != 'custom') != (metrics.project_id IS NOT NULL) ),
CONSTRAINT unique_key UNIQUE (predefined_key)
@ -1048,55 +1064,236 @@ $$
$$
LANGUAGE plpgsql;
INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type, view_type)
VALUES ('Captured sessions', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_sessions', 'predefined', 'overview'),
('Request Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'),
('Page Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'),
('Image Load Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'),
('DOM Content Load Start', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'),
('First Meaningful paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'),
('No. of Visited Pages', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'),
('Session Duration', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_session_duration', 'predefined', 'overview'),
('DOM Build Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'),
('Pages Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'),
('Response Time', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_response_time', 'predefined', 'overview'),
('First Paint', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_first_paint', 'predefined', 'overview'),
('DOM Content Loaded', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'),
('Time Till First byte', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'),
('Time To Interactive', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'),
('Captured requests', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'count_requests', 'predefined', 'overview'),
('Time To Render', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'),
('Memory Consumption', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'),
('CPU Load', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_cpu', 'predefined', 'overview'),
('Frame rate', 'overview', '{"col":1,"row":1,"position":0}', true, true, true, 'avg_fps', 'predefined', 'overview'),
INSERT INTO metrics (name, category, default_config, is_predefined, is_template, is_public, predefined_key, metric_type,
view_type)
VALUES ('Captured sessions', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'count_sessions', 'predefined', 'overview'),
('Request Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_request_load_time', 'predefined', 'overview'),
('Page Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_page_load_time', 'predefined', 'overview'),
('Image Load Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_image_load_time', 'predefined', 'overview'),
('DOM Content Load Start', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_dom_content_load_start', 'predefined', 'overview'),
('First Meaningful paint', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_first_contentful_pixel', 'predefined', 'overview'),
('No. of Visited Pages', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_visited_pages', 'predefined', 'overview'),
('Session Duration', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_session_duration', 'predefined', 'overview'),
('DOM Build Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_pages_dom_buildtime', 'predefined', 'overview'),
('Pages Response Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_pages_response_time', 'predefined', 'overview'),
('Response Time', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_response_time', 'predefined', 'overview'),
('First Paint', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_first_paint', 'predefined', 'overview'),
('DOM Content Loaded', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_dom_content_loaded', 'predefined', 'overview'),
('Time Till First byte', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_till_first_byte', 'predefined', 'overview'),
('Time To Interactive', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_time_to_interactive', 'predefined', 'overview'),
('Captured requests', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'count_requests', 'predefined', 'overview'),
('Time To Render', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_time_to_render', 'predefined', 'overview'),
('Memory Consumption', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_used_js_heap_size', 'predefined', 'overview'),
('CPU Load', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_cpu', 'predefined', 'overview'),
('Frame rate', 'overview', '{
"col": 1,
"row": 1,
"position": 0
}', true, true, true, 'avg_fps', 'predefined', 'overview'),
('Sessions Affected by JS Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'),
('Top Domains with 4xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'),
('Top Domains with 5xx Fetch Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'),
('Errors per Domain', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_domains', 'predefined', 'table'),
('Fetch Calls with Errors', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'calls_errors', 'predefined', 'table'),
('Errors by Type', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'errors_per_type', 'predefined', 'barChart'),
('Errors by Origin', 'errors', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'),
('Sessions Affected by JS Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'impacted_sessions_by_js_errors', 'predefined', 'barChart'),
('Top Domains with 4xx Fetch Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'domains_errors_4xx', 'predefined', 'lineChart'),
('Top Domains with 5xx Fetch Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'domains_errors_5xx', 'predefined', 'lineChart'),
('Errors per Domain', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'errors_per_domains', 'predefined', 'table'),
('Fetch Calls with Errors', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'calls_errors', 'predefined', 'table'),
('Errors by Type', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'errors_per_type', 'predefined', 'barChart'),
('Errors by Origin', 'errors', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_by_party', 'predefined', 'stackedBarChart'),
('Speed Index by Location', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'speed_location', 'predefined', 'map'),
('Slowest Domains', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'slowest_domains', 'predefined', 'table'),
('Sessions per Browser', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'sessions_per_browser', 'predefined', 'table'),
('Time To Render', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'time_to_render', 'predefined', 'areaChart'),
('Sessions Impacted by Slow Pages', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'),
('Memory Consumption', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'),
('CPU Load', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'cpu', 'predefined', 'areaChart'),
('Frame Rate', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'fps', 'predefined', 'areaChart'),
('Crashes', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'crashes', 'predefined', 'areaChart'),
('Resources Loaded vs Visually Complete', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'),
('DOM Build Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'),
('Pages Response Time', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'),
('Pages Response Time Distribution', 'performance', '{"col":2,"row":2,"position":0}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'),
('Speed Index by Location', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'speed_location', 'predefined', 'map'),
('Slowest Domains', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'slowest_domains', 'predefined', 'table'),
('Sessions per Browser', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'sessions_per_browser', 'predefined', 'table'),
('Time To Render', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'time_to_render', 'predefined', 'areaChart'),
('Sessions Impacted by Slow Pages', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'impacted_sessions_by_slow_pages', 'predefined', 'areaChart'),
('Memory Consumption', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'memory_consumption', 'predefined', 'areaChart'),
('CPU Load', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'cpu', 'predefined', 'areaChart'),
('Frame Rate', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'fps', 'predefined', 'areaChart'),
('Crashes', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'crashes', 'predefined', 'areaChart'),
('Resources Loaded vs Visually Complete', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_vs_visually_complete', 'predefined', 'areaChart'),
('DOM Build Time', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_dom_buildtime', 'predefined', 'areaChart'),
('Pages Response Time', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_response_time', 'predefined', 'areaChart'),
('Pages Response Time Distribution', 'performance', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'pages_response_time_distribution', 'predefined', 'barChart'),
('Missing Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'missing_resources', 'predefined', 'table'),
('Slowest Resources', 'resources', '{"col":4,"row":2,"position":0}', true, true, true, 'slowest_resources', 'predefined', 'table'),
('Resources Fetch Time', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_loading_time', 'predefined', 'table'),
('Resource Loaded vs Response End', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'),
('Breakdown of Loaded Resources', 'resources', '{"col":2,"row":2,"position":0}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart')
('Missing Resources', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'missing_resources', 'predefined', 'table'),
('Slowest Resources', 'resources', '{
"col": 4,
"row": 2,
"position": 0
}', true, true, true, 'slowest_resources', 'predefined', 'table'),
('Resources Fetch Time', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_loading_time', 'predefined', 'table'),
('Resource Loaded vs Response End', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resource_type_vs_response_end', 'predefined', 'stackedBarLineChart'),
('Breakdown of Loaded Resources', 'resources', '{
"col": 2,
"row": 2,
"position": 0
}', true, true, true, 'resources_count_by_type', 'predefined', 'stackedBarChart')
ON CONFLICT (predefined_key) DO UPDATE
SET name=excluded.name,
category=excluded.category,

6
sourcemap-reader/.gitignore vendored Normal file
View file

@ -0,0 +1,6 @@
.idea
node_modules
npm-debug.log
.cache
test.html
/utils/

1113
sourcemap-reader/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

25
sourcemap-reader/package.json Normal file
View file

@ -0,0 +1,25 @@
{
"name": "sourcemaps-reader",
"version": "1.0.0",
"description": "assist server to get live sessions & sourcemaps reader to get stack trace",
"main": "peerjs-server.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node server.js"
},
"repository": {
"type": "git",
"url": "git+https://github.com/openreplay/openreplay.git"
},
"author": "KRAIEM Taha Yassine <tahayk2@gmail.com>",
"license": "Elastic License 2.0 (ELv2)",
"bugs": {
"url": "https://github.com/openreplay/openreplay/issues"
},
"homepage": "https://github.com/openreplay/openreplay#readme",
"dependencies": {
"aws-sdk": "^2.992.0",
"express": "^4.17.1",
"source-map": "^0.7.3"
}
}

19
sourcemap-reader/server.js Normal file
View file

@ -0,0 +1,19 @@
const dumps = require('./utils/HeapSnapshot');
const sourcemapsReaderServer = require('./servers/sourcemaps-server');
const express = require('express');
const {request_logger} = require("./utils/helper");
const HOST = '0.0.0.0';
const PORT = 9000;
const app = express();
app.use(request_logger("[wsapp]"));
app.use('/sourcemaps', sourcemapsReaderServer);
app.use('/heapdump', dumps.router);
const server = app.listen(PORT, HOST, () => {
console.log(`WS App listening on http://${HOST}:${PORT}`);
console.log('Press Ctrl+C to quit.');
});
module.exports = {server};

0
utilities/servers/sourcemaps-handler.js → sourcemap-reader/servers/sourcemaps-handler.js
View file

0
utilities/servers/sourcemaps-server.js → sourcemap-reader/servers/sourcemaps-server.js
View file

891
utilities/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

7
utilities/package.json
View file

@ -1,5 +1,5 @@
{
"name": "utilities_server",
"name": "utilities-server",
"version": "1.0.0",
"description": "assist server to get live sessions & sourcemaps reader to get stack trace",
"main": "peerjs-server.js",
@ -12,18 +12,15 @@
"url": "git+https://github.com/openreplay/openreplay.git"
},
"author": "KRAIEM Taha Yassine <tahayk2@gmail.com>",
"license": "MIT",
"license": "Elastic License 2.0 (ELv2)",
"bugs": {
"url": "https://github.com/openreplay/openreplay/issues"
},
"homepage": "https://github.com/openreplay/openreplay#readme",
"dependencies": {
"@maxmind/geoip2-node": "^3.4.0",
"aws-sdk": "^2.992.0",
"express": "^4.17.1",
"peer": "^0.6.1",
"socket.io": "^4.4.1",
"source-map": "^0.7.3",
"ua-parser-js": "^1.0.2"
}
}

50
utilities/server.js
View file

@ -1,57 +1,21 @@
const dumps = require('./utils/HeapSnapshot');
const sourcemapsReaderServer = require('./servers/sourcemaps-server');
const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server');
const express = require('express');
const {ExpressPeerServer} = require('peer');
const socket = require("./servers/websocket");
const {request_logger} = require("./utils/helper");
const HOST = '0.0.0.0';
const PORT = 9000;
const PORT = 9001;
const app = express();
const wsapp = express();
let debug = process.env.debug === "1" || false;
const request_logger = (identity) => {
return (req, res, next) => {
debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl);
res.on('finish', function () {
if (this.statusCode !== 200 || debug) {
console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode);
}
})
next();
}
};
app.use(request_logger("[app]"));
wsapp.use(request_logger("[wsapp]"));
app.use('/sourcemaps', sourcemapsReaderServer);
app.use('/assist', peerRouter);
wsapp.use('/assist', socket.wsRouter);
wsapp.use(`/assist/${process.env.S3_KEY}`, socket.wsRouter);
wsapp.use(`/heapdump/${process.env.S3_KEY}`, dumps.router);
app.use('/heapdump', dumps.router);
const server = app.listen(PORT, HOST, () => {
console.log(`App listening on http://${HOST}:${PORT}`);
const wsserver = wsapp.listen(PORT, HOST, () => {
console.log(`WS App listening on http://${HOST}:${PORT}`);
console.log('Press Ctrl+C to quit.');
});
const wsserver = wsapp.listen(PORT + 1, HOST, () => {
console.log(`WS App listening on http://${HOST}:${PORT + 1}`);
console.log('Press Ctrl+C to quit.');
});
const peerServer = ExpressPeerServer(server, {
debug: true,
path: '/',
proxied: true,
allow_discovery: false
});
peerServer.on('connection', peerConnection);
peerServer.on('disconnect', peerDisconnect);
peerServer.on('error', peerError);
app.use('/', peerServer);
app.enable('trust proxy');
wsapp.enable('trust proxy');
socket.start(wsserver);
module.exports = {wsserver, server};
console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`);
module.exports = {wsserver};

25
utilities/servers/websocket.js
View file

@ -1,8 +1,8 @@
const _io = require('socket.io');
const express = require('express');
const uaParser = require('ua-parser-js');
const geoip2Reader = require('@maxmind/geoip2-node').Reader;
const {extractPeerId} = require('./peerjs-server');
const {extractPeerId} = require('../utils/helper');
const {geoip} = require('../utils/geoIP');
const wsRouter = express.Router();
const UPDATE_EVENT = "UPDATE_SESSION";
const IDENTITIES = {agent: 'agent', session: 'session'};
@ -79,7 +79,7 @@ const socketsList = async function (req, res) {
}
respond(res, liveSessions);
}
wsRouter.get(`/${process.env.S3_KEY}/sockets-list`, socketsList);
wsRouter.get(`/sockets-list`, socketsList);
const socketsListByProject = async function (req, res) {
debug && console.log("[WS]looking for available sessions");
@ -105,7 +105,7 @@ const socketsListByProject = async function (req, res) {
}
respond(res, liveSessions[_projectKey] || []);
}
wsRouter.get(`/${process.env.S3_KEY}/sockets-list/:projectKey`, socketsListByProject);
wsRouter.get(`/sockets-list/:projectKey`, socketsListByProject);
const socketsLive = async function (req, res) {
debug && console.log("[WS]looking for all available LIVE sessions");
@ -113,7 +113,7 @@ const socketsLive = async function (req, res) {
let liveSessions = {};
let rooms = await getAvailableRooms();
for (let peerId of rooms) {
let {projectKey, sessionId} = extractPeerId(peerId);
let {projectKey} = extractPeerId(peerId);
if (projectKey !== undefined) {
let connected_sockets = await io.in(peerId).fetchSockets();
for (let item of connected_sockets) {
@ -132,7 +132,7 @@ const socketsLive = async function (req, res) {
}
respond(res, liveSessions);
}
wsRouter.get(`/${process.env.S3_KEY}/sockets-live`, socketsLive);
wsRouter.get(`/sockets-live`, socketsLive);
const socketsLiveByProject = async function (req, res) {
debug && console.log("[WS]looking for available LIVE sessions");
@ -141,7 +141,7 @@ const socketsLiveByProject = async function (req, res) {
let liveSessions = {};
let rooms = await getAvailableRooms();
for (let peerId of rooms) {
let {projectKey, sessionId} = extractPeerId(peerId);
let {projectKey} = extractPeerId(peerId);
if (projectKey === _projectKey) {
let connected_sockets = await io.in(peerId).fetchSockets();
for (let item of connected_sockets) {
@ -160,7 +160,7 @@ const socketsLiveByProject = async function (req, res) {
}
respond(res, liveSessions[_projectKey] || []);
}
wsRouter.get(`/${process.env.S3_KEY}/sockets-live/:projectKey`, socketsLiveByProject);
wsRouter.get(`/sockets-live/:projectKey`, socketsLiveByProject);
const findSessionSocketId = async (io, peerId) => {
const connected_sockets = await io.in(peerId).fetchSockets();
@ -204,15 +204,6 @@ async function get_all_agents_ids(io, socket) {
return agents;
}
let geoip = null;
geoip2Reader.open(process.env.MAXMINDDB_FILE, {})
.then(reader => {
geoip = reader;
})
.catch(error => {
console.log("Error while opening the MAXMINDDB_FILE.")
console.error(error);
});
function extractSessionInfo(socket) {
if (socket.handshake.query.sessionInfo !== undefined) {

9
utilities/utils/HeapSnapshot.js
View file

@ -31,6 +31,9 @@ async function downloadHeapSnapshot(req, res) {
return res.end("should wait for done status");
}
res.download(location + fileName, function (err) {
if (err) {
return console.error("error while uploading HeapSnapshot file");
}
try {
fs.unlinkSync(location + fileName)
} catch (err) {
@ -57,7 +60,9 @@ function createNewHeapSnapshot(req, res) {
res.end(JSON.stringify({path: location + fileName, 'done': creationStatus}));
}
router.get('/status', getHeapSnapshotStatus);
router.get(`/status`, getHeapSnapshotStatus);
router.get(`/new`, createNewHeapSnapshot);
router.get(`/download`, downloadHeapSnapshot);
module.exports = {router}
module.exports = {router}
console.log(`HeapSnapshot enabled. Send a request to "/heapdump/new" to generate a heapdump.`);

16
utilities/utils/geoIP.js Normal file
View file

@ -0,0 +1,16 @@
const geoip2Reader = require('@maxmind/geoip2-node').Reader;
let geoip = null;
if (process.env.MAXMINDDB_FILE !== undefined) {
geoip2Reader.open(process.env.MAXMINDDB_FILE, {})
.then(reader => {
geoip = reader;
})
.catch(error => {
console.log("Error while opening the MAXMINDDB_FILE.")
console.error(error);
});
} else {
console.error("!!! please provide a valid value for MAXMINDDB_FILE env var.");
}
module.exports = {geoip}

30
utilities/utils/helper.js Normal file
View file

@ -0,0 +1,30 @@
let PROJECT_KEY_LENGTH = parseInt(process.env.PROJECT_KEY_LENGTH) || 20;
let debug = process.env.debug === "1" || false;
const extractPeerId = (peerId) => {
let splited = peerId.split("-");
if (splited.length !== 2) {
debug && console.error(`cannot split peerId: ${peerId}`);
return {};
}
if (PROJECT_KEY_LENGTH > 0 && splited[0].length !== PROJECT_KEY_LENGTH) {
debug && console.error(`wrong project key length for peerId: ${peerId}`);
return {};
}
return {projectKey: splited[0], sessionId: splited[1]};
};
const request_logger = (identity) => {
return (req, res, next) => {
debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl);
res.on('finish', function () {
if (this.statusCode !== 200 || debug) {
console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode);
}
})
next();
}
};
module.exports = {
extractPeerId, request_logger
};