jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(chalice): devtools permission

Browse source 
feat(chalice): devtools as a mob file
feat(chalice): unprocessed devtools endpoint
This commit is contained in:
Taha Yassine Kraiem 2022-09-22 17:32:47 +01:00
parent 9b207d3a80
commit 3f6a90cd43
19 changed files with 1472 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
api/chalicelib/core/assist.py
View file

@ -1,10 +1,12 @@
from os import access, R_OK
from os.path import exists
import requests
from decouple import config
from os.path import exists
from starlette.exceptions import HTTPException
import schemas
from chalicelib.core import projects
from starlette.exceptions import HTTPException
from os import access, R_OK
ASSIST_KEY = config("ASSIST_KEY")
ASSIST_URL = config("ASSIST_URL") % ASSIST_KEY
@ -165,16 +167,24 @@ def get_ice_servers():
and len(config("iceServers")) > 0 else None
def get_raw_mob_by_id(project_id, session_id):
def __get_efs_path():
efs_path = config("FS_DIR")
if not exists(efs_path):
raise HTTPException(400, f"EFS not found in path: {efs_path}")
if not access(efs_path, R_OK):
raise HTTPException(400, f"EFS found under: {efs_path}; but it is not readable, please check permissions")
return efs_path
path_to_file = efs_path + "/" + str(session_id)
def __get_mob_path(project_id, session_id):
params = {"projectId": project_id, "sessionId": session_id}
return config("EFS_SESSION_MOB_PATTERN", default="%(sessionId)s") % params
def get_raw_mob_by_id(project_id, session_id):
efs_path = __get_efs_path()
path_to_file = efs_path + "/" + __get_mob_path(project_id=project_id, session_id=session_id)
if exists(path_to_file):
if not access(path_to_file, R_OK):
raise HTTPException(400, f"Replay file found under: {efs_path};"
@ -183,3 +193,21 @@ def get_raw_mob_by_id(project_id, session_id):
return path_to_file
return None
def __get_devtools_path(project_id, session_id):
params = {"projectId": project_id, "sessionId": session_id}
return config("EFS_DEVTOOLS_MOB_PATTERN", default="%(sessionId)s") % params
def get_raw_devtools_by_id(project_id, session_id):
efs_path = __get_efs_path()
path_to_file = efs_path + "/" + __get_devtools_path(project_id=project_id, session_id=session_id)
if exists(path_to_file):
if not access(path_to_file, R_OK):
raise HTTPException(400, f"Devtools file found under: {efs_path};"
f" but it is not readable, please check permissions")
return path_to_file
return None

2
api/chalicelib/core/jobs.py
View file

@ -144,7 +144,7 @@ def execute_jobs():
)
sessions.delete_sessions_by_session_ids(session_ids)
sessions_mobs.delete_mobs(session_ids)
sessions_mobs.delete_mobs(session_ids=session_ids, project_id=job["projectId"])
else:
raise Exception(f"The action {job['action']} not supported.")

15
api/chalicelib/core/sessions.py
View file

@ -2,7 +2,8 @@ from typing import List
import schemas
from chalicelib.core import events, metadata, events_ios, \
sessions_mobs, issues, projects, errors, resources, assist, performance_event, sessions_viewed, sessions_favorite
sessions_mobs, issues, projects, errors, resources, assist, performance_event, sessions_viewed, sessions_favorite, \
sessions_devtool
from chalicelib.utils import pg_client, helper, metrics_helper
SESSION_PROJECTION_COLS = """s.project_id,
@ -81,7 +82,7 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_
data['crashes'] = events_ios.get_crashes_by_session_id(session_id=session_id)
data['userEvents'] = events_ios.get_customs_by_sessionId(project_id=project_id,
session_id=session_id)
data['mobsUrl'] = sessions_mobs.get_ios(sessionId=session_id)
data['mobsUrl'] = sessions_mobs.get_ios(session_id=session_id)
else:
data['events'] = events.get_by_sessionId2_pg(project_id=project_id, session_id=session_id,
group_clickrage=True)
@ -89,14 +90,14 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_
data['stackEvents'] = [e for e in all_errors if e['source'] != "js_exception"]
# to keep only the first stack
data['errors'] = [errors.format_first_stack_frame(e) for e in all_errors if
e['source'] == "js_exception"][
:500] # limit the number of errors to reduce the response-body size
# limit the number of errors to reduce the response-body size
e['source'] == "js_exception"][:500]
data['userEvents'] = events.get_customs_by_sessionId2_pg(project_id=project_id,
session_id=session_id)
data['mobsUrl'] = sessions_mobs.get_web(sessionId=session_id)
data['domURL'] = sessions_mobs.get_urls(session_id=session_id, project_id=project_id)
data['devtoolsURL'] = sessions_devtool.get_urls(session_id=session_id, project_id=project_id)
data['resources'] = resources.get_by_session_id(session_id=session_id, project_id=project_id,
start_ts=data["startTs"],
duration=data["duration"])
start_ts=data["startTs"], duration=data["duration"])
data['metadata'] = __group_metadata(project_metadata=data.pop("projectMetadata"), session=data)
data['issues'] = issues.get_by_session_id(session_id=session_id, project_id=project_id)

24
api/chalicelib/core/sessions_devtool.py Normal file
View file

@ -0,0 +1,24 @@
from decouple import config
from chalicelib.utils.s3 import client
def __get_devtools_keys(project_id, session_id):
params = {
"sessionId": session_id,
"projectId": project_id
}
return [
config("DEVTOOLS_MOB_PATTERN", default="%(sessionId)sdevtools") % params
]
def get_urls(session_id, project_id):
results = []
for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
results.append(client.generate_presigned_url(
'get_object',
Params={'Bucket': config("sessions_bucket"), 'Key': k},
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
))
return results

12
api/chalicelib/core/sessions_favorite.py
View file

@ -7,8 +7,8 @@ def add_favorite_session(project_id, user_id, session_id):
cur.execute(
cur.mogrify(f"""\
INSERT INTO public.user_favorite_sessions(user_id, session_id)
VALUES (%(userId)s,%(sessionId)s);""",
{"userId": user_id, "sessionId": session_id})
VALUES (%(userId)s,%(session_id)s);""",
{"userId": user_id, "session_id": session_id})
)
return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False,
include_fav_viewed=True)
@ -20,8 +20,8 @@ def remove_favorite_session(project_id, user_id, session_id):
cur.mogrify(f"""\
DELETE FROM public.user_favorite_sessions
WHERE user_id = %(userId)s
AND session_id = %(sessionId)s;""",
{"userId": user_id, "sessionId": session_id})
AND session_id = %(session_id)s;""",
{"userId": user_id, "session_id": session_id})
)
return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False,
include_fav_viewed=True)
@ -42,8 +42,8 @@ def favorite_session_exists(user_id, session_id):
FROM public.user_favorite_sessions
WHERE
user_id = %(userId)s
AND session_id = %(sessionId)s""",
{"userId": user_id, "sessionId": session_id})
AND session_id = %(session_id)s""",
{"userId": user_id, "session_id": session_id})
)
r = cur.fetchone()
return r is not None

47
api/chalicelib/core/sessions_mobs.py
View file

@ -4,37 +4,40 @@ from chalicelib.utils import s3
from chalicelib.utils.s3 import client
def get_web(sessionId):
def __get_mob_keys(project_id, session_id):
params = {
"sessionId": session_id,
"projectId": project_id
}
return [
client.generate_presigned_url(
'get_object',
Params={
'Bucket': config("sessions_bucket"),
'Key': str(sessionId)
},
ExpiresIn=100000
),
client.generate_presigned_url(
'get_object',
Params={
'Bucket': config("sessions_bucket"),
'Key': str(sessionId) + "e"
},
ExpiresIn=100000
)]
config("SESSION_MOB_PATTERN_S", default="%(sessionId)s") % params,
config("SESSION_MOB_PATTERN_E", default="%(sessionId)se") % params
]
def get_ios(sessionId):
def get_urls(project_id, session_id):
results = []
for k in __get_mob_keys(project_id=project_id, session_id=session_id):
results.append(client.generate_presigned_url(
'get_object',
Params={'Bucket': config("sessions_bucket"), 'Key': k},
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
))
return results
def get_ios(session_id):
return client.generate_presigned_url(
'get_object',
Params={
'Bucket': config("ios_bucket"),
'Key': str(sessionId)
'Key': str(session_id)
},
ExpiresIn=100000
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
)
def delete_mobs(session_ids):
def delete_mobs(project_id, session_ids):
for session_id in session_ids:
s3.schedule_for_deletion(config("sessions_bucket"), session_id)
for k in __get_mob_keys(project_id=project_id, session_id=session_id):
s3.schedule_for_deletion(config("sessions_bucket"), k)

4
api/chalicelib/core/sessions_viewed.py
View file

@ -5,7 +5,7 @@ def view_session(project_id, user_id, session_id):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify("""INSERT INTO public.user_viewed_sessions(user_id, session_id)
VALUES (%(userId)s,%(sessionId)s)
VALUES (%(userId)s,%(session_id)s)
ON CONFLICT DO NOTHING;""",
{"userId": user_id, "sessionId": session_id})
{"userId": user_id, "session_id": session_id})
)

2
api/chalicelib/core/socket_ios.py
View file

@ -7,7 +7,7 @@ def start_replay(project_id, session_id, device, os_version, mob_url):
r = requests.post(config("IOS_MIDDLEWARE") + "/replay", json={
"projectId": project_id,
"projectKey": projects.get_project_key(project_id),
"sessionId": session_id,
"session_id": session_id,
"device": device,
"osVersion": os_version,
"mobUrl": mob_url

8
api/env.default
View file

@ -48,4 +48,10 @@ sourcemaps_bucket=sourcemaps
sourcemaps_reader=http://127.0.0.1:9000/sourcemaps/%s/sourcemaps
stage=default-foss
version_number=1.4.0
FS_DIR=/mnt/efs
FS_DIR=/mnt/efs
EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob
EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs
SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe
DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
PRESIGNED_URL_EXPIRATION=3600

27
api/routers/core_dynamic.py
View file

@ -1,12 +1,13 @@
from typing import Optional
from typing import Optional, Union
from decouple import config
from fastapi import Body, Depends, BackgroundTasks
from starlette.responses import RedirectResponse
from starlette.responses import RedirectResponse, FileResponse
import schemas
from chalicelib.core import integrations_manager
from chalicelib.core import sessions
from chalicelib.core import sessions, errors, errors_viewed, errors_favorite, sessions_assignments, heatmaps, \
sessions_favorite, assist
from chalicelib.core import sessions_viewed
from chalicelib.core import tenants, users, projects, license
from chalicelib.core import webhook
from chalicelib.core.collaboration_slack import Slack
@ -292,6 +293,24 @@ def get_live_session_replay_file(projectId: int, sessionId: Union[int, str],
return FileResponse(path=path, media_type="application/octet-stream")
@app.get('/{projectId}/unprocessed/{sessionId}/devtools', tags=["assist"])
@app.get('/{projectId}/assist/sessions/{sessionId}/devtools', tags=["assist"])
def get_live_session_devtools_file(projectId: int, sessionId: Union[int, str],
context: schemas.CurrentContext = Depends(OR_context)):
if isinstance(sessionId, str) or not sessions.session_exists(project_id=projectId, session_id=sessionId):
if isinstance(sessionId, str):
print(f"{sessionId} not a valid number.")
else:
print(f"{projectId}/{sessionId} not found in DB.")
return {"errors": ["Devtools file not found"]}
path = assist.get_raw_devtools_by_id(project_id=projectId, session_id=sessionId)
if path is None:
return {"errors": ["Devtools file not found"]}
return FileResponse(path=path, media_type="application/octet-stream")
@app.post('/{projectId}/heatmaps/url', tags=["heatmaps"])
def get_heatmaps_by_url(projectId: int, data: schemas.GetHeatmapPayloadSchema = Body(...),
context: schemas.CurrentContext = Depends(OR_context)):

1
ee/api/.gitignore vendored
View file

@ -210,7 +210,6 @@ Pipfile
/chalicelib/core/log_tool_sumologic.py
/chalicelib/core/metadata.py
/chalicelib/core/mobile.py
/chalicelib/core/sessions.py
/chalicelib/core/sessions_assignments.py
/chalicelib/core/sessions_metas.py
/chalicelib/core/sessions_mobs.py

10
ee/api/chalicelib/core/permissions.py Normal file
View file

@ -0,0 +1,10 @@
from fastapi.security import SecurityScopes
import schemas_ee
def check(security_scopes: SecurityScopes, context: schemas_ee.CurrentContext):
for scope in security_scopes.scopes:
if scope not in context.permissions:
return False
return True

1254
ee/api/chalicelib/core/sessions.py Normal file
View file

File diff suppressed because it is too large Load diff

31
ee/api/chalicelib/core/sessions_devtool.py Normal file
View file

@ -0,0 +1,31 @@
from decouple import config
from fastapi.security import SecurityScopes
import schemas_ee
from chalicelib.core import permissions
from chalicelib.utils.s3 import client
SCOPES = SecurityScopes([schemas_ee.Permissions.dev_tools])
def __get_devtools_keys(project_id, session_id):
params = {
"sessionId": session_id,
"projectId": project_id
}
return [
config("DEVTOOLS_MOB_PATTERN", default="%(sessionId)sdevtools") % params
]
def get_urls(session_id, project_id, context: schemas_ee.CurrentContext):
if not permissions.check(security_scopes=SCOPES, context=context):
return []
results = []
for k in __get_devtools_keys(project_id=project_id, session_id=session_id):
results.append(client.generate_presigned_url(
'get_object',
Params={'Bucket': config("sessions_bucket"), 'Key': k},
ExpiresIn=config("PRESIGNED_URL_EXPIRATION", cast=int, default=900)
))
return results

13
ee/api/chalicelib/core/sessions_favorite.py
View file

@ -1,10 +1,11 @@
from decouple import config
import schemas_ee
from chalicelib.core import sessions, sessions_favorite_exp
from chalicelib.utils import pg_client, s3_extra
def add_favorite_session(project_id, user_id, session_id):
def add_favorite_session(project_id, user_id, session_id, context: schemas_ee.CurrentContext):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify(f"""\
@ -15,10 +16,10 @@ def add_favorite_session(project_id, user_id, session_id):
sessions_favorite_exp.add_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id)
return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False,
include_fav_viewed=True)
include_fav_viewed=True, context=context)
def remove_favorite_session(project_id, user_id, session_id):
def remove_favorite_session(project_id, user_id, session_id, context: schemas_ee.CurrentContext):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify(f"""\
@ -29,10 +30,10 @@ def remove_favorite_session(project_id, user_id, session_id):
)
sessions_favorite_exp.remove_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id)
return sessions.get_by_id2_pg(project_id=project_id, session_id=session_id, user_id=user_id, full_data=False,
include_fav_viewed=True)
include_fav_viewed=True, context=context)
def favorite_session(project_id, user_id, session_id):
def favorite_session(project_id, user_id, session_id, context: schemas_ee.CurrentContext):
if favorite_session_exists(user_id=user_id, session_id=session_id):
key = str(session_id)
try:
@ -59,7 +60,7 @@ def favorite_session(project_id, user_id, session_id):
except Exception as e:
print(f"!!!Error while tagging: {key} to vault")
print(str(e))
return add_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id)
return add_favorite_session(project_id=project_id, user_id=user_id, session_id=session_id, context=context)
def favorite_session_exists(user_id, session_id):

1
ee/api/clean.sh
View file

@ -32,7 +32,6 @@ rm -rf ./chalicelib/core/log_tool_stackdriver.py
rm -rf ./chalicelib/core/log_tool_sumologic.py
rm -rf ./chalicelib/core/metadata.py
rm -rf ./chalicelib/core/mobile.py
rm -rf ./chalicelib/core/sessions.py
rm -rf ./chalicelib/core/sessions_assignments.py
rm -rf ./chalicelib/core/sessions_metas.py
rm -rf ./chalicelib/core/sessions_mobs.py

8
ee/api/env.default
View file

@ -68,4 +68,10 @@ EXP_7D_MV=false
EXP_ALERTS=false
EXP_FUNNELS=false
EXP_RESOURCES=true
TRACE_PERIOD=300
TRACE_PERIOD=300
EFS_SESSION_MOB_PATTERN=%(sessionId)s/dom.mob
EFS_DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
SESSION_MOB_PATTERN_S=%(sessionId)s/dom.mobs
SESSION_MOB_PATTERN_E=%(sessionId)s/dom.mobe
DEVTOOLS_MOB_PATTERN=%(sessionId)s/devtools.mob
PRESIGNED_URL_EXPIRATION=3600

5
ee/api/or_dependencies.py
View file

@ -12,6 +12,7 @@ from starlette.responses import Response, JSONResponse
import schemas_ee
from chalicelib.core import traces
from chalicelib.core import permissions
async def OR_context(request: Request) -> schemas_ee.CurrentContext:
@ -48,7 +49,7 @@ class ORRoute(APIRoute):
return custom_route_handler
def check_permissions(security_scopes: SecurityScopes, context: schemas_ee.CurrentContext = Depends(OR_context)):
def __check(security_scopes: SecurityScopes, context: schemas_ee.CurrentContext = Depends(OR_context)):
for scope in security_scopes.scopes:
if scope not in context.permissions:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED,
@ -56,4 +57,4 @@ def check_permissions(security_scopes: SecurityScopes, context: schemas_ee.Curre
def OR_scope(*scopes):
return Security(check_permissions, scopes=list(scopes))
return Security(__check, scopes=list(scopes))

37
ee/api/routers/core_dynamic.py
View file

@ -2,11 +2,12 @@ from typing import Optional, Union
from decouple import config
from fastapi import Body, Depends, BackgroundTasks
from starlette.responses import RedirectResponse
from starlette.responses import RedirectResponse, FileResponse
import schemas
import schemas_ee
from chalicelib.core import sessions
from chalicelib.core import sessions, assist, heatmaps, sessions_favorite, sessions_assignments, errors, errors_viewed, \
errors_favorite
from chalicelib.core import sessions_viewed
from chalicelib.core import tenants, users, projects, license
from chalicelib.core import webhook
@ -183,7 +184,7 @@ def get_session(projectId: int, sessionId: Union[int, str], background_tasks: Ba
if isinstance(sessionId, str):
return {"errors": ["session not found"]}
data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True, user_id=context.user_id,
include_fav_viewed=True, group_metadata=True)
include_fav_viewed=True, group_metadata=True, context=context)
if data is None:
return {"errors": ["session not found"]}
if data.get("inDB"):
@ -270,11 +271,12 @@ def add_remove_favorite_error(projectId: int, errorId: str, action: str, startDa
@app.get('/{projectId}/assist/sessions/{sessionId}', tags=["assist"], dependencies=[OR_scope(Permissions.assist_live)])
def get_live_session(projectId: int, sessionId: str, background_tasks: BackgroundTasks,
context: schemas.CurrentContext = Depends(OR_context)):
context: schemas_ee.CurrentContext = Depends(OR_context)):
data = assist.get_live_session_by_id(project_id=projectId, session_id=sessionId)
if data is None:
data = sessions.get_by_id2_pg(project_id=projectId, session_id=sessionId, full_data=True,
user_id=context.user_id, include_fav_viewed=True, group_metadata=True, live=False)
user_id=context.user_id, include_fav_viewed=True, group_metadata=True, live=False,
context=context)
if data is None:
return {"errors": ["session not found"]}
if data.get("inDB"):
@ -303,6 +305,26 @@ def get_live_session_replay_file(projectId: int, sessionId: Union[int, str],
return FileResponse(path=path, media_type="application/octet-stream")
@app.get('/{projectId}/unprocessed/{sessionId}/devtools', tags=["assist"],
dependencies=[OR_scope(Permissions.assist_live, Permissions.session_replay, Permissions.dev_tools)])
@app.get('/{projectId}/assist/sessions/{sessionId}/devtools', tags=["assist"],
dependencies=[OR_scope(Permissions.assist_live, Permissions.session_replay, Permissions.dev_tools)])
def get_live_session_devtools_file(projectId: int, sessionId: Union[int, str],
context: schemas.CurrentContext = Depends(OR_context)):
if isinstance(sessionId, str) or not sessions.session_exists(project_id=projectId, session_id=sessionId):
if isinstance(sessionId, str):
print(f"{sessionId} not a valid number.")
else:
print(f"{projectId}/{sessionId} not found in DB.")
return {"errors": ["Devtools file not found"]}
path = assist.get_raw_devtools_by_id(project_id=projectId, session_id=sessionId)
if path is None:
return {"errors": ["Devtools file not found"]}
return FileResponse(path=path, media_type="application/octet-stream")
@app.post('/{projectId}/heatmaps/url', tags=["heatmaps"], dependencies=[OR_scope(Permissions.session_replay)])
def get_heatmaps_by_url(projectId: int, data: schemas.GetHeatmapPayloadSchema = Body(...),
context: schemas.CurrentContext = Depends(OR_context)):
@ -313,10 +335,11 @@ def get_heatmaps_by_url(projectId: int, data: schemas.GetHeatmapPayloadSchema =
dependencies=[OR_scope(Permissions.session_replay)])
@app.get('/{projectId}/sessions2/{sessionId}/favorite', tags=["sessions"],
dependencies=[OR_scope(Permissions.session_replay)])
def add_remove_favorite_session2(projectId: int, sessionId: int, context: schemas.CurrentContext = Depends(OR_context)):
def add_remove_favorite_session2(projectId: int, sessionId: int,
context: schemas_ee.CurrentContext = Depends(OR_context)):
return {
"data": sessions_favorite.favorite_session(project_id=projectId, user_id=context.user_id,
session_id=sessionId)}
session_id=sessionId, context=context)}
@app.get('/{projectId}/sessions/{sessionId}/assign', tags=["sessions"],