jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #286 from openreplay/dev

Browse source 
v1.4.0: Improvements
This commit is contained in:
Mehdi Osman 2022-01-23 22:16:13 +01:00 • committed by GitHub
commit af3b6c950a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
55 changed files with 1337 additions and 814 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
LICENSE
View file

@ -1,6 +1,6 @@
MIT License
Copyright (c) 2021 Asayer SAS.
Copyright (c) 2022 Asayer SAS.
Portions of this software are licensed as follows:

13
api/chalicelib/core/alerts.py
View file

@ -1,11 +1,12 @@
import json
import logging
import time
import schemas
from chalicelib.core import notifications, slack, webhook
from chalicelib.utils import pg_client, helper, email_helper
from chalicelib.utils.TimeUTC import TimeUTC
import logging
def get(id):
with pg_client.PostgresClient() as cur:
@ -157,3 +158,13 @@ def delete(project_id, alert_id):
{"alert_id": alert_id, "project_id": project_id})
)
return {"data": {"state": "success"}}
def get_predefined_values():
values = [e.value for e in schemas.AlertColumn]
values = [{"name": v, "value": v,
"unit": "count" if v.endswith(".count") else "ms",
"predefined": True,
"metricId": None,
"seriesId": None} for v in values]
return values

33
api/chalicelib/core/custom_metrics.py
View file

@ -26,7 +26,9 @@ def try_live(project_id, data: schemas.TryCustomMetricsSchema):
def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema):
metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id)
metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False)
if metric is None:
return None
metric: schemas.TryCustomMetricsSchema = schemas.TryCustomMetricsSchema.parse_obj({**data.dict(), **metric})
return try_live(project_id=project_id, data=metric)
@ -113,10 +115,10 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche
u AS (UPDATE metric_series
SET name=series.name,
filter=series.filter,
index=series.filter.index
index=series.index
FROM (VALUES {",".join([f"(%(u_series_id_{s['i']})s,%(u_index_{s['i']})s,%(u_name_{s['i']})s,%(u_filter_{s['i']})s::jsonb)"
for s in n_series])}) AS series(series_id, index, name, filter)
WHERE metric_id =%(metric_id)s AND series_id=series.series_id
for s in u_series])}) AS series(series_id, index, name, filter)
WHERE metric_series.metric_id =%(metric_id)s AND metric_series.series_id=series.series_id
RETURNING 1)""")
if len(d_series_ids) > 0:
sub_queries.append("""\
@ -133,7 +135,6 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche
cur.execute(
query
)
r = cur.fetchone()
return get(metric_id=metric_id, project_id=project_id, user_id=user_id)
@ -158,6 +159,8 @@ def get_all(project_id, user_id):
rows = cur.fetchall()
for r in rows:
r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"])
for s in r["series"]:
s["filter"] = helper.old_search_payload_to_flat(s["filter"])
rows = helper.list_to_camel_case(rows)
return rows
@ -177,7 +180,7 @@ def delete(project_id, metric_id, user_id):
return {"state": "success"}
def get(metric_id, project_id, user_id):
def get(metric_id, project_id, user_id, flatten=True):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify(
@ -197,7 +200,12 @@ def get(metric_id, project_id, user_id):
)
)
row = cur.fetchone()
if row is None:
return None
row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"])
if flatten:
for s in row["series"]:
s["filter"] = helper.old_search_payload_to_flat(s["filter"])
return helper.dict_to_camel_case(row)
@ -205,17 +213,18 @@ def get_series_for_alert(project_id, user_id):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify(
"""SELECT metric_id,
series_id,
metrics.name AS metric_name,
metric_series.name AS series_name,
index AS series_index
"""SELECT series_id AS value,
metrics.name || '.' || (COALESCE(metric_series.name, 'series ' || index)) || '.count' AS name,
'count' AS unit,
FALSE AS predefined,
metric_id,
series_id
FROM metric_series
INNER JOIN metrics USING (metric_id)
WHERE metrics.deleted_at ISNULL
AND metrics.project_id = %(project_id)s
AND (user_id = %(user_id)s OR is_public)
ORDER BY metric_name, series_index, series_name;""",
ORDER BY name;""",
{"project_id": project_id, "user_id": user_id}
)
)

18
api/chalicelib/core/funnels.py
View file

@ -99,9 +99,11 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date
# row["filter"]["events"] = filter_stages(row["filter"]["events"])
get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date,
end_date=end_date)
counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True)
counts = sessions.search2_pg(data=schemas.SessionsSearchPayloadSchema.parse_obj(row["filter"]),
project_id=project_id, user_id=None, count_only=True)
row["sessionsCount"] = counts["countSessions"]
row["usersCount"] = counts["countUsers"]
filter_clone = dict(row["filter"])
overview = significance.get_overview(filter_d=row["filter"], project_id=project_id)
row["stages"] = overview["stages"]
row.pop("filter")
@ -110,6 +112,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date
row["criticalIssuesCount"] = overview["criticalIssuesCount"]
row["missedConversions"] = 0 if len(row["stages"]) < 2 \
else row["stages"][0]["sessionsCount"] - row["stages"][-1]["sessionsCount"]
row["filter"] = helper.old_search_payload_to_flat(filter_clone)
return rows
@ -147,11 +150,12 @@ def delete(project_id, funnel_id, user_id):
def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None):
f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id)
f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False)
if f is None:
return {"errors": ["funnel not found"]}
get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date)
return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id)
return sessions.search2_pg(data=schemas.SessionsSearchPayloadSchema.parse_obj(f["filter"]), project_id=project_id,
user_id=user_id)
def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema):
@ -168,7 +172,7 @@ def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.Funnel
def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None):
f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id)
f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False)
if f is None:
return {"errors": ["funnel not found"]}
get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date)
@ -196,7 +200,7 @@ def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data):
def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None):
f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id)
f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False)
if f is None:
return {"errors": ["funnel not found"]}
get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date)
@ -224,7 +228,7 @@ def get_issues_on_the_fly(funnel_id, user_id, project_id, data):
last_stage=last_stage))}
def get(funnel_id, project_id, user_id):
def get(funnel_id, project_id, user_id, flatten=True):
with pg_client.PostgresClient() as cur:
cur.execute(
cur.mogrify(
@ -246,6 +250,8 @@ def get(funnel_id, project_id, user_id):
f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"])
# f["filter"]["events"] = filter_stages(stages=f["filter"]["events"])
if flatten:
f["filter"] = helper.old_search_payload_to_flat(f["filter"])
return f

5
api/chalicelib/core/saved_search.py
View file

@ -18,6 +18,7 @@ def create(project_id, user_id, data: schemas.SavedSearchSchema):
)
r = cur.fetchone()
r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"])
r["filter"] = helper.old_search_payload_to_flat(r["filter"])
r = helper.dict_to_camel_case(r)
return {"data": r}
@ -40,6 +41,7 @@ def update(search_id, project_id, user_id, data: schemas.SavedSearchSchema):
)
r = cur.fetchone()
r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"])
r["filter"] = helper.old_search_payload_to_flat(r["filter"])
r = helper.dict_to_camel_case(r)
# r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"])
return r
@ -74,6 +76,8 @@ def get_all(project_id, user_id, details=False):
rows = helper.list_to_camel_case(rows)
for row in rows:
row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"])
if details:
row["filter"] = helper.old_search_payload_to_flat(row["filter"])
return rows
@ -112,4 +116,5 @@ def get(search_id, project_id, user_id):
return None
f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"])
f["filter"] = helper.old_search_payload_to_flat(f["filter"])
return f

11
api/chalicelib/core/significance.py
View file

@ -2,7 +2,7 @@ __author__ = "AZNAUROV David"
__maintainer__ = "KRAIEM Taha Yassine"
import schemas
from chalicelib.core import events, sessions_metas, metadata, sessions
from chalicelib.core import events, metadata, sessions
from chalicelib.utils import dev
"""
@ -617,6 +617,15 @@ def get_overview(filter_d, project_id, first_stage=None, last_stage=None):
# The result of the multi-stage query
rows = get_stages_and_events(filter_d=filter_d, project_id=project_id)
if len(rows) == 0:
# PS: not sure what to return if rows are empty
output["stages"] = [{
"type": stages[0]["type"],
"value": stages[0]["value"],
"sessionsCount": None,
"dropPercentage": None,
"usersCount": None
}]
output['criticalIssuesCount'] = 0
return output
# Obtain the first part of the output
stages_list = get_stages(stages, rows)

11
api/chalicelib/utils/helper.py
View file

@ -366,3 +366,14 @@ def has_smtp():
def get_edition():
return "ee" if "ee" in config("ENTERPRISE_BUILD", default="").lower() else "foss"
def old_search_payload_to_flat(values):
# in case the old search body was passed
if values.get("events") is not None:
for v in values["events"]:
v["isEvent"] = True
for v in values.get("filters", []):
v["isEvent"] = False
values["filters"] = values.pop("events") + values.get("filters", [])
return values

2
api/entrypoint.sh
View file

@ -1,2 +1,2 @@
#!/bin/bash
uvicorn app:app --host 0.0.0.0
uvicorn app:app --host 0.0.0.0 --reload

13
api/routers/core.py
View file

@ -108,7 +108,7 @@ def events_search(projectId: int, q: str, type: Union[schemas.FilterType, schema
@app.post('/{projectId}/sessions/search2', tags=["sessions"])
def sessions_search2(projectId: int, data: schemas.SessionsSearchPayloadSchema = Body(...),
def sessions_search2(projectId: int, data: schemas.FlatSessionsSearchPayloadSchema = Body(...),
context: schemas.CurrentContext = Depends(OR_context)):
data = sessions.search2_pg(data, projectId, user_id=context.user_id)
return {'data': data}
@ -621,6 +621,12 @@ def get_all_alerts(projectId: int, context: schemas.CurrentContext = Depends(OR_
return {"data": alerts.get_all(projectId)}
@app.get('/{projectId}/alerts/triggers', tags=["alerts", "customMetrics"])
def get_alerts_triggers(projectId: int, context: schemas.CurrentContext = Depends(OR_context)):
return {"data": alerts.get_predefined_values() \
+ custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)}
@app.get('/{projectId}/alerts/{alertId}', tags=["alerts"])
def get_alert(projectId: int, alertId: int, context: schemas.CurrentContext = Depends(OR_context)):
return {"data": alerts.get(alertId)}
@ -1072,11 +1078,6 @@ def get_custom_metric_chart(projectId: int, data: schemas.CustomMetricChartPaylo
data=data)}
@app.get('/{projectId}/custom_metrics/series', tags=["customMetrics"])
def get_series_for_alert(projectId: int, context: schemas.CurrentContext = Depends(OR_context)):
return {"data": custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)}
@app.post('/{projectId}/custom_metrics', tags=["customMetrics"])
@app.put('/{projectId}/custom_metrics', tags=["customMetrics"])
def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...),

50
api/schemas.py
View file

@ -458,7 +458,15 @@ class IssueType(str, Enum):
js_exception = 'js_exception'
class _SessionSearchEventRaw(BaseModel):
class __MixedSearchFilter(BaseModel):
is_event: bool = Field(...)
class Config:
alias_generator = attribute_to_camel_case
class _SessionSearchEventRaw(__MixedSearchFilter):
is_event: bool = Field(True, const=True)
custom: Optional[List[Union[int, str]]] = Field(None, min_items=1)
customOperator: Optional[MathOperator] = Field(None)
key: Optional[str] = Field(None)
@ -491,7 +499,8 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw):
value: Union[List[_SessionSearchEventRaw], str, List[str]] = Field(...)
class _SessionSearchFilterSchema(BaseModel):
class _SessionSearchFilterSchema(__MixedSearchFilter):
is_event: bool = Field(False, const=False)
custom: Optional[List[str]] = Field(None)
key: Optional[str] = Field(None)
value: Union[Optional[Union[IssueType, PlatformType, int, str]],
@ -536,12 +545,39 @@ class SessionsSearchPayloadSchema(BaseModel):
alias_generator = attribute_to_camel_case
class SessionsSearchCountSchema(SessionsSearchPayloadSchema):
class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema):
events: Optional[List[_SessionSearchEventSchema]] = Field([])
filters: List[Union[_SessionSearchFilterSchema, _SessionSearchEventSchema]] = Field([])
@root_validator(pre=True)
def flat_to_original(cls, values):
# in case the old search body was passed
if len(values.get("events", [])) > 0:
for v in values["events"]:
v["isEvent"] = True
for v in values.get("filters", []):
v["isEvent"] = False
else:
n_filters = []
n_events = []
for v in values.get("filters", []):
if v["isEvent"]:
n_events.append(v)
else:
n_filters.append(v)
values["events"] = n_events
values["filters"] = n_filters
return values
class SessionsSearchCountSchema(FlatSessionsSearchPayloadSchema):
# class SessionsSearchCountSchema(SessionsSearchPayloadSchema):
sort: Optional[str] = Field(default=None)
order: Optional[str] = Field(default=None)
class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema):
class FunnelSearchPayloadSchema(FlatSessionsSearchPayloadSchema):
# class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema):
range_value: Optional[str] = Field(None)
sort: Optional[str] = Field(None)
order: Optional[str] = Field(None)
@ -565,7 +601,8 @@ class UpdateFunnelSchema(FunnelSchema):
is_public: Optional[bool] = Field(None)
class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema):
class FunnelInsightsPayloadSchema(FlatSessionsSearchPayloadSchema):
# class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema):
sort: Optional[str] = Field(None)
order: Optional[str] = Field(None)
@ -595,7 +632,8 @@ class MobileSignPayloadSchema(BaseModel):
keys: List[str] = Field(...)
class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema):
class CustomMetricSeriesFilterSchema(FlatSessionsSearchPayloadSchema):
# class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema):
startDate: Optional[int] = Field(None)
endDate: Optional[int] = Field(None)
sort: Optional[str] = Field(None)

2
backend/pkg/db/cache/messages_common.go vendored
View file

@ -38,7 +38,7 @@ func (c *PGCache) InsertUserID(sessionID uint64, userID *IOSUserID) error {
if err != nil {
return err
}
session.UserID = &userID.Value
session.UserID = userID.Value
return nil
}

1
backend/pkg/db/cache/messages_web.go vendored
View file

@ -30,6 +30,7 @@ func (c *PGCache) InsertWebSessionStart(sessionID uint64, s *SessionStart) error
UserDeviceType: s.UserDeviceType,
UserDeviceMemorySize: s.UserDeviceMemorySize,
UserDeviceHeapSize: s.UserDeviceHeapSize,
UserID: s.UserID,
}
if err := c.Conn.InsertSessionStart(sessionID, c.sessions[ sessionID ]); err != nil {
c.sessions[ sessionID ] = nil

7
backend/pkg/db/postgres/messages_common.go
View file

@ -47,7 +47,8 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error {
rev_id,
tracker_version, issue_score,
platform,
user_agent, user_browser, user_browser_version, user_device_memory_size, user_device_heap_size
user_agent, user_browser, user_browser_version, user_device_memory_size, user_device_heap_size,
user_id
) VALUES (
$1, $2, $3,
$4, $5, $6, $7,
@ -55,7 +56,8 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error {
NULLIF($10, ''),
$11, $12,
$13,
NULLIF($14, ''), NULLIF($15, ''), NULLIF($16, ''), NULLIF($17, 0), NULLIF($18, 0::bigint)
NULLIF($14, ''), NULLIF($15, ''), NULLIF($16, ''), NULLIF($17, 0), NULLIF($18, 0::bigint),
NULLIF($19, '')
)`,
sessionID, s.ProjectID, s.Timestamp,
s.UserUUID, s.UserDevice, s.UserDeviceType, s.UserCountry,
@ -64,6 +66,7 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error {
s.TrackerVersion, s.Timestamp/1000,
s.Platform,
s.UserAgent, s.UserBrowser, s.UserBrowserVersion, s.UserDeviceMemorySize, s.UserDeviceHeapSize,
s.UserID,
); err != nil {
return err;
}

2
backend/pkg/db/types/session.go
View file

@ -16,7 +16,7 @@ type Session struct {
PagesCount int
EventsCount int
ErrorsCount int
UserID *string
UserID string // pointer??
UserAnonymousID *string
Metadata1 *string
Metadata2 *string

4
backend/pkg/messages/messages.go
View file

@ -63,9 +63,10 @@ UserDeviceType string
UserDeviceMemorySize uint64
UserDeviceHeapSize uint64
UserCountry string
UserID string
}
func (msg *SessionStart) Encode() []byte{
buf := make([]byte, 151 + len(msg.TrackerVersion)+ len(msg.RevID)+ len(msg.UserUUID)+ len(msg.UserAgent)+ len(msg.UserOS)+ len(msg.UserOSVersion)+ len(msg.UserBrowser)+ len(msg.UserBrowserVersion)+ len(msg.UserDevice)+ len(msg.UserDeviceType)+ len(msg.UserCountry))
buf := make([]byte, 161 + len(msg.TrackerVersion)+ len(msg.RevID)+ len(msg.UserUUID)+ len(msg.UserAgent)+ len(msg.UserOS)+ len(msg.UserOSVersion)+ len(msg.UserBrowser)+ len(msg.UserBrowserVersion)+ len(msg.UserDevice)+ len(msg.UserDeviceType)+ len(msg.UserCountry)+ len(msg.UserID))
buf[0] = 1
p := 1
p = WriteUint(msg.Timestamp, buf, p)
@ -83,6 +84,7 @@ p = WriteString(msg.UserDeviceType, buf, p)
p = WriteUint(msg.UserDeviceMemorySize, buf, p)
p = WriteUint(msg.UserDeviceHeapSize, buf, p)
p = WriteString(msg.UserCountry, buf, p)
p = WriteString(msg.UserID, buf, p)
return buf[:p]
}

1
backend/pkg/messages/read_message.go
View file

@ -42,6 +42,7 @@ if msg.UserDeviceType, err = ReadString(reader); err != nil { return nil, err }
if msg.UserDeviceMemorySize, err = ReadUint(reader); err != nil { return nil, err }
if msg.UserDeviceHeapSize, err = ReadUint(reader); err != nil { return nil, err }
if msg.UserCountry, err = ReadString(reader); err != nil { return nil, err }
if msg.UserID, err = ReadString(reader); err != nil { return nil, err }
return msg, nil
case 2:

2
backend/services/http/handlers_web.go
View file

@ -27,6 +27,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) {
JsHeapSizeLimit uint64 `json:"jsHeapSizeLimit"`
ProjectKey *string `json:"projectKey"`
Reset bool `json:"reset"`
UserID string `json:"userID"`
}
type response struct {
Timestamp int64 `json:"timestamp"`
@ -101,6 +102,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) {
UserCountry: country,
UserDeviceMemorySize: req.DeviceMemory,
UserDeviceHeapSize: req.JsHeapSizeLimit,
UserID: req.UserID,
}))
}

36
ee/LICENSE.md
View file

@ -1,36 +1,4 @@
The OpenReplay Enterprise license (the “Enterprise License”)
Copyright (c) 2021 Asayer SAS.
Copyright (c) 2022 Asayer SAS.
With regard to the OpenReplay Software:
This software and associated documentation files (the "Software") may only be
used in production, if you (and any entity that you represent) have agreed to,
and are in compliance with, the OpenReplay Subscription Terms of Service, available
at https://openreplay.com/terms.html (the “Enterprise Edition”), or other
agreement governing the use of the Software, as agreed by you and OpenReplay,
and otherwise have a valid OpenReplay Enterprise license for the
correct usage. Subject to the foregoing sentence, you are free to
modify this Software and publish patches to the Software. You agree that OpenReplay
and/or its licensors (as applicable) retain all right, title and interest in and
to all such modifications and/or patches, and all such modifications and/or
patches may only be used, copied, modified, displayed, distributed, or otherwise
exploited with a valid OpenReplay Enterprise license for the correct
number of user seats and profiles. Notwithstanding the foregoing, you may copy and modify
the Software for development and testing purposes, without requiring a
subscription. You agree that OpenReplay and/or its licensors (as applicable) retain
all right, title and interest in and to all such modifications. You are not
granted any other rights beyond what is expressly stated herein. Subject to the
foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
and/or sell the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
For all third party components incorporated into the OpenReplay Software, those
components are licensed under the original license provided by the owner of the
applicable component.
To license the Enterprise Edition of OpenReplay, and take advantage of its additional features, functionality and support, you must agree to the terms of the OpenReplay Enterprise License Agreement. Please contact OpenReplay at [sales@openreplay.com](mailto:sales@openreplay.com).

2
ee/api/.gitignore vendored
View file

@ -253,7 +253,7 @@ Pipfile
/db_changes.sql
/Dockerfile.bundle
/entrypoint.bundle.sh
/entrypoint.sh
#/entrypoint.sh
/chalicelib/core/heatmaps.py
/routers/subs/insights.py
/schemas.py

29
ee/api/chalicelib/utils/SAML2_helper.py
View file

@ -12,11 +12,11 @@ SAML2 = {
"sp": {
"entityId": config("SITE_URL") + "/api/sso/saml2/metadata/",
"assertionConsumerService": {
"url": config("SITE_URL") + "/api/sso/saml2/acs",
"url": config("SITE_URL") + "/api/sso/saml2/acs/",
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
},
"singleLogoutService": {
"url": config("SITE_URL") + "/api/sso/saml2/sls",
"url": config("SITE_URL") + "/api/sso/saml2/sls/",
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
},
"NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress",
@ -25,6 +25,12 @@ SAML2 = {
},
"idp": None
}
# in case tenantKey is included in the URL
sp_acs = config("idp_tenantKey", default="")
if sp_acs is not None and len(sp_acs) > 0:
SAML2["sp"]["assertionConsumerService"]["url"] += sp_acs + "/"
idp = None
# SAML2 config handler
if config("SAML2_MD_URL", default=None) is not None and len(config("SAML2_MD_URL")) > 0:
@ -60,12 +66,9 @@ else:
def init_saml_auth(req):
# auth = OneLogin_Saml2_Auth(req, custom_base_path=environ['SAML_PATH'])
if idp is None:
raise Exception("No SAML2 config provided")
auth = OneLogin_Saml2_Auth(req, old_settings=SAML2)
return auth
return OneLogin_Saml2_Auth(req, old_settings=SAML2)
async def prepare_request(request: Request):
@ -86,12 +89,20 @@ async def prepare_request(request: Request):
session = {}
# If server is behind proxys or balancers use the HTTP_X_FORWARDED fields
headers = request.headers
url_data = urlparse('%s://%s' % (headers.get('x-forwarded-proto', 'http'), headers['host']))
proto = headers.get('x-forwarded-proto', 'http')
if headers.get('x-forwarded-proto') is not None:
print(f"x-forwarded-proto: {proto}")
url_data = urlparse('%s://%s' % (proto, headers['host']))
path = request.url.path
# add / to /acs
if not path.endswith("/"):
path = path + '/'
return {
'https': 'on' if request.headers.get('x-forwarded-proto', 'http') == 'https' else 'off',
'https': 'on' if proto == 'https' else 'off',
'http_host': request.headers['host'],
'server_port': url_data.port,
'script_name': "/api" + request.url.path,
'script_name': "/api" + path,
'get_data': request.args.copy(),
# Uncomment if using ADFS as IdP, https://github.com/onelogin/python-saml/pull/144
# 'lowercase_urlencoding': True,

2
ee/api/entrypoint.sh Executable file
View file

@ -0,0 +1,2 @@
#!/bin/bash
uvicorn app:app --host 0.0.0.0 --reload

89
ee/api/routers/saml.py
View file

@ -16,6 +16,7 @@ from starlette import status
@public_app.get("/sso/saml2", tags=["saml2"])
@public_app.get("/sso/saml2/", tags=["saml2"])
async def start_sso(request: Request):
request.path = ''
req = await prepare_request(request=request)
@ -25,6 +26,7 @@ async def start_sso(request: Request):
@public_app.post('/sso/saml2/acs', tags=["saml2"])
@public_app.post('/sso/saml2/acs/', tags=["saml2"])
async def process_sso_assertion(request: Request):
req = await prepare_request(request=request)
session = req["cookie"]["session"]
@ -43,6 +45,8 @@ async def process_sso_assertion(request: Request):
user_data = auth.get_attributes()
elif auth.get_settings().is_debug_active():
error_reason = auth.get_last_error_reason()
print("SAML2 error:")
print(error_reason)
return {"errors": [error_reason]}
email = auth.get_nameid()
@ -108,7 +112,91 @@ async def process_sso_assertion(request: Request):
headers={'Location': SAML2_helper.get_landing_URL(jwt)})
@public_app.post('/sso/saml2/acs/{tenantKey}', tags=["saml2"])
@public_app.post('/sso/saml2/acs/{tenantKey}/', tags=["saml2"])
async def process_sso_assertion_tk(tenantKey: str, request: Request):
req = await prepare_request(request=request)
session = req["cookie"]["session"]
auth = init_saml_auth(req)
request_id = None
if 'AuthNRequestID' in session:
request_id = session['AuthNRequestID']
auth.process_response(request_id=request_id)
errors = auth.get_errors()
user_data = {}
if len(errors) == 0:
if 'AuthNRequestID' in session:
del session['AuthNRequestID']
user_data = auth.get_attributes()
elif auth.get_settings().is_debug_active():
error_reason = auth.get_last_error_reason()
print("SAML2 error:")
print(error_reason)
return {"errors": [error_reason]}
email = auth.get_nameid()
print("received nameId:")
print(email)
existing = users.get_by_email_only(auth.get_nameid())
internal_id = next(iter(user_data.get("internalId", [])), None)
t = tenants.get_by_tenant_key(tenantKey)
if t is None:
print("invalid tenantKey, please copy the correct value from Preferences > Account")
return {"errors": ["invalid tenantKey, please copy the correct value from Preferences > Account"]}
print(user_data)
role_name = user_data.get("role", [])
if len(role_name) == 0:
print("No role specified, setting role to member")
role_name = ["member"]
role_name = role_name[0]
role = roles.get_role_by_name(tenant_id=t['tenantId'], name=role_name)
if role is None:
return {"errors": [f"role {role_name} not found, please create it in openreplay first"]}
admin_privileges = user_data.get("adminPrivileges", [])
admin_privileges = not (len(admin_privileges) == 0
or admin_privileges[0] is None
or admin_privileges[0].lower() == "false")
if existing is None:
deleted = users.get_deleted_user_by_email(auth.get_nameid())
if deleted is not None:
print("== restore deleted user ==")
users.restore_sso_user(user_id=deleted["userId"], tenant_id=t['tenantId'], email=email,
admin=admin_privileges, origin=SAML2_helper.get_saml2_provider(),
name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])),
internal_id=internal_id, role_id=role["roleId"])
else:
print("== new user ==")
users.create_sso_user(tenant_id=t['tenantId'], email=email, admin=admin_privileges,
origin=SAML2_helper.get_saml2_provider(),
name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])),
internal_id=internal_id, role_id=role["roleId"])
else:
if t['tenantId'] != existing["tenantId"]:
print("user exists for a different tenant")
return {"errors": ["user exists for a different tenant"]}
if existing.get("origin") is None:
print(f"== migrating user to {SAML2_helper.get_saml2_provider()} ==")
users.update(tenant_id=t['tenantId'], user_id=existing["id"],
changes={"origin": SAML2_helper.get_saml2_provider(), "internal_id": internal_id})
expiration = auth.get_session_expiration()
expiration = expiration if expiration is not None and expiration > 10 * 60 \
else int(config("sso_exp_delta_seconds", cast=int, default=24 * 60 * 60))
jwt = users.authenticate_sso(email=email, internal_id=internal_id, exp=expiration)
if jwt is None:
return {"errors": ["null JWT"]}
return Response(
status_code=status.HTTP_302_FOUND,
headers={'Location': SAML2_helper.get_landing_URL(jwt)})
@public_app.get('/sso/saml2/sls', tags=["saml2"])
@public_app.get('/sso/saml2/sls/', tags=["saml2"])
async def process_sls_assertion(request: Request):
req = await prepare_request(request=request)
session = req["cookie"]["session"]
@ -144,6 +232,7 @@ async def process_sls_assertion(request: Request):
@public_app.get('/sso/saml2/metadata', tags=["saml2"])
@public_app.get('/sso/saml2/metadata/', tags=["saml2"])
async def saml2_metadata(request: Request):
req = await prepare_request(request=request)
auth = init_saml_auth(req)

1
scripts/helm/app/chalice.yaml
View file

@ -64,5 +64,6 @@ env:
idp_x509cert: ''
idp_sls_url: ''
idp_name: ''
idp_tenantKey: ''
assist_secret: ''
iceServers: ''

4
scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml
View file

@ -69,9 +69,9 @@ spec:
- name: sessions_region
value: '{{ .Values.global.s3.region }}'
- name: sessions_bucket
value: '{{ .Values.global.s3.recordings_bucket }}'
value: '{{ .Values.global.s3.recordingsBucket }}'
- name: sourcemaps_bucket
value: '{{ .Values.global.s3.sourcemaps_bucket }}'
value: '{{ .Values.global.s3.sourcemapsBucket }}'
- name: js_cache_bucket
value: '{{ .Values.global.s3.assetsBucket }}'
- name: EMAIL_HOST

1
scripts/helmcharts/openreplay/charts/chalice/values.yaml
View file

@ -94,6 +94,7 @@ env:
idp_x509cert: ''
idp_sls_url: ''
idp_name: ''
idp_tenantKey: ''
assist_secret: ''
iceServers: ''

1
scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt Symbolic link
View file

@ -0,0 +1 @@
../../../files/site.crt

1
scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key Symbolic link
View file

@ -0,0 +1 @@
../../../files/site.key

6
scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml
View file

@ -4,6 +4,6 @@ kind: Secret
metadata:
name: ssl
data:
ca.crt: ""
site.crt: "{{ .Values.ssl_certificate | b64enc }}"
site.key: "{{ .Values.ssl_privatekey | b64enc }}"
ca.crt: ''
site.crt: '{{ .Files.Get "files/site.crt" | b64enc }}'
site.key: '{{ .Files.Get "files/site.key" | b64enc }}'

87
scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml
View file

@ -84,90 +84,3 @@ nodeSelector: {}
tolerations: []
affinity: {}
ssl_certificate: |-
-----BEGIN CERTIFICATE-----
MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL
BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3
NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh
bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+
LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w
hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds
QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX
sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o
immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l
fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ
ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ
OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u
tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p
CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA
YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2
z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs
MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z
qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA
dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p
0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6
5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh
0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s
/JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP
XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc
HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG
T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi
qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S
w7yMPvE+TIh6+HwDZc2jNLkv/8tY
-----END CERTIFICATE-----
ssl_privatekey: |-
-----BEGIN PRIVATE KEY-----
MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t
K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE
BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB
OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew
6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK
aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9
hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK
lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6
VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62
RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK
Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh
QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe
K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s
dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2
CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh
ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl
5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw
TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk
x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c
Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g
DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj
OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm
5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p
/CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA
0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6
tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E
ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy
HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4
IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA
PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q
p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO
GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw
1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ
cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d
vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ
n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+
Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x
OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9
Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP
kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB
De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr
TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T
e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP
3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9
WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa
8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb
esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn
PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6
PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5
7qINKoGovqXB1oAdopIl1z64e7MWVE4=
-----END PRIVATE KEY-----

4
scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml
View file

@ -49,9 +49,9 @@ spec:
- name: AWS_REGION_IOS
value: '{{ .Values.global.s3.region }}'
- name: S3_BUCKET_WEB
value: '{{ .Values.global.s3.recordings_bucket }}'
value: '{{ .Values.global.s3.recordingsBucket }}'
- name: S3_BUCKET_IOS
value: '{{ .Values.global.s3.recordings_bucket }}'
value: '{{ .Values.global.s3.recordingsBucket }}'
- name: REDIS_STRING
value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}'
- name: LICENSE_KEY

30
scripts/helmcharts/openreplay/files/site.crt Normal file
View file

@ -0,0 +1,30 @@
-----BEGIN CERTIFICATE-----
MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL
BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3
NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh
bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+
LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w
hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds
QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX
sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o
immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l
fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ
ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ
OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u
tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p
CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA
YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2
z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs
MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z
qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA
dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p
0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6
5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh
0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s
/JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP
XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc
HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG
T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi
qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S
w7yMPvE+TIh6+HwDZc2jNLkv/8tY
-----END CERTIFICATE-----

52
scripts/helmcharts/openreplay/files/site.key Normal file
View file

@ -0,0 +1,52 @@
-----BEGIN PRIVATE KEY-----
MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t
K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE
BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB
OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew
6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK
aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9
hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK
lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6
VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62
RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK
Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh
QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe
K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s
dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2
CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh
ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl
5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw
TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk
x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c
Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g
DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj
OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm
5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p
/CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA
0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6
tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E
ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy
HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4
IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA
PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q
p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO
GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw
1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ
cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d
vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ
n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+
Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x
OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9
Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP
kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB
De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr
TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T
e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP
3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9
WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa
8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb
esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn
PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6
PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5
7qINKoGovqXB1oAdopIl1z64e7MWVE4=
-----END PRIVATE KEY-----

21
scripts/helmcharts/vars.yaml
View file

@ -8,6 +8,13 @@ postgresql: &postgres
postgresqlPort: "5432"
postgresqlUser: "postgres"
postgresqlDatabase: "postgres"
# resources:
# requests:
# memory: 256Mi
# cpu: 250m
# limits:
# memory: 3000Mi
# cpu: 2
clickhouse: {}
# For enterpriseEdition
@ -43,8 +50,8 @@ global:
region: "us-east-1"
endpoint: "http://minio.db.svc.cluster.local:9000"
assetsBucket: "sessions-assets"
recordings_bucket: "mobs"
sourcemaps_bucket: "sourcemaps"
recordingsBucket: "mobs"
sourcemapsBucket: "sourcemaps"
# if you're using one node installation, where
# you're using local s3, make sure these variables
# are same as minio.global.minio.accesskey and secretKey
@ -75,15 +82,14 @@ chalice:
# idp_x509cert: ''
# idp_sls_url: ''
# idp_name: ''
# idp_tenantKey: ''
# If you want to override something
# chartname:
# filedFrom chart/Values.yaml:
# key: value
# For example:
#
# For example (http):
# http:
# resources:
# limits:
@ -96,7 +102,10 @@ chalice:
## Changes to nginx
#
# nginx-ingress:
# # Key and certificate files must be named site.key and site.crt
# # and copied to ../openreplay/files/
# sslKey: site.key
# sslCert: site.crt
# customServerConfigs: |
# # Redirecting http to https
# return 301 https://$host$request_uri;
#

111
scripts/helmcharts/vars_template.yaml Normal file
View file

@ -0,0 +1,111 @@
fromVersion: "{{ openreplay_version }}"
# Databases specific variables
postgresql: &postgres
# For generating passwords
# `openssl rand -hex 20`
postgresqlPassword: "{{ postgres_db_password }}"
postgresqlHost: "{{ postgres_endpoint }}"
postgresqlPort: "5432"
postgresqlUser: "{{ postgres_db_user }}"
postgresqlDatabase: "{{ postgres_db_name }}"
# resources:
# requests:
# memory: 256Mi
# cpu: 250m
# limits:
# memory: 3000Mi
# cpu: 2
clickhouse: {}
# For enterpriseEdition
# enabled: true
kafka: &kafka
# For enterpriseEdition
# enabled: true
kafkaHost: "{{ kafka_endpoint }}"
kafkaPort: "{{ kafka_endpoint.split(':')[-1] }}"
kafkaUseSsl: "{{ kafka_ssl }}"
redis: &redis
# For enterpriseEdition
# enabled: false
redisHost: "{{ redis_endpoint }}"
redisPort: "{{ redis_endpoint.split(':')[-1] }}"
minio:
global:
minio:
# For generating passwords
# `openssl rand -hex 20`
accessKey: "{{ minio_access_key }}"
secretKey: "{{ minio_secret_key }}"
# Application specific variables
global:
postgresql: *postgres
kafka: *kafka
redis: *redis
s3:
region: "us-east-1"
endpoint: "http://minio.db.svc.cluster.local:9000"
assetsBucket: "sessions-assets"
recordingsBucket: "mobs"
sourcemapsBucket: "sourcemaps"
# if you're using one node installation, where
# you're using local s3, make sure these variables
# are same as minio.global.minio.accesskey and secretKey
accessKey: "{{ minio_access_key }}"
secretKey: "{{ minio_secret_key }}"
email:
emailHost: '{{ email_host }}'
emailPort: '{{ email_port }}'
emailUser: '{{ email_user }}'
emailPassword: '{{ email_password }}'
emailUseTls: '{{ email_use_tls }}'
emailUseSsl: '{{ email_use_ssl }}'
emailSslKey: '{{ email_ssl_key }}'
emailSslCert: '{{ email_ssl_cert }}'
emailFrom: '{{ email_from }}'
enterpriseEditionLicense: "{{ enterprise_edition_license }}"
domainName: "{{ domain_name }}"
chalice:
env:
jwt_secret: "{{ jwt_secret_key }}"
# captcha_server: ''
# captcha_key: ''
# SAML2_MD_URL: ''
# idp_entityId: ''
# idp_sso_url: ''
# idp_x509cert: ''
# idp_sls_url: ''
# idp_name: ''
# idp_tenantKey: ''
# If you want to override something
# chartname:
# filedFrom chart/Values.yaml:
# key: value
# For example:
#
# http:
# resources:
# limits:
# cpu: 1024m
# memory: 4096Mi
# requests:
# cpu: 512m
# memory: 2056Mi
## Changes to nginx
#
# nginx-ingress:
# customServerConfigs: |
# # Redirecting http to https
# return 301 https://$host$request_uri;
#

5
third-party.md
View file

@ -1,4 +1,4 @@
## Licenses (as of January 16, 2022)
## Licenses (as of January 21, 2022)
Below is the list of dependencies used in OpenReplay software. Licenses may change between versions, so please keep this up to date with every new library you use.
@ -84,10 +84,9 @@ Below is the list of dependencies used in OpenReplay software. Licenses may chan
| redux-immutable | BSD3 | JavaScript |
| redux-thunk | MIT | JavaScript |
| semantic-ui-react | MIT | JavaScript |
| socket.io-client | MIT | JavaScript |
| socketio | MIT | JavaScript |
| source-map | BSD3 | JavaScript |
| aws-sdk | Apache2 | JavaScript |
| serverless | MIT | JavaScript |
| lib/pq | MIT | Go |
| peerjs | MIT | JavaScript |
| antonmedv/finder | MIT | JavaScript |

2
tracker/tracker-assist/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@openreplay/tracker-assist",
"description": "Tracker plugin for screen assistance through the WebRTC",
"version": "3.4.13",
"version": "3.4.16",
"keywords": [
"WebRTC",
"assistance",

6
tracker/tracker-assist/src/BufferingConnection.ts
View file

@ -6,12 +6,13 @@ interface Message {
}
// 16kb should be max according to specification
// 64kb chrome
const crOrFf: boolean =
typeof navigator !== "undefined" &&
(navigator.userAgent.indexOf("Chrom") !== -1 || // Chrome && Chromium
navigator.userAgent.indexOf("Firefox") !== -1);
const MESSAGES_PER_SEND = crOrFf ? 500 : 100
const MESSAGES_PER_SEND = crOrFf ? 200 : 50
// Bffering required in case of webRTC
export default class BufferingConnection {
@ -34,7 +35,10 @@ export default class BufferingConnection {
send(messages: Message[]) {
if (!this.conn.open) { return; }
let i = 0;
//@ts-ignore
messages=messages.filter(m => m._id !== 39)
while (i < messages.length) {
this.buffer.push(messages.slice(i, i+=this.msgsPerSend))
}
if (!this.buffering) {

5
tracker/tracker-assist/src/Mouse.ts
View file

@ -47,11 +47,12 @@ export default class Mouse {
}
}
private readonly pScrEl = document.scrollingElement || document.documentElement
private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct
private lastScrEl: Element | "window" | null = null
private resetLastScrEl = () => { this.lastScrEl = null }
private handleWScroll = e => {
if (e.target !== this.lastScrEl) {
if (e.target !== this.lastScrEl &&
this.lastScrEl !== "window") {
this.resetLastScrEl()
}
}

4
tracker/tracker-assist/src/index.ts
View file

@ -27,7 +27,7 @@ enum CallingState {
};
//@ts-ignore peerjs hack for webpack5 (?!) TODO: ES/node modules;
//Peer = Peer.default || Peer;
Peer = Peer.default || Peer;
// type IncomeMessages =
// "call_end" |
@ -86,7 +86,7 @@ export default function(opts?: Partial<Options>) {
host: app.getHost(),
path: '/assist',
port: location.protocol === 'http:' && appOptions.__DISABLE_SECURE_MODE ? 80 : 443,
//debug: // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs.
debug: appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs.
}
if (options.config) {
_opt['config'] = options.config

2
tracker/tracker/package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "@openreplay/tracker",
"version": "3.4.7",
"version": "3.4.12",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
tracker/tracker/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@openreplay/tracker",
"description": "The OpenReplay tracker main package",
"version": "3.4.12",
"version": "3.4.17",
"keywords": [
"logging",
"replay"

72
tracker/tracker/src/main/app/context.ts Normal file
View file

@ -0,0 +1,72 @@
// TODO: global type
export interface Window extends globalThis.Window {
HTMLInputElement: typeof HTMLInputElement,
HTMLLinkElement: typeof HTMLLinkElement,
HTMLStyleElement: typeof HTMLStyleElement,
SVGStyleElement: typeof SVGStyleElement,
HTMLIFrameElement: typeof HTMLIFrameElement,
Text: typeof Text,
Element: typeof Element,
ShadowRoot: typeof ShadowRoot,
//parent: Window,
}
type WindowConstructor =
Document |
Element |
Text |
ShadowRoot |
HTMLInputElement |
HTMLLinkElement |
HTMLStyleElement |
HTMLIFrameElement
// type ConstructorNames =
// 'Element' |
// 'Text' |
// 'HTMLInputElement' |
// 'HTMLLinkElement' |
// 'HTMLStyleElement' |
// 'HTMLIFrameElement'
type Constructor<T> = { new (...args: any[]): T , name: string };
// TODO: we need a type expert here so we won't have to ignore the lines
// TODO: use it everywhere (static function; export from which file? <-- global Window typing required)
export function isInstance<T extends WindowConstructor>(node: Node, constr: Constructor<T>): node is T {
const doc = node.ownerDocument;
if (!doc) { // null if Document
return constr.name === 'Document';
}
let context: Window =
// @ts-ignore (for EI, Safary)
doc.parentWindow ||
doc.defaultView; // TODO: smart global typing for Window object
while(context.parent && context.parent !== context) {
// @ts-ignore
if (node instanceof context[constr.name]) {
return true
}
// @ts-ignore
context = context.parent
}
// @ts-ignore
return node instanceof context[constr.name]
}
export function inDocument(node: Node): boolean {
const doc = node.ownerDocument
if (!doc) { return false }
if (doc.contains(node)) { return true }
let context: Window =
// @ts-ignore (for EI, Safary)
doc.parentWindow ||
doc.defaultView;
while(context.parent && context.parent !== context) {
if (context.document.contains(node)) {
return true
}
// @ts-ignore
context = context.parent
}
return false;
}

246
tracker/tracker/src/main/app/index.ts
View file

@ -2,12 +2,15 @@ import { timestamp, log, warn } from "../utils.js";
import { Timestamp, PageClose } from "../../messages/index.js";
import Message from "../../messages/message.js";
import Nodes from "./nodes.js";
import Observer from "./observer.js";
import Observer from "./observer/top_observer.js";
import Sanitizer from "./sanitizer.js";
import Ticker from "./ticker.js";
import { deviceMemory, jsHeapSizeLimit } from "../modules/performance.js";
import type { Options as ObserverOptions } from "./observer.js";
import type { Options as ObserverOptions } from "./observer/top_observer.js";
import type { Options as SanitizerOptions } from "./sanitizer.js";
import type { Options as WebworkerOptions, WorkerMessageData } from "../../messages/webworker.js";
@ -17,11 +20,17 @@ export interface OnStartInfo {
userUUID: string,
}
export type Options = {
export interface StartOptions {
userID?: string,
forceNew: boolean,
}
type AppOptions = {
revID: string;
node_id: string;
session_token_key: string;
session_pageno_key: string;
session_reset_key: string;
local_uuid_key: string;
ingestPoint: string;
resourceBaseHref: string | null, // resourceHref?
@ -30,7 +39,9 @@ export type Options = {
__debug_report_edp: string | null;
__debug_log: boolean;
onStart?: (info: OnStartInfo) => void;
} & ObserverOptions & WebworkerOptions;
} & WebworkerOptions;
export type Options = AppOptions & ObserverOptions & SanitizerOptions
type Callback = () => void;
type CommitCallback = (messages: Array<Message>) => void;
@ -43,21 +54,23 @@ export default class App {
readonly nodes: Nodes;
readonly ticker: Ticker;
readonly projectKey: string;
readonly sanitizer: Sanitizer;
private readonly messages: Array<Message> = [];
/*private*/ readonly observer: Observer; // temp, for fast security fix. TODO: separate security/obscure module with nodeCallback that incapsulates `textMasked` functionality from Observer
private readonly observer: Observer;
private readonly startCallbacks: Array<Callback> = [];
private readonly stopCallbacks: Array<Callback> = [];
private readonly commitCallbacks: Array<CommitCallback> = [];
private readonly options: Options;
private readonly options: AppOptions;
private readonly revID: string;
private _sessionID: string | null = null;
private _userID: string | undefined;
private isActive = false;
private version = 'TRACKER_VERSION';
private readonly worker?: Worker;
constructor(
projectKey: string,
sessionToken: string | null | undefined,
opts: Partial<Options>,
options: Partial<Options>,
) {
this.projectKey = projectKey;
this.options = Object.assign(
@ -66,24 +79,23 @@ export default class App {
node_id: '__openreplay_id',
session_token_key: '__openreplay_token',
session_pageno_key: '__openreplay_pageno',
session_reset_key: '__openreplay_reset',
local_uuid_key: '__openreplay_uuid',
ingestPoint: DEFAULT_INGEST_POINT,
resourceBaseHref: null,
__is_snippet: false,
__debug_report_edp: null,
__debug_log: false,
obscureTextEmails: true,
obscureTextNumbers: false,
captureIFrames: false,
},
opts,
options,
);
if (sessionToken != null) {
sessionStorage.setItem(this.options.session_token_key, sessionToken);
}
this.revID = this.options.revID;
this.sanitizer = new Sanitizer(this, options);
this.nodes = new Nodes(this.options.node_id);
this.observer = new Observer(this, this.options);
this.observer = new Observer(this, options);
this.ticker = new Ticker(this);
this.ticker.attach(() => this.commit());
try {
@ -102,7 +114,10 @@ export default class App {
this.stop();
} else if (data === "restart") {
this.stop();
this.start(true);
this.start({
forceNew: true,
userID: this._userID,
});
}
};
const alertWorker = () => {
@ -244,104 +259,132 @@ export default class App {
active(): boolean {
return this.isActive;
}
private _start(reset: boolean): Promise<OnStartInfo> {
if (!this.isActive) {
if (!this.worker) {
return Promise.reject("No worker found: perhaps, CSP is not set.");
}
this.isActive = true;
let pageNo: number = 0;
const pageNoStr = sessionStorage.getItem(this.options.session_pageno_key);
if (pageNoStr != null) {
pageNo = parseInt(pageNoStr);
pageNo++;
}
sessionStorage.setItem(this.options.session_pageno_key, pageNo.toString());
const startTimestamp = timestamp();
const messageData: WorkerMessageData = {
ingestPoint: this.options.ingestPoint,
pageNo,
startTimestamp,
connAttemptCount: this.options.connAttemptCount,
connAttemptGap: this.options.connAttemptGap,
}
this.worker.postMessage(messageData); // brings delay of 10th ms?
return window.fetch(this.options.ingestPoint + '/v1/web/start', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
token: sessionStorage.getItem(this.options.session_token_key),
userUUID: localStorage.getItem(this.options.local_uuid_key),
projectKey: this.projectKey,
revID: this.revID,
timestamp: startTimestamp,
trackerVersion: this.version,
isSnippet: this.options.__is_snippet,
deviceMemory,
jsHeapSizeLimit,
reset,
}),
})
.then(r => {
if (r.status === 200) {
return r.json()
} else { // TODO: handle canceling && 403
return r.text().then(text => {
throw new Error(`Server error: ${r.status}. ${text}`);
});
}
})
.then(r => {
const { token, userUUID, sessionID, beaconSizeLimit } = r;
if (typeof token !== 'string' ||
typeof userUUID !== 'string' ||
(typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')) {
throw new Error(`Incorrect server response: ${ JSON.stringify(r) }`);
}
sessionStorage.setItem(this.options.session_token_key, token);
localStorage.setItem(this.options.local_uuid_key, userUUID);
if (typeof sessionID === 'string') {
this._sessionID = sessionID;
}
if (!this.worker) {
throw new Error("no worker found after start request (this might not happen)");
}
this.worker.postMessage({ token, beaconSizeLimit });
this.startCallbacks.forEach((cb) => cb());
this.observer.observe();
this.ticker.start();
log("OpenReplay tracking started.");
const onStartInfo = { sessionToken: token, userUUID, sessionID };
if (typeof this.options.onStart === 'function') {
this.options.onStart(onStartInfo);
}
return onStartInfo;
})
.catch(e => {
sessionStorage.removeItem(this.options.session_token_key)
this.stop()
warn("OpenReplay was unable to start. ", e)
this._debug("session_start", e);
throw e
})
resetNextPageSession(flag: boolean) {
if (flag) {
sessionStorage.setItem(this.options.session_reset_key, 't');
} else {
sessionStorage.removeItem(this.options.session_reset_key);
}
return Promise.reject("Player is already active");
}
private _start(startOpts: StartOptions): Promise<OnStartInfo> {
if (!this.worker) {
return Promise.reject("No worker found: perhaps, CSP is not set.");
}
if (this.isActive) {
return Promise.reject("OpenReplay: trying to call `start()` on the instance that has been started already.")
}
this.isActive = true;
let pageNo: number = 0;
const pageNoStr = sessionStorage.getItem(this.options.session_pageno_key);
if (pageNoStr != null) {
pageNo = parseInt(pageNoStr);
pageNo++;
}
sessionStorage.setItem(this.options.session_pageno_key, pageNo.toString());
const startTimestamp = timestamp();
const messageData: WorkerMessageData = {
ingestPoint: this.options.ingestPoint,
pageNo,
startTimestamp,
connAttemptCount: this.options.connAttemptCount,
connAttemptGap: this.options.connAttemptGap,
}
this.worker.postMessage(messageData); // brings delay of 10th ms?
// let token = sessionStorage.getItem(this.options.session_token_key)
// const tokenIsActive = localStorage.getItem("__or_at_" + token)
// if (tokenIsActive) {
// token = null
// }
const sReset = sessionStorage.getItem(this.options.session_reset_key);
sessionStorage.removeItem(this.options.session_reset_key);
this._userID = startOpts.userID || undefined
return window.fetch(this.options.ingestPoint + '/v1/web/start', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
token: sessionStorage.getItem(this.options.session_token_key),
userUUID: localStorage.getItem(this.options.local_uuid_key),
projectKey: this.projectKey,
revID: this.revID,
timestamp: startTimestamp,
trackerVersion: this.version,
isSnippet: this.options.__is_snippet,
deviceMemory,
jsHeapSizeLimit,
reset: startOpts.forceNew || sReset !== null,
userID: this._userID,
}),
})
.then(r => {
if (r.status === 200) {
return r.json()
} else { // TODO: handle canceling && 403
return r.text().then(text => {
throw new Error(`Server error: ${r.status}. ${text}`);
});
}
})
.then(r => {
const { token, userUUID, sessionID, beaconSizeLimit } = r;
if (typeof token !== 'string' ||
typeof userUUID !== 'string' ||
(typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')) {
throw new Error(`Incorrect server response: ${ JSON.stringify(r) }`);
}
sessionStorage.setItem(this.options.session_token_key, token);
localStorage.setItem(this.options.local_uuid_key, userUUID);
// localStorage.setItem("__or_at_" + token, "true")
// this.attachEventListener(window, 'beforeunload', ()=>{
// localStorage.removeItem("__or_at_" + token)
// }, false);
// this.attachEventListener(window, 'pagehide', ()=>{
// localStorage.removeItem("__or_at_" + token)
// }, false);
if (typeof sessionID === 'string') {
this._sessionID = sessionID;
}
if (!this.worker) {
throw new Error("no worker found after start request (this might not happen)");
}
this.worker.postMessage({ token, beaconSizeLimit });
this.startCallbacks.forEach((cb) => cb());
this.observer.observe();
this.ticker.start();
log("OpenReplay tracking started.");
const onStartInfo = { sessionToken: token, userUUID, sessionID };
if (typeof this.options.onStart === 'function') {
this.options.onStart(onStartInfo);
}
return onStartInfo;
})
.catch(e => {
sessionStorage.removeItem(this.options.session_token_key)
this.stop()
warn("OpenReplay was unable to start. ", e)
this._debug("session_start", e);
throw e
})
}
start(reset: boolean = false): Promise<OnStartInfo> {
start(options: StartOptions = { forceNew: false }): Promise<OnStartInfo> {
if (!document.hidden) {
return this._start(reset);
return this._start(options);
} else {
return new Promise((resolve) => {
const onVisibilityChange = () => {
if (!document.hidden) {
document.removeEventListener("visibilitychange", onVisibilityChange);
resolve(this._start(reset));
resolve(this._start(options));
}
}
document.addEventListener("visibilitychange", onVisibilityChange);
@ -354,6 +397,7 @@ export default class App {
if (this.worker) {
this.worker.postMessage("stop");
}
this.sanitizer.clear();
this.observer.disconnect();
this.nodes.clear();
this.ticker.stop();

484
tracker/tracker/src/main/app/observer.ts
View file

@ -1,484 +0,0 @@
import { stars, hasOpenreplayAttribute } from "../utils.js";
import {
CreateDocument,
CreateElementNode,
CreateTextNode,
SetNodeData,
SetCSSDataURLBased,
SetNodeAttribute,
SetNodeAttributeURLBased,
RemoveNodeAttribute,
MoveNode,
RemoveNode,
CreateIFrameDocument,
} from "../../messages/index.js";
import App from "./index.js";
interface Window extends WindowProxy {
HTMLInputElement: typeof HTMLInputElement,
HTMLLinkElement: typeof HTMLLinkElement,
HTMLStyleElement: typeof HTMLStyleElement,
SVGStyleElement: typeof SVGStyleElement,
HTMLIFrameElement: typeof HTMLIFrameElement,
Text: typeof Text,
Element: typeof Element,
//parent: Window,
}
type WindowConstructor =
Document |
Element |
Text |
HTMLInputElement |
HTMLLinkElement |
HTMLStyleElement |
HTMLIFrameElement
// type ConstructorNames =
// 'Element' |
// 'Text' |
// 'HTMLInputElement' |
// 'HTMLLinkElement' |
// 'HTMLStyleElement' |
// 'HTMLIFrameElement'
type Constructor<T> = { new (...args: any[]): T , name: string };
function isSVGElement(node: Element): node is SVGElement {
return node.namespaceURI === 'http://www.w3.org/2000/svg';
}
export interface Options {
obscureTextEmails: boolean;
obscureTextNumbers: boolean;
captureIFrames: boolean;
}
export default class Observer {
private readonly observer: MutationObserver;
private readonly commited: Array<boolean | undefined>;
private readonly recents: Array<boolean | undefined>;
private readonly indexes: Array<number>;
private readonly attributesList: Array<Set<string> | undefined>;
private readonly textSet: Set<number>;
private readonly textMasked: Set<number>;
constructor(private readonly app: App, private readonly options: Options, private readonly context: Window = window) {
this.observer = new MutationObserver(
this.app.safe((mutations) => {
for (const mutation of mutations) {
const target = mutation.target;
const type = mutation.type;
// Special case
// Document 'childList' might happen in case of iframe.
// TODO: generalize as much as possible
if (this.isInstance(target, Document)
&& type === 'childList'
//&& new Array(mutation.addedNodes).some(node => this.isInstance(node, HTMLHtmlElement))
) {
const parentFrame = target.defaultView?.frameElement
if (!parentFrame) { continue }
this.bindTree(target.documentElement)
const frameID = this.app.nodes.getID(parentFrame)
const docID = this.app.nodes.getID(target.documentElement)
if (frameID === undefined || docID === undefined) { continue }
this.app.send(CreateIFrameDocument(frameID, docID));
continue;
}
if (this.isIgnored(target) || !context.document.contains(target)) {
continue;
}
if (type === 'childList') {
for (let i = 0; i < mutation.removedNodes.length; i++) {
this.bindTree(mutation.removedNodes[i]);
}
for (let i = 0; i < mutation.addedNodes.length; i++) {
this.bindTree(mutation.addedNodes[i]);
}
continue;
}
const id = this.app.nodes.getID(target);
if (id === undefined) {
continue;
}
if (id >= this.recents.length) {
this.recents[id] = undefined;
}
if (type === 'attributes') {
const name = mutation.attributeName;
if (name === null) {
continue;
}
let attr = this.attributesList[id];
if (attr === undefined) {
this.attributesList[id] = attr = new Set();
}
attr.add(name);
continue;
}
if (type === 'characterData') {
this.textSet.add(id);
continue;
}
}
this.commitNodes();
}),
);
this.commited = [];
this.recents = [];
this.indexes = [0];
this.attributesList = [];
this.textSet = new Set();
this.textMasked = new Set();
}
private clear(): void {
this.commited.length = 0;
this.recents.length = 0;
this.indexes.length = 1;
this.attributesList.length = 0;
this.textSet.clear();
this.textMasked.clear();
}
// TODO: we need a type expert here so we won't have to ignore the lines
private isInstance<T extends WindowConstructor>(node: Node, constr: Constructor<T>): node is T {
let context = this.context;
while(context.parent && context.parent !== context) {
// @ts-ignore
if (node instanceof context[constr.name]) {
return true
}
// @ts-ignore
context = context.parent
}
// @ts-ignore
return node instanceof context[constr.name]
}
private isIgnored(node: Node): boolean {
if (this.isInstance(node, Text)) {
return false;
}
if (!this.isInstance(node, Element)) {
return true;
}
const tag = node.tagName.toUpperCase();
if (tag === 'LINK') {
const rel = node.getAttribute('rel');
const as = node.getAttribute('as');
return !(rel?.includes('stylesheet') || as === "style" || as === "font");
}
return (
tag === 'SCRIPT' ||
tag === 'NOSCRIPT' ||
tag === 'META' ||
tag === 'TITLE' ||
tag === 'BASE'
);
}
private sendNodeAttribute(
id: number,
node: Element,
name: string,
value: string | null,
): void {
if (isSVGElement(node)) {
if (name.substr(0, 6) === 'xlink:') {
name = name.substr(6);
}
if (value === null) {
this.app.send(new RemoveNodeAttribute(id, name));
} else if (name === 'href') {
if (value.length > 1e5) {
value = '';
}
this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref()));
} else {
this.app.send(new SetNodeAttribute(id, name, value));
}
return;
}
if (
name === 'src' ||
name === 'srcset' ||
name === 'integrity' ||
name === 'crossorigin' ||
name === 'autocomplete' ||
name.substr(0, 2) === 'on'
) {
return;
}
if (
name === 'value' &&
this.isInstance(node, HTMLInputElement) &&
node.type !== 'button' &&
node.type !== 'reset' &&
node.type !== 'submit'
) {
return;
}
if (value === null) {
this.app.send(new RemoveNodeAttribute(id, name));
return;
}
if (name === 'style' || name === 'href' && this.isInstance(node, HTMLLinkElement)) {
this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref()));
return;
}
if (name === 'href' || value.length > 1e5) {
value = '';
}
this.app.send(new SetNodeAttribute(id, name, value));
}
/* TODO: abstract sanitation */
getInnerTextSecure(el: HTMLElement): string {
const id = this.app.nodes.getID(el)
if (!id) { return '' }
return this.checkObscure(id, el.innerText)
}
private checkObscure(id: number, data: string): string {
if (this.textMasked.has(id)) {
return data.replace(
/[^\f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/g,
'█',
);
}
if (this.options.obscureTextNumbers) {
data = data.replace(/\d/g, '0');
}
if (this.options.obscureTextEmails) {
data = data.replace(
/([^\s]+)@([^\s]+)\.([^\s]+)/g,
(...f: Array<string>) =>
stars(f[1]) + '@' + stars(f[2]) + '.' + stars(f[3]),
);
}
return data
}
private sendNodeData(id: number, parentElement: Element, data: string): void {
if (this.isInstance(parentElement, HTMLStyleElement) || this.isInstance(parentElement, SVGStyleElement)) {
this.app.send(new SetCSSDataURLBased(id, data, this.app.getBaseHref()));
return;
}
data = this.checkObscure(id, data)
this.app.send(new SetNodeData(id, data));
}
/* end TODO: abstract sanitation */
private bindNode(node: Node): void {
const r = this.app.nodes.registerNode(node);
const id = r[0];
this.recents[id] = r[1] || this.recents[id] || false;
}
private bindTree(node: Node): void {
if (this.isIgnored(node)) {
return;
}
this.bindNode(node);
const walker = document.createTreeWalker(
node,
NodeFilter.SHOW_ELEMENT + NodeFilter.SHOW_TEXT,
{
acceptNode: (node) =>
this.isIgnored(node) || this.app.nodes.getID(node) !== undefined
? NodeFilter.FILTER_REJECT
: NodeFilter.FILTER_ACCEPT,
},
// @ts-ignore
false,
);
while (walker.nextNode()) {
this.bindNode(walker.currentNode);
}
}
private unbindNode(node: Node): void {
const id = this.app.nodes.unregisterNode(node);
if (id !== undefined && this.recents[id] === false) {
this.app.send(new RemoveNode(id));
}
}
private _commitNode(id: number, node: Node): boolean {
const parent = node.parentNode;
let parentID: number | undefined;
if (this.isInstance(node, HTMLHtmlElement)) {
this.indexes[id] = 0
} else {
if (parent === null) {
this.unbindNode(node);
return false;
}
parentID = this.app.nodes.getID(parent);
if (parentID === undefined) {
this.unbindNode(node);
return false;
}
if (!this.commitNode(parentID)) {
this.unbindNode(node);
return false;
}
if (
this.textMasked.has(parentID) ||
(this.isInstance(node, Element) && hasOpenreplayAttribute(node, 'masked'))
) {
this.textMasked.add(id);
}
let sibling = node.previousSibling;
while (sibling !== null) {
const siblingID = this.app.nodes.getID(sibling);
if (siblingID !== undefined) {
this.commitNode(siblingID);
this.indexes[id] = this.indexes[siblingID] + 1;
break;
}
sibling = sibling.previousSibling;
}
if (sibling === null) {
this.indexes[id] = 0;
}
}
const isNew = this.recents[id];
const index = this.indexes[id];
if (index === undefined) {
throw 'commitNode: missing node index';
}
if (isNew === true) {
if (this.isInstance(node, Element)) {
if (parentID !== undefined) {
this.app.send(new
CreateElementNode(
id,
parentID,
index,
node.tagName,
isSVGElement(node),
),
);
}
for (let i = 0; i < node.attributes.length; i++) {
const attr = node.attributes[i];
this.sendNodeAttribute(id, node, attr.nodeName, attr.value);
}
if (this.isInstance(node, HTMLIFrameElement) &&
(this.options.captureIFrames || node.getAttribute("data-openreplay-capture"))) {
this.handleIframe(node);
}
} else if (this.isInstance(node, Text)) {
// for text node id != 0, hence parentID !== undefined and parent is Element
this.app.send(new CreateTextNode(id, parentID as number, index));
this.sendNodeData(id, parent as Element, node.data);
}
return true;
}
if (isNew === false && parentID !== undefined) {
this.app.send(new MoveNode(id, parentID, index));
}
const attr = this.attributesList[id];
if (attr !== undefined) {
if (!this.isInstance(node, Element)) {
throw 'commitNode: node is not an element';
}
for (const name of attr) {
this.sendNodeAttribute(id, node, name, node.getAttribute(name));
}
}
if (this.textSet.has(id)) {
if (!this.isInstance(node, Text)) {
throw 'commitNode: node is not a text';
}
// for text node id != 0, hence parent is Element
this.sendNodeData(id, parent as Element, node.data);
}
return true;
}
private commitNode(id: number): boolean {
const node = this.app.nodes.getNode(id);
if (node === undefined) {
return false;
}
const cmt = this.commited[id];
if (cmt !== undefined) {
return cmt;
}
return (this.commited[id] = this._commitNode(id, node));
}
private commitNodes(): void {
let node;
for (let id = 0; id < this.recents.length; id++) {
this.commitNode(id);
if (this.recents[id] === true && (node = this.app.nodes.getNode(id))) {
this.app.nodes.callNodeCallbacks(node);
}
}
this.clear();
}
private iframeObservers: Observer[] = [];
private handleIframe(iframe: HTMLIFrameElement): void {
let context: Window | null = null
const handle = this.app.safe(() => {
const id = this.app.nodes.getID(iframe)
if (id === undefined) { return }
if (iframe.contentWindow === context) { return }
context = iframe.contentWindow as Window | null;
if (!context) { return }
const observer = new Observer(this.app, this.options, context)
this.iframeObservers.push(observer)
observer.observeIframe(id, context)
})
this.app.attachEventListener(iframe, "load", handle)
handle()
}
// TODO: abstract common functionality, separate FrameObserver
private observeIframe(id: number, context: Window) {
const doc = context.document;
this.observer.observe(doc, {
childList: true,
attributes: true,
characterData: true,
subtree: true,
attributeOldValue: false,
characterDataOldValue: false,
});
this.bindTree(doc.documentElement);
const docID = this.app.nodes.getID(doc.documentElement);
if (docID === undefined) {
console.log("Wrong")
return;
}
this.app.send(CreateIFrameDocument(id,docID));
this.commitNodes();
}
observe(): void {
this.observer.observe(this.context.document, {
childList: true,
attributes: true,
characterData: true,
subtree: true,
attributeOldValue: false,
characterDataOldValue: false,
});
this.app.send(new CreateDocument());
this.bindTree(this.context.document.documentElement);
this.commitNodes();
}
disconnect(): void {
this.iframeObservers.forEach(o => o.disconnect());
this.iframeObservers = [];
this.observer.disconnect();
this.clear();
}
}

19
tracker/tracker/src/main/app/observer/iframe_observer.ts Normal file
View file

@ -0,0 +1,19 @@
import Observer from "./observer.js";
import { CreateIFrameDocument } from "../../../messages/index.js";
export default class IFrameObserver extends Observer {
observe(iframe: HTMLIFrameElement) {
const doc = iframe.contentDocument;
const hostID = this.app.nodes.getID(iframe);
if (!doc || hostID === undefined) { return } //log TODO common app.logger
// Have to observe document, because the inner <html> might be changed
this.observeRoot(doc, (docID) => {
if (docID === undefined) {
console.log("OpenReplay: Iframe document not bound")
return;
}
this.app.send(CreateIFrameDocument(hostID, docID));
});
}
}

353
tracker/tracker/src/main/app/observer/observer.ts Normal file
View file

@ -0,0 +1,353 @@
import { hasOpenreplayAttribute } from "../../utils.js";
import {
RemoveNodeAttribute,
SetNodeAttribute,
SetNodeAttributeURLBased,
SetCSSDataURLBased,
SetNodeData,
CreateTextNode,
CreateElementNode,
MoveNode,
RemoveNode,
} from "../../../messages/index.js";
import App from "../index.js";
import { isInstance, inDocument } from "../context.js";
function isSVGElement(node: Element): node is SVGElement {
return node.namespaceURI === 'http://www.w3.org/2000/svg';
}
function isIgnored(node: Node): boolean {
if (isInstance(node, Text)) {
return false;
}
if (!isInstance(node, Element)) {
return true;
}
const tag = node.tagName.toUpperCase();
if (tag === 'LINK') {
const rel = node.getAttribute('rel');
const as = node.getAttribute('as');
return !(rel?.includes('stylesheet') || as === "style" || as === "font");
}
return (
tag === 'SCRIPT' ||
tag === 'NOSCRIPT' ||
tag === 'META' ||
tag === 'TITLE' ||
tag === 'BASE'
);
}
function isRootNode(node: Node): boolean {
return isInstance(node, Document) || isInstance(node, ShadowRoot);
}
function isObservable(node: Node): boolean {
if (isRootNode(node)) {
return true;
}
return !isIgnored(node);
}
export default abstract class Observer {
private readonly observer: MutationObserver;
private readonly commited: Array<boolean | undefined> = [];
private readonly recents: Array<boolean | undefined> = [];
private readonly myNodes: Array<boolean | undefined> = [];
private readonly indexes: Array<number> = [];
private readonly attributesList: Array<Set<string> | undefined> = [];
private readonly textSet: Set<number> = new Set();
private readonly inUpperContext: boolean;
constructor(protected readonly app: App, protected readonly context: Window = window) {
this.inUpperContext = context.parent === context //TODO: get rid of context here
this.observer = new MutationObserver(
this.app.safe((mutations) => {
for (const mutation of mutations) {
const target = mutation.target;
const type = mutation.type;
if (!isObservable(target) || !inDocument(target)) {
continue;
}
if (type === 'childList') {
for (let i = 0; i < mutation.removedNodes.length; i++) {
this.bindTree(mutation.removedNodes[i]);
}
for (let i = 0; i < mutation.addedNodes.length; i++) {
this.bindTree(mutation.addedNodes[i]);
}
continue;
}
const id = this.app.nodes.getID(target);
if (id === undefined) {
continue;
}
if (id >= this.recents.length) { // TODO: something more convinient
this.recents[id] = undefined;
}
if (type === 'attributes') {
const name = mutation.attributeName;
if (name === null) {
continue;
}
let attr = this.attributesList[id];
if (attr === undefined) {
this.attributesList[id] = attr = new Set();
}
attr.add(name);
continue;
}
if (type === 'characterData') {
this.textSet.add(id);
continue;
}
}
this.commitNodes();
}),
);
}
private clear(): void {
this.commited.length = 0;
this.recents.length = 0;
this.indexes.length = 1;
this.attributesList.length = 0;
this.textSet.clear();
}
private sendNodeAttribute(
id: number,
node: Element,
name: string,
value: string | null,
): void {
if (isSVGElement(node)) {
if (name.substr(0, 6) === 'xlink:') {
name = name.substr(6);
}
if (value === null) {
this.app.send(new RemoveNodeAttribute(id, name));
} else if (name === 'href') {
if (value.length > 1e5) {
value = '';
}
this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref()));
} else {
this.app.send(new SetNodeAttribute(id, name, value));
}
return;
}
if (
name === 'src' ||
name === 'srcset' ||
name === 'integrity' ||
name === 'crossorigin' ||
name === 'autocomplete' ||
name.substr(0, 2) === 'on'
) {
return;
}
if (
name === 'value' &&
isInstance(node, HTMLInputElement) &&
node.type !== 'button' &&
node.type !== 'reset' &&
node.type !== 'submit'
) {
return;
}
if (value === null) {
this.app.send(new RemoveNodeAttribute(id, name));
return;
}
if (name === 'style' || name === 'href' && isInstance(node, HTMLLinkElement)) {
this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref()));
return;
}
if (name === 'href' || value.length > 1e5) {
value = '';
}
this.app.send(new SetNodeAttribute(id, name, value));
}
private sendNodeData(id: number, parentElement: Element, data: string): void {
if (isInstance(parentElement, HTMLStyleElement) || isInstance(parentElement, SVGStyleElement)) {
this.app.send(new SetCSSDataURLBased(id, data, this.app.getBaseHref()));
return;
}
data = this.app.sanitizer.sanitize(id, data)
this.app.send(new SetNodeData(id, data));
}
private bindNode(node: Node): void {
const r = this.app.nodes.registerNode(node);
const id = r[0];
this.recents[id] = r[1] || this.recents[id] || false;
this.myNodes[id] = true;
}
private bindTree(node: Node): void {
if (!isObservable(node)) {
return
}
this.bindNode(node);
const walker = document.createTreeWalker(
node,
NodeFilter.SHOW_ELEMENT + NodeFilter.SHOW_TEXT,
{
acceptNode: (node) =>
isIgnored(node) || this.app.nodes.getID(node) !== undefined
? NodeFilter.FILTER_REJECT
: NodeFilter.FILTER_ACCEPT,
},
// @ts-ignore
false,
);
while (walker.nextNode()) {
this.bindNode(walker.currentNode);
}
}
private unbindNode(node: Node): void {
const id = this.app.nodes.unregisterNode(node);
if (id !== undefined && this.recents[id] === false) {
this.app.send(new RemoveNode(id));
}
}
private _commitNode(id: number, node: Node): boolean {
if (isRootNode(node)) {
return true;
}
const parent = node.parentNode;
let parentID: number | undefined;
// Disable parent check for the upper context HTMLHtmlElement, because it is root there... (before)
// TODO: get rid of "special" cases (there is an issue with CreateDocument altered behaviour though)
// TODO: Clean the logic (though now it workd fine)
if (!isInstance(node, HTMLHtmlElement) || !this.inUpperContext) {
if (parent === null) {
this.unbindNode(node);
return false;
}
parentID = this.app.nodes.getID(parent);
if (parentID === undefined) {
this.unbindNode(node);
return false;
}
if (!this.commitNode(parentID)) {
this.unbindNode(node);
return false;
}
this.app.sanitizer.handleNode(id, parentID, node);
}
let sibling = node.previousSibling;
while (sibling !== null) {
const siblingID = this.app.nodes.getID(sibling);
if (siblingID !== undefined) {
this.commitNode(siblingID);
this.indexes[id] = this.indexes[siblingID] + 1;
break;
}
sibling = sibling.previousSibling;
}
if (sibling === null) {
this.indexes[id] = 0; //
}
const isNew = this.recents[id];
const index = this.indexes[id];
if (index === undefined) {
throw 'commitNode: missing node index';
}
if (isNew === true) {
if (isInstance(node, Element)) {
if (parentID !== undefined) {
this.app.send(new
CreateElementNode(
id,
parentID,
index,
node.tagName,
isSVGElement(node),
),
);
}
for (let i = 0; i < node.attributes.length; i++) {
const attr = node.attributes[i];
this.sendNodeAttribute(id, node, attr.nodeName, attr.value);
}
} else if (isInstance(node, Text)) {
// for text node id != 0, hence parentID !== undefined and parent is Element
this.app.send(new CreateTextNode(id, parentID as number, index));
this.sendNodeData(id, parent as Element, node.data);
}
return true;
}
if (isNew === false && parentID !== undefined) {
this.app.send(new MoveNode(id, parentID, index));
}
const attr = this.attributesList[id];
if (attr !== undefined) {
if (!isInstance(node, Element)) {
throw 'commitNode: node is not an element';
}
for (const name of attr) {
this.sendNodeAttribute(id, node, name, node.getAttribute(name));
}
}
if (this.textSet.has(id)) {
if (!isInstance(node, Text)) {
throw 'commitNode: node is not a text';
}
// for text node id != 0, hence parent is Element
this.sendNodeData(id, parent as Element, node.data);
}
return true;
}
private commitNode(id: number): boolean {
const node = this.app.nodes.getNode(id);
if (node === undefined) {
return false;
}
const cmt = this.commited[id];
if (cmt !== undefined) {
return cmt;
}
return (this.commited[id] = this._commitNode(id, node));
}
private commitNodes(): void {
let node;
for (let id = 0; id < this.recents.length; id++) {
// TODO: make things/logic nice here.
// commit required in any case if recents[id] true or false (in case of unbinding) or undefined (in case of attr change).
if (!this.myNodes[id]) { continue }
this.commitNode(id);
if (this.recents[id] === true && (node = this.app.nodes.getNode(id))) {
this.app.nodes.callNodeCallbacks(node);
}
}
this.clear();
}
// ISSSUE
protected observeRoot(node: Node, beforeCommit: (id?: number) => unknown, nodeToBind: Node = node) {
this.observer.observe(node, {
childList: true,
attributes: true,
characterData: true,
subtree: true,
attributeOldValue: false,
characterDataOldValue: false,
});
this.bindTree(nodeToBind);
beforeCommit(this.app.nodes.getID(node))
this.commitNodes();
}
disconnect(): void {
this.observer.disconnect();
this.clear();
this.myNodes.length = 0;
}
}

18
tracker/tracker/src/main/app/observer/shadow_root_observer.ts Normal file
View file

@ -0,0 +1,18 @@
import Observer from "./observer.js";
import { CreateIFrameDocument } from "../../../messages/index.js";
export default class ShadowRootObserver extends Observer {
observe(el: Element) {
const shRoot = el.shadowRoot;
const hostID = this.app.nodes.getID(el);
if (!shRoot || hostID === undefined) { return } // log
this.observeRoot(shRoot, (rootID) => {
if (rootID === undefined) {
console.log("OpenReplay: Shadow Root was not bound")
return;
}
this.app.send(CreateIFrameDocument(hostID,rootID));
});
}
}

98
tracker/tracker/src/main/app/observer/top_observer.ts Normal file
View file

@ -0,0 +1,98 @@
import Observer from "./observer.js";
import { isInstance } from "../context.js";
import type { Window } from "../context.js";
import IFrameObserver from "./iframe_observer.js";
import ShadowRootObserver from "./shadow_root_observer.js";
import { CreateDocument } from "../../../messages/index.js";
import App from "../index.js";
import { IN_BROWSER } from '../../utils.js'
export interface Options {
captureIFrames: boolean
}
const attachShadowNativeFn = IN_BROWSER ? Element.prototype.attachShadow : ()=>new ShadowRoot();
export default class TopObserver extends Observer {
private readonly options: Options;
constructor(app: App, options: Partial<Options>) {
super(app);
this.options = Object.assign({
captureIFrames: false
}, options);
// IFrames
this.app.nodes.attachNodeCallback(node => {
if (isInstance(node, HTMLIFrameElement) &&
(this.options.captureIFrames || node.getAttribute("data-openreplay-capture"))
) {
this.handleIframe(node)
}
})
// ShadowDOM
this.app.nodes.attachNodeCallback(node => {
if (isInstance(node, Element) && node.shadowRoot !== null) {
this.handleShadowRoot(node.shadowRoot)
}
})
}
private iframeObservers: IFrameObserver[] = [];
private handleIframe(iframe: HTMLIFrameElement): void {
let context: Window | null = null
const handle = this.app.safe(() => {
const id = this.app.nodes.getID(iframe)
if (id === undefined) { return } //log
if (iframe.contentWindow === context) { return } //Does this happen frequently?
context = iframe.contentWindow as Window | null;
if (!context) { return }
const observer = new IFrameObserver(this.app, context)
this.iframeObservers.push(observer)
observer.observe(iframe)
})
this.app.attachEventListener(iframe, "load", handle)
handle()
}
private shadowRootObservers: ShadowRootObserver[] = []
private handleShadowRoot(shRoot: ShadowRoot) {
const observer = new ShadowRootObserver(this.app, this.context)
this.shadowRootObservers.push(observer)
observer.observe(shRoot.host)
}
observe(): void {
// Protection from several subsequent calls?
const observer = this;
Element.prototype.attachShadow = function() {
const shadow = attachShadowNativeFn.apply(this, arguments)
observer.handleShadowRoot(shadow)
return shadow
}
// Can observe documentElement (<html>) here, because it is not supposed to be changing.
// However, it is possible in some exotic cases and may cause an ignorance of the newly created <html>
// In this case context.document have to be observed, but this will cause
// the change in the re-player behaviour caused by CreateDocument message:
// the 0-node ("fRoot") will become #document rather than documentElement as it is now.
// Alternatively - observe(#document) then bindNode(documentElement)
this.observeRoot(this.context.document, () => {
this.app.send(new CreateDocument())
}, this.context.document.documentElement);
}
disconnect() {
Element.prototype.attachShadow = attachShadowNativeFn
this.iframeObservers.forEach(o => o.disconnect())
this.iframeObservers = []
this.shadowRootObservers.forEach(o => o.disconnect())
this.shadowRootObservers = []
super.disconnect()
}
}

66
tracker/tracker/src/main/app/sanitizer.ts Normal file
View file

@ -0,0 +1,66 @@
import { stars, hasOpenreplayAttribute } from "../utils.js";
import App from "./index.js";
import { isInstance } from "./context.js";
export interface Options {
obscureTextEmails: boolean;
obscureTextNumbers: boolean;
}
export default class Sanitizer {
private readonly masked: Set<number> = new Set();
private readonly options: Options;
constructor(private readonly app: App, options: Partial<Options>) {
this.options = Object.assign({
obscureTextEmails: true,
obscureTextNumbers: false,
}, options);
}
handleNode(id: number, parentID: number, node: Node) {
if (
this.masked.has(parentID) ||
(isInstance(node, Element) && hasOpenreplayAttribute(node, 'masked'))
) {
this.masked.add(id);
}
}
sanitize(id: number, data: string): string {
if (this.masked.has(id)) {
// TODO: is it the best place to put trim() ? Might trimmed spaces be considered in layout in certain cases?
return data.trim().replace(
/[^\f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/g,
'█',
);
}
if (this.options.obscureTextNumbers) {
data = data.replace(/\d/g, '0');
}
if (this.options.obscureTextEmails) {
data = data.replace(
/([^\s]+)@([^\s]+)\.([^\s]+)/g,
(...f: Array<string>) =>
stars(f[1]) + '@' + stars(f[2]) + '.' + stars(f[3]),
);
}
return data
}
isMasked(id: number): boolean {
return this.masked.has(id);
}
getInnerTextSecure(el: HTMLElement): string {
const id = this.app.nodes.getID(el)
if (!id) { return '' }
return this.sanitize(id, el.innerText)
}
clear(): void {
this.masked.clear();
}
}

65
tracker/tracker/src/main/index.ts
View file

@ -19,14 +19,15 @@ import Longtasks from "./modules/longtasks.js";
import CSSRules from "./modules/cssrules.js";
import { IN_BROWSER, deprecationWarn, DOCS_HOST } from "./utils.js";
import { Options as AppOptions } from "./app/index.js";
import { Options as ConsoleOptions } from "./modules/console.js";
import { Options as ExceptionOptions } from "./modules/exception.js";
import { Options as InputOptions } from "./modules/input.js";
import { Options as PerformanceOptions } from "./modules/performance.js";
import { Options as TimingOptions } from "./modules/timing.js";
export type { OnStartInfo } from './app/index.js';
import type { Options as AppOptions } from "./app/index.js";
import type { Options as ConsoleOptions } from "./modules/console.js";
import type { Options as ExceptionOptions } from "./modules/exception.js";
import type { Options as InputOptions } from "./modules/input.js";
import type { Options as PerformanceOptions } from "./modules/performance.js";
import type { Options as TimingOptions } from "./modules/timing.js";
import type { StartOptions } from './app/index.js'
//TODO: unique options init
import type { OnStartInfo } from './app/index.js';
export type Options = Partial<
AppOptions & ConsoleOptions & ExceptionOptions & InputOptions & PerformanceOptions & TimingOptions
@ -35,6 +36,7 @@ export type Options = Partial<
projectKey: string;
sessionToken?: string;
respectDoNotTrack?: boolean;
autoResetOnWindowOpen?: boolean;
// dev only
__DISABLE_SECURE_MODE?: boolean;
};
@ -84,7 +86,7 @@ export default class API {
(navigator.doNotTrack == '1'
// @ts-ignore
|| window.doNotTrack == '1');
this.app = doNotTrack ||
const app = this.app = doNotTrack ||
!('Map' in window) ||
!('Set' in window) ||
!('MutationObserver' in window) ||
@ -95,20 +97,35 @@ export default class API {
!('Worker' in window)
? null
: new App(options.projectKey, options.sessionToken, options);
if (this.app !== null) {
Viewport(this.app);
CSSRules(this.app);
Connection(this.app);
Console(this.app, options);
Exception(this.app, options);
Img(this.app);
Input(this.app, options);
Mouse(this.app);
Timing(this.app, options);
Performance(this.app, options);
Scroll(this.app);
Longtasks(this.app);
if (app !== null) {
Viewport(app);
CSSRules(app);
Connection(app);
Console(app, options);
Exception(app, options);
Img(app);
Input(app, options);
Mouse(app);
Timing(app, options);
Performance(app, options);
Scroll(app);
Longtasks(app);
(window as any).__OPENREPLAY__ = this;
if (options.autoResetOnWindowOpen) {
const wOpen = window.open;
app.attachStartCallback(() => {
// @ts-ignore ?
window.open = function(...args) {
app.resetNextPageSession(true)
wOpen.call(window, ...args)
app.resetNextPageSession(false)
}
})
app.attachStopCallback(() => {
window.open = wOpen;
})
}
} else {
console.log("OpenReplay: browser doesn't support API required for tracking or doNotTrack is set to 1.")
const req = new XMLHttpRequest();
@ -140,7 +157,7 @@ export default class API {
return this.isActive();
}
start() /*: Promise<OnStartInfo>*/ {
start(startOpts?: StartOptions) : Promise<OnStartInfo> {
if (!IN_BROWSER) {
console.error(`OpenReplay: you are trying to start Tracker on a node.js environment. If you want to use OpenReplay with SSR, please, use componentDidMount or useEffect API for placing the \`tracker.start()\` line. Check documentation on ${DOCS_HOST}${DOCS_SETUP}`)
return Promise.reject("Trying to start not in browser.");
@ -148,7 +165,7 @@ export default class API {
if (this.app === null) {
return Promise.reject("Browser doesn't support required api, or doNotTrack is active.");
}
return this.app.start();
return this.app.start(startOpts);
}
stop(): void {
if (this.app === null) {

8
tracker/tracker/src/main/modules/exception.ts
View file

@ -50,7 +50,13 @@ export function getExceptionMessageFromEvent(e: ErrorEvent | PromiseRejectionEve
if (e.reason instanceof Error) {
return getExceptionMessage(e.reason, [])
} else {
return new JSException('Unhandled Promise Rejection', String(e.reason), '[]');
let message: string;
try {
message = JSON.stringify(e.reason)
} catch(_) {
message = String(e.reason)
}
return new JSException('Unhandled Promise Rejection', message, '[]');
}
}
return null;

19
tracker/tracker/src/main/modules/img.ts
View file

@ -1,8 +1,21 @@
import { timestamp, isURL } from "../utils.js";
import App from "../app/index.js";
import { ResourceTiming, SetNodeAttributeURLBased } from "../../messages/index.js";
import { ResourceTiming, SetNodeAttributeURLBased, SetNodeAttribute } from "../../messages/index.js";
const PLACEHOLDER_SRC = "https://static.openreplay.com/tracker/placeholder.jpeg";
export default function (app: App): void {
function sendPlaceholder(id: number, node: HTMLImageElement): void {
app.send(new SetNodeAttribute(id, "src", PLACEHOLDER_SRC))
const { width, height } = node.getBoundingClientRect();
if (!node.hasAttribute("width")){
app.send(new SetNodeAttribute(id, "width", String(width)))
}
if (!node.hasAttribute("height")){
app.send(new SetNodeAttribute(id, "height", String(height)))
}
}
const sendImgSrc = app.safe(function (this: HTMLImageElement): void {
const id = app.nodes.getID(this);
if (id === undefined) {
@ -16,7 +29,9 @@ export default function (app: App): void {
if (src != null && isURL(src)) { // TODO: How about relative urls ? Src type is null sometimes.
app.send(new ResourceTiming(timestamp(), 0, 0, 0, 0, 0, src, 'img'));
}
} else if (src.length < 1e5) {
} else if (src.length >= 1e5 || app.sanitizer.isMasked(id)) {
sendPlaceholder(id, this)
} else {
app.send(new SetNodeAttributeURLBased(id, 'src', src, app.getBaseHref()));
}
});

20
tracker/tracker/src/main/modules/input.ts
View file

@ -2,7 +2,12 @@ import { normSpaces, IN_BROWSER, getLabelAttribute, hasOpenreplayAttribute } fro
import App from "../app/index.js";
import { SetInputTarget, SetInputValue, SetInputChecked } from "../../messages/index.js";
function isInput(node: any): node is HTMLInputElement {
// TODO: take into consideration "contenteditable" attribute
type TextEditableElement = HTMLInputElement | HTMLTextAreaElement
function isTextEditable(node: any): node is TextEditableElement {
if (node instanceof HTMLTextAreaElement) {
return true;
}
if (!(node instanceof HTMLInputElement)) {
return false;
}
@ -16,6 +21,7 @@ function isInput(node: any): node is HTMLInputElement {
type === 'range'
);
}
function isCheckable(node: any): node is HTMLInputElement {
if (!(node instanceof HTMLInputElement)) {
return false;
@ -25,7 +31,7 @@ function isCheckable(node: any): node is HTMLInputElement {
}
const labelElementFor: (
node: HTMLInputElement,
node: TextEditableElement,
) => HTMLLabelElement | undefined =
IN_BROWSER && 'labels' in HTMLInputElement.prototype
? (node): HTMLLabelElement | undefined => {
@ -56,7 +62,7 @@ const labelElementFor: (
}
};
export function getInputLabel(node: HTMLInputElement): string {
export function getInputLabel(node: TextEditableElement): string {
let label = getLabelAttribute(node);
if (label === null) {
const labelElement = labelElementFor(node);
@ -89,13 +95,13 @@ export default function (app: App, opts: Partial<Options>): void {
},
opts,
);
function sendInputTarget(id: number, node: HTMLInputElement): void {
function sendInputTarget(id: number, node: TextEditableElement): void {
const label = getInputLabel(node);
if (label !== '') {
app.send(new SetInputTarget(id, label));
}
}
function sendInputValue(id: number, node: HTMLInputElement): void {
function sendInputValue(id: number, node: TextEditableElement): void {
let value = node.value;
let inputMode: InputMode = options.defaultInputMode;
if (node.type === 'password' || hasOpenreplayAttribute(node, 'hidden')) {
@ -136,7 +142,7 @@ export default function (app: App, opts: Partial<Options>): void {
app.ticker.attach((): void => {
inputValues.forEach((value, id) => {
const node = app.nodes.getNode(id);
if (!isInput(node)) {
if (!isTextEditable(node)) {
inputValues.delete(id);
return;
}
@ -169,7 +175,7 @@ export default function (app: App, opts: Partial<Options>): void {
if (id === undefined) {
return;
}
if (isInput(node)) {
if (isTextEditable(node)) {
inputValues.set(id, node.value);
sendInputValue(id, node);
return;

2
tracker/tracker/src/main/modules/mouse.ts
View file

@ -92,7 +92,7 @@ export default function (app: App): void {
(target as HTMLElement).onclick != null ||
target.getAttribute('role') === 'button'
) {
const label: string = app.observer.getInnerTextSecure(target as HTMLElement);
const label: string = app.sanitizer.getInnerTextSecure(target as HTMLElement);
return normSpaces(label).slice(0, 100);
}
return '';

1
tracker/tracker/src/webworker/index.ts
View file

@ -47,6 +47,7 @@ function sendBatch(batch: Uint8Array):void {
return; // happens simultaneously with onerror TODO: clear codeflow
}
if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason
busy = false;
reset();
sendQueue.length = 0;
if (this.status === 401) { // Unauthorised (Token expired)