diff --git a/LICENSE b/LICENSE index 940c97860..406fe9608 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Asayer SAS. +Copyright (c) 2022 Asayer SAS. Portions of this software are licensed as follows: diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index c701f0ce0..6fe799c19 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -1,11 +1,12 @@ import json +import logging import time import schemas from chalicelib.core import notifications, slack, webhook from chalicelib.utils import pg_client, helper, email_helper from chalicelib.utils.TimeUTC import TimeUTC -import logging + def get(id): with pg_client.PostgresClient() as cur: @@ -157,3 +158,13 @@ def delete(project_id, alert_id): {"alert_id": alert_id, "project_id": project_id}) ) return {"data": {"state": "success"}} + + +def get_predefined_values(): + values = [e.value for e in schemas.AlertColumn] + values = [{"name": v, "value": v, + "unit": "count" if v.endswith(".count") else "ms", + "predefined": True, + "metricId": None, + "seriesId": None} for v in values] + return values diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 62bddfbb3..10e86024c 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -26,7 +26,9 @@ def try_live(project_id, data: schemas.TryCustomMetricsSchema): def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): - metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id) + metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) + if metric is None: + return None metric: schemas.TryCustomMetricsSchema = schemas.TryCustomMetricsSchema.parse_obj({**data.dict(), **metric}) return try_live(project_id=project_id, data=metric) @@ -113,10 +115,10 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche u AS (UPDATE metric_series SET name=series.name, filter=series.filter, - index=series.filter.index + index=series.index FROM (VALUES {",".join([f"(%(u_series_id_{s['i']})s,%(u_index_{s['i']})s,%(u_name_{s['i']})s,%(u_filter_{s['i']})s::jsonb)" - for s in n_series])}) AS series(series_id, index, name, filter) - WHERE metric_id =%(metric_id)s AND series_id=series.series_id + for s in u_series])}) AS series(series_id, index, name, filter) + WHERE metric_series.metric_id =%(metric_id)s AND metric_series.series_id=series.series_id RETURNING 1)""") if len(d_series_ids) > 0: sub_queries.append("""\ @@ -133,7 +135,6 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche cur.execute( query ) - r = cur.fetchone() return get(metric_id=metric_id, project_id=project_id, user_id=user_id) @@ -158,6 +159,8 @@ def get_all(project_id, user_id): rows = cur.fetchall() for r in rows: r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + for s in r["series"]: + s["filter"] = helper.old_search_payload_to_flat(s["filter"]) rows = helper.list_to_camel_case(rows) return rows @@ -177,7 +180,7 @@ def delete(project_id, metric_id, user_id): return {"state": "success"} -def get(metric_id, project_id, user_id): +def get(metric_id, project_id, user_id, flatten=True): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( @@ -197,7 +200,12 @@ def get(metric_id, project_id, user_id): ) ) row = cur.fetchone() + if row is None: + return None row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) + if flatten: + for s in row["series"]: + s["filter"] = helper.old_search_payload_to_flat(s["filter"]) return helper.dict_to_camel_case(row) @@ -205,17 +213,18 @@ def get_series_for_alert(project_id, user_id): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( - """SELECT metric_id, - series_id, - metrics.name AS metric_name, - metric_series.name AS series_name, - index AS series_index + """SELECT series_id AS value, + metrics.name || '.' || (COALESCE(metric_series.name, 'series ' || index)) || '.count' AS name, + 'count' AS unit, + FALSE AS predefined, + metric_id, + series_id FROM metric_series INNER JOIN metrics USING (metric_id) WHERE metrics.deleted_at ISNULL AND metrics.project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) - ORDER BY metric_name, series_index, series_name;""", + ORDER BY name;""", {"project_id": project_id, "user_id": user_id} ) ) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 835a655f4..c33bed586 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -99,9 +99,11 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date # row["filter"]["events"] = filter_stages(row["filter"]["events"]) get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True) + counts = sessions.search2_pg(data=schemas.SessionsSearchPayloadSchema.parse_obj(row["filter"]), + project_id=project_id, user_id=None, count_only=True) row["sessionsCount"] = counts["countSessions"] row["usersCount"] = counts["countUsers"] + filter_clone = dict(row["filter"]) overview = significance.get_overview(filter_d=row["filter"], project_id=project_id) row["stages"] = overview["stages"] row.pop("filter") @@ -110,6 +112,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date row["criticalIssuesCount"] = overview["criticalIssuesCount"] row["missedConversions"] = 0 if len(row["stages"]) < 2 \ else row["stages"][0]["sessionsCount"] - row["stages"][-1]["sessionsCount"] + row["filter"] = helper.old_search_payload_to_flat(filter_clone) return rows @@ -147,11 +150,12 @@ def delete(project_id, funnel_id, user_id): def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id) + return sessions.search2_pg(data=schemas.SessionsSearchPayloadSchema.parse_obj(f["filter"]), project_id=project_id, + user_id=user_id) def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): @@ -168,7 +172,7 @@ def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.Funnel def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -196,7 +200,7 @@ def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -224,7 +228,7 @@ def get_issues_on_the_fly(funnel_id, user_id, project_id, data): last_stage=last_stage))} -def get(funnel_id, project_id, user_id): +def get(funnel_id, project_id, user_id, flatten=True): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( @@ -246,6 +250,8 @@ def get(funnel_id, project_id, user_id): f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) # f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + if flatten: + f["filter"] = helper.old_search_payload_to_flat(f["filter"]) return f diff --git a/api/chalicelib/core/saved_search.py b/api/chalicelib/core/saved_search.py index dfa9a1dcf..732fc1596 100644 --- a/api/chalicelib/core/saved_search.py +++ b/api/chalicelib/core/saved_search.py @@ -18,6 +18,7 @@ def create(project_id, user_id, data: schemas.SavedSearchSchema): ) r = cur.fetchone() r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["filter"] = helper.old_search_payload_to_flat(r["filter"]) r = helper.dict_to_camel_case(r) return {"data": r} @@ -40,6 +41,7 @@ def update(search_id, project_id, user_id, data: schemas.SavedSearchSchema): ) r = cur.fetchone() r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["filter"] = helper.old_search_payload_to_flat(r["filter"]) r = helper.dict_to_camel_case(r) # r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) return r @@ -74,6 +76,8 @@ def get_all(project_id, user_id, details=False): rows = helper.list_to_camel_case(rows) for row in rows: row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) + if details: + row["filter"] = helper.old_search_payload_to_flat(row["filter"]) return rows @@ -112,4 +116,5 @@ def get(search_id, project_id, user_id): return None f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) + f["filter"] = helper.old_search_payload_to_flat(f["filter"]) return f diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index d81378ddb..f2261ce59 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -2,7 +2,7 @@ __author__ = "AZNAUROV David" __maintainer__ = "KRAIEM Taha Yassine" import schemas -from chalicelib.core import events, sessions_metas, metadata, sessions +from chalicelib.core import events, metadata, sessions from chalicelib.utils import dev """ @@ -617,6 +617,15 @@ def get_overview(filter_d, project_id, first_stage=None, last_stage=None): # The result of the multi-stage query rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) if len(rows) == 0: + # PS: not sure what to return if rows are empty + output["stages"] = [{ + "type": stages[0]["type"], + "value": stages[0]["value"], + "sessionsCount": None, + "dropPercentage": None, + "usersCount": None + }] + output['criticalIssuesCount'] = 0 return output # Obtain the first part of the output stages_list = get_stages(stages, rows) diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index f8ce9fab5..6887fa5da 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -366,3 +366,14 @@ def has_smtp(): def get_edition(): return "ee" if "ee" in config("ENTERPRISE_BUILD", default="").lower() else "foss" + + +def old_search_payload_to_flat(values): + # in case the old search body was passed + if values.get("events") is not None: + for v in values["events"]: + v["isEvent"] = True + for v in values.get("filters", []): + v["isEvent"] = False + values["filters"] = values.pop("events") + values.get("filters", []) + return values diff --git a/api/entrypoint.sh b/api/entrypoint.sh index 60fefb5c0..a092737be 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -1,2 +1,2 @@ #!/bin/bash -uvicorn app:app --host 0.0.0.0 +uvicorn app:app --host 0.0.0.0 --reload diff --git a/api/routers/core.py b/api/routers/core.py index 9c5bec86e..bbeb30bcd 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -108,7 +108,7 @@ def events_search(projectId: int, q: str, type: Union[schemas.FilterType, schema @app.post('/{projectId}/sessions/search2', tags=["sessions"]) -def sessions_search2(projectId: int, data: schemas.SessionsSearchPayloadSchema = Body(...), +def sessions_search2(projectId: int, data: schemas.FlatSessionsSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): data = sessions.search2_pg(data, projectId, user_id=context.user_id) return {'data': data} @@ -621,6 +621,12 @@ def get_all_alerts(projectId: int, context: schemas.CurrentContext = Depends(OR_ return {"data": alerts.get_all(projectId)} +@app.get('/{projectId}/alerts/triggers', tags=["alerts", "customMetrics"]) +def get_alerts_triggers(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": alerts.get_predefined_values() \ + + custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)} + + @app.get('/{projectId}/alerts/{alertId}', tags=["alerts"]) def get_alert(projectId: int, alertId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": alerts.get(alertId)} @@ -1072,11 +1078,6 @@ def get_custom_metric_chart(projectId: int, data: schemas.CustomMetricChartPaylo data=data)} -@app.get('/{projectId}/custom_metrics/series', tags=["customMetrics"]) -def get_series_for_alert(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)} - - @app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) @app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), diff --git a/api/schemas.py b/api/schemas.py index 0dfd949ac..d66b92444 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -458,7 +458,15 @@ class IssueType(str, Enum): js_exception = 'js_exception' -class _SessionSearchEventRaw(BaseModel): +class __MixedSearchFilter(BaseModel): + is_event: bool = Field(...) + + class Config: + alias_generator = attribute_to_camel_case + + +class _SessionSearchEventRaw(__MixedSearchFilter): + is_event: bool = Field(True, const=True) custom: Optional[List[Union[int, str]]] = Field(None, min_items=1) customOperator: Optional[MathOperator] = Field(None) key: Optional[str] = Field(None) @@ -491,7 +499,8 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): value: Union[List[_SessionSearchEventRaw], str, List[str]] = Field(...) -class _SessionSearchFilterSchema(BaseModel): +class _SessionSearchFilterSchema(__MixedSearchFilter): + is_event: bool = Field(False, const=False) custom: Optional[List[str]] = Field(None) key: Optional[str] = Field(None) value: Union[Optional[Union[IssueType, PlatformType, int, str]], @@ -536,12 +545,39 @@ class SessionsSearchPayloadSchema(BaseModel): alias_generator = attribute_to_camel_case -class SessionsSearchCountSchema(SessionsSearchPayloadSchema): +class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema): + events: Optional[List[_SessionSearchEventSchema]] = Field([]) + filters: List[Union[_SessionSearchFilterSchema, _SessionSearchEventSchema]] = Field([]) + + @root_validator(pre=True) + def flat_to_original(cls, values): + # in case the old search body was passed + if len(values.get("events", [])) > 0: + for v in values["events"]: + v["isEvent"] = True + for v in values.get("filters", []): + v["isEvent"] = False + else: + n_filters = [] + n_events = [] + for v in values.get("filters", []): + if v["isEvent"]: + n_events.append(v) + else: + n_filters.append(v) + values["events"] = n_events + values["filters"] = n_filters + return values + + +class SessionsSearchCountSchema(FlatSessionsSearchPayloadSchema): + # class SessionsSearchCountSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(default=None) order: Optional[str] = Field(default=None) -class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): +class FunnelSearchPayloadSchema(FlatSessionsSearchPayloadSchema): + # class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): range_value: Optional[str] = Field(None) sort: Optional[str] = Field(None) order: Optional[str] = Field(None) @@ -565,7 +601,8 @@ class UpdateFunnelSchema(FunnelSchema): is_public: Optional[bool] = Field(None) -class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema): +class FunnelInsightsPayloadSchema(FlatSessionsSearchPayloadSchema): + # class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(None) order: Optional[str] = Field(None) @@ -595,7 +632,8 @@ class MobileSignPayloadSchema(BaseModel): keys: List[str] = Field(...) -class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): +class CustomMetricSeriesFilterSchema(FlatSessionsSearchPayloadSchema): + # class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): startDate: Optional[int] = Field(None) endDate: Optional[int] = Field(None) sort: Optional[str] = Field(None) diff --git a/backend/pkg/db/cache/messages_common.go b/backend/pkg/db/cache/messages_common.go index c05422cb2..dcf860835 100644 --- a/backend/pkg/db/cache/messages_common.go +++ b/backend/pkg/db/cache/messages_common.go @@ -38,7 +38,7 @@ func (c *PGCache) InsertUserID(sessionID uint64, userID *IOSUserID) error { if err != nil { return err } - session.UserID = &userID.Value + session.UserID = userID.Value return nil } diff --git a/backend/pkg/db/cache/messages_web.go b/backend/pkg/db/cache/messages_web.go index 3afae8592..b259e49da 100644 --- a/backend/pkg/db/cache/messages_web.go +++ b/backend/pkg/db/cache/messages_web.go @@ -30,6 +30,7 @@ func (c *PGCache) InsertWebSessionStart(sessionID uint64, s *SessionStart) error UserDeviceType: s.UserDeviceType, UserDeviceMemorySize: s.UserDeviceMemorySize, UserDeviceHeapSize: s.UserDeviceHeapSize, + UserID: s.UserID, } if err := c.Conn.InsertSessionStart(sessionID, c.sessions[ sessionID ]); err != nil { c.sessions[ sessionID ] = nil diff --git a/backend/pkg/db/postgres/messages_common.go b/backend/pkg/db/postgres/messages_common.go index a6f624651..df539e05c 100644 --- a/backend/pkg/db/postgres/messages_common.go +++ b/backend/pkg/db/postgres/messages_common.go @@ -47,7 +47,8 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { rev_id, tracker_version, issue_score, platform, - user_agent, user_browser, user_browser_version, user_device_memory_size, user_device_heap_size + user_agent, user_browser, user_browser_version, user_device_memory_size, user_device_heap_size, + user_id ) VALUES ( $1, $2, $3, $4, $5, $6, $7, @@ -55,7 +56,8 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { NULLIF($10, ''), $11, $12, $13, - NULLIF($14, ''), NULLIF($15, ''), NULLIF($16, ''), NULLIF($17, 0), NULLIF($18, 0::bigint) + NULLIF($14, ''), NULLIF($15, ''), NULLIF($16, ''), NULLIF($17, 0), NULLIF($18, 0::bigint), + NULLIF($19, '') )`, sessionID, s.ProjectID, s.Timestamp, s.UserUUID, s.UserDevice, s.UserDeviceType, s.UserCountry, @@ -64,6 +66,7 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { s.TrackerVersion, s.Timestamp/1000, s.Platform, s.UserAgent, s.UserBrowser, s.UserBrowserVersion, s.UserDeviceMemorySize, s.UserDeviceHeapSize, + s.UserID, ); err != nil { return err; } diff --git a/backend/pkg/db/types/session.go b/backend/pkg/db/types/session.go index 0205f76f3..d354b0cd2 100644 --- a/backend/pkg/db/types/session.go +++ b/backend/pkg/db/types/session.go @@ -16,7 +16,7 @@ type Session struct { PagesCount int EventsCount int ErrorsCount int - UserID *string + UserID string // pointer?? UserAnonymousID *string Metadata1 *string Metadata2 *string diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 3d8bae7f6..cdff71e1d 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -63,9 +63,10 @@ UserDeviceType string UserDeviceMemorySize uint64 UserDeviceHeapSize uint64 UserCountry string +UserID string } func (msg *SessionStart) Encode() []byte{ - buf := make([]byte, 151 + len(msg.TrackerVersion)+ len(msg.RevID)+ len(msg.UserUUID)+ len(msg.UserAgent)+ len(msg.UserOS)+ len(msg.UserOSVersion)+ len(msg.UserBrowser)+ len(msg.UserBrowserVersion)+ len(msg.UserDevice)+ len(msg.UserDeviceType)+ len(msg.UserCountry)) + buf := make([]byte, 161 + len(msg.TrackerVersion)+ len(msg.RevID)+ len(msg.UserUUID)+ len(msg.UserAgent)+ len(msg.UserOS)+ len(msg.UserOSVersion)+ len(msg.UserBrowser)+ len(msg.UserBrowserVersion)+ len(msg.UserDevice)+ len(msg.UserDeviceType)+ len(msg.UserCountry)+ len(msg.UserID)) buf[0] = 1 p := 1 p = WriteUint(msg.Timestamp, buf, p) @@ -83,6 +84,7 @@ p = WriteString(msg.UserDeviceType, buf, p) p = WriteUint(msg.UserDeviceMemorySize, buf, p) p = WriteUint(msg.UserDeviceHeapSize, buf, p) p = WriteString(msg.UserCountry, buf, p) +p = WriteString(msg.UserID, buf, p) return buf[:p] } diff --git a/backend/pkg/messages/read_message.go b/backend/pkg/messages/read_message.go index d0148bbc6..c226df728 100644 --- a/backend/pkg/messages/read_message.go +++ b/backend/pkg/messages/read_message.go @@ -42,6 +42,7 @@ if msg.UserDeviceType, err = ReadString(reader); err != nil { return nil, err } if msg.UserDeviceMemorySize, err = ReadUint(reader); err != nil { return nil, err } if msg.UserDeviceHeapSize, err = ReadUint(reader); err != nil { return nil, err } if msg.UserCountry, err = ReadString(reader); err != nil { return nil, err } +if msg.UserID, err = ReadString(reader); err != nil { return nil, err } return msg, nil case 2: diff --git a/backend/services/http/handlers_web.go b/backend/services/http/handlers_web.go index 5e144f1cc..09d2511d8 100644 --- a/backend/services/http/handlers_web.go +++ b/backend/services/http/handlers_web.go @@ -27,6 +27,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { JsHeapSizeLimit uint64 `json:"jsHeapSizeLimit"` ProjectKey *string `json:"projectKey"` Reset bool `json:"reset"` + UserID string `json:"userID"` } type response struct { Timestamp int64 `json:"timestamp"` @@ -101,6 +102,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserCountry: country, UserDeviceMemorySize: req.DeviceMemory, UserDeviceHeapSize: req.JsHeapSizeLimit, + UserID: req.UserID, })) } diff --git a/ee/LICENSE.md b/ee/LICENSE.md index d99b63d76..5f6043f8f 100644 --- a/ee/LICENSE.md +++ b/ee/LICENSE.md @@ -1,36 +1,4 @@ The OpenReplay Enterprise license (the “Enterprise License”) -Copyright (c) 2021 Asayer SAS. +Copyright (c) 2022 Asayer SAS. -With regard to the OpenReplay Software: - -This software and associated documentation files (the "Software") may only be -used in production, if you (and any entity that you represent) have agreed to, -and are in compliance with, the OpenReplay Subscription Terms of Service, available -at https://openreplay.com/terms.html (the “Enterprise Edition”), or other -agreement governing the use of the Software, as agreed by you and OpenReplay, -and otherwise have a valid OpenReplay Enterprise license for the -correct usage. Subject to the foregoing sentence, you are free to -modify this Software and publish patches to the Software. You agree that OpenReplay -and/or its licensors (as applicable) retain all right, title and interest in and -to all such modifications and/or patches, and all such modifications and/or -patches may only be used, copied, modified, displayed, distributed, or otherwise -exploited with a valid OpenReplay Enterprise license for the correct -number of user seats and profiles. Notwithstanding the foregoing, you may copy and modify -the Software for development and testing purposes, without requiring a -subscription. You agree that OpenReplay and/or its licensors (as applicable) retain -all right, title and interest in and to all such modifications. You are not -granted any other rights beyond what is expressly stated herein. Subject to the -foregoing, it is forbidden to copy, merge, publish, distribute, sublicense, -and/or sell the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -For all third party components incorporated into the OpenReplay Software, those -components are licensed under the original license provided by the owner of the -applicable component. +To license the Enterprise Edition of OpenReplay, and take advantage of its additional features, functionality and support, you must agree to the terms of the OpenReplay Enterprise License Agreement. Please contact OpenReplay at [sales@openreplay.com](mailto:sales@openreplay.com). diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 8afea0ab6..f1ff9550b 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -253,7 +253,7 @@ Pipfile /db_changes.sql /Dockerfile.bundle /entrypoint.bundle.sh -/entrypoint.sh +#/entrypoint.sh /chalicelib/core/heatmaps.py /routers/subs/insights.py /schemas.py diff --git a/ee/api/chalicelib/utils/SAML2_helper.py b/ee/api/chalicelib/utils/SAML2_helper.py index a2a4e1e6e..c00081d2c 100644 --- a/ee/api/chalicelib/utils/SAML2_helper.py +++ b/ee/api/chalicelib/utils/SAML2_helper.py @@ -12,11 +12,11 @@ SAML2 = { "sp": { "entityId": config("SITE_URL") + "/api/sso/saml2/metadata/", "assertionConsumerService": { - "url": config("SITE_URL") + "/api/sso/saml2/acs", + "url": config("SITE_URL") + "/api/sso/saml2/acs/", "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" }, "singleLogoutService": { - "url": config("SITE_URL") + "/api/sso/saml2/sls", + "url": config("SITE_URL") + "/api/sso/saml2/sls/", "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" }, "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress", @@ -25,6 +25,12 @@ SAML2 = { }, "idp": None } + +# in case tenantKey is included in the URL +sp_acs = config("idp_tenantKey", default="") +if sp_acs is not None and len(sp_acs) > 0: + SAML2["sp"]["assertionConsumerService"]["url"] += sp_acs + "/" + idp = None # SAML2 config handler if config("SAML2_MD_URL", default=None) is not None and len(config("SAML2_MD_URL")) > 0: @@ -60,12 +66,9 @@ else: def init_saml_auth(req): # auth = OneLogin_Saml2_Auth(req, custom_base_path=environ['SAML_PATH']) - if idp is None: raise Exception("No SAML2 config provided") - auth = OneLogin_Saml2_Auth(req, old_settings=SAML2) - - return auth + return OneLogin_Saml2_Auth(req, old_settings=SAML2) async def prepare_request(request: Request): @@ -86,12 +89,20 @@ async def prepare_request(request: Request): session = {} # If server is behind proxys or balancers use the HTTP_X_FORWARDED fields headers = request.headers - url_data = urlparse('%s://%s' % (headers.get('x-forwarded-proto', 'http'), headers['host'])) + proto = headers.get('x-forwarded-proto', 'http') + if headers.get('x-forwarded-proto') is not None: + print(f"x-forwarded-proto: {proto}") + url_data = urlparse('%s://%s' % (proto, headers['host'])) + path = request.url.path + # add / to /acs + if not path.endswith("/"): + path = path + '/' + return { - 'https': 'on' if request.headers.get('x-forwarded-proto', 'http') == 'https' else 'off', + 'https': 'on' if proto == 'https' else 'off', 'http_host': request.headers['host'], 'server_port': url_data.port, - 'script_name': "/api" + request.url.path, + 'script_name': "/api" + path, 'get_data': request.args.copy(), # Uncomment if using ADFS as IdP, https://github.com/onelogin/python-saml/pull/144 # 'lowercase_urlencoding': True, diff --git a/ee/api/entrypoint.sh b/ee/api/entrypoint.sh new file mode 100755 index 000000000..a092737be --- /dev/null +++ b/ee/api/entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/bash +uvicorn app:app --host 0.0.0.0 --reload diff --git a/ee/api/routers/saml.py b/ee/api/routers/saml.py index 50723a1db..ee0f0333b 100644 --- a/ee/api/routers/saml.py +++ b/ee/api/routers/saml.py @@ -16,6 +16,7 @@ from starlette import status @public_app.get("/sso/saml2", tags=["saml2"]) +@public_app.get("/sso/saml2/", tags=["saml2"]) async def start_sso(request: Request): request.path = '' req = await prepare_request(request=request) @@ -25,6 +26,7 @@ async def start_sso(request: Request): @public_app.post('/sso/saml2/acs', tags=["saml2"]) +@public_app.post('/sso/saml2/acs/', tags=["saml2"]) async def process_sso_assertion(request: Request): req = await prepare_request(request=request) session = req["cookie"]["session"] @@ -43,6 +45,8 @@ async def process_sso_assertion(request: Request): user_data = auth.get_attributes() elif auth.get_settings().is_debug_active(): error_reason = auth.get_last_error_reason() + print("SAML2 error:") + print(error_reason) return {"errors": [error_reason]} email = auth.get_nameid() @@ -108,7 +112,91 @@ async def process_sso_assertion(request: Request): headers={'Location': SAML2_helper.get_landing_URL(jwt)}) +@public_app.post('/sso/saml2/acs/{tenantKey}', tags=["saml2"]) +@public_app.post('/sso/saml2/acs/{tenantKey}/', tags=["saml2"]) +async def process_sso_assertion_tk(tenantKey: str, request: Request): + req = await prepare_request(request=request) + session = req["cookie"]["session"] + auth = init_saml_auth(req) + + request_id = None + if 'AuthNRequestID' in session: + request_id = session['AuthNRequestID'] + + auth.process_response(request_id=request_id) + errors = auth.get_errors() + user_data = {} + if len(errors) == 0: + if 'AuthNRequestID' in session: + del session['AuthNRequestID'] + user_data = auth.get_attributes() + elif auth.get_settings().is_debug_active(): + error_reason = auth.get_last_error_reason() + print("SAML2 error:") + print(error_reason) + return {"errors": [error_reason]} + + email = auth.get_nameid() + print("received nameId:") + print(email) + existing = users.get_by_email_only(auth.get_nameid()) + + internal_id = next(iter(user_data.get("internalId", [])), None) + + t = tenants.get_by_tenant_key(tenantKey) + if t is None: + print("invalid tenantKey, please copy the correct value from Preferences > Account") + return {"errors": ["invalid tenantKey, please copy the correct value from Preferences > Account"]} + print(user_data) + role_name = user_data.get("role", []) + if len(role_name) == 0: + print("No role specified, setting role to member") + role_name = ["member"] + role_name = role_name[0] + role = roles.get_role_by_name(tenant_id=t['tenantId'], name=role_name) + if role is None: + return {"errors": [f"role {role_name} not found, please create it in openreplay first"]} + + admin_privileges = user_data.get("adminPrivileges", []) + admin_privileges = not (len(admin_privileges) == 0 + or admin_privileges[0] is None + or admin_privileges[0].lower() == "false") + + if existing is None: + deleted = users.get_deleted_user_by_email(auth.get_nameid()) + if deleted is not None: + print("== restore deleted user ==") + users.restore_sso_user(user_id=deleted["userId"], tenant_id=t['tenantId'], email=email, + admin=admin_privileges, origin=SAML2_helper.get_saml2_provider(), + name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), + internal_id=internal_id, role_id=role["roleId"]) + else: + print("== new user ==") + users.create_sso_user(tenant_id=t['tenantId'], email=email, admin=admin_privileges, + origin=SAML2_helper.get_saml2_provider(), + name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), + internal_id=internal_id, role_id=role["roleId"]) + else: + if t['tenantId'] != existing["tenantId"]: + print("user exists for a different tenant") + return {"errors": ["user exists for a different tenant"]} + if existing.get("origin") is None: + print(f"== migrating user to {SAML2_helper.get_saml2_provider()} ==") + users.update(tenant_id=t['tenantId'], user_id=existing["id"], + changes={"origin": SAML2_helper.get_saml2_provider(), "internal_id": internal_id}) + expiration = auth.get_session_expiration() + expiration = expiration if expiration is not None and expiration > 10 * 60 \ + else int(config("sso_exp_delta_seconds", cast=int, default=24 * 60 * 60)) + jwt = users.authenticate_sso(email=email, internal_id=internal_id, exp=expiration) + if jwt is None: + return {"errors": ["null JWT"]} + return Response( + status_code=status.HTTP_302_FOUND, + headers={'Location': SAML2_helper.get_landing_URL(jwt)}) + + @public_app.get('/sso/saml2/sls', tags=["saml2"]) +@public_app.get('/sso/saml2/sls/', tags=["saml2"]) async def process_sls_assertion(request: Request): req = await prepare_request(request=request) session = req["cookie"]["session"] @@ -144,6 +232,7 @@ async def process_sls_assertion(request: Request): @public_app.get('/sso/saml2/metadata', tags=["saml2"]) +@public_app.get('/sso/saml2/metadata/', tags=["saml2"]) async def saml2_metadata(request: Request): req = await prepare_request(request=request) auth = init_saml_auth(req) diff --git a/scripts/helm/app/chalice.yaml b/scripts/helm/app/chalice.yaml index fcbea8ed6..2d6b53ead 100644 --- a/scripts/helm/app/chalice.yaml +++ b/scripts/helm/app/chalice.yaml @@ -64,5 +64,6 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' + idp_tenantKey: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index 41df5e2f5..25ec6c387 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -69,9 +69,9 @@ spec: - name: sessions_region value: '{{ .Values.global.s3.region }}' - name: sessions_bucket - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: sourcemaps_bucket - value: '{{ .Values.global.s3.sourcemaps_bucket }}' + value: '{{ .Values.global.s3.sourcemapsBucket }}' - name: js_cache_bucket value: '{{ .Values.global.s3.assetsBucket }}' - name: EMAIL_HOST diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index e5c579739..5e76420e8 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -94,6 +94,7 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' + idp_tenantKey: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt new file mode 120000 index 000000000..12e23824a --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt @@ -0,0 +1 @@ +../../../files/site.crt \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key new file mode 120000 index 000000000..3805a27d1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key @@ -0,0 +1 @@ +../../../files/site.key \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml index e5e4d7dd9..91b7cc09c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml @@ -4,6 +4,6 @@ kind: Secret metadata: name: ssl data: - ca.crt: "" - site.crt: "{{ .Values.ssl_certificate | b64enc }}" - site.key: "{{ .Values.ssl_privatekey | b64enc }}" + ca.crt: '' + site.crt: '{{ .Files.Get "files/site.crt" | b64enc }}' + site.key: '{{ .Files.Get "files/site.key" | b64enc }}' diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml index 8c595aafa..821ad9e3c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml @@ -84,90 +84,3 @@ nodeSelector: {} tolerations: [] affinity: {} - - -ssl_certificate: |- - -----BEGIN CERTIFICATE----- - MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL - BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 - NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh - bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+ - LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w - hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds - QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX - sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o - immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l - fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ - ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ - OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u - tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p - CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA - YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2 - z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs - MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z - qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA - dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p - 0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6 - 5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh - 0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s - /JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP - XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc - HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG - T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi - qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S - w7yMPvE+TIh6+HwDZc2jNLkv/8tY - -----END CERTIFICATE----- - -ssl_privatekey: |- - -----BEGIN PRIVATE KEY----- - MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t - K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE - BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB - OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew - 6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK - aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9 - hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK - lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6 - VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62 - RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK - Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh - QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe - K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s - dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2 - CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh - ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl - 5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw - TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk - x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c - Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g - DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj - OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm - 5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p - /CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA - 0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6 - tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E - ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy - HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4 - IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA - PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q - p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO - GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw - 1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ - cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d - vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ - n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+ - Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x - OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9 - Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP - kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB - De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr - TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T - e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP - 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 - WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa - 8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb - esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn - PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 - PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 - 7qINKoGovqXB1oAdopIl1z64e7MWVE4= - -----END PRIVATE KEY----- diff --git a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml index 4ca3be6e2..c5ec92cf4 100644 --- a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml @@ -49,9 +49,9 @@ spec: - name: AWS_REGION_IOS value: '{{ .Values.global.s3.region }}' - name: S3_BUCKET_WEB - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: S3_BUCKET_IOS - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: REDIS_STRING value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - name: LICENSE_KEY diff --git a/scripts/helmcharts/openreplay/files/site.crt b/scripts/helmcharts/openreplay/files/site.crt new file mode 100644 index 000000000..90f6a68c2 --- /dev/null +++ b/scripts/helmcharts/openreplay/files/site.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL +BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 +NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh +bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+ +LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w +hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds +QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX +sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o +immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l +fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ +ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ +OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u +tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p +CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA +YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2 +z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs +MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z +qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA +dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p +0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6 +5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh +0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s +/JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP +XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc +HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG +T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi +qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S +w7yMPvE+TIh6+HwDZc2jNLkv/8tY +-----END CERTIFICATE----- diff --git a/scripts/helmcharts/openreplay/files/site.key b/scripts/helmcharts/openreplay/files/site.key new file mode 100644 index 000000000..641fd1b7b --- /dev/null +++ b/scripts/helmcharts/openreplay/files/site.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t +K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE +BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB +OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew +6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK +aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9 +hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK +lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6 +VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62 +RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK +Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh +QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe +K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s +dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2 +CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh +ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl +5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw +TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk +x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c +Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g +DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj +OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm +5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p +/CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA +0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6 +tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E +ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy +HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4 +IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA +PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q +p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO +GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw +1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ +cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d +vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ +n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+ +Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x +OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9 +Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP +kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB +De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr +TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T +e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP +3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 +WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa +8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb +esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn +PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 +PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 +7qINKoGovqXB1oAdopIl1z64e7MWVE4= +-----END PRIVATE KEY----- diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 53272762b..6edb0f32c 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -8,6 +8,13 @@ postgresql: &postgres postgresqlPort: "5432" postgresqlUser: "postgres" postgresqlDatabase: "postgres" + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 3000Mi + # cpu: 2 clickhouse: {} # For enterpriseEdition @@ -43,8 +50,8 @@ global: region: "us-east-1" endpoint: "http://minio.db.svc.cluster.local:9000" assetsBucket: "sessions-assets" - recordings_bucket: "mobs" - sourcemaps_bucket: "sourcemaps" + recordingsBucket: "mobs" + sourcemapsBucket: "sourcemaps" # if you're using one node installation, where # you're using local s3, make sure these variables # are same as minio.global.minio.accesskey and secretKey @@ -75,15 +82,14 @@ chalice: # idp_x509cert: '' # idp_sls_url: '' # idp_name: '' - + # idp_tenantKey: '' # If you want to override something # chartname: # filedFrom chart/Values.yaml: # key: value - -# For example: # +# For example (http): # http: # resources: # limits: @@ -96,7 +102,10 @@ chalice: ## Changes to nginx # # nginx-ingress: +# # Key and certificate files must be named site.key and site.crt +# # and copied to ../openreplay/files/ +# sslKey: site.key +# sslCert: site.crt # customServerConfigs: | # # Redirecting http to https # return 301 https://$host$request_uri; -# diff --git a/scripts/helmcharts/vars_template.yaml b/scripts/helmcharts/vars_template.yaml new file mode 100644 index 000000000..84ce36ee6 --- /dev/null +++ b/scripts/helmcharts/vars_template.yaml @@ -0,0 +1,111 @@ +fromVersion: "{{ openreplay_version }}" +# Databases specific variables +postgresql: &postgres + # For generating passwords + # `openssl rand -hex 20` + postgresqlPassword: "{{ postgres_db_password }}" + postgresqlHost: "{{ postgres_endpoint }}" + postgresqlPort: "5432" + postgresqlUser: "{{ postgres_db_user }}" + postgresqlDatabase: "{{ postgres_db_name }}" + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 3000Mi + # cpu: 2 + +clickhouse: {} + # For enterpriseEdition + # enabled: true + +kafka: &kafka + # For enterpriseEdition + # enabled: true + + kafkaHost: "{{ kafka_endpoint }}" + kafkaPort: "{{ kafka_endpoint.split(':')[-1] }}" + kafkaUseSsl: "{{ kafka_ssl }}" + +redis: &redis + # For enterpriseEdition + # enabled: false + redisHost: "{{ redis_endpoint }}" + redisPort: "{{ redis_endpoint.split(':')[-1] }}" + +minio: + global: + minio: + # For generating passwords + # `openssl rand -hex 20` + accessKey: "{{ minio_access_key }}" + secretKey: "{{ minio_secret_key }}" + +# Application specific variables +global: + postgresql: *postgres + kafka: *kafka + redis: *redis + s3: + region: "us-east-1" + endpoint: "http://minio.db.svc.cluster.local:9000" + assetsBucket: "sessions-assets" + recordingsBucket: "mobs" + sourcemapsBucket: "sourcemaps" + # if you're using one node installation, where + # you're using local s3, make sure these variables + # are same as minio.global.minio.accesskey and secretKey + accessKey: "{{ minio_access_key }}" + secretKey: "{{ minio_secret_key }}" + email: + emailHost: '{{ email_host }}' + emailPort: '{{ email_port }}' + emailUser: '{{ email_user }}' + emailPassword: '{{ email_password }}' + emailUseTls: '{{ email_use_tls }}' + emailUseSsl: '{{ email_use_ssl }}' + emailSslKey: '{{ email_ssl_key }}' + emailSslCert: '{{ email_ssl_cert }}' + emailFrom: '{{ email_from }}' + + enterpriseEditionLicense: "{{ enterprise_edition_license }}" + domainName: "{{ domain_name }}" + +chalice: + env: + jwt_secret: "{{ jwt_secret_key }}" + # captcha_server: '' + # captcha_key: '' + # SAML2_MD_URL: '' + # idp_entityId: '' + # idp_sso_url: '' + # idp_x509cert: '' + # idp_sls_url: '' + # idp_name: '' + # idp_tenantKey: '' + + +# If you want to override something +# chartname: +# filedFrom chart/Values.yaml: +# key: value + +# For example: +# +# http: +# resources: +# limits: +# cpu: 1024m +# memory: 4096Mi +# requests: +# cpu: 512m +# memory: 2056Mi + +## Changes to nginx +# +# nginx-ingress: +# customServerConfigs: | +# # Redirecting http to https +# return 301 https://$host$request_uri; +# diff --git a/third-party.md b/third-party.md index a26f646c3..98aefe5b0 100644 --- a/third-party.md +++ b/third-party.md @@ -1,4 +1,4 @@ -## Licenses (as of January 16, 2022) +## Licenses (as of January 21, 2022) Below is the list of dependencies used in OpenReplay software. Licenses may change between versions, so please keep this up to date with every new library you use. @@ -84,10 +84,9 @@ Below is the list of dependencies used in OpenReplay software. Licenses may chan | redux-immutable | BSD3 | JavaScript | | redux-thunk | MIT | JavaScript | | semantic-ui-react | MIT | JavaScript | -| socket.io-client | MIT | JavaScript | +| socketio | MIT | JavaScript | | source-map | BSD3 | JavaScript | | aws-sdk | Apache2 | JavaScript | | serverless | MIT | JavaScript | | lib/pq | MIT | Go | | peerjs | MIT | JavaScript | -| antonmedv/finder | MIT | JavaScript | diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 506f05e2d..4d327410b 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "3.4.13", + "version": "3.4.16", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/BufferingConnection.ts b/tracker/tracker-assist/src/BufferingConnection.ts index e90970c21..5fb3b7349 100644 --- a/tracker/tracker-assist/src/BufferingConnection.ts +++ b/tracker/tracker-assist/src/BufferingConnection.ts @@ -6,12 +6,13 @@ interface Message { } // 16kb should be max according to specification +// 64kb chrome const crOrFf: boolean = typeof navigator !== "undefined" && (navigator.userAgent.indexOf("Chrom") !== -1 || // Chrome && Chromium navigator.userAgent.indexOf("Firefox") !== -1); -const MESSAGES_PER_SEND = crOrFf ? 500 : 100 +const MESSAGES_PER_SEND = crOrFf ? 200 : 50 // Bffering required in case of webRTC export default class BufferingConnection { @@ -34,7 +35,10 @@ export default class BufferingConnection { send(messages: Message[]) { if (!this.conn.open) { return; } let i = 0; + //@ts-ignore + messages=messages.filter(m => m._id !== 39) while (i < messages.length) { + this.buffer.push(messages.slice(i, i+=this.msgsPerSend)) } if (!this.buffering) { diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index d2c89cfe6..b183413bd 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -47,11 +47,12 @@ export default class Mouse { } } - private readonly pScrEl = document.scrollingElement || document.documentElement + private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct private lastScrEl: Element | "window" | null = null private resetLastScrEl = () => { this.lastScrEl = null } private handleWScroll = e => { - if (e.target !== this.lastScrEl) { + if (e.target !== this.lastScrEl && + this.lastScrEl !== "window") { this.resetLastScrEl() } } diff --git a/tracker/tracker-assist/src/index.ts b/tracker/tracker-assist/src/index.ts index 74514378a..d2067ff91 100644 --- a/tracker/tracker-assist/src/index.ts +++ b/tracker/tracker-assist/src/index.ts @@ -27,7 +27,7 @@ enum CallingState { }; //@ts-ignore peerjs hack for webpack5 (?!) TODO: ES/node modules; -//Peer = Peer.default || Peer; +Peer = Peer.default || Peer; // type IncomeMessages = // "call_end" | @@ -86,7 +86,7 @@ export default function(opts?: Partial) { host: app.getHost(), path: '/assist', port: location.protocol === 'http:' && appOptions.__DISABLE_SECURE_MODE ? 80 : 443, - //debug: // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. + debug: appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. } if (options.config) { _opt['config'] = options.config diff --git a/tracker/tracker/package-lock.json b/tracker/tracker/package-lock.json index 287203b30..6dcbc4e81 100644 --- a/tracker/tracker/package-lock.json +++ b/tracker/tracker/package-lock.json @@ -1,6 +1,6 @@ { "name": "@openreplay/tracker", - "version": "3.4.7", + "version": "3.4.12", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index a2dbc60f1..16b10b8f4 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "3.4.12", + "version": "3.4.17", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/app/context.ts b/tracker/tracker/src/main/app/context.ts new file mode 100644 index 000000000..aa9a5dfb3 --- /dev/null +++ b/tracker/tracker/src/main/app/context.ts @@ -0,0 +1,72 @@ +// TODO: global type +export interface Window extends globalThis.Window { + HTMLInputElement: typeof HTMLInputElement, + HTMLLinkElement: typeof HTMLLinkElement, + HTMLStyleElement: typeof HTMLStyleElement, + SVGStyleElement: typeof SVGStyleElement, + HTMLIFrameElement: typeof HTMLIFrameElement, + Text: typeof Text, + Element: typeof Element, + ShadowRoot: typeof ShadowRoot, + //parent: Window, +} + +type WindowConstructor = + Document | + Element | + Text | + ShadowRoot | + HTMLInputElement | + HTMLLinkElement | + HTMLStyleElement | + HTMLIFrameElement + +// type ConstructorNames = +// 'Element' | +// 'Text' | +// 'HTMLInputElement' | +// 'HTMLLinkElement' | +// 'HTMLStyleElement' | +// 'HTMLIFrameElement' +type Constructor = { new (...args: any[]): T , name: string }; + + // TODO: we need a type expert here so we won't have to ignore the lines + // TODO: use it everywhere (static function; export from which file? <-- global Window typing required) +export function isInstance(node: Node, constr: Constructor): node is T { + const doc = node.ownerDocument; + if (!doc) { // null if Document + return constr.name === 'Document'; + } + let context: Window = + // @ts-ignore (for EI, Safary) + doc.parentWindow || + doc.defaultView; // TODO: smart global typing for Window object + while(context.parent && context.parent !== context) { + // @ts-ignore + if (node instanceof context[constr.name]) { + return true + } + // @ts-ignore + context = context.parent + } + // @ts-ignore + return node instanceof context[constr.name] +} + +export function inDocument(node: Node): boolean { + const doc = node.ownerDocument + if (!doc) { return false } + if (doc.contains(node)) { return true } + let context: Window = + // @ts-ignore (for EI, Safary) + doc.parentWindow || + doc.defaultView; + while(context.parent && context.parent !== context) { + if (context.document.contains(node)) { + return true + } + // @ts-ignore + context = context.parent + } + return false; +} diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 54fe9050f..b02d15f91 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -2,12 +2,15 @@ import { timestamp, log, warn } from "../utils.js"; import { Timestamp, PageClose } from "../../messages/index.js"; import Message from "../../messages/message.js"; import Nodes from "./nodes.js"; -import Observer from "./observer.js"; +import Observer from "./observer/top_observer.js"; +import Sanitizer from "./sanitizer.js"; import Ticker from "./ticker.js"; import { deviceMemory, jsHeapSizeLimit } from "../modules/performance.js"; -import type { Options as ObserverOptions } from "./observer.js"; +import type { Options as ObserverOptions } from "./observer/top_observer.js"; +import type { Options as SanitizerOptions } from "./sanitizer.js"; + import type { Options as WebworkerOptions, WorkerMessageData } from "../../messages/webworker.js"; @@ -17,11 +20,17 @@ export interface OnStartInfo { userUUID: string, } -export type Options = { +export interface StartOptions { + userID?: string, + forceNew: boolean, +} + +type AppOptions = { revID: string; node_id: string; session_token_key: string; session_pageno_key: string; + session_reset_key: string; local_uuid_key: string; ingestPoint: string; resourceBaseHref: string | null, // resourceHref? @@ -30,7 +39,9 @@ export type Options = { __debug_report_edp: string | null; __debug_log: boolean; onStart?: (info: OnStartInfo) => void; -} & ObserverOptions & WebworkerOptions; +} & WebworkerOptions; + +export type Options = AppOptions & ObserverOptions & SanitizerOptions type Callback = () => void; type CommitCallback = (messages: Array) => void; @@ -43,21 +54,23 @@ export default class App { readonly nodes: Nodes; readonly ticker: Ticker; readonly projectKey: string; + readonly sanitizer: Sanitizer; private readonly messages: Array = []; - /*private*/ readonly observer: Observer; // temp, for fast security fix. TODO: separate security/obscure module with nodeCallback that incapsulates `textMasked` functionality from Observer + private readonly observer: Observer; private readonly startCallbacks: Array = []; private readonly stopCallbacks: Array = []; private readonly commitCallbacks: Array = []; - private readonly options: Options; + private readonly options: AppOptions; private readonly revID: string; private _sessionID: string | null = null; + private _userID: string | undefined; private isActive = false; private version = 'TRACKER_VERSION'; private readonly worker?: Worker; constructor( projectKey: string, sessionToken: string | null | undefined, - opts: Partial, + options: Partial, ) { this.projectKey = projectKey; this.options = Object.assign( @@ -66,24 +79,23 @@ export default class App { node_id: '__openreplay_id', session_token_key: '__openreplay_token', session_pageno_key: '__openreplay_pageno', + session_reset_key: '__openreplay_reset', local_uuid_key: '__openreplay_uuid', ingestPoint: DEFAULT_INGEST_POINT, resourceBaseHref: null, __is_snippet: false, __debug_report_edp: null, __debug_log: false, - obscureTextEmails: true, - obscureTextNumbers: false, - captureIFrames: false, }, - opts, + options, ); if (sessionToken != null) { sessionStorage.setItem(this.options.session_token_key, sessionToken); } this.revID = this.options.revID; + this.sanitizer = new Sanitizer(this, options); this.nodes = new Nodes(this.options.node_id); - this.observer = new Observer(this, this.options); + this.observer = new Observer(this, options); this.ticker = new Ticker(this); this.ticker.attach(() => this.commit()); try { @@ -102,7 +114,10 @@ export default class App { this.stop(); } else if (data === "restart") { this.stop(); - this.start(true); + this.start({ + forceNew: true, + userID: this._userID, + }); } }; const alertWorker = () => { @@ -244,104 +259,132 @@ export default class App { active(): boolean { return this.isActive; } - private _start(reset: boolean): Promise { - if (!this.isActive) { - if (!this.worker) { - return Promise.reject("No worker found: perhaps, CSP is not set."); - } - this.isActive = true; - let pageNo: number = 0; - const pageNoStr = sessionStorage.getItem(this.options.session_pageno_key); - if (pageNoStr != null) { - pageNo = parseInt(pageNoStr); - pageNo++; - } - sessionStorage.setItem(this.options.session_pageno_key, pageNo.toString()); - const startTimestamp = timestamp(); - - const messageData: WorkerMessageData = { - ingestPoint: this.options.ingestPoint, - pageNo, - startTimestamp, - connAttemptCount: this.options.connAttemptCount, - connAttemptGap: this.options.connAttemptGap, - } - this.worker.postMessage(messageData); // brings delay of 10th ms? - return window.fetch(this.options.ingestPoint + '/v1/web/start', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - token: sessionStorage.getItem(this.options.session_token_key), - userUUID: localStorage.getItem(this.options.local_uuid_key), - projectKey: this.projectKey, - revID: this.revID, - timestamp: startTimestamp, - trackerVersion: this.version, - isSnippet: this.options.__is_snippet, - deviceMemory, - jsHeapSizeLimit, - reset, - }), - }) - .then(r => { - if (r.status === 200) { - return r.json() - } else { // TODO: handle canceling && 403 - return r.text().then(text => { - throw new Error(`Server error: ${r.status}. ${text}`); - }); - } - }) - .then(r => { - const { token, userUUID, sessionID, beaconSizeLimit } = r; - if (typeof token !== 'string' || - typeof userUUID !== 'string' || - (typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')) { - throw new Error(`Incorrect server response: ${ JSON.stringify(r) }`); - } - sessionStorage.setItem(this.options.session_token_key, token); - localStorage.setItem(this.options.local_uuid_key, userUUID); - if (typeof sessionID === 'string') { - this._sessionID = sessionID; - } - if (!this.worker) { - throw new Error("no worker found after start request (this might not happen)"); - } - this.worker.postMessage({ token, beaconSizeLimit }); - this.startCallbacks.forEach((cb) => cb()); - this.observer.observe(); - this.ticker.start(); - - log("OpenReplay tracking started."); - const onStartInfo = { sessionToken: token, userUUID, sessionID }; - if (typeof this.options.onStart === 'function') { - this.options.onStart(onStartInfo); - } - return onStartInfo; - }) - .catch(e => { - sessionStorage.removeItem(this.options.session_token_key) - this.stop() - warn("OpenReplay was unable to start. ", e) - this._debug("session_start", e); - throw e - }) + resetNextPageSession(flag: boolean) { + if (flag) { + sessionStorage.setItem(this.options.session_reset_key, 't'); + } else { + sessionStorage.removeItem(this.options.session_reset_key); } - return Promise.reject("Player is already active"); + } + private _start(startOpts: StartOptions): Promise { + if (!this.worker) { + return Promise.reject("No worker found: perhaps, CSP is not set."); + } + if (this.isActive) { + return Promise.reject("OpenReplay: trying to call `start()` on the instance that has been started already.") + } + this.isActive = true; + + let pageNo: number = 0; + const pageNoStr = sessionStorage.getItem(this.options.session_pageno_key); + if (pageNoStr != null) { + pageNo = parseInt(pageNoStr); + pageNo++; + } + sessionStorage.setItem(this.options.session_pageno_key, pageNo.toString()); + const startTimestamp = timestamp(); + + const messageData: WorkerMessageData = { + ingestPoint: this.options.ingestPoint, + pageNo, + startTimestamp, + connAttemptCount: this.options.connAttemptCount, + connAttemptGap: this.options.connAttemptGap, + } + this.worker.postMessage(messageData); // brings delay of 10th ms? + + + // let token = sessionStorage.getItem(this.options.session_token_key) + // const tokenIsActive = localStorage.getItem("__or_at_" + token) + // if (tokenIsActive) { + // token = null + // } + + const sReset = sessionStorage.getItem(this.options.session_reset_key); + sessionStorage.removeItem(this.options.session_reset_key); + + this._userID = startOpts.userID || undefined + return window.fetch(this.options.ingestPoint + '/v1/web/start', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + token: sessionStorage.getItem(this.options.session_token_key), + userUUID: localStorage.getItem(this.options.local_uuid_key), + projectKey: this.projectKey, + revID: this.revID, + timestamp: startTimestamp, + trackerVersion: this.version, + isSnippet: this.options.__is_snippet, + deviceMemory, + jsHeapSizeLimit, + reset: startOpts.forceNew || sReset !== null, + userID: this._userID, + }), + }) + .then(r => { + if (r.status === 200) { + return r.json() + } else { // TODO: handle canceling && 403 + return r.text().then(text => { + throw new Error(`Server error: ${r.status}. ${text}`); + }); + } + }) + .then(r => { + const { token, userUUID, sessionID, beaconSizeLimit } = r; + if (typeof token !== 'string' || + typeof userUUID !== 'string' || + (typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')) { + throw new Error(`Incorrect server response: ${ JSON.stringify(r) }`); + } + sessionStorage.setItem(this.options.session_token_key, token); + localStorage.setItem(this.options.local_uuid_key, userUUID); + // localStorage.setItem("__or_at_" + token, "true") + // this.attachEventListener(window, 'beforeunload', ()=>{ + // localStorage.removeItem("__or_at_" + token) + // }, false); + // this.attachEventListener(window, 'pagehide', ()=>{ + // localStorage.removeItem("__or_at_" + token) + // }, false); + if (typeof sessionID === 'string') { + this._sessionID = sessionID; + } + if (!this.worker) { + throw new Error("no worker found after start request (this might not happen)"); + } + this.worker.postMessage({ token, beaconSizeLimit }); + this.startCallbacks.forEach((cb) => cb()); + this.observer.observe(); + this.ticker.start(); + + log("OpenReplay tracking started."); + const onStartInfo = { sessionToken: token, userUUID, sessionID }; + if (typeof this.options.onStart === 'function') { + this.options.onStart(onStartInfo); + } + return onStartInfo; + }) + .catch(e => { + sessionStorage.removeItem(this.options.session_token_key) + this.stop() + warn("OpenReplay was unable to start. ", e) + this._debug("session_start", e); + throw e + }) } - start(reset: boolean = false): Promise { + start(options: StartOptions = { forceNew: false }): Promise { if (!document.hidden) { - return this._start(reset); + return this._start(options); } else { return new Promise((resolve) => { const onVisibilityChange = () => { if (!document.hidden) { document.removeEventListener("visibilitychange", onVisibilityChange); - resolve(this._start(reset)); + resolve(this._start(options)); } } document.addEventListener("visibilitychange", onVisibilityChange); @@ -354,6 +397,7 @@ export default class App { if (this.worker) { this.worker.postMessage("stop"); } + this.sanitizer.clear(); this.observer.disconnect(); this.nodes.clear(); this.ticker.stop(); diff --git a/tracker/tracker/src/main/app/observer.ts b/tracker/tracker/src/main/app/observer.ts deleted file mode 100644 index 3ed5088af..000000000 --- a/tracker/tracker/src/main/app/observer.ts +++ /dev/null @@ -1,484 +0,0 @@ -import { stars, hasOpenreplayAttribute } from "../utils.js"; -import { - CreateDocument, - CreateElementNode, - CreateTextNode, - SetNodeData, - SetCSSDataURLBased, - SetNodeAttribute, - SetNodeAttributeURLBased, - RemoveNodeAttribute, - MoveNode, - RemoveNode, - CreateIFrameDocument, -} from "../../messages/index.js"; -import App from "./index.js"; - -interface Window extends WindowProxy { - HTMLInputElement: typeof HTMLInputElement, - HTMLLinkElement: typeof HTMLLinkElement, - HTMLStyleElement: typeof HTMLStyleElement, - SVGStyleElement: typeof SVGStyleElement, - HTMLIFrameElement: typeof HTMLIFrameElement, - Text: typeof Text, - Element: typeof Element, - //parent: Window, -} - - -type WindowConstructor = - Document | - Element | - Text | - HTMLInputElement | - HTMLLinkElement | - HTMLStyleElement | - HTMLIFrameElement - -// type ConstructorNames = -// 'Element' | -// 'Text' | -// 'HTMLInputElement' | -// 'HTMLLinkElement' | -// 'HTMLStyleElement' | -// 'HTMLIFrameElement' -type Constructor = { new (...args: any[]): T , name: string }; - - -function isSVGElement(node: Element): node is SVGElement { - return node.namespaceURI === 'http://www.w3.org/2000/svg'; -} - -export interface Options { - obscureTextEmails: boolean; - obscureTextNumbers: boolean; - captureIFrames: boolean; -} - -export default class Observer { - private readonly observer: MutationObserver; - private readonly commited: Array; - private readonly recents: Array; - private readonly indexes: Array; - private readonly attributesList: Array | undefined>; - private readonly textSet: Set; - private readonly textMasked: Set; - constructor(private readonly app: App, private readonly options: Options, private readonly context: Window = window) { - this.observer = new MutationObserver( - this.app.safe((mutations) => { - for (const mutation of mutations) { - const target = mutation.target; - const type = mutation.type; - - // Special case - // Document 'childList' might happen in case of iframe. - // TODO: generalize as much as possible - if (this.isInstance(target, Document) - && type === 'childList' - //&& new Array(mutation.addedNodes).some(node => this.isInstance(node, HTMLHtmlElement)) - ) { - const parentFrame = target.defaultView?.frameElement - if (!parentFrame) { continue } - this.bindTree(target.documentElement) - const frameID = this.app.nodes.getID(parentFrame) - const docID = this.app.nodes.getID(target.documentElement) - if (frameID === undefined || docID === undefined) { continue } - this.app.send(CreateIFrameDocument(frameID, docID)); - continue; - } - - if (this.isIgnored(target) || !context.document.contains(target)) { - continue; - } - if (type === 'childList') { - for (let i = 0; i < mutation.removedNodes.length; i++) { - this.bindTree(mutation.removedNodes[i]); - } - for (let i = 0; i < mutation.addedNodes.length; i++) { - this.bindTree(mutation.addedNodes[i]); - } - continue; - } - const id = this.app.nodes.getID(target); - if (id === undefined) { - continue; - } - if (id >= this.recents.length) { - this.recents[id] = undefined; - } - if (type === 'attributes') { - const name = mutation.attributeName; - if (name === null) { - continue; - } - let attr = this.attributesList[id]; - if (attr === undefined) { - this.attributesList[id] = attr = new Set(); - } - attr.add(name); - continue; - } - if (type === 'characterData') { - this.textSet.add(id); - continue; - } - } - this.commitNodes(); - }), - ); - this.commited = []; - this.recents = []; - this.indexes = [0]; - this.attributesList = []; - this.textSet = new Set(); - this.textMasked = new Set(); - } - private clear(): void { - this.commited.length = 0; - this.recents.length = 0; - this.indexes.length = 1; - this.attributesList.length = 0; - this.textSet.clear(); - this.textMasked.clear(); - } - - // TODO: we need a type expert here so we won't have to ignore the lines - private isInstance(node: Node, constr: Constructor): node is T { - let context = this.context; - while(context.parent && context.parent !== context) { - // @ts-ignore - if (node instanceof context[constr.name]) { - return true - } - // @ts-ignore - context = context.parent - } - // @ts-ignore - return node instanceof context[constr.name] - } - - private isIgnored(node: Node): boolean { - if (this.isInstance(node, Text)) { - return false; - } - if (!this.isInstance(node, Element)) { - return true; - } - const tag = node.tagName.toUpperCase(); - if (tag === 'LINK') { - const rel = node.getAttribute('rel'); - const as = node.getAttribute('as'); - return !(rel?.includes('stylesheet') || as === "style" || as === "font"); - } - return ( - tag === 'SCRIPT' || - tag === 'NOSCRIPT' || - tag === 'META' || - tag === 'TITLE' || - tag === 'BASE' - ); - } - - private sendNodeAttribute( - id: number, - node: Element, - name: string, - value: string | null, - ): void { - if (isSVGElement(node)) { - if (name.substr(0, 6) === 'xlink:') { - name = name.substr(6); - } - if (value === null) { - this.app.send(new RemoveNodeAttribute(id, name)); - } else if (name === 'href') { - if (value.length > 1e5) { - value = ''; - } - this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); - } else { - this.app.send(new SetNodeAttribute(id, name, value)); - } - return; - } - if ( - name === 'src' || - name === 'srcset' || - name === 'integrity' || - name === 'crossorigin' || - name === 'autocomplete' || - name.substr(0, 2) === 'on' - ) { - return; - } - if ( - name === 'value' && - this.isInstance(node, HTMLInputElement) && - node.type !== 'button' && - node.type !== 'reset' && - node.type !== 'submit' - ) { - return; - } - if (value === null) { - this.app.send(new RemoveNodeAttribute(id, name)); - return; - } - if (name === 'style' || name === 'href' && this.isInstance(node, HTMLLinkElement)) { - this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); - return; - } - if (name === 'href' || value.length > 1e5) { - value = ''; - } - this.app.send(new SetNodeAttribute(id, name, value)); - } - - /* TODO: abstract sanitation */ - getInnerTextSecure(el: HTMLElement): string { - const id = this.app.nodes.getID(el) - if (!id) { return '' } - return this.checkObscure(id, el.innerText) - - } - - private checkObscure(id: number, data: string): string { - if (this.textMasked.has(id)) { - return data.replace( - /[^\f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/g, - '█', - ); - } - if (this.options.obscureTextNumbers) { - data = data.replace(/\d/g, '0'); - } - if (this.options.obscureTextEmails) { - data = data.replace( - /([^\s]+)@([^\s]+)\.([^\s]+)/g, - (...f: Array) => - stars(f[1]) + '@' + stars(f[2]) + '.' + stars(f[3]), - ); - } - return data - } - - private sendNodeData(id: number, parentElement: Element, data: string): void { - if (this.isInstance(parentElement, HTMLStyleElement) || this.isInstance(parentElement, SVGStyleElement)) { - this.app.send(new SetCSSDataURLBased(id, data, this.app.getBaseHref())); - return; - } - data = this.checkObscure(id, data) - this.app.send(new SetNodeData(id, data)); - } - /* end TODO: abstract sanitation */ - - private bindNode(node: Node): void { - const r = this.app.nodes.registerNode(node); - const id = r[0]; - this.recents[id] = r[1] || this.recents[id] || false; - } - - private bindTree(node: Node): void { - if (this.isIgnored(node)) { - return; - } - this.bindNode(node); - const walker = document.createTreeWalker( - node, - NodeFilter.SHOW_ELEMENT + NodeFilter.SHOW_TEXT, - { - acceptNode: (node) => - this.isIgnored(node) || this.app.nodes.getID(node) !== undefined - ? NodeFilter.FILTER_REJECT - : NodeFilter.FILTER_ACCEPT, - }, - // @ts-ignore - false, - ); - while (walker.nextNode()) { - this.bindNode(walker.currentNode); - } - } - - private unbindNode(node: Node): void { - const id = this.app.nodes.unregisterNode(node); - if (id !== undefined && this.recents[id] === false) { - this.app.send(new RemoveNode(id)); - } - } - - private _commitNode(id: number, node: Node): boolean { - const parent = node.parentNode; - let parentID: number | undefined; - if (this.isInstance(node, HTMLHtmlElement)) { - this.indexes[id] = 0 - } else { - if (parent === null) { - this.unbindNode(node); - return false; - } - parentID = this.app.nodes.getID(parent); - if (parentID === undefined) { - this.unbindNode(node); - return false; - } - if (!this.commitNode(parentID)) { - this.unbindNode(node); - return false; - } - if ( - this.textMasked.has(parentID) || - (this.isInstance(node, Element) && hasOpenreplayAttribute(node, 'masked')) - ) { - this.textMasked.add(id); - } - let sibling = node.previousSibling; - while (sibling !== null) { - const siblingID = this.app.nodes.getID(sibling); - if (siblingID !== undefined) { - this.commitNode(siblingID); - this.indexes[id] = this.indexes[siblingID] + 1; - break; - } - sibling = sibling.previousSibling; - } - if (sibling === null) { - this.indexes[id] = 0; - } - } - const isNew = this.recents[id]; - const index = this.indexes[id]; - if (index === undefined) { - throw 'commitNode: missing node index'; - } - if (isNew === true) { - if (this.isInstance(node, Element)) { - if (parentID !== undefined) { - this.app.send(new - CreateElementNode( - id, - parentID, - index, - node.tagName, - isSVGElement(node), - ), - ); - } - for (let i = 0; i < node.attributes.length; i++) { - const attr = node.attributes[i]; - this.sendNodeAttribute(id, node, attr.nodeName, attr.value); - } - - if (this.isInstance(node, HTMLIFrameElement) && - (this.options.captureIFrames || node.getAttribute("data-openreplay-capture"))) { - this.handleIframe(node); - } - } else if (this.isInstance(node, Text)) { - // for text node id != 0, hence parentID !== undefined and parent is Element - this.app.send(new CreateTextNode(id, parentID as number, index)); - this.sendNodeData(id, parent as Element, node.data); - } - return true; - } - if (isNew === false && parentID !== undefined) { - this.app.send(new MoveNode(id, parentID, index)); - } - const attr = this.attributesList[id]; - if (attr !== undefined) { - if (!this.isInstance(node, Element)) { - throw 'commitNode: node is not an element'; - } - for (const name of attr) { - this.sendNodeAttribute(id, node, name, node.getAttribute(name)); - } - } - if (this.textSet.has(id)) { - if (!this.isInstance(node, Text)) { - throw 'commitNode: node is not a text'; - } - // for text node id != 0, hence parent is Element - this.sendNodeData(id, parent as Element, node.data); - } - return true; - } - private commitNode(id: number): boolean { - const node = this.app.nodes.getNode(id); - if (node === undefined) { - return false; - } - const cmt = this.commited[id]; - if (cmt !== undefined) { - return cmt; - } - return (this.commited[id] = this._commitNode(id, node)); - } - private commitNodes(): void { - let node; - for (let id = 0; id < this.recents.length; id++) { - this.commitNode(id); - if (this.recents[id] === true && (node = this.app.nodes.getNode(id))) { - this.app.nodes.callNodeCallbacks(node); - } - } - this.clear(); - } - - private iframeObservers: Observer[] = []; - private handleIframe(iframe: HTMLIFrameElement): void { - let context: Window | null = null - const handle = this.app.safe(() => { - const id = this.app.nodes.getID(iframe) - if (id === undefined) { return } - if (iframe.contentWindow === context) { return } - context = iframe.contentWindow as Window | null; - if (!context) { return } - const observer = new Observer(this.app, this.options, context) - this.iframeObservers.push(observer) - observer.observeIframe(id, context) - }) - this.app.attachEventListener(iframe, "load", handle) - handle() - } - - // TODO: abstract common functionality, separate FrameObserver - private observeIframe(id: number, context: Window) { - const doc = context.document; - this.observer.observe(doc, { - childList: true, - attributes: true, - characterData: true, - subtree: true, - attributeOldValue: false, - characterDataOldValue: false, - }); - this.bindTree(doc.documentElement); - const docID = this.app.nodes.getID(doc.documentElement); - if (docID === undefined) { - console.log("Wrong") - return; - } - this.app.send(CreateIFrameDocument(id,docID)); - this.commitNodes(); - } - - observe(): void { - this.observer.observe(this.context.document, { - childList: true, - attributes: true, - characterData: true, - subtree: true, - attributeOldValue: false, - characterDataOldValue: false, - }); - this.app.send(new CreateDocument()); - this.bindTree(this.context.document.documentElement); - this.commitNodes(); - } - - disconnect(): void { - this.iframeObservers.forEach(o => o.disconnect()); - this.iframeObservers = []; - this.observer.disconnect(); - this.clear(); - } -} diff --git a/tracker/tracker/src/main/app/observer/iframe_observer.ts b/tracker/tracker/src/main/app/observer/iframe_observer.ts new file mode 100644 index 000000000..be0a7182c --- /dev/null +++ b/tracker/tracker/src/main/app/observer/iframe_observer.ts @@ -0,0 +1,19 @@ +import Observer from "./observer.js"; +import { CreateIFrameDocument } from "../../../messages/index.js"; + +export default class IFrameObserver extends Observer { + observe(iframe: HTMLIFrameElement) { + const doc = iframe.contentDocument; + const hostID = this.app.nodes.getID(iframe); + if (!doc || hostID === undefined) { return } //log TODO common app.logger + // Have to observe document, because the inner might be changed + this.observeRoot(doc, (docID) => { + if (docID === undefined) { + console.log("OpenReplay: Iframe document not bound") + return; + } + this.app.send(CreateIFrameDocument(hostID, docID)); + }); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts new file mode 100644 index 000000000..0f4ff2994 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -0,0 +1,353 @@ +import { hasOpenreplayAttribute } from "../../utils.js"; +import { + RemoveNodeAttribute, + SetNodeAttribute, + SetNodeAttributeURLBased, + SetCSSDataURLBased, + SetNodeData, + CreateTextNode, + CreateElementNode, + MoveNode, + RemoveNode, +} from "../../../messages/index.js"; +import App from "../index.js"; +import { isInstance, inDocument } from "../context.js"; + + +function isSVGElement(node: Element): node is SVGElement { + return node.namespaceURI === 'http://www.w3.org/2000/svg'; +} + +function isIgnored(node: Node): boolean { + if (isInstance(node, Text)) { + return false; + } + if (!isInstance(node, Element)) { + return true; + } + const tag = node.tagName.toUpperCase(); + if (tag === 'LINK') { + const rel = node.getAttribute('rel'); + const as = node.getAttribute('as'); + return !(rel?.includes('stylesheet') || as === "style" || as === "font"); + } + return ( + tag === 'SCRIPT' || + tag === 'NOSCRIPT' || + tag === 'META' || + tag === 'TITLE' || + tag === 'BASE' + ); +} + +function isRootNode(node: Node): boolean { + return isInstance(node, Document) || isInstance(node, ShadowRoot); +} + +function isObservable(node: Node): boolean { + if (isRootNode(node)) { + return true; + } + return !isIgnored(node); +} + +export default abstract class Observer { + private readonly observer: MutationObserver; + private readonly commited: Array = []; + private readonly recents: Array = []; + private readonly myNodes: Array = []; + private readonly indexes: Array = []; + private readonly attributesList: Array | undefined> = []; + private readonly textSet: Set = new Set(); + private readonly inUpperContext: boolean; + constructor(protected readonly app: App, protected readonly context: Window = window) { + this.inUpperContext = context.parent === context //TODO: get rid of context here + this.observer = new MutationObserver( + this.app.safe((mutations) => { + for (const mutation of mutations) { + const target = mutation.target; + const type = mutation.type; + + if (!isObservable(target) || !inDocument(target)) { + continue; + } + if (type === 'childList') { + for (let i = 0; i < mutation.removedNodes.length; i++) { + this.bindTree(mutation.removedNodes[i]); + } + for (let i = 0; i < mutation.addedNodes.length; i++) { + this.bindTree(mutation.addedNodes[i]); + } + continue; + } + const id = this.app.nodes.getID(target); + if (id === undefined) { + continue; + } + if (id >= this.recents.length) { // TODO: something more convinient + this.recents[id] = undefined; + } + if (type === 'attributes') { + const name = mutation.attributeName; + if (name === null) { + continue; + } + let attr = this.attributesList[id]; + if (attr === undefined) { + this.attributesList[id] = attr = new Set(); + } + attr.add(name); + continue; + } + if (type === 'characterData') { + this.textSet.add(id); + continue; + } + } + this.commitNodes(); + }), + ); + } + private clear(): void { + this.commited.length = 0; + this.recents.length = 0; + this.indexes.length = 1; + this.attributesList.length = 0; + this.textSet.clear(); + } + + private sendNodeAttribute( + id: number, + node: Element, + name: string, + value: string | null, + ): void { + if (isSVGElement(node)) { + if (name.substr(0, 6) === 'xlink:') { + name = name.substr(6); + } + if (value === null) { + this.app.send(new RemoveNodeAttribute(id, name)); + } else if (name === 'href') { + if (value.length > 1e5) { + value = ''; + } + this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); + } else { + this.app.send(new SetNodeAttribute(id, name, value)); + } + return; + } + if ( + name === 'src' || + name === 'srcset' || + name === 'integrity' || + name === 'crossorigin' || + name === 'autocomplete' || + name.substr(0, 2) === 'on' + ) { + return; + } + if ( + name === 'value' && + isInstance(node, HTMLInputElement) && + node.type !== 'button' && + node.type !== 'reset' && + node.type !== 'submit' + ) { + return; + } + if (value === null) { + this.app.send(new RemoveNodeAttribute(id, name)); + return; + } + if (name === 'style' || name === 'href' && isInstance(node, HTMLLinkElement)) { + this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); + return; + } + if (name === 'href' || value.length > 1e5) { + value = ''; + } + this.app.send(new SetNodeAttribute(id, name, value)); + } + + private sendNodeData(id: number, parentElement: Element, data: string): void { + if (isInstance(parentElement, HTMLStyleElement) || isInstance(parentElement, SVGStyleElement)) { + this.app.send(new SetCSSDataURLBased(id, data, this.app.getBaseHref())); + return; + } + data = this.app.sanitizer.sanitize(id, data) + this.app.send(new SetNodeData(id, data)); + } + + private bindNode(node: Node): void { + const r = this.app.nodes.registerNode(node); + const id = r[0]; + this.recents[id] = r[1] || this.recents[id] || false; + + this.myNodes[id] = true; + } + + private bindTree(node: Node): void { + if (!isObservable(node)) { + return + } + this.bindNode(node); + const walker = document.createTreeWalker( + node, + NodeFilter.SHOW_ELEMENT + NodeFilter.SHOW_TEXT, + { + acceptNode: (node) => + isIgnored(node) || this.app.nodes.getID(node) !== undefined + ? NodeFilter.FILTER_REJECT + : NodeFilter.FILTER_ACCEPT, + }, + // @ts-ignore + false, + ); + while (walker.nextNode()) { + this.bindNode(walker.currentNode); + } + } + + private unbindNode(node: Node): void { + const id = this.app.nodes.unregisterNode(node); + if (id !== undefined && this.recents[id] === false) { + this.app.send(new RemoveNode(id)); + } + } + + private _commitNode(id: number, node: Node): boolean { + if (isRootNode(node)) { + return true; + } + const parent = node.parentNode; + let parentID: number | undefined; + // Disable parent check for the upper context HTMLHtmlElement, because it is root there... (before) + // TODO: get rid of "special" cases (there is an issue with CreateDocument altered behaviour though) + // TODO: Clean the logic (though now it workd fine) + if (!isInstance(node, HTMLHtmlElement) || !this.inUpperContext) { + if (parent === null) { + this.unbindNode(node); + return false; + } + parentID = this.app.nodes.getID(parent); + if (parentID === undefined) { + this.unbindNode(node); + return false; + } + if (!this.commitNode(parentID)) { + this.unbindNode(node); + return false; + } + this.app.sanitizer.handleNode(id, parentID, node); + } + let sibling = node.previousSibling; + while (sibling !== null) { + const siblingID = this.app.nodes.getID(sibling); + if (siblingID !== undefined) { + this.commitNode(siblingID); + this.indexes[id] = this.indexes[siblingID] + 1; + break; + } + sibling = sibling.previousSibling; + } + if (sibling === null) { + this.indexes[id] = 0; // + } + const isNew = this.recents[id]; + const index = this.indexes[id]; + if (index === undefined) { + throw 'commitNode: missing node index'; + } + if (isNew === true) { + if (isInstance(node, Element)) { + if (parentID !== undefined) { + this.app.send(new + CreateElementNode( + id, + parentID, + index, + node.tagName, + isSVGElement(node), + ), + ); + } + for (let i = 0; i < node.attributes.length; i++) { + const attr = node.attributes[i]; + this.sendNodeAttribute(id, node, attr.nodeName, attr.value); + } + } else if (isInstance(node, Text)) { + // for text node id != 0, hence parentID !== undefined and parent is Element + this.app.send(new CreateTextNode(id, parentID as number, index)); + this.sendNodeData(id, parent as Element, node.data); + } + return true; + } + if (isNew === false && parentID !== undefined) { + this.app.send(new MoveNode(id, parentID, index)); + } + const attr = this.attributesList[id]; + if (attr !== undefined) { + if (!isInstance(node, Element)) { + throw 'commitNode: node is not an element'; + } + for (const name of attr) { + this.sendNodeAttribute(id, node, name, node.getAttribute(name)); + } + } + if (this.textSet.has(id)) { + if (!isInstance(node, Text)) { + throw 'commitNode: node is not a text'; + } + // for text node id != 0, hence parent is Element + this.sendNodeData(id, parent as Element, node.data); + } + return true; + } + private commitNode(id: number): boolean { + const node = this.app.nodes.getNode(id); + if (node === undefined) { + return false; + } + const cmt = this.commited[id]; + if (cmt !== undefined) { + return cmt; + } + return (this.commited[id] = this._commitNode(id, node)); + } + private commitNodes(): void { + let node; + for (let id = 0; id < this.recents.length; id++) { + // TODO: make things/logic nice here. + // commit required in any case if recents[id] true or false (in case of unbinding) or undefined (in case of attr change). + if (!this.myNodes[id]) { continue } + this.commitNode(id); + if (this.recents[id] === true && (node = this.app.nodes.getNode(id))) { + this.app.nodes.callNodeCallbacks(node); + } + } + this.clear(); + } + + // ISSSUE + protected observeRoot(node: Node, beforeCommit: (id?: number) => unknown, nodeToBind: Node = node) { + this.observer.observe(node, { + childList: true, + attributes: true, + characterData: true, + subtree: true, + attributeOldValue: false, + characterDataOldValue: false, + }); + this.bindTree(nodeToBind); + beforeCommit(this.app.nodes.getID(node)) + this.commitNodes(); + } + + disconnect(): void { + this.observer.disconnect(); + this.clear(); + this.myNodes.length = 0; + } +} diff --git a/tracker/tracker/src/main/app/observer/shadow_root_observer.ts b/tracker/tracker/src/main/app/observer/shadow_root_observer.ts new file mode 100644 index 000000000..244348ea1 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/shadow_root_observer.ts @@ -0,0 +1,18 @@ +import Observer from "./observer.js"; +import { CreateIFrameDocument } from "../../../messages/index.js"; + +export default class ShadowRootObserver extends Observer { + observe(el: Element) { + const shRoot = el.shadowRoot; + const hostID = this.app.nodes.getID(el); + if (!shRoot || hostID === undefined) { return } // log + this.observeRoot(shRoot, (rootID) => { + if (rootID === undefined) { + console.log("OpenReplay: Shadow Root was not bound") + return; + } + this.app.send(CreateIFrameDocument(hostID,rootID)); + }); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/observer/top_observer.ts b/tracker/tracker/src/main/app/observer/top_observer.ts new file mode 100644 index 000000000..b35f5d901 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/top_observer.ts @@ -0,0 +1,98 @@ +import Observer from "./observer.js"; +import { isInstance } from "../context.js"; +import type { Window } from "../context.js"; +import IFrameObserver from "./iframe_observer.js"; +import ShadowRootObserver from "./shadow_root_observer.js"; + +import { CreateDocument } from "../../../messages/index.js"; +import App from "../index.js"; +import { IN_BROWSER } from '../../utils.js' + +export interface Options { + captureIFrames: boolean +} + +const attachShadowNativeFn = IN_BROWSER ? Element.prototype.attachShadow : ()=>new ShadowRoot(); + +export default class TopObserver extends Observer { + private readonly options: Options; + constructor(app: App, options: Partial) { + super(app); + this.options = Object.assign({ + captureIFrames: false + }, options); + + // IFrames + this.app.nodes.attachNodeCallback(node => { + if (isInstance(node, HTMLIFrameElement) && + (this.options.captureIFrames || node.getAttribute("data-openreplay-capture")) + ) { + this.handleIframe(node) + } + }) + + // ShadowDOM + this.app.nodes.attachNodeCallback(node => { + if (isInstance(node, Element) && node.shadowRoot !== null) { + this.handleShadowRoot(node.shadowRoot) + } + }) + } + + + private iframeObservers: IFrameObserver[] = []; + private handleIframe(iframe: HTMLIFrameElement): void { + let context: Window | null = null + const handle = this.app.safe(() => { + const id = this.app.nodes.getID(iframe) + if (id === undefined) { return } //log + if (iframe.contentWindow === context) { return } //Does this happen frequently? + context = iframe.contentWindow as Window | null; + if (!context) { return } + const observer = new IFrameObserver(this.app, context) + + this.iframeObservers.push(observer) + observer.observe(iframe) + }) + this.app.attachEventListener(iframe, "load", handle) + handle() + } + + private shadowRootObservers: ShadowRootObserver[] = [] + private handleShadowRoot(shRoot: ShadowRoot) { + const observer = new ShadowRootObserver(this.app, this.context) + + this.shadowRootObservers.push(observer) + observer.observe(shRoot.host) + } + + observe(): void { + // Protection from several subsequent calls? + const observer = this; + Element.prototype.attachShadow = function() { + const shadow = attachShadowNativeFn.apply(this, arguments) + observer.handleShadowRoot(shadow) + return shadow + } + + // Can observe documentElement () here, because it is not supposed to be changing. + // However, it is possible in some exotic cases and may cause an ignorance of the newly created + // In this case context.document have to be observed, but this will cause + // the change in the re-player behaviour caused by CreateDocument message: + // the 0-node ("fRoot") will become #document rather than documentElement as it is now. + // Alternatively - observe(#document) then bindNode(documentElement) + this.observeRoot(this.context.document, () => { + this.app.send(new CreateDocument()) + }, this.context.document.documentElement); + } + + disconnect() { + Element.prototype.attachShadow = attachShadowNativeFn + this.iframeObservers.forEach(o => o.disconnect()) + this.iframeObservers = [] + this.shadowRootObservers.forEach(o => o.disconnect()) + this.shadowRootObservers = [] + super.disconnect() + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/sanitizer.ts b/tracker/tracker/src/main/app/sanitizer.ts new file mode 100644 index 000000000..d085b5739 --- /dev/null +++ b/tracker/tracker/src/main/app/sanitizer.ts @@ -0,0 +1,66 @@ +import { stars, hasOpenreplayAttribute } from "../utils.js"; +import App from "./index.js"; +import { isInstance } from "./context.js"; + +export interface Options { + obscureTextEmails: boolean; + obscureTextNumbers: boolean; +} + +export default class Sanitizer { + private readonly masked: Set = new Set(); + private readonly options: Options; + + constructor(private readonly app: App, options: Partial) { + this.options = Object.assign({ + obscureTextEmails: true, + obscureTextNumbers: false, + }, options); + } + + handleNode(id: number, parentID: number, node: Node) { + if ( + this.masked.has(parentID) || + (isInstance(node, Element) && hasOpenreplayAttribute(node, 'masked')) + ) { + this.masked.add(id); + } + } + + sanitize(id: number, data: string): string { + if (this.masked.has(id)) { + // TODO: is it the best place to put trim() ? Might trimmed spaces be considered in layout in certain cases? + return data.trim().replace( + /[^\f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/g, + '█', + ); + } + if (this.options.obscureTextNumbers) { + data = data.replace(/\d/g, '0'); + } + if (this.options.obscureTextEmails) { + data = data.replace( + /([^\s]+)@([^\s]+)\.([^\s]+)/g, + (...f: Array) => + stars(f[1]) + '@' + stars(f[2]) + '.' + stars(f[3]), + ); + } + return data + } + + isMasked(id: number): boolean { + return this.masked.has(id); + } + + getInnerTextSecure(el: HTMLElement): string { + const id = this.app.nodes.getID(el) + if (!id) { return '' } + return this.sanitize(id, el.innerText) + + } + + clear(): void { + this.masked.clear(); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index 6af325e57..75d195e50 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -19,14 +19,15 @@ import Longtasks from "./modules/longtasks.js"; import CSSRules from "./modules/cssrules.js"; import { IN_BROWSER, deprecationWarn, DOCS_HOST } from "./utils.js"; -import { Options as AppOptions } from "./app/index.js"; -import { Options as ConsoleOptions } from "./modules/console.js"; -import { Options as ExceptionOptions } from "./modules/exception.js"; -import { Options as InputOptions } from "./modules/input.js"; -import { Options as PerformanceOptions } from "./modules/performance.js"; -import { Options as TimingOptions } from "./modules/timing.js"; - -export type { OnStartInfo } from './app/index.js'; +import type { Options as AppOptions } from "./app/index.js"; +import type { Options as ConsoleOptions } from "./modules/console.js"; +import type { Options as ExceptionOptions } from "./modules/exception.js"; +import type { Options as InputOptions } from "./modules/input.js"; +import type { Options as PerformanceOptions } from "./modules/performance.js"; +import type { Options as TimingOptions } from "./modules/timing.js"; +import type { StartOptions } from './app/index.js' +//TODO: unique options init +import type { OnStartInfo } from './app/index.js'; export type Options = Partial< AppOptions & ConsoleOptions & ExceptionOptions & InputOptions & PerformanceOptions & TimingOptions @@ -35,6 +36,7 @@ export type Options = Partial< projectKey: string; sessionToken?: string; respectDoNotTrack?: boolean; + autoResetOnWindowOpen?: boolean; // dev only __DISABLE_SECURE_MODE?: boolean; }; @@ -84,7 +86,7 @@ export default class API { (navigator.doNotTrack == '1' // @ts-ignore || window.doNotTrack == '1'); - this.app = doNotTrack || + const app = this.app = doNotTrack || !('Map' in window) || !('Set' in window) || !('MutationObserver' in window) || @@ -95,20 +97,35 @@ export default class API { !('Worker' in window) ? null : new App(options.projectKey, options.sessionToken, options); - if (this.app !== null) { - Viewport(this.app); - CSSRules(this.app); - Connection(this.app); - Console(this.app, options); - Exception(this.app, options); - Img(this.app); - Input(this.app, options); - Mouse(this.app); - Timing(this.app, options); - Performance(this.app, options); - Scroll(this.app); - Longtasks(this.app); + if (app !== null) { + Viewport(app); + CSSRules(app); + Connection(app); + Console(app, options); + Exception(app, options); + Img(app); + Input(app, options); + Mouse(app); + Timing(app, options); + Performance(app, options); + Scroll(app); + Longtasks(app); (window as any).__OPENREPLAY__ = this; + + if (options.autoResetOnWindowOpen) { + const wOpen = window.open; + app.attachStartCallback(() => { + // @ts-ignore ? + window.open = function(...args) { + app.resetNextPageSession(true) + wOpen.call(window, ...args) + app.resetNextPageSession(false) + } + }) + app.attachStopCallback(() => { + window.open = wOpen; + }) + } } else { console.log("OpenReplay: browser doesn't support API required for tracking or doNotTrack is set to 1.") const req = new XMLHttpRequest(); @@ -140,7 +157,7 @@ export default class API { return this.isActive(); } - start() /*: Promise*/ { + start(startOpts?: StartOptions) : Promise { if (!IN_BROWSER) { console.error(`OpenReplay: you are trying to start Tracker on a node.js environment. If you want to use OpenReplay with SSR, please, use componentDidMount or useEffect API for placing the \`tracker.start()\` line. Check documentation on ${DOCS_HOST}${DOCS_SETUP}`) return Promise.reject("Trying to start not in browser."); @@ -148,7 +165,7 @@ export default class API { if (this.app === null) { return Promise.reject("Browser doesn't support required api, or doNotTrack is active."); } - return this.app.start(); + return this.app.start(startOpts); } stop(): void { if (this.app === null) { diff --git a/tracker/tracker/src/main/modules/exception.ts b/tracker/tracker/src/main/modules/exception.ts index 45fe37465..848df03be 100644 --- a/tracker/tracker/src/main/modules/exception.ts +++ b/tracker/tracker/src/main/modules/exception.ts @@ -50,7 +50,13 @@ export function getExceptionMessageFromEvent(e: ErrorEvent | PromiseRejectionEve if (e.reason instanceof Error) { return getExceptionMessage(e.reason, []) } else { - return new JSException('Unhandled Promise Rejection', String(e.reason), '[]'); + let message: string; + try { + message = JSON.stringify(e.reason) + } catch(_) { + message = String(e.reason) + } + return new JSException('Unhandled Promise Rejection', message, '[]'); } } return null; diff --git a/tracker/tracker/src/main/modules/img.ts b/tracker/tracker/src/main/modules/img.ts index 61e793b89..8c0f911a8 100644 --- a/tracker/tracker/src/main/modules/img.ts +++ b/tracker/tracker/src/main/modules/img.ts @@ -1,8 +1,21 @@ import { timestamp, isURL } from "../utils.js"; import App from "../app/index.js"; -import { ResourceTiming, SetNodeAttributeURLBased } from "../../messages/index.js"; +import { ResourceTiming, SetNodeAttributeURLBased, SetNodeAttribute } from "../../messages/index.js"; + +const PLACEHOLDER_SRC = "https://static.openreplay.com/tracker/placeholder.jpeg"; export default function (app: App): void { + function sendPlaceholder(id: number, node: HTMLImageElement): void { + app.send(new SetNodeAttribute(id, "src", PLACEHOLDER_SRC)) + const { width, height } = node.getBoundingClientRect(); + if (!node.hasAttribute("width")){ + app.send(new SetNodeAttribute(id, "width", String(width))) + } + if (!node.hasAttribute("height")){ + app.send(new SetNodeAttribute(id, "height", String(height))) + } + } + const sendImgSrc = app.safe(function (this: HTMLImageElement): void { const id = app.nodes.getID(this); if (id === undefined) { @@ -16,7 +29,9 @@ export default function (app: App): void { if (src != null && isURL(src)) { // TODO: How about relative urls ? Src type is null sometimes. app.send(new ResourceTiming(timestamp(), 0, 0, 0, 0, 0, src, 'img')); } - } else if (src.length < 1e5) { + } else if (src.length >= 1e5 || app.sanitizer.isMasked(id)) { + sendPlaceholder(id, this) + } else { app.send(new SetNodeAttributeURLBased(id, 'src', src, app.getBaseHref())); } }); diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 746c26f8f..ad8cda673 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -2,7 +2,12 @@ import { normSpaces, IN_BROWSER, getLabelAttribute, hasOpenreplayAttribute } fro import App from "../app/index.js"; import { SetInputTarget, SetInputValue, SetInputChecked } from "../../messages/index.js"; -function isInput(node: any): node is HTMLInputElement { +// TODO: take into consideration "contenteditable" attribute +type TextEditableElement = HTMLInputElement | HTMLTextAreaElement +function isTextEditable(node: any): node is TextEditableElement { + if (node instanceof HTMLTextAreaElement) { + return true; + } if (!(node instanceof HTMLInputElement)) { return false; } @@ -16,6 +21,7 @@ function isInput(node: any): node is HTMLInputElement { type === 'range' ); } + function isCheckable(node: any): node is HTMLInputElement { if (!(node instanceof HTMLInputElement)) { return false; @@ -25,7 +31,7 @@ function isCheckable(node: any): node is HTMLInputElement { } const labelElementFor: ( - node: HTMLInputElement, + node: TextEditableElement, ) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype ? (node): HTMLLabelElement | undefined => { @@ -56,7 +62,7 @@ const labelElementFor: ( } }; -export function getInputLabel(node: HTMLInputElement): string { +export function getInputLabel(node: TextEditableElement): string { let label = getLabelAttribute(node); if (label === null) { const labelElement = labelElementFor(node); @@ -89,13 +95,13 @@ export default function (app: App, opts: Partial): void { }, opts, ); - function sendInputTarget(id: number, node: HTMLInputElement): void { + function sendInputTarget(id: number, node: TextEditableElement): void { const label = getInputLabel(node); if (label !== '') { app.send(new SetInputTarget(id, label)); } } - function sendInputValue(id: number, node: HTMLInputElement): void { + function sendInputValue(id: number, node: TextEditableElement): void { let value = node.value; let inputMode: InputMode = options.defaultInputMode; if (node.type === 'password' || hasOpenreplayAttribute(node, 'hidden')) { @@ -136,7 +142,7 @@ export default function (app: App, opts: Partial): void { app.ticker.attach((): void => { inputValues.forEach((value, id) => { const node = app.nodes.getNode(id); - if (!isInput(node)) { + if (!isTextEditable(node)) { inputValues.delete(id); return; } @@ -169,7 +175,7 @@ export default function (app: App, opts: Partial): void { if (id === undefined) { return; } - if (isInput(node)) { + if (isTextEditable(node)) { inputValues.set(id, node.value); sendInputValue(id, node); return; diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 3ec70e844..0089fa37f 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -92,7 +92,7 @@ export default function (app: App): void { (target as HTMLElement).onclick != null || target.getAttribute('role') === 'button' ) { - const label: string = app.observer.getInnerTextSecure(target as HTMLElement); + const label: string = app.sanitizer.getInnerTextSecure(target as HTMLElement); return normSpaces(label).slice(0, 100); } return ''; diff --git a/tracker/tracker/src/webworker/index.ts b/tracker/tracker/src/webworker/index.ts index 723008c52..cf0d1586a 100644 --- a/tracker/tracker/src/webworker/index.ts +++ b/tracker/tracker/src/webworker/index.ts @@ -47,6 +47,7 @@ function sendBatch(batch: Uint8Array):void { return; // happens simultaneously with onerror TODO: clear codeflow } if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason + busy = false; reset(); sendQueue.length = 0; if (this.status === 401) { // Unauthorised (Token expired)