Spot network refactoring (#2617)

* start refactoring network

* separate network module, refactor spot network capture

Signed-off-by: nick-delirium <nikita@openreplay.com>

* some console refactoring, display network results in ui

* detect gql error param

* fix proxy ignore file, fix network tracking, fix tab tracking

* some code quality improvements...

* handle graphql in network lib (.2 ver), update tracker to use last version of lib

* remove debug logs, change request type to gql (if its gql!) in lib, display gql in ui

---------

Signed-off-by: nick-delirium <nikita@openreplay.com>
This commit is contained in:
Delirium 2024-09-30 09:47:27 +02:00 committed by GitHub
parent 264abc986f
commit e66423dcf4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
40 changed files with 1128 additions and 684 deletions

View file

@ -43,6 +43,8 @@ const mapSpotNetworkToEv = (ev: SpotNetworkRequest): any => {
return 'xhr';
case 'fetch':
return 'fetch';
case 'graphql':
return 'graphql';
case 'resource':
return 'resource';
default:
@ -56,7 +58,7 @@ const mapSpotNetworkToEv = (ev: SpotNetworkRequest): any => {
})
const response = JSON.stringify({
headers: ev.responseHeaders,
body: { warn: "Chrome Manifest V3 -- No response body available in Chrome 93+" }
body: ev.responseBody ?? { warn: "Chrome Manifest V3 -- No response body available in Chrome 93+" }
})
return ({
...ev,

View file

@ -10,6 +10,7 @@ export const enum ResourceType {
IMG = 'img',
MEDIA = 'media',
WS = 'websocket',
GRAPHQL = 'graphql',
OTHER = 'other',
}
@ -47,6 +48,8 @@ export function getResourceType(initiator: string, url: string): ResourceType {
case "avi":
case "mp3":
return ResourceType.MEDIA
case "graphql":
return ResourceType.GRAPHQL
default:
return ResourceType.OTHER
}

31
networkProxy/.gitignore vendored Normal file
View file

@ -0,0 +1,31 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
.output
stats.html
stats-*.json
.wxt
web-ext.config.ts
!public
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
dist/*
dist

19
networkProxy/LICENSE Normal file
View file

@ -0,0 +1,19 @@
Copyright (c) 2024 Asayer, Inc
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

46
networkProxy/README.md Normal file
View file

@ -0,0 +1,46 @@
this tiny library helps us (OpenReplay folks) to create proxy objects for fetch,
XHR and beacons for proper request tracking in @openreplay/tracker and Spot extension.
example usage:
```
import createNetworkProxy from '@openreplay/network-proxy';
const context = this;
const ignoreHeaders = ['Authorization'];
const tokenUrlMatcher = /\/auth\/token/;
function setSessionTokenHeader(setRequestHeader: (name: string, value: string) => void) {
const header = 'X-Session-Token
const sessionToken = getToken() // for exmaple, => `session #123123`;
if (sessionToken) {
setRequestHeader(header, sessionToken)
}
}
function sanitize(reqResInfo) {
if (reqResInfo.request) {
delete reqResInfo.request.body
}
return reqResInfo
}
const onMsg = (networkReq) => console.log(networkReq)
const isIgnoredUrl = (url) => url.includes('google.com')
// Gets current tracker requests url and returns boolean. If present,
// sessionTokenHeader will only be applied when this function returns true.
// Default: undefined
const tokenUrlMatcher = (url) => url.includes('google.com');
// this will observe global network requests
createNetworkProxy(
context,
options.ignoreHeaders,
setSessionTokenHeader,
sanitize,
(message) => app.send(message),
(url) => app.isServiceURL(url),
options.tokenUrlMatcher,
)
// to stop it, you can save this.fetch/other apis before appliying the proxy
// and then restore them
```

29
networkProxy/package-lock.json generated Normal file
View file

@ -0,0 +1,29 @@
{
"name": "network-proxy",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "network-proxy",
"version": "1.0.0",
"license": "ISC",
"devDependencies": {
"typescript": "^5.6.2"
}
},
"node_modules/typescript": {
"version": "5.6.2",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz",
"integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
}
}
}

19
networkProxy/package.json Normal file
View file

@ -0,0 +1,19 @@
{
"name": "@openreplay/network-proxy",
"version": "1.0.3",
"description": "this library helps us to create proxy objects for fetch, XHR and beacons for proper request tracking.",
"main": "dist/index.js",
"module": "dist/index.js",
"types": "dist/index.d.ts",
"files": [
"dist"
],
"scripts": {
"build": "tsc"
},
"author": "Nikita <nikita@openreplay.com>",
"license": "MIT",
"devDependencies": {
"typescript": "^5.6.2"
}
}

View file

@ -1,7 +1,6 @@
import { NetworkRequest } from '../../../common/messages.gen.js'
import NetworkMessage from './networkMessage.js'
import { RequestResponseData } from './types.js'
import { genStringBody, getURL } from './utils.js'
import NetworkMessage from './networkMessage'
import { RequestState, INetworkMessage, RequestResponseData } from './types';
import { genStringBody, getURL } from './utils'
// https://fetch.spec.whatwg.org/#concept-bodyinit-extract
const getContentType = (data?: BodyInit) => {
@ -22,7 +21,7 @@ export class BeaconProxyHandler<T extends typeof navigator.sendBeacon> implement
private readonly ignoredHeaders: boolean | string[],
private readonly setSessionTokenHeader: (cb: (name: string, value: string) => void) => void,
private readonly sanitize: (data: RequestResponseData) => RequestResponseData | null,
private readonly sendMessage: (item: NetworkRequest) => void,
private readonly sendMessage: (item: INetworkMessage) => void,
private readonly isServiceUrl: (url: string) => boolean,
) {}
@ -85,7 +84,7 @@ export default class BeaconProxy {
ignoredHeaders: boolean | string[],
setSessionTokenHeader: (cb: (name: string, value: string) => void) => void,
sanitize: (data: RequestResponseData) => RequestResponseData | null,
sendMessage: (item: NetworkRequest) => void,
sendMessage: (item: INetworkMessage) => void,
isServiceUrl: (url: string) => boolean,
) {
if (!BeaconProxy.hasSendBeacon()) {

View file

@ -5,10 +5,9 @@
* we can intercept the network requests
* in not-so-hacky way
* */
import NetworkMessage, { RequestState } from './networkMessage.js'
import { formatByteSize, genStringBody, getStringResponseByType, getURL } from './utils.js'
import { RequestResponseData } from './types.js'
import { NetworkRequest } from '../../../common/messages.gen.js'
import NetworkMessage from './networkMessage'
import { RequestState, INetworkMessage, RequestResponseData } from './types';
import { formatByteSize, genStringBody, getStringResponseByType, getURL } from './utils'
export class ResponseProxyHandler<T extends Response> implements ProxyHandler<T> {
public resp: Response
@ -123,7 +122,7 @@ export class FetchProxyHandler<T extends typeof fetch> implements ProxyHandler<T
private readonly ignoredHeaders: boolean | string[],
private readonly setSessionTokenHeader: (cb: (name: string, value: string) => void) => void,
private readonly sanitize: (data: RequestResponseData) => RequestResponseData | null,
private readonly sendMessage: (item: NetworkRequest) => void,
private readonly sendMessage: (item: INetworkMessage) => void,
private readonly isServiceUrl: (url: string) => boolean,
private readonly tokenUrlMatcher?: (url: string) => boolean,
) {}
@ -311,7 +310,7 @@ export default class FetchProxy {
ignoredHeaders: boolean | string[],
setSessionTokenHeader: (cb: (name: string, value: string) => void) => void,
sanitize: (data: RequestResponseData) => RequestResponseData | null,
sendMessage: (item: NetworkRequest) => void,
sendMessage: (item: INetworkMessage) => void,
isServiceUrl: (url: string) => boolean,
tokenUrlMatcher?: (url: string) => boolean,
) {

96
networkProxy/src/index.ts Normal file
View file

@ -0,0 +1,96 @@
import BeaconProxy from "./beaconProxy";
import FetchProxy from "./fetchProxy";
import XHRProxy from "./xhrProxy";
import { INetworkMessage, RequestResponseData } from "./types";
export {
BeaconProxy,
FetchProxy,
XHRProxy,
INetworkMessage,
RequestResponseData,
};
const getWarning = (api: string) => {
const str = `Openreplay: Can't find ${api} in global context.`;
console.warn(str);
};
/**
* Creates network proxies for XMLHttpRequest, fetch, and sendBeacon to intercept and monitor network requests and
* responses.
*
* @param {Window | typeof globalThis} context - The global context object (e.g., window or globalThis).
* @param {boolean | string[]} ignoredHeaders - Headers to ignore from requests. If `true`, all headers are ignored; if
* an array of strings, those header names are ignored.
* @param {(cb: (name: string, value: string) => void) => void} setSessionTokenHeader - Function to set a session token
* header; accepts a callback that sets the header name and value.
* @param {(data: RequestResponseData) => RequestResponseData | null} sanitize - Function to sanitize request and
* response data; should return sanitized data or `null` to ignore the data.
* @param {(message: INetworkMessage) => void} sendMessage - Function to send network messages for further processing
* or logging.
* @param {(url: string) => boolean} isServiceUrl - Function to determine if a URL is a service URL that should be
* ignored by the proxy.
* @param {Object} [modules] - Modules to apply the proxies to.
* @param {boolean} [modules.xhr=true] - Whether to proxy XMLHttpRequest.
* @param {boolean} [modules.fetch=true] - Whether to proxy the fetch API.
* @param {boolean} [modules.beacon=true] - Whether to proxy navigator.sendBeacon.
* @param {(url: string) => boolean} [tokenUrlMatcher] - Optional function; the session token header will only be
* applied to requests matching this function.
*
* @returns {void}
*/
export default function createNetworkProxy(
context: typeof globalThis,
ignoredHeaders: boolean | string[],
setSessionTokenHeader: (cb: (name: string, value: string) => void) => void,
sanitize: (data: RequestResponseData) => RequestResponseData | null,
sendMessage: (message: INetworkMessage) => void,
isServiceUrl: (url: string) => boolean,
modules: { xhr: boolean; fetch: boolean; beacon: boolean } = {
xhr: true,
fetch: true,
beacon: true,
},
tokenUrlMatcher?: (url: string) => boolean,
): void {
if (modules.xhr) {
if (context.XMLHttpRequest) {
context.XMLHttpRequest = XHRProxy.create(
ignoredHeaders,
setSessionTokenHeader,
sanitize,
sendMessage,
isServiceUrl,
tokenUrlMatcher,
);
} else {
getWarning("XMLHttpRequest");
}
}
if (modules.fetch) {
if (context.fetch) {
context.fetch = FetchProxy.create(
ignoredHeaders,
setSessionTokenHeader,
sanitize,
sendMessage,
isServiceUrl,
tokenUrlMatcher,
);
} else {
getWarning("fetch");
}
}
if (modules.beacon) {
if (context?.navigator?.sendBeacon) {
context.navigator.sendBeacon = BeaconProxy.create(
ignoredHeaders,
setSessionTokenHeader,
sanitize,
sendMessage,
isServiceUrl,
);
}
}
}

View file

@ -1,19 +1,9 @@
import { NetworkRequest } from '../../app/messages.gen.js'
import { RequestResponseData } from './types.js'
import { getTimeOrigin } from '../../utils.js'
export type httpMethod =
// '' is a rare case of error
'' | 'GET' | 'POST' | 'PUT' | 'DELETE' | 'HEAD' | 'CONNECT' | 'OPTIONS' | 'TRACE' | 'PATCH'
export enum RequestState {
UNSENT = 0,
OPENED = 1,
HEADERS_RECEIVED = 2,
LOADING = 3,
DONE = 4,
}
import {
RequestResponseData,
INetworkMessage,
httpMethod,
RequestState,
} from './types'
/**
* I know we're not using most of the information from this class
* but it can be useful in the future if we will decide to display more stuff in our ui
@ -30,9 +20,9 @@ export default class NetworkMessage {
readyState?: RequestState = 0
header: { [key: string]: string } = {}
responseType: XMLHttpRequest['responseType'] = ''
requestType: 'xhr' | 'fetch' | 'ping' | 'custom' | 'beacon'
requestType: 'xhr' | 'fetch' | 'ping' | 'custom' | 'beacon' | 'graphql' = 'xhr'
requestHeader: HeadersInit = {}
response: any
response: string
responseSize = 0 // bytes
responseSizeText = ''
startTime = 0
@ -47,7 +37,7 @@ export default class NetworkMessage {
private readonly sanitize: (data: RequestResponseData) => RequestResponseData | null,
) {}
getMessage() {
getMessage(): INetworkMessage | null {
const { reqHs, resHs } = this.writeHeaders()
const request = {
headers: reqHs,
@ -63,19 +53,26 @@ export default class NetworkMessage {
response,
})
if (!messageInfo) return
if (!messageInfo) return null;
return NetworkRequest(
this.requestType,
messageInfo.method,
messageInfo.url,
JSON.stringify(messageInfo.request),
JSON.stringify(messageInfo.response),
messageInfo.status,
this.startTime + getTimeOrigin(),
this.duration,
this.responseSize,
)
const isGraphql = messageInfo.url.includes("/graphql");
if (isGraphql && messageInfo.response.body && typeof messageInfo.response.body === 'string') {
const isError = messageInfo.response.body.includes("errors");
messageInfo.status = isError ? 400 : 200;
this.requestType = 'graphql';
}
return {
requestType: this.requestType,
method: messageInfo.method as httpMethod,
url: messageInfo.url,
request: JSON.stringify(messageInfo.request),
response: JSON.stringify(messageInfo.response),
status: messageInfo.status,
startTime: this.startTime,
duration: this.duration,
responseSize: this.responseSize,
}
}
writeHeaders() {

39
networkProxy/src/types.ts Normal file
View file

@ -0,0 +1,39 @@
export interface RequestResponseData {
status: number
readonly method: string
url: string
request: {
body: string | null
headers: Record<string, string>
}
response: {
body: string | null
headers: Record<string, string>
}
}
export interface INetworkMessage {
requestType: 'xhr' | 'fetch' | 'ping' | 'custom' | 'beacon' | 'graphql',
method: httpMethod,
url: string,
/** stringified JSON { headers: {}, body: {} } */
request: string,
/** stringified JSON { headers: {}, body: {} } */
response: string,
status: number,
startTime: number,
duration: number,
responseSize: number,
}
export type httpMethod =
// '' is a rare case of error
'' | 'GET' | 'POST' | 'PUT' | 'DELETE' | 'HEAD' | 'CONNECT' | 'OPTIONS' | 'TRACE' | 'PATCH'
export enum RequestState {
UNSENT = 0,
OPENED = 1,
HEADERS_RECEIVED = 2,
LOADING = 3,
DONE = 4,
}

View file

@ -6,10 +6,9 @@
* in not-so-hacky way
* */
import NetworkMessage, { RequestState } from './networkMessage.js'
import { genGetDataByUrl, formatByteSize, genStringBody, getStringResponseByType } from './utils.js'
import { RequestResponseData } from './types.js'
import { NetworkRequest } from '../../../common/messages.gen.js'
import NetworkMessage from './networkMessage'
import { RequestState, INetworkMessage, RequestResponseData } from './types';
import { genGetDataByUrl, formatByteSize, genStringBody, getStringResponseByType } from './utils'
export class XHRProxyHandler<T extends XMLHttpRequest> implements ProxyHandler<T> {
public XMLReq: XMLHttpRequest
@ -20,7 +19,7 @@ export class XHRProxyHandler<T extends XMLHttpRequest> implements ProxyHandler<T
private readonly ignoredHeaders: boolean | string[],
private readonly setSessionTokenHeader: (cb: (name: string, value: string) => void) => void,
private readonly sanitize: (data: RequestResponseData) => RequestResponseData | null,
private readonly sendMessage: (message: NetworkRequest) => void,
private readonly sendMessage: (message: INetworkMessage) => void,
private readonly isServiceUrl: (url: string) => boolean,
private readonly tokenUrlMatcher?: (url: string) => boolean,
) {
@ -239,7 +238,7 @@ export default class XHRProxy {
ignoredHeaders: boolean | string[],
setSessionTokenHeader: (cb: (name: string, value: string) => void) => void,
sanitize: (data: RequestResponseData) => RequestResponseData | null,
sendMessage: (data: NetworkRequest) => void,
sendMessage: (data: INetworkMessage) => void,
isServiceUrl: (url: string) => boolean,
tokenUrlMatcher?: (url: string) => boolean,
) {

View file

@ -0,0 +1,14 @@
{
"compilerOptions": {
"target": "ES2017",
"module": "ES2022",
"declaration": true,
"outDir": "./dist",
"strict": false,
"esModuleInterop": true,
"moduleResolution": "node",
"sourceMap": true,
"forceConsistentCasingInFileNames": true
},
"include": ["src/**/*"]
}

BIN
spot/bun.lockb Executable file

Binary file not shown.

View file

@ -1,13 +1,4 @@
import {
createSpotNetworkRequest,
stopTrackingNetwork,
startTrackingNetwork,
SpotNetworkRequest,
rawRequests,
} from "../utils/networkTracking";
import {
isTokenExpired
} from '../utils/jwt';
import { isTokenExpired } from "~/utils/jwt";
let checkBusy = false;
@ -65,6 +56,7 @@ export default defineBackground(() => {
injected: {
from: {
bumpLogs: "ort:bump-logs",
bumpNetwork: "ort:bump-network",
},
},
offscreen: {
@ -218,7 +210,7 @@ export default defineBackground(() => {
setJWTToken("");
}
const { jwtToken, settings } = data;
const refreshUrl = `${safeApiUrl(settings.ingestPoint)}/api/spot/refresh`
const refreshUrl = `${safeApiUrl(settings.ingestPoint)}/api/spot/refresh`;
if (!isTokenExpired(jwtToken) || !jwtToken) {
if (refreshInt) {
clearInterval(refreshInt);
@ -305,12 +297,12 @@ export default defineBackground(() => {
void browser.runtime.sendMessage({
type: messages.popup.to.noLogin,
});
checkBusy = false
checkBusy = false;
return;
}
const ok = await refreshToken();
if (ok) {
setJWTToken(data.jwtToken)
setJWTToken(data.jwtToken);
if (!refreshInt) {
refreshInt = setInterval(() => {
void refreshToken();
@ -322,7 +314,7 @@ export default defineBackground(() => {
}, PING_INT);
}
}
checkBusy = false
checkBusy = false;
}
// @ts-ignore
browser.runtime.onMessage.addListener((request, sender, respond) => {
@ -408,6 +400,9 @@ export default defineBackground(() => {
settings.networkLogs,
settings.consoleLogs,
() => recordingState.recording,
(hook) => {
onStop = hook;
},
);
// @ts-ignore this is false positive
respond(true);
@ -579,6 +574,10 @@ export default defineBackground(() => {
finalSpotObj.logs.push(...request.logs);
return "pong";
}
if (request.type === messages.injected.from.bumpNetwork) {
finalSpotObj.network.push(request.event);
return "pong";
}
if (request.type === messages.content.from.bumpClicks) {
finalSpotObj.clicks.push(...request.clicks);
return "pong";
@ -739,25 +738,7 @@ export default defineBackground(() => {
});
}
if (request.type === messages.content.from.saveSpotData) {
stopTrackingNetwork();
const finalNetwork: SpotNetworkRequest[] = [];
const tab =
recordingState.area === "tab" ? recordingState.activeTabId : undefined;
let lastIn = 0;
try {
rawRequests.forEach((r, i) => {
lastIn = i;
const spotNetworkRequest = createSpotNetworkRequest(r, tab);
if (spotNetworkRequest) {
finalNetwork.push(spotNetworkRequest);
}
});
} catch (e) {
console.error("cant parse network", e, rawRequests[lastIn]);
}
Object.assign(finalSpotObj, request.spot, {
network: finalNetwork,
});
Object.assign(finalSpotObj, request.spot);
return "pong";
}
if (request.type === messages.content.from.saveSpotVidChunk) {
@ -897,7 +878,7 @@ export default defineBackground(() => {
url: `${link}/view-spot/${id}`,
active: settings.openInNewTab,
});
}, 250)
}, 250);
const blob = base64ToBlob(videoData);
const mPromise = fetch(mobURL, {
@ -974,7 +955,7 @@ export default defineBackground(() => {
async function initializeOffscreenDocument() {
const existingContexts = await browser.runtime.getContexts({
contextTypes: ['OFFSCREEN_DOCUMENT'],
contextTypes: ["OFFSCREEN_DOCUMENT"],
});
const offscreenDocument = existingContexts.find(
@ -997,7 +978,7 @@ export default defineBackground(() => {
justification: "Recording from chrome.tabCapture API",
});
} catch (e) {
console.log('cant create new offscreen document', e)
console.error("cant create new offscreen document", e);
}
return;
@ -1025,7 +1006,12 @@ export default defineBackground(() => {
if (contentArmy[sendTo]) {
await browser.tabs.sendMessage(sendTo, message);
} else {
console.error("Content script might not be ready in tab", sendTo);
console.trace(
"Content script might not be ready in tab",
sendTo,
contentArmy,
message,
);
await browser.tabs.sendMessage(sendTo, message);
}
}
@ -1063,7 +1049,7 @@ export default defineBackground(() => {
withNetwork: boolean,
withConsole: boolean,
getRecState: () => string,
setOnStop?: (hook: any) => void,
setOnStop: (hook: any) => void,
) {
let activeTabs = await browser.tabs.query({
active: true,
@ -1108,17 +1094,16 @@ export default defineBackground(() => {
slackChannels,
activeTabId,
withConsole,
withNetwork,
state: "recording",
// by default this is already handled by :start event
// that triggers mount with countdown
shouldMount: false,
};
void sendToActiveTab(mountMsg);
if (withNetwork) {
startTrackingNetwork();
}
let previousTab: number | null = usedTab ?? null;
/** moves ui to active tab when screen recording */
function tabActivatedListener({ tabId }: { tabId: number }) {
const state = getRecState();
if (state === REC_STATE.stopped) {
@ -1147,14 +1132,17 @@ export default defineBackground(() => {
previousTab = tabId;
}
}
/** moves ui to active tab when screen recording */
function startTabActivationListening() {
browser.tabs.onActivated.addListener(tabActivatedListener);
}
/** moves ui to active tab when screen recording */
function stopTabActivationListening() {
browser.tabs.onActivated.removeListener(tabActivatedListener);
}
const trackedTab: number | null = usedTab ?? null;
/** reloads ui on currently active tab once its reloads itself */
function tabUpdateListener(tabId: number, changeInfo: any) {
const state = getRecState();
if (state === REC_STATE.stopped) {
@ -1186,6 +1174,7 @@ export default defineBackground(() => {
});
}
/** discards recording if was recording single tab and its now closed */
function tabRemovedListener(tabId: number) {
if (tabId === trackedTab) {
void browser.runtime.sendMessage({
@ -1221,12 +1210,14 @@ export default defineBackground(() => {
startTabListening();
if (area === "desktop") {
// if desktop, watch for tab change events
startTabActivationListening();
}
if (area === "tab") {
// if tab, watch for tab remove changes to discard recording
startRemovedListening();
}
setOnStop?.(() => {
setOnStop(() => {
stopTabListening();
if (area === "desktop") {
stopTabActivationListening();

View file

@ -1,4 +1,4 @@
import Countdown from "@/entrypoints/content/Countdown";
import Countdown from "~/entrypoints/content/Countdown";
import "~/assets/main.css";
import "./style.css";
import { createSignal } from "solid-js";

View file

@ -1,6 +1,6 @@
import { createSignal, onCleanup, createEffect } from "solid-js";
import { STATES, formatMsToTime } from "@/entrypoints/content/utils";
import micOn from "@/assets/mic-on.svg";
import { STATES, formatMsToTime } from "~/entrypoints/content/utils";
import micOn from "~/assets/mic-on.svg";
import { createDraggable } from "@neodrag/solid";
interface IRControls {
@ -128,7 +128,7 @@ function RecordingControls({
handleRef.classList.remove("popupanimated");
}, 250);
const audioPerm = getAudioPerm()
const audioPerm = getAudioPerm();
return (
<div
class={"rec-controls popupanimated cursor-grab"}
@ -202,7 +202,13 @@ function RecordingControls({
class={`btn btn-sm btn-circle btn-ghost tooltip tooltip-top flex items-center ${
mic() ? "bg-black/20" : "bg-black"
}`}
data-tip={audioPerm > 0 ? mic() ? "Switch Off Mic" : "Switch On Mic" : "Microphone disabled"}
data-tip={
audioPerm > 0
? mic()
? "Switch Off Mic"
: "Switch On Mic"
: "Microphone disabled"
}
onClick={audioPerm > 0 ? toggleMic : undefined}
>
{mic() ? (

View file

@ -1,7 +1,7 @@
// noinspection SpellCheckingInspection
import { createSignal, onCleanup, createEffect } from "solid-js";
import { formatMsToTime } from "@/entrypoints/content/utils";
import { formatMsToTime } from "~/entrypoints/content/utils";
import "./style.css";
import "./dragControls.css";
@ -152,7 +152,7 @@ function SavingControls({
const trim =
bounds[0] + bounds[1] === 0
? null
: [Math.floor(bounds[0] * 1000), Math.ceil(bounds[1] * 1000)]
: [Math.floor(bounds[0] * 1000), Math.ceil(bounds[1] * 1000)];
const dataObj = {
blob: videoBlob(),
name: name(),

View file

@ -5,7 +5,7 @@ import {
startClickRecording,
stopClickRecording,
} from "./eventTrackers";
import ControlsBox from "@/entrypoints/content/ControlsBox";
import ControlsBox from "~/entrypoints/content/ControlsBox";
import { convertBlobToBase64, getChromeFullVersion } from "./utils";
import "./style.css";
@ -253,20 +253,41 @@ export default defineContentScript({
logs: event.data.logs,
});
}
if (event.data.type === "ort:bump-network") {
void chrome.runtime.sendMessage({
type: "ort:bump-network",
event: event.data.event,
});
}
});
function startConsoleTracking() {
let injected = false;
function injectScript() {
if (injected) return;
injected = true;
const scriptEl = document.createElement("script");
scriptEl.src = browser.runtime.getURL("/injected.js");
document.head.appendChild(scriptEl);
}
function startConsoleTracking() {
injectScript()
setTimeout(() => {
window.postMessage({ type: "injected:start" });
window.postMessage({ type: "injected:c-start" });
}, 100);
}
function startNetworkTracking() {
injectScript()
setTimeout(() => {
window.postMessage({ type: "injected:n-start" });
}, 100)
}
function stopConsoleTracking() {
window.postMessage({ type: "injected:stop" });
window.postMessage({ type: "injected:c-stop" });
}
function stopNetworkTracking() {
window.postMessage({ type: "injected:n-stop" });
}
function onRestart() {
@ -323,7 +344,12 @@ export default defineContentScript({
micResponse = null;
startClickRecording();
startLocationRecording();
startConsoleTracking();
if (message.withConsole) {
startConsoleTracking();
}
if (message.withNetwork) {
startNetworkTracking();
}
browser.runtime.sendMessage({ type: "ort:started" });
if (message.shouldMount) {
ui.mount();
@ -343,6 +369,7 @@ export default defineContentScript({
stopClickRecording();
stopLocationRecording();
stopConsoleTracking();
stopNetworkTracking();
recState = "stopped";
ui.remove();
return "unmounted";

View file

@ -1,153 +0,0 @@
export default defineUnlistedScript(() => {
const printError =
"InstallTrigger" in window // detect Firefox
? (e) => e.message + "\n" + e.stack
: (e) => e.stack || e.message;
function printString(arg) {
if (arg === undefined) {
return "undefined";
}
if (arg === null) {
return "null";
}
if (arg instanceof Error) {
return printError(arg);
}
if (Array.isArray(arg)) {
return `Array(${arg.length})`;
}
return String(arg);
}
function printFloat(arg) {
if (typeof arg !== "number") return "NaN";
return arg.toString();
}
function printInt(arg) {
if (typeof arg !== "number") return "NaN";
return Math.floor(arg).toString();
}
function printObject(arg) {
if (arg === undefined) {
return "undefined";
}
if (arg === null) {
return "null";
}
if (arg instanceof Error) {
return printError(arg);
}
if (Array.isArray(arg)) {
const length = arg.length;
const values = arg.slice(0, 10).map(printString).join(", ");
return `Array(${length})[${values}]`;
}
if (typeof arg === "object") {
const res = [];
let i = 0;
for (const k in arg) {
if (++i === 10) {
break;
}
const v = arg[k];
res.push(k + ": " + printString(v));
}
return "{" + res.join(", ") + "}";
}
return arg.toString();
}
function printf(args) {
if (typeof args[0] === "string") {
args.unshift(
args.shift().replace(/%(o|s|f|d|i)/g, (s, t) => {
const arg = args.shift();
if (arg === undefined) return s;
switch (t) {
case "o":
return printObject(arg);
case "s":
return printString(arg);
case "f":
return printFloat(arg);
case "d":
case "i":
return printInt(arg);
default:
return s;
}
}),
);
}
return args.map(printObject).join(" ");
}
const consoleMethods = ["log", "info", "warn", "error", "debug", "assert"];
const patchConsole = (console, ctx) => {
if (window.revokeSpotPatch || window.__or_proxy_revocable) {
return;
}
let n = 0;
const reset = () => {
n = 0;
};
let int = setInterval(reset, 1000);
const sendConsoleLog = (level, args) => {
const msg = printf(args);
const truncated =
msg.length > 5000 ? `Truncated: ${msg.slice(0, 5000)}...` : msg;
const logs = [{ level, msg: truncated, time: Date.now() }];
window.postMessage({ type: "ort:bump-logs", logs }, "*");
};
const handler = (level) => ({
apply: function (target, thisArg, argumentsList) {
Reflect.apply(target, ctx, argumentsList);
n = n + 1;
if (n > 10) {
return;
} else {
sendConsoleLog(level, argumentsList); // Pass the correct level
}
},
});
window.__or_proxy_revocable = [];
consoleMethods.forEach((method) => {
if (consoleMethods.indexOf(method) === -1) {
return;
}
const fn = ctx.console[method];
// is there any way to preserve the original console trace?
const revProxy = Proxy.revocable(fn, handler(method));
console[method] = revProxy.proxy;
window.__or_proxy_revocable.push(revProxy);
});
return () => {
clearInterval(int);
window.__or_proxy_revocable.forEach((revocable) => {
revocable.revoke();
});
};
};
window.addEventListener("message", (event) => {
if (event.data.type === "injected:start") {
if (!window.__or_revokeSpotPatch) {
window.__or_revokeSpotPatch = patchConsole(console, window);
}
}
if (event.data.type === "injected:stop") {
if (window.__or_revokeSpotPatch) {
window.__or_revokeSpotPatch();
window.__or_revokeSpotPatch = null;
}
}
});
});

View file

@ -0,0 +1,24 @@
import { startNetwork, stopNetwork } from "~/utils/proxyNetworkTracking";
import { patchConsole } from "~/utils/consoleTracking";
export default defineUnlistedScript(() => {
window.addEventListener("message", (event) => {
if (event.data.type === "injected:c-start") {
if (!window.__or_revokeSpotPatch) {
window.__or_revokeSpotPatch = patchConsole(console, window);
}
}
if (event.data.type === "injected:n-start") {
startNetwork();
}
if (event.data.type === "injected:n-stop") {
stopNetwork();
}
if (event.data.type === "injected:c-stop") {
if (window.__or_revokeSpotPatch) {
window.__or_revokeSpotPatch();
window.__or_revokeSpotPatch = null;
}
}
});
});

View file

@ -1,11 +1,11 @@
import orLogo from "@/assets/orSpot.svg";
import micOff from "@/assets/mic-off-red.svg";
import micOn from "@/assets/mic-on-dark.svg";
import Login from "@/entrypoints/popup/Login";
import Settings from "@/entrypoints/popup/Settings";
import orLogo from "~/assets/orSpot.svg";
import micOff from "~/assets/mic-off-red.svg";
import micOn from "~/assets/mic-on-dark.svg";
import Login from "~/entrypoints/popup/Login";
import Settings from "~/entrypoints/popup/Settings";
import { createSignal, createEffect, onMount } from "solid-js";
import Dropdown from "@/entrypoints/popup/Dropdown";
import Button from "@/entrypoints/popup/Button";
import Dropdown from "~/entrypoints/popup/Dropdown";
import Button from "~/entrypoints/popup/Button";
import {
ChevronSvg,
RecordDesktopSvg,

View file

@ -1,5 +1,3 @@
import Button from "@/entrypoints/popup/Button";
function Login() {
const onOpenLoginPage = async () => {
const { settings } = await chrome.storage.local.get("settings");
@ -11,8 +9,20 @@ function Login() {
};
return (
<div class={"flex flex-row gap-2"}>
<button onClick={onOpenLoginPage} name={"Login"} class="btn btn-primary text-white shadow-sm text-lg w-2/4 ">Login</button>
<button onClick={onOpenSignupPage} name={"Create Account"} class="btn btn-primary btn-outline bg-white shadow-sm text-lg w-2/4 ">Create Account</button>
<button
onClick={onOpenLoginPage}
name={"Login"}
class="btn btn-primary text-white shadow-sm text-lg w-2/4 "
>
Login
</button>
<button
onClick={onOpenSignupPage}
name={"Create Account"}
class="btn btn-primary btn-outline bg-white shadow-sm text-lg w-2/4 "
>
Create Account
</button>
</div>
);
}
@ -46,4 +56,4 @@ function openLoginPage(instanceUrl: string) {
});
}
export default Login;
export default Login;

View file

@ -1,6 +1,6 @@
import { createSignal, onMount } from "solid-js";
import orLogo from "@/assets/orSpot.svg";
import arrowLeft from "@/assets/arrow-left.svg";
import orLogo from "~/assets/orSpot.svg";
import arrowLeft from "~/assets/arrow-left.svg";
function Settings({ goBack }: { goBack: () => void }) {
const [includeDevTools, setIncludeDevTools] = createSignal(true);
@ -13,7 +13,8 @@ function Settings({ goBack }: { goBack: () => void }) {
onMount(() => {
chrome.storage.local.get("settings", (data: any) => {
if (data.settings) {
const ingest = data.settings.ingestPoint || "https://app.openreplay.com";
const ingest =
data.settings.ingestPoint || "https://app.openreplay.com";
const devToolsEnabled =
data.settings.consoleLogs && data.settings.networkLogs;
setOpenInNewTab(data.settings.openInNewTab ?? false);
@ -89,7 +90,8 @@ function Settings({ goBack }: { goBack: () => void }) {
return (
<div class={"flex flex-col"}>
<div class={"flex gap-2 items-center justify-between p-4"}>
<button class="btn btn-xs btn-circle bg-white hover:bg-indigo-50"
<button
class="btn btn-xs btn-circle bg-white hover:bg-indigo-50"
onClick={goBack}
>
<img src={arrowLeft} alt={"Go back"} />
@ -106,21 +108,20 @@ function Settings({ goBack }: { goBack: () => void }) {
<div class="flex flex-col">
<div class="p-4 border-b border-slate-300 hover:bg-indigo-50">
<div class="flex flex-row justify-between items-center">
<p class="font-semibold mb-1 flex items-center">
View Recording
</p>
<p class="font-semibold mb-1 flex items-center">View Recording</p>
<label class="label cursor-pointer pr-0">
<input
type="checkbox"
class="toggle toggle-primary toggle-sm cursor-pointer"
checked={openInNewTab()}
onChange={toggleOpenInNewTab}
/>
</label>
<input
type="checkbox"
class="toggle toggle-primary toggle-sm cursor-pointer"
checked={openInNewTab()}
onChange={toggleOpenInNewTab}
/>
</label>
</div>
<p class="text-xs">Take me to newly created Spot tab after saving a recording.</p>
<p class="text-xs">
Take me to newly created Spot tab after saving a recording.
</p>
</div>
<div class="flex flex-col border-b border-slate-300 cursor-default justify-between p-4 hover:bg-indigo-50">
@ -140,7 +141,8 @@ function Settings({ goBack }: { goBack: () => void }) {
</div>
</div>
<p class="text-xs">
Include console logs, network calls and other useful debugging information for developers.
Include console logs, network calls and other useful debugging
information for developers.
</p>
</div>
@ -159,7 +161,8 @@ function Settings({ goBack }: { goBack: () => void }) {
</div>
</div>
<p class="text-xs">
Set this URL if you are self-hosting OpenReplay so it points to your instance.
Set this URL if you are self-hosting OpenReplay so it points to your
instance.
</p>
{showIngest() && (
@ -191,16 +194,16 @@ function Settings({ goBack }: { goBack: () => void }) {
</div>
</div>
) : (
<div class={"flex items-center gap-2"}>
<span class={"text-gray-700"}>{ingest()}</span>
<button
class="btn btn-sm btn-link font-normal no-underline hover:no-underline hover:opacity-75"
onClick={() => toggleEditIngest(true)}
>
Edit
</button>
</div>
)}
<div class={"flex items-center gap-2"}>
<span class={"text-gray-700"}>{ingest()}</span>
<button
class="btn btn-sm btn-link font-normal no-underline hover:no-underline hover:opacity-75"
onClick={() => toggleEditIngest(true)}
>
Edit
</button>
</div>
)}
</div>
)}
</div>

View file

@ -2,7 +2,7 @@
"name": "wxt-starter",
"description": "manifest.json description",
"private": true,
"version": "1.0.8",
"version": "1.0.9",
"type": "module",
"scripts": {
"dev": "wxt",
@ -17,6 +17,7 @@
},
"dependencies": {
"@neodrag/solid": "^2.0.4",
"@openreplay/network-proxy": "^1.0.3",
"@thedutchcoder/postcss-rem-to-px": "^0.0.2",
"autoprefixer": "^10.4.19",
"install": "^0.13.0",
@ -31,6 +32,6 @@
"@wxt-dev/module-solid": "^1.1.2",
"daisyui": "^4.12.10",
"typescript": "^5.4.5",
"wxt": "0.19.9"
"wxt": "0.19.10"
}
}

View file

@ -1,5 +1,14 @@
{
"extends": "./.wxt/tsconfig.json",
"paths": {
"~": [".."],
"~/*": ["../*"],
"@@": [".."],
"@@/*": ["../*"],
"~~": [".."],
"~~/*": ["../*"]
},
"lib": ["es2022", "DOM"],
"compilerOptions": {
"jsx": "preserve",
"jsxImportSource": "solid-js"

View file

@ -0,0 +1,142 @@
function printString(arg) {
const printError =
"InstallTrigger" in window // detect Firefox
? (e) => e.message + "\n" + e.stack
: (e) => e.stack || e.message;
if (arg === undefined) {
return "undefined";
}
if (arg === null) {
return "null";
}
if (arg instanceof Error) {
return printError(arg);
}
if (Array.isArray(arg)) {
return `Array(${arg.length})`;
}
return String(arg);
}
function printFloat(arg) {
if (typeof arg !== "number") return "NaN";
return arg.toString();
}
function printInt(arg) {
if (typeof arg !== "number") return "NaN";
return Math.floor(arg).toString();
}
function printObject(arg) {
const printError =
"InstallTrigger" in window // detect Firefox
? (e) => e.message + "\n" + e.stack
: (e) => e.stack || e.message;
if (arg === undefined) {
return "undefined";
}
if (arg === null) {
return "null";
}
if (arg instanceof Error) {
return printError(arg);
}
if (Array.isArray(arg)) {
const length = arg.length;
const values = arg.slice(0, 10).map(printString).join(", ");
return `Array(${length})[${values}]`;
}
if (typeof arg === "object") {
const res = [];
let i = 0;
for (const k in arg) {
if (++i === 10) {
break;
}
const v = arg[k];
res.push(k + ": " + printString(v));
}
return "{" + res.join(", ") + "}";
}
return arg.toString();
}
function printf(args) {
if (typeof args[0] === "string") {
args.unshift(
args.shift().replace(/%(o|s|f|d|i)/g, (s, t) => {
const arg = args.shift();
if (arg === undefined) return s;
switch (t) {
case "o":
return printObject(arg);
case "s":
return printString(arg);
case "f":
return printFloat(arg);
case "d":
case "i":
return printInt(arg);
default:
return s;
}
}),
);
}
return args.map(printObject).join(" ");
}
const consoleMethods = ["log", "info", "warn", "error", "debug", "assert"];
export const patchConsole = (console, ctx) => {
if (window.revokeSpotPatch || window.__or_proxy_revocable) {
return;
}
let n = 0;
const reset = () => {
n = 0;
};
let int = setInterval(reset, 1000);
const sendConsoleLog = (level, args) => {
const msg = printf(args);
const truncated =
msg.length > 5000 ? `Truncated: ${msg.slice(0, 5000)}...` : msg;
const logs = [{ level, msg: truncated, time: Date.now() }];
window.postMessage({ type: "ort:bump-logs", logs }, "*");
};
const handler = (level) => ({
apply: function (target, thisArg, argumentsList) {
Reflect.apply(target, ctx, argumentsList);
n = n + 1;
if (n > 10) {
return;
} else {
sendConsoleLog(level, argumentsList); // Pass the correct level
}
},
});
window.__or_proxy_revocable = [];
consoleMethods.forEach((method) => {
if (consoleMethods.indexOf(method) === -1) {
return;
}
const fn = ctx.console[method];
// is there any way to preserve the original console trace?
const revProxy = Proxy.revocable(fn, handler(method));
console[method] = revProxy.proxy;
window.__or_proxy_revocable.push(revProxy);
});
return () => {
clearInterval(int);
window.__or_proxy_revocable.forEach((revocable) => {
revocable.revoke();
});
};
};

View file

@ -1,305 +1,169 @@
import { WebRequest } from "webextension-polyfill";
export type TrackedRequest = {
statusCode: number;
requestHeaders: Record<string, string>;
responseHeaders: Record<string, string>;
} & (
| WebRequest.OnBeforeRequestDetailsType
| WebRequest.OnBeforeSendHeadersDetailsType
| WebRequest.OnCompletedDetailsType
| WebRequest.OnErrorOccurredDetailsType
| WebRequest.OnResponseStartedDetailsType
);
export interface SpotNetworkRequest {
encodedBodySize: number;
responseBodySize: number;
duration: number;
method: TrackedRequest["method"];
type: string;
time: TrackedRequest["timeStamp"];
statusCode: number;
error?: string;
url: TrackedRequest["url"];
fromCache: boolean;
body: string;
requestHeaders: Record<string, string>;
responseHeaders: Record<string, string>;
}
export const rawRequests: (TrackedRequest & {
startTs: number;
duration: number;
})[] = [];
const sensitiveParams = new Set([
"password",
"pass",
"pwd",
"mdp",
"token",
"bearer",
"jwt",
"api_key",
"api-key",
"apiKey",
"key",
"secret",
"id",
"user",
"userId",
"email",
"ssn",
"name",
"firstname",
"lastname",
"birthdate",
"dob",
"address",
"zip",
"zipcode",
"x-api-key",
"www-authenticate",
"x-csrf-token",
"x-requested-with",
"x-forwarded-for",
"x-real-ip",
"cookie",
"authorization",
"auth",
"proxy-authorization",
"set-cookie",
"account_key",
]);
function filterHeaders(headers: Record<string, string>) {
const filteredHeaders: Record<string, string> = {};
if (Array.isArray(headers)) {
headers.forEach(({ name, value }) => {
if (sensitiveParams.has(name.toLowerCase())) {
filteredHeaders[name] = "******";
} else {
filteredHeaders[name] = value;
}
});
} else {
for (const [key, value] of Object.entries(headers)) {
if (sensitiveParams.has(key.toLowerCase())) {
filteredHeaders[key] = "******";
} else {
filteredHeaders[key] = value;
}
}
}
return filteredHeaders;
}
// JSON or form data
function filterBody(body: any) {
if (!body) {
return body;
}
let parsedBody;
let isJSON = false;
try {
parsedBody = JSON.parse(body);
isJSON = true;
} catch (e) {
// not json
}
if (isJSON) {
obscureSensitiveData(parsedBody);
return JSON.stringify(parsedBody);
} else {
const params = new URLSearchParams(body);
for (const key of params.keys()) {
if (sensitiveParams.has(key.toLowerCase())) {
params.set(key, "******");
}
}
return params.toString();
}
}
function obscureSensitiveData(obj: Record<string, any> | any[]) {
if (Array.isArray(obj)) {
obj.forEach(obscureSensitiveData);
} else if (obj && typeof obj === "object") {
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
if (sensitiveParams.has(key.toLowerCase())) {
obj[key] = "******";
} else if (obj[key] !== null && typeof obj[key] === "object") {
obscureSensitiveData(obj[key]);
}
}
}
}
}
function tryFilterUrl(url: string) {
if (!url) return "";
try {
const urlObj = new URL(url);
if (urlObj.searchParams) {
for (const key of urlObj.searchParams.keys()) {
if (sensitiveParams.has(key.toLowerCase())) {
urlObj.searchParams.set(key, "******");
}
}
}
return urlObj.toString();
} catch (e) {
return url;
}
}
export function createSpotNetworkRequest(
trackedRequest: TrackedRequest,
trackedTab?: number,
) {
if (trackedRequest.tabId === -1) {
return;
}
if (trackedTab && trackedTab !== trackedRequest.tabId) {
return;
}
if (
["ping", "beacon", "image", "script", "font"].includes(trackedRequest.type)
) {
if (!trackedRequest.statusCode || trackedRequest.statusCode < 400) {
return;
}
}
const type = ["stylesheet", "script", "image", "media", "font"].includes(
trackedRequest.type,
)
? "resource"
: trackedRequest.type;
const requestHeaders = trackedRequest.requestHeaders
? filterHeaders(trackedRequest.requestHeaders)
: {};
const responseHeaders = trackedRequest.responseHeaders
? filterHeaders(trackedRequest.responseHeaders)
: {};
const reqSize = trackedRequest.reqBody
? trackedRequest.requestSize || trackedRequest.reqBody.length
: 0;
const status = getRequestStatus(trackedRequest);
let body;
if (trackedRequest.reqBody) {
try {
body = filterBody(trackedRequest.reqBody);
} catch (e) {
body = "Error parsing body";
console.error(e);
}
} else {
body = "";
}
const request: SpotNetworkRequest = {
method: trackedRequest.method,
type,
body,
requestHeaders,
responseHeaders,
time: trackedRequest.timeStamp,
statusCode: status,
error: trackedRequest.error,
url: tryFilterUrl(trackedRequest.url),
fromCache: trackedRequest.fromCache || false,
encodedBodySize: reqSize,
responseBodySize: trackedRequest.responseSize,
duration: trackedRequest.duration,
};
return request;
}
function modifyOnSpot(request: TrackedRequest) {
const id = request.requestId;
const index = rawRequests.findIndex((r) => r.requestId === id);
const ts = Date.now();
const start = rawRequests[index]?.startTs ?? ts;
rawRequests[index] = {
...rawRequests[index],
...request,
duration: ts - start,
};
}
const trackOnBefore = (
details: WebRequest.OnBeforeRequestDetailsType & { reqBody: string },
) => {
if (details.method === "POST" && details.requestBody) {
const requestBody = details.requestBody;
if (requestBody.formData) {
details.reqBody = JSON.stringify(requestBody.formData);
} else if (requestBody.raw) {
const raw = requestBody.raw[0]?.bytes;
if (raw) {
details.reqBody = new TextDecoder("utf-8").decode(raw);
}
}
}
rawRequests.push({ ...details, startTs: Date.now(), duration: 0 });
};
const trackOnCompleted = (details: WebRequest.OnCompletedDetailsType) => {
modifyOnSpot(details);
};
const trackOnHeaders = (details: WebRequest.OnBeforeSendHeadersDetailsType) => {
modifyOnSpot(details);
};
const trackOnError = (details: WebRequest.OnErrorOccurredDetailsType) => {
modifyOnSpot(details);
};
export function startTrackingNetwork() {
rawRequests.length = 0;
browser.webRequest.onBeforeRequest.addListener(
// @ts-ignore
trackOnBefore,
{ urls: ["<all_urls>"] },
["requestBody"],
);
browser.webRequest.onBeforeSendHeaders.addListener(
trackOnHeaders,
{ urls: ["<all_urls>"] },
["requestHeaders"],
);
browser.webRequest.onCompleted.addListener(
trackOnCompleted,
{
urls: ["<all_urls>"],
},
["responseHeaders"],
);
browser.webRequest.onErrorOccurred.addListener(
trackOnError,
{
urls: ["<all_urls>"],
},
["extraHeaders"],
);
}
export function stopTrackingNetwork() {
browser.webRequest.onBeforeRequest.removeListener(trackOnBefore);
browser.webRequest.onCompleted.removeListener(trackOnCompleted);
browser.webRequest.onErrorOccurred.removeListener(trackOnError);
}
function getRequestStatus(request: any): number {
if (request.statusCode) {
return request.statusCode;
}
if (request.error) {
return 0;
}
return 200;
}
// import {
// SpotNetworkRequest,
// filterBody,
// filterHeaders,
// tryFilterUrl,
// TrackedRequest,
// } from "./networkTrackingUtils";
//
// export const rawRequests: (TrackedRequest & {
// startTs: number;
// duration: number;
// })[] = [];
//
// export function createSpotNetworkRequestV1(
// trackedRequest: TrackedRequest,
// trackedTab?: number,
// ) {
// if (trackedRequest.tabId === -1) {
// return;
// }
// if (trackedTab && trackedTab !== trackedRequest.tabId) {
// return;
// }
// if (
// ["ping", "beacon", "image", "script", "font"].includes(trackedRequest.type)
// ) {
// if (!trackedRequest.statusCode || trackedRequest.statusCode < 400) {
// return;
// }
// }
// const type = ["stylesheet", "script", "image", "media", "font"].includes(
// trackedRequest.type,
// )
// ? "resource"
// : trackedRequest.type;
//
// const requestHeaders = trackedRequest.requestHeaders
// ? filterHeaders(trackedRequest.requestHeaders)
// : {};
// const responseHeaders = trackedRequest.responseHeaders
// ? filterHeaders(trackedRequest.responseHeaders)
// : {};
//
// const reqSize = trackedRequest.reqBody
// ? trackedRequest.requestSize || trackedRequest.reqBody.length
// : 0;
//
// const status = getRequestStatus(trackedRequest);
// let body;
// if (trackedRequest.reqBody) {
// try {
// body = filterBody(trackedRequest.reqBody);
// } catch (e) {
// body = "Error parsing body";
// console.error(e);
// }
// } else {
// body = "";
// }
// const request: SpotNetworkRequest = {
// method: trackedRequest.method,
// type,
// body,
// responseBody: "",
// requestHeaders,
// responseHeaders,
// time: trackedRequest.timeStamp,
// statusCode: status,
// error: trackedRequest.error,
// url: tryFilterUrl(trackedRequest.url),
// fromCache: trackedRequest.fromCache || false,
// encodedBodySize: reqSize,
// responseBodySize: trackedRequest.responseSize,
// duration: trackedRequest.duration,
// };
//
// return request;
// }
//
// function modifyOnSpot(request: TrackedRequest) {
// const id = request.requestId;
// const index = rawRequests.findIndex((r) => r.requestId === id);
// const ts = Date.now();
// const start = rawRequests[index]?.startTs ?? ts;
// rawRequests[index] = {
// ...rawRequests[index],
// ...request,
// duration: ts - start,
// };
// }
//
// const trackOnBefore = (
// details: WebRequest.OnBeforeRequestDetailsType & { reqBody: string },
// ) => {
// if (details.method === "POST" && details.requestBody) {
// const requestBody = details.requestBody;
// if (requestBody.formData) {
// details.reqBody = JSON.stringify(requestBody.formData);
// } else if (requestBody.raw) {
// const raw = requestBody.raw[0]?.bytes;
// if (raw) {
// details.reqBody = new TextDecoder("utf-8").decode(raw);
// }
// }
// }
// rawRequests.push({ ...details, startTs: Date.now(), duration: 0 });
// };
// const trackOnCompleted = (details: WebRequest.OnCompletedDetailsType) => {
// modifyOnSpot(details);
// };
// const trackOnHeaders = (details: WebRequest.OnBeforeSendHeadersDetailsType) => {
// modifyOnSpot(details);
// };
// const trackOnError = (details: WebRequest.OnErrorOccurredDetailsType) => {
// modifyOnSpot(details);
// };
// export function startTrackingNetwork() {
// rawRequests.length = 0;
// browser.webRequest.onBeforeRequest.addListener(
// // @ts-ignore
// trackOnBefore,
// { urls: ["<all_urls>"] },
// ["requestBody"],
// );
// browser.webRequest.onBeforeSendHeaders.addListener(
// trackOnHeaders,
// { urls: ["<all_urls>"] },
// ["requestHeaders"],
// );
// browser.webRequest.onCompleted.addListener(
// trackOnCompleted,
// {
// urls: ["<all_urls>"],
// },
// ["responseHeaders"],
// );
// browser.webRequest.onErrorOccurred.addListener(
// trackOnError,
// {
// urls: ["<all_urls>"],
// },
// ["extraHeaders"],
// );
// }
//
// export function stopTrackingNetwork() {
// browser.webRequest.onBeforeRequest.removeListener(trackOnBefore);
// browser.webRequest.onCompleted.removeListener(trackOnCompleted);
// browser.webRequest.onErrorOccurred.removeListener(trackOnError);
// }
//
// function getRequestStatus(request: any): number {
// if (request.statusCode) {
// return request.statusCode;
// }
// if (request.error) {
// return 0;
// }
// return 200;
// }
//
// export function getFinalRequests(tabId: number): SpotNetworkRequest[] {
// const finalRequests = rawRequests
// .map((r) => createSpotNetworkRequest(r, tabId))
// .filter((r) => r !== undefined);
// rawRequests.length = 0;
//
// return finalRequests;
// }

View file

@ -0,0 +1,168 @@
import { WebRequest } from "webextension-polyfill";
export type TrackedRequest = {
statusCode: number;
requestHeaders: Record<string, string>;
responseHeaders: Record<string, string>;
} & (
| WebRequest.OnBeforeRequestDetailsType
| WebRequest.OnBeforeSendHeadersDetailsType
| WebRequest.OnCompletedDetailsType
| WebRequest.OnErrorOccurredDetailsType
| WebRequest.OnResponseStartedDetailsType
);
export function getTopWindow(): Window {
let currentWindow = window;
try {
while (currentWindow !== currentWindow.parent) {
currentWindow = currentWindow.parent;
}
} catch (e) {
// Accessing currentWindow.parent threw an exception due to cross-origin policy
// currentWindow is the topmost accessible window
}
return currentWindow;
}
export interface SpotNetworkRequest {
encodedBodySize: number;
responseBodySize: number;
duration: number;
method: TrackedRequest["method"];
type: string;
time: TrackedRequest["timeStamp"];
statusCode: number;
error?: string;
url: TrackedRequest["url"];
fromCache: boolean;
body: string;
responseBody: string;
requestHeaders: Record<string, string>;
responseHeaders: Record<string, string>;
}
export const sensitiveParams = new Set([
"password",
"pass",
"pwd",
"mdp",
"token",
"bearer",
"jwt",
"api_key",
"api-key",
"apiKey",
"key",
"secret",
"id",
"user",
"userId",
"email",
"ssn",
"name",
"firstname",
"lastname",
"birthdate",
"dob",
"address",
"zip",
"zipcode",
"x-api-key",
"www-authenticate",
"x-csrf-token",
"x-requested-with",
"x-forwarded-for",
"x-real-ip",
"cookie",
"authorization",
"auth",
"proxy-authorization",
"set-cookie",
"account_key",
]);
export function filterHeaders(headers: Record<string, string> | { name: string; value: string }[]) {
const filteredHeaders: Record<string, string> = {};
if (Array.isArray(headers)) {
headers.forEach(({ name, value }) => {
if (sensitiveParams.has(name.toLowerCase())) {
filteredHeaders[name] = "******";
} else {
filteredHeaders[name] = value;
}
});
} else {
for (const [key, value] of Object.entries(headers)) {
if (sensitiveParams.has(key.toLowerCase())) {
filteredHeaders[key] = "******";
} else {
filteredHeaders[key] = value;
}
}
}
return filteredHeaders;
}
// JSON or form data
export function filterBody(body: any): string {
if (!body) {
return body;
}
let parsedBody;
let isJSON = false;
try {
parsedBody = JSON.parse(body);
isJSON = true;
} catch (e) {
// not json
}
if (isJSON) {
obscureSensitiveData(parsedBody);
return JSON.stringify(parsedBody);
} else {
const params = new URLSearchParams(body);
for (const key of params.keys()) {
if (sensitiveParams.has(key.toLowerCase())) {
params.set(key, "******");
}
}
return params.toString();
}
}
export function obscureSensitiveData(obj: Record<string, any> | any[]) {
if (Array.isArray(obj)) {
obj.forEach(obscureSensitiveData);
} else if (obj && typeof obj === "object") {
for (const key in obj) {
if (Object.hasOwn(obj, key)) {
if (sensitiveParams.has(key.toLowerCase())) {
obj[key] = "******";
} else if (obj[key] !== null && typeof obj[key] === "object") {
obscureSensitiveData(obj[key]);
}
}
}
}
}
export function tryFilterUrl(url: string) {
if (!url) return "";
try {
const urlObj = new URL(url);
if (urlObj.searchParams) {
for (const key of urlObj.searchParams.keys()) {
if (sensitiveParams.has(key.toLowerCase())) {
urlObj.searchParams.set(key, "******");
}
}
}
return urlObj.toString();
} catch (e) {
return url;
}
}

View file

@ -0,0 +1,101 @@
import createNetworkProxy, { INetworkMessage } from "@openreplay/network-proxy";
import {
SpotNetworkRequest,
filterBody,
filterHeaders,
tryFilterUrl,
getTopWindow,
} from "./networkTrackingUtils";
let defaultFetch: typeof fetch | undefined;
let defaultXhr: typeof XMLHttpRequest | undefined;
let defaultBeacon: typeof navigator.sendBeacon | undefined;
export function startNetwork() {
const context = getTopWindow();
defaultXhr = context.XMLHttpRequest;
defaultBeacon = context.navigator.sendBeacon;
defaultFetch = context.fetch;
createNetworkProxy(
context,
[], // headers
() => null,
(reqRes) => reqRes,
(msg) => {
const event = createSpotNetworkRequest(msg);
window.postMessage({ type: "ort:bump-network", event }, "*");
},
(url) =>
url.includes("/spot/") || url.includes(".mob?") || url.includes(".mobe?"),
{ xhr: true, fetch: true, beacon: true },
);
}
function getBody(req: { body?: string | Record<string, any> }): string {
let body;
if (req.body) {
try {
body = filterBody(req.body);
} catch (e) {
body = "Error parsing body";
console.error(e);
}
} else {
body = "";
}
return body;
}
export function createSpotNetworkRequest(
msg: INetworkMessage,
): SpotNetworkRequest {
let request: Record<string, any> = {}
let response: Record<string, any> = {};
try {
request = JSON.parse(msg.request);
} catch (e) {
console.error("Error parsing request", e);
}
try {
response = JSON.parse(msg.response);
} catch (e) {
console.error("Error parsing response", e);
}
const reqHeaders = request.headers ? filterHeaders(request.headers) : {};
const resHeaders = response.headers ? filterHeaders(response.headers) : {};
const responseBodySize = msg.responseSize || 0;
const reqSize = msg.request ? msg.request.length : 0;
const body = getBody(request);
const responseBody = getBody(response);
return {
method: msg.method,
type: msg.requestType,
body,
responseBody,
requestHeaders: reqHeaders,
responseHeaders: resHeaders,
time: msg.startTime,
statusCode: msg.status || 0,
error: undefined,
url: tryFilterUrl(msg.url),
fromCache: false,
encodedBodySize: reqSize,
responseBodySize,
duration: msg.duration,
};
}
export function stopNetwork() {
if (defaultFetch) {
window.fetch = defaultFetch;
}
if (defaultXhr) {
window.XMLHttpRequest = defaultXhr;
}
if (defaultBeacon) {
window.navigator.sendBeacon = defaultBeacon;
}
}

View file

@ -707,6 +707,11 @@
proc-log "^4.0.0"
which "^4.0.0"
"@openreplay/network-proxy@^1.0.3":
version "1.0.3"
resolved "https://registry.yarnpkg.com/@openreplay/network-proxy/-/network-proxy-1.0.3.tgz#82c07f0742db01456f3764bae6c0b5ff280557a9"
integrity sha512-vm/x8Ioo1BKJIyZf58tK44CgtzHA0tIwu2uZ2WdIzrBOODF+A2qV4mELC8Zdp9WqV1O7uxXGaA4J38HU3th14g==
"@pkgjs/parseargs@^0.11.0":
version "0.11.0"
resolved "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz"
@ -5331,10 +5336,10 @@ ws@8.18.0:
resolved "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz"
integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
wxt@0.19.9:
version "0.19.9"
resolved "https://registry.yarnpkg.com/wxt/-/wxt-0.19.9.tgz#b8f7f838cab00d66f4ee22f483c49ad8f6527af8"
integrity sha512-XUbF4JNyx2jTDpXwx2c/esaJcUD2Dr482C2GGenkGRMH2UnerzOIchGCtaa1hb2U8eAed7Akda0yRoMJU0uxUw==
wxt@0.19.10:
version "0.19.10"
resolved "https://registry.yarnpkg.com/wxt/-/wxt-0.19.10.tgz#557d57e63ab5fcf3b026791aae3706c400b5f7cb"
integrity sha512-lX/dzAaau79SDsU7QZKUgxJUf9nBsaQXMiqsDNgGoqZO1wrRpFq0kijcN3/mNjGbXM989VJhEl7B6f3ttxKAnQ==
dependencies:
"@aklinker1/rollup-plugin-visualizer" "5.12.0"
"@types/chrome" "^0.0.269"
@ -5350,6 +5355,7 @@ wxt@0.19.9:
consola "^3.2.3"
defu "^6.1.4"
dequal "^2.0.3"
dotenv "^16.4.5"
esbuild "^0.23.0"
execa "^9.3.1"
fast-glob "^3.3.2"
@ -5371,6 +5377,7 @@ wxt@0.19.9:
ohash "^1.1.3"
open "^10.1.0"
ora "^8.1.0"
perfect-debounce "^1.0.0"
picocolors "^1.0.1"
prompts "^2.4.2"
publish-browser-extension "^2.1.3"

View file

@ -1,3 +1,7 @@
# 14.0.8
- use separate library to handle network requests ([@openreplay/network-proxy](https://www.npmjs.com/package/@openreplay/network-proxy))
# 14.0.7
- check for stopping status during restarts

Binary file not shown.

View file

@ -1,7 +1,7 @@
{
"name": "@openreplay/tracker",
"description": "The OpenReplay tracker main package",
"version": "14.0.7",
"version": "14.0.8",
"keywords": [
"logging",
"replay"
@ -50,6 +50,7 @@
},
"dependencies": {
"@medv/finder": "^3.2.0",
"@openreplay/network-proxy": "^1.0.2",
"error-stack-parser": "^2.0.6",
"fflate": "^0.8.2"
},

View file

@ -1,53 +0,0 @@
import FetchProxy from './fetchProxy.js'
import XHRProxy from './xhrProxy.js'
import BeaconProxy from './beaconProxy.js'
import { RequestResponseData } from './types.js'
import { NetworkRequest } from '../../../common/messages.gen.js'
const getWarning = (api: string) =>
console.warn(`Openreplay: Can't find ${api} in global context.
If you're using serverside rendering in your app, make sure that tracker is loaded dynamically, otherwise ${api} won't be tracked.`)
export default function setProxy(
context: typeof globalThis,
ignoredHeaders: boolean | string[],
setSessionTokenHeader: (cb: (name: string, value: string) => void) => void,
sanitize: (data: RequestResponseData) => RequestResponseData | null,
sendMessage: (message: NetworkRequest) => void,
isServiceUrl: (url: string) => boolean,
tokenUrlMatcher?: (url: string) => boolean,
) {
if (context.XMLHttpRequest) {
context.XMLHttpRequest = XHRProxy.create(
ignoredHeaders,
setSessionTokenHeader,
sanitize,
sendMessage,
isServiceUrl,
tokenUrlMatcher,
)
} else {
getWarning('XMLHttpRequest')
}
if (context.fetch) {
context.fetch = FetchProxy.create(
ignoredHeaders,
setSessionTokenHeader,
sanitize,
sendMessage,
isServiceUrl,
tokenUrlMatcher,
)
} else {
getWarning('fetch')
}
if (context?.navigator?.sendBeacon) {
context.navigator.sendBeacon = BeaconProxy.create(
ignoredHeaders,
setSessionTokenHeader,
sanitize,
sendMessage,
isServiceUrl,
)
}
}

View file

@ -1,15 +0,0 @@
export interface RequestResponseData {
readonly status: number
readonly method: string
url: string
request: {
body: string | null
headers: Record<string, string>
}
response: {
body: string | null
headers: Record<string, string>
}
}
// we only support sanitizing for json/string data because how you're gonna sanitize binary data?

View file

@ -3,7 +3,7 @@ import { NetworkRequest } from '../app/messages.gen.js'
import { getTimeOrigin } from '../utils.js'
import type { AxiosInstance } from './axiosSpy.js'
import axiosSpy from './axiosSpy.js'
import setProxy from './Network/index.js'
import createNetworkProxy from '@openreplay/network-proxy'
type WindowFetch = typeof window.fetch
type XHRRequestBody = Parameters<XMLHttpRequest['send']>[0]
@ -130,13 +130,28 @@ export default function (app: App, opts: Partial<Options> = {}) {
const patchWindow = (context: typeof globalThis) => {
/* ====== modern way ====== */
if (options.useProxy) {
return setProxy(
return createNetworkProxy(
context,
options.ignoreHeaders,
setSessionTokenHeader,
sanitize,
(message) => app.send(message),
(message) => {
app.send(
NetworkRequest(
message.requestType,
message.method,
message.url,
message.request,
message.response,
message.status,
message.startTime + getTimeOrigin(),
message.duration,
message.responseSize,
),
)
},
(url) => app.isServiceURL(url),
{ xhr: true, fetch: true, beacon: true },
options.tokenUrlMatcher,
)
}

View file

@ -9,8 +9,8 @@
"alwaysStrict": true,
"target": "es2020",
"module": "es6",
"moduleResolution": "nodenext",
"esModuleInterop": true
"moduleResolution": "node",
"esModuleInterop": true,
},
"exclude": ["**/*.test.ts"]
}