diff --git a/frontend/app/components/Spots/SpotPlayer/spotPlayerStore.ts b/frontend/app/components/Spots/SpotPlayer/spotPlayerStore.ts index 45e7ac7c2..d5ceb9c65 100644 --- a/frontend/app/components/Spots/SpotPlayer/spotPlayerStore.ts +++ b/frontend/app/components/Spots/SpotPlayer/spotPlayerStore.ts @@ -43,6 +43,8 @@ const mapSpotNetworkToEv = (ev: SpotNetworkRequest): any => { return 'xhr'; case 'fetch': return 'fetch'; + case 'graphql': + return 'graphql'; case 'resource': return 'resource'; default: @@ -56,7 +58,7 @@ const mapSpotNetworkToEv = (ev: SpotNetworkRequest): any => { }) const response = JSON.stringify({ headers: ev.responseHeaders, - body: { warn: "Chrome Manifest V3 -- No response body available in Chrome 93+" } + body: ev.responseBody ?? { warn: "Chrome Manifest V3 -- No response body available in Chrome 93+" } }) return ({ ...ev, diff --git a/frontend/app/player/web/types/resource.ts b/frontend/app/player/web/types/resource.ts index d4520d8aa..8344e0c33 100644 --- a/frontend/app/player/web/types/resource.ts +++ b/frontend/app/player/web/types/resource.ts @@ -10,6 +10,7 @@ export const enum ResourceType { IMG = 'img', MEDIA = 'media', WS = 'websocket', + GRAPHQL = 'graphql', OTHER = 'other', } @@ -47,6 +48,8 @@ export function getResourceType(initiator: string, url: string): ResourceType { case "avi": case "mp3": return ResourceType.MEDIA + case "graphql": + return ResourceType.GRAPHQL default: return ResourceType.OTHER } diff --git a/networkProxy/.gitignore b/networkProxy/.gitignore new file mode 100644 index 000000000..a4c70b46e --- /dev/null +++ b/networkProxy/.gitignore @@ -0,0 +1,31 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +.output +stats.html +stats-*.json +.wxt +web-ext.config.ts + +!public + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +dist/* +dist \ No newline at end of file diff --git a/networkProxy/LICENSE b/networkProxy/LICENSE new file mode 100644 index 000000000..4933f212f --- /dev/null +++ b/networkProxy/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2024 Asayer, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/networkProxy/README.md b/networkProxy/README.md new file mode 100644 index 000000000..3b267f787 --- /dev/null +++ b/networkProxy/README.md @@ -0,0 +1,46 @@ +this tiny library helps us (OpenReplay folks) to create proxy objects for fetch, +XHR and beacons for proper request tracking in @openreplay/tracker and Spot extension. + +example usage: +``` +import createNetworkProxy from '@openreplay/network-proxy'; + +const context = this; +const ignoreHeaders = ['Authorization']; +const tokenUrlMatcher = /\/auth\/token/; +function setSessionTokenHeader(setRequestHeader: (name: string, value: string) => void) { + const header = 'X-Session-Token + const sessionToken = getToken() // for exmaple, => `session #123123`; + if (sessionToken) { + setRequestHeader(header, sessionToken) + } +} +function sanitize(reqResInfo) { + if (reqResInfo.request) { + delete reqResInfo.request.body + } + return reqResInfo +} + +const onMsg = (networkReq) => console.log(networkReq) +const isIgnoredUrl = (url) => url.includes('google.com') + +// Gets current tracker request’s url and returns boolean. If present, +// sessionTokenHeader will only be applied when this function returns true. +// Default: undefined +const tokenUrlMatcher = (url) => url.includes('google.com'); + +// this will observe global network requests +createNetworkProxy( + context, + options.ignoreHeaders, + setSessionTokenHeader, + sanitize, + (message) => app.send(message), + (url) => app.isServiceURL(url), + options.tokenUrlMatcher, +) + +// to stop it, you can save this.fetch/other apis before appliying the proxy +// and then restore them +``` diff --git a/networkProxy/package-lock.json b/networkProxy/package-lock.json new file mode 100644 index 000000000..5e3c8ae4e --- /dev/null +++ b/networkProxy/package-lock.json @@ -0,0 +1,29 @@ +{ + "name": "network-proxy", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "network-proxy", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "typescript": "^5.6.2" + } + }, + "node_modules/typescript": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/networkProxy/package.json b/networkProxy/package.json new file mode 100644 index 000000000..032d91af9 --- /dev/null +++ b/networkProxy/package.json @@ -0,0 +1,19 @@ +{ + "name": "@openreplay/network-proxy", + "version": "1.0.3", + "description": "this library helps us to create proxy objects for fetch, XHR and beacons for proper request tracking.", + "main": "dist/index.js", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "scripts": { + "build": "tsc" + }, + "author": "Nikita ", + "license": "MIT", + "devDependencies": { + "typescript": "^5.6.2" + } +} diff --git a/tracker/tracker/src/main/modules/Network/beaconProxy.ts b/networkProxy/src/beaconProxy.ts similarity index 89% rename from tracker/tracker/src/main/modules/Network/beaconProxy.ts rename to networkProxy/src/beaconProxy.ts index 2433bd950..11eac6e33 100644 --- a/tracker/tracker/src/main/modules/Network/beaconProxy.ts +++ b/networkProxy/src/beaconProxy.ts @@ -1,7 +1,6 @@ -import { NetworkRequest } from '../../../common/messages.gen.js' -import NetworkMessage from './networkMessage.js' -import { RequestResponseData } from './types.js' -import { genStringBody, getURL } from './utils.js' +import NetworkMessage from './networkMessage' +import { RequestState, INetworkMessage, RequestResponseData } from './types'; +import { genStringBody, getURL } from './utils' // https://fetch.spec.whatwg.org/#concept-bodyinit-extract const getContentType = (data?: BodyInit) => { @@ -22,7 +21,7 @@ export class BeaconProxyHandler implement private readonly ignoredHeaders: boolean | string[], private readonly setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, private readonly sanitize: (data: RequestResponseData) => RequestResponseData | null, - private readonly sendMessage: (item: NetworkRequest) => void, + private readonly sendMessage: (item: INetworkMessage) => void, private readonly isServiceUrl: (url: string) => boolean, ) {} @@ -85,7 +84,7 @@ export default class BeaconProxy { ignoredHeaders: boolean | string[], setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, sanitize: (data: RequestResponseData) => RequestResponseData | null, - sendMessage: (item: NetworkRequest) => void, + sendMessage: (item: INetworkMessage) => void, isServiceUrl: (url: string) => boolean, ) { if (!BeaconProxy.hasSendBeacon()) { diff --git a/tracker/tracker/src/main/modules/Network/fetchProxy.ts b/networkProxy/src/fetchProxy.ts similarity index 97% rename from tracker/tracker/src/main/modules/Network/fetchProxy.ts rename to networkProxy/src/fetchProxy.ts index 22d3b5163..483efb46b 100644 --- a/tracker/tracker/src/main/modules/Network/fetchProxy.ts +++ b/networkProxy/src/fetchProxy.ts @@ -5,10 +5,9 @@ * we can intercept the network requests * in not-so-hacky way * */ -import NetworkMessage, { RequestState } from './networkMessage.js' -import { formatByteSize, genStringBody, getStringResponseByType, getURL } from './utils.js' -import { RequestResponseData } from './types.js' -import { NetworkRequest } from '../../../common/messages.gen.js' +import NetworkMessage from './networkMessage' +import { RequestState, INetworkMessage, RequestResponseData } from './types'; +import { formatByteSize, genStringBody, getStringResponseByType, getURL } from './utils' export class ResponseProxyHandler implements ProxyHandler { public resp: Response @@ -123,7 +122,7 @@ export class FetchProxyHandler implements ProxyHandler void) => void, private readonly sanitize: (data: RequestResponseData) => RequestResponseData | null, - private readonly sendMessage: (item: NetworkRequest) => void, + private readonly sendMessage: (item: INetworkMessage) => void, private readonly isServiceUrl: (url: string) => boolean, private readonly tokenUrlMatcher?: (url: string) => boolean, ) {} @@ -311,7 +310,7 @@ export default class FetchProxy { ignoredHeaders: boolean | string[], setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, sanitize: (data: RequestResponseData) => RequestResponseData | null, - sendMessage: (item: NetworkRequest) => void, + sendMessage: (item: INetworkMessage) => void, isServiceUrl: (url: string) => boolean, tokenUrlMatcher?: (url: string) => boolean, ) { diff --git a/networkProxy/src/index.ts b/networkProxy/src/index.ts new file mode 100644 index 000000000..f5cd0ca68 --- /dev/null +++ b/networkProxy/src/index.ts @@ -0,0 +1,96 @@ +import BeaconProxy from "./beaconProxy"; +import FetchProxy from "./fetchProxy"; +import XHRProxy from "./xhrProxy"; +import { INetworkMessage, RequestResponseData } from "./types"; + +export { + BeaconProxy, + FetchProxy, + XHRProxy, + INetworkMessage, + RequestResponseData, +}; + +const getWarning = (api: string) => { + const str = `Openreplay: Can't find ${api} in global context.`; + console.warn(str); +}; + +/** + * Creates network proxies for XMLHttpRequest, fetch, and sendBeacon to intercept and monitor network requests and + * responses. + * + * @param {Window | typeof globalThis} context - The global context object (e.g., window or globalThis). + * @param {boolean | string[]} ignoredHeaders - Headers to ignore from requests. If `true`, all headers are ignored; if + * an array of strings, those header names are ignored. + * @param {(cb: (name: string, value: string) => void) => void} setSessionTokenHeader - Function to set a session token + * header; accepts a callback that sets the header name and value. + * @param {(data: RequestResponseData) => RequestResponseData | null} sanitize - Function to sanitize request and + * response data; should return sanitized data or `null` to ignore the data. + * @param {(message: INetworkMessage) => void} sendMessage - Function to send network messages for further processing + * or logging. + * @param {(url: string) => boolean} isServiceUrl - Function to determine if a URL is a service URL that should be + * ignored by the proxy. + * @param {Object} [modules] - Modules to apply the proxies to. + * @param {boolean} [modules.xhr=true] - Whether to proxy XMLHttpRequest. + * @param {boolean} [modules.fetch=true] - Whether to proxy the fetch API. + * @param {boolean} [modules.beacon=true] - Whether to proxy navigator.sendBeacon. + * @param {(url: string) => boolean} [tokenUrlMatcher] - Optional function; the session token header will only be + * applied to requests matching this function. + * + * @returns {void} + */ +export default function createNetworkProxy( + context: typeof globalThis, + ignoredHeaders: boolean | string[], + setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, + sanitize: (data: RequestResponseData) => RequestResponseData | null, + sendMessage: (message: INetworkMessage) => void, + isServiceUrl: (url: string) => boolean, + modules: { xhr: boolean; fetch: boolean; beacon: boolean } = { + xhr: true, + fetch: true, + beacon: true, + }, + tokenUrlMatcher?: (url: string) => boolean, +): void { + if (modules.xhr) { + if (context.XMLHttpRequest) { + context.XMLHttpRequest = XHRProxy.create( + ignoredHeaders, + setSessionTokenHeader, + sanitize, + sendMessage, + isServiceUrl, + tokenUrlMatcher, + ); + } else { + getWarning("XMLHttpRequest"); + } + } + if (modules.fetch) { + if (context.fetch) { + context.fetch = FetchProxy.create( + ignoredHeaders, + setSessionTokenHeader, + sanitize, + sendMessage, + isServiceUrl, + tokenUrlMatcher, + ); + } else { + getWarning("fetch"); + } + } + if (modules.beacon) { + if (context?.navigator?.sendBeacon) { + context.navigator.sendBeacon = BeaconProxy.create( + ignoredHeaders, + setSessionTokenHeader, + sanitize, + sendMessage, + isServiceUrl, + ); + } + } +} diff --git a/tracker/tracker/src/main/modules/Network/networkMessage.ts b/networkProxy/src/networkMessage.ts similarity index 68% rename from tracker/tracker/src/main/modules/Network/networkMessage.ts rename to networkProxy/src/networkMessage.ts index 7ef4da96a..b0dade2cc 100644 --- a/tracker/tracker/src/main/modules/Network/networkMessage.ts +++ b/networkProxy/src/networkMessage.ts @@ -1,19 +1,9 @@ -import { NetworkRequest } from '../../app/messages.gen.js' -import { RequestResponseData } from './types.js' -import { getTimeOrigin } from '../../utils.js' - -export type httpMethod = - // '' is a rare case of error - '' | 'GET' | 'POST' | 'PUT' | 'DELETE' | 'HEAD' | 'CONNECT' | 'OPTIONS' | 'TRACE' | 'PATCH' - -export enum RequestState { - UNSENT = 0, - OPENED = 1, - HEADERS_RECEIVED = 2, - LOADING = 3, - DONE = 4, -} - +import { + RequestResponseData, + INetworkMessage, + httpMethod, + RequestState, +} from './types' /** * I know we're not using most of the information from this class * but it can be useful in the future if we will decide to display more stuff in our ui @@ -30,9 +20,9 @@ export default class NetworkMessage { readyState?: RequestState = 0 header: { [key: string]: string } = {} responseType: XMLHttpRequest['responseType'] = '' - requestType: 'xhr' | 'fetch' | 'ping' | 'custom' | 'beacon' + requestType: 'xhr' | 'fetch' | 'ping' | 'custom' | 'beacon' | 'graphql' = 'xhr' requestHeader: HeadersInit = {} - response: any + response: string responseSize = 0 // bytes responseSizeText = '' startTime = 0 @@ -47,7 +37,7 @@ export default class NetworkMessage { private readonly sanitize: (data: RequestResponseData) => RequestResponseData | null, ) {} - getMessage() { + getMessage(): INetworkMessage | null { const { reqHs, resHs } = this.writeHeaders() const request = { headers: reqHs, @@ -63,19 +53,26 @@ export default class NetworkMessage { response, }) - if (!messageInfo) return + if (!messageInfo) return null; - return NetworkRequest( - this.requestType, - messageInfo.method, - messageInfo.url, - JSON.stringify(messageInfo.request), - JSON.stringify(messageInfo.response), - messageInfo.status, - this.startTime + getTimeOrigin(), - this.duration, - this.responseSize, - ) + const isGraphql = messageInfo.url.includes("/graphql"); + if (isGraphql && messageInfo.response.body && typeof messageInfo.response.body === 'string') { + const isError = messageInfo.response.body.includes("errors"); + messageInfo.status = isError ? 400 : 200; + this.requestType = 'graphql'; + } + + return { + requestType: this.requestType, + method: messageInfo.method as httpMethod, + url: messageInfo.url, + request: JSON.stringify(messageInfo.request), + response: JSON.stringify(messageInfo.response), + status: messageInfo.status, + startTime: this.startTime, + duration: this.duration, + responseSize: this.responseSize, + } } writeHeaders() { diff --git a/networkProxy/src/types.ts b/networkProxy/src/types.ts new file mode 100644 index 000000000..8e2ae038a --- /dev/null +++ b/networkProxy/src/types.ts @@ -0,0 +1,39 @@ +export interface RequestResponseData { + status: number + readonly method: string + url: string + request: { + body: string | null + headers: Record + } + response: { + body: string | null + headers: Record + } +} + +export interface INetworkMessage { + requestType: 'xhr' | 'fetch' | 'ping' | 'custom' | 'beacon' | 'graphql', + method: httpMethod, + url: string, + /** stringified JSON { headers: {}, body: {} } */ + request: string, + /** stringified JSON { headers: {}, body: {} } */ + response: string, + status: number, + startTime: number, + duration: number, + responseSize: number, +} + +export type httpMethod = + // '' is a rare case of error + '' | 'GET' | 'POST' | 'PUT' | 'DELETE' | 'HEAD' | 'CONNECT' | 'OPTIONS' | 'TRACE' | 'PATCH' + +export enum RequestState { + UNSENT = 0, + OPENED = 1, + HEADERS_RECEIVED = 2, + LOADING = 3, + DONE = 4, +} diff --git a/tracker/tracker/src/main/modules/Network/utils.ts b/networkProxy/src/utils.ts similarity index 100% rename from tracker/tracker/src/main/modules/Network/utils.ts rename to networkProxy/src/utils.ts diff --git a/tracker/tracker/src/main/modules/Network/xhrProxy.ts b/networkProxy/src/xhrProxy.ts similarity index 96% rename from tracker/tracker/src/main/modules/Network/xhrProxy.ts rename to networkProxy/src/xhrProxy.ts index f300b0a4a..a15806481 100644 --- a/tracker/tracker/src/main/modules/Network/xhrProxy.ts +++ b/networkProxy/src/xhrProxy.ts @@ -6,10 +6,9 @@ * in not-so-hacky way * */ -import NetworkMessage, { RequestState } from './networkMessage.js' -import { genGetDataByUrl, formatByteSize, genStringBody, getStringResponseByType } from './utils.js' -import { RequestResponseData } from './types.js' -import { NetworkRequest } from '../../../common/messages.gen.js' +import NetworkMessage from './networkMessage' +import { RequestState, INetworkMessage, RequestResponseData } from './types'; +import { genGetDataByUrl, formatByteSize, genStringBody, getStringResponseByType } from './utils' export class XHRProxyHandler implements ProxyHandler { public XMLReq: XMLHttpRequest @@ -20,7 +19,7 @@ export class XHRProxyHandler implements ProxyHandler void) => void, private readonly sanitize: (data: RequestResponseData) => RequestResponseData | null, - private readonly sendMessage: (message: NetworkRequest) => void, + private readonly sendMessage: (message: INetworkMessage) => void, private readonly isServiceUrl: (url: string) => boolean, private readonly tokenUrlMatcher?: (url: string) => boolean, ) { @@ -239,7 +238,7 @@ export default class XHRProxy { ignoredHeaders: boolean | string[], setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, sanitize: (data: RequestResponseData) => RequestResponseData | null, - sendMessage: (data: NetworkRequest) => void, + sendMessage: (data: INetworkMessage) => void, isServiceUrl: (url: string) => boolean, tokenUrlMatcher?: (url: string) => boolean, ) { diff --git a/networkProxy/tsconfig.json b/networkProxy/tsconfig.json new file mode 100644 index 000000000..6378fdbb1 --- /dev/null +++ b/networkProxy/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2017", + "module": "ES2022", + "declaration": true, + "outDir": "./dist", + "strict": false, + "esModuleInterop": true, + "moduleResolution": "node", + "sourceMap": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*"] +} diff --git a/spot/bun.lockb b/spot/bun.lockb new file mode 100755 index 000000000..0ff64f229 Binary files /dev/null and b/spot/bun.lockb differ diff --git a/spot/entrypoints/background.ts b/spot/entrypoints/background.ts index 58b7e4e48..9373275be 100644 --- a/spot/entrypoints/background.ts +++ b/spot/entrypoints/background.ts @@ -1,13 +1,4 @@ -import { - createSpotNetworkRequest, - stopTrackingNetwork, - startTrackingNetwork, - SpotNetworkRequest, - rawRequests, -} from "../utils/networkTracking"; -import { - isTokenExpired -} from '../utils/jwt'; +import { isTokenExpired } from "~/utils/jwt"; let checkBusy = false; @@ -65,6 +56,7 @@ export default defineBackground(() => { injected: { from: { bumpLogs: "ort:bump-logs", + bumpNetwork: "ort:bump-network", }, }, offscreen: { @@ -218,7 +210,7 @@ export default defineBackground(() => { setJWTToken(""); } const { jwtToken, settings } = data; - const refreshUrl = `${safeApiUrl(settings.ingestPoint)}/api/spot/refresh` + const refreshUrl = `${safeApiUrl(settings.ingestPoint)}/api/spot/refresh`; if (!isTokenExpired(jwtToken) || !jwtToken) { if (refreshInt) { clearInterval(refreshInt); @@ -305,12 +297,12 @@ export default defineBackground(() => { void browser.runtime.sendMessage({ type: messages.popup.to.noLogin, }); - checkBusy = false + checkBusy = false; return; } const ok = await refreshToken(); if (ok) { - setJWTToken(data.jwtToken) + setJWTToken(data.jwtToken); if (!refreshInt) { refreshInt = setInterval(() => { void refreshToken(); @@ -322,7 +314,7 @@ export default defineBackground(() => { }, PING_INT); } } - checkBusy = false + checkBusy = false; } // @ts-ignore browser.runtime.onMessage.addListener((request, sender, respond) => { @@ -408,6 +400,9 @@ export default defineBackground(() => { settings.networkLogs, settings.consoleLogs, () => recordingState.recording, + (hook) => { + onStop = hook; + }, ); // @ts-ignore this is false positive respond(true); @@ -579,6 +574,10 @@ export default defineBackground(() => { finalSpotObj.logs.push(...request.logs); return "pong"; } + if (request.type === messages.injected.from.bumpNetwork) { + finalSpotObj.network.push(request.event); + return "pong"; + } if (request.type === messages.content.from.bumpClicks) { finalSpotObj.clicks.push(...request.clicks); return "pong"; @@ -739,25 +738,7 @@ export default defineBackground(() => { }); } if (request.type === messages.content.from.saveSpotData) { - stopTrackingNetwork(); - const finalNetwork: SpotNetworkRequest[] = []; - const tab = - recordingState.area === "tab" ? recordingState.activeTabId : undefined; - let lastIn = 0; - try { - rawRequests.forEach((r, i) => { - lastIn = i; - const spotNetworkRequest = createSpotNetworkRequest(r, tab); - if (spotNetworkRequest) { - finalNetwork.push(spotNetworkRequest); - } - }); - } catch (e) { - console.error("cant parse network", e, rawRequests[lastIn]); - } - Object.assign(finalSpotObj, request.spot, { - network: finalNetwork, - }); + Object.assign(finalSpotObj, request.spot); return "pong"; } if (request.type === messages.content.from.saveSpotVidChunk) { @@ -897,7 +878,7 @@ export default defineBackground(() => { url: `${link}/view-spot/${id}`, active: settings.openInNewTab, }); - }, 250) + }, 250); const blob = base64ToBlob(videoData); const mPromise = fetch(mobURL, { @@ -974,7 +955,7 @@ export default defineBackground(() => { async function initializeOffscreenDocument() { const existingContexts = await browser.runtime.getContexts({ - contextTypes: ['OFFSCREEN_DOCUMENT'], + contextTypes: ["OFFSCREEN_DOCUMENT"], }); const offscreenDocument = existingContexts.find( @@ -997,7 +978,7 @@ export default defineBackground(() => { justification: "Recording from chrome.tabCapture API", }); } catch (e) { - console.log('cant create new offscreen document', e) + console.error("cant create new offscreen document", e); } return; @@ -1025,7 +1006,12 @@ export default defineBackground(() => { if (contentArmy[sendTo]) { await browser.tabs.sendMessage(sendTo, message); } else { - console.error("Content script might not be ready in tab", sendTo); + console.trace( + "Content script might not be ready in tab", + sendTo, + contentArmy, + message, + ); await browser.tabs.sendMessage(sendTo, message); } } @@ -1063,7 +1049,7 @@ export default defineBackground(() => { withNetwork: boolean, withConsole: boolean, getRecState: () => string, - setOnStop?: (hook: any) => void, + setOnStop: (hook: any) => void, ) { let activeTabs = await browser.tabs.query({ active: true, @@ -1108,17 +1094,16 @@ export default defineBackground(() => { slackChannels, activeTabId, withConsole, + withNetwork, state: "recording", // by default this is already handled by :start event // that triggers mount with countdown shouldMount: false, }; void sendToActiveTab(mountMsg); - if (withNetwork) { - startTrackingNetwork(); - } - let previousTab: number | null = usedTab ?? null; + + /** moves ui to active tab when screen recording */ function tabActivatedListener({ tabId }: { tabId: number }) { const state = getRecState(); if (state === REC_STATE.stopped) { @@ -1147,14 +1132,17 @@ export default defineBackground(() => { previousTab = tabId; } } + /** moves ui to active tab when screen recording */ function startTabActivationListening() { browser.tabs.onActivated.addListener(tabActivatedListener); } + /** moves ui to active tab when screen recording */ function stopTabActivationListening() { browser.tabs.onActivated.removeListener(tabActivatedListener); } const trackedTab: number | null = usedTab ?? null; + /** reloads ui on currently active tab once its reloads itself */ function tabUpdateListener(tabId: number, changeInfo: any) { const state = getRecState(); if (state === REC_STATE.stopped) { @@ -1186,6 +1174,7 @@ export default defineBackground(() => { }); } + /** discards recording if was recording single tab and its now closed */ function tabRemovedListener(tabId: number) { if (tabId === trackedTab) { void browser.runtime.sendMessage({ @@ -1221,12 +1210,14 @@ export default defineBackground(() => { startTabListening(); if (area === "desktop") { + // if desktop, watch for tab change events startTabActivationListening(); } if (area === "tab") { + // if tab, watch for tab remove changes to discard recording startRemovedListening(); } - setOnStop?.(() => { + setOnStop(() => { stopTabListening(); if (area === "desktop") { stopTabActivationListening(); diff --git a/spot/entrypoints/content/ControlsBox.tsx b/spot/entrypoints/content/ControlsBox.tsx index 38fa3a9de..f9e585245 100644 --- a/spot/entrypoints/content/ControlsBox.tsx +++ b/spot/entrypoints/content/ControlsBox.tsx @@ -1,4 +1,4 @@ -import Countdown from "@/entrypoints/content/Countdown"; +import Countdown from "~/entrypoints/content/Countdown"; import "~/assets/main.css"; import "./style.css"; import { createSignal } from "solid-js"; diff --git a/spot/entrypoints/content/RecordingControls.tsx b/spot/entrypoints/content/RecordingControls.tsx index 989111efd..ecbef8c85 100644 --- a/spot/entrypoints/content/RecordingControls.tsx +++ b/spot/entrypoints/content/RecordingControls.tsx @@ -1,6 +1,6 @@ import { createSignal, onCleanup, createEffect } from "solid-js"; -import { STATES, formatMsToTime } from "@/entrypoints/content/utils"; -import micOn from "@/assets/mic-on.svg"; +import { STATES, formatMsToTime } from "~/entrypoints/content/utils"; +import micOn from "~/assets/mic-on.svg"; import { createDraggable } from "@neodrag/solid"; interface IRControls { @@ -128,7 +128,7 @@ function RecordingControls({ handleRef.classList.remove("popupanimated"); }, 250); - const audioPerm = getAudioPerm() + const audioPerm = getAudioPerm(); return (
0 ? mic() ? "Switch Off Mic" : "Switch On Mic" : "Microphone disabled"} + data-tip={ + audioPerm > 0 + ? mic() + ? "Switch Off Mic" + : "Switch On Mic" + : "Microphone disabled" + } onClick={audioPerm > 0 ? toggleMic : undefined} > {mic() ? ( diff --git a/spot/entrypoints/content/SavingControls.tsx b/spot/entrypoints/content/SavingControls.tsx index 88e34df70..1576eaeb6 100644 --- a/spot/entrypoints/content/SavingControls.tsx +++ b/spot/entrypoints/content/SavingControls.tsx @@ -1,7 +1,7 @@ // noinspection SpellCheckingInspection import { createSignal, onCleanup, createEffect } from "solid-js"; -import { formatMsToTime } from "@/entrypoints/content/utils"; +import { formatMsToTime } from "~/entrypoints/content/utils"; import "./style.css"; import "./dragControls.css"; @@ -152,7 +152,7 @@ function SavingControls({ const trim = bounds[0] + bounds[1] === 0 ? null - : [Math.floor(bounds[0] * 1000), Math.ceil(bounds[1] * 1000)] + : [Math.floor(bounds[0] * 1000), Math.ceil(bounds[1] * 1000)]; const dataObj = { blob: videoBlob(), name: name(), diff --git a/spot/entrypoints/content/index.tsx b/spot/entrypoints/content/index.tsx index e4d824d5e..56a913776 100644 --- a/spot/entrypoints/content/index.tsx +++ b/spot/entrypoints/content/index.tsx @@ -5,7 +5,7 @@ import { startClickRecording, stopClickRecording, } from "./eventTrackers"; -import ControlsBox from "@/entrypoints/content/ControlsBox"; +import ControlsBox from "~/entrypoints/content/ControlsBox"; import { convertBlobToBase64, getChromeFullVersion } from "./utils"; import "./style.css"; @@ -253,20 +253,41 @@ export default defineContentScript({ logs: event.data.logs, }); } + if (event.data.type === "ort:bump-network") { + void chrome.runtime.sendMessage({ + type: "ort:bump-network", + event: event.data.event, + }); + } }); - function startConsoleTracking() { + let injected = false; + function injectScript() { + if (injected) return; + injected = true; const scriptEl = document.createElement("script"); scriptEl.src = browser.runtime.getURL("/injected.js"); document.head.appendChild(scriptEl); - + } + function startConsoleTracking() { + injectScript() setTimeout(() => { - window.postMessage({ type: "injected:start" }); + window.postMessage({ type: "injected:c-start" }); }, 100); } + function startNetworkTracking() { + injectScript() + setTimeout(() => { + window.postMessage({ type: "injected:n-start" }); + }, 100) + } function stopConsoleTracking() { - window.postMessage({ type: "injected:stop" }); + window.postMessage({ type: "injected:c-stop" }); + } + + function stopNetworkTracking() { + window.postMessage({ type: "injected:n-stop" }); } function onRestart() { @@ -323,7 +344,12 @@ export default defineContentScript({ micResponse = null; startClickRecording(); startLocationRecording(); - startConsoleTracking(); + if (message.withConsole) { + startConsoleTracking(); + } + if (message.withNetwork) { + startNetworkTracking(); + } browser.runtime.sendMessage({ type: "ort:started" }); if (message.shouldMount) { ui.mount(); @@ -343,6 +369,7 @@ export default defineContentScript({ stopClickRecording(); stopLocationRecording(); stopConsoleTracking(); + stopNetworkTracking(); recState = "stopped"; ui.remove(); return "unmounted"; diff --git a/spot/entrypoints/injected.js b/spot/entrypoints/injected.js deleted file mode 100644 index b810aa469..000000000 --- a/spot/entrypoints/injected.js +++ /dev/null @@ -1,153 +0,0 @@ -export default defineUnlistedScript(() => { - const printError = - "InstallTrigger" in window // detect Firefox - ? (e) => e.message + "\n" + e.stack - : (e) => e.stack || e.message; - - function printString(arg) { - if (arg === undefined) { - return "undefined"; - } - if (arg === null) { - return "null"; - } - if (arg instanceof Error) { - return printError(arg); - } - if (Array.isArray(arg)) { - return `Array(${arg.length})`; - } - return String(arg); - } - - function printFloat(arg) { - if (typeof arg !== "number") return "NaN"; - return arg.toString(); - } - - function printInt(arg) { - if (typeof arg !== "number") return "NaN"; - return Math.floor(arg).toString(); - } - - function printObject(arg) { - if (arg === undefined) { - return "undefined"; - } - if (arg === null) { - return "null"; - } - if (arg instanceof Error) { - return printError(arg); - } - if (Array.isArray(arg)) { - const length = arg.length; - const values = arg.slice(0, 10).map(printString).join(", "); - return `Array(${length})[${values}]`; - } - if (typeof arg === "object") { - const res = []; - let i = 0; - for (const k in arg) { - if (++i === 10) { - break; - } - const v = arg[k]; - res.push(k + ": " + printString(v)); - } - return "{" + res.join(", ") + "}"; - } - return arg.toString(); - } - - function printf(args) { - if (typeof args[0] === "string") { - args.unshift( - args.shift().replace(/%(o|s|f|d|i)/g, (s, t) => { - const arg = args.shift(); - if (arg === undefined) return s; - switch (t) { - case "o": - return printObject(arg); - case "s": - return printString(arg); - case "f": - return printFloat(arg); - case "d": - case "i": - return printInt(arg); - default: - return s; - } - }), - ); - } - return args.map(printObject).join(" "); - } - - const consoleMethods = ["log", "info", "warn", "error", "debug", "assert"]; - - const patchConsole = (console, ctx) => { - if (window.revokeSpotPatch || window.__or_proxy_revocable) { - return; - } - let n = 0; - const reset = () => { - n = 0; - }; - let int = setInterval(reset, 1000); - - const sendConsoleLog = (level, args) => { - const msg = printf(args); - const truncated = - msg.length > 5000 ? `Truncated: ${msg.slice(0, 5000)}...` : msg; - const logs = [{ level, msg: truncated, time: Date.now() }]; - window.postMessage({ type: "ort:bump-logs", logs }, "*"); - }; - - const handler = (level) => ({ - apply: function (target, thisArg, argumentsList) { - Reflect.apply(target, ctx, argumentsList); - n = n + 1; - if (n > 10) { - return; - } else { - sendConsoleLog(level, argumentsList); // Pass the correct level - } - }, - }); - - window.__or_proxy_revocable = []; - consoleMethods.forEach((method) => { - if (consoleMethods.indexOf(method) === -1) { - return; - } - const fn = ctx.console[method]; - // is there any way to preserve the original console trace? - const revProxy = Proxy.revocable(fn, handler(method)); - console[method] = revProxy.proxy; - window.__or_proxy_revocable.push(revProxy); - }); - - return () => { - clearInterval(int); - window.__or_proxy_revocable.forEach((revocable) => { - revocable.revoke(); - }); - }; - }; - - window.addEventListener("message", (event) => { - if (event.data.type === "injected:start") { - if (!window.__or_revokeSpotPatch) { - window.__or_revokeSpotPatch = patchConsole(console, window); - } - } - if (event.data.type === "injected:stop") { - if (window.__or_revokeSpotPatch) { - window.__or_revokeSpotPatch(); - window.__or_revokeSpotPatch = null; - } - } - }); -}); diff --git a/spot/entrypoints/injected.ts b/spot/entrypoints/injected.ts new file mode 100644 index 000000000..9ecebc13a --- /dev/null +++ b/spot/entrypoints/injected.ts @@ -0,0 +1,24 @@ +import { startNetwork, stopNetwork } from "~/utils/proxyNetworkTracking"; +import { patchConsole } from "~/utils/consoleTracking"; + +export default defineUnlistedScript(() => { + window.addEventListener("message", (event) => { + if (event.data.type === "injected:c-start") { + if (!window.__or_revokeSpotPatch) { + window.__or_revokeSpotPatch = patchConsole(console, window); + } + } + if (event.data.type === "injected:n-start") { + startNetwork(); + } + if (event.data.type === "injected:n-stop") { + stopNetwork(); + } + if (event.data.type === "injected:c-stop") { + if (window.__or_revokeSpotPatch) { + window.__or_revokeSpotPatch(); + window.__or_revokeSpotPatch = null; + } + } + }); +}); diff --git a/spot/entrypoints/popup/App.tsx b/spot/entrypoints/popup/App.tsx index 196ebebfd..3d9314f76 100644 --- a/spot/entrypoints/popup/App.tsx +++ b/spot/entrypoints/popup/App.tsx @@ -1,11 +1,11 @@ -import orLogo from "@/assets/orSpot.svg"; -import micOff from "@/assets/mic-off-red.svg"; -import micOn from "@/assets/mic-on-dark.svg"; -import Login from "@/entrypoints/popup/Login"; -import Settings from "@/entrypoints/popup/Settings"; +import orLogo from "~/assets/orSpot.svg"; +import micOff from "~/assets/mic-off-red.svg"; +import micOn from "~/assets/mic-on-dark.svg"; +import Login from "~/entrypoints/popup/Login"; +import Settings from "~/entrypoints/popup/Settings"; import { createSignal, createEffect, onMount } from "solid-js"; -import Dropdown from "@/entrypoints/popup/Dropdown"; -import Button from "@/entrypoints/popup/Button"; +import Dropdown from "~/entrypoints/popup/Dropdown"; +import Button from "~/entrypoints/popup/Button"; import { ChevronSvg, RecordDesktopSvg, diff --git a/spot/entrypoints/popup/Login.tsx b/spot/entrypoints/popup/Login.tsx index 846ae6fc8..33d2a2d85 100644 --- a/spot/entrypoints/popup/Login.tsx +++ b/spot/entrypoints/popup/Login.tsx @@ -1,5 +1,3 @@ -import Button from "@/entrypoints/popup/Button"; - function Login() { const onOpenLoginPage = async () => { const { settings } = await chrome.storage.local.get("settings"); @@ -11,8 +9,20 @@ function Login() { }; return (
- - + +
); } @@ -46,4 +56,4 @@ function openLoginPage(instanceUrl: string) { }); } -export default Login; \ No newline at end of file +export default Login; diff --git a/spot/entrypoints/popup/Settings.tsx b/spot/entrypoints/popup/Settings.tsx index 664b6e872..4f0cfb049 100644 --- a/spot/entrypoints/popup/Settings.tsx +++ b/spot/entrypoints/popup/Settings.tsx @@ -1,6 +1,6 @@ import { createSignal, onMount } from "solid-js"; -import orLogo from "@/assets/orSpot.svg"; -import arrowLeft from "@/assets/arrow-left.svg"; +import orLogo from "~/assets/orSpot.svg"; +import arrowLeft from "~/assets/arrow-left.svg"; function Settings({ goBack }: { goBack: () => void }) { const [includeDevTools, setIncludeDevTools] = createSignal(true); @@ -13,7 +13,8 @@ function Settings({ goBack }: { goBack: () => void }) { onMount(() => { chrome.storage.local.get("settings", (data: any) => { if (data.settings) { - const ingest = data.settings.ingestPoint || "https://app.openreplay.com"; + const ingest = + data.settings.ingestPoint || "https://app.openreplay.com"; const devToolsEnabled = data.settings.consoleLogs && data.settings.networkLogs; setOpenInNewTab(data.settings.openInNewTab ?? false); @@ -89,7 +90,8 @@ function Settings({ goBack }: { goBack: () => void }) { return (
-
@@ -159,7 +161,8 @@ function Settings({ goBack }: { goBack: () => void }) {

- Set this URL if you are self-hosting OpenReplay so it points to your instance. + Set this URL if you are self-hosting OpenReplay so it points to your + instance.

{showIngest() && ( @@ -191,16 +194,16 @@ function Settings({ goBack }: { goBack: () => void }) { ) : ( -
- {ingest()} - -
- )} +
+ {ingest()} + +
+ )} )} diff --git a/spot/package.json b/spot/package.json index f3ac9c69f..0d5997b74 100644 --- a/spot/package.json +++ b/spot/package.json @@ -2,7 +2,7 @@ "name": "wxt-starter", "description": "manifest.json description", "private": true, - "version": "1.0.8", + "version": "1.0.9", "type": "module", "scripts": { "dev": "wxt", @@ -17,6 +17,7 @@ }, "dependencies": { "@neodrag/solid": "^2.0.4", + "@openreplay/network-proxy": "^1.0.3", "@thedutchcoder/postcss-rem-to-px": "^0.0.2", "autoprefixer": "^10.4.19", "install": "^0.13.0", @@ -31,6 +32,6 @@ "@wxt-dev/module-solid": "^1.1.2", "daisyui": "^4.12.10", "typescript": "^5.4.5", - "wxt": "0.19.9" + "wxt": "0.19.10" } } diff --git a/spot/tsconfig.json b/spot/tsconfig.json index db3e31422..70a38bef5 100644 --- a/spot/tsconfig.json +++ b/spot/tsconfig.json @@ -1,5 +1,14 @@ { "extends": "./.wxt/tsconfig.json", + "paths": { + "~": [".."], + "~/*": ["../*"], + "@@": [".."], + "@@/*": ["../*"], + "~~": [".."], + "~~/*": ["../*"] + }, + "lib": ["es2022", "DOM"], "compilerOptions": { "jsx": "preserve", "jsxImportSource": "solid-js" diff --git a/spot/utils/consoleTracking.ts b/spot/utils/consoleTracking.ts new file mode 100644 index 000000000..80de7ffde --- /dev/null +++ b/spot/utils/consoleTracking.ts @@ -0,0 +1,142 @@ +function printString(arg) { + const printError = + "InstallTrigger" in window // detect Firefox + ? (e) => e.message + "\n" + e.stack + : (e) => e.stack || e.message; + + if (arg === undefined) { + return "undefined"; + } + if (arg === null) { + return "null"; + } + if (arg instanceof Error) { + return printError(arg); + } + if (Array.isArray(arg)) { + return `Array(${arg.length})`; + } + return String(arg); +} + +function printFloat(arg) { + if (typeof arg !== "number") return "NaN"; + return arg.toString(); +} + +function printInt(arg) { + if (typeof arg !== "number") return "NaN"; + return Math.floor(arg).toString(); +} + +function printObject(arg) { + const printError = + "InstallTrigger" in window // detect Firefox + ? (e) => e.message + "\n" + e.stack + : (e) => e.stack || e.message; + + if (arg === undefined) { + return "undefined"; + } + if (arg === null) { + return "null"; + } + if (arg instanceof Error) { + return printError(arg); + } + if (Array.isArray(arg)) { + const length = arg.length; + const values = arg.slice(0, 10).map(printString).join(", "); + return `Array(${length})[${values}]`; + } + if (typeof arg === "object") { + const res = []; + let i = 0; + for (const k in arg) { + if (++i === 10) { + break; + } + const v = arg[k]; + res.push(k + ": " + printString(v)); + } + return "{" + res.join(", ") + "}"; + } + return arg.toString(); +} + +function printf(args) { + if (typeof args[0] === "string") { + args.unshift( + args.shift().replace(/%(o|s|f|d|i)/g, (s, t) => { + const arg = args.shift(); + if (arg === undefined) return s; + switch (t) { + case "o": + return printObject(arg); + case "s": + return printString(arg); + case "f": + return printFloat(arg); + case "d": + case "i": + return printInt(arg); + default: + return s; + } + }), + ); + } + return args.map(printObject).join(" "); +} + +const consoleMethods = ["log", "info", "warn", "error", "debug", "assert"]; + +export const patchConsole = (console, ctx) => { + if (window.revokeSpotPatch || window.__or_proxy_revocable) { + return; + } + let n = 0; + const reset = () => { + n = 0; + }; + let int = setInterval(reset, 1000); + + const sendConsoleLog = (level, args) => { + const msg = printf(args); + const truncated = + msg.length > 5000 ? `Truncated: ${msg.slice(0, 5000)}...` : msg; + const logs = [{ level, msg: truncated, time: Date.now() }]; + window.postMessage({ type: "ort:bump-logs", logs }, "*"); + }; + + const handler = (level) => ({ + apply: function (target, thisArg, argumentsList) { + Reflect.apply(target, ctx, argumentsList); + n = n + 1; + if (n > 10) { + return; + } else { + sendConsoleLog(level, argumentsList); // Pass the correct level + } + }, + }); + + window.__or_proxy_revocable = []; + consoleMethods.forEach((method) => { + if (consoleMethods.indexOf(method) === -1) { + return; + } + const fn = ctx.console[method]; + // is there any way to preserve the original console trace? + const revProxy = Proxy.revocable(fn, handler(method)); + console[method] = revProxy.proxy; + window.__or_proxy_revocable.push(revProxy); + }); + + return () => { + clearInterval(int); + window.__or_proxy_revocable.forEach((revocable) => { + revocable.revoke(); + }); + }; +}; \ No newline at end of file diff --git a/spot/utils/networkTracking.ts b/spot/utils/networkTracking.ts index 069de9593..e6e4d67fe 100644 --- a/spot/utils/networkTracking.ts +++ b/spot/utils/networkTracking.ts @@ -1,305 +1,169 @@ -import { WebRequest } from "webextension-polyfill"; -export type TrackedRequest = { - statusCode: number; - requestHeaders: Record; - responseHeaders: Record; -} & ( - | WebRequest.OnBeforeRequestDetailsType - | WebRequest.OnBeforeSendHeadersDetailsType - | WebRequest.OnCompletedDetailsType - | WebRequest.OnErrorOccurredDetailsType - | WebRequest.OnResponseStartedDetailsType -); - -export interface SpotNetworkRequest { - encodedBodySize: number; - responseBodySize: number; - duration: number; - method: TrackedRequest["method"]; - type: string; - time: TrackedRequest["timeStamp"]; - statusCode: number; - error?: string; - url: TrackedRequest["url"]; - fromCache: boolean; - body: string; - requestHeaders: Record; - responseHeaders: Record; -} -export const rawRequests: (TrackedRequest & { - startTs: number; - duration: number; -})[] = []; - -const sensitiveParams = new Set([ - "password", - "pass", - "pwd", - "mdp", - "token", - "bearer", - "jwt", - "api_key", - "api-key", - "apiKey", - "key", - "secret", - "id", - "user", - "userId", - "email", - "ssn", - "name", - "firstname", - "lastname", - "birthdate", - "dob", - "address", - "zip", - "zipcode", - "x-api-key", - "www-authenticate", - "x-csrf-token", - "x-requested-with", - "x-forwarded-for", - "x-real-ip", - "cookie", - "authorization", - "auth", - "proxy-authorization", - "set-cookie", - "account_key", -]); - -function filterHeaders(headers: Record) { - const filteredHeaders: Record = {}; - if (Array.isArray(headers)) { - headers.forEach(({ name, value }) => { - if (sensitiveParams.has(name.toLowerCase())) { - filteredHeaders[name] = "******"; - } else { - filteredHeaders[name] = value; - } - }); - } else { - for (const [key, value] of Object.entries(headers)) { - if (sensitiveParams.has(key.toLowerCase())) { - filteredHeaders[key] = "******"; - } else { - filteredHeaders[key] = value; - } - } - } - return filteredHeaders; -} - -// JSON or form data -function filterBody(body: any) { - if (!body) { - return body; - } - - let parsedBody; - let isJSON = false; - - try { - parsedBody = JSON.parse(body); - isJSON = true; - } catch (e) { - // not json - } - - if (isJSON) { - obscureSensitiveData(parsedBody); - return JSON.stringify(parsedBody); - } else { - const params = new URLSearchParams(body); - for (const key of params.keys()) { - if (sensitiveParams.has(key.toLowerCase())) { - params.set(key, "******"); - } - } - - return params.toString(); - } -} - -function obscureSensitiveData(obj: Record | any[]) { - if (Array.isArray(obj)) { - obj.forEach(obscureSensitiveData); - } else if (obj && typeof obj === "object") { - for (const key in obj) { - if (obj.hasOwnProperty(key)) { - if (sensitiveParams.has(key.toLowerCase())) { - obj[key] = "******"; - } else if (obj[key] !== null && typeof obj[key] === "object") { - obscureSensitiveData(obj[key]); - } - } - } - } -} - -function tryFilterUrl(url: string) { - if (!url) return ""; - try { - const urlObj = new URL(url); - if (urlObj.searchParams) { - for (const key of urlObj.searchParams.keys()) { - if (sensitiveParams.has(key.toLowerCase())) { - urlObj.searchParams.set(key, "******"); - } - } - } - return urlObj.toString(); - } catch (e) { - return url; - } -} - -export function createSpotNetworkRequest( - trackedRequest: TrackedRequest, - trackedTab?: number, -) { - if (trackedRequest.tabId === -1) { - return; - } - if (trackedTab && trackedTab !== trackedRequest.tabId) { - return; - } - if ( - ["ping", "beacon", "image", "script", "font"].includes(trackedRequest.type) - ) { - if (!trackedRequest.statusCode || trackedRequest.statusCode < 400) { - return; - } - } - const type = ["stylesheet", "script", "image", "media", "font"].includes( - trackedRequest.type, - ) - ? "resource" - : trackedRequest.type; - - const requestHeaders = trackedRequest.requestHeaders - ? filterHeaders(trackedRequest.requestHeaders) - : {}; - const responseHeaders = trackedRequest.responseHeaders - ? filterHeaders(trackedRequest.responseHeaders) - : {}; - - const reqSize = trackedRequest.reqBody - ? trackedRequest.requestSize || trackedRequest.reqBody.length - : 0; - - const status = getRequestStatus(trackedRequest); - let body; - if (trackedRequest.reqBody) { - try { - body = filterBody(trackedRequest.reqBody); - } catch (e) { - body = "Error parsing body"; - console.error(e); - } - } else { - body = ""; - } - const request: SpotNetworkRequest = { - method: trackedRequest.method, - type, - body, - requestHeaders, - responseHeaders, - time: trackedRequest.timeStamp, - statusCode: status, - error: trackedRequest.error, - url: tryFilterUrl(trackedRequest.url), - fromCache: trackedRequest.fromCache || false, - encodedBodySize: reqSize, - responseBodySize: trackedRequest.responseSize, - duration: trackedRequest.duration, - }; - - return request; -} - -function modifyOnSpot(request: TrackedRequest) { - const id = request.requestId; - const index = rawRequests.findIndex((r) => r.requestId === id); - const ts = Date.now(); - const start = rawRequests[index]?.startTs ?? ts; - rawRequests[index] = { - ...rawRequests[index], - ...request, - duration: ts - start, - }; -} - -const trackOnBefore = ( - details: WebRequest.OnBeforeRequestDetailsType & { reqBody: string }, -) => { - if (details.method === "POST" && details.requestBody) { - const requestBody = details.requestBody; - if (requestBody.formData) { - details.reqBody = JSON.stringify(requestBody.formData); - } else if (requestBody.raw) { - const raw = requestBody.raw[0]?.bytes; - if (raw) { - details.reqBody = new TextDecoder("utf-8").decode(raw); - } - } - } - rawRequests.push({ ...details, startTs: Date.now(), duration: 0 }); -}; -const trackOnCompleted = (details: WebRequest.OnCompletedDetailsType) => { - modifyOnSpot(details); -}; -const trackOnHeaders = (details: WebRequest.OnBeforeSendHeadersDetailsType) => { - modifyOnSpot(details); -}; -const trackOnError = (details: WebRequest.OnErrorOccurredDetailsType) => { - modifyOnSpot(details); -}; -export function startTrackingNetwork() { - rawRequests.length = 0; - browser.webRequest.onBeforeRequest.addListener( - // @ts-ignore - trackOnBefore, - { urls: [""] }, - ["requestBody"], - ); - browser.webRequest.onBeforeSendHeaders.addListener( - trackOnHeaders, - { urls: [""] }, - ["requestHeaders"], - ); - browser.webRequest.onCompleted.addListener( - trackOnCompleted, - { - urls: [""], - }, - ["responseHeaders"], - ); - browser.webRequest.onErrorOccurred.addListener( - trackOnError, - { - urls: [""], - }, - ["extraHeaders"], - ); -} - -export function stopTrackingNetwork() { - browser.webRequest.onBeforeRequest.removeListener(trackOnBefore); - browser.webRequest.onCompleted.removeListener(trackOnCompleted); - browser.webRequest.onErrorOccurred.removeListener(trackOnError); -} - -function getRequestStatus(request: any): number { - if (request.statusCode) { - return request.statusCode; - } - if (request.error) { - return 0; - } - return 200; -} +// import { +// SpotNetworkRequest, +// filterBody, +// filterHeaders, +// tryFilterUrl, +// TrackedRequest, +// } from "./networkTrackingUtils"; +// +// export const rawRequests: (TrackedRequest & { +// startTs: number; +// duration: number; +// })[] = []; +// +// export function createSpotNetworkRequestV1( +// trackedRequest: TrackedRequest, +// trackedTab?: number, +// ) { +// if (trackedRequest.tabId === -1) { +// return; +// } +// if (trackedTab && trackedTab !== trackedRequest.tabId) { +// return; +// } +// if ( +// ["ping", "beacon", "image", "script", "font"].includes(trackedRequest.type) +// ) { +// if (!trackedRequest.statusCode || trackedRequest.statusCode < 400) { +// return; +// } +// } +// const type = ["stylesheet", "script", "image", "media", "font"].includes( +// trackedRequest.type, +// ) +// ? "resource" +// : trackedRequest.type; +// +// const requestHeaders = trackedRequest.requestHeaders +// ? filterHeaders(trackedRequest.requestHeaders) +// : {}; +// const responseHeaders = trackedRequest.responseHeaders +// ? filterHeaders(trackedRequest.responseHeaders) +// : {}; +// +// const reqSize = trackedRequest.reqBody +// ? trackedRequest.requestSize || trackedRequest.reqBody.length +// : 0; +// +// const status = getRequestStatus(trackedRequest); +// let body; +// if (trackedRequest.reqBody) { +// try { +// body = filterBody(trackedRequest.reqBody); +// } catch (e) { +// body = "Error parsing body"; +// console.error(e); +// } +// } else { +// body = ""; +// } +// const request: SpotNetworkRequest = { +// method: trackedRequest.method, +// type, +// body, +// responseBody: "", +// requestHeaders, +// responseHeaders, +// time: trackedRequest.timeStamp, +// statusCode: status, +// error: trackedRequest.error, +// url: tryFilterUrl(trackedRequest.url), +// fromCache: trackedRequest.fromCache || false, +// encodedBodySize: reqSize, +// responseBodySize: trackedRequest.responseSize, +// duration: trackedRequest.duration, +// }; +// +// return request; +// } +// +// function modifyOnSpot(request: TrackedRequest) { +// const id = request.requestId; +// const index = rawRequests.findIndex((r) => r.requestId === id); +// const ts = Date.now(); +// const start = rawRequests[index]?.startTs ?? ts; +// rawRequests[index] = { +// ...rawRequests[index], +// ...request, +// duration: ts - start, +// }; +// } +// +// const trackOnBefore = ( +// details: WebRequest.OnBeforeRequestDetailsType & { reqBody: string }, +// ) => { +// if (details.method === "POST" && details.requestBody) { +// const requestBody = details.requestBody; +// if (requestBody.formData) { +// details.reqBody = JSON.stringify(requestBody.formData); +// } else if (requestBody.raw) { +// const raw = requestBody.raw[0]?.bytes; +// if (raw) { +// details.reqBody = new TextDecoder("utf-8").decode(raw); +// } +// } +// } +// rawRequests.push({ ...details, startTs: Date.now(), duration: 0 }); +// }; +// const trackOnCompleted = (details: WebRequest.OnCompletedDetailsType) => { +// modifyOnSpot(details); +// }; +// const trackOnHeaders = (details: WebRequest.OnBeforeSendHeadersDetailsType) => { +// modifyOnSpot(details); +// }; +// const trackOnError = (details: WebRequest.OnErrorOccurredDetailsType) => { +// modifyOnSpot(details); +// }; +// export function startTrackingNetwork() { +// rawRequests.length = 0; +// browser.webRequest.onBeforeRequest.addListener( +// // @ts-ignore +// trackOnBefore, +// { urls: [""] }, +// ["requestBody"], +// ); +// browser.webRequest.onBeforeSendHeaders.addListener( +// trackOnHeaders, +// { urls: [""] }, +// ["requestHeaders"], +// ); +// browser.webRequest.onCompleted.addListener( +// trackOnCompleted, +// { +// urls: [""], +// }, +// ["responseHeaders"], +// ); +// browser.webRequest.onErrorOccurred.addListener( +// trackOnError, +// { +// urls: [""], +// }, +// ["extraHeaders"], +// ); +// } +// +// export function stopTrackingNetwork() { +// browser.webRequest.onBeforeRequest.removeListener(trackOnBefore); +// browser.webRequest.onCompleted.removeListener(trackOnCompleted); +// browser.webRequest.onErrorOccurred.removeListener(trackOnError); +// } +// +// function getRequestStatus(request: any): number { +// if (request.statusCode) { +// return request.statusCode; +// } +// if (request.error) { +// return 0; +// } +// return 200; +// } +// +// export function getFinalRequests(tabId: number): SpotNetworkRequest[] { +// const finalRequests = rawRequests +// .map((r) => createSpotNetworkRequest(r, tabId)) +// .filter((r) => r !== undefined); +// rawRequests.length = 0; +// +// return finalRequests; +// } diff --git a/spot/utils/networkTrackingUtils.ts b/spot/utils/networkTrackingUtils.ts new file mode 100644 index 000000000..2bd5e4208 --- /dev/null +++ b/spot/utils/networkTrackingUtils.ts @@ -0,0 +1,168 @@ +import { WebRequest } from "webextension-polyfill"; +export type TrackedRequest = { + statusCode: number; + requestHeaders: Record; + responseHeaders: Record; +} & ( + | WebRequest.OnBeforeRequestDetailsType + | WebRequest.OnBeforeSendHeadersDetailsType + | WebRequest.OnCompletedDetailsType + | WebRequest.OnErrorOccurredDetailsType + | WebRequest.OnResponseStartedDetailsType +); + +export function getTopWindow(): Window { + let currentWindow = window; + try { + while (currentWindow !== currentWindow.parent) { + currentWindow = currentWindow.parent; + } + } catch (e) { + // Accessing currentWindow.parent threw an exception due to cross-origin policy + // currentWindow is the topmost accessible window + } + return currentWindow; +} + +export interface SpotNetworkRequest { + encodedBodySize: number; + responseBodySize: number; + duration: number; + method: TrackedRequest["method"]; + type: string; + time: TrackedRequest["timeStamp"]; + statusCode: number; + error?: string; + url: TrackedRequest["url"]; + fromCache: boolean; + body: string; + responseBody: string; + requestHeaders: Record; + responseHeaders: Record; +} + +export const sensitiveParams = new Set([ + "password", + "pass", + "pwd", + "mdp", + "token", + "bearer", + "jwt", + "api_key", + "api-key", + "apiKey", + "key", + "secret", + "id", + "user", + "userId", + "email", + "ssn", + "name", + "firstname", + "lastname", + "birthdate", + "dob", + "address", + "zip", + "zipcode", + "x-api-key", + "www-authenticate", + "x-csrf-token", + "x-requested-with", + "x-forwarded-for", + "x-real-ip", + "cookie", + "authorization", + "auth", + "proxy-authorization", + "set-cookie", + "account_key", +]); + +export function filterHeaders(headers: Record | { name: string; value: string }[]) { + const filteredHeaders: Record = {}; + if (Array.isArray(headers)) { + headers.forEach(({ name, value }) => { + if (sensitiveParams.has(name.toLowerCase())) { + filteredHeaders[name] = "******"; + } else { + filteredHeaders[name] = value; + } + }); + } else { + for (const [key, value] of Object.entries(headers)) { + if (sensitiveParams.has(key.toLowerCase())) { + filteredHeaders[key] = "******"; + } else { + filteredHeaders[key] = value; + } + } + } + return filteredHeaders; +} + +// JSON or form data +export function filterBody(body: any): string { + if (!body) { + return body; + } + + let parsedBody; + let isJSON = false; + + try { + parsedBody = JSON.parse(body); + isJSON = true; + } catch (e) { + // not json + } + + if (isJSON) { + obscureSensitiveData(parsedBody); + return JSON.stringify(parsedBody); + } else { + const params = new URLSearchParams(body); + for (const key of params.keys()) { + if (sensitiveParams.has(key.toLowerCase())) { + params.set(key, "******"); + } + } + + return params.toString(); + } +} + +export function obscureSensitiveData(obj: Record | any[]) { + if (Array.isArray(obj)) { + obj.forEach(obscureSensitiveData); + } else if (obj && typeof obj === "object") { + for (const key in obj) { + if (Object.hasOwn(obj, key)) { + if (sensitiveParams.has(key.toLowerCase())) { + obj[key] = "******"; + } else if (obj[key] !== null && typeof obj[key] === "object") { + obscureSensitiveData(obj[key]); + } + } + } + } +} + +export function tryFilterUrl(url: string) { + if (!url) return ""; + try { + const urlObj = new URL(url); + if (urlObj.searchParams) { + for (const key of urlObj.searchParams.keys()) { + if (sensitiveParams.has(key.toLowerCase())) { + urlObj.searchParams.set(key, "******"); + } + } + } + return urlObj.toString(); + } catch (e) { + return url; + } +} diff --git a/spot/utils/proxyNetworkTracking.ts b/spot/utils/proxyNetworkTracking.ts new file mode 100644 index 000000000..fb3a7185e --- /dev/null +++ b/spot/utils/proxyNetworkTracking.ts @@ -0,0 +1,101 @@ +import createNetworkProxy, { INetworkMessage } from "@openreplay/network-proxy"; +import { + SpotNetworkRequest, + filterBody, + filterHeaders, + tryFilterUrl, + getTopWindow, +} from "./networkTrackingUtils"; + +let defaultFetch: typeof fetch | undefined; +let defaultXhr: typeof XMLHttpRequest | undefined; +let defaultBeacon: typeof navigator.sendBeacon | undefined; + +export function startNetwork() { + const context = getTopWindow(); + defaultXhr = context.XMLHttpRequest; + defaultBeacon = context.navigator.sendBeacon; + defaultFetch = context.fetch; + createNetworkProxy( + context, + [], // headers + () => null, + (reqRes) => reqRes, + (msg) => { + const event = createSpotNetworkRequest(msg); + window.postMessage({ type: "ort:bump-network", event }, "*"); + }, + (url) => + url.includes("/spot/") || url.includes(".mob?") || url.includes(".mobe?"), + { xhr: true, fetch: true, beacon: true }, + ); +} + +function getBody(req: { body?: string | Record }): string { + let body; + + if (req.body) { + try { + body = filterBody(req.body); + } catch (e) { + body = "Error parsing body"; + console.error(e); + } + } else { + body = ""; + } + + return body; +} + +export function createSpotNetworkRequest( + msg: INetworkMessage, +): SpotNetworkRequest { + let request: Record = {} + let response: Record = {}; + try { + request = JSON.parse(msg.request); + } catch (e) { + console.error("Error parsing request", e); + } + try { + response = JSON.parse(msg.response); + } catch (e) { + console.error("Error parsing response", e); + } + const reqHeaders = request.headers ? filterHeaders(request.headers) : {}; + const resHeaders = response.headers ? filterHeaders(response.headers) : {}; + const responseBodySize = msg.responseSize || 0; + const reqSize = msg.request ? msg.request.length : 0; + const body = getBody(request); + const responseBody = getBody(response); + + return { + method: msg.method, + type: msg.requestType, + body, + responseBody, + requestHeaders: reqHeaders, + responseHeaders: resHeaders, + time: msg.startTime, + statusCode: msg.status || 0, + error: undefined, + url: tryFilterUrl(msg.url), + fromCache: false, + encodedBodySize: reqSize, + responseBodySize, + duration: msg.duration, + }; +} + +export function stopNetwork() { + if (defaultFetch) { + window.fetch = defaultFetch; + } + if (defaultXhr) { + window.XMLHttpRequest = defaultXhr; + } + if (defaultBeacon) { + window.navigator.sendBeacon = defaultBeacon; + } +} diff --git a/spot/yarn.lock b/spot/yarn.lock index 127633c78..468f7f6a5 100644 --- a/spot/yarn.lock +++ b/spot/yarn.lock @@ -707,6 +707,11 @@ proc-log "^4.0.0" which "^4.0.0" +"@openreplay/network-proxy@^1.0.3": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@openreplay/network-proxy/-/network-proxy-1.0.3.tgz#82c07f0742db01456f3764bae6c0b5ff280557a9" + integrity sha512-vm/x8Ioo1BKJIyZf58tK44CgtzHA0tIwu2uZ2WdIzrBOODF+A2qV4mELC8Zdp9WqV1O7uxXGaA4J38HU3th14g== + "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz" @@ -5331,10 +5336,10 @@ ws@8.18.0: resolved "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz" integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== -wxt@0.19.9: - version "0.19.9" - resolved "https://registry.yarnpkg.com/wxt/-/wxt-0.19.9.tgz#b8f7f838cab00d66f4ee22f483c49ad8f6527af8" - integrity sha512-XUbF4JNyx2jTDpXwx2c/esaJcUD2Dr482C2GGenkGRMH2UnerzOIchGCtaa1hb2U8eAed7Akda0yRoMJU0uxUw== +wxt@0.19.10: + version "0.19.10" + resolved "https://registry.yarnpkg.com/wxt/-/wxt-0.19.10.tgz#557d57e63ab5fcf3b026791aae3706c400b5f7cb" + integrity sha512-lX/dzAaau79SDsU7QZKUgxJUf9nBsaQXMiqsDNgGoqZO1wrRpFq0kijcN3/mNjGbXM989VJhEl7B6f3ttxKAnQ== dependencies: "@aklinker1/rollup-plugin-visualizer" "5.12.0" "@types/chrome" "^0.0.269" @@ -5350,6 +5355,7 @@ wxt@0.19.9: consola "^3.2.3" defu "^6.1.4" dequal "^2.0.3" + dotenv "^16.4.5" esbuild "^0.23.0" execa "^9.3.1" fast-glob "^3.3.2" @@ -5371,6 +5377,7 @@ wxt@0.19.9: ohash "^1.1.3" open "^10.1.0" ora "^8.1.0" + perfect-debounce "^1.0.0" picocolors "^1.0.1" prompts "^2.4.2" publish-browser-extension "^2.1.3" diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index f1dad5d24..060b724b0 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,3 +1,7 @@ +# 14.0.8 + +- use separate library to handle network requests ([@openreplay/network-proxy](https://www.npmjs.com/package/@openreplay/network-proxy)) + # 14.0.7 - check for stopping status during restarts diff --git a/tracker/tracker/bun.lockb b/tracker/tracker/bun.lockb index 2b5bba11e..fca47c24e 100755 Binary files a/tracker/tracker/bun.lockb and b/tracker/tracker/bun.lockb differ diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 739874541..06a3202e3 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "14.0.7", + "version": "14.0.8", "keywords": [ "logging", "replay" @@ -50,6 +50,7 @@ }, "dependencies": { "@medv/finder": "^3.2.0", + "@openreplay/network-proxy": "^1.0.2", "error-stack-parser": "^2.0.6", "fflate": "^0.8.2" }, diff --git a/tracker/tracker/src/main/modules/Network/index.ts b/tracker/tracker/src/main/modules/Network/index.ts deleted file mode 100644 index 1e9c81ac1..000000000 --- a/tracker/tracker/src/main/modules/Network/index.ts +++ /dev/null @@ -1,53 +0,0 @@ -import FetchProxy from './fetchProxy.js' -import XHRProxy from './xhrProxy.js' -import BeaconProxy from './beaconProxy.js' -import { RequestResponseData } from './types.js' -import { NetworkRequest } from '../../../common/messages.gen.js' - -const getWarning = (api: string) => - console.warn(`Openreplay: Can't find ${api} in global context. -If you're using serverside rendering in your app, make sure that tracker is loaded dynamically, otherwise ${api} won't be tracked.`) - -export default function setProxy( - context: typeof globalThis, - ignoredHeaders: boolean | string[], - setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, - sanitize: (data: RequestResponseData) => RequestResponseData | null, - sendMessage: (message: NetworkRequest) => void, - isServiceUrl: (url: string) => boolean, - tokenUrlMatcher?: (url: string) => boolean, -) { - if (context.XMLHttpRequest) { - context.XMLHttpRequest = XHRProxy.create( - ignoredHeaders, - setSessionTokenHeader, - sanitize, - sendMessage, - isServiceUrl, - tokenUrlMatcher, - ) - } else { - getWarning('XMLHttpRequest') - } - if (context.fetch) { - context.fetch = FetchProxy.create( - ignoredHeaders, - setSessionTokenHeader, - sanitize, - sendMessage, - isServiceUrl, - tokenUrlMatcher, - ) - } else { - getWarning('fetch') - } - if (context?.navigator?.sendBeacon) { - context.navigator.sendBeacon = BeaconProxy.create( - ignoredHeaders, - setSessionTokenHeader, - sanitize, - sendMessage, - isServiceUrl, - ) - } -} diff --git a/tracker/tracker/src/main/modules/Network/types.ts b/tracker/tracker/src/main/modules/Network/types.ts deleted file mode 100644 index 14364700e..000000000 --- a/tracker/tracker/src/main/modules/Network/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -export interface RequestResponseData { - readonly status: number - readonly method: string - url: string - request: { - body: string | null - headers: Record - } - response: { - body: string | null - headers: Record - } -} - -// we only support sanitizing for json/string data because how you're gonna sanitize binary data? diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts index b16657b03..d50b11f8c 100644 --- a/tracker/tracker/src/main/modules/network.ts +++ b/tracker/tracker/src/main/modules/network.ts @@ -3,7 +3,7 @@ import { NetworkRequest } from '../app/messages.gen.js' import { getTimeOrigin } from '../utils.js' import type { AxiosInstance } from './axiosSpy.js' import axiosSpy from './axiosSpy.js' -import setProxy from './Network/index.js' +import createNetworkProxy from '@openreplay/network-proxy' type WindowFetch = typeof window.fetch type XHRRequestBody = Parameters[0] @@ -130,13 +130,28 @@ export default function (app: App, opts: Partial = {}) { const patchWindow = (context: typeof globalThis) => { /* ====== modern way ====== */ if (options.useProxy) { - return setProxy( + return createNetworkProxy( context, options.ignoreHeaders, setSessionTokenHeader, sanitize, - (message) => app.send(message), + (message) => { + app.send( + NetworkRequest( + message.requestType, + message.method, + message.url, + message.request, + message.response, + message.status, + message.startTime + getTimeOrigin(), + message.duration, + message.responseSize, + ), + ) + }, (url) => app.isServiceURL(url), + { xhr: true, fetch: true, beacon: true }, options.tokenUrlMatcher, ) } diff --git a/tracker/tracker/tsconfig-base.json b/tracker/tracker/tsconfig-base.json index f74da1469..5bda21c51 100644 --- a/tracker/tracker/tsconfig-base.json +++ b/tracker/tracker/tsconfig-base.json @@ -9,8 +9,8 @@ "alwaysStrict": true, "target": "es2020", "module": "es6", - "moduleResolution": "nodenext", - "esModuleInterop": true + "moduleResolution": "node", + "esModuleInterop": true, }, "exclude": ["**/*.test.ts"] }