spot: mix network requests with webRequest data for better tracking
This commit is contained in:
nick-delirium
Delirium
parent
28dea3b225
commit
8ba35b1324
5 changed files with 195 additions and 185 deletions
|
|
@ -1,4 +1,10 @@
|
|||
import { isTokenExpired } from "~/utils/jwt";
|
||||
import {
|
||||
startTrackingNetwork,
|
||||
getFinalRequests,
|
||||
stopTrackingNetwork,
|
||||
} from "~/utils/networkTracking";
|
||||
import { mergeRequests } from "~/utils/networkTrackingUtils";
|
||||
|
||||
let checkBusy = false;
|
||||
|
||||
|
|
@ -136,6 +142,7 @@ export default defineBackground(() => {
|
|||
let finalVideoBase64 = "";
|
||||
let finalReady = false;
|
||||
let finalSpotObj: SpotObj = defaultSpotObj;
|
||||
let injectNetworkRequests = [];
|
||||
let onStop: (() => void) | null = null;
|
||||
let settings = defaultSettings;
|
||||
let recordingState = {
|
||||
|
|
@ -339,6 +346,7 @@ export default defineBackground(() => {
|
|||
recording: REC_STATE.stopped,
|
||||
audioPerm: request.permissions ? (request.mic ? 2 : 1) : 0,
|
||||
};
|
||||
startTrackingNetwork();
|
||||
if (request.area === "tab") {
|
||||
browser.tabs
|
||||
.query({
|
||||
|
|
@ -577,7 +585,7 @@ export default defineBackground(() => {
|
|||
return "pong";
|
||||
}
|
||||
if (request.type === messages.injected.from.bumpNetwork) {
|
||||
finalSpotObj.network.push(request.event);
|
||||
injectNetworkRequests.push(request.event);
|
||||
return "pong";
|
||||
}
|
||||
if (request.type === messages.content.from.bumpClicks) {
|
||||
|
|
@ -649,7 +657,7 @@ export default defineBackground(() => {
|
|||
title: "JS Error",
|
||||
time: (l.time - finalSpotObj.startTs) / 1000,
|
||||
}));
|
||||
const network = finalSpotObj.network
|
||||
const network = [...injectNetworkRequests, ...finalSpotObj.network]
|
||||
.filter((net) => net.statusCode >= 400 || net.error)
|
||||
.map((n) => ({
|
||||
title: "Network Error",
|
||||
|
|
@ -665,8 +673,19 @@ export default defineBackground(() => {
|
|||
}
|
||||
if (request.type === messages.content.from.toStop) {
|
||||
if (recordingState.recording === REC_STATE.stopped) {
|
||||
return console.error('Calling stopped recording?')
|
||||
return console.error("Calling stopped recording?");
|
||||
}
|
||||
const networkRequests = getFinalRequests(
|
||||
recordingState.activeTabId ?? false,
|
||||
);
|
||||
|
||||
stopTrackingNetwork();
|
||||
const mappedNetwork = mergeRequests(
|
||||
networkRequests,
|
||||
injectNetworkRequests,
|
||||
);
|
||||
injectNetworkRequests = [];
|
||||
finalSpotObj.network = mappedNetwork;
|
||||
browser.runtime
|
||||
.sendMessage({
|
||||
type: messages.offscreen.to.stopRecording,
|
||||
|
|
@ -749,12 +768,12 @@ export default defineBackground(() => {
|
|||
if (request.type === messages.content.from.saveSpotVidChunk) {
|
||||
finalVideoBase64 += request.part;
|
||||
finalReady = request.index === request.total - 1;
|
||||
const getPlatformData = async () => {
|
||||
const vendor = await browser.runtime.getPlatformInfo();
|
||||
const platform = `${vendor.os} ${vendor.arch}`;
|
||||
return { platform };
|
||||
};
|
||||
if (finalReady) {
|
||||
const getPlatformData = async () => {
|
||||
const vendor = await browser.runtime.getPlatformInfo();
|
||||
const platform = `${vendor.os} ${vendor.arch}`;
|
||||
return { platform };
|
||||
};
|
||||
const duration = finalSpotObj.crop
|
||||
? finalSpotObj.crop[1] - finalSpotObj.crop[0]
|
||||
: finalSpotObj.duration;
|
||||
|
|
@ -1155,7 +1174,7 @@ export default defineBackground(() => {
|
|||
if (state === REC_STATE.stopped) {
|
||||
return stopNavListening();
|
||||
}
|
||||
contentArmy[details.tabId] = false
|
||||
contentArmy[details.tabId] = false;
|
||||
|
||||
if (area === "tab" && (!trackedTab || details.tabId !== trackedTab)) {
|
||||
return;
|
||||
|
|
@ -1179,10 +1198,10 @@ export default defineBackground(() => {
|
|||
}
|
||||
|
||||
function startNavListening() {
|
||||
browser.webNavigation.onCompleted.addListener(tabNavigatedListener)
|
||||
browser.webNavigation.onCompleted.addListener(tabNavigatedListener);
|
||||
}
|
||||
function stopNavListening() {
|
||||
browser.webNavigation.onCompleted.removeListener(tabNavigatedListener)
|
||||
browser.webNavigation.onCompleted.removeListener(tabNavigatedListener);
|
||||
}
|
||||
|
||||
/** discards recording if was recording single tab and its now closed */
|
||||
|
|
|
|||
|
|
@ -270,16 +270,16 @@ export default defineContentScript({
|
|||
document.head.appendChild(scriptEl);
|
||||
}
|
||||
function startConsoleTracking() {
|
||||
injectScript()
|
||||
injectScript();
|
||||
setTimeout(() => {
|
||||
window.postMessage({ type: "injected:c-start" });
|
||||
}, 100);
|
||||
}
|
||||
function startNetworkTracking() {
|
||||
injectScript()
|
||||
injectScript();
|
||||
setTimeout(() => {
|
||||
window.postMessage({ type: "injected:n-start" });
|
||||
}, 100)
|
||||
}, 100);
|
||||
}
|
||||
|
||||
function stopConsoleTracking() {
|
||||
|
|
@ -325,7 +325,7 @@ export default defineContentScript({
|
|||
|
||||
setInterval(() => {
|
||||
void browser.runtime.sendMessage({ type: "ort:content-ready" });
|
||||
}, 250)
|
||||
}, 250);
|
||||
// @ts-ignore false positive
|
||||
browser.runtime.onMessage.addListener((message: any, resp) => {
|
||||
if (message.type === "content:mount") {
|
||||
|
|
|
|||
|
|
@ -1,169 +1,156 @@
|
|||
// import {
|
||||
// SpotNetworkRequest,
|
||||
// filterBody,
|
||||
// filterHeaders,
|
||||
// tryFilterUrl,
|
||||
// TrackedRequest,
|
||||
// } from "./networkTrackingUtils";
|
||||
//
|
||||
// export const rawRequests: (TrackedRequest & {
|
||||
// startTs: number;
|
||||
// duration: number;
|
||||
// })[] = [];
|
||||
//
|
||||
// export function createSpotNetworkRequestV1(
|
||||
// trackedRequest: TrackedRequest,
|
||||
// trackedTab?: number,
|
||||
// ) {
|
||||
// if (trackedRequest.tabId === -1) {
|
||||
// return;
|
||||
// }
|
||||
// if (trackedTab && trackedTab !== trackedRequest.tabId) {
|
||||
// return;
|
||||
// }
|
||||
// if (
|
||||
// ["ping", "beacon", "image", "script", "font"].includes(trackedRequest.type)
|
||||
// ) {
|
||||
// if (!trackedRequest.statusCode || trackedRequest.statusCode < 400) {
|
||||
// return;
|
||||
// }
|
||||
// }
|
||||
// const type = ["stylesheet", "script", "image", "media", "font"].includes(
|
||||
// trackedRequest.type,
|
||||
// )
|
||||
// ? "resource"
|
||||
// : trackedRequest.type;
|
||||
//
|
||||
// const requestHeaders = trackedRequest.requestHeaders
|
||||
// ? filterHeaders(trackedRequest.requestHeaders)
|
||||
// : {};
|
||||
// const responseHeaders = trackedRequest.responseHeaders
|
||||
// ? filterHeaders(trackedRequest.responseHeaders)
|
||||
// : {};
|
||||
//
|
||||
// const reqSize = trackedRequest.reqBody
|
||||
// ? trackedRequest.requestSize || trackedRequest.reqBody.length
|
||||
// : 0;
|
||||
//
|
||||
// const status = getRequestStatus(trackedRequest);
|
||||
// let body;
|
||||
// if (trackedRequest.reqBody) {
|
||||
// try {
|
||||
// body = filterBody(trackedRequest.reqBody);
|
||||
// } catch (e) {
|
||||
// body = "Error parsing body";
|
||||
// console.error(e);
|
||||
// }
|
||||
// } else {
|
||||
// body = "";
|
||||
// }
|
||||
// const request: SpotNetworkRequest = {
|
||||
// method: trackedRequest.method,
|
||||
// type,
|
||||
// body,
|
||||
// responseBody: "",
|
||||
// requestHeaders,
|
||||
// responseHeaders,
|
||||
// time: trackedRequest.timeStamp,
|
||||
// statusCode: status,
|
||||
// error: trackedRequest.error,
|
||||
// url: tryFilterUrl(trackedRequest.url),
|
||||
// fromCache: trackedRequest.fromCache || false,
|
||||
// encodedBodySize: reqSize,
|
||||
// responseBodySize: trackedRequest.responseSize,
|
||||
// duration: trackedRequest.duration,
|
||||
// };
|
||||
//
|
||||
// return request;
|
||||
// }
|
||||
//
|
||||
// function modifyOnSpot(request: TrackedRequest) {
|
||||
// const id = request.requestId;
|
||||
// const index = rawRequests.findIndex((r) => r.requestId === id);
|
||||
// const ts = Date.now();
|
||||
// const start = rawRequests[index]?.startTs ?? ts;
|
||||
// rawRequests[index] = {
|
||||
// ...rawRequests[index],
|
||||
// ...request,
|
||||
// duration: ts - start,
|
||||
// };
|
||||
// }
|
||||
//
|
||||
// const trackOnBefore = (
|
||||
// details: WebRequest.OnBeforeRequestDetailsType & { reqBody: string },
|
||||
// ) => {
|
||||
// if (details.method === "POST" && details.requestBody) {
|
||||
// const requestBody = details.requestBody;
|
||||
// if (requestBody.formData) {
|
||||
// details.reqBody = JSON.stringify(requestBody.formData);
|
||||
// } else if (requestBody.raw) {
|
||||
// const raw = requestBody.raw[0]?.bytes;
|
||||
// if (raw) {
|
||||
// details.reqBody = new TextDecoder("utf-8").decode(raw);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// rawRequests.push({ ...details, startTs: Date.now(), duration: 0 });
|
||||
// };
|
||||
// const trackOnCompleted = (details: WebRequest.OnCompletedDetailsType) => {
|
||||
// modifyOnSpot(details);
|
||||
// };
|
||||
// const trackOnHeaders = (details: WebRequest.OnBeforeSendHeadersDetailsType) => {
|
||||
// modifyOnSpot(details);
|
||||
// };
|
||||
// const trackOnError = (details: WebRequest.OnErrorOccurredDetailsType) => {
|
||||
// modifyOnSpot(details);
|
||||
// };
|
||||
// export function startTrackingNetwork() {
|
||||
// rawRequests.length = 0;
|
||||
// browser.webRequest.onBeforeRequest.addListener(
|
||||
// // @ts-ignore
|
||||
// trackOnBefore,
|
||||
// { urls: ["<all_urls>"] },
|
||||
// ["requestBody"],
|
||||
// );
|
||||
// browser.webRequest.onBeforeSendHeaders.addListener(
|
||||
// trackOnHeaders,
|
||||
// { urls: ["<all_urls>"] },
|
||||
// ["requestHeaders"],
|
||||
// );
|
||||
// browser.webRequest.onCompleted.addListener(
|
||||
// trackOnCompleted,
|
||||
// {
|
||||
// urls: ["<all_urls>"],
|
||||
// },
|
||||
// ["responseHeaders"],
|
||||
// );
|
||||
// browser.webRequest.onErrorOccurred.addListener(
|
||||
// trackOnError,
|
||||
// {
|
||||
// urls: ["<all_urls>"],
|
||||
// },
|
||||
// ["extraHeaders"],
|
||||
// );
|
||||
// }
|
||||
//
|
||||
// export function stopTrackingNetwork() {
|
||||
// browser.webRequest.onBeforeRequest.removeListener(trackOnBefore);
|
||||
// browser.webRequest.onCompleted.removeListener(trackOnCompleted);
|
||||
// browser.webRequest.onErrorOccurred.removeListener(trackOnError);
|
||||
// }
|
||||
//
|
||||
// function getRequestStatus(request: any): number {
|
||||
// if (request.statusCode) {
|
||||
// return request.statusCode;
|
||||
// }
|
||||
// if (request.error) {
|
||||
// return 0;
|
||||
// }
|
||||
// return 200;
|
||||
// }
|
||||
//
|
||||
// export function getFinalRequests(tabId: number): SpotNetworkRequest[] {
|
||||
// const finalRequests = rawRequests
|
||||
// .map((r) => createSpotNetworkRequest(r, tabId))
|
||||
// .filter((r) => r !== undefined);
|
||||
// rawRequests.length = 0;
|
||||
//
|
||||
// return finalRequests;
|
||||
// }
|
||||
import {
|
||||
SpotNetworkRequest,
|
||||
filterBody,
|
||||
filterHeaders,
|
||||
tryFilterUrl,
|
||||
TrackedRequest,
|
||||
} from "./networkTrackingUtils";
|
||||
|
||||
export const rawRequests: Array<
|
||||
TrackedRequest & { startTs: number; duration: number }
|
||||
> = [];
|
||||
|
||||
function getRequestStatus(request: TrackedRequest): number {
|
||||
if (request.statusCode) return request.statusCode;
|
||||
if (request.error) return 0;
|
||||
return 200;
|
||||
}
|
||||
|
||||
function modifyOnSpot(request: TrackedRequest) {
|
||||
const id = request.requestId;
|
||||
const index = rawRequests.findIndex((r) => r.requestId === id);
|
||||
const ts = Date.now();
|
||||
const start = rawRequests[index]?.startTs ?? ts;
|
||||
rawRequests[index] = {
|
||||
...rawRequests[index],
|
||||
...request,
|
||||
duration: ts - start,
|
||||
};
|
||||
}
|
||||
|
||||
function trackOnBefore(
|
||||
details: browser.webRequest._OnBeforeRequestDetails & { reqBody?: string },
|
||||
) {
|
||||
if (details.method === "POST" && details.requestBody) {
|
||||
if (details.requestBody.formData) {
|
||||
details.reqBody = JSON.stringify(details.requestBody.formData);
|
||||
} else if (details.requestBody.raw) {
|
||||
const raw = details.requestBody.raw[0]?.bytes;
|
||||
if (raw) details.reqBody = new TextDecoder("utf-8").decode(raw);
|
||||
}
|
||||
}
|
||||
rawRequests.push({ ...details, startTs: Date.now(), duration: 0 });
|
||||
}
|
||||
|
||||
function trackOnHeaders(
|
||||
details: browser.webRequest._OnBeforeSendHeadersDetails,
|
||||
) {
|
||||
modifyOnSpot(details);
|
||||
}
|
||||
|
||||
function trackOnCompleted(details: browser.webRequest._OnCompletedDetails) {
|
||||
modifyOnSpot(details);
|
||||
}
|
||||
|
||||
function trackOnError(details: browser.webRequest._OnErrorOccurredDetails) {
|
||||
modifyOnSpot(details);
|
||||
}
|
||||
|
||||
// Build final SpotNetworkRequest objects
|
||||
function createSpotNetworkRequest(
|
||||
trackedRequest: TrackedRequest,
|
||||
trackedTab?: number,
|
||||
): SpotNetworkRequest | undefined {
|
||||
if (trackedRequest.tabId === -1) return;
|
||||
if (trackedTab && trackedTab !== trackedRequest.tabId) return;
|
||||
|
||||
if (
|
||||
["ping", "beacon", "image", "script", "font"].includes(trackedRequest.type)
|
||||
) {
|
||||
if (!trackedRequest.statusCode || trackedRequest.statusCode < 400) return;
|
||||
}
|
||||
|
||||
const type = ["stylesheet", "script", "image", "media", "font"].includes(
|
||||
trackedRequest.type,
|
||||
)
|
||||
? "resource"
|
||||
: trackedRequest.type;
|
||||
|
||||
const requestHeaders = trackedRequest.requestHeaders
|
||||
? filterHeaders(trackedRequest.requestHeaders)
|
||||
: {};
|
||||
const responseHeaders = trackedRequest.responseHeaders
|
||||
? filterHeaders(trackedRequest.responseHeaders)
|
||||
: {};
|
||||
|
||||
const reqSize = trackedRequest.reqBody
|
||||
? trackedRequest.requestSize || trackedRequest.reqBody.length
|
||||
: 0;
|
||||
const status = getRequestStatus(trackedRequest);
|
||||
|
||||
let body = "";
|
||||
if (trackedRequest.reqBody) {
|
||||
try {
|
||||
body = filterBody(trackedRequest.reqBody);
|
||||
} catch (e) {
|
||||
body = "Error parsing body";
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
const request: SpotNetworkRequest = {
|
||||
method: trackedRequest.method,
|
||||
type,
|
||||
body,
|
||||
responseBody: "",
|
||||
requestHeaders,
|
||||
responseHeaders,
|
||||
timestamp: trackedRequest.timeStamp,
|
||||
statusCode: status,
|
||||
error: trackedRequest.error,
|
||||
url: tryFilterUrl(trackedRequest.url),
|
||||
fromCache: trackedRequest.fromCache || false,
|
||||
encodedBodySize: reqSize,
|
||||
responseBodySize: trackedRequest.responseSize,
|
||||
duration: trackedRequest.duration,
|
||||
};
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
export function startTrackingNetwork() {
|
||||
rawRequests.length = 0;
|
||||
browser.webRequest.onBeforeRequest.addListener(
|
||||
trackOnBefore,
|
||||
{ urls: ["<all_urls>"] },
|
||||
["requestBody"], // allows capturing POST bodies
|
||||
);
|
||||
browser.webRequest.onBeforeSendHeaders.addListener(
|
||||
trackOnHeaders,
|
||||
{ urls: ["<all_urls>"] },
|
||||
["requestHeaders"],
|
||||
);
|
||||
browser.webRequest.onCompleted.addListener(
|
||||
trackOnCompleted,
|
||||
{ urls: ["<all_urls>"] },
|
||||
["responseHeaders"],
|
||||
);
|
||||
browser.webRequest.onErrorOccurred.addListener(trackOnError, {
|
||||
urls: ["<all_urls>"],
|
||||
});
|
||||
}
|
||||
|
||||
export function stopTrackingNetwork() {
|
||||
browser.webRequest.onBeforeRequest.removeListener(trackOnBefore);
|
||||
browser.webRequest.onBeforeSendHeaders.removeListener(trackOnHeaders);
|
||||
browser.webRequest.onCompleted.removeListener(trackOnCompleted);
|
||||
browser.webRequest.onErrorOccurred.removeListener(trackOnError);
|
||||
}
|
||||
|
||||
export function getFinalRequests(tabId?: number): SpotNetworkRequest[] {
|
||||
const finalRequests = rawRequests
|
||||
.map((r) => createSpotNetworkRequest(r, tabId))
|
||||
.filter((r) => r !== undefined) as SpotNetworkRequest[];
|
||||
rawRequests.length = 0;
|
||||
return finalRequests;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,7 +82,9 @@ export const sensitiveParams = new Set([
|
|||
"account_key",
|
||||
]);
|
||||
|
||||
export function filterHeaders(headers: Record<string, string> | { name: string; value: string }[]) {
|
||||
export function filterHeaders(
|
||||
headers: Record<string, string> | { name: string; value: string }[],
|
||||
) {
|
||||
const filteredHeaders: Record<string, string> = {};
|
||||
if (Array.isArray(headers)) {
|
||||
headers.forEach(({ name, value }) => {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ export default defineConfig({
|
|||
"offscreen",
|
||||
"unlimitedStorage",
|
||||
"webNavigation",
|
||||
"webRequest",
|
||||
"<all_urls>",
|
||||
],
|
||||
},
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue