diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 9bedaa6f6..e53c457b5 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "8.0.0", + "version": "8.0.1-beta14.3", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index c1a5212ba..2a7113ba6 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -147,7 +147,7 @@ export default class API { app.attachStartCallback(() => { if (options.flags?.onFlagsLoad) { - this.featureFlags.onFlagsLoad(options.flags.onFlagsLoad) + this.onFlagsLoad(options.flags.onFlagsLoad) } void this.featureFlags.reloadFlags() }) diff --git a/tracker/tracker/src/main/modules/Network/fetchProxy.ts b/tracker/tracker/src/main/modules/Network/fetchProxy.ts new file mode 100644 index 000000000..6667a3e4f --- /dev/null +++ b/tracker/tracker/src/main/modules/Network/fetchProxy.ts @@ -0,0 +1,288 @@ +/** + * I took inspiration in few stack exchange posts + * and Tencent vConsole library (MIT) + * by wrapping the XMLHttpRequest object in a Proxy + * we can intercept the network requests + * in not-so-hacky way + * */ +import NetworkMessage, { RequestState } from './networkMessage.js' +import { formatByteSize, genStringBody, getStringResponseByType, getURL } from './utils.js' +import { RequestResponseData } from './types.js' +import { NetworkRequest } from '../../../common/messages.gen.js' + +export class ResponseProxyHandler implements ProxyHandler { + public resp: Response + public item: NetworkMessage + + constructor(resp: T, item: NetworkMessage) { + this.resp = resp + this.item = item + this.mockReader() + } + + public set(target: T, key: string, value: (args: any[]) => any) { + return Reflect.set(target, key, value) + } + + public get(target: T, key: string) { + const value = Reflect.get(target, key) + switch (key) { + case 'arrayBuffer': + case 'blob': + case 'formData': + case 'json': + case 'text': + return () => { + this.item.responseType = key.toLowerCase() + // @ts-ignore + return value.apply(target).then((resp: any) => { + this.item.response = getStringResponseByType(this.item.responseType, resp) + return resp + }) + } + } + if (typeof value === 'function') { + return value.bind(target) + } else { + return value + } + } + + protected mockReader() { + let readerReceivedValue: Uint8Array + if (!this.resp.body) { + // some browsers do not return `body` in some cases, like `OPTIONS` method + return + } + if (typeof this.resp.body.getReader !== 'function') { + return + } + const _getReader = this.resp.body.getReader + // @ts-ignore + this.resp.body.getReader = () => { + const reader = >_getReader.apply(this.resp.body) + + // when readyState is already 4, + // it's not a chunked stream, or it had already been read. + // so should not update status. + if (this.item.readyState === RequestState.DONE) { + return reader + } + + const _read = reader.read + const _cancel = reader.cancel + this.item.responseType = 'arraybuffer' + + // @ts-ignore + reader.read = () => { + return (>_read.apply(reader)).then( + (result: ReadableStreamReadResult) => { + if (!readerReceivedValue) { + // @ts-ignore + readerReceivedValue = new Uint8Array(result.value) + } else { + const newValue = new Uint8Array(readerReceivedValue.length + result.value!.length) + newValue.set(readerReceivedValue) + newValue.set(result.value!, readerReceivedValue.length) + readerReceivedValue = newValue + } + this.item.endTime = performance.now() + this.item.duration = this.item.endTime - (this.item.startTime || this.item.endTime) + this.item.readyState = result.done ? 4 : 3 + this.item.statusText = result.done ? String(this.item.status) : 'Loading' + this.item.responseSize = readerReceivedValue.length + this.item.responseSizeText = formatByteSize(this.item.responseSize) + if (result.done) { + this.item.response = getStringResponseByType( + this.item.responseType, + readerReceivedValue, + ) + } + return result + }, + ) + } + + reader.cancel = (...args) => { + this.item.cancelState = 2 + this.item.statusText = 'Cancel' + this.item.endTime = performance.now() + this.item.duration = this.item.endTime - (this.item.startTime || this.item.endTime) + this.item.response = getStringResponseByType(this.item.responseType, readerReceivedValue) + return _cancel.apply(reader, args) + } + return reader + } + } +} + +export class FetchProxyHandler implements ProxyHandler { + constructor( + private readonly ignoredHeaders: boolean | string[], + private readonly setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, + private readonly sanitize: (data: RequestResponseData) => RequestResponseData, + private readonly sendMessage: (item: NetworkRequest) => void, + private readonly isServiceUrl: (url: string) => boolean, + ) {} + + public apply(target: T, thisArg: typeof window, argsList: [RequestInfo | URL, RequestInit]) { + const input = argsList[0] + const init = argsList[1] + + const isORUrl = + input instanceof URL || typeof input === 'string' + ? this.isServiceUrl(String(input)) + : this.isServiceUrl(String(input.url)) + + if (isORUrl) { + return target.apply(window, argsList) + } + + const item = new NetworkMessage(this.ignoredHeaders, this.setSessionTokenHeader, this.sanitize) + this.beforeFetch(item, input as RequestInfo, init) + + return (>target.apply(window, argsList)) + .then(this.afterFetch(item)) + .catch((e) => { + // mock finally + item.endTime = performance.now() + item.duration = item.endTime - (item.startTime || item.endTime) + throw e + }) + } + + protected beforeFetch(item: NetworkMessage, input: RequestInfo, init?: RequestInit) { + let url: URL, + method = 'GET', + requestHeader: HeadersInit = {} + + // handle `input` content + if (typeof input === 'string') { + // when `input` is a string + method = init?.method || 'GET' + url = getURL(input) + requestHeader = init?.headers || {} + } else { + // when `input` is a `Request` object + method = input.method || 'GET' + url = getURL(input.url) + requestHeader = input.headers + } + + item.method = method + item.requestType = 'fetch' + item.requestHeader = requestHeader + item.url = url.toString() + item.name = (url.pathname.split('/').pop() || '') + url.search + item.status = 0 + item.statusText = 'Pending' + item.readyState = 1 + if (!item.startTime) { + // UNSENT + item.startTime = performance.now() + } + + if (Object.prototype.toString.call(requestHeader) === '[object Headers]') { + item.requestHeader = {} + for (const [key, value] of requestHeader) { + item.requestHeader[key] = value + } + } else { + item.requestHeader = requestHeader + } + + // save GET data + if (url.search && url.searchParams) { + item.getData = {} + for (const [key, value] of url.searchParams) { + item.getData[key] = value + } + } + + // save POST data + if (init?.body) { + item.requestData = genStringBody(init.body) + } + } + + protected afterFetch(item: NetworkMessage) { + return (resp: Response) => { + item.endTime = performance.now() + item.duration = item.endTime - (item.startTime || item.endTime) + item.status = resp.status + item.statusText = String(resp.status) + + let isChunked = false + item.header = {} + for (const [key, value] of resp.headers) { + item.header[key] = value + isChunked = value.toLowerCase().indexOf('chunked') > -1 ? true : isChunked + } + + if (isChunked) { + // when `transfer-encoding` is chunked, the response is a stream which is under loading, + // so the `readyState` should be 3 (Loading), + // and the response should NOT be `clone()` which will affect stream reading. + item.readyState = 3 + } else { + // Otherwise, not chunked, the response is not a stream, + // so it's completed and can be cloned for `text()` calling. + item.readyState = 4 + + void this.handleResponseBody(resp.clone(), item).then( + (responseValue: string | ArrayBuffer) => { + item.responseSize = + typeof responseValue === 'string' ? responseValue.length : responseValue.byteLength + item.responseSizeText = formatByteSize(item.responseSize) + item.response = getStringResponseByType(item.responseType, responseValue) + + this.sendMessage(item.getMessage()) + }, + ) + } + + return new Proxy(resp, new ResponseProxyHandler(resp, item)) + } + } + + protected handleResponseBody(resp: Response, item: NetworkMessage) { + // parse response body by Content-Type + const contentType = resp.headers.get('content-type') + if (contentType && contentType.includes('application/json')) { + item.responseType = 'json' + return resp.text() + } else if ( + contentType && + (contentType.includes('text/html') || contentType.includes('text/plain')) + ) { + item.responseType = 'text' + return resp.text() + } else { + item.responseType = 'arraybuffer' + return resp.arrayBuffer() + } + } +} + +export default class FetchProxy { + public static origFetch = fetch + + public static create( + ignoredHeaders: boolean | string[], + setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, + sanitize: (data: RequestResponseData) => RequestResponseData, + sendMessage: (item: NetworkRequest) => void, + isServiceUrl: (url: string) => boolean, + ) { + return new Proxy( + fetch, + new FetchProxyHandler( + ignoredHeaders, + setSessionTokenHeader, + sanitize, + sendMessage, + isServiceUrl, + ), + ) + } +} diff --git a/tracker/tracker/src/main/modules/Network/index.ts b/tracker/tracker/src/main/modules/Network/index.ts new file mode 100644 index 000000000..60a5e8f6f --- /dev/null +++ b/tracker/tracker/src/main/modules/Network/index.ts @@ -0,0 +1,28 @@ +import FetchProxy from './fetchProxy.js' +import XHRProxy from './xhrProxy.js' +import { RequestResponseData } from './types.js' +import { NetworkRequest } from '../../../common/messages.gen.js' + +export default function setProxy( + context: typeof globalThis, + ignoredHeaders: boolean | string[], + setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, + sanitize: (data: RequestResponseData) => RequestResponseData, + sendMessage: (message: NetworkRequest) => void, + isServiceUrl: (url: string) => boolean, +) { + context.XMLHttpRequest = XHRProxy.create( + ignoredHeaders, + setSessionTokenHeader, + sanitize, + sendMessage, + isServiceUrl, + ) + context.fetch = FetchProxy.create( + ignoredHeaders, + setSessionTokenHeader, + sanitize, + sendMessage, + isServiceUrl, + ) +} diff --git a/tracker/tracker/src/main/modules/Network/networkMessage.ts b/tracker/tracker/src/main/modules/Network/networkMessage.ts new file mode 100644 index 000000000..833895864 --- /dev/null +++ b/tracker/tracker/src/main/modules/Network/networkMessage.ts @@ -0,0 +1,99 @@ +import { NetworkRequest } from '../../app/messages.gen.js' +import { RequestResponseData } from './types.js' +import { getTimeOrigin } from '../../utils.js' + +export type httpMethod = + // '' is a rare case of error + '' | 'GET' | 'POST' | 'PUT' | 'DELETE' | 'HEAD' | 'CONNECT' | 'OPTIONS' | 'TRACE' | 'PATCH' + +export enum RequestState { + UNSENT = 0, + OPENED = 1, + HEADERS_RECEIVED = 2, + LOADING = 3, + DONE = 4, +} + +/** + * I know we're not using most of the information from this class + * but it can be useful in the future if we will decide to display more stuff in our ui + * */ + +export default class NetworkMessage { + id = '' + name?: string = '' + method: httpMethod = '' + url = '' + status = 0 + statusText?: string = '' + cancelState?: 0 | 1 | 2 | 3 = 0 + readyState?: RequestState = 0 + header: { [key: string]: string } = {} + responseType: XMLHttpRequest['responseType'] = '' + requestType: 'xhr' | 'fetch' | 'ping' | 'custom' + requestHeader: HeadersInit = {} + response: any + responseSize = 0 // bytes + responseSizeText = '' + startTime = 0 + endTime = 0 + duration = 0 + getData: { [key: string]: string } = {} + requestData: string | null = null + + constructor( + private readonly ignoredHeaders: boolean | string[] = [], + private readonly setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, + private readonly sanitize: (data: RequestResponseData) => RequestResponseData, + ) {} + + getMessage() { + const { reqHs, resHs } = this.writeHeaders() + const request = { + headers: reqHs, + body: this.method === 'GET' ? JSON.stringify(this.getData) : this.requestData, + } + const response = { headers: resHs, body: this.response } + + const messageInfo = this.sanitize({ + url: this.url, + method: this.method, + status: this.status, + request, + response, + }) + + return NetworkRequest( + this.requestType, + messageInfo.method, + messageInfo.url, + JSON.stringify(messageInfo.request), + JSON.stringify(messageInfo.response), + messageInfo.status, + this.startTime + getTimeOrigin(), + this.duration, + ) + } + + writeHeaders() { + const reqHs: Record = {} + Object.entries(this.requestHeader).forEach(([key, value]) => { + if (this.isHeaderIgnored(key)) return + reqHs[key] = value + }) + this.setSessionTokenHeader((name, value) => { + reqHs[name] = value + }) + const resHs: Record = {} + Object.entries(this.header).forEach(([key, value]) => { + if (this.isHeaderIgnored(key)) return + resHs[key] = value + }) + return { reqHs, resHs } + } + + isHeaderIgnored(key: string) { + if (Array.isArray(this.ignoredHeaders)) return this.ignoredHeaders.includes(key) + return this.ignoredHeaders + } +} diff --git a/tracker/tracker/src/main/modules/Network/types.ts b/tracker/tracker/src/main/modules/Network/types.ts new file mode 100644 index 000000000..14364700e --- /dev/null +++ b/tracker/tracker/src/main/modules/Network/types.ts @@ -0,0 +1,15 @@ +export interface RequestResponseData { + readonly status: number + readonly method: string + url: string + request: { + body: string | null + headers: Record + } + response: { + body: string | null + headers: Record + } +} + +// we only support sanitizing for json/string data because how you're gonna sanitize binary data? diff --git a/tracker/tracker/src/main/modules/Network/utils.ts b/tracker/tracker/src/main/modules/Network/utils.ts new file mode 100644 index 000000000..4dd90b3d3 --- /dev/null +++ b/tracker/tracker/src/main/modules/Network/utils.ts @@ -0,0 +1,205 @@ +export const genResponseByType = ( + responseType: XMLHttpRequest['responseType'], + response: any, +): string | Record => { + let result = '' + switch (responseType) { + case '': + case 'text': + case 'json': + if (typeof response == 'string') { + try { + result = JSON.parse(response) + } catch (e) { + // not a JSON string + result = response.slice(0, 10000) + } + } else if (isPureObject(response) || Array.isArray(response)) { + result = JSON.stringify(response) + } else if (typeof response !== 'undefined') { + result = Object.prototype.toString.call(response) + } + break + + case 'blob': + case 'document': + case 'arraybuffer': + default: + if (typeof response !== 'undefined') { + result = Object.prototype.toString.call(response) + } + break + } + return result +} + +export const getStringResponseByType = ( + responseType: XMLHttpRequest['responseType'], + response: any, +) => { + let result = '' + switch (responseType) { + case '': + case 'text': + case 'json': + if (typeof response == 'string') { + result = response + } else if (isPureObject(response) || Array.isArray(response)) { + result = JSON.stringify(response) + } else if (typeof response !== 'undefined') { + result = Object.prototype.toString.call(response) + } + break + case 'blob': + case 'document': + case 'arraybuffer': + default: + if (typeof response !== 'undefined') { + result = Object.prototype.toString.call(response) + } + break + } + return result +} + +export const genStringBody = (body?: BodyInit) => { + if (!body) { + return null + } + let result: string + + if (typeof body === 'string') { + if (body[0] === '{' || body[0] === '[') { + result = body + } + // 'a=1&b=2' => try to parse as query + const arr = body.split('&') + if (arr.length === 1) { + // not a query, parse as original string + result = body + } else { + // 'a=1&b=2&c' => parse as query + result = arr.join(',') + } + } else if (isIterable(body)) { + // FormData or URLSearchParams or Array + const arr = [] + for (const [key, value] of body) { + arr.push(`${key}=${typeof value === 'string' ? value : '[object Object]'}`) + } + result = arr.join(',') + } else if ( + body instanceof Blob || + body instanceof ReadableStream || + body instanceof ArrayBuffer + ) { + result = 'byte data' + } else if (isPureObject(body)) { + // overriding ArrayBufferView which is not convertable to string + result = body + } else { + result = `can't parse body ${typeof body}` + } + return result +} + +export const genGetDataByUrl = (url: string, getData: Record = {}) => { + if (!isPureObject(getData)) { + getData = {} + } + let query: string[] = url ? url.split('?') : [] // a.php?b=c&d=?e => ['a.php', 'b=c&d=', 'e'] + query.shift() // => ['b=c&d=', 'e'] + if (query.length > 0) { + query = query.join('?').split('&') // => 'b=c&d=?e' => ['b=c', 'd=?e'] + for (const q of query) { + const kv = q.split('=') + try { + getData[kv[0]] = decodeURIComponent(kv[1]) + } catch (e) { + // "URIError: URI malformed" will be thrown when `kv[1]` contains "%", so just use raw data + // @issue #470 + // @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Malformed_URI + getData[kv[0]] = kv[1] + } + } + } + return getData +} + +export const genFormattedBody = (body?: BodyInit) => { + if (!body) { + return null + } + let result: string | { [key: string]: string } + + if (typeof body === 'string') { + try { + // '{a:1}' => + result = JSON.parse(body) + } catch (e) { + // 'a=1&b=2' => try to parse as query + const arr = body.split('&') + result = {} + // eslint-disable-next-line + for (let q of arr) { + const kv = q.split('=') + result[kv[0]] = kv[1] === undefined ? 'undefined' : kv[1] + } + } + } else if (isIterable(body)) { + // FormData or URLSearchParams or Array + result = {} + for (const [key, value] of body) { + result[key] = typeof value === 'string' ? value : '[object Object]' + } + } else if ( + body instanceof Blob || + body instanceof ReadableStream || + body instanceof ArrayBuffer + ) { + result = 'byte data' + } else if (isPureObject(body)) { + // overriding ArrayBufferView which is not convertable to string + result = body + } else { + result = `can't parse body ${typeof body}` + } + return result +} + +export function isPureObject(input: any): input is Record { + return null !== input && typeof input === 'object' +} + +export function isIterable(value: any) { + if (value === null || value === undefined) { + return false + } + return typeof Symbol !== 'undefined' && typeof value[Symbol.iterator] === 'function' +} + +export function formatByteSize(bytes: number) { + if (bytes <= 0) { + // shouldn't happen? + return '' + } + if (bytes >= 1000 * 1000) { + return (bytes / 1000 / 1000).toFixed(1) + ' MB' + } + if (bytes >= 1000) { + return (bytes / 1000).toFixed(1) + ' KB' + } + return `${bytes}B` +} + +export const getURL = (urlString: string) => { + if (urlString.startsWith('//')) { + const baseUrl = new URL(window.location.href) + urlString = `${baseUrl.protocol}${urlString}` + } + if (urlString.startsWith('http')) { + return new URL(urlString) + } else { + return new URL(urlString, window.location.href) + } +} diff --git a/tracker/tracker/src/main/modules/Network/xhrProxy.ts b/tracker/tracker/src/main/modules/Network/xhrProxy.ts new file mode 100644 index 000000000..c97048b03 --- /dev/null +++ b/tracker/tracker/src/main/modules/Network/xhrProxy.ts @@ -0,0 +1,246 @@ +/** + * I took inspiration in few stack exchange posts + * and Tencent vConsole library (MIT) + * by wrapping the XMLHttpRequest object in a Proxy + * we can intercept the network requests + * in not-so-hacky way + * */ + +import NetworkMessage, { RequestState } from './networkMessage.js' +import { genGetDataByUrl, formatByteSize, genStringBody, getStringResponseByType } from './utils.js' +import { RequestResponseData } from './types.js' +import { NetworkRequest } from '../../../common/messages.gen.js' + +export class XHRProxyHandler implements ProxyHandler { + public XMLReq: XMLHttpRequest + public item: NetworkMessage + + constructor( + XMLReq: XMLHttpRequest, + private readonly ignoredHeaders: boolean | string[], + private readonly setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, + private readonly sanitize: (data: RequestResponseData) => RequestResponseData, + private readonly sendMessage: (message: NetworkRequest) => void, + private readonly isServiceUrl: (url: string) => boolean, + ) { + this.XMLReq = XMLReq + this.XMLReq.onreadystatechange = () => { + this.onReadyStateChange() + } + this.XMLReq.onabort = () => { + this.onAbort() + } + this.XMLReq.ontimeout = () => { + this.onTimeout() + } + this.item = new NetworkMessage(ignoredHeaders, setSessionTokenHeader, sanitize) + this.item.requestType = 'xhr' + } + + public get(target: T, key: string) { + switch (key) { + case 'open': + return this.getOpen(target) + case 'send': + return this.getSend(target) + case 'setRequestHeader': + return this.getSetRequestHeader(target) + default: + // eslint-disable-next-line no-case-declarations + const value = Reflect.get(target, key) + if (typeof value === 'function') { + return value.bind(target) + } else { + return value + } + } + } + + public set(target: T, key: string, value: (args: any[]) => any) { + switch (key) { + case 'onreadystatechange': + return this.setOnReadyStateChange(target, key, value) + case 'onabort': + return this.setOnAbort(target, key, value) + case 'ontimeout': + return this.setOnTimeout(target, key, value) + default: + // not tracked methods + } + return Reflect.set(target, key, value) + } + + public onReadyStateChange() { + if (this.item.url && this.isServiceUrl(this.item.url)) return + this.item.readyState = this.XMLReq.readyState + this.item.responseType = this.XMLReq.responseType + this.item.endTime = performance.now() + this.item.duration = this.item.endTime - this.item.startTime + this.updateItemByReadyState() + setTimeout(() => { + this.item.response = getStringResponseByType(this.item.responseType, this.item.response) + }, 0) + + if (this.XMLReq.readyState === RequestState.DONE) { + this.sendMessage(this.item.getMessage()) + } + } + + public onAbort() { + this.item.cancelState = 1 + this.item.statusText = 'Abort' + + this.sendMessage(this.item.getMessage()) + } + + public onTimeout() { + this.item.cancelState = 3 + this.item.statusText = 'Timeout' + + this.sendMessage(this.item.getMessage()) + } + + protected getOpen(target: T) { + const targetFunction = Reflect.get(target, 'open') + return (...args: any[]) => { + const method = args[0] + const url = args[1] + this.item.method = method ? method.toUpperCase() : 'GET' + this.item.url = url || '' + this.item.name = this.item.url.replace(new RegExp('/*$'), '').split('/').pop() || '' + this.item.getData = genGetDataByUrl(this.item.url, {}) + return targetFunction.apply(target, args) + } + } + + protected getSend(target: T) { + const targetFunction = Reflect.get(target, 'send') + return (...args: any[]) => { + const data: XMLHttpRequestBodyInit = args[0] + this.item.requestData = genStringBody(data) + return targetFunction.apply(target, args) + } + } + + protected getSetRequestHeader(target: T) { + const targetFunction = Reflect.get(target, 'setRequestHeader') + return (...args: any[]) => { + if (!this.item.requestHeader) { + this.item.requestHeader = {} + } + // @ts-ignore + this.item.requestHeader[args[0]] = args[1] + return targetFunction.apply(target, args) + } + } + + protected setOnReadyStateChange(target: T, key: string, orscFunction: (args: any[]) => any) { + return Reflect.set(target, key, (...args: any[]) => { + this.onReadyStateChange() + orscFunction.apply(target, args) + }) + } + + protected setOnAbort(target: T, key: string, oaFunction: (args: any[]) => any) { + return Reflect.set(target, key, (...args: any[]) => { + this.onAbort() + oaFunction.apply(target, args) + }) + } + + protected setOnTimeout(target: T, key: string, otFunction: (args: any[]) => any) { + return Reflect.set(target, key, (...args: any[]) => { + this.onTimeout() + otFunction.apply(target, args) + }) + } + + /** + * Update item's properties according to readyState. + */ + protected updateItemByReadyState() { + switch (this.XMLReq.readyState) { + case RequestState.UNSENT: + case RequestState.OPENED: + this.item.status = RequestState.UNSENT + this.item.statusText = 'Pending' + if (!this.item.startTime) { + this.item.startTime = performance.now() + } + break + case RequestState.HEADERS_RECEIVED: + this.item.status = this.XMLReq.status + this.item.statusText = 'Loading' + this.item.header = {} + // eslint-disable-next-line no-case-declarations + const header = this.XMLReq.getAllResponseHeaders() || '', + headerArr = header.split('\n') + // extract plain text to key-value format + for (let i = 0; i < headerArr.length; i++) { + const line = headerArr[i] + if (!line) { + continue + } + const arr = line.split(': ') + const key = arr[0] + this.item.header[key] = arr.slice(1).join(': ') + } + break + case RequestState.LOADING: + this.item.status = this.XMLReq.status + this.item.statusText = 'Loading' + if (!!this.XMLReq.response && this.XMLReq.response.length) { + this.item.responseSize = this.XMLReq.response.length + this.item.responseSizeText = formatByteSize(this.item.responseSize) + } + break + case RequestState.DONE: + // `XMLReq.abort()` will change `status` from 200 to 0, so use previous value in this case + this.item.status = this.XMLReq.status || this.item.status || 0 + // show status code when request completed + this.item.statusText = String(this.item.status) + this.item.endTime = performance.now() + this.item.duration = this.item.endTime - (this.item.startTime || this.item.endTime) + this.item.response = this.XMLReq.response + + if (!!this.XMLReq.response && this.XMLReq.response.length) { + this.item.responseSize = this.XMLReq.response.length + this.item.responseSizeText = formatByteSize(this.item.responseSize) + } + break + default: + this.item.status = this.XMLReq.status + this.item.statusText = 'Unknown' + break + } + } +} + +export default class XHRProxy { + public static origXMLHttpRequest = XMLHttpRequest + + public static create( + ignoredHeaders: boolean | string[], + setSessionTokenHeader: (cb: (name: string, value: string) => void) => void, + sanitize: (data: RequestResponseData) => RequestResponseData, + sendMessage: (data: NetworkRequest) => void, + isServiceUrl: (url: string) => boolean, + ) { + return new Proxy(XMLHttpRequest, { + construct(original: any) { + const XMLReq = new original() + return new Proxy( + XMLReq, + new XHRProxyHandler( + XMLReq as XMLHttpRequest, + ignoredHeaders, + setSessionTokenHeader, + sanitize, + sendMessage, + isServiceUrl, + ), + ) + }, + }) + } +} diff --git a/tracker/tracker/src/main/modules/console.ts b/tracker/tracker/src/main/modules/console.ts index 2a2b61c1d..2e74bcfb2 100644 --- a/tracker/tracker/src/main/modules/console.ts +++ b/tracker/tracker/src/main/modules/console.ts @@ -1,5 +1,4 @@ import type App from '../app/index.js' -import { hasTag } from '../app/guards.js' import { IN_BROWSER } from '../utils.js' import { ConsoleLog } from '../app/messages.gen.js' @@ -104,6 +103,7 @@ export default function (app: App, opts: Partial): void { }, opts, ) + if (!Array.isArray(options.consoleMethods) || options.consoleMethods.length === 0) { return } @@ -112,29 +112,40 @@ export default function (app: App, opts: Partial): void { app.send(ConsoleLog(level, printf(args))), ) - let n: number + let n = 0 const reset = (): void => { n = 0 } app.attachStartCallback(reset) app.ticker.attach(reset, 33, false) - const patchConsole = (console: Console) => + const patchConsole = (console: Console, ctx: typeof globalThis) => { + const handler = { + apply: function (target: Console['log'], thisArg: typeof this, argumentsList: unknown[]) { + Reflect.apply(target, ctx, argumentsList) + n = n + 1 + if (n > options.consoleThrottling) { + return + } else { + sendConsoleLog(target.name, argumentsList) + } + }, + } + options.consoleMethods!.forEach((method) => { if (consoleMethods.indexOf(method) === -1) { app.debug.error(`OpenReplay: unsupported console method "${method}"`) return } - const fn = (console as any)[method] - ;(console as any)[method] = function (...args: unknown[]): void { - fn.apply(this, args) - if (n++ > options.consoleThrottling) { - return - } - sendConsoleLog(method, args) - } + const fn = (ctx.console as any)[method] + // is there any way to preserve the original console trace? + ;(console as any)[method] = new Proxy(fn, handler) }) - const patchContext = app.safe((context: typeof globalThis) => patchConsole(context.console)) + } + + const patchContext = app.safe((context: typeof globalThis) => + patchConsole(context.console, context), + ) patchContext(window) app.observer.attachContextCallback(patchContext) diff --git a/tracker/tracker/src/main/modules/network.ts b/tracker/tracker/src/main/modules/network.ts index b8374a5b7..86fee1426 100644 --- a/tracker/tracker/src/main/modules/network.ts +++ b/tracker/tracker/src/main/modules/network.ts @@ -3,42 +3,13 @@ import { NetworkRequest } from '../app/messages.gen.js' import { getTimeOrigin } from '../utils.js' import type { AxiosInstance } from './axiosSpy.js' import axiosSpy from './axiosSpy.js' +import setProxy from './Network/index.js' type WindowFetch = typeof window.fetch type XHRRequestBody = Parameters[0] -type FetchRequestBody = RequestInit['body'] - -// Request: -// declare const enum BodyType { -// Blob = "Blob", -// ArrayBuffer = "ArrayBuffer", -// TypedArray = "TypedArray", -// DataView = "DataView", -// FormData = "FormData", -// URLSearchParams = "URLSearchParams", -// Document = "Document", // XHR only -// ReadableStream = "ReadableStream", // Fetch only -// Literal = "literal", -// Unknown = "unk", -// } -// XHRResponse body: ArrayBuffer, a Blob, a Document, a JavaScript Object, or a string - -// TODO: extract maximum of useful information from any type of Request/Response bodies -// function objectifyBody(body: any): RequestBody { -// if (body instanceof Blob) { -// return { -// body: `; size: ${body.size}`, -// bodyType: BodyType.Blob, -// } -// } -// return { -// body, -// bodyType: BodyType.Literal, -// } -// } interface RequestData { - body: XHRRequestBody | FetchRequestBody + body: string | null headers: Record } @@ -74,7 +45,7 @@ function strMethod(method?: string) { return typeof method === 'string' ? method.toUpperCase() : 'GET' } -type Sanitizer = (data: RequestResponseData) => RequestResponseData | null +type Sanitizer = (data: RequestResponseData) => RequestResponseData export interface Options { sessionTokenHeader: string | boolean @@ -84,6 +55,7 @@ export interface Options { captureInIframes: boolean sanitizer?: Sanitizer axiosInstances?: Array + useProxy?: boolean } export default function (app: App, opts: Partial = {}) { @@ -95,10 +67,17 @@ export default function (app: App, opts: Partial = {}) { sessionTokenHeader: false, captureInIframes: true, axiosInstances: undefined, + useProxy: false, }, opts, ) + if (options.useProxy === false) { + app.debug.warn( + 'Network module is migrating to proxy api, to gradually migrate and test it set useProxy to true', + ) + } + const ignoreHeaders = options.ignoreHeaders const isHIgnored = Array.isArray(ignoreHeaders) ? (name: string) => ignoreHeaders.includes(name) @@ -118,13 +97,14 @@ export default function (app: App, opts: Partial = {}) { function sanitize(reqResInfo: RequestResponseData) { if (!options.capturePayload) { + // @ts-ignore delete reqResInfo.request.body delete reqResInfo.response.body } if (options.sanitizer) { const resBody = reqResInfo.response.body if (typeof resBody === 'string') { - // Parse response in order to have handy view in sanitisation function + // Parse response in order to have handy view in sanitization function try { reqResInfo.response.body = JSON.parse(resBody) } catch {} @@ -147,6 +127,17 @@ export default function (app: App, opts: Partial = {}) { } const patchWindow = (context: typeof globalThis) => { + /* ====== modern way ====== */ + if (options.useProxy) { + return setProxy( + context, + options.ignoreHeaders, + setSessionTokenHeader, + sanitize, + (message) => app.send(message), + (url) => app.isServiceURL(url), + ) + } /* ====== Fetch ====== */ const origFetch = context.fetch.bind(context) as WindowFetch @@ -207,7 +198,8 @@ export default function (app: App, opts: Partial = {}) { status: r.status, request: { headers: reqHs, - body: init.body, + // @ts-ignore + body: init.body || null, }, response: { headers: resHs, @@ -277,7 +269,8 @@ export default function (app: App, opts: Partial = {}) { status: xhr.status, request: { headers: reqHs, - body: reqBody, + // @ts-ignore + body: reqBody || null, }, response: { headers: headerMap, diff --git a/tracker/tracker/src/tests/console.test.ts b/tracker/tracker/src/tests/console.test.ts new file mode 100644 index 000000000..085b008be --- /dev/null +++ b/tracker/tracker/src/tests/console.test.ts @@ -0,0 +1,124 @@ +// @ts-nocheck +import mainFunction, { Options } from '../main/modules/console.js' // replace with actual module path +import { describe, beforeEach, afterEach, it, expect } from '@jest/globals' + +jest.useFakeTimers() + +describe('Console logging module', () => { + let originalConsole + let mockApp + + beforeEach(() => { + originalConsole = global.console + global.console = { + log: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + assert: jest.fn(), + } + + mockApp = { + safe: jest.fn((callback) => callback), + send: jest.fn(), + attachStartCallback: jest.fn(), + ticker: { + attach: jest.fn(), + }, + debug: { + error: jest.fn(), + }, + observer: { + attachContextCallback: jest.fn(), + }, + } + }) + + afterEach(() => { + global.console = originalConsole + }) + + it('should patch console methods', () => { + mainFunction(mockApp, {}) + jest.useFakeTimers() + global.console.log('test log') + jest.advanceTimersByTime(9999) + // 22 - Console message + expect(mockApp.send).toHaveBeenCalledWith([22, 'mockConstructor', 'test log']) + }) + + it('should respect consoleThrottling', async () => { + const options: Options = { + consoleMethods: ['log'], + consoleThrottling: 1, + } + mainFunction(mockApp, options) + jest.runAllTimers() + global.console.log('test log 1') + global.console.log('test log 2') + global.console.log('test log 3') + global.console.log('test log 4') + + expect(mockApp.send).toHaveBeenCalledTimes(1) + }) + + it('should not patch console methods when consoleMethods is null', () => { + const options: Options = { + consoleMethods: null, + consoleThrottling: 30, + } + mainFunction(mockApp, options) + global.console.log('test log') + expect(mockApp.send).not.toHaveBeenCalled() + }) + + it('should not patch console methods when consoleMethods is an empty array', () => { + const options: Options = { + consoleMethods: [], + consoleThrottling: 30, + } + mainFunction(mockApp, options) + global.console.log('test log') + expect(mockApp.send).not.toHaveBeenCalled() + }) + + it('should log an error when an unsupported console method is provided', () => { + const options: Options = { + consoleMethods: ['unsupportedMethod'], + consoleThrottling: 30, + } + mainFunction(mockApp, options) + expect(mockApp.debug.error).toHaveBeenCalledWith( + 'OpenReplay: unsupported console method "unsupportedMethod"', + ) + }) + + // More tests for the printf function + it('should correctly print different argument types', () => { + const options: Options = { + consoleMethods: ['log'], + consoleThrottling: 30, + } + mainFunction(mockApp, options) + global.console.log('%s %f %d %o', 'test', 3.14, 42, { key: 'value' }) + jest.advanceTimersByTimeAsync(110) + expect(mockApp.send).toHaveBeenCalledWith([22, 'mockConstructor', 'test 3.14 42 {key: value}']) + }) + + // More tests for the printObject function + it('should correctly print different object types', () => { + const options: Options = { + consoleMethods: ['log'], + consoleThrottling: 30, + } + mainFunction(mockApp, options) + global.console.log([1, 2, 3], { key1: 'value1', key2: 'value2' }) + jest.advanceTimersByTimeAsync(110) + expect(mockApp.send).toHaveBeenCalledWith([ + 22, + 'mockConstructor', + 'Array(3)[1, 2, 3] {key1: value1, key2: value2}', + ]) + }) +}) diff --git a/tracker/tracker/src/tests/network.utils.test.ts b/tracker/tracker/src/tests/network.utils.test.ts new file mode 100644 index 000000000..1aae41eb6 --- /dev/null +++ b/tracker/tracker/src/tests/network.utils.test.ts @@ -0,0 +1,80 @@ +import { + genResponseByType, + getStringResponseByType, + genStringBody, + genGetDataByUrl, + genFormattedBody, + isPureObject, + isIterable, + formatByteSize, + getURL, +} from '../main/modules/Network/utils.js' +import { describe, it, expect, jest } from '@jest/globals' + +describe('Network utility function tests', () => { + it('genResponseByType should handle response types correctly', () => { + expect(genResponseByType('json', '{"key":"value"}')).toEqual({ key: 'value' }) + expect(genResponseByType('blob', new Blob())).toBe('[object Blob]') + }) + + it('getStringResponseByType should handle response types correctly', () => { + expect(getStringResponseByType('json', '{"key":"value"}')).toBe('{"key":"value"}') + expect(getStringResponseByType('json', { key: 'value' })).toBe('{"key":"value"}') + expect(getStringResponseByType('blob', new Blob())).toBe('[object Blob]') + }) + + it('genStringBody should handle body types correctly', () => { + expect(genStringBody('{"key":"value"}')).toBe('{"key":"value"}') + expect(genStringBody(new URLSearchParams('key=value'))).toBe('key=value') + // Add more cases as needed + }) + + it('genGetDataByUrl should get data from URL', () => { + expect(genGetDataByUrl('http://localhost/?key=value')).toEqual({ key: 'value' }) + // Add more cases as needed + }) + it('genGetDataByUrl handles wrong format', () => { + // @ts-ignore + expect(genGetDataByUrl('http://localhost/?key=value', '')).toEqual({ key: 'value' }) + }) + + it('genFormattedBody should format body correctly', () => { + const param = new URLSearchParams('key=value&other=test') + const blob = new Blob([param.toString()], { type: 'text/plain' }) + expect(genFormattedBody('{"key":"value"}')).toEqual({ key: 'value' }) + expect(genFormattedBody('key=value&other=test')).toEqual({ key: 'value', other: 'test' }) + expect(genFormattedBody(param)).toEqual({ key: 'value', other: 'test' }) + expect(genFormattedBody(blob)).toEqual('byte data') + }) + + it('isPureObject should return true for objects', () => { + expect(isPureObject({})).toBe(true) + expect(isPureObject([])).toBe(true) + expect(isPureObject(null)).toBe(false) + expect(isPureObject(undefined)).toBe(false) + }) + + it('isIterable should return true for iterables', () => { + expect(isIterable([])).toBe(true) + expect(isIterable(new Map())).toBe(true) + expect(isIterable('string')).toBe(true) + expect(isIterable(undefined)).toBe(false) + }) + + it('formatByteSize should format byte sizes correctly', () => { + expect(formatByteSize(500)).toBe('500B') + expect(formatByteSize(1500)).toBe('1.5 KB') + expect(formatByteSize(1500000)).toBe('1.5 MB') + expect(formatByteSize(-1)).toBe('') + }) + + it('getURL should get a URL', () => { + // @ts-ignore + delete window.location + // @ts-ignore + window.location = new URL('https://www.example.com') + expect(getURL('https://example.com').toString()).toBe('https://example.com/') + expect(getURL('//example.com').toString()).toBe('https://example.com/') + expect(getURL('/path').toString()).toBe('https://www.example.com/path') + }) +}) diff --git a/tracker/tracker/src/tests/networkMessage.test.ts b/tracker/tracker/src/tests/networkMessage.test.ts new file mode 100644 index 000000000..66e8a1268 --- /dev/null +++ b/tracker/tracker/src/tests/networkMessage.test.ts @@ -0,0 +1,90 @@ +import NetworkMessage, { RequestState, httpMethod } from '../main/modules/Network/networkMessage.js' +import { NetworkRequest } from '../main/app/messages.gen.js' +import { describe, it, expect, beforeEach, jest } from '@jest/globals' + +describe('NetworkMessage', () => { + const ignoredHeaders = ['cookie'] + const setSessionTokenHeader = jest.fn() + const sanitize = jest.fn() + + beforeEach(() => { + jest.clearAllMocks() + }) + + describe('getMessage', () => { + it('should properly construct and return a NetworkRequest', () => { + // @ts-ignore + const networkMessage = new NetworkMessage(ignoredHeaders, setSessionTokenHeader, sanitize) + + // Set some basic properties + networkMessage.method = 'GET' + networkMessage.url = 'https://example.com' + networkMessage.status = 200 + networkMessage.requestType = 'xhr' + networkMessage.startTime = 0 + networkMessage.duration = 500 + networkMessage.getData = { key: 'value' } + + // Expect sanitized message + sanitize.mockReturnValueOnce({ + url: 'https://example.com', + method: 'GET', + status: 200, + request: {}, + response: {}, + }) + + const result = networkMessage.getMessage() + + const expected = NetworkRequest( + 'xhr', + 'GET', + 'https://example.com', + JSON.stringify({}), + JSON.stringify({}), + 200, + // yeah + result[7], + 500, + ) + expect(result).toBeDefined() + expect(result).toEqual(expected) + expect(sanitize).toHaveBeenCalledTimes(1) + }) + }) + + describe('writeHeaders', () => { + it('should properly write request and response headers', () => { + // @ts-ignore + const networkMessage = new NetworkMessage(ignoredHeaders, setSessionTokenHeader, sanitize) + + networkMessage.requestHeader = { 'Content-Type': 'application/json', cookie: 'test' } + networkMessage.header = { 'Content-Type': 'application/json', cookie: 'test' } + + const result = networkMessage.writeHeaders() + + expect(result).toBeDefined() + expect(result.reqHs).toEqual({ 'Content-Type': 'application/json' }) + expect(result.resHs).toEqual({ 'Content-Type': 'application/json' }) + expect(setSessionTokenHeader).toHaveBeenCalledTimes(1) + }) + }) + + describe('isHeaderIgnored', () => { + it('should properly identify ignored headers', () => { + // @ts-ignore + const networkMessage = new NetworkMessage(ignoredHeaders, setSessionTokenHeader, sanitize) + + expect(networkMessage.isHeaderIgnored('cookie')).toBe(true) + expect(networkMessage.isHeaderIgnored('Content-Type')).toBe(false) + }) + it('if ignoreHeaders is true should ignore all headers', () => { + // @ts-ignore + const networkMessage = new NetworkMessage(true, setSessionTokenHeader, sanitize) + + expect(networkMessage.isHeaderIgnored('cookie')).toBe(true) + expect(networkMessage.isHeaderIgnored('Content-Type')).toBe(true) + expect(networkMessage.isHeaderIgnored('Random-Header')).toBe(true) + }) + }) +})