diff --git a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx index 13e9c2655..f433a29d0 100644 --- a/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx +++ b/frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx @@ -1,9 +1,17 @@ /* eslint-disable i18next/no-literal-string */ import { ResourceType, Timed } from 'Player'; +import { WsChannel } from 'Player/web/messages'; import MobilePlayer from 'Player/mobile/IOSPlayer'; import WebPlayer from 'Player/web/WebPlayer'; import { observer } from 'mobx-react-lite'; -import React, { useMemo, useState } from 'react'; +import React, { + useMemo, + useState, + useEffect, + useCallback, + useRef, +} from 'react'; +import i18n from 'App/i18n' import { useModal } from 'App/components/Modal'; import { @@ -12,25 +20,27 @@ import { } from 'App/components/Session/playerContext'; import { formatMs } from 'App/date'; import { useStore } from 'App/mstore'; -import { formatBytes } from 'App/utils'; +import { formatBytes, debounceCall } from 'App/utils'; import { Icon, NoContent, Tabs } from 'UI'; import { Tooltip, Input, Switch, Form } from 'antd'; -import { SearchOutlined, InfoCircleOutlined } from '@ant-design/icons'; +import { + SearchOutlined, + InfoCircleOutlined, +} from '@ant-design/icons'; import FetchDetailsModal from 'Shared/FetchDetailsModal'; -import { WsChannel } from 'App/player/web/messages'; import BottomBlock from '../BottomBlock'; import InfoLine from '../BottomBlock/InfoLine'; import TabSelector from '../TabSelector'; import TimeTable from '../TimeTable'; import useAutoscroll, { getLastItemTime } from '../useAutoscroll'; -import { useRegExListFilterMemo, useTabListFilterMemo } from '../useListFilter'; import WSPanel from './WSPanel'; import { useTranslation } from 'react-i18next'; +import { mergeListsWithZoom, processInChunks } from './utils' +// Constants remain the same const INDEX_KEY = 'network'; - const ALL = 'ALL'; const XHR = 'xhr'; const JS = 'js'; @@ -62,6 +72,9 @@ export const NETWORK_TABS = TAP_KEYS.map((tab) => ({ const DOM_LOADED_TIME_COLOR = 'teal'; const LOAD_TIME_COLOR = 'red'; +const BATCH_SIZE = 2500; +const INITIAL_LOAD_SIZE = 5000; + export function renderType(r: any) { return ( {r.type}}> @@ -79,13 +92,17 @@ export function renderName(r: any) { } function renderSize(r: any) { - const { t } = useTranslation(); - if (r.responseBodySize) return formatBytes(r.responseBodySize); + const t = i18n.t; + const notCaptured = t('Not captured'); + const resSizeStr = t('Resource size') let triggerText; let content; - if (r.decodedBodySize == null || r.decodedBodySize === 0) { + if (r.responseBodySize) { + triggerText = formatBytes(r.responseBodySize); + content = undefined; + } else if (r.decodedBodySize == null || r.decodedBodySize === 0) { triggerText = 'x'; - content = t('Not captured'); + content = notCaptured; } else { const headerSize = r.headerSize || 0; const showTransferred = r.headerSize != null; @@ -100,7 +117,7 @@ function renderSize(r: any) { )} transferred over network`} )} -
  • {`${t('Resource size')}: ${formatBytes(r.decodedBodySize)} `}
    • +
  • {`${resSizeStr}: ${formatBytes(r.decodedBodySize)} `}
    • ); } @@ -168,6 +185,8 @@ function renderStatus({ ); } + +// Main component for Network Panel function NetworkPanelCont({ panelHeight }: { panelHeight: number }) { const { player, store } = React.useContext(PlayerContext); const { sessionStore, uiPlayerStore } = useStore(); @@ -216,6 +235,7 @@ function NetworkPanelCont({ panelHeight }: { panelHeight: number }) { const getTabNum = (tab: string) => tabsArr.findIndex((t) => t === tab) + 1; const getTabName = (tabId: string) => tabNames[tabId]; + return ( void, hasMore: boolean) => { + const observerRef = useRef(null); + const loadingRef = useRef(null); + + useEffect(() => { + const observer = new IntersectionObserver( + (entries) => { + if (entries[0]?.isIntersecting && hasMore) { + loadMoreCallback(); + } + }, + { threshold: 0.1 }, + ); + + if (loadingRef.current) { + observer.observe(loadingRef.current); + } + + // @ts-ignore + observerRef.current = observer; + + return () => { + if (observerRef.current) { + observerRef.current.disconnect(); + } + }; + }, [loadMoreCallback, hasMore, loadingRef]); + + return loadingRef; }; interface Props { @@ -302,8 +344,8 @@ interface Props { resourceList: Timed[]; fetchListNow: Timed[]; resourceListNow: Timed[]; - websocketList: Array; - websocketListNow: Array; + websocketList: Array; + websocketListNow: Array; player: WebPlayer | MobilePlayer; startedAt: number; isMobile?: boolean; @@ -349,107 +391,190 @@ export const NetworkPanelComp = observer( >(null); const { showModal } = useModal(); const [showOnlyErrors, setShowOnlyErrors] = useState(false); - const [isDetailsModalActive, setIsDetailsModalActive] = useState(false); + const [isLoading, setIsLoading] = useState(true); + const [isProcessing, setIsProcessing] = useState(false); + const [displayedItems, setDisplayedItems] = useState([]); + const [totalItems, setTotalItems] = useState(0); + const [summaryStats, setSummaryStats] = useState({ + resourcesSize: 0, + transferredSize: 0, + }); + + // Store original data in refs to avoid reprocessing + const originalListRef = useRef([]); + const socketListRef = useRef([]); + const { sessionStore: { devTools }, } = useStore(); const { filter } = devTools[INDEX_KEY]; const { activeTab } = devTools[INDEX_KEY]; const activeIndex = activeOutsideIndex ?? devTools[INDEX_KEY].index; + const [inputFilterValue, setInputFilterValue] = useState(filter); - const socketList = useMemo( - () => - websocketList.filter( - (ws, i, arr) => - arr.findIndex((it) => it.channelName === ws.channelName) === i, - ), - [websocketList], + const debouncedFilter = useCallback( + debounceCall((filterValue) => { + devTools.update(INDEX_KEY, { filter: filterValue }); + }, 300), + [], ); - const list = useMemo( - () => - // TODO: better merge (with body size info) - do it in player - resourceList - .filter( - (res) => - !fetchList.some((ft) => { - // res.url !== ft.url doesn't work on relative URLs appearing within fetchList (to-fix in player) - if (res.name === ft.name) { - if (res.time === ft.time) return true; - if (res.url.includes(ft.url)) { - return ( - Math.abs(res.time - ft.time) < 350 || - Math.abs(res.timestamp - ft.timestamp) < 350 - ); - } - } - - if (res.name !== ft.name) { - return false; - } - if (Math.abs(res.time - ft.time) > 250) { - return false; - } // TODO: find good epsilons - if (Math.abs(res.duration - ft.duration) > 200) { - return false; - } - - return true; - }), - ) - .concat(fetchList) - .concat( - socketList.map((ws) => ({ - ...ws, - type: 'websocket', - method: 'ws', - url: ws.channelName, - name: ws.channelName, - status: '101', - duration: 0, - transferredBodySize: 0, - })), - ) - .filter((req) => - zoomEnabled - ? req.time >= zoomStartTs! && req.time <= zoomEndTs! - : true, - ) - .sort((a, b) => a.time - b.time), - [resourceList.length, fetchList.length, socketList.length], - ); - - let filteredList = useMemo(() => { - if (!showOnlyErrors) { - return list; - } - return list.filter( - (it) => parseInt(it.status) >= 400 || !it.success || it.error, + // Process socket lists once + useEffect(() => { + const uniqueSocketList = websocketList.filter( + (ws, i, arr) => + arr.findIndex((it) => it.channelName === ws.channelName) === i, ); - }, [showOnlyErrors, list]); - filteredList = useRegExListFilterMemo( - filteredList, - (it) => [it.status, it.name, it.type, it.method], - filter, - ); - filteredList = useTabListFilterMemo( - filteredList, - (it) => TYPE_TO_TAB[it.type], - ALL, - activeTab, - ); + socketListRef.current = uniqueSocketList; + }, [websocketList.length]); - const onTabClick = (activeTab: (typeof TAP_KEYS)[number]) => + // Initial data processing - do this only once when data changes + useEffect(() => { + setIsLoading(true); + + // Heaviest operation here, will create a final merged network list + const processData = async () => { + const fetchUrls = new Set( + fetchList.map((ft) => { + return `${ft.name}-${Math.floor(ft.time / 100)}-${Math.floor(ft.duration / 100)}`; + }), + ); + + // We want to get resources that aren't in fetch list + const filteredResources = await processInChunks(resourceList, (chunk) => + chunk.filter((res: any) => { + const key = `${res.name}-${Math.floor(res.time / 100)}-${Math.floor(res.duration / 100)}`; + return !fetchUrls.has(key); + }), + BATCH_SIZE, + 25, + ); + + const processedSockets = socketListRef.current.map((ws: any) => ({ + ...ws, + type: 'websocket', + method: 'ws', + url: ws.channelName, + name: ws.channelName, + status: '101', + duration: 0, + transferredBodySize: 0, + })); + + const mergedList: Timed[] = mergeListsWithZoom( + filteredResources as Timed[], + fetchList, + processedSockets as Timed[], + { enabled: Boolean(zoomEnabled), start: zoomStartTs ?? 0, end: zoomEndTs ?? 0 } + ) + + originalListRef.current = mergedList; + setTotalItems(mergedList.length); + + calculateResourceStats(resourceList); + + // Only display initial chunk + setDisplayedItems(mergedList.slice(0, INITIAL_LOAD_SIZE)); + setIsLoading(false); + }; + + void processData(); + }, [ + resourceList.length, + fetchList.length, + socketListRef.current.length, + zoomEnabled, + zoomStartTs, + zoomEndTs, + ]); + + const calculateResourceStats = (resourceList: Record) => { + setTimeout(() => { + let resourcesSize = 0 + let transferredSize = 0 + resourceList.forEach(({ decodedBodySize, headerSize, encodedBodySize }: any) => { + resourcesSize += decodedBodySize || 0 + transferredSize += (headerSize || 0) + (encodedBodySize || 0) + }) + + setSummaryStats({ + resourcesSize, + transferredSize, + }); + }, 0); + } + + useEffect(() => { + if (originalListRef.current.length === 0) return; + setIsProcessing(true); + const applyFilters = async () => { + let filteredItems: any[] = originalListRef.current; + + filteredItems = await processInChunks(filteredItems, (chunk) => + chunk.filter( + (it) => { + let valid = true; + if (showOnlyErrors) { + valid = parseInt(it.status) >= 400 || !it.success || it.error + } + if (filter) { + try { + const regex = new RegExp(filter, 'i'); + valid = valid && regex.test(it.status) || regex.test(it.name) || regex.test(it.type) || regex.test(it.method); + } catch (e) { + valid = valid && String(it.status).includes(filter) || it.name.includes(filter) || it.type.includes(filter) || (it.method && it.method.includes(filter)); + } + } + if (activeTab !== ALL) { + valid = valid && TYPE_TO_TAB[it.type] === activeTab; + } + + return valid; + }, + ), + ); + + // Update displayed items + setDisplayedItems(filteredItems.slice(0, INITIAL_LOAD_SIZE)); + setTotalItems(filteredItems.length); + setIsProcessing(false); + }; + + void applyFilters(); + }, [filter, activeTab, showOnlyErrors]); + + const loadMoreItems = useCallback(() => { + if (isProcessing) return; + + setIsProcessing(true); + setTimeout(() => { + setDisplayedItems((prevItems) => { + const currentLength = prevItems.length; + const newItems = originalListRef.current.slice( + currentLength, + currentLength + BATCH_SIZE, + ); + return [...prevItems, ...newItems]; + }); + setIsProcessing(false); + }, 10); + }, [isProcessing]); + + const hasMoreItems = displayedItems.length < totalItems; + const loadingRef = useInfiniteScroll(loadMoreItems, hasMoreItems); + + const onTabClick = (activeTab) => { devTools.update(INDEX_KEY, { activeTab }); - const onFilterChange = ({ - target: { value }, - }: React.ChangeEvent) => - devTools.update(INDEX_KEY, { filter: value }); + }; + + const onFilterChange = ({ target: { value } }) => { + setInputFilterValue(value) + debouncedFilter(value); + }; - // AutoScroll const [timeoutStartAutoscroll, stopAutoscroll] = useAutoscroll( - filteredList, + displayedItems, getLastItemTime(fetchListNow, resourceListNow), activeIndex, (index) => devTools.update(INDEX_KEY, { index }), @@ -462,24 +587,6 @@ export const NetworkPanelComp = observer( timeoutStartAutoscroll(); }; - const resourcesSize = useMemo( - () => - resourceList.reduce( - (sum, { decodedBodySize }) => sum + (decodedBodySize || 0), - 0, - ), - [resourceList.length], - ); - const transferredSize = useMemo( - () => - resourceList.reduce( - (sum, { headerSize, encodedBodySize }) => - sum + (headerSize || 0) + (encodedBodySize || 0), - 0, - ), - [resourceList.length], - ); - const referenceLines = useMemo(() => { const arr = []; @@ -513,7 +620,7 @@ export const NetworkPanelComp = observer( isSpot={isSpot} time={item.time + startedAt} resource={item} - rows={filteredList} + rows={displayedItems} fetchPresented={fetchList.length > 0} />, { @@ -525,12 +632,12 @@ export const NetworkPanelComp = observer( }, }, ); - devTools.update(INDEX_KEY, { index: filteredList.indexOf(item) }); + devTools.update(INDEX_KEY, { index: displayedItems.indexOf(item) }); stopAutoscroll(); }; - const tableCols = React.useMemo(() => { - const cols: any[] = [ + const tableCols = useMemo(() => { + const cols = [ { label: t('Status'), dataKey: 'status', @@ -585,7 +692,7 @@ export const NetworkPanelComp = observer( }); } return cols; - }, [showSingleTab]); + }, [showSingleTab, activeTab, t, getTabName, getTabNum, isSpot]); return ( } /> @@ -625,7 +732,7 @@ export const NetworkPanelComp = observer(
    -
    +
    + + {isProcessing && ( + + Processing data... + + )}
    + 0} + display={summaryStats.transferredSize > 0} /> 0} + display={summaryStats.resourcesSize > 0} />
    - - - {t('No Data')} + + {isLoading ? ( +
    +
    +
    +

    Loading network data...

    - } - size="small" - show={filteredList.length === 0} - > - {/* @ts-ignore */} - { - devTools.update(INDEX_KEY, { - index: filteredList.indexOf(row), - }); - player.jump(row.time); - }} - activeIndex={activeIndex} +
    + ) : ( + + + {t('No Data')} +
    + } + size="small" + show={displayedItems.length === 0} > - {tableCols} - - {selectedWsChannel ? ( - setSelectedWsChannel(null)} - /> - ) : null} - +
    + { + devTools.update(INDEX_KEY, { + index: displayedItems.indexOf(row), + }); + player.jump(row.time); + }} + activeIndex={activeIndex} + > + {tableCols} + + + {hasMoreItems && ( +
    +
    +
    + Loading more data ({totalItems - displayedItems.length}{' '} + remaining) +
    +
    + )} +
    + + {selectedWsChannel ? ( + setSelectedWsChannel(null)} + /> + ) : null} + + )}     ); @@ -722,7 +862,6 @@ export const NetworkPanelComp = observer( ); const WebNetworkPanel = observer(NetworkPanelCont); - const MobileNetworkPanel = observer(MobileNetworkPanelCont); export { WebNetworkPanel, MobileNetworkPanel }; diff --git a/frontend/app/components/shared/DevTools/NetworkPanel/utils.ts b/frontend/app/components/shared/DevTools/NetworkPanel/utils.ts new file mode 100644 index 000000000..ee3ac743a --- /dev/null +++ b/frontend/app/components/shared/DevTools/NetworkPanel/utils.ts @@ -0,0 +1,182 @@ +export function mergeListsWithZoom< + T extends Record, + Y extends Record, + Z extends Record, +>( + arr1: T[], + arr2: Y[], + arr3: Z[], + zoom?: { enabled: boolean; start: number; end: number }, +): Array { + // Early return for empty arrays + if (arr1.length === 0 && arr2.length === 0 && arr3.length === 0) { + return []; + } + + // Create result array with pre-allocated size estimation + // (We'll set an initial capacity based on rough estimate) + const merged = []; + + // Optimize for common case - no zoom + if (!zoom?.enabled) { + return mergeThreeSortedArrays(arr1, arr2, arr3); + } + + // Binary search for start indexes (faster than linear search for large arrays) + const index1 = binarySearchStartIndex(arr1, zoom.start); + const index2 = binarySearchStartIndex(arr2, zoom.start); + const index3 = binarySearchStartIndex(arr3, zoom.start); + + // Merge arrays within zoom range + return mergeThreeSortedArraysWithinRange( + arr1, + arr2, + arr3, + index1, + index2, + index3, + zoom.start, + zoom.end, + ); +} + +// Helper function to perform binary search to find index of first element >= threshold +function binarySearchStartIndex>( + arr: T[], + threshold: number, +): number { + if (arr.length === 0) return 0; + + let low = 0; + let high = arr.length - 1; + + // Handle edge cases first for better performance + if (arr[high].time < threshold) return arr.length; + if (arr[low].time >= threshold) return 0; + + while (low <= high) { + const mid = Math.floor((low + high) / 2); + + if (arr[mid].time < threshold) { + low = mid + 1; + } else { + high = mid - 1; + } + } + + return low; +} + +// Specialized function for when no zoom is applied +function mergeThreeSortedArrays< + T extends Record, + Y extends Record, + Z extends Record, +>(arr1: T[], arr2: Y[], arr3: Z[]): Array { + const totalLength = arr1.length + arr2.length + arr3.length; + const result = new Array(totalLength); + + let i = 0, + j = 0, + k = 0, + index = 0; + + while (i < arr1.length || j < arr2.length || k < arr3.length) { + const val1 = i < arr1.length ? arr1[i].time : Infinity; + const val2 = j < arr2.length ? arr2[j].time : Infinity; + const val3 = k < arr3.length ? arr3[k].time : Infinity; + + if (val1 <= val2 && val1 <= val3) { + result[index++] = arr1[i++]; + } else if (val2 <= val1 && val2 <= val3) { + result[index++] = arr2[j++]; + } else { + result[index++] = arr3[k++]; + } + } + + return result; +} + +// Specialized function for merging with zoom range +function mergeThreeSortedArraysWithinRange< + T extends Record, + Y extends Record, + Z extends Record, +>( + arr1: T[], + arr2: Y[], + arr3: Z[], + startIdx1: number, + startIdx2: number, + startIdx3: number, + start: number, + end: number, +): Array { + const result = []; + + let i = startIdx1; + let j = startIdx2; + let k = startIdx3; + + while (i < arr1.length || j < arr2.length || k < arr3.length) { + const val1 = i < arr1.length ? arr1[i].time : Infinity; + const val2 = j < arr2.length ? arr2[j].time : Infinity; + const val3 = k < arr3.length ? arr3[k].time : Infinity; + + // Early termination: if all remaining values exceed end time + if (Math.min(val1, val2, val3) > end) { + break; + } + + if (val1 <= val2 && val1 <= val3) { + if (val1 <= end) { + result.push(arr1[i]); + } + i++; + } else if (val2 <= val1 && val2 <= val3) { + if (val2 <= end) { + result.push(arr2[j]); + } + j++; + } else { + if (val3 <= end) { + result.push(arr3[k]); + } + k++; + } + } + + return result; +} + +export function processInChunks( + items: any[], + processFn: (item: any) => any, + chunkSize = 1000, + overscan = 0, +) { + return new Promise((resolve) => { + if (items.length === 0) { + resolve([]); + return; + } + + let result: any[] = []; + let index = 0; + + const processNextChunk = () => { + const chunk = items.slice(index, index + chunkSize + overscan); + result = result.concat(processFn(chunk)); + index += chunkSize; + + if (index < items.length) { + setTimeout(processNextChunk, 0); + } else { + resolve(result); + } + }; + + processNextChunk(); + }); +} diff --git a/frontend/app/utils/index.ts b/frontend/app/utils/index.ts index 02caac58e..e74a56f87 100644 --- a/frontend/app/utils/index.ts +++ b/frontend/app/utils/index.ts @@ -38,7 +38,6 @@ export function debounceCall(func, wait) { }; } - export function randomInt(a, b) { const min = (b ? a : 0) - 0.5; const max = b || a || Number.MAX_SAFE_INTEGER;