ui: improve network panel row mapping

frontend/app/components/shared/DevTools/NetworkPanel/NetworkPanel.tsx

@@ -1,9 +1,17 @@
 /* eslint-disable i18next/no-literal-string */
 import { ResourceType, Timed } from 'Player';
+import { WsChannel } from 'Player/web/messages';
 import MobilePlayer from 'Player/mobile/IOSPlayer';
 import WebPlayer from 'Player/web/WebPlayer';
 import { observer } from 'mobx-react-lite';
-import React, { useMemo, useState } from 'react';
+import React, {
+  useMemo,
+  useState,
+  useEffect,
+  useCallback,
+  useRef,
+} from 'react';
+import i18n from 'App/i18n'
 
 import { useModal } from 'App/components/Modal';
 import {
@@ -12,25 +20,27 @@ import {
 } from 'App/components/Session/playerContext';
 import { formatMs } from 'App/date';
 import { useStore } from 'App/mstore';
-import { formatBytes } from 'App/utils';
+import { formatBytes, debounceCall } from 'App/utils';
 import { Icon, NoContent, Tabs } from 'UI';
 import { Tooltip, Input, Switch, Form } from 'antd';
-import { SearchOutlined, InfoCircleOutlined } from '@ant-design/icons';
+import {
+  SearchOutlined,
+  InfoCircleOutlined,
+} from '@ant-design/icons';
 
 import FetchDetailsModal from 'Shared/FetchDetailsModal';
-import { WsChannel } from 'App/player/web/messages';
 
 import BottomBlock from '../BottomBlock';
 import InfoLine from '../BottomBlock/InfoLine';
 import TabSelector from '../TabSelector';
 import TimeTable from '../TimeTable';
 import useAutoscroll, { getLastItemTime } from '../useAutoscroll';
-import { useRegExListFilterMemo, useTabListFilterMemo } from '../useListFilter';
 import WSPanel from './WSPanel';
 import { useTranslation } from 'react-i18next';
+import { mergeListsWithZoom, processInChunks } from './utils'
 
+// Constants remain the same
 const INDEX_KEY = 'network';
-
 const ALL = 'ALL';
 const XHR = 'xhr';
 const JS = 'js';
@@ -62,6 +72,9 @@ export const NETWORK_TABS = TAP_KEYS.map((tab) => ({
 const DOM_LOADED_TIME_COLOR = 'teal';
 const LOAD_TIME_COLOR = 'red';
 
+const BATCH_SIZE = 2500;
+const INITIAL_LOAD_SIZE = 5000;
+
 export function renderType(r: any) {
   return (
     <Tooltip style={{ width: '100%' }} title={<div>{r.type}</div>}>
@@ -79,13 +92,17 @@ export function renderName(r: any) {
 }
 
 function renderSize(r: any) {
-  const { t } = useTranslation();
+  const t = i18n.t;
-  if (r.responseBodySize) return formatBytes(r.responseBodySize);
+  const notCaptured = t('Not captured');
+  const resSizeStr = t('Resource size')
   let triggerText;
   let content;
-  if (r.decodedBodySize == null || r.decodedBodySize === 0) {
+  if (r.responseBodySize) {
+    triggerText = formatBytes(r.responseBodySize);
+    content = undefined;
+  } else if (r.decodedBodySize == null || r.decodedBodySize === 0) {
     triggerText = 'x';
-    content = t('Not captured');
+    content = notCaptured;
   } else {
     const headerSize = r.headerSize || 0;
     const showTransferred = r.headerSize != null;
@@ -100,7 +117,7 @@ function renderSize(r: any) {
             )} transferred over network`}
           </li>
         )}
-        <li>{`${t('Resource size')}: ${formatBytes(r.decodedBodySize)} `}</li>
+        <li>{`${resSizeStr}: ${formatBytes(r.decodedBodySize)} `}</li>
       </ul>
     );
   }
@@ -168,6 +185,8 @@ function renderStatus({
   );
 }
 
+
+// Main component for Network Panel
 function NetworkPanelCont({ panelHeight }: { panelHeight: number }) {
   const { player, store } = React.useContext(PlayerContext);
   const { sessionStore, uiPlayerStore } = useStore();
@@ -216,6 +235,7 @@ function NetworkPanelCont({ panelHeight }: { panelHeight: number }) {
 
   const getTabNum = (tab: string) => tabsArr.findIndex((t) => t === tab) + 1;
   const getTabName = (tabId: string) => tabNames[tabId];
+
   return (
     <NetworkPanelComp
       loadTime={loadTime}
@@ -228,8 +248,8 @@ function NetworkPanelCont({ panelHeight }: { panelHeight: number }) {
       resourceListNow={resourceListNow}
       player={player}
       startedAt={startedAt}
-      websocketList={websocketList as WSMessage[]}
+      websocketList={websocketList}
-      websocketListNow={websocketListNow as WSMessage[]}
+      websocketListNow={websocketListNow}
       getTabNum={getTabNum}
       getTabName={getTabName}
       showSingleTab={showSingleTab}
@@ -269,9 +289,7 @@ function MobileNetworkPanelCont({ panelHeight }: { panelHeight: number }) {
       resourceListNow={resourceListNow}
       player={player}
       startedAt={startedAt}
-      // @ts-ignore
       websocketList={websocketList}
-      // @ts-ignore
       websocketListNow={websocketListNow}
       zoomEnabled={zoomEnabled}
       zoomStartTs={zoomStartTs}
@@ -280,12 +298,36 @@ function MobileNetworkPanelCont({ panelHeight }: { panelHeight: number }) {
   );
 }
 
-type WSMessage = Timed & {
+// Custom hook for infinite scrolling
-  channelName: string;
+const useInfiniteScroll = (loadMoreCallback: () => void, hasMore: boolean) => {
-  data: string;
+  const observerRef = useRef<IntersectionObserver>(null);
-  timestamp: number;
+  const loadingRef = useRef<HTMLDivElement>(null);
-  dir: 'up' | 'down';
+
-  messageType: string;
+  useEffect(() => {
+    const observer = new IntersectionObserver(
+      (entries) => {
+        if (entries[0]?.isIntersecting && hasMore) {
+          loadMoreCallback();
+        }
+      },
+      { threshold: 0.1 },
+    );
+
+    if (loadingRef.current) {
+      observer.observe(loadingRef.current);
+    }
+
+    // @ts-ignore
+    observerRef.current = observer;
+
+    return () => {
+      if (observerRef.current) {
+        observerRef.current.disconnect();
+      }
+    };
+  }, [loadMoreCallback, hasMore, loadingRef]);
+
+  return loadingRef;
 };
 
 interface Props {
@@ -302,8 +344,8 @@ interface Props {
   resourceList: Timed[];
   fetchListNow: Timed[];
   resourceListNow: Timed[];
-  websocketList: Array<WSMessage>;
+  websocketList: Array<WsChannel>;
-  websocketListNow: Array<WSMessage>;
+  websocketListNow: Array<WsChannel>;
   player: WebPlayer | MobilePlayer;
   startedAt: number;
   isMobile?: boolean;
@@ -349,107 +391,190 @@ export const NetworkPanelComp = observer(
     >(null);
     const { showModal } = useModal();
     const [showOnlyErrors, setShowOnlyErrors] = useState(false);
-
     const [isDetailsModalActive, setIsDetailsModalActive] = useState(false);
+    const [isLoading, setIsLoading] = useState(true);
+    const [isProcessing, setIsProcessing] = useState(false);
+    const [displayedItems, setDisplayedItems] = useState([]);
+    const [totalItems, setTotalItems] = useState(0);
+    const [summaryStats, setSummaryStats] = useState({
+      resourcesSize: 0,
+      transferredSize: 0,
+    });
+
+    // Store original data in refs to avoid reprocessing
+    const originalListRef = useRef([]);
+    const socketListRef = useRef([]);
+
     const {
       sessionStore: { devTools },
     } = useStore();
     const { filter } = devTools[INDEX_KEY];
     const { activeTab } = devTools[INDEX_KEY];
     const activeIndex = activeOutsideIndex ?? devTools[INDEX_KEY].index;
+    const [inputFilterValue, setInputFilterValue] = useState(filter);
 
-    const socketList = useMemo(
+    const debouncedFilter = useCallback(
-      () =>
+      debounceCall((filterValue) => {
-        websocketList.filter(
+        devTools.update(INDEX_KEY, { filter: filterValue });
-          (ws, i, arr) =>
+      }, 300),
-            arr.findIndex((it) => it.channelName === ws.channelName) === i,
+      [],
-        ),
-      [websocketList],
     );
 
-    const list = useMemo(
+    // Process socket lists once
-      () =>
+    useEffect(() => {
-        // TODO: better merge (with body size info) - do it in player
+      const uniqueSocketList = websocketList.filter(
-        resourceList
+        (ws, i, arr) =>
-          .filter(
+          arr.findIndex((it) => it.channelName === ws.channelName) === i,
-            (res) =>
-              !fetchList.some((ft) => {
-                // res.url !== ft.url doesn't work on relative URLs appearing within fetchList (to-fix in player)
-                if (res.name === ft.name) {
-                  if (res.time === ft.time) return true;
-                  if (res.url.includes(ft.url)) {
-                    return (
-                      Math.abs(res.time - ft.time) < 350 ||
-                      Math.abs(res.timestamp - ft.timestamp) < 350
-                    );
-                  }
-                }
-
-                if (res.name !== ft.name) {
-                  return false;
-                }
-                if (Math.abs(res.time - ft.time) > 250) {
-                  return false;
-                } // TODO: find good epsilons
-                if (Math.abs(res.duration - ft.duration) > 200) {
-                  return false;
-                }
-
-                return true;
-              }),
-          )
-          .concat(fetchList)
-          .concat(
-            socketList.map((ws) => ({
-              ...ws,
-              type: 'websocket',
-              method: 'ws',
-              url: ws.channelName,
-              name: ws.channelName,
-              status: '101',
-              duration: 0,
-              transferredBodySize: 0,
-            })),
-          )
-          .filter((req) =>
-            zoomEnabled
-              ? req.time >= zoomStartTs! && req.time <= zoomEndTs!
-              : true,
-          )
-          .sort((a, b) => a.time - b.time),
-      [resourceList.length, fetchList.length, socketList.length],
-    );
-
-    let filteredList = useMemo(() => {
-      if (!showOnlyErrors) {
-        return list;
-      }
-      return list.filter(
-        (it) => parseInt(it.status) >= 400 || !it.success || it.error,
       );
-    }, [showOnlyErrors, list]);
+      socketListRef.current = uniqueSocketList;
-    filteredList = useRegExListFilterMemo(
+    }, [websocketList.length]);
-      filteredList,
-      (it) => [it.status, it.name, it.type, it.method],
-      filter,
-    );
-    filteredList = useTabListFilterMemo(
-      filteredList,
-      (it) => TYPE_TO_TAB[it.type],
-      ALL,
-      activeTab,
-    );
 
-    const onTabClick = (activeTab: (typeof TAP_KEYS)[number]) =>
+    // Initial data processing - do this only once when data changes
+    useEffect(() => {
+      setIsLoading(true);
+
+      // Heaviest operation here, will create a final merged network list
+      const processData = async () => {
+        const fetchUrls = new Set(
+          fetchList.map((ft) => {
+            return `${ft.name}-${Math.floor(ft.time / 100)}-${Math.floor(ft.duration / 100)}`;
+          }),
+        );
+
+        // We want to get resources that aren't in fetch list
+        const filteredResources = await processInChunks(resourceList, (chunk) =>
+          chunk.filter((res: any) => {
+            const key = `${res.name}-${Math.floor(res.time / 100)}-${Math.floor(res.duration / 100)}`;
+            return !fetchUrls.has(key);
+          }),
+          BATCH_SIZE,
+          25,
+        );
+
+        const processedSockets = socketListRef.current.map((ws: any) => ({
+          ...ws,
+          type: 'websocket',
+          method: 'ws',
+          url: ws.channelName,
+          name: ws.channelName,
+          status: '101',
+          duration: 0,
+          transferredBodySize: 0,
+        }));
+
+        const mergedList: Timed[] = mergeListsWithZoom(
+          filteredResources as Timed[],
+          fetchList,
+          processedSockets as Timed[],
+          { enabled: Boolean(zoomEnabled), start: zoomStartTs ?? 0, end: zoomEndTs ?? 0 }
+        )
+
+        originalListRef.current = mergedList;
+        setTotalItems(mergedList.length);
+
+        calculateResourceStats(resourceList);
+
+        // Only display initial chunk
+        setDisplayedItems(mergedList.slice(0, INITIAL_LOAD_SIZE));
+        setIsLoading(false);
+      };
+
+      void processData();
+    }, [
+      resourceList.length,
+      fetchList.length,
+      socketListRef.current.length,
+      zoomEnabled,
+      zoomStartTs,
+      zoomEndTs,
+    ]);
+
+    const calculateResourceStats = (resourceList: Record<string, any>) => {
+      setTimeout(() => {
+        let resourcesSize = 0
+        let transferredSize = 0
+        resourceList.forEach(({ decodedBodySize, headerSize, encodedBodySize }: any) => {
+          resourcesSize += decodedBodySize || 0
+          transferredSize += (headerSize || 0) + (encodedBodySize || 0)
+        })
+
+        setSummaryStats({
+          resourcesSize,
+          transferredSize,
+        });
+      }, 0);
+    }
+
+    useEffect(() => {
+      if (originalListRef.current.length === 0) return;
+      setIsProcessing(true);
+      const applyFilters = async () => {
+        let filteredItems: any[] = originalListRef.current;
+
+        filteredItems = await processInChunks(filteredItems, (chunk) =>
+          chunk.filter(
+            (it) => {
+              let valid = true;
+              if (showOnlyErrors) {
+                valid = parseInt(it.status) >= 400 || !it.success || it.error
+              }
+              if (filter) {
+                try {
+                  const regex = new RegExp(filter, 'i');
+                  valid = valid && regex.test(it.status) || regex.test(it.name) || regex.test(it.type) || regex.test(it.method);
+                } catch (e) {
+                  valid = valid && String(it.status).includes(filter) || it.name.includes(filter) || it.type.includes(filter) || (it.method && it.method.includes(filter));
+                }
+              }
+              if (activeTab !== ALL) {
+                valid = valid && TYPE_TO_TAB[it.type] === activeTab;
+              }
+
+              return valid;
+            },
+          ),
+        );
+
+        // Update displayed items
+        setDisplayedItems(filteredItems.slice(0, INITIAL_LOAD_SIZE));
+        setTotalItems(filteredItems.length);
+        setIsProcessing(false);
+      };
+
+      void applyFilters();
+    }, [filter, activeTab, showOnlyErrors]);
+
+    const loadMoreItems = useCallback(() => {
+      if (isProcessing) return;
+
+      setIsProcessing(true);
+      setTimeout(() => {
+        setDisplayedItems((prevItems) => {
+          const currentLength = prevItems.length;
+          const newItems = originalListRef.current.slice(
+            currentLength,
+            currentLength + BATCH_SIZE,
+          );
+          return [...prevItems, ...newItems];
+        });
+        setIsProcessing(false);
+      }, 10);
+    }, [isProcessing]);
+
+    const hasMoreItems = displayedItems.length < totalItems;
+    const loadingRef = useInfiniteScroll(loadMoreItems, hasMoreItems);
+
+    const onTabClick = (activeTab) => {
       devTools.update(INDEX_KEY, { activeTab });
-    const onFilterChange = ({
+    };
-      target: { value },
+
-    }: React.ChangeEvent<HTMLInputElement>) =>
+    const onFilterChange = ({ target: { value } }) => {
-      devTools.update(INDEX_KEY, { filter: value });
+      setInputFilterValue(value)
+      debouncedFilter(value);
+    };
 
-    // AutoScroll
     const [timeoutStartAutoscroll, stopAutoscroll] = useAutoscroll(
-      filteredList,
+      displayedItems,
       getLastItemTime(fetchListNow, resourceListNow),
       activeIndex,
       (index) => devTools.update(INDEX_KEY, { index }),
@@ -462,24 +587,6 @@ export const NetworkPanelComp = observer(
       timeoutStartAutoscroll();
     };
 
-    const resourcesSize = useMemo(
-      () =>
-        resourceList.reduce(
-          (sum, { decodedBodySize }) => sum + (decodedBodySize || 0),
-          0,
-        ),
-      [resourceList.length],
-    );
-    const transferredSize = useMemo(
-      () =>
-        resourceList.reduce(
-          (sum, { headerSize, encodedBodySize }) =>
-            sum + (headerSize || 0) + (encodedBodySize || 0),
-          0,
-        ),
-      [resourceList.length],
-    );
-
     const referenceLines = useMemo(() => {
       const arr = [];
 
@@ -513,7 +620,7 @@ export const NetworkPanelComp = observer(
           isSpot={isSpot}
           time={item.time + startedAt}
           resource={item}
-          rows={filteredList}
+          rows={displayedItems}
           fetchPresented={fetchList.length > 0}
         />,
         {
@@ -525,12 +632,12 @@ export const NetworkPanelComp = observer(
           },
         },
       );
-      devTools.update(INDEX_KEY, { index: filteredList.indexOf(item) });
+      devTools.update(INDEX_KEY, { index: displayedItems.indexOf(item) });
       stopAutoscroll();
     };
 
-    const tableCols = React.useMemo(() => {
+    const tableCols = useMemo(() => {
-      const cols: any[] = [
+      const cols = [
         {
           label: t('Status'),
           dataKey: 'status',
@@ -585,7 +692,7 @@ export const NetworkPanelComp = observer(
         });
       }
       return cols;
-    }, [showSingleTab]);
+    }, [showSingleTab, activeTab, t, getTabName, getTabNum, isSpot]);
 
     return (
       <BottomBlock
@@ -617,7 +724,7 @@ export const NetworkPanelComp = observer(
               name="filter"
               onChange={onFilterChange}
               width={280}
-              value={filter}
+              value={inputFilterValue}
               size="small"
               prefix={<SearchOutlined className="text-neutral-400" />}
             />
@@ -625,7 +732,7 @@ export const NetworkPanelComp = observer(
         </BottomBlock.Header>
         <BottomBlock.Content>
           <div className="flex items-center justify-between px-4 border-b bg-teal/5 h-8">
-            <div>
+            <div className="flex items-center">
               <Form.Item name="show-errors-only" className="mb-0">
                 <label
                   style={{
@@ -642,21 +749,29 @@ export const NetworkPanelComp = observer(
                   <span className="text-sm ms-2">4xx-5xx Only</span>
                 </label>
               </Form.Item>
+
+              {isProcessing && (
+                <span className="text-xs text-gray-500 ml-4">
+                  Processing data...
+                </span>
+              )}
             </div>
             <InfoLine>
+              <InfoLine.Point label={`${totalItems}`} value="requests" />
               <InfoLine.Point
-                label={`${filteredList.length}`}
+                label={`${displayedItems.length}/${totalItems}`}
-                value=" requests"
+                value="displayed"
+                display={displayedItems.length < totalItems}
               />
               <InfoLine.Point
-                label={formatBytes(transferredSize)}
+                label={formatBytes(summaryStats.transferredSize)}
                 value="transferred"
-                display={transferredSize > 0}
+                display={summaryStats.transferredSize > 0}
               />
               <InfoLine.Point
-                label={formatBytes(resourcesSize)}
+                label={formatBytes(summaryStats.resourcesSize)}
                 value="resources"
-                display={resourcesSize > 0}
+                display={summaryStats.resourcesSize > 0}
               />
               <InfoLine.Point
                 label={formatMs(domBuildingTime)}
@@ -679,42 +794,67 @@ export const NetworkPanelComp = observer(
               />
             </InfoLine>
           </div>
-          <NoContent
+
-            title={
+          {isLoading ? (
-              <div className="capitalize flex items-center gap-2">
+            <div className="flex items-center justify-center h-full">
-                <InfoCircleOutlined size={18} />
+              <div className="text-center">
-                {t('No Data')}
+                <div className="animate-spin rounded-full h-8 w-8 border-b-2 border-gray-900 mx-auto mb-2"></div>
+                <p>Loading network data...</p>
               </div>
-            }
+            </div>
-            size="small"
+          ) : (
-            show={filteredList.length === 0}
+            <NoContent
-          >
+              title={
-            {/* @ts-ignore */}
+                <div className="capitalize flex items-center gap-2">
-            <TimeTable
+                  <InfoCircleOutlined size={18} />
-              rows={filteredList}
+                  {t('No Data')}
-              tableHeight={panelHeight - 102}
+                </div>
-              referenceLines={referenceLines}
+              }
-              renderPopup
+              size="small"
-              onRowClick={showDetailsModal}
+              show={displayedItems.length === 0}
-              sortBy="time"
-              sortAscending
-              onJump={(row: any) => {
-                devTools.update(INDEX_KEY, {
-                  index: filteredList.indexOf(row),
-                });
-                player.jump(row.time);
-              }}
-              activeIndex={activeIndex}
             >
-              {tableCols}
+              <div>
-            </TimeTable>
+                <TimeTable
-            {selectedWsChannel ? (
+                  rows={displayedItems}
-              <WSPanel
+                  tableHeight={panelHeight - 102 - (hasMoreItems ? 30 : 0)}
-                socketMsgList={selectedWsChannel}
+                  referenceLines={referenceLines}
-                onClose={() => setSelectedWsChannel(null)}
+                  renderPopup
-              />
+                  onRowClick={showDetailsModal}
-            ) : null}
+                  sortBy="time"
-          </NoContent>
+                  sortAscending
+                  onJump={(row) => {
+                    devTools.update(INDEX_KEY, {
+                      index: displayedItems.indexOf(row),
+                    });
+                    player.jump(row.time);
+                  }}
+                  activeIndex={activeIndex}
+                >
+                  {tableCols}
+                </TimeTable>
+
+                {hasMoreItems && (
+                  <div
+                    ref={loadingRef}
+                    className="flex justify-center items-center text-xs text-gray-500"
+                  >
+                      <div className="flex items-center">
+                        <div className="animate-spin rounded-full h-4 w-4 border-b-2 border-gray-600 mr-2"></div>
+                        Loading more data ({totalItems - displayedItems.length}{' '}
+                        remaining)
+                      </div>
+                  </div>
+                )}
+              </div>
+
+              {selectedWsChannel ? (
+                <WSPanel
+                  socketMsgList={selectedWsChannel}
+                  onClose={() => setSelectedWsChannel(null)}
+                />
+              ) : null}
+            </NoContent>
+          )}
         </BottomBlock.Content>
       </BottomBlock>
     );
@@ -722,7 +862,6 @@ export const NetworkPanelComp = observer(
 );
 
 const WebNetworkPanel = observer(NetworkPanelCont);
-
 const MobileNetworkPanel = observer(MobileNetworkPanelCont);
 
 export { WebNetworkPanel, MobileNetworkPanel };

frontend/app/components/shared/DevTools/NetworkPanel/utils.ts

@@ -0,0 +1,182 @@
+export function mergeListsWithZoom<
+  T extends Record<string, any>,
+  Y extends Record<string, any>,
+  Z extends Record<string, any>,
+>(
+  arr1: T[],
+  arr2: Y[],
+  arr3: Z[],
+  zoom?: { enabled: boolean; start: number; end: number },
+): Array<T | Y | Z> {
+  // Early return for empty arrays
+  if (arr1.length === 0 && arr2.length === 0 && arr3.length === 0) {
+    return [];
+  }
+
+  // Create result array with pre-allocated size estimation
+  // (We'll set an initial capacity based on rough estimate)
+  const merged = [];
+
+  // Optimize for common case - no zoom
+  if (!zoom?.enabled) {
+    return mergeThreeSortedArrays(arr1, arr2, arr3);
+  }
+
+  // Binary search for start indexes (faster than linear search for large arrays)
+  const index1 = binarySearchStartIndex(arr1, zoom.start);
+  const index2 = binarySearchStartIndex(arr2, zoom.start);
+  const index3 = binarySearchStartIndex(arr3, zoom.start);
+
+  // Merge arrays within zoom range
+  return mergeThreeSortedArraysWithinRange(
+    arr1,
+    arr2,
+    arr3,
+    index1,
+    index2,
+    index3,
+    zoom.start,
+    zoom.end,
+  );
+}
+
+// Helper function to perform binary search to find index of first element >= threshold
+function binarySearchStartIndex<T extends Record<string, any>>(
+  arr: T[],
+  threshold: number,
+): number {
+  if (arr.length === 0) return 0;
+
+  let low = 0;
+  let high = arr.length - 1;
+
+  // Handle edge cases first for better performance
+  if (arr[high].time < threshold) return arr.length;
+  if (arr[low].time >= threshold) return 0;
+
+  while (low <= high) {
+    const mid = Math.floor((low + high) / 2);
+
+    if (arr[mid].time < threshold) {
+      low = mid + 1;
+    } else {
+      high = mid - 1;
+    }
+  }
+
+  return low;
+}
+
+// Specialized function for when no zoom is applied
+function mergeThreeSortedArrays<
+  T extends Record<string, any>,
+  Y extends Record<string, any>,
+  Z extends Record<string, any>,
+>(arr1: T[], arr2: Y[], arr3: Z[]): Array<T | Y | Z> {
+  const totalLength = arr1.length + arr2.length + arr3.length;
+  const result = new Array(totalLength);
+
+  let i = 0,
+    j = 0,
+    k = 0,
+    index = 0;
+
+  while (i < arr1.length || j < arr2.length || k < arr3.length) {
+    const val1 = i < arr1.length ? arr1[i].time : Infinity;
+    const val2 = j < arr2.length ? arr2[j].time : Infinity;
+    const val3 = k < arr3.length ? arr3[k].time : Infinity;
+
+    if (val1 <= val2 && val1 <= val3) {
+      result[index++] = arr1[i++];
+    } else if (val2 <= val1 && val2 <= val3) {
+      result[index++] = arr2[j++];
+    } else {
+      result[index++] = arr3[k++];
+    }
+  }
+
+  return result;
+}
+
+// Specialized function for merging with zoom range
+function mergeThreeSortedArraysWithinRange<
+  T extends Record<string, any>,
+  Y extends Record<string, any>,
+  Z extends Record<string, any>,
+>(
+  arr1: T[],
+  arr2: Y[],
+  arr3: Z[],
+  startIdx1: number,
+  startIdx2: number,
+  startIdx3: number,
+  start: number,
+  end: number,
+): Array<T | Y | Z> {
+  const result = [];
+
+  let i = startIdx1;
+  let j = startIdx2;
+  let k = startIdx3;
+
+  while (i < arr1.length || j < arr2.length || k < arr3.length) {
+    const val1 = i < arr1.length ? arr1[i].time : Infinity;
+    const val2 = j < arr2.length ? arr2[j].time : Infinity;
+    const val3 = k < arr3.length ? arr3[k].time : Infinity;
+
+    // Early termination: if all remaining values exceed end time
+    if (Math.min(val1, val2, val3) > end) {
+      break;
+    }
+
+    if (val1 <= val2 && val1 <= val3) {
+      if (val1 <= end) {
+        result.push(arr1[i]);
+      }
+      i++;
+    } else if (val2 <= val1 && val2 <= val3) {
+      if (val2 <= end) {
+        result.push(arr2[j]);
+      }
+      j++;
+    } else {
+      if (val3 <= end) {
+        result.push(arr3[k]);
+      }
+      k++;
+    }
+  }
+
+  return result;
+}
+
+export function processInChunks(
+  items: any[],
+  processFn: (item: any) => any,
+  chunkSize = 1000,
+  overscan = 0,
+) {
+  return new Promise((resolve) => {
+    if (items.length === 0) {
+      resolve([]);
+      return;
+    }
+
+    let result: any[] = [];
+    let index = 0;
+
+    const processNextChunk = () => {
+      const chunk = items.slice(index, index + chunkSize + overscan);
+      result = result.concat(processFn(chunk));
+      index += chunkSize;
+
+      if (index < items.length) {
+        setTimeout(processNextChunk, 0);
+      } else {
+        resolve(result);
+      }
+    };
+
+    processNextChunk();
+  });
+}

frontend/app/utils/index.ts

@@ -38,7 +38,6 @@ export function debounceCall(func, wait) {
   };
 }
 
-
 export function randomInt(a, b) {
   const min = (b ? a : 0) - 0.5;
   const max = b || a || Number.MAX_SAFE_INTEGER;