tracker css batching/inlining (#3334)

* tracker: initial css inlining functionality

* tracker: add tests, adjust sheet id, stagger rule sending

* removed sorting

* upgrade css inliner

* ui: better logging for ocunter

* tracker: force-fetch mode for cssInliner

* tracker: fix ts warns

* tracker: use debug opts

* tracker: 16.2.0 changelogs, inliner opts

* tracker: remove debug options

---------

Co-authored-by: Андрей Бабушкин <andreybabushkin2000@gmail.com>

frontend/app/player/web/TabManager.ts

@@ -348,19 +348,19 @@ export default class TabSessionManager {
             break;
           case MType.CreateTextNode:
           case MType.CreateElementNode:
-            this.windowNodeCounter.addNode(msg.id, msg.parentID);
+            this.windowNodeCounter.addNode(msg);
             this.performanceTrackManager.setCurrentNodesCount(
               this.windowNodeCounter.count,
             );
             break;
           case MType.MoveNode:
-            this.windowNodeCounter.moveNode(msg.id, msg.parentID);
+            this.windowNodeCounter.moveNode(msg);
             this.performanceTrackManager.setCurrentNodesCount(
               this.windowNodeCounter.count,
             );
             break;
           case MType.RemoveNode:
-            this.windowNodeCounter.removeNode(msg.id);
+            this.windowNodeCounter.removeNode(msg);
             this.performanceTrackManager.setCurrentNodesCount(
               this.windowNodeCounter.count,
             );

frontend/app/player/web/managers/DOM/VirtualDOM.ts

@@ -257,6 +257,7 @@ export class VElement extends VParent<Element> {
 
   applyChanges() {
     this.prioritized && this.applyPrioritizedChanges();
+    this.node.data = this.data;
     this.applyAttributeChanges();
     super.applyChanges();
   }

frontend/app/player/web/managers/WindowNodeCounter.ts

@@ -54,40 +54,45 @@ export default class WindowNodeCounter {
     this.nodes = [this.root];
   }
 
-  addNode(id: number, parentID: number) {
+  addNode(msg: { id: number, parentID: number, time: number }): boolean {
+    const { id, parentID } = msg;
     if (!this.nodes[parentID]) {
       // TODO: iframe case
       // console.error(`Wrong! Node with id ${ parentID } (parentId) not found.`);
-      return;
+      return false;
     }
     if (this.nodes[id]) {
       // console.error(`Wrong! Node with id ${ id } already exists.`);
-      return;
+      return false;
     }
     this.nodes[id] = this.nodes[parentID].newChild();
+    return true;
   }
 
-  removeNode(id: number) {
+  removeNode({ id }: { id: number }) {
     if (!this.nodes[id]) {
       // Might be text node
       // console.error(`Wrong! Node with id ${ id } not found.`);
-      return;
+      return false;
     }
     this.nodes[id].removeNode();
+    return true;
   }
 
-  moveNode(id: number, parentId: number) {
+  moveNode(msg: { id: number, parentID: number, time: number }) {
+    const { id, parentID, time } = msg;
     if (!this.nodes[id]) {
-      console.warn(`Node Counter: Node with id ${id} not found.`);
+      console.warn(`Node Counter: Node with id ${id} (parent: ${parentID}) not found. time: ${time}`);
-      return;
+      return false;
     }
-    if (!this.nodes[parentId]) {
+    if (!this.nodes[parentID]) {
       console.warn(
-        `Node Counter: Node with id ${parentId} (parentId) not found.`,
+        `Node Counter: Node with id ${parentID} (parentId) not found. time: ${time}`,
       );
-      return;
+      return false;
     }
-    this.nodes[id].moveNode(this.nodes[parentId]);
+    this.nodes[id].moveNode(this.nodes[parentID]);
+    return true;
   }
 
   get count() {

tracker/tracker/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 16.2.0
+
+- css batching and inlining via (!plain mode will cause fake text nodes in style tags occupying 99*10^6 id space, can conflict with crossdomain iframes!)
+
+```
+inlineRemoteCss: boolean
+  inlinerOptions?: {
+    forceFetch?: boolean,
+    forcePlain?: boolean,
+  }
+```
+
 ## 16.1.4
 
 - bump proxy version to .3

tracker/tracker/src/main/app/index.ts

@@ -319,16 +319,14 @@ export default class App {
       __save_canvas_locally: false,
       localStorage: null,
       sessionStorage: null,
-      disableStringDict: true,
       forceSingleTab: false,
       assistSocketHost: '',
       fixedCanvasScaling: false,
       disableCanvas: false,
       captureIFrames: true,
-      disableSprites: false,
+      obscureTextEmails: false,
-      inlineRemoteCss: true,
-      obscureTextEmails: true,
       obscureTextNumbers: false,
+      disableStringDict: false,
       crossdomain: {
         parentDomain: '*',
       },
@@ -339,6 +337,12 @@ export default class App {
         useAnimationFrame: false,
       },
       forceNgOff: false,
+      inlineRemoteCss: false,
+      disableSprites: false,
+      inlinerOptions: {
+        forceFetch: false,
+        forcePlain: false,
+      }
     }
     this.options = simpleMerge(defaultOptions, options)
 

tracker/tracker/src/main/app/observer/cssInliner.ts

@@ -1,3 +1,5 @@
+let fakeIdHolder = 1000000 * 99;
+
 export function inlineRemoteCss(
   node: HTMLLinkElement,
   id: number,
@@ -5,83 +7,286 @@ export function inlineRemoteCss(
   getNextID: () => number,
   insertRule: (id: number, cssText: string, index: number, baseHref: string) => any[],
   addOwner: (sheetId: number, ownerId: number) => any[],
+  forceFetch?: boolean,
+  sendPlain?: boolean,
+  onPlain?: (cssText: string, id: number) => void,
 ) {
-  const sheet = node.sheet;
+  const sheetId = getNextID();
-  const sheetId = getNextID()
   addOwner(sheetId, id);
 
-  const processRules = (rules: CSSRuleList) => {
+  const sheet = node.sheet;
-    if (rules.length) {
-      setTimeout(() => {
-        for (let i = 0; i < rules.length; i++) {
-          const rule = rules[i];
-          insertRule(sheetId, rule.cssText, i, baseHref);
-        }
-      }, 0)
-    }
-  };
 
-  const processCssText = (cssText: string) => {
+  if (sheet && !forceFetch) {
-    cssText = cssText.replace(/\/\*[\s\S]*?\*\//g, '');
-
-    const ruleTexts: string[] = [];
-    let depth = 0;
-    let currentRule = '';
-
-    for (let i = 0; i < cssText.length; i++) {
-      const char = cssText[i];
-
-      if (char === '{') {
-        depth++;
-      } else if (char === '}') {
-        depth--;
-        if (depth === 0) {
-          currentRule += char;
-          ruleTexts.push(currentRule.trim());
-          currentRule = '';
-          continue;
-        }
-      }
-
-      currentRule += char;
-    }
-
-    for (let i = 0; i < ruleTexts.length; i++) {
-      const ruleText = ruleTexts[i];
-      insertRule(sheetId, ruleText, i, baseHref);
-    }
-  };
-
-  if (sheet) {
     try {
-      const rules = sheet.cssRules;
+      const cssText = stringifyStylesheet(sheet);
-      processRules(rules);
+
-    } catch (e) {
+      if (cssText) {
-      const href = node.href;
+        processCssText(cssText);
-      if (href) {
+        return;
-        fetch(href)
-          .then(response => {
-            if (!response.ok) {
-              throw new Error(`Failed to fetch CSS: ${response.status}`);
-            }
-            return response.text();
-          })
-          .then(cssText => {
-            processCssText(cssText);
-          })
-          .catch(error => {
-            console.error(`Failed to fetch or process CSS from ${href}:`, error);
-          });
       }
+    } catch (e) {
+      console.warn("Could not stringify sheet, falling back to fetch:", e);
     }
-  } else if (node.href) {
+  }
+
+  // Fall back to fetching if we couldn't get or stringify the sheet
+  if (node.href) {
     fetch(node.href)
-      .then(response => response.text())
+      .then(response => {
+        if (!response.ok) {
+          throw new Error(`Failed to fetch CSS: ${response.status}`);
+        }
+        return response.text();
+      })
       .then(cssText => {
+        if (sendPlain && onPlain) {
+          onPlain(cssText, fakeIdHolder++);
+        }
         processCssText(cssText);
       })
       .catch(error => {
         console.error(`Failed to fetch CSS from ${node.href}:`, error);
       });
   }
-}
+
+  function processCssText(cssText: string) {
+    // Remove comments
+    cssText = cssText.replace(/\/\*[\s\S]*?\*\//g, '');
+
+    // Parse and process the CSS text to extract rules
+    const ruleTexts = parseCSS(cssText);
+
+    for (let i = 0; i < ruleTexts.length; i++) {
+      insertRule(sheetId, ruleTexts[i], i, baseHref);
+    }
+  }
+
+
+  function parseCSS(cssText: string): string[] {
+    const rules: string[] = [];
+    let inComment = false;
+    let inString = false;
+    let stringChar = '';
+    let braceLevel = 0;
+    let currentRule = '';
+
+    for (let i = 0; i < cssText.length; i++) {
+      const char = cssText[i];
+      const nextChar = cssText[i + 1] || '';
+
+      // comments
+      if (!inString && char === '/' && nextChar === '*') {
+        inComment = true;
+        i++; // Skip the next character
+        continue;
+      }
+
+      if (inComment) {
+        if (char === '*' && nextChar === '/') {
+          inComment = false;
+          i++; // Skip the next character
+        }
+        continue;
+      }
+
+
+      if (!inString && (char === '"' || char === "'")) {
+        inString = true;
+        stringChar = char;
+        currentRule += char;
+        continue;
+      }
+
+      if (inString) {
+        currentRule += char;
+        if (char === stringChar && cssText[i - 1] !== '\\') {
+          inString = false;
+        }
+        continue;
+      }
+
+
+      currentRule += char;
+
+      if (char === '{') {
+        braceLevel++;
+      } else if (char === '}') {
+        braceLevel--;
+
+        if (braceLevel === 0) {
+          // End of a top-level rule
+          rules.push(currentRule.trim());
+          currentRule = '';
+        }
+      }
+    }
+
+    // Handle any remaining text (should be rare)
+    if (currentRule.trim()) {
+      rules.push(currentRule.trim());
+    }
+
+    return rules;
+  }
+
+
+  function stringifyStylesheet(s: CSSStyleSheet): string | null {
+    try {
+      const rules = s.rules || s.cssRules;
+      if (!rules) {
+        return null;
+      }
+
+      let sheetHref = s.href;
+      if (!sheetHref && s.ownerNode && (s.ownerNode as HTMLElement).ownerDocument) {
+        // an inline <style> element
+        sheetHref = (s.ownerNode as HTMLElement).ownerDocument.location.href;
+      }
+
+      const stringifiedRules = Array.from(rules, (rule: CSSRule) =>
+        stringifyRule(rule, sheetHref)
+      ).join('');
+
+      return fixBrowserCompatibilityIssuesInCSS(stringifiedRules);
+    } catch (error) {
+      return null;
+    }
+  }
+  function stringifyRule(rule: CSSRule, sheetHref: string | null): string {
+    if (isCSSImportRule(rule)) {
+      let importStringified;
+      try {
+        importStringified =
+          // for same-origin stylesheets,
+          // we can access the imported stylesheet rules directly
+          stringifyStylesheet((rule as any).styleSheet) ||
+          // work around browser issues with the raw string `@import url(...)` statement
+          escapeImportStatement(rule as any);
+      } catch (error) {
+        importStringified = rule.cssText;
+      }
+      if ((rule as any).styleSheet.href) {
+        // url()s within the imported stylesheet are relative to _that_ sheet's href
+        return absolutifyURLs(importStringified, (rule as any).styleSheet.href);
+      }
+      return importStringified;
+    } else {
+      let ruleStringified = rule.cssText;
+      if (isCSSStyleRule(rule) && (rule as any).selectorText.includes(':')) {
+        // Safari does not escape selectors with : properly
+        ruleStringified = fixSafariColons(ruleStringified);
+      }
+      if (sheetHref) {
+        return absolutifyURLs(ruleStringified, sheetHref);
+      }
+      return ruleStringified;
+    }
+  }
+  function fixBrowserCompatibilityIssuesInCSS(cssText: string): string {
+    // Fix for Chrome's handling of webkit-background-clip
+    if (
+      cssText.includes(' background-clip: text;') &&
+      !cssText.includes(' -webkit-background-clip: text;')
+    ) {
+      cssText = cssText.replace(
+        /\sbackground-clip:\s*text;/g,
+        ' -webkit-background-clip: text; background-clip: text;',
+      );
+    }
+    return cssText;
+  }
+
+  function escapeImportStatement(rule: any): string {
+    const { cssText } = rule;
+    if (cssText.split('"').length < 3) return cssText;
+
+    const statement = ['@import', `url(${JSON.stringify(rule.href)})`];
+    if (rule.layerName === '') {
+      statement.push(`layer`);
+    } else if (rule.layerName) {
+      statement.push(`layer(${rule.layerName})`);
+    }
+    if (rule.supportsText) {
+      statement.push(`supports(${rule.supportsText})`);
+    }
+    if (rule.media.length) {
+      statement.push(rule.media.mediaText);
+    }
+    return statement.join(' ') + ';';
+  }
+
+  function fixSafariColons(cssStringified: string): string {
+    const regex = /(\[(?:[\w-]+)[^\\])(:(?:[\w-]+)\])/gm;
+    return cssStringified.replace(regex, '$1\\$2');
+  }
+
+  function isCSSImportRule(rule: CSSRule): boolean {
+    return 'styleSheet' in rule;
+  }
+
+  function isCSSStyleRule(rule: CSSRule): boolean {
+    return 'selectorText' in rule;
+  }
+
+  function absolutifyURLs(cssText: string | null, href: string): string {
+    if (!cssText) return '';
+
+    const URL_IN_CSS_REF = /url\((?:(')([^']*)'|(")(.*?)"|([^)]*))\)/gm;
+    const URL_PROTOCOL_MATCH = /^(?:[a-z+]+:)?\/\//i;
+    const URL_WWW_MATCH = /^www\..*/i;
+    const DATA_URI = /^(data:)([^,]*),(.*)/i;
+
+    return cssText.replace(
+      URL_IN_CSS_REF,
+      (
+        origin: string,
+        quote1: string,
+        path1: string,
+        quote2: string,
+        path2: string,
+        path3: string,
+      ) => {
+        const filePath = path1 || path2 || path3;
+        const maybeQuote = quote1 || quote2 || '';
+        if (!filePath) {
+          return origin;
+        }
+        if (URL_PROTOCOL_MATCH.test(filePath) || URL_WWW_MATCH.test(filePath)) {
+          return `url(${maybeQuote}${filePath}${maybeQuote})`;
+        }
+        if (DATA_URI.test(filePath)) {
+          return `url(${maybeQuote}${filePath}${maybeQuote})`;
+        }
+        if (filePath[0] === '/') {
+          return `url(${maybeQuote}${
+            extractOrigin(href) + filePath
+          }${maybeQuote})`;
+        }
+        const stack = href.split('/');
+        const parts = filePath.split('/');
+        stack.pop();
+        for (const part of parts) {
+          if (part === '.') {
+            continue;
+          } else if (part === '..') {
+            stack.pop();
+          } else {
+            stack.push(part);
+          }
+        }
+        return `url(${maybeQuote}${stack.join('/')}${maybeQuote})`;
+      },
+    );
+  }
+
+  function extractOrigin(url: string): string {
+    let origin = '';
+    if (url.indexOf('//') > -1) {
+      origin = url.split('/').slice(0, 3).join('/');
+    } else {
+      origin = url.split('/')[0];
+    }
+    origin = origin.split('?')[0];
+    return origin;
+  }
+}

tracker/tracker/src/main/app/observer/observer.ts

@@ -184,6 +184,15 @@ enum RecentsType {
   Changed,
 }
 
+interface Options {
+  inlineRemoteCss?: boolean,
+  disableSprites?: boolean,
+  inlinerOptions?: {
+    forceFetch?: boolean,
+    forcePlain?: boolean,
+  }
+}
+
 export default abstract class Observer {
   private readonly observer: MutationObserver
   private readonly commited: Array<boolean | undefined> = []
@@ -198,14 +207,16 @@ export default abstract class Observer {
    * can (and will) affect performance
    * */
   private readonly inlineRemoteCss: boolean = false
+  private readonly inlinerOptions: Options['inlinerOptions'] = undefined
   private readonly domParser = new DOMParser()
   constructor(
     protected readonly app: App,
-    protected readonly isTopContext = false,
+    protected readonly isTopContext: boolean = false,
-    options: { disableSprites: boolean, inlineRemoteCss: boolean } = { disableSprites: false, inlineRemoteCss: false },
+    options: Options = {},
   ) {
-    this.disableSprites = options.disableSprites
+    this.disableSprites = Boolean(options.disableSprites)
-    this.inlineRemoteCss = options.inlineRemoteCss
+    this.inlineRemoteCss = Boolean(options.inlineRemoteCss)
+    this.inlinerOptions = options.inlinerOptions
     this.observer = createMutationObserver(
       this.app.safe((mutations) => {
         for (const mutation of mutations) {
@@ -368,15 +379,23 @@ export default abstract class Observer {
         setTimeout(() => {
           inlineRemoteCss(
             // @ts-ignore
-          node,
+            node,
-          id,
+            id,
-          this.app.getBaseHref(),
+            this.app.getBaseHref(),
-          nextID,
+            nextID,
             (id: number, cssText: string, index: number, baseHref: string) => {
               this.app.send(AdoptedSSInsertRuleURLBased(id, cssText, index, baseHref))
             },
             (sheetId: number, ownerId: number) => {
               this.app.send(AdoptedSSAddOwner(sheetId, ownerId))
+            },
+            this.inlinerOptions?.forceFetch,
+            this.inlinerOptions?.forcePlain,
+            (cssText: string, fakeTextId: number) => {
+              this.app.send(CreateTextNode(fakeTextId, id, 0))
+              setTimeout(() => {
+                this.app.send(SetNodeData(fakeTextId, cssText))
+              }, 10)
             }
           )
         }, 0)

tracker/tracker/src/main/app/observer/top_observer.ts

@@ -20,6 +20,10 @@ export interface Options {
    * @default false
    * */
   inlineRemoteCss: boolean
+  inlinerOptions?: {
+    forceFetch?: boolean,
+    forcePlain?: boolean,
+  }
 }
 
 type Context = Window & typeof globalThis
@@ -95,7 +99,7 @@ export default class TopObserver extends Observer {
           this.app.debug.info('doc already observed for', id)
           return
         }
-        const observer = new IFrameObserver(this.app)
+        const observer = new IFrameObserver(this.app, false, {})
         this.iframeObservers.set(iframe, observer)
         this.docObservers.set(currentDoc, observer)
         this.iframeObserversArr.push(observer)

tracker/tracker/src/tests/cssInliner.test.ts

@@ -53,7 +53,7 @@ describe('inlineRemoteCss', () => {
     jest.runAllTimers();
     expect(mockNextID).toHaveBeenCalled();
     expect(mockAdoptedSSAddOwner).toHaveBeenCalledWith(123, 456);
-    expect(mockAdoptedSSInsertRuleURLBased).toHaveBeenCalledWith(456, 'body { color: red; }', 0, 'http://example.com');
+    expect(mockAdoptedSSInsertRuleURLBased).toHaveBeenCalledWith(123, 'body { color: red; }', 0, 'http://example.com');
     jest.useRealTimers();
   });
 
@@ -73,6 +73,12 @@ describe('inlineRemoteCss', () => {
   test('should handle successful fetch and process CSS text', async () => {
     mockNode.href = 'http://example.com/style.css';
     const mockSheet = {};
+    global.fetch.mockImplementationOnce(() =>
+      Promise.resolve({
+        ok: true,
+        text: () => Promise.resolve('body { color: red; }')
+      })
+    );
     Object.defineProperty(mockSheet, 'cssRules', {
       get: () => { throw new Error('CORS error'); }
     });
@@ -80,18 +86,26 @@ describe('inlineRemoteCss', () => {
       get: () => mockSheet
     });
     inlineRemoteCss(mockNode, 456,  'http://example.com',mockNextID,mockAdoptedSSInsertRuleURLBased, mockAdoptedSSAddOwner);
-    await new Promise(process.nextTick);
+    await new Promise(resolve => setTimeout(resolve, 0));
     expect(mockNextID).toHaveBeenCalled();
     expect(mockAdoptedSSAddOwner).toHaveBeenCalledWith(123, 456);
     expect(mockAdoptedSSInsertRuleURLBased).toHaveBeenCalledWith(123, 'body { color: red; }', 0, 'http://example.com');
+    jest.useRealTimers();
   });
 
-  test('should fetch CSS if node has no sheet but has href', () => {
+  test('should fetch CSS if node has no sheet but has href', async () => {
     Object.defineProperty(mockNode, 'sheet', {
       get: () => null
     });
+    global.fetch.mockImplementationOnce(() =>
+      Promise.resolve({
+        ok: true,
+        text: () => Promise.resolve('body { color: red; }')
+      })
+    );
     mockNode.href = 'http://example.com/style.css';
     inlineRemoteCss(mockNode, 456,  'http://example.com',mockNextID,mockAdoptedSSInsertRuleURLBased, mockAdoptedSSAddOwner);
+    await new Promise(resolve => setTimeout(resolve, 0));
     expect(global.fetch).toHaveBeenCalledWith('http://example.com/style.css');
   });