From ca7dd4ae4636af5196895e7f2997194be2f5c612 Mon Sep 17 00:00:00 2001 From: nick-delirium Date: Thu, 11 Jan 2024 16:53:44 +0100 Subject: [PATCH] fix(tracker): expose canvas tracking restart method, remove context creation in assist --- tracker/tracker-assist/CHANGELOG.md | 4 ++++ tracker/tracker-assist/bun.lockb | Bin 249605 -> 241984 bytes tracker/tracker-assist/package.json | 4 ++-- tracker/tracker-assist/src/Canvas.ts | 1 - tracker/tracker-assist/src/version.ts | 2 +- tracker/tracker/CHANGELOG.md | 5 +++++ tracker/tracker/package.json | 2 +- tracker/tracker/src/main/app/canvas.ts | 18 ++++++++++++++---- tracker/tracker/src/main/app/index.ts | 22 ++++++++++++++++------ tracker/tracker/src/main/index.ts | 7 +++++++ 10 files changed, 50 insertions(+), 15 deletions(-) diff --git a/tracker/tracker-assist/CHANGELOG.md b/tracker/tracker-assist/CHANGELOG.md index 4422b4446..464985ec8 100644 --- a/tracker/tracker-assist/CHANGELOG.md +++ b/tracker/tracker-assist/CHANGELOG.md @@ -1,3 +1,7 @@ +## 7.0.3 + +- small fix for canvas context tracking + ## 7.0.1 - mark live sessions with ux test active diff --git a/tracker/tracker-assist/bun.lockb b/tracker/tracker-assist/bun.lockb index 7136f43beb6bd0b0021b5fcda14ef1399eaf7489..9f375c61a44fb98b192a5a588906d75101d9af5a 100755 GIT binary patch delta 2908 zcmYk8dr(wW9LLW+yNfI@VNp>YLLeZxyDXrH50HnV0zOcY6l5d?QyC*9VJTZQF#m`m zEf14uqnS0+R2Vv&2tG=4w3n7P7Fxd2tb8GwW@+Ew@?)km``!D!pZh!a{LVS|o^uMPLLSzb2=ebd{#cfQB(zo?= zoH7~S(F>t#yWY{&XSZC;`8Hn@kNQ8-gea7*MqGdT`ZqJOjM;_zvnStb9NkiCE`Ih{ z$IY|HoKJt1RFt)9?<%kMbw?6rJS@4N^1QyFIq{Xx>gS%>o;IQEP{@&^4Lg7Ia%#u5 zObyBCo#r;}z6D7Pa1Ohm<*ZZ-_i6E-t@*vJ^%O~!=d1HqjdNtIt$@nClLfAy+*$4#7LWH@@H{hjux4f# zm>I!hm|bK>iQMI}yu{2C>;q<(!N__+Nf4DQF6I!z8z%!?NQmFKvJcL$aU<85^#{Ah z?006qV5gb=!ORcr5VJp-nZP=kU1#PGw%Gwly}ZGY5}D3f-()rrERWeOW`n@yGP}(z z0Bjz!Zf1kQ7BjoUYzWu{W{w_)fq)jy@Gi3;usCM-m<59+GW&~J2v{()`^<)d{mqR$ z03#a;{mbkj7^*l#7~o@u^iI%yo`7~U({Kb9;m{stl34_plbMd$FtAUV>6t}>?PX?Q z76sNRG5_?SxG{`|x1cq8Q2H?&4j)GuptytKCt{#PW*!_D3pSFOC&!Hd+YZrl;tfWf zGeb@ARD2yURL24}a%DeeR% zYz(|X$MZjo;aI@K@Khq1je~y+qN#}j!+pefs0^b;Qxn5+De$>qG&QlzQsGm8=^)i`>7l_2c8-smdz|5p5mxSIn3t5Qyj5eW(DvRM;)5R z>`8cvBStGMwL4FMQa04f84Tyc6QiVNGAm?83D8nZm5Z2BW3!lrEr3BOFH=9%%WzGo zsg*L*NM8Xvpm)`tN*SSbJ5N{2P)$oxw^z!D_d2Si%rN7YR)|&_8x#*EK(xxxsxk^n zf|4Oxx+0+-wahlIMFI`b%c`+j253!cTQ!)X1Se z#kf;Drhhir98LMy~gIiK9R&yXc@E| zDp%gMGFrQ=;%jA4;0lmmp&O|2J&x7`Dsk-o}YC*9q?b5&73^|;jJ0Bxz4?jFr$ zxC@HS?s|mu^o{8Jr(Ui%1q^_tjes_Yg?9X*jPI7}H6Z(btvzzb-~fkRFtl$ hEUhRnvByjE0CpzW?iw)v}# z)vL!d$11JfR(WGiv(+^kH0!-!bJMZosvR7=F+1&-Ik^KT-JNnMf7#3@=e%@kRl& zR&vpaJ&`jli@rD-8JJ&w--s}NmYQ>8^GAj~qVa=s=ZiNElJxHp?GYQtG)!1=Xb_aP z4?J07DxhP4f=iG3-XEyJw3>>r!e(H=8h>L@Xk^#mBH8WQnrO~jbSn$i-D z0;Aj1td{IslRf^;3@;OmdbqO9YKyHfYc57#Go+PfIb!+?>PYj=T06F`*aEXQX7$7h z#prC?Q1j2wXG)6O z2g#1F|CB5H=DH<8qU=ov%NIlY@YUvW(SE;Vq2&^u`EEQjIGeDlkAorg+*qa#afvyHtWLmw=-N~ z*j0`hW=qYwiG3}b?N;qEyG!4iEz_Qm%cb*XE6sX{J#KcJ_H??Zv=_$suQBW;yU31f z&8`sJXSU9)x7d?r>&>ndJ0P1hz1^&j?3QYcG`+*@D%q{fHi$*(O<$?K;hlEuCw9L? z?%pNF)%BNlY97eld+c|$>|9QwZIjsm+2hPMn++73AV!McD@KM5;`#>(lH*(Lc#Z50 zYJ?nr&}^{mjVu6d+sv+&eYe?mvms&yV&we}F;Z-(#3*EH{}cNSlg%h(GX6g`94^Qx z;P{By2-%E6L+Md5Mq#A1OClxrm|Z8EsbG7;Y?SPcvPm_jOQ#M-OY3EjX-|ojVg9d| zHt?_7p0(o*vPnx)?V$GD!Ny2SBvS1Kg0&wjaSpbZ9h)or&NvBzSIoxg_#w4HO8&}h zyzJ;sfE63mhEbakZKr}n!= z_5%`Gb<%9E><5`U;Ae*UvRSR9%jahEWV2dfUziohX0gIfna!6?XBfrPVsvJK8Ck_G z9k$Sn%!r;bEYtzLCN<9bgo|X;YqH=Qv&FI>G^6?{B)%x|BQwsKEtQQQjEacgtu}n^Az3Gg~2>QGhZ3GJ}=UBN*CJ#5naT=`k@f zjroVImh!p(lWLV6e4A_<9;1D#*&5k24696-xF+X$IfPqK$ck-GZtGDv#w@a#hRLRGh+cZGwW`)O)T5&@+8ea_I9bc z3@TF(IWjamBo-P4=PC#9l+7Ao>ua`4HXWiC^fP-%HjTsjn>{R>cBlncn|)vH$LE;5 zA0Vi|;0F?^NZwy##~;e35psO6*^gw?2<%$3f02zJc|XMLUuEM58*27r+4w~l>SLJU zzsbfCHr(tdvgtLoV1(IE&6a3C((Do0EKG9zIMz(MF$&AIAM4=9WgiqHpL4~y>Fkw=C3Pn|_6gadP20xb=h1;3` z0>h_eb0yS*`C|GDo{^X*YC)m>o|Vls!4{bvlua!6{l$)bPBx=Jec(GI4L>i@L23cD zC;t5R?>eA|z-4y)583n(#urI~UyulLeaz5qy=IWm*>r1mt7#n6ufBmhU|Q?ZQ36)`?YLFYrFQp z5tA9bDKT0wzV1iVe(;vWZ~+h7@i($xB|_UfW^c>pX0ucKcg+sVCT^GXp4mIHxrpye z@0-0Vn-0T{n7t=EO7IV*qlWLxhEYh4nH@2sXdE{?Dw_pI-k%WDUvNxf!IAeL+V8k* zI$jtM58|6ukz*#yJBHv1phbcTFBE2h8TONq{q&gbm+ zm2CW8kj~rh_p5DZH(i|CkG>O%#yP{+i=Dw^?S zgIHK4vt%8&B>yNRl?_V?5)4ZX=|ESWUA^IwlrY%dE1FiG^Kema1c7DRi~X zqE!S424)zh>6l4xjAKKyS~@0}65Gh^G9A+}tg%^b9TN-7GRx30tDjQbM2sw{BgX2d6lW)E z{$&Ss^|4)hN@sHi^8`#P9+g^{Wr~rCu$E>#0TX*%$}!^!n1(58t;A@UCtw<;sI|3U zBkooNQ^DF9@&rsU)vCRNvm8vZ>SVtrVswzg)Y*(DU^)ovV#X6NZBUrHn(+h-i&Aj9 z8S(_o2`Mh!&06?`u*=Pu4*V!CJ}*GB z!Eifvk&PdDKf-=p?ML2^vR^m*k>jK7*WG@k_!#?LZoi7m|5!Wrup{|A-i|%(NbXLs zUoZQSx|8j9h5bm=Dfa7aziQe~v)`5WBPFMc@eCOBu_HM-!;V+kk@TDC6ZW+q={MVc z{baKiNHyM;YZvs_2a6DPi`mt(S#V_4T(be>Kf$C+zTrUG1jFW;@s1EbGK2Sf+6C9> z13wt={b7UkK_g@a@BLxdnz6VSnhi0F*MEKtz;UQPXoOYEZvn7j`k)cmVzc40@ng~Q ztA=*L2z}rOTWU5^HhwHx{n8QlyUu>FcI%&Vc><0V6 zR+^2mAB%RC*;xC*R-5tKHcAg!wYM4a-j*wYtuY%f8$VXvRxym)SJ?v1oZutX(i&ANav|PYkLp9 zKgBeC>};ehI^H5L2-#IFqez{3wZ8)C~RhV^0(PYg3_y{iuL1aep>eqVZ2_HJo! z>?A&^lfr$mZzhEq16Y=nA(kxp!18Cgv%F78_B zywYy36LyrSr_|#`5_c1B)ODo>(zo(GmK3{ZN|+tZ(K)HfZ6qEPcnF|Wa!=0CdAL{a zRLle&lf&FB21^tc@|3DnpnW5W2S^?mn@CNiW>U7)Txub;lyan2QfrQ>S{LiU|K)@? z>ZEj-hr)K!-#*8LzwQ;^xA}s%ODB3x`=_OcC4PZgudCf6@kN3km6mA#kd6;)zh62a zot1dhIxkg<#6Fo4-VmisKA_{R5~Y-qN^#--u2Hz?AaF5=TRHdrzkNY}eLaeg#-Hm- zb|SGu{Fv~N^aJU6HF&8`J*JM#mfswSx|S~$NEeUkUi_YZvrf#(xuO%c2`ah%SbJ|u zZ%EA}vD>DGz2k3Jcqc?GH?B)vOyY$x(%h(sy|6FY(Zn^+kD`ffyrB3OS*HhOJSWXB z;m}sc?Id1cmOOD{LUGc>jwLp7skq{Q)7`D)D3PV>q&3oNX{EGWib_kR#ZsZf{is0V zMt7-(N^6btQtZMg?MUo!1s(ipC}UUTg;ir?^TM*x#H&cWI($nuc43%H`gPIBb&@(u zZKNEjWNgWi^b$x(7}D6esbPAQh7&uLXkY52|Db=1{zVb}2c7iqIH%Uh#R}>_qce%4 zdq^WcKUs6PBQ`WItTC+QG+$U(i80^BE*7^V`11^x^hrGDKiQ66ba^EWef--UaA7K`kS GZ22R2FFlO_ diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 363e9910c..e46b95669 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "7.0.4", + "version": "7.0.3", "keywords": [ "WebRTC", "assistance", @@ -34,7 +34,7 @@ "socket.io-client": "^4.7.2" }, "peerDependencies": { - "@openreplay/tracker": "^12.0.0" + "@openreplay/tracker": "^11.0.5" }, "devDependencies": { "@openreplay/tracker": "file:../tracker", diff --git a/tracker/tracker-assist/src/Canvas.ts b/tracker/tracker-assist/src/Canvas.ts index 068ecaa26..7d6c13c54 100644 --- a/tracker/tracker-assist/src/Canvas.ts +++ b/tracker/tracker-assist/src/Canvas.ts @@ -9,7 +9,6 @@ export default class CanvasRecorder { private readonly onStream: (stream: MediaStream) => void, private readonly logError: (...args: any[]) => void, ) { - this.canvas.getContext('2d', { alpha: true, }) const stream = this.canvas.captureStream(this.fps) this.emitStream(stream) } diff --git a/tracker/tracker-assist/src/version.ts b/tracker/tracker-assist/src/version.ts index 39847a820..f4abd5f6f 100644 --- a/tracker/tracker-assist/src/version.ts +++ b/tracker/tracker-assist/src/version.ts @@ -1 +1 @@ -export const pkgVersion = '7.0.4' +export const pkgVersion = '7.0.3' diff --git a/tracker/tracker/CHANGELOG.md b/tracker/tracker/CHANGELOG.md index 901521e4a..bd0491928 100644 --- a/tracker/tracker/CHANGELOG.md +++ b/tracker/tracker/CHANGELOG.md @@ -1,3 +1,8 @@ +# 11.0.5 + +- add method to restart canvas tracking (in case of context recreation) +- scan dom tree for canvas els on tracker start + # 11.0.4 - some additional security for canvas capture (check if canvas el itself is obscured/ignored) diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index 7cedd088c..8295276b0 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "11.0.4", + "version": "11.0.5", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/app/canvas.ts b/tracker/tracker/src/main/app/canvas.ts index 43c809155..fa54c6772 100644 --- a/tracker/tracker/src/main/app/canvas.ts +++ b/tracker/tracker/src/main/app/canvas.ts @@ -10,6 +10,7 @@ interface CanvasSnapshot { interface Options { fps: number quality: 'low' | 'medium' | 'high' + isDebug?: boolean } class CanvasRecorder { @@ -25,10 +26,17 @@ class CanvasRecorder { } startTracking() { + setTimeout(() => { + this.app.nodes.scanTree(this.handleCanvasEl) + this.app.nodes.attachNodeCallback((node: Node): void => { + this.handleCanvasEl(node) + }) + }, 500) + } + + restartTracking = () => { + this.clear() this.app.nodes.scanTree(this.handleCanvasEl) - this.app.nodes.attachNodeCallback((node: Node): void => { - this.handleCanvasEl(node) - }) } handleCanvasEl = (node: Node) => { @@ -74,7 +82,9 @@ class CanvasRecorder { images.forEach((snapshot) => { const blob = dataUrlToBlob(snapshot.data)[0] formData.append('snapshot', blob, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`) - // saveImageData(snapshot.data, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`) + if (this.options.isDebug) { + saveImageData(snapshot.data, `${createdAt}_${canvasId}_${snapshot.id}.jpeg`) + } }) fetch(this.app.options.ingestPoint + '/v1/web/images', { diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 8565b2696..5cae53b74 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -106,6 +106,7 @@ type AppOptions = { __is_snippet: boolean __debug_report_edp: string | null __debug__?: LoggerOptions + __save_canvas_locally?: boolean localStorage: Storage | null sessionStorage: Storage | null forceSingleTab?: boolean @@ -183,6 +184,7 @@ export default class App { verbose: false, __is_snippet: false, __debug_report_edp: null, + __save_canvas_locally: false, localStorage: null, sessionStorage: null, disableStringDict: false, @@ -702,17 +704,21 @@ export default class App { this.compressionThreshold = compressionThreshold const onStartInfo = { sessionToken: token, userUUID, sessionID } + if (canvasEnabled) { + this.canvasRecorder = + this.canvasRecorder ?? + new CanvasRecorder(this, { + fps: canvasFPS, + quality: canvasQuality, + isDebug: this.options.__save_canvas_locally, + }) + this.canvasRecorder.startTracking() + } // TODO: start as early as possible (before receiving the token) this.startCallbacks.forEach((cb) => cb(onStartInfo)) // MBTODO: callbacks after DOM "mounted" (observed) this.observer.observe() this.ticker.start() - if (canvasEnabled) { - this.canvasRecorder = - this.canvasRecorder ?? - new CanvasRecorder(this, { fps: canvasFPS, quality: canvasQuality }) - this.canvasRecorder.startTracking() - } this.activityState = ActivityState.Active this.notify.log('OpenReplay tracking started.') @@ -767,6 +773,10 @@ export default class App { }) } + restartCanvasTracking = () => { + this.canvasRecorder?.restartTracking() + } + onUxtCb = [] addOnUxtCb(cb: (id: number) => void) { diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index 3c70078a3..ef91071a3 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -223,6 +223,13 @@ export default class API { return this.featureFlags.flags } + public restartCanvasTracking = () => { + if (this.app === null) { + return + } + this.app.restartCanvasTracking() + } + use(fn: (app: App | null, options?: Options) => T): T { return fn(this.app, this.options) }