From 4315ac9f062ca808a3ab0d91edd7ee1b71cf0d0e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 27 Jan 2023 17:08:39 +0100 Subject: [PATCH 1/3] feat(chalice): obfuscate Github token after insert/update --- api/chalicelib/core/integration_github.py | 11 +++++++---- api/chalicelib/utils/helper.py | 6 ++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/integration_github.py b/api/chalicelib/core/integration_github.py index b300aa7f7..0be412122 100644 --- a/api/chalicelib/core/integration_github.py +++ b/api/chalicelib/core/integration_github.py @@ -24,8 +24,7 @@ class GitHubIntegration(integration_base.BaseIntegration): integration = self.get() if integration is None: return None - token = "*" * (len(integration["token"]) - 4) + integration["token"][-4:] - return {"token": token, "provider": self.provider.lower()} + return {"token": helper.obfuscate(text=integration["token"]), "provider": self.provider.lower()} def update(self, changes, obfuscate=False): with pg_client.PostgresClient() as cur: @@ -40,12 +39,14 @@ class GitHubIntegration(integration_base.BaseIntegration): **changes}) ) w = helper.dict_to_camel_case(cur.fetchone()) + if w and w.get("token") and obfuscate: + w["token"] = helper.obfuscate(w["token"]) return w def _add(self, data): pass - def add(self, token): + def add(self, token, obfuscate=False): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify("""\ @@ -56,6 +57,8 @@ class GitHubIntegration(integration_base.BaseIntegration): "token": token}) ) w = helper.dict_to_camel_case(cur.fetchone()) + if w and w.get("token") and obfuscate: + w["token"] = helper.obfuscate(w["token"]) return w # TODO: make a revoke token call @@ -81,4 +84,4 @@ class GitHubIntegration(integration_base.BaseIntegration): obfuscate=True ) else: - return self.add(token=data["token"]) + return self.add(token=data["token"], obfuscate=True) diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index 458691f0c..85e34ec80 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -312,3 +312,9 @@ def get_domain(): if not _url.startswith("http"): _url = "http://" + _url return '.'.join(urlparse(_url).netloc.split(".")[-2:]) + + +def obfuscate(text, keep_last: int = 4): + if text is None or not isinstance(text, str): + return text + return "*" * (len(text) - keep_last) + text[-keep_last:] From b0d0d47da26a2de31bc8dd522591eba842a78998 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 27 Jan 2023 16:47:45 +0100 Subject: [PATCH 2/3] fix(player): start load simultaneously, but handle files data sequentially --- frontend/app/player/web/MessageManager.ts | 17 +++---- frontend/app/player/web/network/loadFiles.ts | 53 ++++++++------------ 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/frontend/app/player/web/MessageManager.ts b/frontend/app/player/web/MessageManager.ts index 93d571303..877a920df 100644 --- a/frontend/app/player/web/MessageManager.ts +++ b/frontend/app/player/web/MessageManager.ts @@ -30,7 +30,7 @@ import type { MouseClick, } from './messages'; -import { loadFiles, requestEFSDom, requestEFSDevtools, NO_FILE_OK, NO_URLS } from './network/loadFiles'; +import { loadFiles, requestEFSDom, requestEFSDevtools } from './network/loadFiles'; import { decryptSessionBytes } from './network/crypto'; import Lists, { INITIAL_STATE as LISTS_INITIAL_STATE, State as ListsState } from './Lists'; @@ -226,22 +226,19 @@ export default class MessageManager { loadFiles(loadMethod.url, loadMethod.parser()) // EFS fallback - .catch((e) => { - if (e === NO_FILE_OK || e === NO_URLS) { - requestEFSDom(this.session.sessionId) - .then(createNewParser(false)) - .catch(this.onFileReadFailed); - } else { - this.onFileReadFailed(e); - } - }) + .catch((e) => + requestEFSDom(this.session.sessionId) + .then(createNewParser(false)) + ) .then(this.onFileReadSuccess) + .catch(this.onFileReadFailed) .finally(this.onFileReadFinally); // load devtools if (this.session.devtoolsURL?.length) { this.state.update({ devtoolsLoading: true }) loadFiles(this.session.devtoolsURL, createNewParser()) + // EFS fallback .catch(() => requestEFSDevtools(this.session.sessionId) .then(createNewParser(false)) diff --git a/frontend/app/player/web/network/loadFiles.ts b/frontend/app/player/web/network/loadFiles.ts index 65ee67d8f..ea7d44a5c 100644 --- a/frontend/app/player/web/network/loadFiles.ts +++ b/frontend/app/player/web/network/loadFiles.ts @@ -1,41 +1,33 @@ import APIClient from 'App/api_client'; -export const NO_FILE_OK = "No-file-but-this-is-ok" -export const NO_SECOND_FILE = 'No-second-file-but-this-is-ok-too' +const ALLOWED_404 = "No-file-and-this-is-ok" const NO_BACKUP_FILE = "No-efs-file" export const NO_URLS = 'No-urls-provided' -async function loadFile(url: string, onData: (d: Uint8Array) => void, skippable: boolean) { - try { - const stream = await window.fetch(url) - const data = await processAPIStreamResponse(stream, skippable) - // Messages are being loaded and processed async, we can go on - onData(data) - return Promise.resolve('success') - } catch (e) { - throw e - } -} -export const loadFiles = async ( +export async function loadFiles( urls: string[], onData: (data: Uint8Array) => void, -): Promise => { +): Promise { if (!urls.length) { - return Promise.reject(NO_URLS) + throw NO_URLS } - - return Promise.allSettled(urls.map(url => - loadFile(url, onData, url === urls[0] && !url.match(/devtools/)) - )).then(results => { - if (results[0].status === 'rejected') { - // if no 1st file, we should fall back to EFS storage or display error - return Promise.reject(results[0].reason) - } else { - // we don't care if second file is missing (expected) - return Promise.resolve() + const fileLoads = urls.map((url, index) => + // loads can start simultaneously + window.fetch(url).then(r => processAPIStreamResponse(r, index === 0)) + ) + try { + for (let fileLoad of fileLoads) { + // binary data should be added sequentially + const data = await fileLoad + onData(data) } - }) + } catch(e) { + if (e === ALLOWED_404) { + return + } + throw e + } } export async function requestEFSDom(sessionId: string) { @@ -55,11 +47,10 @@ async function requestEFSMobFile(filename: string) { return await processAPIStreamResponse(res, false) } -const processAPIStreamResponse = (response: Response, canBeMissed: boolean) => { +const processAPIStreamResponse = (response: Response, skippable: boolean) => { return new Promise((res, rej) => { - if (response.status === 404) { - if (canBeMissed) return rej(NO_FILE_OK) - else return rej(NO_SECOND_FILE); + if (response.status === 404 && skippable) { + return rej(ALLOWED_404) } if (response.status >= 400) { return rej(`Bad file status code ${response.status}. Url: ${response.url}`) From c26706cba487bb2bbfba224198cff0365a7c93f1 Mon Sep 17 00:00:00 2001 From: Alex Kaminskii Date: Fri, 27 Jan 2023 17:07:45 +0100 Subject: [PATCH 3/3] fix(player): load files sequentially (sinse the throughput is common) --- frontend/app/player/web/network/loadFiles.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/frontend/app/player/web/network/loadFiles.ts b/frontend/app/player/web/network/loadFiles.ts index ea7d44a5c..8d8142716 100644 --- a/frontend/app/player/web/network/loadFiles.ts +++ b/frontend/app/player/web/network/loadFiles.ts @@ -12,14 +12,10 @@ export async function loadFiles( if (!urls.length) { throw NO_URLS } - const fileLoads = urls.map((url, index) => - // loads can start simultaneously - window.fetch(url).then(r => processAPIStreamResponse(r, index === 0)) - ) try { - for (let fileLoad of fileLoads) { - // binary data should be added sequentially - const data = await fileLoad + for (let url of urls) { + const response = await window.fetch(url) + const data = await processAPIStreamResponse(response, url !== url[0]) onData(data) } } catch(e) {