[sourcemaps-reader] Azure blob storage support (#1259)

* feat(sourcemaps-reader): implemented azure blob storage support for sourcemaps reader

* feat(sourcemaps-reader): azure blob storage support - cleaned code

---------

Co-authored-by: Taha Yassine Kraiem <tahayk2@gmail.com>
This commit is contained in:
Alexander 2023-05-17 17:09:16 +02:00 committed by GitHub
parent 8d132edb58
commit caf599d955
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 363 additions and 37 deletions

View file

@ -9,11 +9,196 @@
"version": "v1.12.0",
"license": "Elastic License 2.0 (ELv2)",
"dependencies": {
"@azure/storage-blob": "^12.14.0",
"aws-sdk": "^2.1367.0",
"express": "^4.18.2",
"source-map": "^0.7.4"
}
},
"node_modules/@azure/abort-controller": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz",
"integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==",
"dependencies": {
"tslib": "^2.2.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/@azure/core-auth": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.4.0.tgz",
"integrity": "sha512-HFrcTgmuSuukRf/EdPmqBrc5l6Q5Uu+2TbuhaKbgaCpP2TfAeiNaQPAadxO+CYBRHGUzIDteMAjFspFLDLnKVQ==",
"dependencies": {
"@azure/abort-controller": "^1.0.0",
"tslib": "^2.2.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/@azure/core-http": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.1.tgz",
"integrity": "sha512-A3x+um3cAPgQe42Lu7Iv/x8/fNjhL/nIoEfqFxfn30EyxK6zC13n+OUxzZBRC0IzQqssqIbt4INf5YG7lYYFtw==",
"dependencies": {
"@azure/abort-controller": "^1.0.0",
"@azure/core-auth": "^1.3.0",
"@azure/core-tracing": "1.0.0-preview.13",
"@azure/core-util": "^1.1.1",
"@azure/logger": "^1.0.0",
"@types/node-fetch": "^2.5.0",
"@types/tunnel": "^0.0.3",
"form-data": "^4.0.0",
"node-fetch": "^2.6.7",
"process": "^0.11.10",
"tslib": "^2.2.0",
"tunnel": "^0.0.6",
"uuid": "^8.3.0",
"xml2js": "^0.5.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@azure/core-http/node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/@azure/core-lro": {
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.5.3.tgz",
"integrity": "sha512-ubkOf2YCnVtq7KqEJQqAI8dDD5rH1M6OP5kW0KO/JQyTaxLA0N0pjFWvvaysCj9eHMNBcuuoZXhhl0ypjod2DA==",
"dependencies": {
"@azure/abort-controller": "^1.0.0",
"@azure/core-util": "^1.2.0",
"@azure/logger": "^1.0.0",
"tslib": "^2.2.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@azure/core-paging": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.5.0.tgz",
"integrity": "sha512-zqWdVIt+2Z+3wqxEOGzR5hXFZ8MGKK52x4vFLw8n58pR6ZfKRx3EXYTxTaYxYHc/PexPUTyimcTWFJbji9Z6Iw==",
"dependencies": {
"tslib": "^2.2.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@azure/core-tracing": {
"version": "1.0.0-preview.13",
"resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz",
"integrity": "sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ==",
"dependencies": {
"@opentelemetry/api": "^1.0.1",
"tslib": "^2.2.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/@azure/core-util": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.3.2.tgz",
"integrity": "sha512-2bECOUh88RvL1pMZTcc6OzfobBeWDBf5oBbhjIhT1MV9otMVWCzpOJkkiKtrnO88y5GGBelgY8At73KGAdbkeQ==",
"dependencies": {
"@azure/abort-controller": "^1.0.0",
"tslib": "^2.2.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@azure/logger": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.4.tgz",
"integrity": "sha512-ustrPY8MryhloQj7OWGe+HrYx+aoiOxzbXTtgblbV3xwCqpzUK36phH3XNHQKj3EPonyFUuDTfR3qFhTEAuZEg==",
"dependencies": {
"tslib": "^2.2.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@azure/storage-blob": {
"version": "12.14.0",
"resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.14.0.tgz",
"integrity": "sha512-g8GNUDpMisGXzBeD+sKphhH5yLwesB4JkHr1U6be/X3F+cAMcyGLPD1P89g2M7wbEtUJWoikry1rlr83nNRBzg==",
"dependencies": {
"@azure/abort-controller": "^1.0.0",
"@azure/core-http": "^3.0.0",
"@azure/core-lro": "^2.2.0",
"@azure/core-paging": "^1.1.1",
"@azure/core-tracing": "1.0.0-preview.13",
"@azure/logger": "^1.0.0",
"events": "^3.0.0",
"tslib": "^2.2.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@azure/storage-blob/node_modules/events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
"engines": {
"node": ">=0.8.x"
}
},
"node_modules/@opentelemetry/api": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz",
"integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==",
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/@types/node": {
"version": "20.1.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.5.tgz",
"integrity": "sha512-IvGD1CD/nego63ySR7vrAKEX3AJTcmrAN2kn+/sDNLi1Ff5kBzDeEdqWDplK+0HAEoLYej137Sk0cUU8OLOlMg=="
},
"node_modules/@types/node-fetch": {
"version": "2.6.4",
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz",
"integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==",
"dependencies": {
"@types/node": "*",
"form-data": "^3.0.0"
}
},
"node_modules/@types/node-fetch/node_modules/form-data": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/@types/tunnel": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz",
"integrity": "sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/accepts": {
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@ -31,6 +216,11 @@
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"node_modules/available-typed-arrays": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
@ -134,6 +324,17 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@ -174,6 +375,14 @@
"ms": "2.0.0"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@ -291,6 +500,19 @@
"is-callable": "^1.1.3"
}
},
"node_modules/form-data": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -551,6 +773,25 @@
"node": ">= 0.6"
}
},
"node_modules/node-fetch": {
"version": "2.6.11",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.11.tgz",
"integrity": "sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/object-inspect": {
"version": "1.12.3",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz",
@ -583,6 +824,14 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
},
"node_modules/process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
"integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
"engines": {
"node": ">= 0.6.0"
}
},
"node_modules/proxy-addr": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@ -758,6 +1007,24 @@
"node": ">=0.6"
}
},
"node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"node_modules/tslib": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
},
"node_modules/tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
"engines": {
"node": ">=0.6.11 <=0.7.0 || >=0.7.3"
}
},
"node_modules/type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@ -823,6 +1090,20 @@
"node": ">= 0.8"
}
},
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"dependencies": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"
}
},
"node_modules/which-typed-array": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.9.tgz",

View file

@ -18,6 +18,7 @@
},
"homepage": "https://github.com/openreplay/openreplay#readme",
"dependencies": {
"@azure/storage-blob": "^12.14.0",
"aws-sdk": "^2.1367.0",
"express": "^4.18.2",
"source-map": "^0.7.4"

View file

@ -2,6 +2,7 @@
const fs = require('fs');
const sourceMap = require('source-map');
const AWS = require('aws-sdk');
const { BlobServiceClient, StorageSharedKeyCredential} = require("@azure/storage-blob");
const URL = require('url');
const http = require('http');
const wasm = fs.readFileSync(process.env.MAPPING_WASM || '/mappings.wasm');
@ -134,46 +135,89 @@ module.exports.sourcemapReader = async event => {
});
});
} else {
let s3;
if (process.env.S3_HOST) {
s3 = new AWS.S3({
endpoint: process.env.S3_HOST,
accessKeyId: process.env.S3_KEY,
secretAccessKey: process.env.S3_SECRET,
s3ForcePathStyle: true, // needed with minio?
signatureVersion: 'v4'
});
} else {
s3 = new AWS.S3({
'AccessKeyID': process.env.aws_access_key_id,
'SecretAccessKey': process.env.aws_secret_access_key,
'Region': process.env.aws_region
});
}
if (process.env.CLOUD === 'azure') {
// Download the file from Azure Blob Storage
const name = process.env.AZURE_ACCOUNT_NAME;
const key = process.env.AZURE_ACCOUNT_KEY;
const url = `https://${name}.blob.core.windows.net/`;
let options = {
Bucket: event.bucket,
Key: event.key
};
return new Promise(function (resolve, reject) {
const getObjectStart = Date.now();
s3.getObject(options, (err, data) => {
if (err) {
console.error("[SR] Get S3 object failed");
console.error(err);
return new Promise(async function (resolve, reject) {
try {
// Init ABS client and get account info to check connection
let client = new BlobServiceClient(url, new StorageSharedKeyCredential(name, key));
await client.getAccountInfo();
let containerClient = client.getContainerClient(event.bucket);
const getObjectStart = Date.now();
const response = await containerClient.getBlobClient(event.key).downloadToBuffer();
const getObjectEnd = Date.now();
let options = {
Bucket: event.bucket,
Key: event.key,
fileSize: (data.ContentLength / 1024) / 1024,
fileSizeUnit: 'Mb',
downloadTime: (getObjectEnd - getObjectStart) / 1000,
downloadTimeUnit: 's',
};
if (options.fileSize >= 3) {
console.log("[SR] large file:" + JSON.stringify(options));
}
let sourcemap = response.toString();
return parseSourcemap(sourcemap, event, options, resolve, reject);
} catch (err) {
if (err.statusCode && err.statusCode === 404) {
console.log("blob not found")
} else {
console.log("unknown error:", err);
}
return reject(err);
}
const getObjectEnd = Date.now();
options.fileSize = (data.ContentLength / 1024) / 1024;
options.fileSizeUnit = 'Mb';
options.downloadTime = (getObjectEnd - getObjectStart) / 1000;
options.downloadTimeUnit = 's';
if (options.fileSize >= 3) {
console.log("[SR] large file:" + JSON.stringify(options));
}
let sourcemap = data.Body.toString();
return parseSourcemap(sourcemap, event, options, resolve, reject);
});
});
} else {
// Download the file from S3
let s3;
if (process.env.S3_HOST) {
s3 = new AWS.S3({
endpoint: process.env.S3_HOST,
accessKeyId: process.env.S3_KEY,
secretAccessKey: process.env.S3_SECRET,
s3ForcePathStyle: true, // needed with minio?
signatureVersion: 'v4'
});
} else {
s3 = new AWS.S3({
'AccessKeyID': process.env.aws_access_key_id,
'SecretAccessKey': process.env.aws_secret_access_key,
'Region': process.env.aws_region
});
}
let options = {
Bucket: event.bucket,
Key: event.key
};
return new Promise(function (resolve, reject) {
const getObjectStart = Date.now();
s3.getObject(options, (err, data) => {
if (err) {
console.error("[SR] Get S3 object failed");
console.error(err);
return reject(err);
}
const getObjectEnd = Date.now();
options.fileSize = (data.ContentLength / 1024) / 1024;
options.fileSizeUnit = 'Mb';
options.downloadTime = (getObjectEnd - getObjectStart) / 1000;
options.downloadTimeUnit = 's';
if (options.fileSize >= 3) {
console.log("[SR] large file:" + JSON.stringify(options));
}
let sourcemap = data.Body.toString();
return parseSourcemap(sourcemap, event, options, resolve, reject);
});
});
}
}
};