feat(utilities): utilities EE-WS as a standalone service

ee/utilities/.gitignore

@@ -11,4 +11,5 @@ servers/peerjs-server.js
 servers/sourcemaps-handler.js
 servers/sourcemaps-server.js
 #servers/websocket.js
-/utils/dump.js
+/utils
+/Dockerfile

ee/utilities/package.json

@@ -20,12 +20,9 @@
   "dependencies": {
     "@maxmind/geoip2-node": "^3.4.0",
     "@socket.io/redis-adapter": "^7.1.0",
-    "aws-sdk": "^2.992.0",
     "express": "^4.17.1",
-    "peer": "^0.6.1",
     "redis": "^4.0.3",
     "socket.io": "^4.4.1",
-    "source-map": "^0.7.3",
     "ua-parser-js": "^1.0.2",
     "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.6.0"
   }

ee/utilities/server.js

@@ -1,8 +1,6 @@
 const dumps = require('./utils/HeapSnapshot');
-const sourcemapsReaderServer = require('./servers/sourcemaps-server');
-const {peerRouter, peerConnection, peerDisconnect, peerError} = require('./servers/peerjs-server');
+const {request_logger} = require('./utils/helper');
 const express = require('express');
-const {ExpressPeerServer} = require('peer');
 let socket;
 if (process.env.redis === "true") {
     console.log("Using Redis");
@@ -12,63 +10,29 @@ if (process.env.redis === "true") {
 }
 
 const HOST = '0.0.0.0';
-const PORT = 9000;
-
-const app = express();
+const PORT = 9001;
 
 let debug = process.env.debug === "1" || false;
-const request_logger = (identity) => {
-    return (req, res, next) => {
-        debug && console.log(identity, new Date().toTimeString(), 'REQUEST', req.method, req.originalUrl);
-        res.on('finish', function () {
-            if (this.statusCode !== 200 || debug) {
-                console.log(new Date().toTimeString(), 'RESPONSE', req.method, req.originalUrl, this.statusCode);
-            }
-        })
-
-        next();
-    }
-};
-app.use(request_logger("[app]"));
-
-
-app.use('/sourcemaps', sourcemapsReaderServer);
-app.use('/assist', peerRouter);
-
-const server = app.listen(PORT, HOST, () => {
-    console.log(`App listening on http://${HOST}:${PORT}`);
-    console.log('Press Ctrl+C to quit.');
-});
-
-const peerServer = ExpressPeerServer(server, {
-    debug: true,
-    path: '/',
-    proxied: true,
-    allow_discovery: false
-});
-peerServer.on('connection', peerConnection);
-peerServer.on('disconnect', peerDisconnect);
-peerServer.on('error', peerError);
-app.use('/', peerServer);
-app.enable('trust proxy');
-app.use('/heapdump', dumps.router);
+const PREFIX = process.env.prefix || `/assist`
 
 if (process.env.uws !== "true") {
-    var wsapp = express();
+    let wsapp = express();
     wsapp.use(request_logger("[wsapp]"));
-    wsapp.use('/assist', socket.wsRouter);
-
-    const wsserver = wsapp.listen(PORT + 1, HOST, () => {
-        console.log(`WS App listening on http://${HOST}:${PORT + 1}`);
+    wsapp.use(request_logger("[app]"));
+    wsapp.use(`/heapdump/${process.env.S3_KEY}`, dumps.router);
+    wsapp.use(`${PREFIX}/${process.env.S3_KEY}`, socket.wsRouter);
+    wsapp.enable('trust proxy');
+    const wsserver = wsapp.listen(PORT, HOST, () => {
+        console.log(`WS App listening on http://${HOST}:${PORT}`);
         console.log('Press Ctrl+C to quit.');
     });
-    wsapp.enable('trust proxy');
+
     socket.start(wsserver);
-    module.exports = {wsserver, server};
+    module.exports = {wsserver};
 } else {
     console.log("Using uWebSocket");
     const {App} = require("uWebSockets.js");
-    const PREFIX = process.env.prefix || '/assist'
+
 
     const uapp = new App();
 
@@ -77,6 +41,7 @@ if (process.env.uws !== "true") {
     }
     uapp.get(PREFIX, healthFn);
     uapp.get(`${PREFIX}/`, healthFn);
+    uapp.get(`${PREFIX}/${process.env.S3_KEY}`, healthFn);
 
 
     /* Either onAborted or simply finished request */
@@ -112,11 +77,11 @@ if (process.env.uws !== "true") {
 
     socket.start(uapp);
 
-    uapp.listen(HOST, PORT + 1, (token) => {
+    uapp.listen(HOST, PORT, (token) => {
         if (!token) {
             console.warn("port already in use");
         }
-        console.log(`WS App listening on http://${HOST}:${PORT + 1}`);
+        console.log(`WS App listening on http://${HOST}:${PORT}`);
         console.log('Press Ctrl+C to quit.');
     });
 
@@ -126,6 +91,5 @@ if (process.env.uws !== "true") {
         debug && console.log(err.stack);
         // process.exit(1);
     });
-    module.exports = {uapp, server};
-}
-console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`);
+    module.exports = {uapp};
+}

ee/utilities/servers/websocket-cluster.js

@@ -1,8 +1,8 @@
 const _io = require('socket.io');
 const express = require('express');
 const uaParser = require('ua-parser-js');
-const geoip2Reader = require('@maxmind/geoip2-node').Reader;
-const {extractPeerId} = require('./peerjs-server');
+const {extractPeerId} = require('../utils/helper');
+const {geoip} = require('../utils/geoIP');
 const {createAdapter} = require("@socket.io/redis-adapter");
 const {createClient} = require("redis");
 const wsRouter = express.Router();
@@ -159,7 +159,7 @@ const socketsLive = async function (req, res) {
     let liveSessions = {};
     let rooms = await getAvailableRooms();
     for (let peerId of rooms) {
-        let {projectKey, sessionId} = extractPeerId(peerId);
+        let {projectKey} = extractPeerId(peerId);
         if (projectKey !== undefined) {
             let connected_sockets = await io.in(peerId).fetchSockets();
             for (let item of connected_sockets) {
@@ -188,7 +188,7 @@ const socketsLiveByProject = async function (req, res) {
     let liveSessions = {};
     let rooms = await getAvailableRooms();
     for (let peerId of rooms) {
-        let {projectKey, sessionId} = extractPeerId(peerId);
+        let {projectKey} = extractPeerId(peerId);
         if (projectKey === _projectKey) {
             let connected_sockets = await io.in(peerId).fetchSockets();
             for (let item of connected_sockets) {
@@ -254,15 +254,6 @@ async function get_all_agents_ids(io, socket) {
     return agents;
 }
 
-let geoip = null;
-geoip2Reader.open(process.env.MAXMINDDB_FILE, {})
-    .then(reader => {
-        geoip = reader;
-    })
-    .catch(error => {
-        console.log("Error while opening the MAXMINDDB_FILE.")
-        console.error(error);
-    });
 
 function extractSessionInfo(socket) {
     if (socket.handshake.query.sessionInfo !== undefined) {
@@ -379,7 +370,7 @@ module.exports = {
                 let rooms = await io.of('/').adapter.allRooms();
                 let validRooms = [];
                 console.log(` ====== Rooms: ${rooms.size} ====== `);
-                const arr = Array.from(rooms)
+                // const arr = Array.from(rooms)
                 // const filtered = arr.filter(room => !room[1].has(room[0]))
                 for (let i of rooms) {
                     let {projectKey, sessionId} = extractPeerId(i);

ee/utilities/servers/websocket.js

@@ -1,8 +1,8 @@
 const _io = require('socket.io');
 const express = require('express');
 const uaParser = require('ua-parser-js');
-const geoip2Reader = require('@maxmind/geoip2-node').Reader;
-const {extractPeerId} = require('./peerjs-server');
+const {extractPeerId} = require('../utils/helper');
+const {geoip} = require('../utils/geoIP');
 const wsRouter = express.Router();
 const UPDATE_EVENT = "UPDATE_SESSION";
 const IDENTITIES = {agent: 'agent', session: 'session'};
@@ -108,7 +108,7 @@ const socketsList = async function (req, res) {
     }
     respond(res, liveSessions);
 }
-wsRouter.get(`/${process.env.S3_KEY}/sockets-list`, socketsList);
+wsRouter.get(`/sockets-list`, socketsList);
 
 const socketsListByProject = async function (req, res) {
     debug && console.log("[WS]looking for available sessions");
@@ -134,7 +134,7 @@ const socketsListByProject = async function (req, res) {
     }
     respond(res, liveSessions[_projectKey] || []);
 }
-wsRouter.get(`/${process.env.S3_KEY}/sockets-list/:projectKey`, socketsListByProject);
+wsRouter.get(`/sockets-list/:projectKey`, socketsListByProject);
 
 const socketsLive = async function (req, res) {
     debug && console.log("[WS]looking for all available LIVE sessions");
@@ -142,7 +142,7 @@ const socketsLive = async function (req, res) {
     let liveSessions = {};
     let rooms = await getAvailableRooms();
     for (let peerId of rooms) {
-        let {projectKey, sessionId} = extractPeerId(peerId);
+        let {projectKey} = extractPeerId(peerId);
         if (projectKey !== undefined) {
             let connected_sockets = await io.in(peerId).fetchSockets();
             for (let item of connected_sockets) {
@@ -161,7 +161,7 @@ const socketsLive = async function (req, res) {
     }
     respond(res, liveSessions);
 }
-wsRouter.get(`/${process.env.S3_KEY}/sockets-live`, socketsLive);
+wsRouter.get(`/sockets-live`, socketsLive);
 
 const socketsLiveByProject = async function (req, res) {
     debug && console.log("[WS]looking for available LIVE sessions");
@@ -170,7 +170,7 @@ const socketsLiveByProject = async function (req, res) {
     let liveSessions = {};
     let rooms = await getAvailableRooms();
     for (let peerId of rooms) {
-        let {projectKey, sessionId} = extractPeerId(peerId);
+        let {projectKey} = extractPeerId(peerId);
         if (projectKey === _projectKey) {
             let connected_sockets = await io.in(peerId).fetchSockets();
             for (let item of connected_sockets) {
@@ -189,7 +189,7 @@ const socketsLiveByProject = async function (req, res) {
     }
     respond(res, liveSessions[_projectKey] || []);
 }
-wsRouter.get(`/${process.env.S3_KEY}/sockets-live/:projectKey`, socketsLiveByProject);
+wsRouter.get(`/sockets-live/:projectKey`, socketsLiveByProject);
 
 const findSessionSocketId = async (io, peerId) => {
     const connected_sockets = await io.in(peerId).fetchSockets();
@@ -233,15 +233,6 @@ async function get_all_agents_ids(io, socket) {
     return agents;
 }
 
-let geoip = null;
-geoip2Reader.open(process.env.MAXMINDDB_FILE, {})
-    .then(reader => {
-        geoip = reader;
-    })
-    .catch(error => {
-        console.log("Error while opening the MAXMINDDB_FILE.")
-        console.error(error);
-    });
 
 function extractSessionInfo(socket) {
     if (socket.handshake.query.sessionInfo !== undefined) {

utilities/server.js

@@ -18,5 +18,4 @@ const wsserver = wsapp.listen(PORT, HOST, () => {
 });
 wsapp.enable('trust proxy');
 socket.start(wsserver);
-module.exports = {wsserver};
-console.log(`Heapdump enabled. Send a request to "/heapdump" to download a heapdump,\nor "/heapdump/save" to only generate a heapdump.`);
+module.exports = {wsserver};

utilities/utils/HeapSnapshot.js

@@ -63,4 +63,6 @@ function createNewHeapSnapshot(req, res) {
 router.get(`/status`, getHeapSnapshotStatus);
 router.get(`/new`, createNewHeapSnapshot);
 router.get(`/download`, downloadHeapSnapshot);
-module.exports = {router}
+module.exports = {router}
+
+console.log(`HeapSnapshot enabled. Send a request to "/heapdump/new" to generate a heapdump.`);