Merge commit

Mere branch 'master' of https://git.cse.iitb.ac.in/synerg/xanadu

dispatch_system/constants.json

@@ -3,5 +3,8 @@
     "master_port": 8080,
     "master_address": "localhost",
     "kafka_host": "10.129.6.5:9092",
-    "log_channel": "LOG_COMMON"
+    "grunt_host": "https://www.namandixit.net/lovecraftian_nightmares/grunt",
+    "log_channel": "LOG_COMMON",
+    "couchdb_host": "10.129.6.5:5984",
+    "couchdb_db_name": "serverless"
 }
\ No newline at end of file

dispatch_system/dispatch_daemon/execute.js

@@ -8,38 +8,47 @@ const { Worker, isMainThread, workerData } = require('worker_threads');
 const registry_url = constants.registry_url
 const logger = libSupport.logger
 
-function runIsolate(local_repository, functionHash, port, resource_id) {
+function runIsolate(local_repository, metadata) {
+    let port = metadata.port,
+        functionHash = metadata.functionHash,
+        resource_id = metadata.resource_id,
+        memory = metadata.resources.memory
     let filename = local_repository + functionHash + ".js"
     return new Promise((resolve, reject) => {
 
-        const worker = new Worker(filename);
+        const worker = new Worker(filename, {
+            argv: [resource_id, functionHash, port, "isolate"],
+            resourceLimits: {
+                maxOldGenerationSizeMb: memory
+            }
+        });
         worker.on('message', resolve);
         worker.on('error', reject);
         worker.on('exit', (code) => {
             if (code !== 0)
                 reject(new Error(`Worker stopped with exit code ${code}`));
             logger.info(`Isolate Worker with resource_id ${resource_id} blown`);
-        })
-        worker.on('online', () => {
             resolve()
         })
-        
     });
 
 }
 
-function runProcess(local_repository, functionHash, port, resource_id) {
+function runProcess(local_repository, metadata) {
+    let port = metadata.port,
+        functionHash = metadata.functionHash,
+        resource_id = metadata.resource_id,
+        memory = metadata.resources.memory
     let filename = local_repository + functionHash + ".js"
     return new Promise((resolve, reject) => {
         let timeStart = Date.now()
-        const process = spawn('node', [filename, port]);
-        let result = "";
+        
+        const process = spawn('node', [filename, resource_id, functionHash, port, "process", `--max-old-space-size=${memory}` ]);
         process.stdout.on('data', (data) => {
             console.log(`stdout: ${data}`);
-            result += data;
             let timeDifference = Math.ceil((Date.now() - timeStart))
             console.log("process time taken: ", timeDifference);
-            resolve(result);
+            
         });
 
         process.stderr.on('data', (data) => {
@@ -48,6 +57,7 @@ function runProcess(local_repository, functionHash, port, resource_id) {
         });
 
         process.on('close', (code) => {
+            resolve(code);
             logger.info(`Process Environment with resource_id ${resource_id} blown`);
         });
     })
@@ -55,7 +65,12 @@ function runProcess(local_repository, functionHash, port, resource_id) {
 }
 
 
-function runContainer(imageName, port, resource_id) {
+function runContainer(metadata) {
+    let imageName = metadata.functionHash,
+        port = metadata.port,
+        resource_id = metadata.resource_id,
+        memory = metadata.resources.memory
+
     logger.info(imageName);
 
     return new Promise((resolve, reject) => {
@@ -76,7 +91,7 @@ function runContainer(imageName, port, resource_id) {
                     if (code != 0)
                         reject("error")
                     else {
-                        const process = spawn('docker', ["run", "--rm", "-p", `${port}:5000`, "--name", resource_id, registry_url + imageName,
+                        const process = spawn('docker', ["run", "--rm", "-p", `${port}:${port}`, "--name", resource_id, registry_url + imageName,
                             resource_id, imageName, port, "container"]);
                         let result = "";
                         // timeStart = Date.now()
@@ -103,7 +118,7 @@ function runContainer(imageName, port, resource_id) {
             } else {
                     logger.info("container starting at port", port);
                     
-                    const process = spawn('docker', ["run", "--rm", "-p", `${port}:5000`, "--name", resource_id, 
+                    const process = spawn('docker', ["run", "--rm", "-p", `${port}:${port}`, "--name", resource_id, 
                             registry_url + imageName, resource_id, imageName, port, "container"]);
                     let result = "";
                     // timeStart = Date.now()

dispatch_system/dispatch_daemon/index.js

 'use strict';
 const constants = require(".././constants.json")
+const secrets = require('./secrets.json')
 const config = require('./config.json')
 const libSupport = require('./lib')
 libSupport.updateConfig()
@@ -7,6 +8,11 @@ const node_id = config.id
 const {spawn } = require('child_process')
 const execute = require('./execute')
 const fs = require('fs')
+const fetch = require('node-fetch');
+
+let metadataDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
+metadataDB = metadataDB + "/" + constants.couchdb_db_name + "/"
+
 const kafka = require('kafka-node')
 const logger = libSupport.logger
 
@@ -44,9 +50,24 @@ libSupport.makeTopic(node_id).then(() => {
          */
         if (message.type === "execute") {
             logger.info("Received Deployment request for resource_id: " + resource_id);
-            libSupport.download(host_url + "/repository/" + functionHash + ".js", local_repository + functionHash + ".js").then(() => {
-                startWorker(local_repository, functionHash, resource_id, producer, runtime, port)
+            fetch(metadataDB + functionHash).then(res => res.json())
+            .then(json => {
+                console.log("metadata", json);
+                
+                libSupport.download(host_url + "/repository/" + functionHash + ".js", local_repository + functionHash + ".js").then(() => {
+                    let metadata = {
+                        resource_id, functionHash,
+                        runtime, port,
+                        resources: {
+                            memory: json.memory
+                        }
+                    }
+                    startWorker(local_repository, producer, metadata)
             })
+            }).catch(err => {
+                logger.error("something went wrong" + err.toString())
+            });
+            
 
         }
 
@@ -56,7 +77,7 @@ libSupport.makeTopic(node_id).then(() => {
 /**
  * download and start grunt
  */
-libSupport.download(host_url + '/repository/grunt', "grunt").then(() => {
+libSupport.download(constants.grunt_host, "grunt").then(() => {
     logger.info("Downloaded grunt binary from repository")
     fs.chmod('grunt', 0o555, (err) => {
         logger.info("grunt made executable. Starting grunt")
@@ -88,26 +109,47 @@ libSupport.download(host_url + '/repository/grunt', "grunt").then(() => {
  * @param {String} runtime 
  * @param {Number} port 
  */
-function startWorker(local_repository, functionHash, resource_id, producer, runtime, port) {
-    logger.info(`Using port ${port} for functionHash ${functionHash}`);
+function startWorker(local_repository, producer, metadata) {
+    let runtime = metadata.runtime
+    console.log(metadata);
     
-    fs.writeFile('./local_repository/config.json', JSON.stringify({port, functionHash, resource_id, runtime}), () => {
-        if (runtime === "isolate")
-            execute.runIsolate(local_repository, functionHash, port, resource_id)
-        else if (runtime === "process")
-            execute.runProcess(local_repository, functionHash, port, resource_id)
-        else if (runtime === "container")
-            execute.runContainer(functionHash, port, resource_id)
-        else {
-            producer.send(
-                [{
-                    topic: "response",
-                    messages: JSON.stringify({ status: "unknown runtime" })
-                }], () => { })
-
-            return
-        }
-    });
+    logger.info(`Using port ${metadata.port} for functionHash ${metadata.functionHash}`)
+    
+    if (runtime === "isolate")
+        execute.runIsolate(local_repository, metadata)
+        .catch(err => {
+            logger.error("=====================deployment failed=========================");
+            producer.send([{
+                topic: "deployed",
+                messages: JSON.stringify({
+                    "status": false,
+                    resource_id: metadata.resource_id,
+                    "reason": "isolate exit"
+                })
+            }], () => { })
+        })
+    else if (runtime === "process")
+        execute.runProcess(local_repository, metadata)
+        .catch(err => {
+            logger.error("=====================deployment failed=========================");
+            producer.send([{ topic: "deployed",
+                messages: JSON.stringify({ 
+                "status": false, 
+                resource_id: metadata.resource_id,
+                "reason": "process exit"
+            }) }], () => { })
+        })
+    else if (runtime === "container")
+        execute.runContainer(metadata)
+    else {
+        producer.send(
+            [{
+                topic: "response",
+                messages: JSON.stringify({ status: "unknown runtime" })
+            }], () => { })
+
+        return
+    }
     
 }
 

dispatch_system/dispatch_daemon/package.json

@@ -13,7 +13,6 @@
     "body-parser": "^1.19.0",
     "express": "^4.17.1",
     "express-fileupload": "^1.1.6",
-    "isolated-vm": "^3.0.0",
     "kafka-node": "^5.0.0",
     "morgan": "^1.9.1",
     "mqtt": "^3.0.0",

dispatch_system/dispatch_manager/lib.js

 const crypto = require('crypto');
 const fs = require('fs')
 const rp = require('request-promise');
+const fetch = require('node-fetch');
 const winston = require('winston')
 const { createLogger, format, transports } = winston;
-
+const heap = require('heap')
 
 /**
  * Generates unique IDs of arbitrary length
@@ -20,13 +21,13 @@ function makeid(length) {
  }
 
 
- /**
-  * generates the runtime executor after inserting the received function
-  * TODO: make this asynchronous
-  * @param {string Path from where to extract the function} functionPath 
-  * @param {string Function Hash value} functionHash 
-  */
- function generateExecutor(functionPath, functionHash) {
+/**
+ * generates the runtime executor after inserting the received function
+ * TODO: make this asynchronous
+ * @param {string Path from where to extract the function} functionPath 
+ * @param {string Function Hash value} functionHash 
+ */
+function generateExecutor(functionPath, functionHash) {
    input = fs.readFileSync('./repository/worker_env/env.js')
    functionFile = fs.readFileSync(functionPath + functionHash)
    searchSize = "(resolve, reject) => {".length
@@ -42,36 +43,58 @@ function makeid(length) {
     return hash
  }
 
- function reverseProxy(req, res, url, tryout) {
-    return new Promise((resolve, reject) => {
-       logger.info("Request received at reverseproxy. Forwarding to: " + url);
+function reverseProxy(req, res, functionToResource, resourceMap) {
 
-       var options = {
-          method: 'POST',
-          uri: url,
-          body: req.body,
-          json: true // Automatically stringifies the body to JSON
-       };
+   return new Promise((resolve, reject) => {
+      let runtime = req.body.runtime
+      let id = req.params.id + runtime
+      /**
+       * Bypass deployment pipeline if resource available
+       */
+      let functionHeap = functionToResource.get(id)
+      let forwardTo = functionHeap[0]
+      let resource = resourceMap.get(forwardTo.resource_id)
+      logger.info(`Choosing resource ${JSON.stringify(forwardTo.resource_id)}` +
+         "\n forwarding via reverse proxy to: " + JSON.stringify(resource));
+      let url = `http://${resource.node_id}:${resource.port}/serverless/function/execute`
+   
+      logger.info("Request received at reverseproxy. Forwarding to: " + url);
+      forwardTo.metric += 1
+      heap.heapify(functionHeap, compare)
+      logger.info(functionHeap);
+      
+      var options = {
+         method: 'POST',
+         uri: url,
+         body: req.body,
+         json: true // Automatically stringifies the body to JSON
+      };
 
-       rp(options)
-          .then(function (parsedBody) {
-            //  console.log("parsed body:", parsedBody);
-             res.json(parsedBody)
-             resolve()
-          })
-          .catch(function (err) {
-             if (err.error.errno === "ECONNREFUSED") {
-                reverseProxy(req, res, url, (tryout != null) ? tryout + 1 : 1)
-             } else {
-                logger.error("error", err.error.errno);
-                res.json(err.message).status(err.statusCode)
-                resolve()
-             }
-          });
-    })
- }
+      // console.log(options);
+      
+      
+      rp(options)
+         .then(function (parsedBody) {
+            
+            res.json(parsedBody)
+            forwardTo.metric -= 1
+            heap.heapify(functionHeap, compare)
+            console.log(functionHeap);
+            resolve()
+         })
+         .catch(function (err) {
+            forwardTo.metric -= 1
+            heap.heapify(functionHeap, compare)
+            console.log(functionHeap);
+            logger.error("error" + err.error.errno);
+            res.json(err.message).status(err.statusCode)
+            resolve()
+         });
+      
+   })
+}
 
- function getPort(usedPort) {
+function getPort(usedPort) {
    let port = -1, ctr = 0
    do {
        min = Math.ceil(30000);
@@ -112,6 +135,10 @@ const logger = winston.createLogger({
    ]
 });
 
+function compare(a, b) {
+   return a.metric - b.metric
+}
+
  module.exports = {
-    makeid, generateExecutor, reverseProxy, getPort, logger
+    makeid, generateExecutor, reverseProxy, getPort, logger, compare
  }
\ No newline at end of file

dispatch_system/dispatch_manager/package.json

@@ -13,11 +13,12 @@
     "body-parser": "^1.19.0",
     "express": "^4.17.1",
     "express-fileupload": "^1.1.6",
+    "heap": "^0.2.6",
     "isolated-vm": "^3.0.0",
     "kafka-node": "^5.0.0",
     "morgan": "^1.9.1",
     "mqtt": "^3.0.0",
-    "nano": "^8.1.0",
+    "node-fetch": "^2.6.0",
     "redis": "^2.8.0",
     "request": "^2.88.0",
     "request-promise": "^4.2.5",

dispatch_system/dispatch_manager/repository/worker_env/env.js

@@ -2,21 +2,12 @@
 const express = require('express')
 const bodyParser = require('body-parser')
 const app = express()
-let port = 5000, resource_id, functionHash, portExternal, runtime
-let config = null;
-try {
-    config = require('./config.json')
-    port = config.port
-    resource_id = config.resource_id
-    functionHash = config.functionHash
-    runtime = config.runtime
-} catch (e) {
-    port = 5000
-    resource_id = process.argv[2]
-    functionHash = process.argv[3]
-    portExternal = process.argv[4]
-    runtime = process.argv[5]
-}
+let port = 5000, resource_id, functionHash, runtime
+
+resource_id = process.argv[2]
+functionHash = process.argv[3]
+port = process.argv[4]
+runtime = process.argv[5]
 
 let kafka = require('kafka-node'),
     Producer = kafka.Producer,
@@ -47,18 +38,19 @@ app.listen(port, () => {
     producer.send(
     [{
         topic: "deployed",
-        messages: JSON.stringify({ functionHash, portExternal, runtime, resource_id })
+        messages: JSON.stringify({ functionHash, portExternal: port, runtime, resource_id }),
+        "status": true
     }], () => { })
 })
 
 function shouldDie() {
     
-    if (Date.now() - lastRequest > 5 * 1000) {
+    if (Date.now() - lastRequest > 30 * 1000) {
         console.log("Idle for too long. Exiting");
         producer.send(
             [{
                 topic: "removeWorker",
-                messages: JSON.stringify({ functionHash, portExternal, runtime, resource_id })
+                messages: JSON.stringify({ functionHash, portExternal: port, runtime, resource_id })
             }], () => {
                 console.log("Ending worker for function", functionHash, "resource_id", resource_id);
                 process.exit(0)

readme.md

@@ -23,7 +23,7 @@ The Dispatch Manager (DM) sends a request to the Resource Manager (RM), detailin
 ```javascript
 {
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "memory": 1024, // in MiB
     ... // Any other resources
 }
@@ -34,7 +34,7 @@ Format:
 ```javascript
 {
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "grunts": [  
            { node_id: some unique ID, port: port address}, ...
      ] // List of machine IDs
@@ -44,10 +44,13 @@ Format:
 Once the runtime entity has been launched (or the launch has failed), the Executor sends back a status message on the `LOG_COMMON` topic.
 ```javascript
 {
-    "node_id" : "uique-machine-id",
+    "message_type" : "deployment_launch",
+    "node_id" : "unique-machine-id",
+    "entity_id" : "handle for the actual container/VM/etc.",
+    "entity_type" : "docker/libvirt/etc.",
     "resource_id": "logical-entity-id",
     "function_id": "unique-function-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "reason": "deployment"/"termination"
     "status": true/false // Only valid if reason==deployment
 }
@@ -57,28 +60,31 @@ Instrumentation data is also sent on the `LOG_COMMON` topic. This data is sent f
 and whoever needs the data is allowed to read it. Each message is required to have atleast three fields: `node_id`, `resource_id` and `function_id`.
  ```javascript
 { // Example message from Executor
-    "node_id" : "uique-machine-id",
+    "message_type" : "instrumentation",
+    "node_id" : "unique-machine-id",
     "resource_id": "logical-entity-id",
     "function_id": "unique-function-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "cpu" : 343, // in MHz
     "memory": 534, // in MiB
     "network": 234 // in KBps
 }
 
 { // Example message from reverse proxy
-    "node_id" : "uique-machine-id",
+    "message_type" : "instrumentation",
+    "node_id" : "unique-machine-id",
     "resource_id": "logical-entity-id",
     "function_id": "unique-function-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "average_fn_time" : 23 // in ms
 }
 
 { // Example message from dispatch manager
-    "node_id" : "uique-machine-id",
+    "message_type" : "instrumentation",
+    "node_id" : "unique-machine-id",
     "resource_id": "logical-entity-id",
     "function_id": "unique-function-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "coldstart_time" 
 }
 ```
@@ -223,7 +229,7 @@ resources being tracked by RDs on each machine. This data is cached by the RM.
 ```javascript
 {
     "node_id": "unique-machine-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "memory": 1024, // in MiB
     ... // Any other resources
 }
@@ -246,7 +252,7 @@ DM on topic `RESPONSE_RM_2_DM`.
 ```javascript
 {
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
 //    "port": 2343 --- NOT IMPLEMENTED YET
     "nodes": ["a", "b", ...] // List of unique machine IDs
 }
@@ -258,7 +264,7 @@ Format:
 ```javascript
 {
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "memory": 1024, // in MiB
     ... // Any other resources
 }
@@ -269,7 +275,7 @@ The RDs recieve this message and send back whether on not they satisfy the const
 {
     "node_id": "unique-machine-id",
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "success" : 0/1 // 0 = fail, 1 = success
 }
 ```