Merge branch 'explicit_function_chaining'

dispatch_system/constants.json

 {
-    "registry_url" :"localhost:5000/",
+    "registry_url" :"10.129.6.5:5000/",
     "master_port": 8080,
     "master_address": "localhost",
     "grunt_host": "https://www.namandixit.net/lovecraftian_nightmares/grunt",
-    "couchdb_host": "localhost:5984",
-    "function_db_name": "serverless",
-    "metrics_db_name": "metrics",
-    "implicit_chain_db_name": "implicit_chain",
+    "couchdb_host": "10.129.6.5:5984",
+    "db": {
+        "function_meta": "serverless",
+        "metrics": "metrics",
+        "implicit_chain_meta": "implicit_chain",
+        "explicit_chain_meta": "explicit_chain"
+    },
     "network": {
         "network_bridge": "hybrid_kafka-serverless",
+        "use_bridge": false,
         "internal": {
-            "kafka_host": "kafka:9092"
+            "kafka_host": "10.129.6.5:9092"
         },
         "external": {
-            "kafka_host": "localhost:29092"
+            "kafka_host": "10.129.6.5:9092"
         }
     },
     "topics": {
@@ -28,7 +32,10 @@
     "autoscalar_metrics": {
         "open_request_threshold": 100
     },
-    "speculative_deployment": false,
-    "JIT_deployment": false,
+    "metrics": {
+        "alpha": 0.7
+    },
+    "speculative_deployment": true,
+    "JIT_deployment": true,
     "id_size": 20
 }

dispatch_system/constants_server.json

+{
+    "registry_url" :"10.129.6.5:5000/",
+    "master_port": 8080,
+    "master_address": "10.129.6.5",
+    "grunt_host": "https://www.namandixit.net/lovecraftian_nightmares/grunt",
+    "couchdb_host": "10.129.6.5:5984",
+    "db": {
+        "function_meta": "serverless",
+        "metrics": "metrics",
+        "implicit_chain_meta": "implicit_chain",
+        "explicit_chain_meta": "explicit_chain"
+    },
+    "network": {
+        "network_bridge": "hybrid_kafka-serverless",
+        "internal": {
+            "kafka_host": "kafka:9092"
+        },
+        "external": {
+            "kafka_host": "10.129.6.5:9092"
+        }
+    },
+    "topics": {
+        "request_dm_2_rm": "request",
+        "heartbeat": "heartbeat",
+        "deployed": "deployed",
+        "remove_worker": "removeWorker",
+        "response_rm_2_dm": "RESPONSE_RM_2_DM_DUMMY",
+        "hscale": "hscale",
+        "log_channel": "LOG_COMMON"
+    },
+    "autoscalar_metrics": {
+        "open_request_threshold": 100
+    },
+    "metrics": {
+        "alpha": 0.7
+    },
+    "speculative_deployment": false,
+    "JIT_deployment": true,
+    "id_size": 20
+}

dispatch_system/dispatch_daemon/execute.js

@@ -92,9 +92,15 @@ function runContainer(metadata) {
                     if (code != 0)
                         reject("error")
                     else {
-                        const process = spawn('docker', ["run", "--rm", `--network=${constants.network.network_bridge}`, "-p", `${port}:${port}`, 
-                            "--name", resource_id, registry_url + imageName,
-                            resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
+                        let process = null;
+                        if (constants.network.use_bridge)
+                            process = spawn('docker', ["run", "--rm", `--network=${constants.network.network_bridge}`, "-p", `${port}:${port}`, 
+                                "--name", resource_id, registry_url + imageName,
+                                resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
+                        else
+                            process = spawn('docker', ["run", "--rm", "-p", `${port}:${port}`,
+                                "--name", resource_id, registry_url + imageName,
+                                resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
                         let result = "";
                         // timeStart = Date.now()
                         process.stdout.on('data', (data) => {
@@ -119,10 +125,15 @@ function runContainer(metadata) {
                 })
             } else {
                     logger.info("container starting at port", port);
-                    
-                    const process = spawn('docker', ["run", "--rm", `--network=${constants.network.network_bridge}`, 
-                            "-p", `${port}:${port}`, "--name", resource_id, 
-                        registry_url + imageName, resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
+                    let process = null;
+                    if (constants.network.use_bridge)
+                        process = spawn('docker', ["run", "--rm", `--network=${constants.network.network_bridge}`, 
+                                "-p", `${port}:${port}`, "--name", resource_id, 
+                            registry_url + imageName, resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
+                    else
+                        process = spawn('docker', ["run", "--rm",
+                            "-p", `${port}:${port}`, "--name", resource_id,
+                            registry_url + imageName, resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
                     let result = "";
                     // timeStart = Date.now()
                     process.stdout.on('data', (data) => {

dispatch_system/dispatch_daemon/index.js

@@ -158,7 +158,7 @@ function heartbeat() {
         topic: "heartbeat",
         messages: JSON.stringify({"address": node_id, "timestamp": Date.now()})
     }]
-    producer.send(payload, function() {})
+    producer.send(payload, function(cb) {})
 }
 
 

dispatch_system/dispatch_daemon/lib.js

@@ -27,7 +27,7 @@ function updateConfig() {
 }
 
 function makeTopic(id) {
-    console.log("Using Primary IP", id, "as topic");
+    console.log("Using Primary IP", id, "as topic", "publishing to:", constants.network.external.kafka_host);
     
     let client = new kafka.KafkaClient({
         kafkaHost: constants.network.external.kafka_host,

dispatch_system/dispatch_manager/index.js

@@ -10,18 +10,11 @@ const { spawn } = require('child_process');
 const morgan = require('morgan');
 const heap = require('heap');
 const fetch = require('node-fetch');
-const swStats = require('swagger-stats');
-const apiSpec = require('./swagger.json');
+// const swStats = require('swagger-stats');
+// const apiSpec = require('./swagger.json');
 const util = require('util')
+const sharedMeta = require('./shared_meta')
 
-/**
- * URL to the couchdb database server used to store function metadata
- */
-let metadataDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
-metadataDB = metadataDB + "/" + constants.function_db_name + "/"
-
-let metricsDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
-metricsDB = metricsDB + "/" + constants.metrics_db_name + "/"
 
 const app = express()
 const libSupport = require('./lib')
@@ -30,13 +23,15 @@ let date = new Date();
 let log_channel = constants.topics.log_channel
 
 let usedPort = new Map(), // TODO: remove after integration with RM
-    db = new Map(), // queue holding request to be dispatched
-    resourceMap = new Map(), // map between resource_id and resource details like node_id, port, associated function etc
-    functionToResource = new Map(), // a function to resource map. Each map contains a minheap of
+    db = sharedMeta.db, // queue holding request to be dispatched
+    resourceMap = sharedMeta.resourceMap, // map between resource_id and resource details like node_id, port, associated function etc
+    functionToResource = sharedMeta.functionToResource, // a function to resource map. Each map contains a minheap of
                                    // resources associated with the function
-    workerNodes = new Map(), // list of worker nodes currently known to the DM
-    functionBranchTree = new Map() // a tree to store function branch predictions
-    
+    workerNodes = sharedMeta.workerNodes, // list of worker nodes currently known to the DM
+    functionBranchTree = sharedMeta.functionBranchTree, // a tree to store function branch predictions
+
+    metricsDB = sharedMeta.metricsDB,
+    metadataDB = sharedMeta.metadataDB
 
 let kafka = require('kafka-node'),
     Producer = kafka.Producer,
@@ -64,11 +59,10 @@ app.use(morgan('combined', {
 app.use(express.json());
 app.use(express.urlencoded({ extended: true }));
 const file_path = __dirname + "/repository/"
-
 app.use('/repository', express.static(file_path)); // file server hosting deployed functions
 app.use(fileUpload())
-app.use(swStats.getMiddleware({ swaggerSpec: apiSpec })); // statistics middleware
-app.use('/serverless/chain', chainHandler); // chain router (explicit_chain_handler.js) for handling explicit chains
+// app.use(swStats.getMiddleware({ swaggerSpec: apiSpec })); // statistics middleware
+app.use('/serverless/chain', chainHandler.router); // chain router (explicit_chain_handler.js) for handling explicit chains
 let requestQueue = []
 
 const WINDOW_SIZE = 10
@@ -225,7 +219,7 @@ app.post('/serverless/execute/:id', (req, res) => {
     res.timestamp = Date.now() 
     if (functionToResource.has(id)) {
         res.start = 'warmstart'
-        libSupport.reverseProxy(req, res, functionToResource, resourceMap, functionBranchTree)
+        libSupport.reverseProxy(req, res)
     } else {
         res.start = 'coldstart'
         /**
@@ -334,17 +328,14 @@ function postDeploy(message) {
             "reason": "deployment",
             "status": true,
             starttime: (Date.now() - resource.deploy_request_time)
-        }, message.resource_id, resourceMap)
+        }, message.resource_id)
         
         if (db.has(id)) {
             let sendQueue = db.get(id)
             logger.info("forwarding request via reverse proxy to: " + JSON.stringify(resource));
             while (sendQueue && sendQueue.length != 0) {
                 let { req, res } = sendQueue.shift()
-                libSupport.reverseProxy(req, res, functionToResource, resourceMap, functionBranchTree)
-                    .then(() => {
-
-                    })
+                libSupport.reverseProxy(req, res)
             }
             db.delete(id)
         }
@@ -410,7 +401,7 @@ consumer.on('message', function (message) {
                 "reason": "terminate",
                 "total_request": message.total_request,
                 "status": true
-            }, message.resource_id, resourceMap)
+            }, message.resource_id)
             .then(() => {
                 resourceMap.delete(message.resource_id)
                 if (resourceArray.length == 0)
@@ -428,7 +419,11 @@ consumer.on('message', function (message) {
             functionHash = message.functionHash
         logger.info(`Generated new resource ID: ${resource_id} for runtime: ${runtime}`);
         console.log("Resource Status: ", functionToResource);
-        
+        if (!functionToResource.has(functionHash + runtime) && !db.has(functionHash + runtime)) {
+            console.log("adding db");
+            
+            db.set(functionHash + runtime, [])
+        }
         /**
          * Request RM for resource
          */
@@ -451,6 +446,7 @@ consumer.on('message', function (message) {
             }),
             partition: 0
         }]
+        
         producer.send(payloadToRM, () => {
             // db.set(functionHash + runtime, { req, res })
             console.log("sent rm");
@@ -521,7 +517,7 @@ function autoscalar() {
  */
 async function speculative_deployment(req, runtime) {
     if (constants.speculative_deployment && req.headers['x-resource-id'] === undefined) {
-        console.log(functionBranchTree, req.params.id);
+        // console.log(functionBranchTree, req.params.id);
 
         if (functionBranchTree.has(req.params.id)) {
             let branchInfo = functionBranchTree.get(req.params.id)
@@ -558,7 +554,7 @@ async function speculative_deployment(req, runtime) {
     }
 }
 setInterval(libSupport.metrics.broadcastMetrics, 5000)
-setInterval(libSupport.viterbi, 1000, functionBranchTree)
+// setInterval(libSupport.viterbi, 1000)
 setInterval(autoscalar, 1000);
 setInterval(dispatch, 1000);
 app.listen(port, () => logger.info(`Server listening on port ${port}!`))
\ No newline at end of file

dispatch_system/dispatch_manager/metrics.js

@@ -5,11 +5,12 @@ const secrets = require('./secrets.json')
 const fetch = require('node-fetch');
 const util = require('util')
 const prom = require('prom-client');
+const sharedMeta = require('./shared_meta');
 
 const Registry = prom.Registry;
 const register = new Registry();
 
-const alpha = 0.99
+const alpha = constants.metrics.alpha
 let log_channel = constants.topics.log_channel,
     metrics = {  }
 
@@ -29,8 +30,7 @@ register.registerMetric(coldstartMetric);
 register.registerMetric(starttimeMetric);
 register.registerMetric(requestMetric);
 
-let metricsDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
-metricsDB = metricsDB + "/" + constants.metrics_db_name + "/"
+let metricsDB = sharedMeta.metricsDB
 let kafka = require('kafka-node'),
     Producer = kafka.Producer,
     client = new kafka.KafkaClient({
@@ -129,6 +129,7 @@ async function broadcastMetrics() {
                         warmstart: metric.longterm.warmstart,
                         starttime: metric.longterm.starttime
                     }
+                    
                     let payload = {
                         method: 'put',
                         body: JSON.stringify(dbData),
@@ -136,6 +137,7 @@ async function broadcastMetrics() {
                     }
                     await fetch(metricsDB + functionHash, payload)
                     
+                    
                     metric.timestamp = Date.now()
                 }
             }

dispatch_system/dispatch_manager/shared_meta.js

+const secrets = require('./secrets.json')
+const constants = require('.././constants.json')
+
+let db = new Map(), // queue holding request to be dispatched
+    resourceMap = new Map(), // map between resource_id and resource details like node_id, port, associated function etc
+    functionToResource = new Map(), // a function to resource map. Each map contains a minheap of
+                                    // resources associated with the function
+    workerNodes = new Map(), // list of worker nodes currently known to the DM
+    functionBranchTree = new Map(), // a tree to store function branch predictions
+    conditionProbabilityExplicit = new Map() // tree holding conditional probabilities for explicit chains
+
+/**
+ * URL to the couchdb database server used to store data
+ */
+
+let metadataDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
+metadataDB = metadataDB + "/" + constants.db.function_meta + "/"
+
+let metricsDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
+metricsDB = metricsDB + "/" + constants.db.metrics + "/"
+
+let implicitChainDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
+implicitChainDB = implicitChainDB + "/" + constants.db.implicit_chain_meta + "/"
+
+let explicitChainDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
+explicitChainDB = explicitChainDB + "/" + constants.db.explicit_chain_meta + "/"
+
+module.exports = {
+    db, functionBranchTree, functionToResource, workerNodes, resourceMap,
+    conditionProbabilityExplicit,
+    metadataDB, metricsDB, implicitChainDB, explicitChainDB
+}
\ No newline at end of file