Merge on pull

.gitignore

@@ -6,3 +6,4 @@ firecracker*
 secrets.json
 resource_system/bin/**
 resource_system/version.linux
+local_experiments/

dispatch_system/constants.json

 {
-    "registry_url" :"10.129.6.5:5000/",
+    "registry_url" :"localhost:5000/",
     "master_port": 8080,
     "master_address": "localhost",
-    "kafka_host": "10.129.6.5:9092",
     "grunt_host": "https://www.namandixit.net/lovecraftian_nightmares/grunt",
-    "log_channel": "LOG_COMMON",
-    "couchdb_host": "10.129.6.5:5984",
-    "couchdb_db_name": "serverless",
+    "couchdb_host": "localhost:5984",
+    "function_db_name": "serverless",
+    "metrics_db_name": "metrics",
+    "implicit_chain_db_name": "implicit_chain",
+    "network": {
+        "network_bridge": "hybrid_kafka-serverless",
+        "internal": {
+            "kafka_host": "kafka:9092"
+        },
+        "external": {
+            "kafka_host": "localhost:29092"
+        }
+    },
     "topics": {
         "request_dm_2_rm": "request",
         "heartbeat": "heartbeat",
         "deployed": "deployed",
         "remove_worker": "removeWorker",
         "response_rm_2_dm": "RESPONSE_RM_2_DM_DUMMY",
-        "hscale": "hscale"
+        "hscale": "hscale",
+        "log_channel": "LOG_COMMON"
     },
     "autoscalar_metrics": {
         "open_request_threshold": 100
     },
-    "speculative_deployment": true
+    "speculative_deployment": false,
+    "JIT_deployment": false,
+    "id_size": 20
 }

dispatch_system/dispatch_daemon/config.json

-{"id":"192.168.31.51","master_node":"10.129.6.5"}
+{"id":"192.168.0.105","master_node":"192.168.0.105"}
\ No newline at end of file

dispatch_system/dispatch_daemon/execute.js

@@ -17,7 +17,7 @@ function runIsolate(local_repository, metadata) {
     return new Promise((resolve, reject) => {
 
         const worker = new Worker(filename, {
-            argv: [resource_id, functionHash, port, "isolate"],
+            argv: [resource_id, functionHash, port, "isolate", constants.network.external.kafka_host],
             resourceLimits: {
                 maxOldGenerationSizeMb: memory
             }
@@ -43,7 +43,8 @@ function runProcess(local_repository, metadata) {
     return new Promise((resolve, reject) => {
         let timeStart = Date.now()
         
-        const process = spawn('node', [filename, resource_id, functionHash, port, "process", `--max-old-space-size=${memory}` ]);
+        const process = spawn('node', [filename, resource_id, functionHash, port, "process", 
+                constants.network.external.kafka_host, `--max-old-space-size=${memory}` ]);
         process.stdout.on('data', (data) => {
             console.log(`stdout: ${data}`);
             let timeDifference = Math.ceil((Date.now() - timeStart))
@@ -91,8 +92,9 @@ function runContainer(metadata) {
                     if (code != 0)
                         reject("error")
                     else {
-                        const process = spawn('docker', ["run", "--rm", "-p", `${port}:${port}`, "--name", resource_id, registry_url + imageName,
-                            resource_id, imageName, port, "container"]);
+                        const process = spawn('docker', ["run", "--rm", `--network=${constants.network.network_bridge}`, "-p", `${port}:${port}`, 
+                            "--name", resource_id, registry_url + imageName,
+                            resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
                         let result = "";
                         // timeStart = Date.now()
                         process.stdout.on('data', (data) => {
@@ -118,8 +120,9 @@ function runContainer(metadata) {
             } else {
                     logger.info("container starting at port", port);
                     
-                    const process = spawn('docker', ["run", "--rm", "-p", `${port}:${port}`, "--name", resource_id, 
-                            registry_url + imageName, resource_id, imageName, port, "container"]);
+                    const process = spawn('docker', ["run", "--rm", `--network=${constants.network.network_bridge}`, 
+                            "-p", `${port}:${port}`, "--name", resource_id, 
+                        registry_url + imageName, resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
                     let result = "";
                     // timeStart = Date.now()
                     process.stdout.on('data', (data) => {

dispatch_system/dispatch_daemon/index.js

@@ -11,7 +11,7 @@ const fs = require('fs')
 const fetch = require('node-fetch');
 
 let metadataDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
-metadataDB = metadataDB + "/" + constants.couchdb_db_name + "/"
+metadataDB = metadataDB + "/" + constants.function_db_name + "/"
 
 const kafka = require('kafka-node')
 const logger = libSupport.logger
@@ -21,7 +21,7 @@ const host_url = "http://" + constants.master_address + ":" + constants.master_p
 
 let Producer = kafka.Producer,
     client = new kafka.KafkaClient({ 
-        kafkaHost: constants.kafka_host,
+        kafkaHost: constants.network.external.kafka_host,
         autoConnect: true
     }),
     producer = new Producer(client),
@@ -77,9 +77,9 @@ libSupport.makeTopic(node_id).then(() => {
 /**
  * download and start grunt
  */
-libSupport.download(constants.grunt_host, "grunt").then(() => {
+libSupport.download(constants.grunt_host, "grunt", false).then(() => {
     logger.info("Downloaded grunt binary from repository")
-    fs.chmod('grunt', 0o555, (err) => {
+    fs.chmod('grunt', 0o755, (err) => {
         logger.info("grunt made executable. Starting grunt")
         let grunt = spawn('./grunt', [node_id])
         grunt.stdout.on('data', data => {

dispatch_system/dispatch_daemon/lib.js

-const http = require('http');
+const fetch = require('node-fetch');
 const fs = require('fs');
 const process = require('process')
 const { spawnSync } = require('child_process');
@@ -30,7 +30,7 @@ function makeTopic(id) {
     console.log("Using Primary IP", id, "as topic");
     
     let client = new kafka.KafkaClient({
-        kafkaHost: constants.kafka_host,
+        kafkaHost: constants.network.external.kafka_host,
         autoConnect: true
     }),
     Producer = kafka.Producer,
@@ -50,28 +50,48 @@ function makeTopic(id) {
     })
 }
 
-var download = function (url, dest, cb) {
-    return new Promise((resolve, reject) => {
+// var download = function (url, dest, check = true, cb) {
+//     return new Promise((resolve, reject) => {
+//         console.log(url);
+//         if (!check || !fs.existsSync(dest)) {
+//             var file = fs.createWriteStream(dest);
+//             var request = https.get(url, function (response) {
+//                 response.pipe(file);
+//                 file.on('finish', function () {
+//                     file.close(cb);  // close() is async, call cb after close completes.
+//                     resolve();
+//                 });
+//             }).on('error', function (err) { // Handle errors
+//                 fs.unlink(dest); // Delete the file async. (But we don't check the result)
+//                 logger.error("download failed" + err.message);
+                
+//                 if (cb) cb(err.message);
+//                 reject(err);
+//             });
+//         } else {
+//             resolve();
+//         }
+//     })
+    
+// };
+
+const download = (async (url, path, check = true) => {
+    
+    if (!check || !fs.existsSync(path)) {
         console.log(url);
-        if (!fs.existsSync(dest)) {
-            var file = fs.createWriteStream(dest);
-            var request = https.get(url, function (response) {
-                response.pipe(file);
-                file.on('finish', function () {
-                    file.close(cb);  // close() is async, call cb after close completes.
-                    resolve();
-                });
-            }).on('error', function (err) { // Handle errors
-                fs.unlink(dest); // Delete the file async. (But we don't check the result)
-                if (cb) cb(err.message);
+        const res = await fetch(url);
+        const fileStream = fs.createWriteStream(path);
+        await new Promise((resolve, reject) => {
+            res.body.pipe(fileStream);
+            res.body.on("error", (err) => {
                 reject(err);
             });
-        } else {
+            fileStream.on("finish", function () {
                 resolve();
+            });
+        });
     }
-    })
-    
-};
+});
 
 function makeid(length) {
     var result = '';

dispatch_system/dispatch_daemon/package.json

@@ -16,6 +16,7 @@
     "kafka-node": "^5.0.0",
     "morgan": "^1.9.1",
     "mqtt": "^3.0.0",
+    "node-fetch": "^2.6.0",
     "redis": "^2.8.0",
     "request": "^2.88.2",
     "winston": "^3.2.1"

dispatch_system/dispatch_manager/lib.js

@@ -4,10 +4,23 @@ const rp = require('request-promise');
 const fetch = require('node-fetch');
 const winston = require('winston')
 const constants = require('.././constants.json')
-
+const secrets = require('./secrets.json')
+const metrics = require('./metrics')
 const { createLogger, format, transports } = winston;
 const heap = require('heap')
 
+
+let kafka = require('kafka-node'),
+   Producer = kafka.Producer,
+   client = new kafka.KafkaClient({
+      kafkaHost: constants.network.external.kafka_host,
+      autoConnect: true
+   }),
+   producer = new Producer(client)
+
+let implicitChainDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
+implicitChainDB = implicitChainDB + "/" + constants.implicit_chain_db_names + "/"
+
 /**
  * Generates unique IDs of arbitrary length
  * @param {Length of the ID} length 
@@ -45,6 +58,14 @@ function generateExecutor(functionPath, functionHash) {
     return hash
  }
 
+ /**
+  * Reverse proxy to take user requests and forward them to appropriate workers using a loadbalacer
+  * @param {JSON} req the user request to be forwarded to the worker
+  * @param {JSON} res Object to use to return the response to the user
+  * @param {Map} functionToResource Function to resource Map
+  * @param {Map} resourceMap Map from resource ID to resource metadata
+  * @param {Map} functionBranchTree Holds the function path's and related probability distribution
+  */
 function reverseProxy(req, res, functionToResource, resourceMap, functionBranchTree) {
    branchChainPredictor(req, resourceMap, functionToResource, functionBranchTree)
    return new Promise((resolve, reject) => {
@@ -54,6 +75,7 @@ function reverseProxy(req, res, functionToResource, resourceMap, functionBranchT
        * Bypass deployment pipeline if resource available
        */
       let functionHeap = functionToResource.get(id)
+      // loadbalancing by choosing worker with lowest load
       let forwardTo = functionHeap[0]
       let resource = resourceMap.get(forwardTo.resource_id)
       // logger.info(`Choosing resource ${JSON.stringify(forwardTo.resource_id)}` +
@@ -62,7 +84,7 @@ function reverseProxy(req, res, functionToResource, resourceMap, functionBranchT
    
       // logger.info("Request received at reverseproxy. Forwarding to: " + url);
       forwardTo.open_request_count += 1
-      heap.heapify(functionHeap, compare)
+      heap.heapify(functionHeap, compare) // maintain loadbalancer by heapifying the Map
       // logger.info(functionHeap);
       
       var options = {
@@ -72,21 +94,22 @@ function reverseProxy(req, res, functionToResource, resourceMap, functionBranchT
          json: true // Automatically stringifies the body to JSON
       };
       
-      // console.log(options);
-      
       
       rp(options)
          .then(function (parsedBody) {
+            let serviceTime = Date.now() - res.timestamp
             
             res.json(parsedBody)
             forwardTo.open_request_count -= 1
             heap.heapify(functionHeap, compare)
+            
+            metrics.collectMetrics({type: res.start, value: serviceTime, functionHash: req.params.id, runtime})
             resolve()
          })
          .catch(function (err) {
             forwardTo.open_request_count -= 1
             heap.heapify(functionHeap, compare)
-            logger.error("error" + err.error.errno);
+            logger.error("error" + err);
             res.json(err.message).status(err.statusCode)
             resolve()
          });
@@ -109,7 +132,6 @@ function getPort(usedPort) {
    return port
 }
 
-
 const logger = winston.createLogger({
    level: 'info',
    format: winston.format.combine(
@@ -252,17 +274,61 @@ function viterbi(functionBranchTree) {
                path.push({node: maxSibling, probability: maxProb})
             siblings = new Map()
          }
-         if (path.length > 0)
-            console.log("path", path);
+         // if (path.length > 0)
+         //    console.log("path", path);
          metadata.mle_path = path
+         if (path.length > 1) {
+            let payload = {
+               method: 'put',
+               body: JSON.stringify(path),
+               headers: { 'Content-Type': 'application/json' }
+            }
+            fetch(implicitChainDB + functionHash, payload)
+         }
       }
    });
+}
+
+function logBroadcast(message, resource_id, resourceMap) {
+   return new Promise((resolve, reject) => {
+      try {
 
+         message.timestamp = Date.now()
+         if (resource_id && resourceMap.has(resource_id)) {
+            let resource = resourceMap.get(resource_id)
+            message.resource_id = resource_id
+            message.node_id = resource.node_id
+            message.runtime = resource.runtime
+            message.function_id = resource.functionHash
+         }
+         let log = [{
+            topic: constants.topics.log_channel,
+            messages: JSON.stringify(message),
+            partition: 0
+         }]
+         producer.send(log, () => {
+            resolve()
+         })
+      } catch (err) {
+         console.log(err);
+         reject()
+      }
+   })
+   
+}
 
+async function fetchData(url, data = null) {
+   let res
+   if (data === undefined || data === null)
+      res = await fetch(url)
+   else
+      res = await fetch(url, data)
+   return await res.json()
 }
 
  module.exports = {
     makeid, generateExecutor, reverseProxy, 
     getPort, logger, compare,
-    viterbi
+    viterbi, logBroadcast, fetchData, metrics,
+    producer
  }
\ No newline at end of file

dispatch_system/dispatch_manager/metrics.js

+'use strict';
+
+const constants = require('.././constants.json');
+const secrets = require('./secrets.json')
+const fetch = require('node-fetch');
+const util = require('util')
+const prom = require('prom-client');
+
+const Registry = prom.Registry;
+const register = new Registry();
+
+const alpha = 0.99
+let log_channel = constants.topics.log_channel,
+    metrics = {  }
+
+
+const intervalCollector = prom.collectDefaultMetrics({ prefix: 'xanadu', timeout: 5000, register });
+const workerCountMetric = new prom.Gauge({ name: "worker_count", help: "worker count" });
+const warmstartMetric = new prom.Histogram({ name: "warmstart", help: "warm start latency" });
+const coldstartMetric = new prom.Histogram({ name: "coldstart", help: "cold start latency"});
+const starttimeMetric = new prom.Histogram({ name: "starttime", help: "worker start times" });
+const requestMetric = new prom.Summary({ name: "requests", help: "request RTT times",
+    percentiles: [0.01, 0.05, 0.5, 0.9, 0.95, 0.99, 0.999]
+});
+
+register.registerMetric(workerCountMetric);
+register.registerMetric(warmstartMetric);
+register.registerMetric(coldstartMetric);
+register.registerMetric(starttimeMetric);
+register.registerMetric(requestMetric);
+
+let metricsDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
+metricsDB = metricsDB + "/" + constants.metrics_db_name + "/"
+let kafka = require('kafka-node'),
+    Producer = kafka.Producer,
+    client = new kafka.KafkaClient({
+        kafkaHost: constants.network.external.kafka_host,
+        autoConnect: true
+    }),
+    producer = new Producer(client)
+
+/**
+ * Function called to report metric data related to functions
+ * @param {JSON} metric 
+ */
+function collectMetrics(metric) {
+    /**
+     * If metrics for a new function comes in, 
+     * provision required structure for the function
+     */
+    if (!(metric.functionHash in metrics)) {
+        metrics[metric.functionHash] = {}
+    } 
+    if (!(metric.runtime in metrics[metric.functionHash])) {
+        metrics[metric.functionHash][metric.runtime] = {
+            shortterm: {
+                coldstart: 0,
+                coldstart_total_request: 0,
+                warm_total_request: 0,
+                scale_count: 0,
+                warmstart: 0,
+                worker_count: 0,
+                starttime: 0
+            }
+        }
+    }
+    if (metric.type === 'coldstart') {
+        metrics[metric.functionHash][metric.runtime].shortterm.coldstart += metric.value
+        metrics[metric.functionHash][metric.runtime].shortterm.coldstart_total_request += 1
+        coldstartMetric.observe(metric.value)
+        requestMetric.observe(metric.value)
+    } else if (metric.type === 'warmstart') {
+        metrics[metric.functionHash][metric.runtime].shortterm.warmstart += metric.value
+        metrics[metric.functionHash][metric.runtime].shortterm.warm_total_request += 1
+        warmstartMetric.observe(metric.value)
+        requestMetric.observe(metric.value)
+    } else if (metric.type === 'scale') {
+        metrics[metric.functionHash][metric.runtime].shortterm.worker_count = metric.value
+        workerCountMetric.set(metric.value)
+        if (metric.starttime !== undefined) {
+            metrics[metric.functionHash][metric.runtime].shortterm.starttime += metric.starttime
+            metrics[metric.functionHash][metric.runtime].shortterm.scale_count += 1
+            starttimeMetric.observe(metric.starttime)
+        }
+    }
+    
+}
+
+
+/**
+ * Run periodically to calculate average runtime metrics like coldstart and
+ * warmstart latencies.
+ * The module provides two granularities for metrics - shortterm and longterm
+ * shortterm - realtime data at a granularity of 5s (set in dispatch_manager/lib.js)
+ *             shortterm data is calculated using Simple Moving Average (SMA)
+ * longterm - longterm data is held and averaged out over a period of time.
+ *            longterm data is calculated using Expontential Moving Average (EMA)
+ */
+async function broadcastMetrics() {
+    if (Object.keys(metrics).length !== 0) {
+        for (let [functionHash, data] of Object.entries(metrics)) {
+            for (let [runtime, metricData] of Object.entries(data)) {
+                if (metricData.shortterm.coldstart != 0 || metricData.shortterm.longterm != 0) {
+                    let { metric, dbData } = await fetchData(functionHash, metricData, runtime)
+                    /**
+                    * Shortterm moving average
+                    */
+                    metric.shortterm.coldstart /= (metric.shortterm.coldstart_total_request != 0) ?
+                        metric.shortterm.coldstart_total_request : 1
+                    metric.shortterm.starttime /= (metric.shortterm.scale_count != 0) ?
+                        metric.shortterm.scale_count : 1
+                    metric.shortterm.warmstart /= (metric.shortterm.warm_total_request != 0) ?
+                        metric.shortterm.warm_total_request : 1
+                    
+                    /**
+                     * Longterm exponential moving average
+                     */
+                    if (metric.shortterm.coldstart != 0)
+                        metric.longterm.coldstart = (metric.longterm.coldstart != 0) ? metric.longterm.coldstart * alpha
+                            + metric.shortterm.coldstart * (1 - alpha) : metric.shortterm.coldstart
+                    if (metric.shortterm.starttime && metric.shortterm.starttime != 0)
+                        metric.longterm.starttime = (metric.longterm.starttime != 0) ? metric.longterm.starttime * alpha
+                            + metric.shortterm.starttime * (1 - alpha) : metric.shortterm.starttime
+                    if (metric.shortterm.warmstart != 0)
+                        metric.longterm.warmstart = (metric.longterm.warmstart != 0) ? metric.longterm.warmstart * alpha
+                            + metric.shortterm.warmstart * (1 - alpha) : metric.shortterm.warmstart
+                    dbData[runtime] = {
+                        coldstart: metric.longterm.coldstart,
+                        warmstart: metric.longterm.warmstart,
+                        starttime: metric.longterm.starttime
+                    }
+                    let payload = {
+                        method: 'put',
+                        body: JSON.stringify(dbData),
+                        headers: { 'Content-Type': 'application/json' }
+                    }
+                    await fetch(metricsDB + functionHash, payload)
+                    
+                    metric.timestamp = Date.now()
+                }
+            }
+        }
+        
+        let log = [{
+            topic: log_channel,
+            messages: JSON.stringify({
+                metrics
+            }),
+            partition: 0
+        }]
+        producer.send(log, () => { })
+        
+        for (let [functionHash, data] of Object.entries(metrics)) {
+            for (let [runtime, metric] of Object.entries(data)) {
+                metric.shortterm = {
+                    coldstart: 0,
+                    coldstart_total_request: 0,
+                    warm_total_request: 0,
+                    warmstart: 0,
+                    worker_count: 0,
+                    starttime: 0,
+                    scale_count: 0
+                }
+            }
+        }
+    }
+}
+
+/**
+ * Function to fetch the latest data from metric DB
+ * @param {String} functionHash 
+ * @param {JSON} metric 
+ */
+async function fetchData(functionHash, metric, runtime) {
+    let res = await fetch(metricsDB + functionHash)
+    let json = await res.json()
+    
+    if (json.error === "not_found" || json[runtime] === undefined) {
+        metric.longterm = {
+            coldstart: 0,
+            warmstart: 0,
+            starttime: 0
+        }
+    } else {
+        metric.longterm = {
+            coldstart: json[runtime].coldstart,
+            warmstart: json[runtime].warmstart,
+            starttime: (json[runtime].starttime) ? json[runtime].starttime: 0
+        }
+    }
+    return {
+        metric,
+        dbData: (json.error === "not_found")? {}: json
+    }
+}
+
+module.exports = {
+    collectMetrics, broadcastMetrics, register
+}
\ No newline at end of file

dispatch_system/dispatch_manager/operator.js

+const op = {
+    'lt': function (x, y) { return x < y },
+    'gt': function (x, y) { return x > y },
+    'lte': function (x, y) { return x <= y },
+    'gte': function (x, y) { return x >= y },
+    'eq': function (x, y) { return x === y },
+    'neq': function (x, y) { return x !== y },
+};
+
+module.exports = op
\ No newline at end of file

dispatch_system/dispatch_manager/package.json

@@ -20,6 +20,7 @@
     "morgan": "^1.9.1",
     "mqtt": "^3.0.0",
     "node-fetch": "^2.6.0",
+    "prom-client": "^12.0.0",
     "redis": "^2.8.0",
     "request": "^2.88.0",
     "request-promise": "^4.2.5",

dispatch_system/dispatch_manager/repository/worker_env/env.js

@@ -5,7 +5,8 @@ let request = require('request')
 const process = require('process')
 
 const app = express()
-let port = 5000, resource_id, functionHash, runtime, idleTime = 30
+let port = 5000, resource_id, functionHash, runtime, idleTime = 60, flagFirstRequest = true
+let waitTime
 
 resource_id = process.argv[2]
 functionHash = process.argv[3]
@@ -18,7 +19,7 @@ request = request.defaults({
 let kafka = require('kafka-node'),
     Producer = kafka.Producer,
     client = new kafka.KafkaClient({
-        kafkaHost: '10.129.6.5:9092',
+        kafkaHost: process.argv[6],
         autoConnect: true
     }),
     producer = new Producer(client)
@@ -28,6 +29,10 @@ app.use(bodyParser.json())
 
 let lastRequest = Date.now(), totalRequest = 0
 app.post('/serverless/function/execute/', (req, res) => {
+    if (flagFirstRequest) {
+        waitTime = Date.now() - waitTime
+        flagFirstRequest = false
+    }
     let payload = req.body
     lastRequest = Date.now()
     totalRequest++
@@ -36,13 +41,15 @@ app.post('/serverless/function/execute/', (req, res) => {
     })
 })
 
-app.post('/serverless/worker/timeout', (req, res) => {
+app.post('/serverless/function/timeout', (req, res) => {
     idleTime = req.body.timeout
     console.log("Idle time set to: ", idleTime);
-    
+    res.json({
+        status: "success"
+    })
 })
 
-function executor(payload) {
+async function executor(payload) {
     return new Promise((resolve, reject) => {
         
     })
@@ -57,6 +64,7 @@ app.listen(port, () => {
             runtime, resource_id, entity_id: process.pid}),
         "status": true
     }], () => { })
+    waitTime = Date.now()
 })
 
 function shouldDie() {
@@ -65,14 +73,17 @@ function shouldDie() {
         let message = JSON.stringify({
             functionHash, portExternal: port,
             runtime, resource_id, entity_id: process.pid,
-            total_request: totalRequest
+            total_request: totalRequest, wait_time: waitTime
         })
 
         console.log("Idle for too long. Exiting");
         producer.send(
             [
                 {topic: "removeWorker", messages: message }
-            ], () => {
+            ], (err, data) => {
+                if (err)
+                    console.log(err);
+                    
                 console.log("Ending worker for function", functionHash, "resource_id", resource_id);
                 process.exit(0)
             })

dispatch_system/log_listener.js

+const constants = require('./constants.json')
+const util = require('util')
+let kafka = require('kafka-node'),
+    Producer = kafka.Producer,
+    client = new kafka.KafkaClient({
+        kafkaHost: constants.network.external.kafka_host,
+        autoConnect: true
+    }),
+    producer = new Producer(client),
+    Consumer = kafka.Consumer,
+    consumer = new Consumer(client,
+        [
+            { topic: constants.topics.log_channel }
+        ])
+
+consumer.on('message', function (message) {
+    message = JSON.parse(message.value)
+    console.log(util.inspect(message, false, null, true /* enable colors */))
+    
+})
\ No newline at end of file

dispatch_system/prometheus.yml

+# my global config
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+      monitor: 'codelab-monitor'
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first.rules"
+  # - "second.rules"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'docker'
+         # metrics_path defaults to '/metrics'
+         # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ['localhost:9323']
+  
+  - job_name: 'xanadu'
+    static_configs:
+      - targets: ['localhost:8080']

dispatch_system/setup_topics.js

+const constants = require('./constants.json')
+
+var kafka = require('kafka-node');
+let client = new kafka.KafkaClient({
+    kafkaHost: constants.network.external.kafka_host,
+    autoConnect: true
+})
+var topicsToCreate = [];
+for (const [key, value] of Object.entries(constants.topics)) {
+    topicsToCreate.push({ topic: value, partitions: 1, replicationFactor: 1 })
+}
+client.createTopics(topicsToCreate, (error, result) => {
+    console.log("topic created", result);
+    
+});
\ No newline at end of file

docker-compose.yml

+version: '2'
+networks:
+  kafka-serverless:
+    driver: bridge
+services:
+  zookeeper:
+    image: 'bitnami/zookeeper:3'
+    restart: unless-stopped
+    networks:
+      - kafka-serverless
+    ports:
+      - '2182:2181'
+    volumes:
+      - 'zookeeper_data:/bitnami'
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+  kafka:
+    image: 'bitnami/kafka:2'
+    restart: unless-stopped
+    networks:
+      - kafka-serverless
+    ports:
+      - '9093:9092'
+      - '29092:29092'
+    volumes:
+      - 'kafka_data:/bitnami'
+    environment:
+      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
+      - ALLOW_PLAINTEXT_LISTENER=yes
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://0.0.0.0:9092,PLAINTEXT_HOST://0.0.0.0:29092
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+    depends_on:
+      - zookeeper
+
+volumes:
+  zookeeper_data:
+    driver: local
+  kafka_data:
+    driver: local

local_experiments/function_chain.drawio

+<mxfile host="Electron" modified="2020-03-09T11:38:25.308Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/12.6.5 Chrome/80.0.3987.86 Electron/8.0.0 Safari/537.36" etag="MPKhp0QP3HoRdc4h3Pxj" version="12.6.5" type="device"><diagram id="_vj1HQFHM5RY5pbnoe9b" name="Page-1">5Zlbb5swGIZ/DZebwJzSy+bQbFNTVY20db2JPHDBHcHIOAn018+ACcdkzZoFRG8if58P2M/38kISSZ2sozmFgbsgNvIkINuRpE4lAEbGiH8miThL6IqaJRyK7SylFIklfkUiKYvsBtsorAxkhHgMB9WkRXwfWaySg5SSXXXYM/GqVw2ggxqJpQW9ZvYHtpkrjgXMIv8FYcfNr6wYV1nPGuaDxUlCF9pkV0qpM0mdUEJY1lpHE+Ql7HIu2bybA737jVHks7dMWDy8zFbfngC5vlvZ7sPcBNf4kyH2xuL8wJRsfBslc2RJHRPKXOIQH3q3hAQ8qfDkC2IsFqWCG0Z4ymVrT/SiCLPHUvunWCppT6NyEOeBz2j8WA7SOZ/1PCympVE+L2SU/N6XhQMdN5kITCHZUAsdAZFrC1IHsSPjhHqRXZGNID5HZI34DvkAijzI8LaqIijE6OzH7afeE8y3DGRx3wBNqEbcNkquonyJbKNiVlF13ihto0ilWjhBF+ZldKEMRhfg3LqoVPTU8ol1t9DbiCvdKI2K8nW4i/JgvHMxQ8sApiR23MerlYNhkFnrM44SBZwCeIsoQ9FRJKJXryl+JOJdyXZFyi057kh+P8On8I5tHKB/XdxOv9s0Nlexld/pZWC8uEsR+sRPyL3/rjizxmsC/DfRgzeKXruQxlvro3VaHwP8rUJpdI8o5sdFtE9l07u0JtC0JtB3awJG1ZtA197UYuYl7RcynxXZgVqV2jPNt5arKfCPXYxLPTeObbJsQC2P+n4b0AX953ABqwz1vjPUOzTxw5ZUhdjyTtNviF0/CJsMTX6pphb58ViNl4cdn7ctDiN5PRsnELAFvWvRsca2nT04UYhf4a90qcRCg+RLd3oQfSzp02Qtbs9h5tTKnnMDagv6g5wVtfZtSGuC1lpA138mOBtoY6CgNaUGWu4YtPlBQKugY9CjBmh1EKDr1tHm0RcFfTVQ0JpZA921R+d32PBI1946FP2/keZh8ZdM9mt68b+WOvsD</diagram></mxfile>
\ No newline at end of file

readme.md

@@ -145,6 +145,7 @@ to run and execute the function on the specified worker node.
 - Docker
 - Java
 - Apache Kafka (Configure to allow auto-delete and auto-registration of topics)
+- couchdb (needs a database named serverless)
 
 ### Starting the server