Commit 1e2d3ca4 authored by Naman Dixit's avatar Naman Dixit

Merge on pull

parents 85c0cfb7 0198dbc7
......@@ -6,3 +6,4 @@ firecracker*
secrets.json
resource_system/bin/**
resource_system/version.linux
local_experiments/
{
"registry_url" :"10.129.6.5:5000/",
"registry_url" :"localhost:5000/",
"master_port": 8080,
"master_address": "localhost",
"kafka_host": "10.129.6.5:9092",
"grunt_host": "https://www.namandixit.net/lovecraftian_nightmares/grunt",
"log_channel": "LOG_COMMON",
"couchdb_host": "10.129.6.5:5984",
"couchdb_db_name": "serverless",
"couchdb_host": "localhost:5984",
"function_db_name": "serverless",
"metrics_db_name": "metrics",
"implicit_chain_db_name": "implicit_chain",
"network": {
"network_bridge": "hybrid_kafka-serverless",
"internal": {
"kafka_host": "kafka:9092"
},
"external": {
"kafka_host": "localhost:29092"
}
},
"topics": {
"request_dm_2_rm": "request",
"heartbeat": "heartbeat",
"deployed": "deployed",
"remove_worker": "removeWorker",
"response_rm_2_dm": "RESPONSE_RM_2_DM_DUMMY",
"hscale": "hscale"
"hscale": "hscale",
"log_channel": "LOG_COMMON"
},
"autoscalar_metrics": {
"open_request_threshold": 100
},
"speculative_deployment": true
}
\ No newline at end of file
"speculative_deployment": false,
"JIT_deployment": false,
"id_size": 20
}
{"id":"192.168.31.51","master_node":"10.129.6.5"}
\ No newline at end of file
{"id":"192.168.0.105","master_node":"192.168.0.105"}
\ No newline at end of file
......@@ -17,7 +17,7 @@ function runIsolate(local_repository, metadata) {
return new Promise((resolve, reject) => {
const worker = new Worker(filename, {
argv: [resource_id, functionHash, port, "isolate"],
argv: [resource_id, functionHash, port, "isolate", constants.network.external.kafka_host],
resourceLimits: {
maxOldGenerationSizeMb: memory
}
......@@ -43,7 +43,8 @@ function runProcess(local_repository, metadata) {
return new Promise((resolve, reject) => {
let timeStart = Date.now()
const process = spawn('node', [filename, resource_id, functionHash, port, "process", `--max-old-space-size=${memory}` ]);
const process = spawn('node', [filename, resource_id, functionHash, port, "process",
constants.network.external.kafka_host, `--max-old-space-size=${memory}` ]);
process.stdout.on('data', (data) => {
console.log(`stdout: ${data}`);
let timeDifference = Math.ceil((Date.now() - timeStart))
......@@ -91,8 +92,9 @@ function runContainer(metadata) {
if (code != 0)
reject("error")
else {
const process = spawn('docker', ["run", "--rm", "-p", `${port}:${port}`, "--name", resource_id, registry_url + imageName,
resource_id, imageName, port, "container"]);
const process = spawn('docker', ["run", "--rm", `--network=${constants.network.network_bridge}`, "-p", `${port}:${port}`,
"--name", resource_id, registry_url + imageName,
resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
let result = "";
// timeStart = Date.now()
process.stdout.on('data', (data) => {
......@@ -118,8 +120,9 @@ function runContainer(metadata) {
} else {
logger.info("container starting at port", port);
const process = spawn('docker', ["run", "--rm", "-p", `${port}:${port}`, "--name", resource_id,
registry_url + imageName, resource_id, imageName, port, "container"]);
const process = spawn('docker', ["run", "--rm", `--network=${constants.network.network_bridge}`,
"-p", `${port}:${port}`, "--name", resource_id,
registry_url + imageName, resource_id, imageName, port, "container", constants.network.internal.kafka_host]);
let result = "";
// timeStart = Date.now()
process.stdout.on('data', (data) => {
......
......@@ -11,7 +11,7 @@ const fs = require('fs')
const fetch = require('node-fetch');
let metadataDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
metadataDB = metadataDB + "/" + constants.couchdb_db_name + "/"
metadataDB = metadataDB + "/" + constants.function_db_name + "/"
const kafka = require('kafka-node')
const logger = libSupport.logger
......@@ -21,7 +21,7 @@ const host_url = "http://" + constants.master_address + ":" + constants.master_p
let Producer = kafka.Producer,
client = new kafka.KafkaClient({
kafkaHost: constants.kafka_host,
kafkaHost: constants.network.external.kafka_host,
autoConnect: true
}),
producer = new Producer(client),
......@@ -77,9 +77,9 @@ libSupport.makeTopic(node_id).then(() => {
/**
* download and start grunt
*/
libSupport.download(constants.grunt_host, "grunt").then(() => {
libSupport.download(constants.grunt_host, "grunt", false).then(() => {
logger.info("Downloaded grunt binary from repository")
fs.chmod('grunt', 0o555, (err) => {
fs.chmod('grunt', 0o755, (err) => {
logger.info("grunt made executable. Starting grunt")
let grunt = spawn('./grunt', [node_id])
grunt.stdout.on('data', data => {
......
const http = require('http');
const fetch = require('node-fetch');
const fs = require('fs');
const process = require('process')
const { spawnSync } = require('child_process');
......@@ -30,7 +30,7 @@ function makeTopic(id) {
console.log("Using Primary IP", id, "as topic");
let client = new kafka.KafkaClient({
kafkaHost: constants.kafka_host,
kafkaHost: constants.network.external.kafka_host,
autoConnect: true
}),
Producer = kafka.Producer,
......@@ -50,28 +50,48 @@ function makeTopic(id) {
})
}
var download = function (url, dest, cb) {
return new Promise((resolve, reject) => {
// var download = function (url, dest, check = true, cb) {
// return new Promise((resolve, reject) => {
// console.log(url);
// if (!check || !fs.existsSync(dest)) {
// var file = fs.createWriteStream(dest);
// var request = https.get(url, function (response) {
// response.pipe(file);
// file.on('finish', function () {
// file.close(cb); // close() is async, call cb after close completes.
// resolve();
// });
// }).on('error', function (err) { // Handle errors
// fs.unlink(dest); // Delete the file async. (But we don't check the result)
// logger.error("download failed" + err.message);
// if (cb) cb(err.message);
// reject(err);
// });
// } else {
// resolve();
// }
// })
// };
const download = (async (url, path, check = true) => {
if (!check || !fs.existsSync(path)) {
console.log(url);
if (!fs.existsSync(dest)) {
var file = fs.createWriteStream(dest);
var request = https.get(url, function (response) {
response.pipe(file);
file.on('finish', function () {
file.close(cb); // close() is async, call cb after close completes.
resolve();
});
}).on('error', function (err) { // Handle errors
fs.unlink(dest); // Delete the file async. (But we don't check the result)
if (cb) cb(err.message);
const res = await fetch(url);
const fileStream = fs.createWriteStream(path);
await new Promise((resolve, reject) => {
res.body.pipe(fileStream);
res.body.on("error", (err) => {
reject(err);
});
} else {
resolve();
}
})
};
fileStream.on("finish", function () {
resolve();
});
});
}
});
function makeid(length) {
var result = '';
......
......@@ -16,6 +16,7 @@
"kafka-node": "^5.0.0",
"morgan": "^1.9.1",
"mqtt": "^3.0.0",
"node-fetch": "^2.6.0",
"redis": "^2.8.0",
"request": "^2.88.2",
"winston": "^3.2.1"
......
This diff is collapsed.
This diff is collapsed.
......@@ -4,10 +4,23 @@ const rp = require('request-promise');
const fetch = require('node-fetch');
const winston = require('winston')
const constants = require('.././constants.json')
const secrets = require('./secrets.json')
const metrics = require('./metrics')
const { createLogger, format, transports } = winston;
const heap = require('heap')
let kafka = require('kafka-node'),
Producer = kafka.Producer,
client = new kafka.KafkaClient({
kafkaHost: constants.network.external.kafka_host,
autoConnect: true
}),
producer = new Producer(client)
let implicitChainDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
implicitChainDB = implicitChainDB + "/" + constants.implicit_chain_db_names + "/"
/**
* Generates unique IDs of arbitrary length
* @param {Length of the ID} length
......@@ -45,6 +58,14 @@ function generateExecutor(functionPath, functionHash) {
return hash
}
/**
* Reverse proxy to take user requests and forward them to appropriate workers using a loadbalacer
* @param {JSON} req the user request to be forwarded to the worker
* @param {JSON} res Object to use to return the response to the user
* @param {Map} functionToResource Function to resource Map
* @param {Map} resourceMap Map from resource ID to resource metadata
* @param {Map} functionBranchTree Holds the function path's and related probability distribution
*/
function reverseProxy(req, res, functionToResource, resourceMap, functionBranchTree) {
branchChainPredictor(req, resourceMap, functionToResource, functionBranchTree)
return new Promise((resolve, reject) => {
......@@ -54,6 +75,7 @@ function reverseProxy(req, res, functionToResource, resourceMap, functionBranchT
* Bypass deployment pipeline if resource available
*/
let functionHeap = functionToResource.get(id)
// loadbalancing by choosing worker with lowest load
let forwardTo = functionHeap[0]
let resource = resourceMap.get(forwardTo.resource_id)
// logger.info(`Choosing resource ${JSON.stringify(forwardTo.resource_id)}` +
......@@ -62,7 +84,7 @@ function reverseProxy(req, res, functionToResource, resourceMap, functionBranchT
// logger.info("Request received at reverseproxy. Forwarding to: " + url);
forwardTo.open_request_count += 1
heap.heapify(functionHeap, compare)
heap.heapify(functionHeap, compare) // maintain loadbalancer by heapifying the Map
// logger.info(functionHeap);
var options = {
......@@ -71,22 +93,23 @@ function reverseProxy(req, res, functionToResource, resourceMap, functionBranchT
body: req.body,
json: true // Automatically stringifies the body to JSON
};
// console.log(options);
rp(options)
.then(function (parsedBody) {
let serviceTime = Date.now() - res.timestamp
res.json(parsedBody)
forwardTo.open_request_count -= 1
heap.heapify(functionHeap, compare)
metrics.collectMetrics({type: res.start, value: serviceTime, functionHash: req.params.id, runtime})
resolve()
})
.catch(function (err) {
forwardTo.open_request_count -= 1
heap.heapify(functionHeap, compare)
logger.error("error" + err.error.errno);
logger.error("error" + err);
res.json(err.message).status(err.statusCode)
resolve()
});
......@@ -109,7 +132,6 @@ function getPort(usedPort) {
return port
}
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(
......@@ -252,17 +274,61 @@ function viterbi(functionBranchTree) {
path.push({node: maxSibling, probability: maxProb})
siblings = new Map()
}
if (path.length > 0)
console.log("path", path);
// if (path.length > 0)
// console.log("path", path);
metadata.mle_path = path
if (path.length > 1) {
let payload = {
method: 'put',
body: JSON.stringify(path),
headers: { 'Content-Type': 'application/json' }
}
fetch(implicitChainDB + functionHash, payload)
}
}
});
}
function logBroadcast(message, resource_id, resourceMap) {
return new Promise((resolve, reject) => {
try {
message.timestamp = Date.now()
if (resource_id && resourceMap.has(resource_id)) {
let resource = resourceMap.get(resource_id)
message.resource_id = resource_id
message.node_id = resource.node_id
message.runtime = resource.runtime
message.function_id = resource.functionHash
}
let log = [{
topic: constants.topics.log_channel,
messages: JSON.stringify(message),
partition: 0
}]
producer.send(log, () => {
resolve()
})
} catch (err) {
console.log(err);
reject()
}
})
}
async function fetchData(url, data = null) {
let res
if (data === undefined || data === null)
res = await fetch(url)
else
res = await fetch(url, data)
return await res.json()
}
module.exports = {
makeid, generateExecutor, reverseProxy,
getPort, logger, compare,
viterbi
viterbi, logBroadcast, fetchData, metrics,
producer
}
\ No newline at end of file
'use strict';
const constants = require('.././constants.json');
const secrets = require('./secrets.json')
const fetch = require('node-fetch');
const util = require('util')
const prom = require('prom-client');
const Registry = prom.Registry;
const register = new Registry();
const alpha = 0.99
let log_channel = constants.topics.log_channel,
metrics = { }
const intervalCollector = prom.collectDefaultMetrics({ prefix: 'xanadu', timeout: 5000, register });
const workerCountMetric = new prom.Gauge({ name: "worker_count", help: "worker count" });
const warmstartMetric = new prom.Histogram({ name: "warmstart", help: "warm start latency" });
const coldstartMetric = new prom.Histogram({ name: "coldstart", help: "cold start latency"});
const starttimeMetric = new prom.Histogram({ name: "starttime", help: "worker start times" });
const requestMetric = new prom.Summary({ name: "requests", help: "request RTT times",
percentiles: [0.01, 0.05, 0.5, 0.9, 0.95, 0.99, 0.999]
});
register.registerMetric(workerCountMetric);
register.registerMetric(warmstartMetric);
register.registerMetric(coldstartMetric);
register.registerMetric(starttimeMetric);
register.registerMetric(requestMetric);
let metricsDB = `http://${secrets.couchdb_username}:${secrets.couchdb_password}@${constants.couchdb_host}`
metricsDB = metricsDB + "/" + constants.metrics_db_name + "/"
let kafka = require('kafka-node'),
Producer = kafka.Producer,
client = new kafka.KafkaClient({
kafkaHost: constants.network.external.kafka_host,
autoConnect: true
}),
producer = new Producer(client)
/**
* Function called to report metric data related to functions
* @param {JSON} metric
*/
function collectMetrics(metric) {
/**
* If metrics for a new function comes in,
* provision required structure for the function
*/
if (!(metric.functionHash in metrics)) {
metrics[metric.functionHash] = {}
}
if (!(metric.runtime in metrics[metric.functionHash])) {
metrics[metric.functionHash][metric.runtime] = {
shortterm: {
coldstart: 0,
coldstart_total_request: 0,
warm_total_request: 0,
scale_count: 0,
warmstart: 0,
worker_count: 0,
starttime: 0
}
}
}
if (metric.type === 'coldstart') {
metrics[metric.functionHash][metric.runtime].shortterm.coldstart += metric.value
metrics[metric.functionHash][metric.runtime].shortterm.coldstart_total_request += 1
coldstartMetric.observe(metric.value)
requestMetric.observe(metric.value)
} else if (metric.type === 'warmstart') {
metrics[metric.functionHash][metric.runtime].shortterm.warmstart += metric.value
metrics[metric.functionHash][metric.runtime].shortterm.warm_total_request += 1
warmstartMetric.observe(metric.value)
requestMetric.observe(metric.value)
} else if (metric.type === 'scale') {
metrics[metric.functionHash][metric.runtime].shortterm.worker_count = metric.value
workerCountMetric.set(metric.value)
if (metric.starttime !== undefined) {
metrics[metric.functionHash][metric.runtime].shortterm.starttime += metric.starttime
metrics[metric.functionHash][metric.runtime].shortterm.scale_count += 1
starttimeMetric.observe(metric.starttime)
}
}
}
/**
* Run periodically to calculate average runtime metrics like coldstart and
* warmstart latencies.
* The module provides two granularities for metrics - shortterm and longterm
* shortterm - realtime data at a granularity of 5s (set in dispatch_manager/lib.js)
* shortterm data is calculated using Simple Moving Average (SMA)
* longterm - longterm data is held and averaged out over a period of time.
* longterm data is calculated using Expontential Moving Average (EMA)
*/
async function broadcastMetrics() {
if (Object.keys(metrics).length !== 0) {
for (let [functionHash, data] of Object.entries(metrics)) {
for (let [runtime, metricData] of Object.entries(data)) {
if (metricData.shortterm.coldstart != 0 || metricData.shortterm.longterm != 0) {
let { metric, dbData } = await fetchData(functionHash, metricData, runtime)
/**
* Shortterm moving average
*/
metric.shortterm.coldstart /= (metric.shortterm.coldstart_total_request != 0) ?
metric.shortterm.coldstart_total_request : 1
metric.shortterm.starttime /= (metric.shortterm.scale_count != 0) ?
metric.shortterm.scale_count : 1
metric.shortterm.warmstart /= (metric.shortterm.warm_total_request != 0) ?
metric.shortterm.warm_total_request : 1
/**
* Longterm exponential moving average
*/
if (metric.shortterm.coldstart != 0)
metric.longterm.coldstart = (metric.longterm.coldstart != 0) ? metric.longterm.coldstart * alpha
+ metric.shortterm.coldstart * (1 - alpha) : metric.shortterm.coldstart
if (metric.shortterm.starttime && metric.shortterm.starttime != 0)
metric.longterm.starttime = (metric.longterm.starttime != 0) ? metric.longterm.starttime * alpha
+ metric.shortterm.starttime * (1 - alpha) : metric.shortterm.starttime
if (metric.shortterm.warmstart != 0)
metric.longterm.warmstart = (metric.longterm.warmstart != 0) ? metric.longterm.warmstart * alpha
+ metric.shortterm.warmstart * (1 - alpha) : metric.shortterm.warmstart
dbData[runtime] = {
coldstart: metric.longterm.coldstart,
warmstart: metric.longterm.warmstart,
starttime: metric.longterm.starttime
}
let payload = {
method: 'put',
body: JSON.stringify(dbData),
headers: { 'Content-Type': 'application/json' }
}
await fetch(metricsDB + functionHash, payload)
metric.timestamp = Date.now()
}
}
}
let log = [{
topic: log_channel,
messages: JSON.stringify({
metrics
}),
partition: 0
}]
producer.send(log, () => { })
for (let [functionHash, data] of Object.entries(metrics)) {
for (let [runtime, metric] of Object.entries(data)) {
metric.shortterm = {
coldstart: 0,
coldstart_total_request: 0,
warm_total_request: 0,
warmstart: 0,
worker_count: 0,
starttime: 0,
scale_count: 0
}
}
}
}
}
/**
* Function to fetch the latest data from metric DB
* @param {String} functionHash
* @param {JSON} metric
*/
async function fetchData(functionHash, metric, runtime) {
let res = await fetch(metricsDB + functionHash)
let json = await res.json()
if (json.error === "not_found" || json[runtime] === undefined) {
metric.longterm = {
coldstart: 0,
warmstart: 0,
starttime: 0
}
} else {
metric.longterm = {
coldstart: json[runtime].coldstart,
warmstart: json[runtime].warmstart,
starttime: (json[runtime].starttime) ? json[runtime].starttime: 0
}
}
return {
metric,
dbData: (json.error === "not_found")? {}: json
}
}
module.exports = {
collectMetrics, broadcastMetrics, register
}
\ No newline at end of file
const op = {
'lt': function (x, y) { return x < y },
'gt': function (x, y) { return x > y },
'lte': function (x, y) { return x <= y },
'gte': function (x, y) { return x >= y },
'eq': function (x, y) { return x === y },
'neq': function (x, y) { return x !== y },
};
module.exports = op
\ No newline at end of file
......@@ -20,6 +20,7 @@
"morgan": "^1.9.1",
"mqtt": "^3.0.0",
"node-fetch": "^2.6.0",
"prom-client": "^12.0.0",
"redis": "^2.8.0",
"request": "^2.88.0",
"request-promise": "^4.2.5",
......
......@@ -5,7 +5,8 @@ let request = require('request')
const process = require('process')
const app = express()
let port = 5000, resource_id, functionHash, runtime, idleTime = 30
let port = 5000, resource_id, functionHash, runtime, idleTime = 60, flagFirstRequest = true
let waitTime
resource_id = process.argv[2]
functionHash = process.argv[3]
......@@ -18,7 +19,7 @@ request = request.defaults({
let kafka = require('kafka-node'),
Producer = kafka.Producer,
client = new kafka.KafkaClient({
kafkaHost: '10.129.6.5:9092',
kafkaHost: process.argv[6],
autoConnect: true
}),
producer = new Producer(client)
......@@ -28,6 +29,10 @@ app.use(bodyParser.json())
let lastRequest = Date.now(), totalRequest = 0
app.post('/serverless/function/execute/', (req, res) => {
if (flagFirstRequest) {
waitTime = Date.now() - waitTime
flagFirstRequest = false
}
let payload = req.body
lastRequest = Date.now()
totalRequest++
......@@ -36,13 +41,15 @@ app.post('/serverless/function/execute/', (req, res) => {
})
})
app.post('/serverless/worker/timeout', (req, res) => {
app.post('/serverless/function/timeout', (req, res) => {
idleTime = req.body.timeout
console.log("Idle time set to: ", idleTime);
res.json({
status: "success"
})
})
function executor(payload) {
async function executor(payload) {
return new Promise((resolve, reject) => {
})
......@@ -57,6 +64,7 @@ app.listen(port, () => {
runtime, resource_id, entity_id: process.pid}),
"status": true
}], () => { })
waitTime = Date.now()
})
function shouldDie() {
......@@ -65,14 +73,17 @@ function shouldDie() {
let message = JSON.stringify({
functionHash, portExternal: port,
runtime, resource_id, entity_id: process.pid,
total_request: totalRequest
total_request: totalRequest, wait_time: waitTime
})
console.log("Idle for too long. Exiting");
producer.send(
[
{topic: "removeWorker", messages: message }
], () => {
], (err, data) => {
if (err)
console.log(err);
console.log("Ending worker for function", functionHash, "resource_id", resource_id);
process.exit(0)
})
......
const constants = require('./constants.json')
const util = require('util')
let kafka = require('kafka-node'),
Producer = kafka.Producer,
client = new kafka.KafkaClient({
kafkaHost: constants.network.external.kafka_host,
autoConnect: true
}),
producer = new Producer(client),
Consumer = kafka.Consumer,
consumer = new Consumer(client,
[
{ topic: constants.topics.log_channel }
])
consumer.on('message', function (message) {
message = JSON.parse(message.value)
console.log(util.inspect(message, false, null, true /* enable colors */))
})
\ No newline at end of file