Skip to content

X
xanadu
Commit ebffa4a2 authored by nilanjandaw's avatar nilanjandaw
Browse files
Options

worker UID changed to IP address 

UID of worker nodes has been changed to IP, streamlined DM code base
parent 5846c061
Hide whitespace changes
Inline Side-by-side
Showing with 141 additions and 132 deletions
dispatch_module/constants.json
View file @ ebffa4a2
{
"mqtt_url": "10.129.6.5",
"registry_url" :"10.129.6.5:5000/",
"master_port": 8080,
"master_address": "localhost"
"master_address": "localhost",
"kafka_host": "10.129.6.5:9092"
}
\ No newline at end of file
dispatch_module/dispatch_daemon/config.json
View file @ ebffa4a2
{
"id": "tpt8hqn7ok"
}
\ No newline at end of file
{"id":"192.168.31.51","master_node":"10.129.6.5"}
\ No newline at end of file
dispatch_module/dispatch_daemon/index.js
View file @ ebffa4a2
const mqtt = require('mqtt')
'use strict';
const constants = require(".././constants.json")
const node_id = require("./config.json").id
const config = require('./config.json')
const libSupport = require('./lib')
libSupport.updateConfig()
const node_id = config.id
const execute = require('./execute')
const fs = require('fs')
const kafka = require('kafka-node')
const local_repository = __dirname + "/local_repository/"
const host_url = "http://" + constants.master_address + ":" + constants.master_port
let kafka = require('kafka-node'),
Producer = kafka.Producer,
let Producer = kafka.Producer,
KeyedMessage = kafka.KeyedMessage,
client = new kafka.KafkaClient({
kafkaHost: '10.129.6.5:9092',
kafkaHost: constants.kafka_host,
autoConnect: true
}),
producer = new Producer(client),
Consumer = kafka.Consumer,
consumer = new Consumer(client,
Consumer = kafka.Consumer
libSupport.makeTopic(node_id).then(() => {
console.log("node topic created")
let consumer = new Consumer(client,
[
{ topic: node_id, partition: 0, offset: 0}
{ topic: node_id, partition: 0, offset: 0 }
],
[
{ autoCommit: true }
])
consumer.on('message', function (message) {
console.log(message);
let topic = message.topic
consumer.on('message', function (message) {
console.log(message);
let topic = message.topic
message = message.value
message = JSON.parse(message)
if (topic !== 'heartbeat') {
let runtime = message.runtime
let functionHash = message.functionHash
let function_id = message.function_id
if (message.type === "execute") {
console.log("function_id", function_id);
if (!fs.existsSync(local_repository + functionHash)) {
message = JSON.parse(message)
if (topic !== 'heartbeat') {
let runtime = message.runtime
let functionHash = message.functionHash
let function_id = message.function_id
if (message.type === "execute") {
console.log("function_id", function_id);
libSupport.download(host_url + "/repository/" + functionHash, local_repository + functionHash).then(() => {
if (runtime === "isolate")
execute.runIsolate(local_repository + functionHash).then(result => {
producer.send([{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id})
}], () =>{})
})
else if (runtime === "process")
execute.runProcess(local_repository + functionHash).then(result => {
producer.send(
[{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id})
}], () =>{})
})
else if (runtime === "container")
execute.runContainer(functionHash).then(result => {
producer.send(
[{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id})
}], () =>{})
})
else {
producer.send(
[{
topic: "response",
messages: JSON.stringify({ status: "unknown runtime" })
}], () =>{})
return
}
startWorker(local_repository, functionHash, producer, runtime)
})
} else {
if (runtime === "isolate")
execute.runIsolate(local_repository + functionHash).then(result => {
producer.send(
[{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id})
}], () =>{})
})
else if (runtime === "process")
execute.runProcess(local_repository + functionHash).then(result => {
producer.send(
[{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id})
}], () =>{})
})
else if (runtime === "container")
execute.runContainer(functionHash).then(result => {
producer.send(
[{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id})
}], () =>{})
})
else {
producer.send(
[{
topic: "response",
messages: JSON.stringify({ status: "unknown runtime" })
}], () =>{})
return
}
}
}
}
})
})
function startWorker(local_repository, functionHash, producer, runtime) {
if (runtime === "isolate")
execute.runIsolate(local_repository + functionHash).then(result => {
producer.send([{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id
})
}], () => { })
})
else if (runtime === "process")
execute.runProcess(local_repository + functionHash).then(result => {
producer.send(
[{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id
})
}], () => { })
})
else if (runtime === "container")
execute.runContainer(functionHash).then(result => {
producer.send(
[{
topic: "response",
messages: JSON.stringify({
status: "success",
result,
function_id
})
}], () => { })
})
else {
producer.send(
[{
topic: "response",
messages: JSON.stringify({ status: "unknown runtime" })
}], () => { })
return
}
}
function heartbeat() {
let payload = [{
......
dispatch_module/dispatch_daemon/lib.js
View file @ ebffa4a2
var http = require('http');
var fs = require('fs');
const http = require('http');
const fs = require('fs');
const process = require('process')
const { spawnSync } = require('child_process');
const constants = require(".././constants.json")
const kafka = require('kafka-node')
function updateConfig() {
console.log("Retrieving primary IP");
let file = JSON.parse(fs.readFileSync('./config.json', { encoding: 'utf-8' }))
const getIP = spawnSync("ip", ["route", "get", file.master_node]);
let err = getIP.stderr.toString().trim()
if (err !== '') {
console.log(err);
process.exit(1);
}
let data = getIP.stdout.toString().trim().split(' ')
file.id = data[6]
fs.writeFileSync('./config.json', JSON.stringify(file));
console.log("Updated Config file");
}
function makeTopic(id) {
console.log("Using Primary IP", id, "as topic");
let client = new kafka.KafkaClient({
kafkaHost: constants.kafka_host,
autoConnect: true
}),
Producer = kafka.Producer,
producer = new Producer(client)
return new Promise((resolve, reject) => {
producer.send([{
topic: id,
messages: JSON.stringify({
status: "success",
})
}], (err, data) => {
if (err)
reject();
else
resolve();
})
})
}
var download = function (url, dest, cb) {
return new Promise((resolve, reject) => {
console.log(url);
var file = fs.createWriteStream(dest);
var request = http.get(url, function (response) {
response.pipe(file);
file.on('finish', function () {
file.close(cb); // close() is async, call cb after close completes.
resolve();
if (!fs.existsSync(local_repository + functionHash)) {
var file = fs.createWriteStream(dest);
var request = http.get(url, function (response) {
response.pipe(file);
file.on('finish', function () {
file.close(cb); // close() is async, call cb after close completes.
resolve();
});
}).on('error', function (err) { // Handle errors
fs.unlink(dest); // Delete the file async. (But we don't check the result)
if (cb) cb(err.message);
reject(err);
});
}).on('error', function (err) { // Handle errors
fs.unlink(dest); // Delete the file async. (But we don't check the result)
if (cb) cb(err.message);
reject(err);
});
} else {
resolve();
}
})
};
......@@ -31,6 +76,6 @@ function makeid(length) {
return result;
}
module.exports.download = download
module.exports.makeid = makeid;
\ No newline at end of file
module.exports = {
download, makeid, updateConfig, makeTopic
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment