Merge remote-tracking branch 'origin/master'

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "resource_manager/src/common/nlib"]
 	path = resource_system/src/common/nlib
 	url = https://github.com/namandixit/nlib
+[submodule "resource_system/src/common/inih"]
+	path = resource_system/src/common/inih
+	url = https://github.com/benhoyt/inih

config.ini

+[Kafka]
+Address = 10.129.6.5:9092
+
+[Arbiter]
+MessageReadGap = 10
+GruntTimeToDie = 10000
+GruntResponseWaitTime = 100
+
+[Grunt]
+MessageReadGap = 10
+HeartbeatGap = 1000
\ No newline at end of file

readme.md

@@ -23,7 +23,7 @@ The Dispatch Manager (DM) sends a request to the Resource Manager (RM), detailin
 ```javascript
 {
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "memory": 1024, // in MiB
     ... // Any other resources
 }
@@ -34,7 +34,7 @@ Format:
 ```javascript
 {
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "grunts": [  
            { node_id: some unique ID, port: port address}, ...
      ] // List of machine IDs
@@ -44,10 +44,13 @@ Format:
 Once the runtime entity has been launched (or the launch has failed), the Executor sends back a status message on the `LOG_COMMON` topic.
 ```javascript
 {
-    "node_id" : "uique-machine-id",
+    "message_type" : "deployment_launch",
+    "node_id" : "unique-machine-id",
+    "entity_id" : "handle for the actual container/VM/etc.",
+    "entity_type" : "docker/libvirt/etc.",
     "resource_id": "logical-entity-id",
     "function_id": "unique-function-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "reason": "deployment"/"termination"
     "status": true/false // Only valid if reason==deployment
 }
@@ -57,28 +60,31 @@ Instrumentation data is also sent on the `LOG_COMMON` topic. This data is sent f
 and whoever needs the data is allowed to read it. Each message is required to have atleast three fields: `node_id`, `resource_id` and `function_id`.
  ```javascript
 { // Example message from Executor
-    "node_id" : "uique-machine-id",
+    "message_type" : "instrumentation",
+    "node_id" : "unique-machine-id",
     "resource_id": "logical-entity-id",
     "function_id": "unique-function-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "cpu" : 343, // in MHz
     "memory": 534, // in MiB
     "network": 234 // in KBps
 }
 
 { // Example message from reverse proxy
-    "node_id" : "uique-machine-id",
+    "message_type" : "instrumentation",
+    "node_id" : "unique-machine-id",
     "resource_id": "logical-entity-id",
     "function_id": "unique-function-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "average_fn_time" : 23 // in ms
 }
 
 { // Example message from dispatch manager
-    "node_id" : "uique-machine-id",
+    "message_type" : "instrumentation",
+    "node_id" : "unique-machine-id",
     "resource_id": "logical-entity-id",
     "function_id": "unique-function-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "coldstart_time" 
 }
 ```
@@ -223,7 +229,7 @@ resources being tracked by RDs on each machine. This data is cached by the RM.
 ```javascript
 {
     "node_id": "unique-machine-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "memory": 1024, // in MiB
     ... // Any other resources
 }
@@ -246,7 +252,7 @@ DM on topic `RESPONSE_RM_2_DM`.
 ```javascript
 {
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
 //    "port": 2343 --- NOT IMPLEMENTED YET
     "nodes": ["a", "b", ...] // List of unique machine IDs
 }
@@ -258,7 +264,7 @@ Format:
 ```javascript
 {
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "memory": 1024, // in MiB
     ... // Any other resources
 }
@@ -269,7 +275,7 @@ The RDs recieve this message and send back whether on not they satisfy the const
 {
     "node_id": "unique-machine-id",
     "resource_id": "unique-transaction-id",
-    "timestamp" : "iso-8601-timestamp",
+    "timestamp" : "time(2) compatible timestamp",
     "success" : 0/1 // 0 = fail, 1 = success
 }
 ```

resource_system/src/arbiter/command.c

+/*
+ * Creator: Naman Dixit
+ * Notice: © Copyright 2020 Naman Dixit
+ */
+
+typedef struct Command {
+    enum Command_Kind {
+        Command_NONE,
+        Command_RESPONSE_ARBITER_2_DM,
+        Command_REQUEST_ARBITER_2_GRUNT,
+        Command_JOIN_ACK_ARBITER_2_GRUNT,
+        Command_REJOIN_ARBITER_2_GRUNT,
+    } kind;
+
+    Char *resource_id;
+
+    union {
+        struct {
+            Char **grunt_ids;
+        } res_a2d;
+
+        struct {
+            Sint memory;
+        } req_a2g;
+
+        struct {
+            Char *grunt_id;
+        } rejoin_a2g;
+
+        struct {
+            Char *grunt_id;
+        } join_ack_a2g;
+    };
+} Command;

resource_system/src/arbiter/conf.c

+/*
+ * Creator: Naman Dixit
+ * Notice: © Copyright 2020 Naman Dixit
+ */
+
+typedef struct Configuration {
+    Char *kafka_address;
+
+    U64 grunt_response_wait_time;
+    Sint message_read_gap;
+    Sint grunt_time_to_die;
+} Configuration;
+
+# if defined(COMPILER_CLANG)
+#  pragma clang diagnostic push
+#   pragma clang diagnostic ignored "-Wincompatible-pointer-types-discards-qualifiers"
+# endif
+
+internal_function
+Sint confCallback (void* user, const Char *section,
+                   const Char *name, const Char *value)
+{
+    Configuration* conf = (Configuration*)user;
+
+    if (strequal(section, "Arbiter")) {
+        if (strequal(name, "MessageReadGap")) {
+            conf->message_read_gap = atoi(value);
+        } else if (strequal(name, "GruntTimeToDie")) {
+            conf->grunt_time_to_die = atoi(value);
+        } else if (strequal(name, "GruntResponseWaitTime")) {
+            conf->grunt_response_wait_time = strtoul(value, NULL, 10);
+        } else {
+            return 0;  /* unknown section/name, error */
+        }
+    } else if (strequal(section, "Kafka")) {
+        if (strequal(name, "Address")) {
+            conf->kafka_address = strdup(value);
+        } else {
+            return 0;  /* unknown section/name, error */
+        }
+    }
+
+    return 1;
+}
+
+# if defined(COMPILER_CLANG)
+#  pragma clang diagnostic pop
+# endif

resource_system/src/arbiter/grunt_manager.c

+/*
+ * Creator: Naman Dixit
+ * Notice: © Copyright 2020 Naman Dixit
+ */
+
+typedef struct Grunt {
+    Char *id;
+    Sint memory;
+    B32 rejoin_asked;
+    Sint time_to_die; // in ms
+} Grunt;
+
+typedef struct Grunt_Survey {
+    Char **grunt_ids;
+    U16 *ports;
+    U64 milli_passed;
+    U64 milli_last;
+    Char *resource_id;
+} Grunt_Survey;
+
+typedef struct Grunt_Tracker {
+    Grunt *grunts;
+    Size *free_list;
+    Hash_Table map;
+} Grunt_Tracker;
+
+internal_function
+void gruntTrackBegin (Grunt_Tracker *t, Grunt g)
+{
+    if (t->grunts == NULL) {
+        t->map = htCreate(0);
+        sbufAdd(t->grunts, (Grunt){0}); // SInce 0 index out of hash table will be invalid
+    }
+
+    Size insertion_index = 0;
+
+    if (sbufElemin(t->free_list) > 0) {
+        t->grunts[t->free_list[0]] = g;
+        insertion_index = t->free_list[0];
+        sbufUnsortedRemove(t->free_list, 0);
+    } else {
+        sbufAdd(t->grunts, g);
+        insertion_index = sbufElemin(t->grunts) - 1;
+    }
+
+    htInsert(&t->map, hashString(g.id), insertion_index);
+}
+
+internal_function
+void gruntTrackEnd (Grunt_Tracker *t, Char *grunt_id)
+{
+    U64 hash = hashString(grunt_id);
+    Size index = htLookup(&t->map, hash);
+    sbufAdd(t->free_list, index);
+    free(t->grunts[index].id);
+    t->grunts[index] = (Grunt){0};
+    htRemove(&t->map, hash);
+}

inih @ 35121712

+Subproject commit 351217124ddb3e3fe2b982248a04c672350bb0af

resource_system/src/grunt/conf.c

+/*
+ * Creator: Naman Dixit
+ * Notice: © Copyright 2020 Naman Dixit
+ */
+
+typedef struct Configuration {
+    Char *kafka_address;
+
+    U64 heartbeat_gap;
+    Sint message_read_gap;
+} Configuration;
+
+# if defined(COMPILER_CLANG)
+#  pragma clang diagnostic push
+#   pragma clang diagnostic ignored "-Wincompatible-pointer-types-discards-qualifiers"
+# endif
+
+internal_function
+Sint confCallback (void* user, const Char *section,
+                   const Char *name, const Char *value)
+{
+    Configuration* conf = (Configuration*)user;
+
+    if (strequal(section, "Grunt")) {
+        if (strequal(name, "MessageReadGap")) {
+            conf->message_read_gap = atoi(value);
+        } else if (strequal(name, "HeartbeatGap")) {
+            conf->heartbeat_gap = strtoul(value, NULL, 10);
+        } else {
+            return 0;  /* unknown section/name, error */
+        }
+    } else if (strequal(section, "Kafka")) {
+        if (strequal(name, "Address")) {
+            conf->kafka_address = strdup(value);
+        } else {
+            return 0;  /* unknown section/name, error */
+        }
+    }
+
+    return 1;
+}
+
+# if defined(COMPILER_CLANG)
+#  pragma clang diagnostic pop
+# endif

resource_system/src/grunt/instrument_docker.c

@@ -7,12 +7,22 @@ internal_function
 noreturn
 void* dockerProcessLoop (void *arg)
 {
-    unused_variable(arg);
-
     pthread_setcanceltype (PTHREAD_CANCEL_DEFERRED, NULL);
 
     while (true) {
         // TODO(naman): Get data
+        Char *data_cmd = NULL;
+        sbufPrint(data_cmd, "docker stats %s", (Char*)arg);
+        FILE* data_file = popen(data_cmd, "r");
+
+
+        fseek(data_file, 0, SEEK_END);
+        long size = ftell(data_file);
+        fseek(data_file, 0, SEEK_SET);
+        Char *data = calloc((Size)size + 1, sizeof(*data));
+        fread(data, 1, (Size)size + 1, data_file);
+        fclose(data_file);
+
         Char *json = NULL;
 
         Char *output = NULL;

resource_system/src/grunt/thread_manager.c

@@ -5,7 +5,8 @@
 
 typedef struct Thread {
     pthread_t thread;
-    Char *id;
+    Char *entity_id;
+    Char *resource_id;
 } Thread;
 
 typedef struct Thread_Tracker {
@@ -33,7 +34,7 @@ void threadTrackBegin (Thread_Tracker *t, Thread th)
         insertion_index = sbufElemin(t->threads) - 1;
     }
 
-    htInsert(&t->map, hashString(th.id), insertion_index);
+    htInsert(&t->map, hashString(th.resource_id), insertion_index);
 }
 
 internal_function
@@ -41,7 +42,8 @@ void threadTrackEnd (Thread_Tracker *t, Char *thread_id)
 {
     Size index = htLookup(&t->map, hashString(thread_id));
     sbufAdd(t->free_list, index);
-    free(t->threads[index].id);
+    free(t->threads[index].resource_id);
+    free(t->threads[index].entity_id);
     t->threads[index] = (Thread){0};
     htRemove(&t->map, index);
 }
@@ -61,16 +63,18 @@ void* tmProcessLoop (void *arg)
         switch (command.kind) {
             case Thread_Manager_Command_DOCKER_CREATE: {
                 pthread_t thread;
-                pthread_create(&thread, NULL, &dockerProcessLoop, NULL);
-                threadTrackBegin(&tt, (Thread){.id = command.id, .thread = thread});
+                pthread_create(&thread, NULL, &dockerProcessLoop, command.entity_id);
+                threadTrackBegin(&tt, (Thread){.entity_id = command.entity_id,
+                            .resource_id = command.resource_id,
+                            .thread = thread});
             } break;
 
             case Thread_Manager_Command_DOCKER_DESTROY: {
-                Size index = htLookup(&tt.map, hashString(command.id));
+                Size index = htLookup(&tt.map, hashString(command.resource_id));
                 pthread_t thread = tt.threads[index].thread;
                 pthread_cancel(thread);
                 pthread_join(thread, NULL);
-                threadTrackEnd(&tt, command.id);
+                threadTrackEnd(&tt, command.resource_id);
             } break;
 
             case Thread_Manager_Command_NONE: {

resource_system/src/test/test.c

@@ -85,16 +85,19 @@ int main(int argc, char** argv)
 
     Sint id = (Sint)time(NULL);
     sbufPrint(output, "{\n\"resource_id\": \"%d\"", id);
+    sbufPrint(output, ",\n\"timestamp\": %d", id);
     sbufPrint(output, ",\n\"memory\": %d", memory_required);
     sbufPrint(output, "\n}\n");
 
     printf("Sending to Arbiter:\n%s\n", cJSON_Print(cJSON_Parse(output)));
 
+    printf("%ld\n", time(0));
     if (output != NULL) {
         if (!kafkaWrite(kafka.writer, "REQUEST_DM_2_RM", "rm_test", output)) {
             return -1;
         }
     }
+    printf("%ld\n", time(0));
 
     rd_kafka_message_t *kafka_message_read = rd_kafka_consumer_poll(kafka.reader, 10);
     while (true) {