RAMCloud Offload, Smit MTP

Smit_MTP_RamCloud_Replication_Offload/.gitignore

+*.gch
+*.out
+*.o
+*code-workspace
+.vscode
+.*~
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/ErrMssg.h

+#ifndef ERR_MSSG_H
+#define ERR_MSSG_H
+
+#define D(x) do{x;}while(0)
+
+#endif
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/Offloaded/RDMA/config.conf

+TRANSPORT_TYPE=RDMA_RC
+NUM_THREADS=8
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/Offloaded/TCP/config_file.conf

+TRANSPORT_TYPE=TCP
+NUM_THREAD=8
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/Readme.txt

+###############_Work in Progress_###############
+
+Things to look out for: cpu speed difference between host and nic can cause an issue while programming in RDMA
+
+Current Issue:  Memory leak on exit of connection
+                Keeps creating buffers for dead connection messages (why is select even returning dead connections?)
+Fix to be implemented:  Clean closing of connections
+
+Compilation commands:
+
+NIC: 
+g++ -g integrated_nic.cc include/common.cc transport_api/transport_config.cc config/read_config.cc include/connection_pool.cc include/thread_pool.cc include/threadsafe_queue.cc include/log.cc include/thread_functions.cc include/dispatcher.cc include/client_functions.cc include/hash.cc include/Buffer.cc include/cli_api.cc -libverbs -lpthread -mcmodel=small
+
+SERVER:
+g++ -g integrated_server.cc include/common.cc transport_api/transport_config.cc config/read_config.cc include/connection_pool.cc include/thread_pool.cc include/threadsafe_queue.cc include/log.cc include/thread_functions.cc include/cli_api.cc include/dispatcher.cc include/client_functions.cc include/hash.cc include/Buffer.cc -libverbs -lpthread -mcmodel=medium
+
+
+CLIENT:
+g++ -g threaded_client1.cc include/common.cc transport_api/transport_config.cc config/read_config.cc include/connection_pool.cc include/thread_pool.cc include/threadsafe_queue.cc include/log.cc include/thread_functions.cc include/dispatcher.cc include/client_functions.cc include/cli_api.cc include/hash.cc include/Buffer.cc -libverbs -lpthread  -mcmodel=medium

Smit_MTP_RamCloud_Replication_Offload/ReplicaManager.h

+
+class ReplicaManager {
+    public:
+        
+
+    private: 
+
+}
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/client.cpp

+#include <iostream>
+#include <sys/time.h>
+#include <infiniband/verbs.h>
+
+#include "rdma_states.hpp"
+#include "rdma_helper.hpp"
+#include "metadata.hpp"
+#include "transport_helper.hpp"
+
+const short int MODE = 0;
+char  *SERVER_HOST = "192.168.200.20";
+char *SERVER_NIC = "192.168.200.21";
+const int NIC_PORT = 8090;
+const double err_fraction = 0.5;
+const short int dev_num = 0;
+
+int main(int argc, char * argv[]) {
+    char *temp = (char *) malloc(128);
+    struct resource_base *base;
+    base = (struct resource_base*) malloc(sizeof(struct resource_base));
+    init_resources(base);
+    //base->server_name = SERVER_NIC;
+    base->ib_port = IB_PORT;
+    base->gid_idx = GID_IDX;
+    base->port = NIC_PORT;
+
+    open_dev(base, dev_num);
+    allocate_pd(base);
+    register_mr(base);
+    init_cq(base);
+    init_qp(base);
+
+    struct ibv_port_attr port_attr;
+    if(ibv_query_port(base->ctx, base->ib_port, &port_attr))
+	    D(err_msg("ibv_query_gid", true, base));
+    if(port_attr.state != IBV_PORT_ACTIVE)
+	    D(err_msg("IB PORT NOT ACTIVE", true, base));
+    base->port_attr = &port_attr;
+
+    union ibv_gid my_gid;
+    if(ibv_query_gid(base->ctx, base->ib_port, base->gid_idx, &my_gid))
+	    D(err_msg("ibv_query_gid", true, base));
+    memcpy(base->local_conn->gid, &my_gid, 16);
+
+    connect_qp(base);
+    strcpy(base->mr_buf_addr, "yo");
+    sync_remote_qp(base, "R", temp, 1);
+    post_send(base, IBV_WR_RDMA_WRITE);
+    
+    
+    //base->mr_buf_addr = (char *) malloc(base->mr_size);
+    //strcpy(base->mr_buf_addr, "Hi from client\0");
+
+    //sock_connect(base);
+
+    //sync_remote_qp(base, base->mr_buf_addr, temp, 15);
+    
+    union object test_obj;
+    union object ret_obj;
+    memset(&test_obj, 0, sizeof(test_obj));
+    memset(&ret_obj, 0, sizeof(ret_obj));
+    test_obj.obj.key = 1;
+    test_obj.obj.value[0] = 'T';
+    test_obj.obj.version = 1;
+    test_obj.obj.status = STATUS_OK;
+
+    struct timeval temp_time;
+    double snd_ts, rcvd_ts;
+    double avg = 0;
+    int err_cnt = 0;
+    int succ_cnt = 0;
+    // for(int i=0; i<1000; i++) {
+    //     gettimeofday(&temp_time, NULL);
+    //     //time in ms
+    //     snd_ts = ((double)temp_time.tv_sec*1000.0) + ((double)temp_time.tv_usec/1000.0);
+    //     test_obj.obj.send_ts = snd_ts;
+        
+    //     sync_remote_qp(base, "W", temp, 1);
+    //     sync_remote_qp(base, (char *)&test_obj, (char *)&ret_obj, sizeof(test_obj));
+    //     gettimeofday(&temp_time, NULL);
+
+    //     rcvd_ts = ((double)temp_time.tv_sec*1000.0) + ((double)temp_time.tv_usec/1000.0);
+    //     avg += (rcvd_ts - snd_ts);
+        
+    // }
+    enum Status ret_status;
+    for(int i=0; i<cache_meta_size; i++) {
+        test_obj.obj.key = i;
+        gettimeofday(&temp_time, NULL);
+        snd_ts = ((double)temp_time.tv_sec*1000.0) + ((double)temp_time.tv_usec/1000.0);
+        test_obj.obj.send_ts = snd_ts;
+        //send_and_check(base, (char *)&test_obj, (char *)&ret_status, sizeof(test_obj), sizeof(ret_status));
+	memcpy((void*)base->mr_buf_addr, (void*)&test_obj, sizeof(test_obj));
+	send_obj(base, "R", 1);
+	sync_remote_qp(base, "T", temp, 1);
+	read_obj(base, (char *)&ret_status, sizeof(ret_status));
+        gettimeofday(&temp_time, NULL);
+        rcvd_ts = ((double)temp_time.tv_sec*1000.0) + ((double)temp_time.tv_usec/1000.0);
+        avg += (rcvd_ts - snd_ts);
+        if(ret_status == STATUS_WRONG_VERSION) err_cnt++;
+        else succ_cnt++; 
+    }
+    std::cout<<"Errored requests: "<<err_cnt<<std::endl;
+    std::cout<<"Successful requests: "<<succ_cnt<<std::endl;
+    std::cout<<"Avg. RTT: "<<avg/(double)cache_meta_size<<" ms"<<std::endl;
+    // if(ret_obj.obj.status == STATUS_WRONG_VERSION) 
+    //     std::cout<<"Returned with wrong version status"<<std::endl;
+
+    cleanup(base);
+    return 0;
+    
+}

Smit_MTP_RamCloud_Replication_Offload/config/client_config.conf

+DEBUG=TRUE
+ANALYZE=TRUE
+INTERACTIVE_MODE=FALSE
+TRANSPORT_TYPE=RDMA_RC
+NUM_THREADS=0
+CONN_PORT=8888
+MAX_PACKET_SIZE_MBYTES=4;
+RDMA_MR_SIZE_MBYTES=1
+RDMA_MTU_SIZE=512
+RDMA_MIN_RNR_TIMER=12
+RDMA_TIMEOUT=12
+RDMA_CQ_POLL_TIMEOUT_MS=5
+RDMA_RETRY_CNT=4
+RDMA_IB_PORT=1
+RDMA_GID_IDX=1
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/config/config_parameters.hpp

+#ifndef __CONFIG_PARAMETERS_H__
+#define __CONFIG_PARAMETERS_H__
+
+#include <unordered_map>
+#include <string>
+
+#include "../transport_api/transport_config.hpp"
+
+enum params {
+    DEBUG,
+    ANALYZE,
+    INTERACTIVE_MODE,
+    TRANSPORT_TYPE,
+    NUM_THREADS,
+    CONN_PORT,
+    NUM_REPLICAS,
+    MAX_PACKET_SIZE_BYTES,
+    MAX_PACKET_SIZE_MBYTES,
+    RDMA_MR_SIZE_BYTES,
+    RDMA_MR_SIZE_MBYTES, //max 8
+    RDMA_MTU_SIZE_BYTES, //one of 256,512,1024,2048,4096
+    RDMA_MIN_RNR_TIMER,
+    RDMA_TIMEOUT,
+    RDMA_CQ_POLL_TIMEOUT_MS,
+    RDMA_RETRY_CNT,
+    RDMA_IB_PORT,
+    RDMA_GID_IDX,
+    ARRIVAL_RATE
+};
+
+const std::string param_strs[] = {
+    "DEBUG",
+    "ANALYZE",
+    "INTERACTIVE_MODE",
+    "TRANSPORT_TYPE",
+    "NUM_THREADS",
+    "CONN_PORT",
+    "NUM_REPLICAS",
+    "MAX_PACKET_SIZE_BYTES",
+    "MAX_PACKET_SIZE_MBYTES",
+    "RDMA_MR_SIZE_BYTES",
+    "RDMA_MR_SIZE_MBYTES",
+    "RDMA_MTU_SIZE_BYTES",
+    "RDMA_MIN_RNR_TIMER",
+    "RDMA_TIMEOUT",
+    "RDMA_CQ_POLL_TIMEOUT_MS",
+    "RDMA_RETRY_CNT",
+    "RDMA_IB_PORT",
+    "RDMA_GID_PORT",
+    "ARRIVAL_RATE"
+};
+
+const int num_params = 17;
+
+#endif
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/config/nic_config.conf

+DEBUG=TRUE
+ANALYZE=TRUE
+TRANSPORT_TYPE=RDMA_RC
+NUM_THREADS=3
+CONN_PORT=8888
+MAX_PACKET_SIZE_MBYTES=4;
+RDMA_MR_SIZE_MBYTES=1
+RDMA_MTU_SIZE=512
+RDMA_MIN_RNR_TIMER=12
+RDMA_TIMEOUT=12
+RDMA_CQ_POLL_TIMEOUT_MS=5
+RDMA_RETRY_CNT=4
+RDMA_IB_PORT=1
+RDMA_GID_IDX=1
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/config/read_config.cc

+#ifndef __READ_CONFIG_CC__
+#define __READ_CONFIG_CC__
+
+#include <chrono>
+#include <iostream>
+#include <string>
+#include <iterator>
+#include <vector>
+#include <fstream>
+#include <infiniband/verbs.h>
+
+#include "../transport_api/transport_config.hpp"
+#include "read_config.hpp"
+
+std::unordered_map<std::string, enum Transport_Type> transport_type_map = {
+    {"TCP",         TCP_IP_TRANSPORT},
+    {"UDP",         UDP_TRANSPORT},
+    {"RDMA_RC",     RDMA_RC_TRANSPORT},
+    {"RDMA_UC",     RDMA_UC_TRANSPORT},
+};
+
+std::unordered_map<std::string, enum params> param_map = {
+    {"DEBUG",                       DEBUG},
+    {"ANALYZE",                     ANALYZE},
+    {"INTERACTIVE_MODE",            INTERACTIVE_MODE},
+    {"TRANSPORT_TYPE",              TRANSPORT_TYPE},
+    {"NUM_THREADS",                 NUM_THREADS},
+    {"CONN_PORT",                   CONN_PORT},
+    {"NUM_REPLICAS",                NUM_REPLICAS},
+    {"MAX_PACKET_SIZE_BYTES",       MAX_PACKET_SIZE_BYTES},
+    {"MAX_PACKET_SIZE_MBYTES",      MAX_PACKET_SIZE_MBYTES},
+    {"RDMA_MR_SIZE_BYTES",          RDMA_MR_SIZE_BYTES},
+    {"RDMA_MR_SIZE_MBYTES",         RDMA_MR_SIZE_MBYTES},
+    {"RDMA_MTU_SIZE_BYTES",         RDMA_MTU_SIZE_BYTES},
+    {"RDMA_MIN_RNR_TIMER",          RDMA_MIN_RNR_TIMER},
+    {"RDMA_TIMEOUT",                RDMA_TIMEOUT},
+    {"RDMA_CQ_POLL_TIMEOUT_MS",     RDMA_CQ_POLL_TIMEOUT_MS},
+    {"RDMA_RETRY_CNT",              RDMA_RETRY_CNT},
+    {"RDMA_IB_PORT",                RDMA_IB_PORT},
+    {"RDMA_GID_IDX",                RDMA_GID_IDX},
+    {"ARRIVAL_RATE",                ARRIVAL_RATE},
+};
+
+std::string strip_whitespaces(std::string str) {
+    std::string t;
+    std::string::iterator it = str.begin();
+    while(it!=str.end() && (*it)==' ') it++;
+    while(it!=str.end() && (*it)!=' ') {
+        t.push_back((*it));
+        it++;
+    }
+    return t;
+}
+
+Params::Params() {
+    //pass
+}
+
+Params::Params(std::string f) {
+    this->filename = f;
+    this->debug = false;
+    this->analyze = false;
+    this->interactive_mode = true;
+    this->transport_type = TCP_IP_TRANSPORT;
+    this->num_threads = 0;
+    this->conn_port = 8080;
+    this->num_replicas = 0;
+    this->max_packet_size_bytes = 0;
+    this->rdma_mr_size_bytes = 0;
+    this->rdma_mtu_size_bytes = IBV_MTU_512;
+    this->rdma_min_rnr_timer = 0;
+    this->rdma_timeout = 0;
+    this->rdma_cq_poll_timeout_ms = std::chrono::duration<double>{0.0};
+    this->rdma_retry_cnt = 0;
+    this->rdma_ib_port = 0;
+    this->rdma_gid_idx = 0;
+    this->arrival_rate = 0.0;
+}
+
+//debug functions
+void Params::print_map() {
+    for(auto x: this->param_val_map) {
+        std::cout<<x.first<<" : "<<x.second<<std::endl;
+    }
+    return;
+}
+
+void Params::print_vals() {
+    std::cout<<"DEBUG: ";
+    if(this->debug) std::cout<<"TRUE"<<std::endl;
+    else std::cout<<"FALSE"<<std::endl;
+    std::cout<<"ANALYZE: ";
+    if(this->analyze) std::cout<<"TRUE"<<std::endl;
+    else std::cout<<"FALSE"<<std::endl;
+    std::cout<<"MODE: ";
+    if(this->interactive_mode) std::cout<<"INTERACTIVE MODE"<<std::endl;
+    else std::cout<<"BATCH MODE"<<std::endl;
+    std::cout<<"Transport: ";
+    switch(this->transport_type) {
+        case TCP_IP_TRANSPORT:
+            std::cout<<"TCP"<<std::endl;
+            break;
+        case UDP_TRANSPORT:
+            std::cout<<"UDP"<<std::endl;
+            break;
+        case RDMA_RC_TRANSPORT:
+            std::cout<<"RDMA RC"<<std::endl;
+            break;
+        case RDMA_UC_TRANSPORT:
+            std::cout<<"RDMA UC"<<std::endl;
+            break;
+    }
+    std::cout<<"Num thread: "<<this->num_threads<<std::endl;
+    std::cout<<"Conn Port: "<<this->conn_port<<std::endl;
+    std::cout<<"Num Replicas: "<<this->num_replicas<<std::endl;
+    std::cout<<"Max Packet Size (Bytes): "<<this->max_packet_size_bytes<<std::endl;
+    std::cout<<"RDMA MR size (Bytes): "<<this->rdma_mr_size_bytes<<std::endl;
+    std::cout<<"RDMA MTU size (Bytes): ";
+    switch (this->rdma_mtu_size_bytes) {
+        case IBV_MTU_256:
+            std::cout<<"256B"<<std::endl;
+            break;
+        case IBV_MTU_512:
+            std::cout<<"512B"<<std::endl;
+            break;
+        case IBV_MTU_1024:
+            std::cout<<"1024B"<<std::endl;
+            break;
+        case IBV_MTU_2048:
+            std::cout<<"2048B"<<std::endl;
+            break;
+        case IBV_MTU_4096:
+            std::cout<<"4096B"<<std::endl;
+            break;
+    }
+    std::cout<<"RDMA min rnr timer: "<<this->rdma_min_rnr_timer<<std::endl;
+    std::cout<<"RDMA timeout: "<<this->rdma_timeout<<std::endl;
+    std::cout<<"RDMA CQ poll timeout: "<<this->rdma_cq_poll_timeout_ms.count()<<std::endl;
+    std::cout<<"RDMA IB Port: "<<this->rdma_ib_port<<std::endl;
+    std::cout<<"RDMA GID IDX: "<<this->rdma_gid_idx<<std::endl;
+    std::cout<<"ARRIVAL RATE: "<<this->arrival_rate<<" requests/second"<<std::endl;
+    return;
+}
+
+// Just Reads config file
+// to fill an internal map
+void Params::read_config_file() {
+    std::string line;
+    std::ifstream config_file;
+    config_file.open(this->filename);
+    int pos;
+    while(getline(config_file, line)) {
+        if(line[0]=='\n' || 
+            (line[0]=='/' && line[1]=='/')) continue;
+        pos = line.find_first_of('=');
+        if(pos==std::string::npos) continue;
+        this->param_val_map[line.substr(0, pos)] = line.substr(pos+1, line.size()-pos);
+    }
+    return;
+}
+
+// Fills parameter variables
+// using internal parameter map
+void Params::fill_params() {
+    std::string t1, t2;
+    double tmp;
+    enum params param_type;
+    if(this->param_val_map.empty()) {
+        //error
+        return;
+    }
+
+    for(auto p: this->param_val_map) {
+        t1 = strip_whitespaces(p.first);
+        t2 = strip_whitespaces(p.second);
+        if(param_map.count(t1)==0) continue;
+        param_type = param_map[t1];
+        switch(param_type) {
+            case DEBUG:
+                if(t2.compare("TRUE") == 0) {
+                    this->debug = true;
+                }
+                else {
+                    this->debug = false;
+                }
+                break;
+            case ANALYZE:
+                if(t2.compare("TRUE") == 0) {
+                    this->analyze = true;
+                }
+                else {
+                    this->analyze = false;
+                }
+                break;
+            case INTERACTIVE_MODE:
+                if(t2.compare("TRUE") == 0) {
+                    this->interactive_mode = true;
+                }
+                else {
+                    this->interactive_mode = false;
+                }
+            case TRANSPORT_TYPE:
+                this->transport_type = transport_type_map[t2];
+                break;
+            case NUM_THREADS:
+                this->num_threads = std::stoi(t2);
+                break;
+            case CONN_PORT:
+                this->conn_port = std::stoi(t2);
+                break;
+            case NUM_REPLICAS:
+                this->num_replicas = std::stoi(t2);
+                break;
+            case MAX_PACKET_SIZE_BYTES:
+                this->max_packet_size_bytes = std::stoi(t2);
+                break;
+            case MAX_PACKET_SIZE_MBYTES:
+                this->max_packet_size_bytes = (int) (std::stoi(t2)*(1<<20));
+                break;
+            case RDMA_MR_SIZE_BYTES:
+                this->rdma_mr_size_bytes = (size_t) (std::stoi(t2));
+                break;
+            case RDMA_MR_SIZE_MBYTES:
+                this->rdma_mr_size_bytes = (size_t) (std::stoi(t2)*(1<<20));
+                break;
+            case RDMA_MTU_SIZE_BYTES:
+                switch(std::stoi(t2)) {
+                    case 256:
+                        this->rdma_mtu_size_bytes = IBV_MTU_256;
+                        break;
+                    case 512:
+                        this->rdma_mtu_size_bytes = IBV_MTU_512;
+                        break;
+                    case 1024:
+                        this->rdma_mtu_size_bytes = IBV_MTU_1024;
+                        break;
+                    case 2048:
+                        this->rdma_mtu_size_bytes = IBV_MTU_2048;
+                        break;
+                    case 4096:
+                        this->rdma_mtu_size_bytes = IBV_MTU_4096;
+                        break;
+                    default:
+                        //error
+                        break;
+                }
+                break;
+            case RDMA_MIN_RNR_TIMER:
+                this->rdma_min_rnr_timer = std::stoi(t2);
+                break;
+            case RDMA_TIMEOUT:
+                this->rdma_timeout = std::stoi(t2);
+                break;
+            case RDMA_CQ_POLL_TIMEOUT_MS:
+                tmp = std::stod(t2);
+                tmp *= 1e-3; //convert to ms
+                this->rdma_cq_poll_timeout_ms = std::chrono::duration<double>{tmp};
+                break;
+            case RDMA_RETRY_CNT:
+                this->rdma_retry_cnt = std::stoi(t2);
+                break;
+            case RDMA_IB_PORT:
+                this->rdma_ib_port = std::stoi(t2);
+                break;
+            case RDMA_GID_IDX:
+                this->rdma_gid_idx = std::stoi(t2);
+                break;
+            case ARRIVAL_RATE:
+                this->arrival_rate = std::stod(t2);
+            default:
+                //error
+                break;
+        }
+    }
+
+    return;
+}
+
+// Reads config file and
+// Fills parameter variables
+void Params::read_params() {
+    this->read_config_file();
+    this->fill_params();
+    return;
+}
+
+
+#endif
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/config/read_config.hpp

+#ifndef __READ_CONFIG_H__
+#define __READ_CONFIG_H__
+
+#include <chrono>
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <infiniband/verbs.h>
+
+#include "config_parameters.hpp"
+#include "../transport_api/transport_config.hpp"
+
+class Params {
+    private:
+        std::string filename;
+        std::vector<std::string> param_lines;
+        std::unordered_map<std::string, std::string> param_val_map;
+
+    public:
+        bool debug;
+        bool analyze;
+        bool interactive_mode;
+        enum Transport_Type transport_type;
+        int num_threads;
+        int conn_port;
+        int num_replicas;
+        int max_packet_size_bytes;
+        size_t rdma_mr_size_bytes;
+        enum ibv_mtu rdma_mtu_size_bytes;
+        int rdma_min_rnr_timer;
+        int rdma_timeout;
+        std::chrono::duration<double> rdma_cq_poll_timeout_ms;
+        int rdma_retry_cnt;
+        int rdma_ib_port;
+        int rdma_gid_idx;
+        double arrival_rate;
+
+        Params();
+        Params(std::string f);
+        
+        //debug functions
+        void print_map();
+        void print_vals();
+
+        //required functions
+        void read_params();
+        void read_config_file();
+        void fill_params();
+};
+
+
+std::vector<std::string> get_param_lines(char *filename);
+
+enum params check_token(std::string token);
+
+#endif
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/config/read_configTest.cc

+#include <iostream>
+#include "read_config.hpp"
+
+using namespace std;
+
+int main() {
+
+    Params p("config.conf");
+    p.read_params();
+    p.print_vals();
+
+    return 0;
+
+
+}
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/config/server_config.conf

+DEBUG=TRUE
+ANALYZE=TRUE
+TRANSPORT_TYPE=RDMA_RC
+NUM_THREADS=3
+CONN_PORT=8888
+MAX_PACKET_SIZE_MBYTES=4;
+RDMA_MR_SIZE_MBYTES=1
+RDMA_MTU_SIZE=512
+RDMA_MIN_RNR_TIMER=12
+RDMA_TIMEOUT=12
+RDMA_CQ_POLL_TIMEOUT_MS=5
+RDMA_RETRY_CNT=4
+RDMA_IB_PORT=1
+RDMA_GID_IDX=1
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/config/threaded_client.conf

+DEBUG=TRUE
+ANALYZE=TRUE
+INTERACTIVE_MODE=FALSE
+TRANSPORT_TYPE=RDMA_RC
+NUM_THREADS=0
+CONN_PORT=8888
+MAX_PACKET_SIZE_MBYTES=4;
+RDMA_MR_SIZE_MBYTES=1
+RDMA_MTU_SIZE=512
+RDMA_MIN_RNR_TIMER=12
+RDMA_TIMEOUT=12
+RDMA_CQ_POLL_TIMEOUT_MS=5
+RDMA_RETRY_CNT=4
+RDMA_IB_PORT=1
+RDMA_GID_IDX=1
+ARRIVAL_RATE=80
\ No newline at end of file

Smit_MTP_RamCloud_Replication_Offload/experiments/client.cpp

+#include <iostream>
+#include <stdio.h>
+#include <vector>
+#include <unistd.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/socket.h>