Commit d2d47b86 authored by Smit Gangurde's avatar Smit Gangurde

RAMCloud Offload, Smit MTP

parent 33075e8b
*.gch
*.out
*.o
*code-workspace
.vscode
.*~
\ No newline at end of file
#ifndef ERR_MSSG_H
#define ERR_MSSG_H
#define D(x) do{x;}while(0)
#endif
\ No newline at end of file
TRANSPORT_TYPE=RDMA_RC
NUM_THREADS=8
\ No newline at end of file
TRANSPORT_TYPE=TCP
NUM_THREAD=8
\ No newline at end of file
###############_Work in Progress_###############
Things to look out for: cpu speed difference between host and nic can cause an issue while programming in RDMA
Current Issue: Memory leak on exit of connection
Keeps creating buffers for dead connection messages (why is select even returning dead connections?)
Fix to be implemented: Clean closing of connections
Compilation commands:
NIC:
g++ -g integrated_nic.cc include/common.cc transport_api/transport_config.cc config/read_config.cc include/connection_pool.cc include/thread_pool.cc include/threadsafe_queue.cc include/log.cc include/thread_functions.cc include/dispatcher.cc include/client_functions.cc include/hash.cc include/Buffer.cc include/cli_api.cc -libverbs -lpthread -mcmodel=small
SERVER:
g++ -g integrated_server.cc include/common.cc transport_api/transport_config.cc config/read_config.cc include/connection_pool.cc include/thread_pool.cc include/threadsafe_queue.cc include/log.cc include/thread_functions.cc include/cli_api.cc include/dispatcher.cc include/client_functions.cc include/hash.cc include/Buffer.cc -libverbs -lpthread -mcmodel=medium
CLIENT:
g++ -g threaded_client1.cc include/common.cc transport_api/transport_config.cc config/read_config.cc include/connection_pool.cc include/thread_pool.cc include/threadsafe_queue.cc include/log.cc include/thread_functions.cc include/dispatcher.cc include/client_functions.cc include/cli_api.cc include/hash.cc include/Buffer.cc -libverbs -lpthread -mcmodel=medium
class ReplicaManager {
public:
private:
}
\ No newline at end of file
#include <iostream>
#include <sys/time.h>
#include <infiniband/verbs.h>
#include "rdma_states.hpp"
#include "rdma_helper.hpp"
#include "metadata.hpp"
#include "transport_helper.hpp"
const short int MODE = 0;
char *SERVER_HOST = "192.168.200.20";
char *SERVER_NIC = "192.168.200.21";
const int NIC_PORT = 8090;
const double err_fraction = 0.5;
const short int dev_num = 0;
int main(int argc, char * argv[]) {
char *temp = (char *) malloc(128);
struct resource_base *base;
base = (struct resource_base*) malloc(sizeof(struct resource_base));
init_resources(base);
//base->server_name = SERVER_NIC;
base->ib_port = IB_PORT;
base->gid_idx = GID_IDX;
base->port = NIC_PORT;
open_dev(base, dev_num);
allocate_pd(base);
register_mr(base);
init_cq(base);
init_qp(base);
struct ibv_port_attr port_attr;
if(ibv_query_port(base->ctx, base->ib_port, &port_attr))
D(err_msg("ibv_query_gid", true, base));
if(port_attr.state != IBV_PORT_ACTIVE)
D(err_msg("IB PORT NOT ACTIVE", true, base));
base->port_attr = &port_attr;
union ibv_gid my_gid;
if(ibv_query_gid(base->ctx, base->ib_port, base->gid_idx, &my_gid))
D(err_msg("ibv_query_gid", true, base));
memcpy(base->local_conn->gid, &my_gid, 16);
connect_qp(base);
strcpy(base->mr_buf_addr, "yo");
sync_remote_qp(base, "R", temp, 1);
post_send(base, IBV_WR_RDMA_WRITE);
//base->mr_buf_addr = (char *) malloc(base->mr_size);
//strcpy(base->mr_buf_addr, "Hi from client\0");
//sock_connect(base);
//sync_remote_qp(base, base->mr_buf_addr, temp, 15);
union object test_obj;
union object ret_obj;
memset(&test_obj, 0, sizeof(test_obj));
memset(&ret_obj, 0, sizeof(ret_obj));
test_obj.obj.key = 1;
test_obj.obj.value[0] = 'T';
test_obj.obj.version = 1;
test_obj.obj.status = STATUS_OK;
struct timeval temp_time;
double snd_ts, rcvd_ts;
double avg = 0;
int err_cnt = 0;
int succ_cnt = 0;
// for(int i=0; i<1000; i++) {
// gettimeofday(&temp_time, NULL);
// //time in ms
// snd_ts = ((double)temp_time.tv_sec*1000.0) + ((double)temp_time.tv_usec/1000.0);
// test_obj.obj.send_ts = snd_ts;
// sync_remote_qp(base, "W", temp, 1);
// sync_remote_qp(base, (char *)&test_obj, (char *)&ret_obj, sizeof(test_obj));
// gettimeofday(&temp_time, NULL);
// rcvd_ts = ((double)temp_time.tv_sec*1000.0) + ((double)temp_time.tv_usec/1000.0);
// avg += (rcvd_ts - snd_ts);
// }
enum Status ret_status;
for(int i=0; i<cache_meta_size; i++) {
test_obj.obj.key = i;
gettimeofday(&temp_time, NULL);
snd_ts = ((double)temp_time.tv_sec*1000.0) + ((double)temp_time.tv_usec/1000.0);
test_obj.obj.send_ts = snd_ts;
//send_and_check(base, (char *)&test_obj, (char *)&ret_status, sizeof(test_obj), sizeof(ret_status));
memcpy((void*)base->mr_buf_addr, (void*)&test_obj, sizeof(test_obj));
send_obj(base, "R", 1);
sync_remote_qp(base, "T", temp, 1);
read_obj(base, (char *)&ret_status, sizeof(ret_status));
gettimeofday(&temp_time, NULL);
rcvd_ts = ((double)temp_time.tv_sec*1000.0) + ((double)temp_time.tv_usec/1000.0);
avg += (rcvd_ts - snd_ts);
if(ret_status == STATUS_WRONG_VERSION) err_cnt++;
else succ_cnt++;
}
std::cout<<"Errored requests: "<<err_cnt<<std::endl;
std::cout<<"Successful requests: "<<succ_cnt<<std::endl;
std::cout<<"Avg. RTT: "<<avg/(double)cache_meta_size<<" ms"<<std::endl;
// if(ret_obj.obj.status == STATUS_WRONG_VERSION)
// std::cout<<"Returned with wrong version status"<<std::endl;
cleanup(base);
return 0;
}
DEBUG=TRUE
ANALYZE=TRUE
INTERACTIVE_MODE=FALSE
TRANSPORT_TYPE=RDMA_RC
NUM_THREADS=0
CONN_PORT=8888
MAX_PACKET_SIZE_MBYTES=4;
RDMA_MR_SIZE_MBYTES=1
RDMA_MTU_SIZE=512
RDMA_MIN_RNR_TIMER=12
RDMA_TIMEOUT=12
RDMA_CQ_POLL_TIMEOUT_MS=5
RDMA_RETRY_CNT=4
RDMA_IB_PORT=1
RDMA_GID_IDX=1
\ No newline at end of file
#ifndef __CONFIG_PARAMETERS_H__
#define __CONFIG_PARAMETERS_H__
#include <unordered_map>
#include <string>
#include "../transport_api/transport_config.hpp"
enum params {
DEBUG,
ANALYZE,
INTERACTIVE_MODE,
TRANSPORT_TYPE,
NUM_THREADS,
CONN_PORT,
NUM_REPLICAS,
MAX_PACKET_SIZE_BYTES,
MAX_PACKET_SIZE_MBYTES,
RDMA_MR_SIZE_BYTES,
RDMA_MR_SIZE_MBYTES, //max 8
RDMA_MTU_SIZE_BYTES, //one of 256,512,1024,2048,4096
RDMA_MIN_RNR_TIMER,
RDMA_TIMEOUT,
RDMA_CQ_POLL_TIMEOUT_MS,
RDMA_RETRY_CNT,
RDMA_IB_PORT,
RDMA_GID_IDX,
ARRIVAL_RATE
};
const std::string param_strs[] = {
"DEBUG",
"ANALYZE",
"INTERACTIVE_MODE",
"TRANSPORT_TYPE",
"NUM_THREADS",
"CONN_PORT",
"NUM_REPLICAS",
"MAX_PACKET_SIZE_BYTES",
"MAX_PACKET_SIZE_MBYTES",
"RDMA_MR_SIZE_BYTES",
"RDMA_MR_SIZE_MBYTES",
"RDMA_MTU_SIZE_BYTES",
"RDMA_MIN_RNR_TIMER",
"RDMA_TIMEOUT",
"RDMA_CQ_POLL_TIMEOUT_MS",
"RDMA_RETRY_CNT",
"RDMA_IB_PORT",
"RDMA_GID_PORT",
"ARRIVAL_RATE"
};
const int num_params = 17;
#endif
\ No newline at end of file
DEBUG=TRUE
ANALYZE=TRUE
TRANSPORT_TYPE=RDMA_RC
NUM_THREADS=3
CONN_PORT=8888
MAX_PACKET_SIZE_MBYTES=4;
RDMA_MR_SIZE_MBYTES=1
RDMA_MTU_SIZE=512
RDMA_MIN_RNR_TIMER=12
RDMA_TIMEOUT=12
RDMA_CQ_POLL_TIMEOUT_MS=5
RDMA_RETRY_CNT=4
RDMA_IB_PORT=1
RDMA_GID_IDX=1
\ No newline at end of file
#ifndef __READ_CONFIG_CC__
#define __READ_CONFIG_CC__
#include <chrono>
#include <iostream>
#include <string>
#include <iterator>
#include <vector>
#include <fstream>
#include <infiniband/verbs.h>
#include "../transport_api/transport_config.hpp"
#include "read_config.hpp"
std::unordered_map<std::string, enum Transport_Type> transport_type_map = {
{"TCP", TCP_IP_TRANSPORT},
{"UDP", UDP_TRANSPORT},
{"RDMA_RC", RDMA_RC_TRANSPORT},
{"RDMA_UC", RDMA_UC_TRANSPORT},
};
std::unordered_map<std::string, enum params> param_map = {
{"DEBUG", DEBUG},
{"ANALYZE", ANALYZE},
{"INTERACTIVE_MODE", INTERACTIVE_MODE},
{"TRANSPORT_TYPE", TRANSPORT_TYPE},
{"NUM_THREADS", NUM_THREADS},
{"CONN_PORT", CONN_PORT},
{"NUM_REPLICAS", NUM_REPLICAS},
{"MAX_PACKET_SIZE_BYTES", MAX_PACKET_SIZE_BYTES},
{"MAX_PACKET_SIZE_MBYTES", MAX_PACKET_SIZE_MBYTES},
{"RDMA_MR_SIZE_BYTES", RDMA_MR_SIZE_BYTES},
{"RDMA_MR_SIZE_MBYTES", RDMA_MR_SIZE_MBYTES},
{"RDMA_MTU_SIZE_BYTES", RDMA_MTU_SIZE_BYTES},
{"RDMA_MIN_RNR_TIMER", RDMA_MIN_RNR_TIMER},
{"RDMA_TIMEOUT", RDMA_TIMEOUT},
{"RDMA_CQ_POLL_TIMEOUT_MS", RDMA_CQ_POLL_TIMEOUT_MS},
{"RDMA_RETRY_CNT", RDMA_RETRY_CNT},
{"RDMA_IB_PORT", RDMA_IB_PORT},
{"RDMA_GID_IDX", RDMA_GID_IDX},
{"ARRIVAL_RATE", ARRIVAL_RATE},
};
std::string strip_whitespaces(std::string str) {
std::string t;
std::string::iterator it = str.begin();
while(it!=str.end() && (*it)==' ') it++;
while(it!=str.end() && (*it)!=' ') {
t.push_back((*it));
it++;
}
return t;
}
Params::Params() {
//pass
}
Params::Params(std::string f) {
this->filename = f;
this->debug = false;
this->analyze = false;
this->interactive_mode = true;
this->transport_type = TCP_IP_TRANSPORT;
this->num_threads = 0;
this->conn_port = 8080;
this->num_replicas = 0;
this->max_packet_size_bytes = 0;
this->rdma_mr_size_bytes = 0;
this->rdma_mtu_size_bytes = IBV_MTU_512;
this->rdma_min_rnr_timer = 0;
this->rdma_timeout = 0;
this->rdma_cq_poll_timeout_ms = std::chrono::duration<double>{0.0};
this->rdma_retry_cnt = 0;
this->rdma_ib_port = 0;
this->rdma_gid_idx = 0;
this->arrival_rate = 0.0;
}
//debug functions
void Params::print_map() {
for(auto x: this->param_val_map) {
std::cout<<x.first<<" : "<<x.second<<std::endl;
}
return;
}
void Params::print_vals() {
std::cout<<"DEBUG: ";
if(this->debug) std::cout<<"TRUE"<<std::endl;
else std::cout<<"FALSE"<<std::endl;
std::cout<<"ANALYZE: ";
if(this->analyze) std::cout<<"TRUE"<<std::endl;
else std::cout<<"FALSE"<<std::endl;
std::cout<<"MODE: ";
if(this->interactive_mode) std::cout<<"INTERACTIVE MODE"<<std::endl;
else std::cout<<"BATCH MODE"<<std::endl;
std::cout<<"Transport: ";
switch(this->transport_type) {
case TCP_IP_TRANSPORT:
std::cout<<"TCP"<<std::endl;
break;
case UDP_TRANSPORT:
std::cout<<"UDP"<<std::endl;
break;
case RDMA_RC_TRANSPORT:
std::cout<<"RDMA RC"<<std::endl;
break;
case RDMA_UC_TRANSPORT:
std::cout<<"RDMA UC"<<std::endl;
break;
}
std::cout<<"Num thread: "<<this->num_threads<<std::endl;
std::cout<<"Conn Port: "<<this->conn_port<<std::endl;
std::cout<<"Num Replicas: "<<this->num_replicas<<std::endl;
std::cout<<"Max Packet Size (Bytes): "<<this->max_packet_size_bytes<<std::endl;
std::cout<<"RDMA MR size (Bytes): "<<this->rdma_mr_size_bytes<<std::endl;
std::cout<<"RDMA MTU size (Bytes): ";
switch (this->rdma_mtu_size_bytes) {
case IBV_MTU_256:
std::cout<<"256B"<<std::endl;
break;
case IBV_MTU_512:
std::cout<<"512B"<<std::endl;
break;
case IBV_MTU_1024:
std::cout<<"1024B"<<std::endl;
break;
case IBV_MTU_2048:
std::cout<<"2048B"<<std::endl;
break;
case IBV_MTU_4096:
std::cout<<"4096B"<<std::endl;
break;
}
std::cout<<"RDMA min rnr timer: "<<this->rdma_min_rnr_timer<<std::endl;
std::cout<<"RDMA timeout: "<<this->rdma_timeout<<std::endl;
std::cout<<"RDMA CQ poll timeout: "<<this->rdma_cq_poll_timeout_ms.count()<<std::endl;
std::cout<<"RDMA IB Port: "<<this->rdma_ib_port<<std::endl;
std::cout<<"RDMA GID IDX: "<<this->rdma_gid_idx<<std::endl;
std::cout<<"ARRIVAL RATE: "<<this->arrival_rate<<" requests/second"<<std::endl;
return;
}
// Just Reads config file
// to fill an internal map
void Params::read_config_file() {
std::string line;
std::ifstream config_file;
config_file.open(this->filename);
int pos;
while(getline(config_file, line)) {
if(line[0]=='\n' ||
(line[0]=='/' && line[1]=='/')) continue;
pos = line.find_first_of('=');
if(pos==std::string::npos) continue;
this->param_val_map[line.substr(0, pos)] = line.substr(pos+1, line.size()-pos);
}
return;
}
// Fills parameter variables
// using internal parameter map
void Params::fill_params() {
std::string t1, t2;
double tmp;
enum params param_type;
if(this->param_val_map.empty()) {
//error
return;
}
for(auto p: this->param_val_map) {
t1 = strip_whitespaces(p.first);
t2 = strip_whitespaces(p.second);
if(param_map.count(t1)==0) continue;
param_type = param_map[t1];
switch(param_type) {
case DEBUG:
if(t2.compare("TRUE") == 0) {
this->debug = true;
}
else {
this->debug = false;
}
break;
case ANALYZE:
if(t2.compare("TRUE") == 0) {
this->analyze = true;
}
else {
this->analyze = false;
}
break;
case INTERACTIVE_MODE:
if(t2.compare("TRUE") == 0) {
this->interactive_mode = true;
}
else {
this->interactive_mode = false;
}
case TRANSPORT_TYPE:
this->transport_type = transport_type_map[t2];
break;
case NUM_THREADS:
this->num_threads = std::stoi(t2);
break;
case CONN_PORT:
this->conn_port = std::stoi(t2);
break;
case NUM_REPLICAS:
this->num_replicas = std::stoi(t2);
break;
case MAX_PACKET_SIZE_BYTES:
this->max_packet_size_bytes = std::stoi(t2);
break;
case MAX_PACKET_SIZE_MBYTES:
this->max_packet_size_bytes = (int) (std::stoi(t2)*(1<<20));
break;
case RDMA_MR_SIZE_BYTES:
this->rdma_mr_size_bytes = (size_t) (std::stoi(t2));
break;
case RDMA_MR_SIZE_MBYTES:
this->rdma_mr_size_bytes = (size_t) (std::stoi(t2)*(1<<20));
break;
case RDMA_MTU_SIZE_BYTES:
switch(std::stoi(t2)) {
case 256:
this->rdma_mtu_size_bytes = IBV_MTU_256;
break;
case 512:
this->rdma_mtu_size_bytes = IBV_MTU_512;
break;
case 1024:
this->rdma_mtu_size_bytes = IBV_MTU_1024;
break;
case 2048:
this->rdma_mtu_size_bytes = IBV_MTU_2048;
break;
case 4096:
this->rdma_mtu_size_bytes = IBV_MTU_4096;
break;
default:
//error
break;
}
break;
case RDMA_MIN_RNR_TIMER:
this->rdma_min_rnr_timer = std::stoi(t2);
break;
case RDMA_TIMEOUT:
this->rdma_timeout = std::stoi(t2);
break;
case RDMA_CQ_POLL_TIMEOUT_MS:
tmp = std::stod(t2);
tmp *= 1e-3; //convert to ms
this->rdma_cq_poll_timeout_ms = std::chrono::duration<double>{tmp};
break;
case RDMA_RETRY_CNT:
this->rdma_retry_cnt = std::stoi(t2);
break;
case RDMA_IB_PORT:
this->rdma_ib_port = std::stoi(t2);
break;
case RDMA_GID_IDX:
this->rdma_gid_idx = std::stoi(t2);
break;
case ARRIVAL_RATE:
this->arrival_rate = std::stod(t2);
default:
//error
break;
}
}
return;
}
// Reads config file and
// Fills parameter variables
void Params::read_params() {
this->read_config_file();
this->fill_params();
return;
}
#endif
\ No newline at end of file
#ifndef __READ_CONFIG_H__
#define __READ_CONFIG_H__
#include <chrono>
#include <string>
#include <vector>
#include <unordered_map>
#include <infiniband/verbs.h>
#include "config_parameters.hpp"
#include "../transport_api/transport_config.hpp"
class Params {
private:
std::string filename;
std::vector<std::string> param_lines;
std::unordered_map<std::string, std::string> param_val_map;
public:
bool debug;
bool analyze;
bool interactive_mode;
enum Transport_Type transport_type;
int num_threads;
int conn_port;
int num_replicas;
int max_packet_size_bytes;
size_t rdma_mr_size_bytes;
enum ibv_mtu rdma_mtu_size_bytes;
int rdma_min_rnr_timer;
int rdma_timeout;
std::chrono::duration<double> rdma_cq_poll_timeout_ms;
int rdma_retry_cnt;
int rdma_ib_port;
int rdma_gid_idx;
double arrival_rate;
Params();
Params(std::string f);
//debug functions
void print_map();
void print_vals();
//required functions
void read_params();
void read_config_file();
void fill_params();
};
std::vector<std::string> get_param_lines(char *filename);
enum params check_token(std::string token);
#endif
\ No newline at end of file
#include <iostream>
#include "read_config.hpp"
using namespace std;
int main() {
Params p("config.conf");
p.read_params();
p.print_vals();
return 0;
}
\ No newline at end of file
DEBUG=TRUE
ANALYZE=TRUE
TRANSPORT_TYPE=RDMA_RC
NUM_THREADS=3
CONN_PORT=8888
MAX_PACKET_SIZE_MBYTES=4;
RDMA_MR_SIZE_MBYTES=1
RDMA_MTU_SIZE=512
RDMA_MIN_RNR_TIMER=12
RDMA_TIMEOUT=12
RDMA_CQ_POLL_TIMEOUT_MS=5
RDMA_RETRY_CNT=4
RDMA_IB_PORT=1
RDMA_GID_IDX=1
\ No newline at end of file
DEBUG=TRUE
ANALYZE=TRUE
INTERACTIVE_MODE=FALSE
TRANSPORT_TYPE=RDMA_RC
NUM_THREADS=0
CONN_PORT=8888
MAX_PACKET_SIZE_MBYTES=4;
RDMA_MR_SIZE_MBYTES=1
RDMA_MTU_SIZE=512
RDMA_MIN_RNR_TIMER=12
RDMA_TIMEOUT=12
RDMA_CQ_POLL_TIMEOUT_MS=5
RDMA_RETRY_CNT=4
RDMA_IB_PORT=1
RDMA_GID_IDX=1
ARRIVAL_RATE=80
\ No newline at end of file
#include <iostream>
#include <stdio.h>
#include <vector>
#include <unistd.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/time.h>
#include <string.h>
using namespace std;
char* N