Skip to content

H
hpdos
Commit 7e25adec authored by Paras Garg's avatar Paras Garg
Browse files
Options

fixed bugs in new messageformat

parent 94b24420
Hide whitespace changes
Inline Side-by-side
Showing with 610 additions and 234 deletions
code/cppclient/Makefile
View file @ 7e25adec
......@@ -41,12 +41,15 @@ clean:
count:
find . -type f -name "*.hpp"|xargs wc -l
find . -type f -name "*.cpp"|xargs wc -l
find YCSB/hpdos -type f -name "*.java"| xargs wc -l
find jsrc/ -type f -name "*.java" | xargs wc -l
@echo "Lines of code counted!"
Jclient: $(OBJS)
$(CXX) -o libhpdosclient.so -L/usr/local/lib -shared $^ $(LIBS)
@echo "jclient "$<" successfully!"
sudo rm -f /usr/lib/libhpdosclient.so
sudo cp libhpdosclient.so /usr/lib
@echo "Copied libhpdosclient.so to /usr/lib"
#jcompile: javac $(JSRCS) -d JBUILD
......
code/cppclient/Readme.md
View file @ 7e25adec
......@@ -5,15 +5,17 @@ Steps to build jni client
Running YSCB
> mvn compile <br>
add .build to make
./bin/ycsb load hpdos -P workloads/workloadb -threads 1
mvn -pl site.ycsb:hpdos-binding -am clean package -Dcheckstyle.skip
mvn -pl site.ycsb:hpdos-binding -am clean package -Dcheckstyle.skip
https://medium.com/@pirogov.alexey/gdb-debug-native-part-of-java-application-c-c-libraries-and-jdk-6593af3b4f3f
to do
delete client endpoint on close
threading in client and hashing in client
add more code for success failure not found etc server error client error
add cache add support for invalidation
interface client API through endpointGroup
......@@ -29,4 +31,29 @@ Options:
-p key=value Override workload property
-s Print status to stderr
-target n Target ops/sec (default: unthrottled)
-threads n Number of client threads (default: 1)
\ No newline at end of file
-threads n Number of client threads (default: 1)
# Steps to configure spdk target and linux initiator
Build spdk using
https://spdk.io/doc/nvmf.html
sudo systemctl restart spdknvmeof.service
if error then
1. cd spdk
2. ./scripts/setup.sh
./rpccommands.sh
nvme discover -t rdma -a 192.168.100.8 -s 4420
sudo nvme connect -t rdma -n "nqn.2021-08.hpdos:ub08" -a 192.168.200.40 -s 4420
sudo nvme disconnect -n "nqn.2021-08.hpdos:ub08"
TO check whether file system is mounted
sudo file -s /dev/nvme0n1
else mount a file sytem
sudo mkfs -t xfs /dev/nvme0n1
sudo mkdir /data
sudo mkdir /data
sudo mount /dev/nvme1n1 /data
https://www.hyper-v.io/nvme-part-1-linux-nvme-initiator-linux-spdk-nvmf-target/
\ No newline at end of file
code/cppclient/YCSB/hpdos/src/main/java/site/ycsb/db/HpdosClient.java → code/cppclient/YCSB/hpdos/src/main/java/HpdosClient.java
View file @ 7e25adec
package site.ycsb.db;
import site.ycsb.DB;
import site.ycsb.DBException;
import site.ycsb.ByteArrayByteIterator;
import site.ycsb.ByteIterator;
import site.ycsb.Status;
import site.ycsb.StringByteIterator;
......@@ -9,40 +10,57 @@ import site.ycsb.StringByteIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
public class HpdosClient extends DB {
// private static final Logger LOGGER =
// LoggerFactory.getLogger(RocksDBClient.class);
public HpdosClient() {
private static final Logger LOGGER = LoggerFactory.getLogger(HpdosClient.class);
public HpdosClient() {
}
long endpointGroup;
JClient jclient;
public void init() throws DBException {
System.out.println(System.getProperty("user.dir"));
Properties prop = getProperties();
prop.forEach((k, v) -> System.out.println(k + "" + v));
System.out.println("init hpdos");
synchronized (HpdosClient.class) {
if (endpointGroup == 0)
endpointGroup++;
}
System.out.println("endpoit" + endpointGroup);
jclient = JClient.getInstance();
System.out.println("jclient"+jclient);
//synchronized (HpdosClient.class) {
// if (jclient == null) {
// System.loadLibrary("hpdosclient");
// jclient = new JClient();
// jclient.endpointGroup = jclient.createEndpointGroup("../prop.config");
// }
//}
LOGGER.info("endpoint" + jclient.endpointGroup);
}
public void cleanup() throws DBException {
jclient.closeEndpointGroup(jclient.endpointGroup);
LOGGER.info("closing");
}
// Read a single record
public Status read(String table, String key, Set<String> fields, Map<String, ByteIterator> result) {
System.out.println("Read" + key);
//System.out.println("Read" + key);
byte res[] = jclient.get(jclient.endpointGroup, key.getBytes());
if (res == null)
return Status.NOT_FOUND;
if(fields == null)
{
return Status.OK;
}
Iterator<String> it = fields.iterator();
if (it.hasNext())
result.put(fields.iterator().next(), new ByteArrayByteIterator(res, 0, res.length));
return Status.OK;
}
......@@ -54,7 +72,28 @@ public class HpdosClient extends DB {
// Update a single record
public Status update(String table, String key, Map<String, ByteIterator> values) {
System.out.println("update" + key);
//System.out.println("update" + key);
try {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
for (ByteIterator v : values.values()) {
try {
stream.write(v.toArray());
} catch (IOException ex) {
LOGGER.error(ex.getMessage());
return Status.ERROR;
}
}
int val = jclient.put(jclient.endpointGroup, key.getBytes(), stream.toByteArray());
if (val == -1)
{
System.out.println("got -1\n");
return Status.BAD_REQUEST;
}
} catch (Exception e) {
System.out.println(e.getMessage());
e.printStackTrace();
return Status.ERROR;
}
return Status.OK;
}
......@@ -65,6 +104,9 @@ public class HpdosClient extends DB {
// Delete a single record
public Status delete(String table, String key) {
int status = jclient.delete(jclient.endpointGroup, key.getBytes());
if (status == -1 )
return Status.ERROR;
return Status.OK;
}
......
code/cppclient/YCSB/hpdos/src/main/java/JClient.java 0 → 100644
View file @ 7e25adec
public class JClient {
static {
System.loadLibrary("hpdosclient");
}
public static JClient jclient = null;
public long endpointGroup;
private JClient() {
endpointGroup = createEndpointGroup("../prop.config");
}
public static JClient getInstance() {
synchronized(JClient.class)
{
if (jclient == null) {
System.out.println("creating jclient");
jclient = new JClient();
}
}
return jclient;
}
public native long createEndpointGroup(String file);
public native void closeEndpointGroup(long endpointGroup);
public native int put(long group, byte[] key, byte[] value);
public native byte[] get(long group, byte[] key);
public native int delete(long group, byte[] key);
}
code/cppclient/YCSB/hpdos/src/main/java/site/ycsb/db/JNIClient.java deleted 100644 → 0
View file @ 94b24420
package site.ycsb.db;
public class JNIClient {
public static long endpointGroup;
public long client;
public native int put(long client, byte[] key, byte[] value);
public native byte[] get(long client, byte[] key);
public native int delete(long client, byte[] key);
public native long createEndpointGroup(int sendQSize, int recvQSize, int compQSize,
int sendMsqSize, int recvMsgSize, int maxInLine, int timeout);
public native long createClient(long endpointGroup);
public native void closeClient(long client);
public native void closeEndpointGroup(long endpointGroup);
}
code/cppclient/YCSB/workloads/myworkload 0 → 100644
View file @ 7e25adec
# Copyright (c) 2010 Yahoo! Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License. See accompanying
# LICENSE file.
# Yahoo! Cloud System Benchmark
# Workload A: Update heavy workload
# Application example: Session store recording recent actions
#
# Read/update ratio: 50/50
# Default data size: 1 KB records (10 fields, 100 bytes each, plus key)
# Request distribution: zipfian
recordcount=200
operationcount=200
fieldcount=1
fieldlength=200
workload=site.ycsb.workloads.CoreWorkload
readallfields=true
readproportion=0.5
updateproportion=0.5
# requestdistribution
# readmodifywriteproportion
scanproportion=0
insertproportion=0
requestdistribution=zipfian
code/cppclient/YCSB/workloads/workloada
View file @ 7e25adec
......@@ -22,8 +22,8 @@
# Default data size: 1 KB records (10 fields, 100 bytes each, plus key)
# Request distribution: zipfian
recordcount=1000
operationcount=1000
recordcount=200
operationcount=200
workload=site.ycsb.workloads.CoreWorkload
readallfields=true
......
code/cppclient/header/JClient.h
View file @ 7e25adec
......@@ -7,6 +7,22 @@
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: JClient
* Method: createEndpointGroup
* Signature: (Ljava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_JClient_createEndpointGroup
(JNIEnv *, jobject, jstring);
/*
* Class: JClient
* Method: closeEndpointGroup
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_JClient_closeEndpointGroup
(JNIEnv *, jobject, jlong);
/*
* Class: JClient
* Method: put
......@@ -31,38 +47,6 @@ JNIEXPORT jbyteArray JNICALL Java_JClient_get
JNIEXPORT jint JNICALL Java_JClient_delete
(JNIEnv *, jobject, jlong, jbyteArray);
/*
* Class: JClient
* Method: createEndpointGroup
* Signature: (IIIIIII)J
*/
JNIEXPORT jlong JNICALL Java_JClient_createEndpointGroup
(JNIEnv *, jobject, jint, jint, jint, jint, jint, jint, jint);
/*
* Class: JClient
* Method: createClient
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL Java_JClient_createClient
(JNIEnv *, jobject, jlong);
/*
* Class: JClient
* Method: closeClient
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_JClient_closeClient
(JNIEnv *, jobject, jlong);
/*
* Class: JClient
* Method: closeEndpointGroup
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_JClient_closeEndpointGroup
(JNIEnv *, jobject, jlong);
#ifdef __cplusplus
}
#endif
......
code/cppclient/header/MessageFormats.hpp
View file @ 7e25adec
......@@ -7,7 +7,7 @@ enum MessageType
DELETE = (1u << 2),
INVALIDATE = (1u << 3),
SUCCESS = (1u << 4),
FAILURE = (1u <<5)
FAILURE = (1u << 5)
};
struct __attribute__ ((__packed__)) MessageHeader
......
code/cppclient/header/Properties.hpp 0 → 100644
View file @ 7e25adec
#ifndef __Properties__
#define __Properties__
#include <string>
#include <iostream>
#include <fstream>
#include <map>
class Properties
{
private:
std::map<std::string, std::string> _props;
const std::string _WHITESPACE = " \n\r\t\f\v";
std::string ltrim(const std::string &s);
std::string rtrim(const std::string &s);
std::string trim(const std::string &);
public:
Properties(std::string filename);
std::string getValue(std::string key);
};
#endif
\ No newline at end of file
code/cppclient/header/RdmaClientEndpointGroup.hpp
View file @ 7e25adec
#ifndef __RDMACLIENTENDPOINTGROUP__
#define __RDMACLIENTENDPOINTGROUP__
#include <vector>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>
#include <stdint.h>
#include <errno.h>
#include <unordered_map>
#ifndef __RDMACLIENTENDPOINTGROUP__
#define __RDMACLIENTENDPOINTGROUP__
#include <chrono>
#include <thread>
#include <mutex>
#include "RdmaEndpointGroup.hpp"
#include "RdmaCqProcessor.hpp"
#include "RdmaCmProcessor.hpp"
#include "RdmaClientEndpoint.hpp"
#include "Properties.hpp"
class RdmaClientEndpointGroup : public RdmaEndpointGroup
{
RdmaCmProcessor *_cmProcessor = NULL;
RdmaCqProcessor *_cqProcessor = NULL;
//struct rdma_cm_id *_cm_id;
// struct rdma_cm_id *_cm_id;
int _sendQueueSize;
int _recvQueueSize;
......@@ -25,14 +28,25 @@ class RdmaClientEndpointGroup : public RdmaEndpointGroup
int _recvMsgSize;
int _maxInLine;
int _timeoutMs;
std::vector<RdmaClientEndpoint*> _clientEps;
int _clients;
int _counter;
std::mutex _counterMutex;
public:
RdmaClientEndpointGroup(int sendQueueSize, int recvQueueSize, int compQueueSize, int sendMsgSize,
int recvMsgSize,int maxInLine, int timeout);
int recvMsgSize, int maxInLine, int timeout);
void processCmEvent(struct rdma_cm_event *event);
struct ibv_cq *createCq(struct rdma_cm_id *id);
RdmaClientEndpoint *createEndpoint();
void createClientEps(Properties *);
/* functions return RDMAFuture whose get will wait until data is put
* to avoid busy waiting cv is used*/
RdmaFuture* put(const char *key, int keySize, const char *value, int valueSize);
RdmaFuture* get(const char *key, int keySize);
RdmaFuture* deleteKey(const char *key, int keySize);
};
#endif
\ No newline at end of file
code/cppclient/header/RdmaCqProcessor.hpp
View file @ 7e25adec
......@@ -22,9 +22,10 @@ public:
struct ibv_cq *getCq();
void start();
void processCQEvents();
void dispatchCqEvents(ibv_wc *wc_array, int size);
inline void dispatchCqEvents(ibv_wc *wc_array, int size);
void close();
void registerEp(uint64_t qpum, RdmaClientEndpoint* ep);
void deRegisterEp(uint64_t qpum);
};
#endif
......
code/cppclient/header/RdmaFuture.hpp
View file @ 7e25adec
......@@ -11,12 +11,17 @@ class RdmaFuture
{
static int DONE;
static int PENDING;
uint8_t _status;
/* Used to identify future for a particular request.
* It is same as request id for packet generated
*/
uint64_t _requestId;
char *_data;
std::mutex stateMutex;
std::condition_variable stateCv;
uint8_t state{0};
char *_data;
uint8_t _status;
public:
RdmaFuture(uint64_t requestId);
......
code/cppclient/jsrc/JClient.java
View file @ 7e25adec
import javax.security.auth.login.CredentialException;
public class JClient {
public native int put(long client, byte[] key, byte[] value);
public long endpointGroup;
public native byte[] get(long client, byte[] key);
public native long createEndpointGroup(String path);
public native int delete(long client, byte[] key);
public native void closeEndpointGroup(long endpointGroup);
public native long createEndpointGroup(int sendQSize, int recvQSize, int compQSize, int sendMsqSize, int recvMsgSize,
int maxInLine, int timeout);
public native int put(long group, byte[] key, byte[] value);
public native long createClient(long endpointGroup);
public native byte[] get(long group, byte[] key);
public native void closeClient(long client);
public native int delete(long group, byte[] key);
public native void closeEndpointGroup(long endpointGroup);
public static long endpointGroup;
public long client;
static {
System.loadLibrary("hpdosclient");
......@@ -27,14 +21,14 @@ public class JClient {
System.out.println(System.getProperty("java.library.path"));
try {
//Thread.sleep(1000*40);
JClient jclient = new JClient();
endpointGroup = jclient.createEndpointGroup(5, 5, 5, 500, 500, 0, 1000);
jclient.client = jclient.createClient(endpointGroup);
System.out.println("jc" + jclient.put(jclient.client, "paras".getBytes(), "garg".getBytes()));
var res = jclient.get(jclient.client, "paras".getBytes());
jclient.endpointGroup = jclient.createEndpointGroup("prop.config");
System.out.println("jc" + jclient.put(jclient.endpointGroup, "paras".getBytes(), "garg".getBytes()));
var res = jclient.get(jclient.endpointGroup, "paras".getBytes());
System.out.println("val size"+res.length+" "+new String(res));
System.out.println("jc" + jclient.delete(jclient.client, "paras".getBytes()));
System.out.println("jc" + jclient.delete(jclient.endpointGroup, "paras".getBytes()));
} catch (Exception e) {
e.printStackTrace();
}
......
code/cppclient/src/Client.cpp
View file @ 7e25adec
#include <iostream>
#include "RdmaClientEndpointGroup.hpp"
#include "MessageFormats.hpp"
#include "Properties.cpp"
#include "Properties.hpp"
int main()
{
Properties prop("prop.config");
int sendQS = stoi(prop.getValue("sendQS"));
int recvQS = stoi(prop.getValue("recvQS"));
int compQS = stoi(prop.getValue("compQS"));
int sendMS = stoi(prop.getValue("sendMS"));
int recvMS = stoi(prop.getValue("recvMS"));
Properties* prop = new Properties("prop.config");
int sendQS = stoi(prop->getValue("sendQS"));
int recvQS = stoi(prop->getValue("recvQS"));
int compQS = stoi(prop->getValue("compQS"));
int sendMS = stoi(prop->getValue("sendMS"));
int recvMS = stoi(prop->getValue("recvMS"));
RdmaClientEndpointGroup *group = new RdmaClientEndpointGroup(sendQS, recvQS, compQS, sendMS, recvMS, 0, 1000);
RdmaClientEndpoint *clientEp = group->createEndpoint();
//RdmaClientEndpoint *clientEp = group->createEndpoint();
group->createClientEps(prop);
delete prop;
/*
clientEp->connect("192.168.200.20", "1921", "sal");
while (!clientEp->isConnected())
;
std::cout << "client : connected" << std::endl;
*/
std::cout << "clients : connected" << std::endl;
auto r1 = clientEp->put("paras", 5, "garg", 4);
auto r2 = clientEp->get("paras", 5);
auto r3 = clientEp->deleteKey("paras", 5);
auto r1 = group->put("paras", 5, "garg", 4);
auto r2 = group->get("paras", 5);
auto r3 = group->deleteKey("paras", 5);
std::cout << "waiting for output" << std::endl;
if (r3 != nullptr)
......@@ -47,7 +51,6 @@ int main()
std::cout << " size " << response->valueSize << std::endl;
delete r2;
}
/*char *message = new char[100];
struct SalRequestHeader *request = (struct SalRequestHeader *)message;
request->id = clientEp->_requestId.fetch_add(1, std::memory_order_relaxed);
......
code/cppclient/src/JClient.cpp
View file @ 7e25adec
......@@ -3,53 +3,42 @@
#include "Logger.hpp"
#include "RdmaClientEndpoint.hpp"
#include "RdmaClientEndpointGroup.hpp"
#include <unistd.h>
/*
bool rocksdb_put_helper(JNIEnv* env, ROCKSDB_NAMESPACE::DB* db,
const ROCKSDB_NAMESPACE::WriteOptions& write_options,
ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle,
jbyteArray jkey, jint jkey_off, jint jkey_len,
jbyteArray jval, jint jval_off, jint jval_len) {
jbyte* key = new jbyte[jkey_len];
env->GetByteArrayRegion(jkey, jkey_off, jkey_len, key);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
delete[] key;
return false;
}
JNIEXPORT jlong JNICALL Java_JClient_createEndpointGroup(JNIEnv *jenv, jobject jobj, jstring jpath)
{
// exec_gdb();
jboolean iscopy;
const char *path = jenv->GetStringUTFChars(jpath, &iscopy);
// std::cout<<path<<"\n";
jbyte* value = new jbyte[jval_len];
env->GetByteArrayRegion(jval, jval_off, jval_len, value);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
delete[] value;
delete[] key;
return false;
}
Properties *prop = new Properties(path);
int sendQS = stoi(prop->getValue("sendQS"));
int recvQS = stoi(prop->getValue("recvQS"));
int compQS = stoi(prop->getValue("compQS"));
int sendMS = stoi(prop->getValue("sendMS"));
int recvMS = stoi(prop->getValue("recvMS"));
std::cout << "Creating EndpointGroup SendQS " << sendQS << " recvQS " << recvQS << " compQS ";
std::cout << compQS << " sendMS " << sendMS << " recvMS " << recvMS << "\n";
RdmaClientEndpointGroup *group = new RdmaClientEndpointGroup(sendQS, recvQS, compQS, sendMS, recvMS, 0, 1000);
std::cout << "Connecting Endpoints\n";
group->createClientEps(prop);
delete prop;
*/
JNIEXPORT jlong JNICALL Java_JClient_createEndpointGroup(JNIEnv *jenv, jobject jobj,
jint sendq, jint recvq, jint compq,
jint sendm, jint recvm,
jint maxinline, jint timeout)
{
RdmaClientEndpointGroup *group = new RdmaClientEndpointGroup(sendq, recvq, compq, sendm,
recvm, maxinline, timeout);
return reinterpret_cast<jlong>(group);
}
JNIEXPORT jlong JNICALL Java_JClient_createClient(JNIEnv *jenv, jobject jobj, jlong group)
JNIEXPORT void JNICALL Java_JClient_closeEndpointGroup(JNIEnv *jenv, jobject jobj, jlong jclient)
{
RdmaClientEndpoint *client = (reinterpret_cast<RdmaClientEndpointGroup *>(group))
->createEndpoint();
client->connect("192.168.200.20", "1921", "sal");
while (!client->isConnected())
;
return reinterpret_cast<jlong>(client);
RdmaClientEndpointGroup *group = reinterpret_cast<RdmaClientEndpointGroup *>(jclient);
// group->close();
}
JNIEXPORT jbyteArray JNICALL Java_JClient_get(JNIEnv *jenv, jobject jobj, jlong jclient,
jbyteArray jkey)
{
/* Get Key form Java Side*/
int keyLen = jenv->GetArrayLength(jkey);
char *key = new char[keyLen];
jenv->GetByteArrayRegion(jkey, 0, keyLen, reinterpret_cast<jbyte *>(key));
......@@ -58,32 +47,43 @@ JNIEXPORT jbyteArray JNICALL Java_JClient_get(JNIEnv *jenv, jobject jobj, jlong
CPPLog::LOG_ERROR("exception occurs in jni get");
delete[] key;
}
RdmaClientEndpoint *client = reinterpret_cast<RdmaClientEndpoint *>(jclient);
auto future = client->get(key, keyLen);
/* Get data from server*/
RdmaClientEndpointGroup *group = reinterpret_cast<RdmaClientEndpointGroup *>(jclient);
auto future = group->get(key, keyLen);
delete[] key;
/* Handle failed get request*/
if (future == nullptr)
{
jbyteArray jvalue = jenv->NewByteArray(0);
return jvalue;
}
auto data = future->get();
/* Create Java Byte Array and send data to java side
* after copying data from c char array to java byte array
*/
std::cout << "get future.get()\n";
char *data = future->get();
delete future;
std::cout << "g future.get()\n";
struct MessageHeader *response = (struct MessageHeader *)data;
if (response->type == MessageType::FAILURE)
{
jbyteArray jvalue = jenv->NewByteArray(0);
delete[] data;
return jvalue;
}
jbyteArray jvalue = jenv->NewByteArray(response->valueSize);
std::cout << "value size " << response->valueSize << " " << (data + MessageHeaderSize);
// std::cout << "value size " << response->valueSize << " " << (data + MessageHeaderSize);
jenv->SetByteArrayRegion(jvalue, 0, response->valueSize - 1,
reinterpret_cast<const jbyte *>(data + MessageHeaderSize));
delete[] data;