Commit 731e788d authored by Paras Garg's avatar Paras Garg

updated files

parent a155b14c
app.HOST=192.168.200.20
app.NO_OF_MASTERS=1
app.MASTER_HOST1=192.168.200.20
#app.MASTER_HOST2=192.168.200.51
app.MASTER_HOST2=192.168.200.40
app.INVALIDATION_PORT=1922
app.CACHE_SIZE=1
......@@ -40,8 +40,8 @@ apply plugin: "application"
apply plugin: "java-library"
//mainClassName = "MetadataClient"
mainClassName = "BenchmarkingClient"
//mainClassName= "StandaloneRDMAClient"
//mainClassName = "BenchmarkingClient"
mainClassName= "StandaloneRDMAClient"
version '1.0-SNAPSHOT'
repositories {
......
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
......@@ -15,30 +16,32 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.apache.log4j.Logger;
import hpdos.handlers.NetworkHandler;
import hpdos.protocol.BenchmarkingProperties;
import hpdos.protocol.Property;
public class BenchmarkingClient {
public class BenchmarkingClient
{
final static Logger logger = Logger.getLogger(BenchmarkingClient.class);
public Properties properties;
public double runExperiment(String threadId, MetadataClient client) throws Exception{
public double runExperiment(String threadId, MetadataClient client,BenchmarkingProperties properties) throws Exception{
Double sleepTime = 1000.0 * 1000.0 / BenchmarkingProperties.requestsPerSecond;
Double sleepTime = 1000.0 * 1000.0 / properties.requestsPerSecond;
long sleepTimeMicros = Double.valueOf(sleepTime).longValue();
logger.debug(sleepTimeMicros);
logger.info(sleepTimeMicros);
Thread.sleep((long)(Math.random() * 500)); // randomizing start time to avoid spikes in request
long requestsPerClient = BenchmarkingProperties.requestsPerClient;
double totalBracket = BenchmarkingProperties.reads
+ BenchmarkingProperties.writes
+ BenchmarkingProperties.deletes;
long requestsPerClient = properties.requestsPerClient;
double totalBracket = properties.reads
+ properties.writes
+ properties.deletes;
double writeBracket, readBracket, deleteBracket;
writeBracket = BenchmarkingProperties.writes * 1.0 / totalBracket;
readBracket = writeBracket + BenchmarkingProperties.reads / totalBracket;
deleteBracket = readBracket + BenchmarkingProperties.deletes / totalBracket;
writeBracket = properties.writes * 1.0 / totalBracket;
readBracket = writeBracket + properties.reads / totalBracket;
deleteBracket = readBracket + properties.deletes / totalBracket;
// int temp = 5;
ExecutorService threadpool = Executors.newCachedThreadPool();
// ListeningExecutorService service = MoreExecutors.listeningDecorator(threadpool);
......@@ -52,15 +55,15 @@ public class BenchmarkingClient {
if (toss < writeBracket) {
byte[] key = new byte[BenchmarkingProperties.keySize];
byte[] value = new byte[BenchmarkingProperties.valueSize];
byte[] key = new byte[properties.keySize];
byte[] value = new byte[properties.valueSize];
new Random().nextBytes(key);
new Random().nextBytes(value);
// stopwatch.start();
long writeStartTime = System.nanoTime();
CompletableFuture<Void> cf = CompletableFuture.runAsync(() -> {
try {
client.networkHandler.put(key, value);
client.put(key, value);
} catch (Exception e) {
e.printStackTrace();
}
......@@ -69,47 +72,45 @@ public class BenchmarkingClient {
});
responses.add(cf);
// stopwatch.stop();
}
else if (toss < readBracket) {
byte[] key = new byte[BenchmarkingProperties.keySize];
byte[] key = new byte[properties.keySize];
new Random().nextBytes(key);
// stopwatch.start();
long readStartTime = System.nanoTime();
CompletableFuture<Void> cf = CompletableFuture.runAsync(() -> {
try {
client.networkHandler.get(key);
client.get(key);
} catch (Exception e) {
e.printStackTrace();
}
}).whenComplete((r,t)->{
collectExperimentStatistics((System.nanoTime() - readStartTime)/1000, 101);
collectExperimentStatistics((System.nanoTime() - readStartTime)/1000, 101,properties);
});
responses.add(cf);
// stopwatch.stop();
// collectExperimentStatistics(stopwatch.elapsed(TimeUnit.MICROSECONDS), 101);
}
else if(toss < deleteBracket){
byte[] key = new byte[BenchmarkingProperties.keySize];
byte[] key = new byte[properties.keySize];
new Random().nextBytes(key);
// stopwatch.start();
long deleteStartTime = System.nanoTime();
CompletableFuture<Void> cf = CompletableFuture.runAsync(() -> {
try {
client.networkHandler.delete(key);
client.delete(key);
} catch (Exception e) {
e.printStackTrace();
}
}).whenComplete((r,t)->{
collectExperimentStatistics((System.nanoTime() - deleteStartTime)/1000, 102);
collectExperimentStatistics((System.nanoTime() - deleteStartTime)/1000, 102,properties);
});
responses.add(cf);
// stopwatch.stop();
// collectExperimentStatistics(stopwatch.elapsed(TimeUnit.MICROSECONDS), 102);
}
if (requestsPerClient % 10000 == 0) {
logger.debug("Ten thousand requests completed " + requestsPerClient);
logger.info("Ten thousand requests completed " + requestsPerClient);
}
requestsPerClient -- ;
......@@ -119,9 +120,9 @@ public class BenchmarkingClient {
// stopwatch.start();
testWait(sleepTimeMicros);
long endTime = System.nanoTime();
BenchmarkingProperties.timePerRequest = BenchmarkingProperties.timePerRequest + ((endTime - startTime)/1000);
properties.timePerRequest = properties.timePerRequest + ((endTime - startTime)/1000);
// stopwatch.stop();
// logger.debug("Microseconds waited: " + stopwatch.elapsed(TimeUnit.MICROSECONDS));
// logger.info("Microseconds waited: " + stopwatch.elapsed(TimeUnit.MICROSECONDS));
}
Iterator iteratorValues = responses.iterator();
long startTime = System.nanoTime();
......@@ -130,107 +131,110 @@ public class BenchmarkingClient {
res.get(10, TimeUnit.SECONDS);
}
long endTime = System.nanoTime();
logger.debug("Average request latency: " + ((endTime - startTime)/1000)/(BenchmarkingProperties.requestsPerClient));
logger.info("Average request latency: " + ((endTime - startTime)/1000)/(properties.requestsPerClient));
return 0;
}
public static void collectExperimentStatistics(long time, int type){
// logger.debug("Time: " + time);
public static void collectExperimentStatistics(long time, int type,BenchmarkingProperties properties){
// logger.info("Time: " + time);
if (type == 100) {
BenchmarkingProperties.writeCount ++;
BenchmarkingProperties.writeTime = BenchmarkingProperties.writeTime + time;
properties.writeCount ++;
properties.writeTime = properties.writeTime + time;
// BenchmarkingProperties.actualReqPerSecond = BenchmarkingProperties.actualReqPerSecond + time;
}
else if (type == 101) {
BenchmarkingProperties.readCount ++;
BenchmarkingProperties.readTime = BenchmarkingProperties.readTime + time;
properties.readCount ++;
properties.readTime = properties.readTime + time;
// BenchmarkingProperties.actualReqPerSecond = BenchmarkingProperties.actualReqPerSecond + time;
}
else if(type == 102){
BenchmarkingProperties.deleteCount ++;
BenchmarkingProperties.deleteTime = BenchmarkingProperties.deleteTime + time;
properties.deleteCount ++;
properties.deleteTime = properties.deleteTime + time;
// BenchmarkingProperties.actualReqPerSecond = BenchmarkingProperties.actualReqPerSecond + time;
}
}
public void printStatistics(){
logger.debug("Total writes: " + BenchmarkingProperties.writeCount);
logger.debug("Total reads: " + BenchmarkingProperties.readCount);
logger.debug("Total deletes: " + BenchmarkingProperties.deleteCount);
public void printStatistics(BenchmarkingProperties properties){
logger.info("Total writes: " + properties.writeCount);
logger.info("Total reads: " + properties.readCount);
logger.info("Total deletes: " + properties.deleteCount);
logger.debug("Total write time: " + BenchmarkingProperties.writeTime);
logger.debug("Total read time: " + BenchmarkingProperties.readTime);
logger.debug("Total delete time: " + BenchmarkingProperties.deleteTime);
logger.info("Total write time: " + properties.writeTime);
logger.info("Total read time: " + properties.readTime);
logger.info("Total delete time: " + properties.deleteTime);
logger.debug("Average write time in microseconds: " + (BenchmarkingProperties.writeTime / BenchmarkingProperties.writeCount));
logger.debug("Average read time in microseconds: " + (BenchmarkingProperties.readTime / BenchmarkingProperties.readCount));
logger.debug("Average delete time in microseconds: " + (BenchmarkingProperties.deleteTime / BenchmarkingProperties.deleteCount));
logger.info("Average write time in microseconds: " + (properties.writeTime / properties.writeCount));
logger.info("Average read time in microseconds: " + (properties.readTime / properties.readCount));
logger.info("Average delete time in microseconds: " + (properties.deleteTime / properties.deleteCount));
logger.debug("Average Time/Request : " + BenchmarkingProperties.timePerRequest/(BenchmarkingProperties.writeCount + BenchmarkingProperties.readCount + BenchmarkingProperties.deleteCount));
long avgTimePerReq = BenchmarkingProperties.timePerRequest/(BenchmarkingProperties.writeCount + BenchmarkingProperties.readCount + BenchmarkingProperties.deleteCount);
logger.debug("Actual Requests/second : " + (1000000/avgTimePerReq));
logger.info("Average Time/Request : " + properties.timePerRequest/(properties.writeCount + properties.readCount + properties.deleteCount));
long avgTimePerReq = properties.timePerRequest/(properties.writeCount + properties.readCount + properties.deleteCount);
logger.info("Actual Requests/second : " + (1000000/avgTimePerReq));
}
public void testWait(long microseconds){
// logger.debug("Sleeping for " + microseconds + " microseconds.");
// logger.info("Sleeping for " + microseconds + " microseconds.");
final long INTERVAL = microseconds * 1000;
long start = System.nanoTime();
long end=0;
do{
end = System.nanoTime();
}while(start + INTERVAL >= end);
// logger.debug(end - start);
// logger.info(end - start);
}
final static Logger logger = Logger.getLogger(BenchmarkingClient.class);
public static void main(String[] args) throws Exception {
BenchmarkingClient benchmarkingClient = new BenchmarkingClient();
logger.debug("Staring to benchmark...");
logger.info("Staring to benchmark...");
benchmarkingClient.properties= new Properties();
InputStream inputStream = new FileInputStream(args[0]);
benchmarkingClient.properties.load(inputStream);
BenchmarkingProperties.reads = Double.valueOf((String)benchmarkingClient.properties.get("reads"));
BenchmarkingProperties.writes = Double.valueOf((String)benchmarkingClient.properties.get("writes"));
BenchmarkingProperties.deletes = Double.valueOf((String)benchmarkingClient.properties.get("deletes"));
BenchmarkingProperties.clients = Integer.valueOf((String)benchmarkingClient.properties.get("clients"));
BenchmarkingProperties.requestsPerClient = Long.valueOf((String)benchmarkingClient.properties.get("requests_per_client"));
BenchmarkingProperties.requestsPerSecond = Double.valueOf((String)benchmarkingClient.properties.get("requests_per_second"));
BenchmarkingProperties.keySize = Integer.valueOf((String)benchmarkingClient.properties.get("key_size"));
BenchmarkingProperties.valueSize = Integer.valueOf((String)benchmarkingClient.properties.get("value_size"));
logger.debug("================= Properties loaded for benchmarking ================= ");
logger.debug("Percentage of reads: " + BenchmarkingProperties.reads * 100);
logger.debug("Percentage of writes: " + BenchmarkingProperties.writes * 100);
logger.debug("Percentage of deletes: " + BenchmarkingProperties.deletes * 100);
logger.debug("Clients to be spawned: " + BenchmarkingProperties.clients);
logger.debug("Requests per client: " + BenchmarkingProperties.requestsPerClient);
logger.debug("Requests per second: " + BenchmarkingProperties.requestsPerSecond);
logger.debug("Key Size: " + BenchmarkingProperties.keySize);
logger.debug("Value Size: " + BenchmarkingProperties.valueSize);
logger.debug("====================================================================== ");
benchmarkingClient.properties.load(inputStream);
BenchmarkingProperties properties = new BenchmarkingProperties();
properties.reads = Double.valueOf(benchmarkingClient.properties.getProperty("reads"));
properties.writes = Double.valueOf(benchmarkingClient.properties.getProperty("writes"));
properties.deletes = Double.valueOf(benchmarkingClient.properties.getProperty("deletes"));
properties.clients = Integer.valueOf(benchmarkingClient.properties.getProperty("clients"));
properties.requestsPerClient = Long.valueOf(benchmarkingClient.properties.getProperty("requests_per_client"));
properties.requestsPerSecond = Double.valueOf(benchmarkingClient.properties.getProperty("requests_per_second"));
properties.keySize = Integer.valueOf(benchmarkingClient.properties.getProperty("key_size"));
properties.valueSize = Integer.valueOf(benchmarkingClient.properties.getProperty("value_size"));
logger.info("================= Properties loaded for benchmarking ================= ");
logger.info("Percentage of reads: " + properties.reads * 100);
logger.info("Percentage of writes: " + properties.writes * 100);
logger.info("Percentage of deletes: " + properties.deletes * 100);
logger.info("Clients to be spawned: " + properties.clients);
logger.info("Requests per client: " + properties.requestsPerClient);
logger.info("Requests per second: " + properties.requestsPerSecond);
logger.info("Key Size: " + properties.keySize);
logger.info("Value Size: " + properties.valueSize);
logger.info("====================================================================== ");
// MetadataClient client = null;
MetadataClient client = new MetadataClient(benchmarkingClient.properties);
ExecutorService experimentExecutors = Executors.newFixedThreadPool(BenchmarkingProperties.clients);
Property metaProperties = new Property("./app.config");
MetadataClient client = new MetadataClient(metaProperties);
ExecutorService experimentExecutors = Executors.newFixedThreadPool(properties.clients);
Set<Callable<Double>> callables = new HashSet<>();
logger.debug("INITIALIZING EXPERIMENT...");
for (int i = 1; i <= BenchmarkingProperties.clients; i++) {
logger.info("INITIALIZING EXPERIMENT...");
for (int i = 1; i <= properties.clients; i++) {
int finalI = i;
callables.add(() -> benchmarkingClient.runExperiment(Integer.toString(finalI), client));
callables.add(() -> benchmarkingClient.runExperiment(Integer.toString(finalI), client,properties));
}
Thread.sleep(5000);
logger.debug("STARTING EXPERIMENT...");
logger.info("STARTING EXPERIMENT...");
List<Future<Double>> futures = experimentExecutors.invokeAll(callables);
for (Future<Double> future: futures) {
future.get();
}
logger.debug("DONE...");
logger.info("DONE...");
NetworkHandler networkHandler = new NetworkHandler();
networkHandler.benchmarkingProperties.printStatistics();
experimentExecutors.shutdown();
}
}
}
\ No newline at end of file
......@@ -23,7 +23,7 @@ public class MetadataClient
public MetadataClient(Property properties) throws Exception
{
String invalidationServerIP = properties.invalidationServerIP;
logger.debug("Invalidation server ip: " + invalidationServerIP);
logger.info("Invalidation server ip: " + invalidationServerIP);
this.cache = Caffeine.newBuilder()
.expireAfterWrite(300, TimeUnit.SECONDS)
......@@ -37,14 +37,15 @@ public class MetadataClient
public byte[] get(byte[] key) throws Exception
{
logger.debug("Getting From local cache");
logger.info("Getting From local cache");
byte[] value = this.cache.getIfPresent(new ByteArray(key));
if(value != null)
return value;
logger.debug("Not found in cache Getting from server");
logger.info("Not found in cache Getting from server");
var futureResponse = networkHandlerM.get(key).get(100, TimeUnit.MILLISECONDS);
//var futureResponse = networkHandlerM.get(key).get(1000, TimeUnit.MILLISECONDS);
var futureResponse = networkHandlerM.get(key).get();
if(futureResponse.getAck() == AckType.SUCCESS_WITH_VALUE){
System.out.println("Adding to cache");
this.cache.put(new ByteArray(key), futureResponse.getValue());
......@@ -55,8 +56,9 @@ public class MetadataClient
public int put(byte[] key,byte[] value) throws Exception
{
var response = networkHandlerM.put(key,value).get(100, TimeUnit.MILLISECONDS);
logger.debug("Putting KV pair on server ....");
//var response = networkHandlerM.put(key,value).get(1000, TimeUnit.MILLISECONDS);
var response = networkHandlerM.put(key,value).get();
logger.info("Putting KV pair on server ....");
if(response.getAck() == AckType.SUCCESS){
this.cache.put(new ByteArray(key), value);
System.out.println("Updating cache..");
......@@ -66,9 +68,10 @@ public class MetadataClient
public int delete(byte[] key) throws Exception
{
logger.debug("Invalidating Local Cache and sending delete request to server");
logger.info("Invalidating Local Cache and sending delete request to server");
//this.cache.invalidate(key);
var response = this.networkHandlerM.delete(key).get(100,TimeUnit.MILLISECONDS);
return response.getAck();
// var response = this.networkHandlerM.delete(key).get(1000,TimeUnit.MILLISECONDS);
var response = this.networkHandlerM.delete(key).get();
return response.getAck();
}
}
\ No newline at end of file
......@@ -10,13 +10,12 @@ public class StandaloneRDMAClient {
public static void main(String[] args) throws Exception {
Scanner scanner = new Scanner(System.in);
System.out.println("Enter Invalidation Server IP to bind");
Property properties = new Property(args[0]);
// String address = scanner.nextLine();
System.out.println("prepare");
MetadataClient client = new MetadataClient(properties);
System.out.println("starting");
int option;
while (true) {
System.out.println("Enter 1. for read and 2 for write 3 for delete: ");
......
......@@ -24,6 +24,8 @@ public class NetworkHandlerM {
public NetworkHandlerM( Property properties) throws Exception
{
streams = new ArrayList<>();
eps = new ArrayList<>();
RpcProtocol rpcProtocol = new RpcProtocol();
group = DaRPCClientGroup.createClientGroup(rpcProtocol, 100, 0, 16, 16);
for(var ip : properties.masters)
......
......@@ -49,12 +49,12 @@ public class InvalidationServer
try
{
int conns = numOfConnections;
logger.debug("Waiting for server to send connection request for invalidation");
logger.info("Waiting for server to send connection request for invalidation");
while (conns>0) {
serverEp.accept();
conns -- ;
}
logger.debug("Got Connected for Invalidation");
logger.info("Got Connected for Invalidation");
}
catch(Exception e){
e.printStackTrace();
......@@ -63,7 +63,7 @@ public class InvalidationServer
var t= new Thread(runnable);
t.setName("Server Connection");
t.start();
logger.debug("started Invalidation server");
logger.info("started Invalidation server");
}
public void sendInvalidationRegistrationRequest(String masterIpAddress, String hostIpAddress,NetworkHandler networkHandler) throws IOException, ClassNotFoundException{
......
......@@ -25,24 +25,24 @@ public class BenchmarkingProperties {
final static Logger logger = Logger.getLogger(BenchmarkingProperties.class);
public void printStatistics(){
logger.debug("Total writes: " + writeCount);
logger.debug("Total reads: " + readCount);
logger.debug("Total deletes: " + deleteCount);
logger.info("Total writes: " + writeCount);
logger.info("Total reads: " + readCount);
logger.info("Total deletes: " + deleteCount);
logger.debug("Total write time: " + writeTime);
logger.debug("Total read time: " + readTime);
logger.debug("Total delete time: " + deleteTime);
logger.info("Total write time: " + writeTime);
logger.info("Total read time: " + readTime);
logger.info("Total delete time: " + deleteTime);
logger.debug("Average write time in microseconds: " + (writeTime / writeCount));
logger.debug("Average read time in microseconds: " + (readTime / readCount));
logger.debug("Average delete time in microseconds: " + (deleteTime / deleteCount));
logger.info("Average write time in microseconds: " + (writeTime / writeCount));
logger.info("Average read time in microseconds: " + (readTime / readCount));
logger.info("Average delete time in microseconds: " + (deleteTime / deleteCount));
// logger.debug("Average Time/Request : " + timePerRequest/(writeCount + readCount + deleteCount));
// logger.info("Average Time/Request : " + timePerRequest/(writeCount + readCount + deleteCount));
// long avgTimePerReq = timePerRequest/(writeCount + readCount + deleteCount);
// logger.debug("Actual Requests/second : " + (1000000/avgTimePerReq));
// logger.info("Actual Requests/second : " + (1000000/avgTimePerReq));
}
public void collectExperimentStatistics(long time, int type){
// logger.debug("Time: " + time);
// logger.info("Time: " + time);
if (type == 100) {
writeCount ++;
writeTime = writeTime + time;
......
......@@ -3,9 +3,12 @@ package hpdos.protocol;
public class ByteArray {
public byte[] key;
private int h;
public ByteArray(byte[] key)
{
this.key = key;
System.out.println("new butearray");
h = 0;
}
@Override
public boolean equals(Object other)
......@@ -13,19 +16,29 @@ public class ByteArray {
if( other == null || !(other instanceof ByteArray))
return false;
if(this.key == other)
return true;
return true;
ByteArray b = (ByteArray)other;
int length = b.key.length;
if (this.key.length != length)
return false;
for (int i=0; i<length; i++)
if (this.key[i] != b.key[i])
return false;
return true;
}
@Override
public int hashCode()
{
if(h == 0 && key.length > 0)
{
for(int i = 0;i<key.length;i++)
{
h = 31 * h + key[i];
}
}
return h;
}
}
......@@ -25,7 +25,7 @@ public class Property {
masters = new String[numberOfMasters];
for(int i=1;i<=numberOfMasters;i++)
{
masters[i] = properties.getProperty("app.MASTER_HOST"+i);
masters[i-1] = properties.getProperty("app.MASTER_HOST"+i);
}
}
......
......@@ -27,7 +27,7 @@ public class InvalidationService extends InvalidationRpcProtocol implements DaRP
@Override
public void processServerEvent(DaRPCServerEvent<InvalidationRequest, InvalidationResponse> event) throws IOException
{
logger.debug("Got Invalidation Request");
logger.info("Got Invalidation Request");
InvalidationRequest request = event.getReceiveMessage();
InvalidationResponse response = event.getSendMessage();
try
......@@ -46,17 +46,17 @@ public class InvalidationService extends InvalidationRpcProtocol implements DaRP
public void open(DaRPCServerEndpoint<InvalidationRequest,InvalidationResponse> serverEp)
{
logger.debug("Recieved New Connection for invalidation");
logger.info("Recieved New Connection for invalidation");
try{
logger.debug(serverEp.getDstAddr());
logger.info(serverEp.getDstAddr());
}catch(Exception e){}
}
public void close(DaRPCServerEndpoint<InvalidationRequest,InvalidationResponse> serverEp)
{
logger.debug("Closing Connection for invalidation");
logger.info("Closing Connection for invalidation");
try{
logger.debug(serverEp.getDstAddr());
logger.info(serverEp.getDstAddr());
}catch(Exception e){}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment