Updated

code/hpdos_rdma_offloaded/README.md

+Set up disni
+
+export LD_LIBRARY_PATH=/usr/local/lib
+
 How to setup RocksDB jar
 1. Compiling RocksDb java from source code
     make -j8 rocksjava DEBUG_LEVEL=0

code/hpdos_rdma_offloaded/app.config

 app.HOST=192.168.200.20
 app.MASTER=true
-app.cpu_affinity=4
+#this is how to pair threads to cpu cores
+app.cpu_affinity=2
+//No of thread for each connectin
+app.rdma_cluster_size=1
 app.rdma_receive_queue=16
 app.rdma_send_queue=16
 app.rdma_polling=false

code/hpdos_rdma_offloaded/app_follower.config

 app.HOST=192.168.200.20
 app.MASTER=false
 app.cpu_affinity=4
+app.rdma_cluster_size=3
 app.rdma_receive_queue=16
 app.rdma_send_queue=16
 app.rdma_polling=false
@@ -8,6 +9,7 @@ app.rdma_max_inline=0
 app.rdma_server_port=1921
 app.NETWORK_HANDLER_THREADS=10
 app.REPLICATION_HANDLER_THREADS=10
+app.db_path=/tmp/rocks
 
 #The below properties are for master node only.
 app.follower_registration_port=9876

code/hpdos_rdma_offloaded/build.gradle

 apply plugin: "application"
 apply plugin: "java-library"
 
-application
-{
-    mainClassName = "hpdos_rdma_offloaded.MetadataServer"
-}
+mainClassName = "hpdos_rdma_offloaded.MetadataServer"
 version '1.0-SNAPSHOT'
 
 repositories

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/MetadataServer.java

 package hpdos_rdma_offloaded;
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.Properties;
 
@@ -18,13 +20,15 @@ import hpdos_rdma_offloaded.handler.RegistrationHandler;
 import hpdos_rdma_offloaded.handler.StorageHandler;
 import hpdos_rdma_offloaded.protocol.Request;
 import hpdos_rdma_offloaded.protocol.Response;
-import hpdos_rdma_offloaded.service.MetadataServerService;
+import hpdos_rdma_offloaded.service.FollowerMetadataService;
+import hpdos_rdma_offloaded.service.MasterMetadataServerService;
 
 public class MetadataServer implements Runnable{
     private String host;
     private int poolsize;
     private int recvQueue;
     private int sendQueue;
+    private int clusterSize;
     private int wqSize = recvQueue;
     private boolean polling;
     private int maxinline = 0;
@@ -34,49 +38,68 @@ public class MetadataServer implements Runnable{
     boolean isMaster;
     StorageHandler storageHandler;
     NetworkHandler networkHandler;
-    MetadataServerService metadataServerService;
+    MasterMetadataServerService metadataServerService = null;
+    FollowerMetadataService followerMetadataServerService = null;
     RegistrationHandler registrationHandler = null;
-    public final static Properties properties= new Properties();;
+    public final static Properties properties= new Properties();
+    String propertyPath;
 
-    public MetadataServer(String propertyPath) throws IOException,RocksDBException {
+    public MetadataServer(String propertyPath) throws Exception,RocksDBException {
+        this.propertyPath = propertyPath;
         InputStream inputStream = new FileInputStream(propertyPath);
         properties.load(inputStream);
-        boolean isMaster = Boolean.valueOf((String)properties.get("app.MASTER")); 
-        if(isMaster)
-        {
-            System.out.println("STARTING MASTER SERVICES.");
-        }
-        else
-        {
-            System.out.println("STARTING FOLLOWER SERVICES.");
-        }
-        isMaster = Boolean.valueOf((String)properties.get("app.MASTER"));
-        storageHandler = new StorageHandler(properties.getProperty("app.db_path"));
-        networkHandler = new NetworkHandler(this.storageHandler, properties);
-        metadataServerService =  new MetadataServerService(this.networkHandler);
-        registrationHandler = new RegistrationHandler(properties);
+        this.isMaster = Boolean.valueOf((String)properties.get("app.MASTER")); 
+        
 
-        host = (String) properties.get("app.HOST");
-        poolsize = Integer.valueOf( (String)properties.get("app.cpu_affinity"));
-        recvQueue = Integer.valueOf( (String)properties.get("app.rdma_receive_queue"));
-        sendQueue = Integer.valueOf( (String) properties.get("app.rdma_send_queue"));
+        host =  properties.getProperty("app.HOST");
+        poolsize = Integer.valueOf( properties.getProperty("app.cpu_affinity"));
+        recvQueue = Integer.valueOf(properties.getProperty("app.rdma_receive_queue"));
+        sendQueue = Integer.valueOf( properties.getProperty("app.rdma_send_queue"));
+        clusterSize = Integer.valueOf(properties.getProperty("app.rdma_cluster_size"));
         wqSize = recvQueue;
-        maxinline = Integer.valueOf((String) properties.get("app.rdma_max_inline"));
-        rdma_port = Integer.valueOf((String) properties.get("app.rdma_server_port"));
-        polling = Boolean.valueOf((String) properties.get("app.rdma_polling"));
+        maxinline = Integer.valueOf(properties.getProperty("app.rdma_max_inline"));
+        rdma_port = Integer.valueOf(properties.getProperty("app.rdma_server_port"));
+        polling = Boolean.valueOf(properties.getProperty("app.rdma_polling"));
+        int registrationPort = Integer.valueOf(properties.getProperty("app.follower_registration_port"));
+        
         clusterAffinities = new long[poolsize];
         for (int i = 0; i < poolsize; i++)
         {
             long cpu = 1L << i;
             clusterAffinities[i] = cpu;
+            System.out.println(cpu);
+        }
+
+        isMaster = Boolean.valueOf(properties.getProperty("app.MASTER"));
+        storageHandler = new StorageHandler(properties.getProperty("app.db_path"));
+        networkHandler = new NetworkHandler(isMaster);
+        // metadataServerService =  new MasterMetadataServerService(this.networkHandler,storageHandler);
+       registrationHandler = new RegistrationHandler(isMaster,host,registrationPort, networkHandler);
+       if(isMaster)
+        {
+            
+            System.out.println("STARTING MASTER SERVICES.");
+            metadataServerService =  new MasterMetadataServerService(this.networkHandler,storageHandler);
+        }
+        else
+        {
+            System.out.println("STARTING FOLLOWER SERVICES.");
+            followerMetadataServerService =  new FollowerMetadataService(storageHandler);
         }
+
     }
+
     public void run(){
         // MetadataServerService rpcService = new MetadataServerService();
         DaRPCServerGroup<Request, Response> group = null;
         try
         {
-            group = DaRPCServerGroup.createServerGroup(this.metadataServerService, this.clusterAffinities, -1, maxinline, polling, recvQueue, sendQueue, wqSize, 32);    
+            if (isMaster) {
+                group = DaRPCServerGroup.createServerGroup(this.metadataServerService, this.clusterAffinities, 1000, maxinline, polling, recvQueue, sendQueue, wqSize, clusterSize);        
+            } else {
+                group = DaRPCServerGroup.createServerGroup(this.followerMetadataServerService, this.clusterAffinities, 1000, maxinline, polling, recvQueue, sendQueue, wqSize, clusterSize);    
+            }
+            
             RdmaServerEndpoint<DaRPCServerEndpoint<Request, Response>> serverEp = null;
             serverEp = group.createServerEndpoint();
             InetSocketAddress address = new InetSocketAddress(host, rdma_port);
@@ -102,7 +125,10 @@ public class MetadataServer implements Runnable{
     public static void main(String[] args)  throws Exception
     {
             RocksDB.loadLibrary();
-            new MetadataServer(args[0]).run();   
-            System.out.println("Rocksdb Exception occur");
+            //var process = Runtime.getRuntime().exec("/bin/sh export LD_LIBRARY_PATH=/usr/local/lib");
+            //process.wait();
+           // System.out.println("Exit value "+process.exitValue());
+            Runnable r = new MetadataServer(args[0]);
+            new Thread(r).start();
     }
 }

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/handler/RegistrationHandler.java

@@ -7,31 +7,34 @@ import java.lang.ClassNotFoundException;
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
-import java.util.Properties;
 import java.util.UUID;
 
 public class RegistrationHandler {
     public ServerSocket server;
     public int port = 9876;
     boolean isMaster;
+    String registrationServerIp;
     int follower_registration_port;
     int sal_registration_port;
-    
-    public RegistrationHandler(Properties properties) throws IOException {
-        this.isMaster = Boolean.valueOf((String)properties.get("app.MASTER"));
-        
+    NetworkHandler networkHandler;
 
-        if (this.isMaster) {
-            this.follower_registration_port = Integer.valueOf((String)properties.get("app.follower_registration_port"));
-            this.sal_registration_port = Integer.valueOf((String)properties.get("app.follower_registration_port"));
+    public RegistrationHandler(boolean isMaster, String registrationServerIP, int registrationPort, NetworkHandler networkHandler) throws IOException {
+        this.registrationServerIp = registrationServerIP;
+        this.follower_registration_port = 9876;
+        this.sal_registration_port = 9875;
+        this.networkHandler = networkHandler;
+        if (isMaster)
+        {
+            
             startFollowerRegistrationHandlerServer();
             startSalRegistrationHandlerServer();    
-        } else {
+        }
+        else {
             // Follower follower = new Follower();
             // follower.setFollowerId( UUID.randomUUID().toString());
             // follower.setIpAddress("192.168.200.20");
             String followerUUID = UUID.randomUUID().toString();
-            String followerIp = "192.168.200.20";
+            String followerIp = registrationServerIP;
             String message = followerUUID + ";" + followerIp;
             sendRegistrationRequest(message);
         }
@@ -39,10 +42,10 @@ public class RegistrationHandler {
 
     private void sendRegistrationRequest(String message) throws IOException {
         System.out.println("Sending a registration request");
-        InetAddress host = InetAddress.getByName("192.168.200.20");
+        InetAddress host = InetAddress.getByName(this.registrationServerIp);
         Socket socket = null;
         ObjectOutputStream oos = null;
-        socket = new Socket(host.getHostName(), 9876);
+        socket = new Socket(this.registrationServerIp, 9876);
         oos = new ObjectOutputStream(socket.getOutputStream());
         oos.writeObject(message);
     }
@@ -50,10 +53,12 @@ public class RegistrationHandler {
     public void startFollowerRegistrationHandlerServer(){
         Runnable serverTask = new Runnable() {
             @Override
-            public void run() {
-                try {
-                    InetAddress addr = InetAddress.getByName("192.168.200.20");
-                    ServerSocket serverSocket = new ServerSocket(9876, 50, addr);
+            public void run()
+            {
+                try
+                {
+                    InetAddress addr = InetAddress.getByName(registrationServerIp);
+                    ServerSocket serverSocket = new ServerSocket(follower_registration_port, 50, addr);
                     System.out.println("Started the follower registration service...");
                     while (true) {
                         Socket socket = serverSocket.accept();
@@ -65,8 +70,8 @@ public class RegistrationHandler {
                         String followerIp = arr[1];
                         System.out.println("Got follower ip: " + followerIp + " , with UUID of: " + followerUUID);
                         
-                        NetworkHandler networkHandler = new NetworkHandler();
-                        networkHandler.connectToFollower(followerUUID, followerIp);
+                       NetworkHandler networkHandler = new NetworkHandler();
+                       networkHandler.connectToFollower(followerUUID, followerIp);
                         System.out.println("RDMA connection establised");
                         // Add the follower details to the respective class/component
                         // clientProcessingPool.submit(new ClientTask(clientSocket));
@@ -86,16 +91,18 @@ public class RegistrationHandler {
             @Override
             public void run() {
                 try {
-                    ServerSocket serverSocket = new ServerSocket(9875);
+                    InetAddress addr = InetAddress.getByName(registrationServerIp);
+                    ServerSocket serverSocket = new ServerSocket(sal_registration_port, 50, addr);
                     System.out.println("Started the SAL registration service...");
                     while (true) {
                         Socket socket = serverSocket.accept();
                         ObjectInputStream ois = new ObjectInputStream(socket.getInputStream());
-                        String message = (String) ois.readObject();
-                        // Send a response back with the list of followers to the SAL
+                        String ip = (String) ois.readObject();
+                        NetworkHandler networkHandler = new NetworkHandler();
+                        networkHandler.connectToSal(ip);
                         // clientProcessingPool.submit(new ClientTask(clientSocket));
                     }
-                } catch (IOException | ClassNotFoundException e) {
+                } catch (Exception e) {
                     System.err.println("Unable to process client request");
                     e.printStackTrace();
                 }

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/handler/StorageHandler.java

@@ -4,6 +4,9 @@ import org.rocksdb.RocksDB;
 import org.rocksdb.Options;
 import org.rocksdb.RocksDBException;
 
+import hpdos_rdma_offloaded.protocol.Request;
+
+
 public class StorageHandler implements AutoCloseable{
 
     RocksDB rocksDB;
@@ -27,16 +30,32 @@ public class StorageHandler implements AutoCloseable{
     public void create(byte[] key,byte[] value) throws RocksDBException{
         rocksDB.put(key,value);
     }
+    public void create(Request request) throws RocksDBException {
+        System.out.println("Writing to local");
+        rocksDB.put(request.getKey(),request.getValue());
+    }
     public byte[] read(byte[] key)throws RocksDBException
     {
         return rocksDB.get(key);
     }
+    public byte[] read(Request request)throws RocksDBException
+    {
+        return rocksDB.get(request.getKey());
+    }
     public void update(byte[] key,byte[] value) throws RocksDBException
     {
         rocksDB.put(key,value);
     }
+    public void update(Request request) throws RocksDBException
+    {
+        rocksDB.put(request.getKey(),request.getValue());
+    }
     public void delete(byte[] key) throws RocksDBException
     {
         rocksDB.delete(key);
     }
+    public void delete(Request request) throws RocksDBException
+    {
+        rocksDB.delete(request.getKey());
+    }
 }
\ No newline at end of file

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/protocol/AckType.java

@@ -6,6 +6,8 @@ public interface AckType
     public static int NOTFOUND = 1;
     public static int NOTALLOWED = 2;
     public static int DBFAILED = 3;
-    public static int SUCCESS_WITH_VALUE = 4;
+    public static int FAILED  = 4;
+    public static int SUCCESS_WITH_VALUE = 5;
+    public static int INVALID_REQUEST = 6;
 }
     
\ No newline at end of file

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/rdma_invalidation_protocol/InvalidationRequest.java → code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/protocol/InvalidationRequest.java

-package hpdos_rdma_offloaded.rdma_invalidation_protocol;
+package hpdos_rdma_offloaded.protocol;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -6,34 +6,31 @@ import java.nio.ByteBuffer;
 import com.ibm.darpc.DaRPCMessage;
 
 public class InvalidationRequest implements DaRPCMessage {
-    public String key;
-    private static char[] dst = new char[100];
-    private int SERIALIZED_SIZE = 256;
+    public byte[] key;
+    private static int SERIALIZED_SIZE = 128 ;
     @Override
     public int write(ByteBuffer buffer) throws IOException {
-        buffer.asCharBuffer().put(key);
+        buffer.put(key);
         return SERIALIZED_SIZE;
     }
 
     @Override
     public void update(ByteBuffer buffer) throws IOException {
-        buffer.asCharBuffer().get(dst, 0, 99);
-        String s = new String(dst);
-        s = s.trim();
-        key = s;
+        if(key == null)
+            key = new byte[128];
+        buffer.get(key);
     }
 
     @Override
     public int size() {
-        
         return SERIALIZED_SIZE;
     }
 
-    public String getKey() {
+    public byte[] getKey() {
         return key;
     }
 
-    public void setKey(String key) {
+    public void setKey(byte[] key) {
         this.key = key;
     }
 }

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/rdma_invalidation_protocol/InvalidationResponse.java → code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/protocol/InvalidationResponse.java

-package hpdos_rdma_offloaded.rdma_invalidation_protocol;
+package hpdos_rdma_offloaded.protocol;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -6,23 +6,32 @@ import java.nio.ByteBuffer;
 import com.ibm.darpc.DaRPCMessage;
 
 public class InvalidationResponse implements DaRPCMessage{
+    public static int SERIALIZED_SIZE = 4;
+    private int ack;
 
     @Override
-    public int write(ByteBuffer buffer) throws IOException {
-        
+    public int write(ByteBuffer buffer) throws IOException
+    {
+        buffer.putInt(ack);   
         return 0;
     }
 
     @Override
-    public void update(ByteBuffer buffer) throws IOException {
-        
-        
+    public void update(ByteBuffer buffer) throws IOException
+    {
+        buffer.getInt(ack);
     }
 
     @Override
     public int size() {
         
-        return 0;
+        return SERIALIZED_SIZE;
+    }
+    public void setAck(int ack){
+        this.ack = ack;
+    }
+    public int getAck()
+    {
+        return this.ack;
     }
-    
 }

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/rdma_invalidation_protocol/InvalidationRpcProtocol.java → code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/protocol/InvalidationRpcProtocol.java

-package hpdos_rdma_offloaded.rdma_invalidation_protocol;
+package hpdos_rdma_offloaded.protocol;
 
 import com.ibm.darpc.DaRPCProtocol;
 

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/protocol/Request.java

@@ -16,7 +16,7 @@ public class Request implements DaRPCMessage
     @Override
     public int write(ByteBuffer buffer) throws IOException
     {
-        System.out.println("Request Write Method");
+        // System.out.println("Request Write Method");
         buffer.putInt(requestType);
         buffer.put(key);
         //if operation type is get and delete then value is not required
@@ -33,7 +33,7 @@ public class Request implements DaRPCMessage
     @Override
     public void update(ByteBuffer buffer) throws IOException
     {
-        System.out.println("Request update method");
+        // System.out.println("Request update method");
         requestType = buffer.getInt();
         if(key == null || key.length != 128)
            this.key  = new byte[128];

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/protocol/RequestType.java

@@ -2,8 +2,8 @@ package hpdos_rdma_offloaded.protocol;
 
 public interface RequestType
 {
-    public static int GET =101;
     public static int PUT =100;
+    public static int GET =101;
     public static int DELETE =102;
     public static int INVALIDATE = 103;
     

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/protocol/Response.java

@@ -14,7 +14,7 @@ public class Response implements DaRPCMessage{
     @Override
     public int write(ByteBuffer buffer) throws IOException 
     {
-        System.out.println("Response write Method");
+        // System.out.println("Response write Method");
         buffer.putInt(ack);
         if(ack == AckType.SUCCESS_WITH_VALUE)
         {
@@ -27,7 +27,7 @@ public class Response implements DaRPCMessage{
     @Override
     public void update(ByteBuffer buffer) throws IOException
     {
-        System.out.println("esponse update method");
+        // System.out.println("esponse update method");
         ack = buffer.getInt();
         if(ack == AckType.SUCCESS_WITH_VALUE)
         {

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/service/MetadataServerService.java → code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/service/FollowerMetadataService.java

 package hpdos_rdma_offloaded.service;
 
 import java.io.IOException;
-import java.util.concurrent.ExecutionException;
 
+import com.ibm.darpc.DaRPCClientGroup;
 import com.ibm.darpc.DaRPCServerEndpoint;
 import com.ibm.darpc.DaRPCServerEvent;
 import com.ibm.darpc.DaRPCService;
 
 import org.rocksdb.RocksDBException;
 
-import hpdos_rdma_offloaded.handler.NetworkHandler;
+import hpdos_rdma_offloaded.handler.StorageHandler;
 import hpdos_rdma_offloaded.lib.Packet;
 import hpdos_rdma_offloaded.protocol.AckType;
+import hpdos_rdma_offloaded.protocol.InvalidationRequest;
+import hpdos_rdma_offloaded.protocol.InvalidationResponse;
 import hpdos_rdma_offloaded.protocol.Request;
 import hpdos_rdma_offloaded.protocol.RequestType;
 import hpdos_rdma_offloaded.protocol.Response;
 import hpdos_rdma_offloaded.protocol.RpcProtocol;
 
-public class MetadataServerService extends RpcProtocol implements DaRPCService<Request, Response>{
-    NetworkHandler networkHandler;
-
-    public MetadataServerService(NetworkHandler networkHandler){
-        this.networkHandler = networkHandler;
+public class FollowerMetadataService extends RpcProtocol implements DaRPCService<Request, Response>{
+    StorageHandler storageHandler;
+    DaRPCClientGroup<InvalidationRequest, InvalidationResponse> invalidationClientGroup;
+    public FollowerMetadataService(StorageHandler storageHandler) throws Exception{
+        System.out.println("New MetadataServerService Created");
+        this.storageHandler  = storageHandler;
     }
 
     @Override
     public void processServerEvent(DaRPCServerEvent<Request, Response> event) throws IOException {
         Request request = event.getReceiveMessage();
         Response response = event.getSendMessage();
-        if (request.getRequestType() == RequestType.PUT) {
-            // Write code for create
-           Packet packet =  new Packet(request.getRequestType(), request.getKey(), request.getValue());
-           try
-           {
-                this.networkHandler.create(packet);
+        
+        System.out.print("Received "+request.getRequestType()+" Request for key: " + new String(request.getKey()));
+        if(request.getValue() != null)
+            System.out.print(" value : "+ new String(request.getValue()) );
+        System.out.println();
+        
+        try{
+            if (request.getRequestType() == RequestType.PUT) {
+                this.storageHandler.create(request);
                 response.setAck(AckType.SUCCESS);
                 response.setKey(null);
                 response.setValue(null);
-            }
-            catch(RocksDBException ex)
-            {
-                ex.printStackTrace();
-                response.setAck(AckType.DBFAILED);
-            }
-            catch (InterruptedException | ExecutionException e) {
-                e.printStackTrace();
-            }
-        } 
-        else if(request.getRequestType() == RequestType.GET) {
-            Packet packet =  new Packet(request.getRequestType(), request.getKey(), null);
-            try
-            {
-                byte[] value = this.networkHandler.read(packet);
+            } 
+            else if(request.getRequestType() == RequestType.GET) {
+                byte[] value = this.storageHandler.read(request); 
                 if(value == null)
                 {
                     response.setAck(AckType.NOTFOUND);
@@ -62,37 +56,24 @@ public class MetadataServerService extends RpcProtocol implements DaRPCService<R
                     response.setValue(value);
                 }
             }
-            catch(RocksDBException ex)
-            {
-                ex.printStackTrace();
-                response.setAck(AckType.DBFAILED);
-            }
-            catch (InterruptedException | ExecutionException e) {
-                e.printStackTrace();
-            }
-            // Write code for read
-        }   
-        else if(request.getRequestType() == RequestType.DELETE){
-            try{
-                Packet packet =  new Packet(request.getRequestType(), request.getKey(), null);
-                this.networkHandler.delete(packet);
+            else if(request.getRequestType() == RequestType.DELETE){
+                this.storageHandler.delete(request);
                 response.setAck(AckType.SUCCESS);
             }
-            catch(RocksDBException ex)
-            {
+            else{
+                response.setAck(AckType.INVALID_REQUEST);
+            }
+        }
+        catch(RocksDBException ex){
                 ex.printStackTrace();
                 response.setAck(AckType.DBFAILED);
-            }catch (InterruptedException | ExecutionException e) {
-                e.printStackTrace();
-            }
-            // Write code for delete
         }
         event.triggerResponse();
     }
 
     @Override
     public void open(DaRPCServerEndpoint<Request, Response> rpcClientEndpoint) {
-        System.out.println("Received new connection...");
+        System.out.print("Received new connection  ");
     }
 
     @Override

code/hpdos_rdma_offloaded/src/main/java/hpdos_rdma_offloaded/service/MasterMetadataServerService.java

+package hpdos_rdma_offloaded.service;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+
+import com.ibm.darpc.DaRPCClientEndpoint;
+import com.ibm.darpc.DaRPCClientGroup;
+import com.ibm.darpc.DaRPCServerEndpoint;
+import com.ibm.darpc.DaRPCServerEvent;
+import com.ibm.darpc.DaRPCService;
+import com.ibm.darpc.DaRPCStream;
+
+import org.rocksdb.RocksDBException;
+
+import hpdos_rdma_offloaded.handler.NetworkHandler;
+import hpdos_rdma_offloaded.handler.StorageHandler;
+import hpdos_rdma_offloaded.lib.Packet;
+import hpdos_rdma_offloaded.protocol.AckType;
+import hpdos_rdma_offloaded.protocol.InvalidationRequest;
+import hpdos_rdma_offloaded.protocol.InvalidationResponse;
+import hpdos_rdma_offloaded.protocol.InvalidationRpcProtocol;
+import hpdos_rdma_offloaded.protocol.Request;
+import hpdos_rdma_offloaded.protocol.RequestType;
+import hpdos_rdma_offloaded.protocol.Response;
+import hpdos_rdma_offloaded.protocol.RpcProtocol;
+
+public class MasterMetadataServerService extends RpcProtocol implements DaRPCService<Request, Response>{
+    NetworkHandler networkHandler;
+    StorageHandler storageHandler;
+    ConcurrentHashMap<SocketAddress,DaRPCStream<InvalidationRequest, InvalidationResponse>> invalidationClientEpStreams;
+    ConcurrentHashMap<SocketAddress,DaRPCClientEndpoint<InvalidationRequest, InvalidationResponse>> invalidationClientEps;
+    DaRPCClientGroup<InvalidationRequest, InvalidationResponse> invalidationClientGroup;
+    public MasterMetadataServerService(NetworkHandler networkHandler,StorageHandler storageHandler) throws Exception{
+        System.out.println("New MetadataServerService Created");
+        this.networkHandler = networkHandler;
+        this.storageHandler  = storageHandler;
+        this.invalidationClientEpStreams = new ConcurrentHashMap<>();
+        this.invalidationClientEps = new ConcurrentHashMap<>();
+        this.networkHandler.setClientStreams(this.invalidationClientEpStreams);
+        InvalidationRpcProtocol rpcProtocol = new InvalidationRpcProtocol();