kvcache read offload: mtcp hashmap code

mtcp-dpdk/mtcp_kvcache/.gitignore

+Makefile
+epserver
+epwget
+log_*

mtcp-dpdk/mtcp_kvcache/Makefile.in

+# TODO: Make this Makefile.in pretty
+
+TARGETS = epserver epwget
+CC=@CC@ -g -O3 -Wall -Werror -fgnu89-inline
+DPDK=@DPDK@
+PS=@PSIO@
+NETMAP=@NETMAP@
+ONVM=@ONVM@
+CCP=@CCP@
+CFLAGS=@CFLAGS@
+LDFLAGS=@LDFLAGS@
+
+# If ARCH is not defined, retrive from system
+ARCH ?= $(shell uname -m)
+
+# Add arch-specific optimization
+ifeq ($(ARCH),x86_64)
+LIBS += -m64
+endif
+
+# mtcp library and header 
+MTCP_FLD    =../../mtcp/
+MTCP_INC    =-I${MTCP_FLD}/include
+MTCP_LIB    =-L${MTCP_FLD}/lib
+MTCP_TARGET = ${MTCP_LIB}/libmtcp.a
+
+UTIL_FLD = ../../util
+UTIL_INC = -I${UTIL_FLD}/include
+UTIL_OBJ = ${UTIL_FLD}/http_parsing.o ${UTIL_FLD}/tdate_parse.o ${UTIL_FLD}/netlib.o
+
+# util library and header
+INC = -I./include/ ${UTIL_INC} ${MTCP_INC} -I${UTIL_FLD}/include
+LIBS = ${MTCP_LIB}
+
+# psio-specific variables
+ifeq ($(PS),1)
+PS_DIR = ../../io_engine/
+PS_INC = ${PS_DIR}/include
+INC += -I{PS_INC}
+LIBS += -lmtcp -L${PS_DIR}/lib -lps -lpthread -lnuma -lrt
+endif
+
+# netmap-specific variables
+ifeq ($(NETMAP),1)
+LIBS += -lmtcp -lpthread -lnuma -lrt
+endif
+
+# dpdk-specific variables
+ifeq ($(DPDK),1)
+DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
+DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
+LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS} ${LDFLAGS}
+endif
+
+# onvm-specific variables
+ifeq ($(ONVM),1)
+ifeq ($(RTE_TARGET),)
+$(error "Please define RTE_TARGET environment variable")
+endif
+
+INC += -I@ONVMLIBPATH@/onvm_nflib
+INC += -I@ONVMLIBPATH@/lib
+INC += -DENABLE_ONVM
+LIBS += @ONVMLIBPATH@/onvm_nflib/$(RTE_TARGET)/libonvm.a
+LIBS += @ONVMLIBPATH@/lib/$(RTE_TARGET)/lib/libonvmhelper.a -lm
+endif
+
+ifeq ($V,) # no echo
+	export MSG=@echo
+	export HIDE=@
+else
+	export MSG=@\#
+	export HIDE=
+endif
+
+ifeq ($(CCP), 1)
+# LIBCCP
+LIBCCP = $(MTCP_FLD)/src/libccp
+LIBS += -L$(LIBCCP) -lccp -lstartccp
+INC += -I$(LIBCCP)
+endif
+
+all: epserver epwget
+
+epserver.o: epserver.c
+	$(MSG) "   CC $<"
+	$(HIDE) ${CC} -c $< ${CFLAGS} ${INC}
+
+epserver: epserver.o ${MTCP_FLD}/lib/libmtcp.a
+	$(MSG) "   LD $<"
+	$(HIDE) ${CC} $< ${LIBS} ${UTIL_OBJ} -o $@
+
+epwget.o: epwget.c
+	$(MSG) "   CC $<"
+	$(HIDE) ${CC} -c $< ${CFLAGS} ${INC}
+
+epwget: epwget.o ${MTCP_FLD}/lib/libmtcp.a
+	$(MSG) "   LD $<"
+	$(HIDE) ${CC} $< ${LIBS} ${UTIL_OBJ} -o $@
+
+clean:
+	$(MSG) "   CLEAN $(TARGETS)"
+	$(HIDE) rm -f *~ *.o ${TARGETS} log_*
+
+distclean: clean
+	rm -rf Makefile

mtcp-dpdk/mtcp_kvcache/Makefile.old

+TARGETS = epserver epwget
+CC = gcc -g -O3
+DPDK=0
+PS=0
+
+# DPDK LIBRARY and HEADER
+DPDK_INC=../../dpdk/include
+DPDK_LIB=../../dpdk/lib/
+
+# mtcp library and header 
+MTCP_FLD    =../../mtcp/
+MTCP_INC    =-I${MTCP_FLD}/include
+MTCP_LIB    =-L${MTCP_FLD}/lib
+MTCP_TARGET = ${MTCP_LIB}/libmtcp.a
+
+UTIL_FLD = ../../util
+UTIL_INC = -I${UTIL_FLD}/include
+UTIL_OBJ = ${UTIL_FLD}/http_parsing.o ${UTIL_FLD}/tdate_parse.o
+
+
+PS_DIR = ../../io_engine/
+PS_INC = ${PS_DIR}/include
+INC = -I./include/ ${UTIL_INC} ${MTCP_INC} -I${UTIL_FLD}/include
+LIBS = ${MTCP_LIB}
+ifeq ($(PS),1)
+INC += -I{PS_INC}
+LIBS += -lmtcp -L${PS_DIR}/lib -lps -lpthread -lnuma -lrt
+endif
+
+# CFLAGS for DPDK-related compilation
+INC += ${MTCP_INC}
+ifeq ($(DPDK),1)
+INC += -DENABLE_DPDK -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 \
+        -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 \
+        -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX \
+        -DRTE_COMPILE_TIME_CPUFLAGS=RTE_CPUFLAG_SSE,RTE_CPUFLAG_SSE2,RTE_CPUFLAG_SSE3,RTE_CPUFLAG_SSSE3,RTE_CPUFLAG_SSE4_1,RTE_CPUFLAG_SSE4_2,RTE_CPUFLAG_AES,RTE_CPUFLAG_PCLMULQDQ,RTE_CPUFLAG_AVX -I${DPDK_INC} \
+        -include ${DPDK_INC}/rte_config.h
+endif
+
+ifeq ($(DPDK),1)
+LIBS += -m64 -g -O3 -pthread -lrt -march=native -Wl,-export-dynamic ${MTCP_FLD}/lib/libmtcp.a -L../../dpdk/lib -Wl,-lnuma -Wl,-lmtcp -Wl,-lpthread -Wl,-lrt -Wl,-ldl -Wl,--whole-archive -Wl,-lrte_distributor -Wl,-lrte_kni -Wl,-lrte_pipeline -Wl,-lrte_table -Wl,-lrte_port -Wl,-lrte_timer -Wl,-lrte_hash -Wl,-lrte_lpm -Wl,-lrte_power -Wl,-lrte_acl -Wl,-lrte_meter -Wl,-lrte_sched -Wl,-lm -Wl,-lrt -Wl,--start-group -Wl,-lrte_kvargs -Wl,-lrte_mbuf -Wl,-lrte_ip_frag -Wl,-lethdev -Wl,-lrte_malloc -Wl,-lrte_mempool -Wl,-lrte_ring -Wl,-lrte_eal -Wl,-lrte_cmdline -Wl,-lrte_cfgfile -Wl,-lrte_pmd_bond -Wl,-lrte_pmd_vmxnet3_uio -Wl,-lrte_pmd_i40e -Wl,-lrte_pmd_ixgbe -Wl,-lrte_pmd_e1000 -Wl,-lrte_pmd_ring -Wl,-lrt -Wl,-lm -Wl,-ldl -Wl,--end-group -Wl,--no-whole-archive
+endif
+
+all: epserver epwget
+
+epserver.o: epserver.c
+	${CC} -c $< ${CFLAGS} ${INC}
+
+epserver: epserver.o
+	${CC} $< ${LIBS} ${UTIL_OBJ} -o $@
+
+epwget.o: epwget.c
+	${CC} -c $< ${CFLAGS} ${INC}
+
+epwget: epwget.o
+	${CC} $< ${LIBS} ${UTIL_OBJ} -o $@
+
+clean:
+	rm -f *~ *.o ${TARGETS} log_*

mtcp-dpdk/mtcp_kvcache/README

+========================================================================
+ USAGE OF EXAMPLE APPLICATIONS
+========================================================================
+
+epserver: a simple mtcp-epoll-based web server
+   Single-Process, Multi-threaded Usage:
+      ./epserver -p www_home -f epserver.conf [-N #cores] 
+      ex) ./epserver -p /home/notav/www -f epserver.conf -N 8
+
+   Multi-Process, Single-threaded Usage [DPDK-only]
+      (Master runs on core 0 by default, Slave processes on core 1~N)
+      ex) ./epserver -p /home/notav/www -f epserver-multiprocess.conf -c 0
+      	  for i in {1..7}
+	  do
+	     ./epserver -p /home/notav/www -f epserver-mutliprocess.conf -c $i
+	  done
+
+
+options:
+  www_home: the directory to server. # max files are limited to 
+            MAX_FILES in epserver.c:36
+  -N: number of CPU cores to use. default: all existing cores
+  -p: path to www/ files
+  -f: path to mtcp configuration file
+  -c: the core_id on which the process should run
+      [only works for multi-process mode]
+
+========================================================================
+
+epwget: simple mtcp-epoll-based http request generator
+ Single-Process, Multi-threaded Usage:
+ usage: ./epwget URL #requests [-N #cores] [-c concurrency] -f $mtcp_conf
+    ex) ./epwget 10.0.0.43/example.txt 10000000 -N 8 -c 8000 -f epwget.conf
+
+ Multi-Process, Single-threaded Usage [DPDK-only]
+    (Master runs on core 0 by default, Slave processes on core 1~N)
+ usage: ./epwget URL #requests -n $core [-c concurrency] -f $mtcp_conf
+    ex) ./epwget 10.0.0.43/example.txt 10000000 -n 0 -c 1000 -f epwget-multiprocess.conf
+        for i in {1..7}
+	do
+	   ./epwget 10.0.0.43/example.txt 1000000 -n $i -c 1000 -f epwget-multiprocess.conf
+	done
+
+options:
+  URL: url of the content to download.
+  #requests: number of requests to generate
+  -N: number of CPU cores to use. default: min(# cores, # requests)
+  -c: number of maximum concurrent connections. default: 100
+  -f: path to mtcp configuration file
+  -n: the core_id on which the process should run
+      [only works for multi-process mode]
+
+notes:
+  - epwget can use a range of IP addresses for larger concurrent 
+	connections that cannot be in an IP. you can set it in epwget.c:33.
+  - epwget overrides some part of the settings in epgwet.conf and uses 
+    mtcp_setconf() internally to apply the input arguments to the 
+    configuration.
+
+========================================================================
+
+ONVM setups:
+The config file provides simple onvm mtcp setups such as:
+  - simple endpoint server
+    	  - Enable `io = onvm`, and `onvm_serv = 1` in epserver.conf
+	  - Run onvm
+	    	$ onvm/go.sh 1,2,3 1 -s stdout
+	  - Run epserver
+	    	$ sudo ./epserver -p /path/to/www -f epserver.conf -N 1
+  - local client/server setup
+    	  - Enable `io = onvm`, `onvm_serv` = 1, and `onvm_dest = 2` in epserver.conf
+	  - Enable `io = onvm`, `onvm_serv` = 2, and `onvm_dest = 1` in epwget.conf
+	  - Run onvm
+	    	$ onvm/go.sh 1,2,3 1 -s stdout
+	  - Run epserver
+	    	$ sudo ./epserver -p /path/to/www -f epserver.conf -N 1
+	  - Run epwget
+	    	$ sudo ./epwget $SERVER_IP/foo.html 10000000 -N 1 -c 1024 -f epwget.conf
+  
+
+PLEASE NOTE THAT YOU WILL HAVE TO ADD STATIC ARP TABLE ENTRIES TO RUN
+MTCP IN ONVM MODE.
+
+========================================================================
+
+                   Contact: mtcp-user at list.ndsl.kaist.edu
+                             April 2, 2014. 
+                 EunYoung Jeong <notav at ndsl.kaist.edu>
+		 M. Asim Jamshed <ajamshed at ndsl.kaist.edu>

mtcp-dpdk/mtcp_kvcache/config/arp.conf

+# This file is to configure static arp tables.
+# Rename this file to arp.conf and set the appropriate values.
+# Please save this file as config/arp.conf. Put the config/
+# directory in the same directory where the binary lies.
+#
+# (Destination IP address/IP_prefix) (Destination MAC address)
+
+ARP_ENTRY 5
+192.168.220.35/32 0c:42:a1:df:ac:48
+192.168.220.34/32 0c:42:a1:df:ac:49
+192.168.220.33/32 ee:cc:b5:55:cc:dc
+192.168.220.60/32 0c:42:a1:df:ac:42
+192.168.220.61/32 0c:42:a1:df:ac:43

mtcp-dpdk/mtcp_kvcache/config/route.conf

+# This file is routing table example of our testbed machine
+# Copy this file to route.conf and give appropriate routes
+# Please save this file as config/route.conf. Put the config/
+# directory in the same directory where the binary lies.
+#
+# (Destination address)/(Prefix) (Device name)
+#
+#
+# Add entry for default gateway route as:
+# w.x.y.z/0 dpdk0
+# Always put the default gateway route as the last entry.
+# Make sure that the mask (Prefix) is set to 0. For example,
+# if the default gateway IP address is 10.0.0.10, then the
+# entry will be:
+# 10.0.0.10/0 dpdk0
+#
+
+ROUTES 5
+192.168.220.34/24 ens259f0
+192.168.220.35/24 ens259f1
+192.168.220.33/24 ens259f1
+192.168.220.60/24 ens259f1
+192.168.220.61/24 ens259f0

mtcp-dpdk/mtcp_kvcache/epserver-multiprocess.conf

+############### mtcp configuration file ###############
+
+# The underlying I/O module you want to use. Please
+# enable only one out of the three.
+#io = psio
+#io = netmap
+io = dpdk
+
+# No. of cores setting (enabling this option will override
+# the `cpu' config for those applications that accept
+# num_cores as command line arguments)
+# 
+# e.g. in case ./epserver is executed with `-N 4', the
+# mtcp core will still invoke 8 mTCP threads if the
+# following line is uncommented.
+#num_cores = 8
+
+# Number of memory channels per processor socket (dpdk-only)
+num_mem_ch = 4
+
+# Enable multi-process support
+multiprocess = 1
+
+# Used port (please adjust accordingly)
+#------ PSIO ports -------#
+#port = xge0 xge1
+#port = xge1
+#------ DPDK ports -------#
+port = dpdk0
+#port = dpdk1
+#port = dpdk0 dpdk1
+
+# Maximum concurrency per core (default = 10000)
+#max_concurrency = 10000
+
+# Maximum number of socket buffers per core (default = 10000)
+# Set this to small value if there are many idle connections
+#max_num_buffers = 10000
+
+# Receive buffer size of sockets; if not set: rcvbuf = sndbuf
+rcvbuf = 8192
+
+# Send buffer size of sockets; if not set: sndbuf = rcvbuf
+sndbuf = 8192
+
+# if sndbuf & rcvbuf not set: sndbuf = rcvbuf = 8192
+
+# TCP timeout seconds
+# (tcp_timeout = -1 can disable the timeout check)
+tcp_timeout = 30
+
+# TCP timewait seconds
+tcp_timewait = 0
+
+# Interface to print stats (please adjust accordingly)
+# You can enable multiple ports in separate lines
+#------ PSIO ports -------#
+#stat_print = xge0
+#stat_print = xge1
+#------ DPDK ports -------#
+stat_print = dpdk0
+#stat_print = dpdk1
+
+#######################################################
\ No newline at end of file

mtcp-dpdk/mtcp_kvcache/epserver.conf

+############### mtcp configuration file ###############
+
+# The underlying I/O module you want to use. Please
+# enable only one out of the four.
+#io = psio
+#io = netmap
+#io = onvm
+io = dpdk
+
+# No. of cores setting (enabling this option will override
+# the `cpu' config for those applications that accept
+# num_cores as command line arguments)
+# 
+# e.g. in case ./epserver is executed with `-N 4', the
+# mtcp core will still invoke 8 mTCP threads if the
+# following line is uncommented.
+num_cores = 4
+
+# Core mask
+#core_mask = 0000000F0
+core_mask = 0F
+
+# number of TX descriptor ring size, the default is 128 (dpdk-only)
+# e.g. in case of VMXNET3 PMD, the min TX ring size is 512
+# this allows mTCP app runs on VMware ESXi VM
+num_tx_desc = 512
+
+# number of RX descriptor ring size, the default is 128 (dpdk-only)
+num_rx_desc = 128
+
+# Number of memory channels per processor socket (dpdk-only)
+num_mem_ch = 4
+
+#--- ONVM specific args ---#
+# Service id (required)
+#onvm_serv = 1
+# Instance id (optional)
+#onvm_inst = 1
+# Destination id (will forward to another NF)
+# If not set will send packets out
+#onvm_dest = 2
+
+# Sample ONVM configurations
+# Single node epserver <-> epwget 
+#onvm_serv = 1
+#onvm_dest = 2
+
+# Simple endpoint server multi node setup
+#onvm_serv = 1
+#--------------------------#
+
+# Enable multi-process support
+#multiprocess = 1
+
+# Used port (please adjust accordingly)
+#------ PSIO ports -------#
+#port = xge0 xge1
+#port = xge1
+#------ DPDK ports -------#
+port = ens259f1 #ens259f0
+#port = dpdk1
+#port = dpdk0 dpdk1
+
+# Congestion control algorithm
+# (only available when configured with --enable-ccp)
+# cc = reno
+# cc = cubic
+
+# Maximum concurrency per core (default = 10000)
+#max_concurrency = 10000
+
+# Maximum number of socket buffers per core (default = 10000)
+# Set this to small value if there are many idle connections
+#max_num_buffers = 10000
+
+# Receive buffer size of sockets; if not set: rcvbuf = sndbuf
+rcvbuf = 8192
+
+# Send buffer size of sockets; if not set: sndbuf = rcvbuf
+sndbuf = 8192
+
+# if sndbuf & rcvbuf not set: sndbuf = rcvbuf = 8192
+
+# TCP timeout seconds
+# (tcp_timeout = -1 can disable the timeout check)
+tcp_timeout = 30
+
+# TCP timewait seconds
+tcp_timewait = 0
+
+# Interface to print stats (please adjust accordingly)
+# You can enable multiple ports in a line
+#------ PSIO ports -------#
+#stat_print = xge0
+#stat_print = xge1
+#------ DPDK ports -------#
+stat_print = ens259f1 #ens259f0
+#stat_print = dpdk0 dpdk1
+
+#######################################################

mtcp-dpdk/mtcp_kvcache/epwget-multiprocess.conf

+############### mtcp configuration file ###############
+# The underlying I/O module you want to use. Please
+# enable only one out of the three.
+#io = psio
+#io = netmap
+io = dpdk
+
+# No. of cores setting (enabling this option will override
+# the `cpu' config for those applications that accept
+# num_cores as command line arguments)
+# 
+# e.g. in case ./epserver is executed with `-N 4', the
+# mtcp core will still invoke 8 mTCP threads if the
+# following line is uncommented.
+#num_cores = 8
+
+# Number of memory channels per processor socket (dpdk-only)
+num_mem_ch = 4
+
+# Enable multi-process support
+multiprocess = 1
+
+# Used port (please adjust accordingly)
+#------ PSIO ports -------#
+#port = xge0 xge1
+#port = xge1
+#------ DPDK ports -------#
+port = dpdk0
+#port = dpdk1
+#port = dpdk0 dpdk1
+
+# Maximum concurrency per core (default = 10000)
+#max_concurrency = 10000
+
+# Maximum number of socket buffers per core (default = 10000)
+# Set this to small value if there are many idle connections
+#max_num_buffers = 10000
+
+# Receive buffer size of sockets; if not set: rcvbuf = sndbuf
+rcvbuf = 8192
+
+# Send buffer size of sockets; if not set: sndbuf = rcvbuf
+sndbuf = 8192
+
+# if sndbuf & rcvbuf not set: sndbuf = rcvbuf = 8192
+
+# TCP timeout seconds
+# (tcp_timeout = -1 can disable the timeout check)
+tcp_timeout = 30
+
+# TCP timewait seconds
+tcp_timewait = 0
+
+# Interface to print stats (please adjust accordingly)
+# You can enable multiple ports in separate lines
+#------ PSIO ports -------#
+#stat_print = xge0
+#stat_print = xge1
+#------ DPDK ports -------#
+stat_print = dpdk0
+#stat_print = dpdk1
+
+#######################################################
\ No newline at end of file

mtcp-dpdk/mtcp_kvcache/epwget.conf

+############### mtcp configuration file ###############
+
+# The underlying I/O module you want to use. Please
+# enable only one out of the four.
+#io = psio
+#io = onvm
+#io = netmap
+io = dpdk
+
+# No. of cores setting (enabling this option will override
+# the `cpu' config for those applications that accept
+# num_cores as command line arguments)
+# 
+# e.g. in case ./epwget is executed with `-N 4', the
+# mtcp core will still invoke 8 mTCP threads if the
+# following line is uncommented.
+num_cores = 4
+
+# number of TX descriptor ring size, the default is 128 (dpdk-only)
+# e.g. in case of VMXNET3 PMD, the min TX ring size is 512
+# this allows mTCP app runs on VMware ESXi VM
+num_tx_desc = 512
+
+# number of RX descriptor ring size, the default is 128 (dpdk-only)
+num_rx_desc = 128
+
+# Number of memory channels per processor socket (dpdk-only)
+num_mem_ch = 4
+
+#--- ONVM specific args ---#
+# Service id (required)
+#onvm_serv = 2
+# Dest id (used to forward traffic to specific NF)
+#onvm_dest = 1
+
+# Sample ONVM configurations
+# Single node epserver <-> epwget 
+#onvm_serv = 2
+#onvm_dest = 1
+
+# Simple client for multi node setup
+#onvm_serv = 1
+#--------------------------#
+
+# Used port (please adjust accordingly)
+#------ PSIO ports -------#
+#port = xge0 xge1
+#port = xge1
+#------ DPDK ports -------#
+port = ens259f1 #ens259f0
+#port = dpdk1
+#port = dpdk0 dpdk1
+
+# Enable multi-process support
+#multiprocess = 1
+
+# Congestion control algorithm
+# (only available when configured with --enable-ccp)
+# cc = reno
+# cc = cubic
+
+# Receive buffer size of sockets; if not set: rcvbuf = sndbuf
+rcvbuf = 8192
+
+# Send buffer size of sockets; if not set: sndbuf = rcvbuf
+sndbuf = 8192
+
+# if sndbuf & rcvbuf not set: sndbuf = rcvbuf = 8192
+
+# Maximum concurrency per core (default = 10000)
+#max_concurrency = 10000
+
+# Maximum number of socket buffers per core (default = 10000)
+# Set this to small value if there are many idle connections
+#max_num_buffers = 10000
+
+# TCO timeout seconds
+# (tcp_timeout = -1 can disable the timeout check)
+tcp_timeout = 30
+
+# TCP timewait seconds
+tcp_timewait = 0
+
+# Interface to print stats (please adjust accordingly)
+# You can enable multiple ports in a line
+#------ PSIO ports -------#
+#stat_print = xge0
+#stat_print = xge1
+#------ DPDK ports -------#
+stat_print = ens259f1 #ens259f0
+#stat_print = dpdk0 dpdk1
+
+#######################################################