Working solution for dispatcher NIC offload

3a6cef8c · Shah Rinku · 2867ff9b · 3a6cef8c · 3a6cef8c · 3a6cef8c
Commit 3a6cef8c authored Apr 24, 2021 by Shah Rinku
27 changed files
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,29 @@ secrets.json
 resource_system/bin/**
 resource_system/version.linux
 local_experiments/
+.vscode
+
+p4src/Makefile-nfp4build
+p4src/app_master.list/
+p4src/blm0.list/
+p4src/echo.nffw
+p4src/echo.yml
+p4src/flowcache_timeout_emu0.list/
+p4src/gro0.list/
+p4src/gro1.list/
+p4src/nbi_init_csr.list/
+p4src/nfd_pcie0_notify.list/
+p4src/nfd_pcie0_pci_in_gather.list/
+p4src/nfd_pcie0_pci_in_issue0.list/
+p4src/nfd_pcie0_pci_in_issue1.list/
+p4src/nfd_pcie0_pci_out_me0.list/
+p4src/nfd_pcie0_pd0.list/
+p4src/nfd_pcie0_pd1.list/
+p4src/nfd_pcie0_sb.list/
+p4src/nfd_svc.list/
+p4src/out/
+p4src/pif_app_nfd.list/
+client/Makefile-nfp4build
+*.list
+p4src/out_dir
+*.nffw
--- a/dispatch_system/benchmark_dispatcher.py
+++ b/dispatch_system/benchmark_dispatcher.py
+import socket
+import struct
+import time
+import threading
+import random
+import time
+
+import argparse
+
+parser = argparse.ArgumentParser(description='Mininet demo')
+
+parser.add_argument('--fid', help='Funtion id',
+                    type=int, action="store", required=False)
+parser.add_argument('--c', help='Concurrency',
+                   type=int, action="store", required=True)
+group = parser.add_mutually_exclusive_group(required=True)
+group.add_argument('--rps', help='Requests per second',
+                   type=int, action="store")
+group.add_argument('--n', help='Number of requests to send',
+                   type=int, action="store")
+
+
+args = parser.parse_args()
+
+PORT = 8000
+dataInt = 0
+fid = args.fid
+SERVER_IP = "192.168.2.3"
+
+egress_time = []
+ingress_time = []
+stop_thread = False
+
+
+def receive():
+    global egress_time, stop_thread
+    CLIENT_IP = "0.0.0.0"
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    s.bind((CLIENT_IP, 8080))
+    print "listening to {} at port {}".format(CLIENT_IP, 8080)
+    run_status = {}
+
+    while True:
+        if stop_thread:
+            break
+        packet, addr = s.recvfrom(1024)
+        # print packet
+        base = 0
+        chain_id = struct.unpack(">I", packet[base:base + 4])[0]
+        base += 4
+        exec_id = struct.unpack(">I", packet[base:base + 4])[0]
+        base += 4
+        function_id = struct.unpack(">I", packet[base:base + 4])[0]
+        base += 4
+        data = struct.unpack(">I", packet[base:base + 4])[0]
+        base += 4
+        function_count = struct.unpack("B", packet[base])[0]
+
+        t = int(time.time() * 1000) % 1000000000
+        data = int(data) - t
+        print "rec", chain_id, exec_id, data, function_id, function_count, 
+
+
+def genPacket():
+    global fid
+    packet = None
+    exec_id = random.randint(0, 2 ** 30)
+    print exec_id
+    chain_id = 1
+    function_count = 5
+    function_id = fid if (fid) else 1
+    f0 = 0; f1 = 1; f2 = 2; f3 = 0; f4 = 0
+    print chain_id, exec_id, "function_id", function_id, function_count, \
+        f0, f1, f2, f3, f4
+    
+    chain_id = struct.pack(">I", chain_id)  # chain id
+    exec_id = struct.pack(">I", exec_id)  # execution id
+
+    dataInt = int(time.time() * 1000) % 1000000000
+    data = struct.pack(">I", dataInt)  # data
+    
+    function_count = struct.pack("B", function_count)  # function count
+    function_id_packed = struct.pack(">I", function_id)
+    f0 = struct.pack("B", f0)  # f0
+    f1 = struct.pack("B", f1)  # f1
+    f2 = struct.pack("B", f2)  # f2 -> f0
+    f3 = struct.pack("B", f3)  # f3 -> f1 f2
+    f4 = struct.pack("B", f4)  # f4 -> f3
+    
+    packet = chain_id + exec_id + function_id_packed + data + function_count + f0 + f1 + f2 + f3 + f4
+    # print dataInt, offload_status
+    return packet, function_id
+
+
+def sendThread(start_time, runtime, sleep_time):
+    global ingress_time
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    while True:
+            packet, function_id = genPacket()
+            if time.time() - start_time > runtime:
+                break
+            s.sendto(packet, (SERVER_IP, PORT))
+            ingress_time.append(time.time())
+            time.sleep(sleep_time)
+
+
+def send():
+    global egress_time, ingress_time
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    print "Sending packet to %s at port %s" % (SERVER_IP, PORT)
+    print "chain id, exec id, data, function count, functions dependencies..."
+
+    # op = struct.unpack("B", packet[0])
+    packet, _ = genPacket()
+    if args.n is not None:
+        for i in range(args.req_count):
+            s.sendto(packet, (SERVER_IP, PORT))
+            ingress_time.append(time.time())
+
+    elif args.rps is not None:
+
+        runtime = 10
+        thread_count = args.c
+        start_time = time.time()
+        sleep_time = thread_count / float(args.rps)
+        print "calculated inter-arrival time, offload mode", sleep_time
+        for i in range(thread_count):
+            t = threading.Thread(target=sendThread, args=[
+                                 start_time, runtime, sleep_time])
+            t.daemon = True
+            t.start()
+        time.sleep(runtime)
+
+    stop_thread = True
+    # s.sendto(packet, (SERVER_IP, PORT))
+    # r.join()
+
+
+
+
+r = threading.Thread(name="receive", target=receive)
+r.daemon = True
+r.start()
+
+time.sleep(1)
+send()
+r.join()
--- a/dispatch_system/dispatch_daemon/package.json
+++ b/dispatch_system/dispatch_daemon/package.json
@@ -13,6 +13,7 @@
    "body-parser": "^1.19.0",
    "express": "^4.17.1",
    "express-fileupload": "^1.1.6",
+    "jspack": "^0.0.4",
    "kafka-node": "^5.0.0",
    "morgan": "^1.9.1",
    "mqtt": "^3.0.0",

--- a/dispatch_system/dispatch_manager/Makefile-nfp4build
+++ b/dispatch_system/dispatch_manager/Makefile-nfp4build
+#
+# Generated Makefile for test
+#
+
+ifndef SDKDIR
+export SDKDIR=/opt/netronome
+endif
+
+ifndef SDKTOOLSDIR
+SDKTOOLSDIR=$(SDKDIR)/bin
+endif
+
+ifndef SDKP4DIR
+SDKP4DIR=$(SDKDIR)/p4
+endif
+
+ifndef OUTDIR
+OUTDIR=p4src
+endif
+
+ifndef PIFOUTDIR
+PIFOUTDIR=p4src/out
+endif
+
+NFP4C_FOUND = $(shell if [ -x $(SDKP4DIR)/bin/nfp4c ]; then echo "found";  fi)
+ifneq ($(NFP4C_FOUND),found)
+  $(warning warning: nfp4c not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFIRC_FOUND = $(shell if [ -x $(SDKP4DIR)/bin/nfirc ]; then echo "found";  fi)
+ifneq ($(NFIRC_FOUND),found)
+  $(warning warning: nfirc not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFLD_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfld ]; then echo "found";  fi)
+ifneq ($(NFLD_FOUND),found)
+  $(warning warning: nfld not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFCC_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfcc ]; then echo "found";  fi)
+ifneq ($(NFCC_FOUND),found)
+  $(warning warning: nfcc not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFAS_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfas ]; then echo "found";  fi)
+ifneq ($(NFAS_FOUND),found)
+  $(warning warning: nfas not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFP4C_FOUND = $(shell if [ -x $(SDKP4DIR)/bin/nfp4c ]; then echo "found";  fi)
+ifneq ($(NFP4C_FOUND),found)
+  $(warning warning: nfp4c not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFIRC_FOUND = $(shell if [ -x $(SDKP4DIR)/bin/nfirc ]; then echo "found";  fi)
+ifneq ($(NFIRC_FOUND),found)
+  $(warning warning: nfirc not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFLD_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfld ]; then echo "found";  fi)
+ifneq ($(NFLD_FOUND),found)
+  $(warning warning: nfld not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFCC_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfcc ]; then echo "found";  fi)
+ifneq ($(NFCC_FOUND),found)
+  $(warning warning: nfcc not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFAS_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfas ]; then echo "found";  fi)
+ifneq ($(NFAS_FOUND),found)
+  $(warning warning: nfas not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFP4C_FOUND = $(shell if [ -x $(SDKP4DIR)/bin/nfp4c ]; then echo "found";  fi)
+ifneq ($(NFP4C_FOUND),found)
+  $(warning warning: nfp4c not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFIRC_FOUND = $(shell if [ -x $(SDKP4DIR)/bin/nfirc ]; then echo "found";  fi)
+ifneq ($(NFIRC_FOUND),found)
+  $(warning warning: nfirc not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFLD_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfld ]; then echo "found";  fi)
+ifneq ($(NFLD_FOUND),found)
+  $(warning warning: nfld not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFCC_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfcc ]; then echo "found";  fi)
+ifneq ($(NFCC_FOUND),found)
+  $(warning warning: nfcc not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFAS_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfas ]; then echo "found";  fi)
+ifneq ($(NFAS_FOUND),found)
+  $(warning warning: nfas not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFP4C_FOUND = $(shell if [ -x $(SDKP4DIR)/bin/nfp4c ]; then echo "found";  fi)
+ifneq ($(NFP4C_FOUND),found)
+  $(warning warning: nfp4c not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFIRC_FOUND = $(shell if [ -x $(SDKP4DIR)/bin/nfirc ]; then echo "found";  fi)
+ifneq ($(NFIRC_FOUND),found)
+  $(warning warning: nfirc not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFLD_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfld ]; then echo "found";  fi)
+ifneq ($(NFLD_FOUND),found)
+  $(warning warning: nfld not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFCC_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfcc ]; then echo "found";  fi)
+ifneq ($(NFCC_FOUND),found)
+  $(warning warning: nfcc not found or not executable, on windows please run nfp4term.bat)
+endif
+
+NFAS_FOUND = $(shell if [ -x $(SDKTOOLSDIR)/nfas ]; then echo "found";  fi)
+ifneq ($(NFAS_FOUND),found)
+  $(warning warning: nfas not found or not executable, on windows please run nfp4term.bat)
+endif
+
+$(OUTDIR)/test.nffw: $(OUTDIR)/nfd_pcie0_pd0.list/nfd_pcie0_pd0.list \
+		$(OUTDIR)/nfd_pcie0_pci_in_issue1.list/nfd_pcie0_pci_in_issue1.list \
+		$(OUTDIR)/nfd_pcie0_pci_out_me0.list/nfd_pcie0_pci_out_me0.list \
+		$(OUTDIR)/nbi_init_csr.list/nbi_init_csr.list \
+		$(OUTDIR)/nfd_pcie0_pd1.list/nfd_pcie0_pd1.list \
+		$(OUTDIR)/nfd_pcie0_pci_in_issue0.list/nfd_pcie0_pci_in_issue0.list \
+		$(OUTDIR)/gro1.list/gro1.list \
+		$(OUTDIR)/pif_app_nfd.list/pif_app_nfd.list \
+		$(OUTDIR)/app_master.list/app_master.list \
+		$(OUTDIR)/nfd_svc.list/nfd_svc.list \
+		$(OUTDIR)/blm0.list/blm0.list \
+		$(OUTDIR)/gro0.list/gro0.list \
+		$(OUTDIR)/nfd_pcie0_notify.list/nfd_pcie0_notify.list \
+		$(OUTDIR)/nfd_pcie0_pci_in_gather.list/nfd_pcie0_pci_in_gather.list \
+		$(OUTDIR)/flowcache_timeout_emu0.list/flowcache_timeout_emu0.list \
+		$(OUTDIR)/nfd_pcie0_sb.list/nfd_pcie0_sb.list \
+		$(PIFOUTDIR)/build_info.json \
+		$(MAKEFILE_LIST)
+	@echo ---------
+	@echo linking $@
+	@echo ---------
+	$(SDKTOOLSDIR)/nfld -rtsyms -mip -user_note_f pif_debug_json "$(PIFOUTDIR)/pif_debug.json" -user_note_f build_info_json "$(PIFOUTDIR)/build_info.json" -g -user_note_f pif_design_json "$(PIFOUTDIR)/pif_design.json" \
+		-i i8 -e $(SDKDIR)/components/standardlibrary/picocode/nfp6000/catamaran/catamaran.npfw \
+		-chip nfp-4xxxc-b0 -g -o $@ \
+		-u i36.me9 $(OUTDIR)/nfd_pcie0_pd0.list/nfd_pcie0_pd0.list \
+		-u pcie0.me3 $(OUTDIR)/nfd_pcie0_pci_in_issue1.list/nfd_pcie0_pci_in_issue1.list \
+		-u pcie0.me0 $(OUTDIR)/nfd_pcie0_pci_out_me0.list/nfd_pcie0_pci_out_me0.list \
+		-u i36.me7 $(OUTDIR)/nfd_pcie0_pd1.list/nfd_pcie0_pd1.list \
+		-u pcie0.me2 $(OUTDIR)/nfd_pcie0_pci_in_issue0.list/nfd_pcie0_pci_in_issue0.list \
+		-u i48.me1 $(OUTDIR)/gro1.list/gro1.list \
+		-u i32.me0 i33.me0 i34.me0 i35.me0 i36.me0 i32.me1 i33.me1 \
+			i34.me1 i35.me1 i36.me1 i32.me2 i33.me2 i34.me2 i35.me2 \
+			i36.me2 i32.me3 i33.me3 i34.me3 i35.me3 i36.me3 i32.me4 \
+			i33.me4 i34.me4 i35.me4 i36.me4 i32.me5 i33.me5 i34.me5 \
+			i35.me5 i36.me5 i32.me6 i33.me6 i34.me6 i35.me6 i32.me7 \
+			i33.me7 i34.me7 i35.me7 i32.me8 i33.me8 i34.me8 i35.me8 \
+			i32.me9 i33.me9 i34.me9 i35.me9 i32.me10 i33.me10 i34.me10 \
+			i35.me10 i32.me11 i33.me11 i34.me11 i35.me11 $(OUTDIR)/pif_app_nfd.list/pif_app_nfd.list \
+		-u i36.me11 $(OUTDIR)/app_master.list/app_master.list \
+		-u i48.me2 $(OUTDIR)/nfd_svc.list/nfd_svc.list \
+		-u i48.me3 $(OUTDIR)/blm0.list/blm0.list \
+		-u i48.me0 $(OUTDIR)/gro0.list/gro0.list \
+		-u i36.me6 $(OUTDIR)/nfd_pcie0_notify.list/nfd_pcie0_notify.list \
+		-u pcie0.me1 $(OUTDIR)/nfd_pcie0_pci_in_gather.list/nfd_pcie0_pci_in_gather.list \
+		-u i36.me10 $(OUTDIR)/flowcache_timeout_emu0.list/flowcache_timeout_emu0.list \
+		-u i36.me8 $(OUTDIR)/nfd_pcie0_sb.list/nfd_pcie0_sb.list \
+		-L $(OUTDIR)/nbi_init_csr.list/nbi_init_csr.list 
+
+#
+# Generate build info json
+#
+
+$(PIFOUTDIR)/build_info.json: $(MAKEFILE_LIST)
+	@echo ---------
+	@echo generating $@
+	@echo ---------
+	@mkdir -p $(PIFOUTDIR)
+	@echo -n {\"sku\": \"nfp-4xxxc-b0\", \"worker_mes\": [\"i32.me0\", \"i33.me0\", \"i34.me0\", \"i35.me0\", \"i36.me0\", \"i32.me1\", \"i33.me1\", \"i34.me1\", \"i35.me1\", \"i36.me1\", \"i32.me2\", \"i33.me2\", \"i34.me2\", \"i35.me2\", \"i36.me2\", \"i32.me3\", \"i33.me3\", \"i34.me3\", \"i35.me3\", \"i36.me3\", \"i32.me4\", \"i33.me4\", \"i34.me4\", \"i35.me4\", \"i36.me4\", \"i32.me5\", \"i33.me5\", \"i34.me5\", \"i35.me5\", \"i36.me5\", \"i32.me6\", \"i33.me6\", \"i34.me6\", \"i35.me6\", \"i32.me7\", \"i33.me7\", \"i34.me7\", \"i35.me7\", \"i32.me8\", \"i33.me8\", \"i34.me8\", \"i35.me8\", \"i32.me9\", \"i33.me9\", \"i34.me9\", \"i35.me9\", \"i32.me10\", \"i33.me10\", \"i34.me10\", \"i35.me10\", \"i32.me11\", \"i33.me11\", \"i34.me11\", \"i35.me11\"], \"reduced_thread_usage\": true, \"debug_info\": true, \"simulation\": false} >$@
+
+
+#
+# Generate IR from P4
+#
+
+$(OUTDIR)/test.yml: p4src/test.p4 \
+		$(MAKEFILE_LIST)
+	@echo ---------
+	@echo compiling p4 $@
+	@echo ---------
+	@mkdir -p $(PIFOUTDIR)
+	$(SDKP4DIR)/bin/nfp4c -o $(OUTDIR)/test.yml \
+		--p4-version 16 \
+		--p4-compiler p4c-nfp \
+		--source_info  \
+		p4src/test.p4
+
+
+#
+# Generate PIF from IR
+#
+
+$(PIFOUTDIR)/pif_design%c \
+$(PIFOUTDIR)/pif_design%h \
+$(PIFOUTDIR)/pif_parrep%c \
+$(PIFOUTDIR)/pif_parrep%h \
+$(PIFOUTDIR)/pif_actions%c \
+$(PIFOUTDIR)/pif_actions%h \
+$(PIFOUTDIR)/pif_global%c \
+$(PIFOUTDIR)/pif_global%h \
+$(PIFOUTDIR)/pif_ctlflow%c \
+$(PIFOUTDIR)/pif_ctlflow%h \
+$(PIFOUTDIR)/pif_tables%c \
+$(PIFOUTDIR)/pif_tables%h \
+$(PIFOUTDIR)/pif_deparse%c \
+$(PIFOUTDIR)/pif_digests%c \
+$(PIFOUTDIR)/pif_digests%h \
+$(PIFOUTDIR)/pif_meters%c \
+$(PIFOUTDIR)/pif_meters%h \
+$(PIFOUTDIR)/pif_registers%c \
+$(PIFOUTDIR)/pif_registers%h \
+$(PIFOUTDIR)/pif_pkt_recurse%c \
+$(PIFOUTDIR)/pif_pkt_recurse%h \
+$(PIFOUTDIR)/pif_pkt_clone%c \
+$(PIFOUTDIR)/pif_pkt_clone%h \
+$(PIFOUTDIR)/pif_flcalc%c \
+$(PIFOUTDIR)/pif_flcalc%h \
+$(PIFOUTDIR)/pif_field_lists%h \
+$(PIFOUTDIR)/pif_parrep_pvs_sync%c : $(OUTDIR)/test%yml $(MAKEFILE_LIST) 
+	@echo ---------
+	@echo generating pif $@
+	@echo ---------
+	@mkdir -p $(PIFOUTDIR)
+	$(SDKP4DIR)/bin/nfirc -o $(PIFOUTDIR)/ \
+		--p4info $(OUTDIR)/test.p4info.json \
+		--debugpoints  \
+		--mac_ingress_timestamp  \
+		$(OUTDIR)/test.yml
+
+
+#
+# NFD_PCIE0_PD0
+#
+
+$(OUTDIR)/nfd_pcie0_pd0.list/nfd_pcie0_pd0.list: $(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_out_pd.uc \
+	$(PIFOUTDIR)/pif_design.h \
+	$(MAKEFILE_LIST)
+	@echo ---------
+	@echo assembling $@
+	@echo ---------
+	@mkdir -p $(OUTDIR)/nfd_pcie0_pd0.list
+	$(SDKTOOLSDIR)/nfas -DPLATFORM_PORT_SELECTION=H_PLATFORM \
+	-DPIF_APP_CONFIG=PIF_APP_DEFAULT \
+	-DNUM_IMU=1 \
+	-DNUM_EMU=1 \
+	-DSIMULATION=0 \
+	-DPIF_APP_REDUCED_THREADS=1 \
+	-DPIF_DEBUG=1 \
+	-DSTAGE_BATCH_MANAGER_CTX=0 \
+	-DNFD_OUT_SB_WQ_NUM=15 \
+	-DNFD_OUT_STAGE_ME="__nfp_idstr2meid(\"i36.me8\")" \
+	-DPCIE_ISL=0 \
+	-chip nfp-4xxxc-b0 \
+	-go \
+	-t \
+	-W3 \
+	-R \
+	-lm 0 \
+	-C \
+	-Of \
+	-Os \
+	-O \
+	-I$(SDKDIR)/components/flowenv/me/blocks/blm/_h \
+	-I$(SDKDIR)/components/flowenv/me/blocks/blm/_uc \
+	-I$(SDKDIR)/components/flowenv/me/blocks/blm \
+	-I$(SDKDIR)/components/flowenv/me/blocks \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include \
+	-I$(SDKDIR)/components/ng_nfd/me/blocks/vnic \
+	-I$(SDKDIR)/components/ng_nfd/me/blocks/vnic/shared \
+	-I$(SDKDIR)/components/ng_nfd/me/blocks \
+	-I$(SDKDIR)/components/ng_nfd/shared \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/common/include \
+	-I$(SDKDIR)/components/flowenv/me/include \
+	-I$(PIFOUTDIR)/ \
+	-I$(SDKDIR)/components/standardlibrary/microcode/src \
+	-I$(SDKDIR)/components/standardlibrary/microcode/include \
+	-I$(SDKDIR)/components/standardlibrary/include \
+	-include $(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include/config.h \
+	-o $@ $(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_out_pd.uc
+
+
+#
+# NFD_PCIE0_PCI_IN_ISSUE1
+#
+
+$(OUTDIR)/nfd_pcie0_pci_in_issue1.list/nfd_pcie0_pci_in_issue1.list: $(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_in_me1.c \
+	$(SDKDIR)/components/flowenv/me/blocks/blm/libblm.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/utils/libutils.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/shared/nfd_cfg.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/intrinsic.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/rtl.c \
+	$(SDKDIR)/components/flowenv/me/lib/nfp/libnfp.c \
+	$(SDKDIR)/components/flowenv/me/lib/std/libstd.c \
+	$(SDKDIR)/components/flowenv/me/lib/pkt/libpkt.c \
+	$(SDKDIR)/components/flowenv/me/lib/pktdma/libpktdma.c \
+	$(SDKDIR)/components/flowenv/me/lib/net/libnet.c \
+	$(SDKDIR)/components/flowenv/me/lib/modscript/libmodscript.c \
+	$(PIFOUTDIR)/pif_design.h \
+	$(MAKEFILE_LIST)
+	@echo ---------
+	@echo compiling $@
+	@echo ---------
+	@mkdir -p $(OUTDIR)/nfd_pcie0_pci_in_issue1.list
+	$(SDKTOOLSDIR)/nfcc -DPLATFORM_PORT_SELECTION=H_PLATFORM \
+	-DPIF_APP_CONFIG=PIF_APP_DEFAULT \
+	-DNUM_IMU=1 \
+	-DNUM_EMU=1 \
+	-DSIMULATION=0 \
+	-DPIF_APP_REDUCED_THREADS=1 \
+	-DPIF_DEBUG=1 \
+	-DNFD_IN_NOTIFY_ME="__nfp_idstr2meid(\"i36.me6\")" \
+	-DPCI_IN_ISSUE_DMA_IDX=1 \
+	-DPCIE_ISL=0 \
+	-chip nfp-4xxxc-b0 \
+	-W3 \
+	-Qno_decl_volatile \
+	-O2 \
+	-mIPOPT_nfp=2 \
+	-Zi \
+	-ng \
+	-Qnctx=4 \
+	-Qnctx_mode=4 \
+	-Qlmpt_reserve_option=0xc \
+	-I$(SDKDIR)/components/flowenv/me/blocks/blm/_h \
+	-I$(SDKDIR)/components/flowenv/me/blocks \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include \
+	-I$(SDKDIR)/components/ng_nfd/me/blocks \
+	-I$(SDKDIR)/components/ng_nfd/shared \
+	-I$(SDKDIR)/components/ng_nfd \
+	-I$(SDKDIR)/components/standardlibrary/microc/include \
+	-I$(SDKDIR)/components/standardlibrary/microc/src \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/common/include \
+	-I$(SDKDIR)/components/flowenv/me/include \
+	-I$(SDKDIR)/components/flowenv/me/include/nfp6000 \
+	-I$(SDKDIR)/components/flowenv/me/lib \
+	-I$(SDKDIR)/components/flowenv/me/lib/nfp \
+	-I$(PIFOUTDIR)/ \
+	-I$(PIFOUTDIR)/callbackapi \
+	-I$(SDKDIR)/components/standardlibrary/include \
+	-FI$(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include/config.h \
+	-Fo$(OUTDIR)/nfd_pcie0_pci_in_issue1.list/ \
+	-Fe$(OUTDIR)/nfd_pcie0_pci_in_issue1.list/nfd_pcie0_pci_in_issue1 $(NFCC_FLAGS) \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_in_me1.c \
+	$(SDKDIR)/components/flowenv/me/blocks/blm/libblm.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/utils/libutils.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/shared/nfd_cfg.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/intrinsic.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/rtl.c \
+	$(SDKDIR)/components/flowenv/me/lib/nfp/libnfp.c \
+	$(SDKDIR)/components/flowenv/me/lib/std/libstd.c \
+	$(SDKDIR)/components/flowenv/me/lib/pkt/libpkt.c \
+	$(SDKDIR)/components/flowenv/me/lib/pktdma/libpktdma.c \
+	$(SDKDIR)/components/flowenv/me/lib/net/libnet.c \
+	$(SDKDIR)/components/flowenv/me/lib/modscript/libmodscript.c
+
+#
+# NFD_PCIE0_PCI_OUT_ME0
+#
+
+$(OUTDIR)/nfd_pcie0_pci_out_me0.list/nfd_pcie0_pci_out_me0.list: $(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_out_me0.c \
+	$(SDKDIR)/components/flowenv/me/blocks/blm/libblm.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/utils/libutils.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/shared/nfd_cfg.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/intrinsic.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/rtl.c \
+	$(SDKDIR)/components/flowenv/me/lib/nfp/libnfp.c \
+	$(SDKDIR)/components/flowenv/me/lib/std/libstd.c \
+	$(SDKDIR)/components/flowenv/me/lib/pkt/libpkt.c \
+	$(SDKDIR)/components/flowenv/me/lib/pktdma/libpktdma.c \
+	$(SDKDIR)/components/flowenv/me/lib/net/libnet.c \
+	$(SDKDIR)/components/flowenv/me/lib/modscript/libmodscript.c \
+	$(PIFOUTDIR)/pif_design.h \
+	$(MAKEFILE_LIST)
+	@echo ---------
+	@echo compiling $@
+	@echo ---------
+	@mkdir -p $(OUTDIR)/nfd_pcie0_pci_out_me0.list
+	$(SDKTOOLSDIR)/nfcc -DPLATFORM_PORT_SELECTION=H_PLATFORM \
+	-DPIF_APP_CONFIG=PIF_APP_DEFAULT \
+	-DNUM_IMU=1 \
+	-DNUM_EMU=1 \
+	-DSIMULATION=0 \
+	-DPIF_APP_REDUCED_THREADS=1 \
+	-DPIF_DEBUG=1 \
+	-DNFD_CFG_SIG_NEXT_ME="nfd_cfg_sig_sb" \
+	-DNFD_CFG_NEXT_ME="__nfp_idstr2meid(\"i36.me8\")" \
+	-DPCIE_ISL=0 \
+	-chip nfp-4xxxc-b0 \
+	-W3 \
+	-Qno_decl_volatile \
+	-O2 \
+	-mIPOPT_nfp=2 \
+	-Zi \
+	-ng \
+	-Qnctx=8 \
+	-Qnctx_mode=8 \
+	-Qlm_start=512 \
+	-I$(SDKDIR)/components/flowenv/me/blocks/blm/_h \
+	-I$(SDKDIR)/components/flowenv/me/blocks \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include \
+	-I$(SDKDIR)/components/ng_nfd/me/blocks \
+	-I$(SDKDIR)/components/ng_nfd/shared \
+	-I$(SDKDIR)/components/ng_nfd \
+	-I$(SDKDIR)/components/standardlibrary/microc/include \
+	-I$(SDKDIR)/components/standardlibrary/microc/src \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/common/include \
+	-I$(SDKDIR)/components/flowenv/me/include \
+	-I$(SDKDIR)/components/flowenv/me/include/nfp6000 \
+	-I$(SDKDIR)/components/flowenv/me/lib \
+	-I$(SDKDIR)/components/flowenv/me/lib/nfp \
+	-I$(PIFOUTDIR)/ \
+	-I$(PIFOUTDIR)/callbackapi \
+	-I$(SDKDIR)/components/standardlibrary/include \
+	-FI$(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include/config.h \
+	-Fo$(OUTDIR)/nfd_pcie0_pci_out_me0.list/ \
+	-Fe$(OUTDIR)/nfd_pcie0_pci_out_me0.list/nfd_pcie0_pci_out_me0 $(NFCC_FLAGS) \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_out_me0.c \
+	$(SDKDIR)/components/flowenv/me/blocks/blm/libblm.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/utils/libutils.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/shared/nfd_cfg.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/intrinsic.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/rtl.c \
+	$(SDKDIR)/components/flowenv/me/lib/nfp/libnfp.c \
+	$(SDKDIR)/components/flowenv/me/lib/std/libstd.c \
+	$(SDKDIR)/components/flowenv/me/lib/pkt/libpkt.c \
+	$(SDKDIR)/components/flowenv/me/lib/pktdma/libpktdma.c \
+	$(SDKDIR)/components/flowenv/me/lib/net/libnet.c \
+	$(SDKDIR)/components/flowenv/me/lib/modscript/libmodscript.c
+
+#
+# NBI_INIT_CSR
+#
+
+$(OUTDIR)/nbi_init_csr.list/nbi_init_csr.list: $(SDKP4DIR)/components/nfp_pif/me/apps/common/src/nbi_init.uc \
+	$(PIFOUTDIR)/pif_design.h \
+	$(MAKEFILE_LIST)
+	@echo ---------
+	@echo assembling $@
+	@echo ---------
+	@mkdir -p $(OUTDIR)/nbi_init_csr.list
+	$(SDKTOOLSDIR)/nfas -DPLATFORM_PORT_SELECTION=H_PLATFORM \
+	-DPIF_APP_CONFIG=PIF_APP_DEFAULT \
+	-DNUM_IMU=1 \
+	-DNUM_EMU=1 \
+	-DSIMULATION=0 \
+	-DPIF_APP_REDUCED_THREADS=1 \
+	-DPIF_DEBUG=1 \
+	-chip nfp-4xxxc-b0 \
+	-go \
+	-W3 \
+	-R \
+	-C \
+	-permit_dram_unaligned \
+	-preproc64 \
+	-keep_unreachable_code \
+	-lm 0 \
+	-codeless \
+	-I$(SDKDIR)/components/flowenv/me/blocks/init \
+	-I$(SDKDIR)/components/flowenv/me/blocks/init/_uc \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/common/include \
+	-I$(SDKDIR)/components/standardlibrary/microcode/src \
+	-I$(SDKDIR)/components/standardlibrary/microcode/include \
+	-I$(PIFOUTDIR)/ \
+	-I$(SDKDIR)/components/standardlibrary/include \
+	-include $(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include/config.h \
+	-o $@ $(SDKP4DIR)/components/nfp_pif/me/apps/common/src/nbi_init.uc
+
+
+#
+# NFD_PCIE0_PD1
+#
+
+$(OUTDIR)/nfd_pcie0_pd1.list/nfd_pcie0_pd1.list: $(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_out_pd.uc \
+	$(PIFOUTDIR)/pif_design.h \
+	$(MAKEFILE_LIST)
+	@echo ---------
+	@echo assembling $@
+	@echo ---------
+	@mkdir -p $(OUTDIR)/nfd_pcie0_pd1.list
+	$(SDKTOOLSDIR)/nfas -DPLATFORM_PORT_SELECTION=H_PLATFORM \
+	-DPIF_APP_CONFIG=PIF_APP_DEFAULT \
+	-DNUM_IMU=1 \
+	-DNUM_EMU=1 \
+	-DSIMULATION=0 \
+	-DPIF_APP_REDUCED_THREADS=1 \
+	-DPIF_DEBUG=1 \
+	-DSTAGE_BATCH_MANAGER_CTX=0 \
+	-DNFD_OUT_SB_WQ_NUM=15 \
+	-DNFD_OUT_STAGE_ME="__nfp_idstr2meid(\"i36.me8\")" \
+	-DPCIE_ISL=0 \
+	-chip nfp-4xxxc-b0 \
+	-go \
+	-t \
+	-W3 \
+	-R \
+	-lm 0 \
+	-C \
+	-Of \
+	-Os \
+	-O \
+	-I$(SDKDIR)/components/flowenv/me/blocks/blm/_h \
+	-I$(SDKDIR)/components/flowenv/me/blocks/blm/_uc \
+	-I$(SDKDIR)/components/flowenv/me/blocks/blm \
+	-I$(SDKDIR)/components/flowenv/me/blocks \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include \
+	-I$(SDKDIR)/components/ng_nfd/me/blocks/vnic \
+	-I$(SDKDIR)/components/ng_nfd/me/blocks/vnic/shared \
+	-I$(SDKDIR)/components/ng_nfd/me/blocks \
+	-I$(SDKDIR)/components/ng_nfd/shared \
+	-I$(SDKP4DIR)/components/nfp_pif/me/apps/common/include \
+	-I$(SDKDIR)/components/flowenv/me/include \
+	-I$(PIFOUTDIR)/ \
+	-I$(SDKDIR)/components/standardlibrary/microcode/src \
+	-I$(SDKDIR)/components/standardlibrary/microcode/include \
+	-I$(SDKDIR)/components/standardlibrary/include \
+	-include $(SDKP4DIR)/components/nfp_pif/me/apps/pif_app_nfd/include/config.h \
+	-o $@ $(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_out_pd.uc
+
+
+#
+# NFD_PCIE0_PCI_IN_ISSUE0
+#
+
+$(OUTDIR)/nfd_pcie0_pci_in_issue0.list/nfd_pcie0_pci_in_issue0.list: $(SDKDIR)/components/ng_nfd/me/blocks/vnic/pci_in_me1.c \
+	$(SDKDIR)/components/flowenv/me/blocks/blm/libblm.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/utils/libutils.c \
+	$(SDKDIR)/components/ng_nfd/me/blocks/vnic/shared/nfd_cfg.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/intrinsic.c \
+	$(SDKDIR)/components/standardlibrary/microc/src/rtl.c \
+	$(SDKDIR)/components/flowenv/me/lib/nfp/libnfp.c \
+	$(SDKDIR)/components/flowenv/me/lib/std/libstd.c \
+	$(SDKDIR)/components/flowenv/me/lib/pkt/libpkt.c \