Add inlining support to LLVM JIT provider.

This provides infrastructure to allow JITed code to inline code
implemented in C. This e.g. can be postgres internal functions or
extension code.

This already speeds up long running queries, by allowing the LLVM
optimizer to optimize across function boundaries. The optimization
potential currently doesn't reach its full potential because LLVM
cannot optimize the FunctionCallInfoData argument fully away, because
it's allocated on the heap rather than the stack. Fixing that is
beyond what's realistic for v11.

To be able to do that, use CLANG to convert C code to LLVM bitcode,
and have LLVM build a summary for it. That bitcode can then be used to
to inline functions at runtime. For that the bitcode needs to be
installed. Postgres bitcode goes into $pkglibdir/bitcode/postgres,
extensions go into equivalent directories.  PGXS has been modified so
that happens automatically if postgres has been compiled with LLVM
support.

Currently this isn't the fastest inline implementation, modules are
reloaded from disk during inlining. That's to work around an apparent
LLVM bug, triggering an apparently spurious error in LLVM assertion
enabled builds.  Once that is resolved we can remove the superfluous
read from disk.

Docs will follow in a later commit containing docs for the whole JIT
feature.

Author: Andres Freund
Discussion: https://postgr.es/m/20170901064131.tazjxwus3k2w3ybh@alap3.anarazel.de

src/Makefile.global.in

@@ -171,6 +171,7 @@ endif # PGXS
 includedir_server = $(pkgincludedir)/server
 includedir_internal = $(pkgincludedir)/internal
 pgxsdir = $(pkglibdir)/pgxs
+bitcodedir = $(pkglibdir)/bitcode
 
 
 ##########################################################################
@@ -972,3 +973,36 @@ endif
 
 %.bc : %.cpp
 	$(COMPILE.cxx.bc) -o $@ $<
+
+# Install LLVM bitcode module (for JITing).
+#
+# The arguments are:
+# $(1) name of the module (e.g. an extension's name or postgres for core code)
+# $(2) source objects, with .o suffix
+#
+define install_llvm_module
+# Create target directory
+$(MKDIR_P) "$(DESTDIR)${bitcodedir}/$(1)"
+# Create sub-directories, if files are in subdirectories
+$(MKDIR_P) $(sort $(dir $(addprefix $(DESTDIR)${bitcodedir}/$(1)/, $(2))))
+# Then install files
+#
+# The many INSTALL_DATA invocations aren't particularly fast, it'd be
+# good if we could coalesce them, but I didn't find a good way.
+$(foreach obj, ${2}, $(INSTALL_DATA) $(patsubst %.o,%.bc, $(obj)) $(DESTDIR)/${bitcodedir}/$(1)/$(dir $(obj));
+)
+# and generate index
+(cd "$(DESTDIR)${bitcodedir}" && $(LLVM_BINPATH)/llvm-lto -thinlto -thinlto-action=thinlink -o $(1).index.bc $(addprefix $(1)/,$(patsubst %.o,%.bc, $(2))))
+endef
+
+# Uninstall LLVM bitcode module.
+#
+# The arguments are:
+# $(1) name of the module (e.g. an extension's name or postgres for core code)
+#
+# This intentionally doesn't use the explicit installed file list,
+# seems too likely to change regularly.
+define uninstall_llvm_module
+rm -rf "$(DESTDIR)${bitcodedir}/$(1)/"
+rm -f "$(DESTDIR)${bitcodedir}/$(1).index.bc"
+endef

src/backend/Makefile

@@ -252,6 +252,13 @@ endif
 	$(INSTALL_DATA) $(srcdir)/utils/misc/postgresql.conf.sample '$(DESTDIR)$(datadir)/postgresql.conf.sample'
 	$(INSTALL_DATA) $(srcdir)/access/transam/recovery.conf.sample '$(DESTDIR)$(datadir)/recovery.conf.sample'
 
+ifeq ($(with_llvm), yes)
+install-bin: install-postgres-bitcode
+
+install-postgres-bitcode: $(OBJS) all
+	$(call install_llvm_module,postgres,$(call expand_subsys, $(filter-out $(top_builddir)/src/timezone/objfiles.txt, $(SUBDIROBJS))))
+endif
+
 install-bin: postgres $(POSTGRES_IMP) installdirs
 	$(INSTALL_PROGRAM) postgres$(X) '$(DESTDIR)$(bindir)/postgres$(X)'
 ifneq ($(PORTNAME), win32)
@@ -309,6 +316,9 @@ endif
 	      '$(DESTDIR)$(datadir)/pg_ident.conf.sample' \
 	      '$(DESTDIR)$(datadir)/postgresql.conf.sample' \
 	      '$(DESTDIR)$(datadir)/recovery.conf.sample'
+ifeq ($(with_llvm), yes)
+	$(call uninstall_llvm_module,postgres)
+endif
 
 
 ##########################################################################

src/backend/common.mk

@@ -30,6 +30,10 @@ objfiles.txt: Makefile $(SUBDIROBJS) $(OBJS)
 # Don't rebuild the list if only the OBJS have changed.
 	$(if $(filter-out $(OBJS),$?),( $(if $(SUBDIROBJS),cat $(SUBDIROBJS); )echo $(addprefix $(subdir)/,$(OBJS)) ) >$@,touch $@)
 
+ifeq ($(with_llvm), yes)
+objfiles.txt: $(patsubst %.o,%.bc, $(OBJS))
+endif
+
 # make function to expand objfiles.txt contents
 expand_subsys = $(foreach file,$(1),$(if $(filter %/objfiles.txt,$(file)),$(patsubst ../../src/backend/%,%,$(addprefix $(top_builddir)/,$(shell cat $(file)))),$(file)))
 
@@ -43,7 +47,7 @@ $(SUBDIRS:%=%-recursive):
 $(call recurse,clean)
 clean: clean-local
 clean-local:
-	rm -f $(subsysfilename) $(OBJS)
+	rm -f $(subsysfilename) $(OBJS) $(patsubst %.o,%.bc, $(OBJS))
 
 $(call recurse,coverage)
 $(call recurse,install)

src/backend/jit/jit.c

@@ -40,6 +40,7 @@ bool		jit_expressions = true;
 bool		jit_profiling_support = false;
 bool		jit_tuple_deforming = true;
 double		jit_above_cost = 100000;
+double		jit_inline_above_cost = 500000;
 double		jit_optimize_above_cost = 500000;
 
 static JitProviderCallbacks provider;

src/backend/jit/llvm/Makefile

@@ -37,7 +37,7 @@ override COMPILER = $(CXX) $(CFLAGS)
 OBJS=$(WIN32RES)
 
 # Infrastructure
-OBJS += llvmjit.o llvmjit_error.o llvmjit_wrap.o
+OBJS += llvmjit.o llvmjit_error.o llvmjit_inline.o llvmjit_wrap.o
 # Code generation
 OBJS += llvmjit_expr.o llvmjit_deform.o
 

src/backend/jit/llvm/llvmjit.c

@@ -468,6 +468,10 @@ llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
 	/* always use always-inliner pass */
 	if (!(context->base.flags & PGJIT_OPT3))
 		LLVMAddAlwaysInlinerPass(llvm_mpm);
+	/* if doing inlining, but no expensive optimization, add inlining pass */
+	if (context->base.flags & PGJIT_INLINE
+		&& !(context->base.flags & PGJIT_OPT3))
+		LLVMAddFunctionInliningPass(llvm_mpm);
 	LLVMRunPassManager(llvm_mpm, context->module);
 	LLVMDisposePassManager(llvm_mpm);
 
@@ -491,6 +495,16 @@ llvm_compile_module(LLVMJitContext *context)
 	else
 		compile_orc = llvm_opt0_orc;
 
+	/* perform inlining */
+	if (context->base.flags & PGJIT_INLINE)
+	{
+		INSTR_TIME_SET_CURRENT(starttime);
+		llvm_inline(context->module);
+		INSTR_TIME_SET_CURRENT(endtime);
+		INSTR_TIME_ACCUM_DIFF(context->base.inlining_counter,
+							  endtime, starttime);
+	}
+
 	if (jit_dump_bitcode)
 	{
 		char	   *filename;
@@ -578,7 +592,8 @@ llvm_compile_module(LLVMJitContext *context)
 	MemoryContextSwitchTo(oldcontext);
 
 	ereport(DEBUG1,
-			(errmsg("time to opt: %.3fs, emit: %.3fs",
+			(errmsg("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
+					INSTR_TIME_GET_DOUBLE(context->base.inlining_counter),
 					INSTR_TIME_GET_DOUBLE(context->base.optimization_counter),
 					INSTR_TIME_GET_DOUBLE(context->base.emission_counter)),
 			 errhidestmt(true),

src/backend/optimizer/plan/planner.c

@@ -544,6 +544,9 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams)
 		if (jit_optimize_above_cost >= 0 &&
 			top_plan->total_cost > jit_optimize_above_cost)
 			result->jitFlags |= PGJIT_OPT3;
+		if (jit_inline_above_cost >= 0 &&
+			top_plan->total_cost > jit_inline_above_cost)
+			result->jitFlags |= PGJIT_INLINE;
 
 		/*
 		 * Decide which operations should be JITed.

src/backend/utils/misc/guc.c

@@ -3117,6 +3117,16 @@ static struct config_real ConfigureNamesReal[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"jit_inline_above_cost", PGC_USERSET, QUERY_TUNING_COST,
+			gettext_noop("Perform JIT inlining if query is more expensive."),
+			gettext_noop("-1 disables inlining.")
+		},
+		&jit_inline_above_cost,
+		500000, -1, DBL_MAX,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"cursor_tuple_fraction", PGC_USERSET, QUERY_TUNING_OTHER,
 			gettext_noop("Sets the planner's estimate of the fraction of "

src/backend/utils/misc/postgresql.conf.sample

@@ -323,6 +323,9 @@
 					# and query more expensive, -1 disables
 #jit_optimize_above_cost = 500000	# optimize JITed functions if query is
 					# more expensive, -1 disables
+#jit_inline_above_cost = 500000		# attempt to inline operators and
+					# functions if query is more expensive,
+					# -1 disables
 
 #min_parallel_table_scan_size = 8MB
 #min_parallel_index_scan_size = 512kB

src/include/jit/jit.h

@@ -19,7 +19,7 @@
 #define PGJIT_NONE     0
 #define PGJIT_PERFORM  1 << 0
 #define PGJIT_OPT3     1 << 1
-/* reserved for PGJIT_INLINE */
+#define PGJIT_INLINE   1 << 2
 #define PGJIT_EXPR	   1 << 3
 #define PGJIT_DEFORM   1 << 4
 
@@ -37,6 +37,9 @@ typedef struct JitContext
 	/* accumulated time to generate code */
 	instr_time	generation_counter;
 
+	/* accumulated time for inlining */
+	instr_time	inlining_counter;
+
 	/* accumulated time for optimization */
 	instr_time	optimization_counter;
 
@@ -70,6 +73,7 @@ extern bool jit_expressions;
 extern bool jit_profiling_support;
 extern bool jit_tuple_deforming;
 extern double jit_above_cost;
+extern double jit_inline_above_cost;
 extern double jit_optimize_above_cost;
 
 

src/include/jit/llvmjit.h

@@ -103,6 +103,7 @@ extern LLVMValueRef llvm_function_reference(LLVMJitContext *context,
 						LLVMModuleRef mod,
 						FunctionCallInfo fcinfo);
 
+extern void llvm_inline(LLVMModuleRef mod);
 
 /*
  ****************************************************************************

src/makefiles/pgxs.mk

@@ -101,6 +101,10 @@ endif
 
 all: $(PROGRAM) $(DATA_built) $(SCRIPTS_built) $(addsuffix $(DLSUFFIX), $(MODULES)) $(addsuffix .control, $(EXTENSION))
 
+ifeq ($(with_llvm), yes)
+all: $(addsuffix .bc, $(MODULES)) $(patsubst %.o,%.bc, $(OBJS))
+endif
+
 ifdef MODULE_big
 # shared library parameters
 NAME = $(MODULE_big)
@@ -123,6 +127,9 @@ ifneq (,$(DATA_TSEARCH))
 endif # DATA_TSEARCH
 ifdef MODULES
 	$(INSTALL_SHLIB) $(addsuffix $(DLSUFFIX), $(MODULES)) '$(DESTDIR)$(pkglibdir)/'
+ifeq ($(with_llvm), yes)
+	$(foreach mod, $(MODULES), $(call install_llvm_module,$(mod),$(mod).bc))
+endif # with_llvm
 endif # MODULES
 ifdef DOCS
 ifdef docdir
@@ -138,8 +145,11 @@ endif # SCRIPTS
 ifdef SCRIPTS_built
 	$(INSTALL_SCRIPT) $(SCRIPTS_built) '$(DESTDIR)$(bindir)/'
 endif # SCRIPTS_built
-
 ifdef MODULE_big
+ifeq ($(with_llvm), yes)
+	$(call install_llvm_module,$(MODULE_big),$(OBJS))
+endif # with_llvm
+
 install: install-lib
 endif # MODULE_big
 
@@ -183,7 +193,10 @@ ifneq (,$(DATA_TSEARCH))
 endif
 ifdef MODULES
 	rm -f $(addprefix '$(DESTDIR)$(pkglibdir)'/, $(addsuffix $(DLSUFFIX), $(MODULES)))
-endif
+ifeq ($(with_llvm), yes)
+	$(foreach mod, $(MODULES), $(call uninstall_llvm_module,$(mod)))
+endif # with_llvm
+endif # MODULES
 ifdef DOCS
 	rm -f $(addprefix '$(DESTDIR)$(docdir)/$(docmoduledir)'/, $(DOCS))
 endif
@@ -198,13 +211,18 @@ ifdef SCRIPTS_built
 endif
 
 ifdef MODULE_big
+ifeq ($(with_llvm), yes)
+	$(call uninstall_llvm_module,$(MODULE_big))
+endif # with_llvm
+
 uninstall: uninstall-lib
 endif # MODULE_big
 
 
 clean:
 ifdef MODULES
-	rm -f $(addsuffix $(DLSUFFIX), $(MODULES)) $(addsuffix .o, $(MODULES)) $(if $(PGFILEDESC),$(WIN32RES))
+	rm -f $(addsuffix $(DLSUFFIX), $(MODULES)) $(addsuffix .o, $(MODULES)) $(if $(PGFILEDESC),$(WIN32RES)) \
+	    $(addsuffix .bc, $(MODULES))
 endif
 ifdef DATA_built
 	rm -f $(DATA_built)
@@ -216,7 +234,7 @@ ifdef PROGRAM
 	rm -f $(PROGRAM)$(X)
 endif
 ifdef OBJS
-	rm -f $(OBJS)
+	rm -f $(OBJS) $(patsubst %.o,%.bc, $(OBJS))
 endif
 ifdef EXTRA_CLEAN
 	rm -rf $(EXTRA_CLEAN)