SQL commands in pgbench scripts are now ended by semicolons, not newlines.

To allow multiline SQL commands in scripts, adopt the same rules psql uses
to decide what is the end of a SQL command, to wit, an unquoted semicolon
not encased in parentheses.  Do this by importing the same flex lexer that
psql uses, since coping with stuff like dollar-quoted literals is hard to
get right without going the full nine yards.

This makes use of the infrastructure added in commit 0ea9efbe to
support independently-written flex lexers scanning the same PsqlScanState
input-buffer data structure.  Since that infrastructure isn't very
friendly to ad-hoc parsing code such as strtok(), improve exprscan.l
so that it can parse either whitespace-separated words or expression
tokens, on demand, and rewrite pgbench.c's backslash-command parsing
code to always use the lexer to fetch tokens.

It's still the case that pgbench backslash commands extend to the end
of the line, no more and no less.  That could be changed in a fairly
localized way now, and there was some interest in doing so, but it
seems like material for a separate patch.

In passing, make some marginal cleanups in syntax error reporting,
const-ify a few data structures that could use it, and run some of
this code through pgindent.

I can't tell whether the MSVC build scripts need to be taught explicitly
about the changes here or not, but the buildfarm will soon tell us.

Kyotaro Horiguchi and Tom Lane

doc/src/sgml/ref/pgbench.sgml

@@ -743,13 +743,25 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</>
   </para>
 
   <para>
-   The format of a script file is one SQL command per line; multiline
-   SQL commands are not supported.  Empty lines and lines beginning with
-   <literal>--</> are ignored.  Script file lines can also be
+   A script file contains one or more SQL commands terminated by
+   semicolons.  Empty lines and lines beginning with
+   <literal>--</> are ignored.  Script files can also contain
    <quote>meta commands</>, which are interpreted by <application>pgbench</>
    itself, as described below.
   </para>
 
+  <note>
+   <para>
+    Before <productname>PostgreSQL</> 9.6, SQL commands in script files
+    were terminated by newlines, and so they could not be continued across
+    lines.  Now a semicolon is <emphasis>required</> to separate consecutive
+    SQL commands (though a SQL command does not need one if it is followed
+    by a meta command).  If you need to create a script file that works with
+    both old and new versions of <application>pgbench</>, be sure to write
+    each SQL command on a single line ending with a semicolon.
+   </para>
+  </note>
+
   <para>
    There is a simple variable-substitution facility for script files.
    Variables can be set by the command-line <option>-D</> option,
@@ -789,7 +801,8 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</>
    </table>
 
   <para>
-   Script file meta commands begin with a backslash (<literal>\</>).
+   Script file meta commands begin with a backslash (<literal>\</>) and
+   extend to the end of the line.
    Arguments to a meta command are separated by white space.
    These meta commands are supported:
   </para>
@@ -806,9 +819,9 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</>
       from <replaceable>expression</>.
       The expression may contain integer constants such as <literal>5432</>,
       references to variables <literal>:</><replaceable>variablename</>,
-      and expressions composed of unary (<literal>-</>) or binary operators
+      unary operators (<literal>+</>, <literal>-</>) and binary operators
       (<literal>+</>, <literal>-</>, <literal>*</>, <literal>/</>,
-      <literal>%</>) with their usual associativity,
+      <literal>%</>) with their usual precedence and associativity,
       <link linkend="pgbench-builtin-functions">function calls</>, and
       parentheses.
      </para>
@@ -938,14 +951,15 @@ f(x) = exp(-parameter * (x - min) / (max - min + 1)) / (1.0 - exp(-parameter))
     <listitem>
      <para>
       Sets variable <replaceable>varname</> to the result of the shell command
-      <replaceable>command</>. The command must return an integer value
-      through its standard output.
+      <replaceable>command</> with the given <replaceable>argument</>(s).
+      The command must return an integer value through its standard output.
      </para>
 
-     <para><replaceable>argument</> can be either a text constant or a
-      <literal>:</><replaceable>variablename</> reference to a variable of
-      any types. If you want to use <replaceable>argument</> starting with
-      colons, you need to add an additional colon at the beginning of
+     <para>
+      <replaceable>command</> and each <replaceable>argument</> can be either
+      a text constant or a <literal>:</><replaceable>variablename</> reference
+      to a variable. If you want to use an <replaceable>argument</> starting
+      with a colon, write an additional colon at the beginning of
       <replaceable>argument</>.
      </para>
 
@@ -964,7 +978,8 @@ f(x) = exp(-parameter * (x - min) / (max - min + 1)) / (1.0 - exp(-parameter))
 
     <listitem>
      <para>
-      Same as <literal>\setshell</literal>, but the result is ignored.
+      Same as <literal>\setshell</literal>, but the result of the command
+      is discarded.
      </para>
 
      <para>
@@ -1010,7 +1025,7 @@ END;
 
    <para>
      The following functions are built into <application>pgbench</> and
-     may be used in conjunction with
+     may be used in expressions appearing in
      <link linkend="pgbench-metacommand-set"><literal>\set</literal></link>.
    </para>
 

src/bin/pgbench/Makefile

@@ -7,9 +7,10 @@ subdir = src/bin/pgbench
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = pgbench.o exprparse.o $(WIN32RES)
+OBJS = pgbench.o exprparse.o psqlscan.o $(WIN32RES)
 
-override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS)
+override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) \
+	-I$(top_srcdir)/src/bin/psql $(CPPFLAGS)
 
 ifneq ($(PORTNAME), win32)
 override CFLAGS += $(PTHREAD_CFLAGS)
@@ -24,6 +25,15 @@ pgbench: $(OBJS) | submake-libpq submake-libpgport
 # exprscan is compiled as part of exprparse
 exprparse.o: exprscan.c
 
+# we import psqlscan.o as-is from psql
+submake-psqlscan:
+	$(MAKE) -C $(top_builddir)/src/bin/psql psqlscan.o
+
+psqlscan.o: | submake-psqlscan
+	rm -f $@ && $(LN_S) $(top_builddir)/src/bin/psql/psqlscan.o .
+
+.PHONY: submake-psqlscan
+
 distprep: exprparse.c exprscan.c
 
 install: all installdirs

src/bin/pgbench/exprparse.y

@@ -22,8 +22,8 @@ static PgBenchExprList *make_elist(PgBenchExpr *exp, PgBenchExprList *list);
 static PgBenchExpr *make_integer_constant(int64 ival);
 static PgBenchExpr *make_variable(char *varname);
 static PgBenchExpr *make_op(yyscan_t yyscanner, const char *operator,
-							PgBenchExpr *lexpr, PgBenchExpr *rexpr);
-static int find_func(yyscan_t yyscanner, const char *fname);
+		PgBenchExpr *lexpr, PgBenchExpr *rexpr);
+static int	find_func(yyscan_t yyscanner, const char *fname);
 static PgBenchExpr *make_func(yyscan_t yyscanner, int fnumber, PgBenchExprList *args);
 
 %}
@@ -114,28 +114,49 @@ make_op(yyscan_t yyscanner, const char *operator,
  * List of available functions:
  * - fname: function name
  * - nargs: number of arguments
- *          -1 is a special value for min & max meaning #args >= 1
+ *			-1 is a special value for min & max meaning #args >= 1
  * - tag: function identifier from PgBenchFunction enum
  */
-static struct
+static const struct
 {
-	char * fname;
-	int nargs;
+	const char *fname;
+	int			nargs;
 	PgBenchFunction tag;
-} PGBENCH_FUNCTIONS[] = {
+}	PGBENCH_FUNCTIONS[] =
+{
 	/* parsed as operators, executed as functions */
-	{ "+", 2, PGBENCH_ADD },
-	{ "-", 2, PGBENCH_SUB },
-	{ "*", 2, PGBENCH_MUL },
-	{ "/", 2, PGBENCH_DIV },
-	{ "%", 2, PGBENCH_MOD },
+	{
+		"+", 2, PGBENCH_ADD
+	},
+	{
+		"-", 2, PGBENCH_SUB
+	},
+	{
+		"*", 2, PGBENCH_MUL
+	},
+	{
+		"/", 2, PGBENCH_DIV
+	},
+	{
+		"%", 2, PGBENCH_MOD
+	},
 	/* actual functions */
-	{ "abs", 1, PGBENCH_ABS },
-	{ "min", -1, PGBENCH_MIN },
-	{ "max", -1, PGBENCH_MAX },
-	{ "debug", 1, PGBENCH_DEBUG },
+	{
+		"abs", 1, PGBENCH_ABS
+	},
+	{
+		"min", -1, PGBENCH_MIN
+	},
+	{
+		"max", -1, PGBENCH_MAX
+	},
+	{
+		"debug", 1, PGBENCH_DEBUG
+	},
 	/* keep as last array element */
-	{ NULL, 0, 0 }
+	{
+		NULL, 0, 0
+	}
 };
 
 /*
@@ -147,7 +168,7 @@ static struct
 static int
 find_func(yyscan_t yyscanner, const char *fname)
 {
-	int i = 0;
+	int			i = 0;
 
 	while (PGBENCH_FUNCTIONS[i].fname)
 	{
@@ -166,7 +187,7 @@ find_func(yyscan_t yyscanner, const char *fname)
 static PgBenchExprList *
 make_elist(PgBenchExpr *expr, PgBenchExprList *list)
 {
-	PgBenchExprLink * cons;
+	PgBenchExprLink *cons;
 
 	if (list == NULL)
 	{
@@ -193,8 +214,8 @@ make_elist(PgBenchExpr *expr, PgBenchExprList *list)
 static int
 elist_length(PgBenchExprList *list)
 {
-	PgBenchExprLink *link = list != NULL? list->head: NULL;
-	int len = 0;
+	PgBenchExprLink *link = list != NULL ? list->head : NULL;
+	int			len = 0;
 
 	for (; link != NULL; link = link->next)
 		len++;
@@ -225,7 +246,7 @@ make_func(yyscan_t yyscanner, int fnumber, PgBenchExprList *args)
 	expr->u.function.function = PGBENCH_FUNCTIONS[fnumber].tag;
 
 	/* only the link is used, the head/tail is not useful anymore */
-	expr->u.function.args = args != NULL? args->head: NULL;
+	expr->u.function.args = args != NULL ? args->head : NULL;
 	if (args)
 		pg_free(args);
 

src/bin/pgbench/pgbench.h

@@ -11,6 +11,8 @@
 #ifndef PGBENCH_H
 #define PGBENCH_H
 
+#include "psqlscan.h"
+
 /*
  * This file is included outside exprscan.l, in places where we can't see
  * flex's definition of typedef yyscan_t.  Fortunately, it's documented as
@@ -84,16 +86,23 @@ extern PgBenchExpr *expr_parse_result;
 
 extern int	expr_yyparse(yyscan_t yyscanner);
 extern int	expr_yylex(yyscan_t yyscanner);
-extern void expr_yyerror(yyscan_t yyscanner, const char *str);
+extern void expr_yyerror(yyscan_t yyscanner, const char *str) pg_attribute_noreturn();
 extern void expr_yyerror_more(yyscan_t yyscanner, const char *str,
-				  const char *more);
-extern yyscan_t expr_scanner_init(const char *str, const char *source,
-				  int lineno, const char *line,
-				  const char *cmd, int ecol);
-extern void syntax_error(const char *source, const int lineno, const char *line,
-			 const char *cmd, const char *msg, const char *more,
-			 const int col);
+				  const char *more) pg_attribute_noreturn();
+extern bool expr_lex_one_word(PsqlScanState state, PQExpBuffer word_buf,
+				  int *offset);
+extern yyscan_t expr_scanner_init(PsqlScanState state,
+				  const char *source, int lineno, int start_offset,
+				  const char *command);
 extern void expr_scanner_finish(yyscan_t yyscanner);
+extern int	expr_scanner_offset(PsqlScanState state);
+extern char *expr_scanner_get_substring(PsqlScanState state,
+						   int start_offset, int end_offset);
+extern int	expr_scanner_get_lineno(PsqlScanState state, int offset);
+
+extern void syntax_error(const char *source, int lineno, const char *line,
+			 const char *cmd, const char *msg,
+			 const char *more, int col) pg_attribute_noreturn();
 
 extern int64 strtoint64(const char *str);