Remove plpgsql's separate lexer (finally!), in favor of using the core lexer

directly.  This was a lot of trouble, but should be worth it in terms of
not having to keep the plpgsql lexer in step with core anymore.  In addition
the handling of keywords is significantly better-structured, allowing us to
de-reserve a number of words that plpgsql formerly treated as reserved.

src/backend/parser/scan.l

@@ -24,7 +24,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/parser/scan.l,v 1.163 2009/11/09 18:38:48 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/parser/scan.l,v 1.164 2009/11/12 00:13:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -85,6 +85,7 @@ static void addlitchar(unsigned char ychar, core_yyscan_t yyscanner);
 static char *litbufdup(core_yyscan_t yyscanner);
 static char *litbuf_udeescape(unsigned char escape, core_yyscan_t yyscanner);
 static unsigned char unescape_single_char(unsigned char c, core_yyscan_t yyscanner);
+static int	process_integer_literal(const char *token, YYSTYPE *lval);
 static bool is_utf16_surrogate_first(pg_wchar c);
 static bool is_utf16_surrogate_second(pg_wchar c);
 static pg_wchar surrogate_pair_to_codepoint(pg_wchar first, pg_wchar second);
@@ -339,12 +340,15 @@ operator		{op_chars}+
  * instead we pass it separately to parser. there it gets
  * coerced via doNegate() -- Leon aug 20 1999
  *
+* {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot, 10.
+*
  * {realfail1} and {realfail2} are added to prevent the need for scanner
  * backup when the {real} rule fails to match completely.
  */
 
 integer			{digit}+
 decimal			(({digit}*\.{digit}+)|({digit}+\.{digit}*))
+decimalfail		{digit}+\.\.
 real			({integer}|{decimal})[Ee][-+]?{digit}+
 realfail1		({integer}|{decimal})[Ee]
 realfail2		({integer}|{decimal})[Ee][-+]
@@ -846,31 +850,20 @@ other			.
 				}
 
 {integer}		{
-					long val;
-					char* endptr;
-
 					SET_YYLLOC();
-					errno = 0;
-					val = strtol(yytext, &endptr, 10);
-					if (*endptr != '\0' || errno == ERANGE
-#ifdef HAVE_LONG_INT_64
-						/* if long > 32 bits, check for overflow of int4 */
-						|| val != (long) ((int32) val)
-#endif
-						)
-					{
-						/* integer too large, treat it as a float */
-						yylval->str = pstrdup(yytext);
-						return FCONST;
-					}
-					yylval->ival = val;
-					return ICONST;
+					return process_integer_literal(yytext, yylval);
 				}
 {decimal}		{
 					SET_YYLLOC();
 					yylval->str = pstrdup(yytext);
 					return FCONST;
 				}
+{decimalfail}	{
+					/* throw back the .., and treat as integer */
+					yyless(yyleng-2);
+					SET_YYLLOC();
+					return process_integer_literal(yytext, yylval);
+				}
 {real}			{
 					SET_YYLLOC();
 					yylval->str = pstrdup(yytext);
@@ -1121,6 +1114,29 @@ litbufdup(core_yyscan_t yyscanner)
 	return new;
 }
 
+static int
+process_integer_literal(const char *token, YYSTYPE *lval)
+{
+	long		val;
+	char	   *endptr;
+
+	errno = 0;
+	val = strtol(token, &endptr, 10);
+	if (*endptr != '\0' || errno == ERANGE
+#ifdef HAVE_LONG_INT_64
+		/* if long > 32 bits, check for overflow of int4 */
+		|| val != (long) ((int32) val)
+#endif
+		)
+	{
+		/* integer too large, treat it as a float */
+		lval->str = pstrdup(token);
+		return FCONST;
+	}
+	lval->ival = val;
+	return ICONST;
+}
+
 static int
 hexval(unsigned char c)
 {

src/backend/parser/scansup.c

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/parser/scansup.c,v 1.37 2009/01/01 17:23:46 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/parser/scansup.c,v 1.38 2009/11/12 00:13:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -197,7 +197,6 @@ bool
 scanner_isspace(char ch)
 {
 	/* This must match scan.l's list of {space} characters */
-	/* and plpgsql's scan.l as well */
 	if (ch == ' ' ||
 		ch == '\t' ||
 		ch == '\n' ||

src/bin/psql/psqlscan.l

@@ -33,7 +33,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/bin/psql/psqlscan.l,v 1.29 2009/09/27 03:27:24 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/bin/psql/psqlscan.l,v 1.30 2009/11/12 00:13:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -364,12 +364,15 @@ operator		{op_chars}+
  * instead we pass it separately to parser. there it gets
  * coerced via doNegate() -- Leon aug 20 1999
  *
+ * {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot, 10.
+ *
  * {realfail1} and {realfail2} are added to prevent the need for scanner
  * backup when the {real} rule fails to match completely.
  */
 
 integer			{digit}+
 decimal			(({digit}*\.{digit}+)|({digit}+\.{digit}*))
+decimalfail		{digit}+\.\.
 real			({integer}|{decimal})[Ee][-+]?{digit}+
 realfail1		({integer}|{decimal})[Ee]
 realfail2		({integer}|{decimal})[Ee][-+]
@@ -776,6 +779,11 @@ other			.
 {decimal}		{
 					ECHO;
 				}
+{decimalfail}	{
+					/* throw back the .., and treat as integer */
+					yyless(yyleng-2);
+					ECHO;
+				}
 {real}			{
 					ECHO;
 				}

src/pl/plpgsql/src/.cvsignore

 pl_gram.c
 pl_gram.h
-pl_scan.c

src/pl/plpgsql/src/Makefile

@@ -2,7 +2,7 @@
 #
 # Makefile for the plpgsql shared object
 #
-# $PostgreSQL: pgsql/src/pl/plpgsql/src/Makefile,v 1.34 2009/08/28 20:26:19 petere Exp $
+# $PostgreSQL: pgsql/src/pl/plpgsql/src/Makefile,v 1.35 2009/11/12 00:13:00 tgl Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -17,7 +17,7 @@ override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
 SHLIB_LINK = $(filter -lintl, $(LIBS))
 rpath =
 
-OBJS = pl_gram.o pl_handler.o pl_comp.o pl_exec.o pl_funcs.o
+OBJS = pl_gram.o pl_handler.o pl_comp.o pl_exec.o pl_funcs.o pl_scanner.o
 
 all: all-lib
 
@@ -33,10 +33,7 @@ uninstall: uninstall-lib
 
 
 # Force these dependencies to be known even without dependency info built:
-pl_gram.o pl_handler.o pl_comp.o pl_exec.o pl_funcs.o: plpgsql.h pl_gram.h
-
-# pl_scan is compiled as part of pl_gram
-pl_gram.o: pl_scan.c
+pl_gram.o pl_handler.o pl_comp.o pl_exec.o pl_funcs.o pl_scanner.o: plpgsql.h pl_gram.h
 
 # See notes in src/backend/parser/Makefile about the following two rules
 
@@ -49,23 +46,12 @@ else
 	@$(missing) bison $< $@
 endif
 
-# Because we use %option case-insensitive, flex's results could vary
-# depending on what the compile-time locale setting is.  Hence, force
-# it to see LC_CTYPE=C to ensure consistent build results.
-
-pl_scan.c: scan.l
-ifdef FLEX
-	LC_CTYPE=C $(FLEX) $(FLEXFLAGS) -o'$@' $<
-else
-	@$(missing) flex $< $@
-endif
-
-distprep: pl_scan.c pl_gram.h pl_gram.c
+distprep: pl_gram.h pl_gram.c
 
-# pl_gram.c, pl_gram.h, and pl_scan.c are in the distribution tarball,
+# pl_gram.c and pl_gram.h are in the distribution tarball,
 # so they are not cleaned here.
 clean distclean: clean-lib
 	rm -f $(OBJS)
 
 maintainer-clean: clean
-	rm -f pl_gram.c pl_gram.h pl_scan.c
+	rm -f pl_gram.c pl_gram.h

src/pl/plpgsql/src/nls.mk

-# $PostgreSQL: pgsql/src/pl/plpgsql/src/nls.mk,v 1.11 2009/10/20 18:23:27 petere Exp $
+# $PostgreSQL: pgsql/src/pl/plpgsql/src/nls.mk,v 1.12 2009/11/12 00:13:00 tgl Exp $
 CATALOG_NAME	:= plpgsql
 AVAIL_LANGUAGES	:= de es fr it ja ro
-GETTEXT_FILES	:= pl_comp.c pl_exec.c pl_gram.c pl_funcs.c pl_handler.c pl_scan.c
+GETTEXT_FILES	:= pl_comp.c pl_exec.c pl_gram.c pl_funcs.c pl_handler.c pl_scanner.c
 GETTEXT_TRIGGERS:= _ errmsg errmsg_plural:1,2 errdetail errdetail_log errdetail_plural:1,2 errhint errcontext yyerror plpgsql_yyerror
 
 .PHONY: gettext-files

src/pl/plpgsql/src/pl_funcs.c

@@ -8,17 +8,13 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_funcs.c,v 1.85 2009/11/07 00:52:26 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_funcs.c,v 1.86 2009/11/12 00:13:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
 
 #include "plpgsql.h"
 
-#include <ctype.h>
-
-#include "parser/scansup.h"
-
 
 /* ----------
  * Local variables for namespace handling
@@ -208,104 +204,6 @@ plpgsql_ns_lookup_label(PLpgSQL_nsitem *ns_cur, const char *name)
 }
 
 
-/* ----------
- * plpgsql_convert_ident
- *
- * Convert a possibly-qualified identifier to internal form: handle
- * double quotes, translate to lower case where not inside quotes,
- * truncate to NAMEDATALEN.
- *
- * There may be several identifiers separated by dots and optional
- * whitespace.	Each one is converted to a separate palloc'd string.
- * The caller passes the expected number of identifiers, as well as
- * a char* array to hold them.	It is an error if we find the wrong
- * number of identifiers (cf grammar processing of fori_varname).
- *
- * NOTE: the input string has already been accepted by the flex lexer,
- * so we don't need a heckuva lot of error checking here.
- * ----------
- */
-void
-plpgsql_convert_ident(const char *s, char **output, int numidents)
-{
-	const char *sstart = s;
-	int			identctr = 0;
-
-	/* Outer loop over identifiers */
-	while (*s)
-	{
-		char	   *curident;
-		char	   *cp;
-
-		/* Process current identifier */
-
-		if (*s == '"')
-		{
-			/* Quoted identifier: copy, collapsing out doubled quotes */
-
-			curident = palloc(strlen(s) + 1);	/* surely enough room */
-			cp = curident;
-			s++;
-			while (*s)
-			{
-				if (*s == '"')
-				{
-					if (s[1] != '"')
-						break;
-					s++;
-				}
-				*cp++ = *s++;
-			}
-			if (*s != '"')		/* should not happen if lexer checked */
-				ereport(ERROR,
-						(errcode(ERRCODE_SYNTAX_ERROR),
-					   errmsg("unterminated \" in identifier: %s", sstart)));
-			s++;
-			*cp = '\0';
-			/* Truncate to NAMEDATALEN */
-			truncate_identifier(curident, cp - curident, false);
-		}
-		else
-		{
-			/* Normal identifier: extends till dot or whitespace */
-			const char *thisstart = s;
-
-			while (*s && *s != '.' && !scanner_isspace(*s))
-				s++;
-			/* Downcase and truncate to NAMEDATALEN */
-			curident = downcase_truncate_identifier(thisstart, s - thisstart,
-													false);
-		}
-
-		/* Pass ident to caller */
-		if (identctr < numidents)
-			output[identctr++] = curident;
-		else
-			ereport(ERROR,
-					(errcode(ERRCODE_SYNTAX_ERROR),
-					 errmsg("qualified identifier cannot be used here: %s",
-							sstart)));
-
-		/* If not done, skip whitespace, dot, whitespace */
-		if (*s)
-		{
-			while (*s && scanner_isspace(*s))
-				s++;
-			if (*s++ != '.')
-				elog(ERROR, "expected dot between identifiers: %s", sstart);
-			while (*s && scanner_isspace(*s))
-				s++;
-			if (*s == '\0')
-				elog(ERROR, "expected another identifier: %s", sstart);
-		}
-	}
-
-	if (identctr != numidents)
-		elog(ERROR, "improperly qualified identifier: %s",
-			 sstart);
-}
-
-
 /*
  * Statement type as a string, for use in error messages etc.
  */

src/pl/plpgsql/src/plpgsql.h

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/plpgsql.h,v 1.123 2009/11/10 02:13:13 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/plpgsql.h,v 1.124 2009/11/12 00:13:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -794,20 +794,18 @@ typedef struct
  * Global variable declarations
  **********************************************************************/
 
+extern bool plpgsql_check_syntax;
 extern bool plpgsql_DumpExecTree;
 extern bool plpgsql_LookupIdentifiers;
+
+extern PLpgSQL_stmt_block *plpgsql_parse_result;
+
 extern int	plpgsql_nDatums;
 extern PLpgSQL_datum **plpgsql_Datums;
 
 extern char *plpgsql_error_funcname;
 
-/* linkage to the real yytext variable */
-extern char *plpgsql_base_yytext;
-
-#define yytext plpgsql_base_yytext
-
 extern PLpgSQL_function *plpgsql_curr_compile;
-extern bool plpgsql_check_syntax;
 extern MemoryContext compile_tmp_cxt;
 
 extern PLpgSQL_plugin **plugin_ptr;
@@ -825,9 +823,12 @@ extern PLpgSQL_function *plpgsql_compile(FunctionCallInfo fcinfo,
 extern PLpgSQL_function *plpgsql_compile_inline(char *proc_source);
 extern void plpgsql_parser_setup(struct ParseState *pstate,
 								 PLpgSQL_expr *expr);
-extern int	plpgsql_parse_word(const char *word);
-extern int	plpgsql_parse_dblword(const char *word);
-extern int	plpgsql_parse_tripword(const char *word);
+extern bool plpgsql_parse_word(char *word1, const char *yytxt,
+							   PLwdatum *wdatum, PLword *word);
+extern bool plpgsql_parse_dblword(char *word1, char *word2,
+								  PLwdatum *wdatum, PLcword *cword);
+extern bool plpgsql_parse_tripword(char *word1, char *word2, char *word3,
+								   PLwdatum *wdatum, PLcword *cword);
 extern PLpgSQL_type *plpgsql_parse_wordtype(char *ident);
 extern PLpgSQL_type *plpgsql_parse_cwordtype(List *idents);
 extern PLpgSQL_type *plpgsql_parse_wordrowtype(char *ident);
@@ -889,16 +890,13 @@ extern PLpgSQL_nsitem *plpgsql_ns_lookup_label(PLpgSQL_nsitem *ns_cur,
  * Other functions in pl_funcs.c
  * ----------
  */
-extern void plpgsql_convert_ident(const char *s, char **output, int numidents);
 extern const char *plpgsql_stmt_typename(PLpgSQL_stmt *stmt);
 extern void plpgsql_dumptree(PLpgSQL_function *func);
 
 /* ----------
- * Externs in gram.y and scan.l
+ * Scanner functions in pl_scanner.c
  * ----------
  */
-extern PLpgSQL_expr *plpgsql_read_expression(int until, const char *expected);
-extern int	plpgsql_yyparse(void);
 extern int	plpgsql_base_yylex(void);
 extern int	plpgsql_yylex(void);
 extern void plpgsql_push_back_token(int token);
@@ -911,4 +909,10 @@ extern int	plpgsql_latest_lineno(void);
 extern void plpgsql_scanner_init(const char *str);
 extern void plpgsql_scanner_finish(void);
 
+/* ----------
+ * Externs in gram.y
+ * ----------
+ */
+extern int	plpgsql_yyparse(void);
+
 #endif   /* PLPGSQL_H */

src/test/regress/expected/plpgsql.out

@@ -3889,6 +3889,10 @@ WARNING:  nonstandard use of \\ in a string literal
 LINE 4:   return 'foo\\bar\041baz';
                  ^
 HINT:  Use the escape string syntax for backslashes, e.g., E'\\'.
+WARNING:  nonstandard use of \\ in a string literal
+LINE 4:   return 'foo\\bar\041baz';
+                 ^
+HINT:  Use the escape string syntax for backslashes, e.g., E'\\'.
 select strtest();
 NOTICE:  foo\bar!baz
 WARNING:  nonstandard use of \\ in a string literal
@@ -4020,3 +4024,19 @@ select * from conflict_test();
 (5 rows)
 
 drop function conflict_test();
+-- Check that an unreserved keyword can be used as a variable name
+create function unreserved_test() returns int as $$
+declare
+  forward int := 21;
+begin
+  forward := forward * 2;
+  return forward;
+end
+$$ language plpgsql;
+select unreserved_test();
+ unreserved_test 
+-----------------
+              42
+(1 row)
+
+drop function unreserved_test();

src/test/regress/sql/plpgsql.sql

@@ -3189,3 +3189,18 @@ $$ language plpgsql;
 select * from conflict_test();
 
 drop function conflict_test();
+
+-- Check that an unreserved keyword can be used as a variable name
+
+create function unreserved_test() returns int as $$
+declare
+  forward int := 21;
+begin
+  forward := forward * 2;
+  return forward;
+end
+$$ language plpgsql;
+
+select unreserved_test();
+
+drop function unreserved_test();

src/tools/msvc/Mkvcbuild.pm

@@ -3,7 +3,7 @@ package Mkvcbuild;
 #
 # Package that generates build files for msvc build
 #
-# $PostgreSQL: pgsql/src/tools/msvc/Mkvcbuild.pm,v 1.43 2009/10/01 01:58:58 tgl Exp $
+# $PostgreSQL: pgsql/src/tools/msvc/Mkvcbuild.pm,v 1.44 2009/11/12 00:13:00 tgl Exp $
 #
 use Carp;
 use Win32;
@@ -80,7 +80,7 @@ sub mkvcbuild
     $snowball->AddReference($postgres);
 
     my $plpgsql = $solution->AddProject('plpgsql','dll','PLs','src\pl\plpgsql\src');
-    $plpgsql->AddFiles('src\pl\plpgsql\src','scan.l','gram.y');
+    $plpgsql->AddFiles('src\pl\plpgsql\src', 'gram.y');
     $plpgsql->AddReference($postgres);
 
     if ($solution->{options}->{perl})

src/tools/msvc/Project.pm

@@ -3,7 +3,7 @@ package Project;
 #
 # Package that encapsulates a Visual C++ project file generation
 #
-# $PostgreSQL: pgsql/src/tools/msvc/Project.pm,v 1.20 2009/07/27 07:11:15 mha Exp $
+# $PostgreSQL: pgsql/src/tools/msvc/Project.pm,v 1.21 2009/11/12 00:13:00 tgl Exp $
 #
 use Carp;
 use strict;
@@ -398,7 +398,6 @@ EOF
         {
             my $of = $f;
             $of =~ s/\.l$/.c/;
-            $of =~ s{^src\\pl\\plpgsql\\src\\scan.c$}{src\\pl\\plpgsql\\src\\pl_scan.c};
             print F '>'
               . GenerateCustomTool('Running flex on ' . $f, 'src\tools\msvc\pgflex.bat ' . $f,$of)
               . '</File>' . "\n";

src/tools/msvc/clean.bat

 @echo off
-REM $PostgreSQL: pgsql/src/tools/msvc/clean.bat,v 1.15 2008/08/30 02:32:24 tgl Exp $
+REM $PostgreSQL: pgsql/src/tools/msvc/clean.bat,v 1.16 2009/11/12 00:13:00 tgl Exp $
 
 set DIST=0
 if "%1"=="dist" set DIST=1
@@ -49,7 +49,6 @@ if %DIST%==1 if exist src\interfaces\ecpg\preproc\preproc.h del /q src\interface
 if exist src\port\pg_config_paths.h del /q src\port\pg_config_paths.h
 
 if exist src\pl\plperl\spi.c del /q src\pl\plperl\spi.c
-if %DIST%==1 if exist src\pl\plpgsql\src\pl_scan.c del /q src\pl\plpgsql\src\pl_scan.c
 if %DIST%==1 if exist src\pl\plpgsql\src\pl_gram.c del /q src\pl\plpgsql\src\pl_gram.c
 if %DIST%==1 if exist src\pl\plpgsql\src\pl_gram.h del /q src\pl\plpgsql\src\pl_gram.h
 

src/tools/msvc/pgflex.bat

 @echo off
-REM $PostgreSQL: pgsql/src/tools/msvc/pgflex.bat,v 1.5 2007/12/19 12:29:36 mha Exp $
+REM $PostgreSQL: pgsql/src/tools/msvc/pgflex.bat,v 1.6 2009/11/12 00:13:00 tgl Exp $
 
 IF NOT EXIST src\tools\msvc\buildenv.pl goto nobuildenv
 perl -e "require 'src/tools/msvc/buildenv.pl'; while(($k,$v) = each %ENV) { print qq[\@SET $k=$v\n]; }" > bldenv.bat
@@ -13,7 +13,6 @@ if errorlevel 1 goto noflex
 if "%1" == "src\backend\parser\scan.l" call :generate %1 src\backend\parser\scan.c -CF
 if "%1" == "src\backend\bootstrap\bootscanner.l" call :generate %1 src\backend\bootstrap\bootscanner.c
 if "%1" == "src\backend\utils\misc\guc-file.l" call :generate %1 src\backend\utils\misc\guc-file.c
-if "%1" == "src\pl\plpgsql\src\scan.l" call :generate %1 src\pl\plpgsql\src\pl_scan.c
 if "%1" == "src\interfaces\ecpg\preproc\pgc.l" call :generate %1 src\interfaces\ecpg\preproc\pgc.c
 if "%1" == "src\bin\psql\psqlscan.l" call :generate %1 src\bin\psql\psqlscan.c
 if "%1" == "contrib\cube\cubescan.l" call :generate %1 contrib\cube\cubescan.c