Get rid of "lookahead" functionality in plpgsql's yylex() function,

and instead make the grammar production for the RETURN statement do the
heavy lifting.  The lookahead idea was copied from the main parser, but
it does not work in plpgsql's parser because here gram.y looks explicitly
at the scanner's yytext variable, which will be out of sync after a
failed lookahead step.  A minimal example is

create or replace function foo() returns void language plpgsql as '
begin
  perform return foo bar;
end';

which can be seen by testing to deliver "foo foo bar" to the main parser
instead of the expected "return foo bar".  This isn't a huge bug since
RETURN is not found in the main grammar, but it could bite someone who
tried to use "return" as an identifier.

Back-patch to 8.1.  Bug exists further back, but HEAD patch doesn't apply
cleanly, and given the lack of field complaints it doesn't seem worth
the effort to develop adjusted patches.

src/pl/plpgsql/src/gram.y

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/gram.y,v 1.93 2006/06/16 23:29:26 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/gram.y,v 1.94 2006/08/14 00:46:53 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -28,8 +28,10 @@ static PLpgSQL_expr		*read_sql_construct(int until,
 											int *endtoken);
 static	PLpgSQL_expr	*read_sql_stmt(const char *sqlstart);
 static	PLpgSQL_type	*read_datatype(int tok);
-static	PLpgSQL_stmt	*make_select_stmt(void);
-static	PLpgSQL_stmt	*make_fetch_stmt(void);
+static	PLpgSQL_stmt	*make_select_stmt(int lineno);
+static	PLpgSQL_stmt	*make_fetch_stmt(int lineno, int curvar);
+static	PLpgSQL_stmt	*make_return_stmt(int lineno);
+static	PLpgSQL_stmt	*make_return_next_stmt(int lineno);
 static	void			 check_assignable(PLpgSQL_datum *datum);
 static	PLpgSQL_row		*read_into_scalar_list(const char *initial_name,
 											   PLpgSQL_datum *initial_datum);
@@ -118,7 +120,7 @@ static	void			 check_labels(const char *start_label,
 %type <loop_body>	loop_body
 %type <stmt>	proc_stmt pl_block
 %type <stmt>	stmt_assign stmt_if stmt_loop stmt_while stmt_exit
-%type <stmt>	stmt_return stmt_return_next stmt_raise stmt_execsql
+%type <stmt>	stmt_return stmt_raise stmt_execsql
 %type <stmt>	stmt_for stmt_select stmt_perform
 %type <stmt>	stmt_dynexecute stmt_getdiag
 %type <stmt>	stmt_open stmt_fetch stmt_close stmt_null
@@ -183,7 +185,6 @@ static	void			 check_labels(const char *start_label,
 %token	K_RENAME
 %token	K_RESULT_OID
 %token	K_RETURN
-%token	K_RETURN_NEXT
 %token	K_REVERSE
 %token	K_SELECT
 %token	K_STRICT
@@ -596,8 +597,6 @@ proc_stmt		: pl_block ';'
 						{ $$ = $1; }
 				| stmt_return
 						{ $$ = $1; }
-				| stmt_return_next
-						{ $$ = $1; }
 				| stmt_raise
 						{ $$ = $1; }
 				| stmt_execsql
@@ -1130,8 +1129,7 @@ for_variable	: T_SCALAR
 
 stmt_select		: K_SELECT lno
 					{
-						$$ = make_select_stmt();
-						$$->lineno = $2;
+						$$ = make_select_stmt($2);
 					}
 				;
 
@@ -1162,109 +1160,18 @@ exit_type		: K_EXIT
 
 stmt_return		: K_RETURN lno
 					{
-						PLpgSQL_stmt_return *new;
-
-						new = palloc0(sizeof(PLpgSQL_stmt_return));
-						new->cmd_type = PLPGSQL_STMT_RETURN;
-						new->lineno   = $2;
-						new->expr	  = NULL;
-						new->retvarno = -1;
+						int	tok;
 
-						if (plpgsql_curr_compile->fn_retset)
-						{
-							if (yylex() != ';')
-								yyerror("RETURN cannot have a parameter in function returning set; use RETURN NEXT");
-						}
-						else if (plpgsql_curr_compile->out_param_varno >= 0)
-						{
-							if (yylex() != ';')
-								yyerror("RETURN cannot have a parameter in function with OUT parameters");
-							new->retvarno = plpgsql_curr_compile->out_param_varno;
-						}
-						else if (plpgsql_curr_compile->fn_rettype == VOIDOID)
+						tok = yylex();
+						if (tok == K_NEXT)
 						{
-							if (yylex() != ';')
-								yyerror("RETURN cannot have a parameter in function returning void");
-						}
-						else if (plpgsql_curr_compile->fn_retistuple)
-						{
-							switch (yylex())
-							{
-								case K_NULL:
-									/* we allow this to support RETURN NULL in triggers */
-									break;
-
-								case T_ROW:
-									new->retvarno = yylval.row->rowno;
-									break;
-
-								case T_RECORD:
-									new->retvarno = yylval.rec->recno;
-									break;
-
-								default:
-									yyerror("RETURN must specify a record or row variable in function returning tuple");
-									break;
-							}
-							if (yylex() != ';')
-								yyerror("RETURN must specify a record or row variable in function returning tuple");
+							$$ = make_return_next_stmt($2);
 						}
 						else
 						{
-							/*
-							 * Note that a well-formed expression is
-							 * _required_ here; anything else is a
-							 * compile-time error.
-							 */
-							new->expr = plpgsql_read_expression(';', ";");
+							plpgsql_push_back_token(tok);
+							$$ = make_return_stmt($2);
 						}
-
-						$$ = (PLpgSQL_stmt *)new;
-					}
-				;
-
-stmt_return_next: K_RETURN_NEXT lno
-					{
-						PLpgSQL_stmt_return_next *new;
-
-						if (!plpgsql_curr_compile->fn_retset)
-							yyerror("cannot use RETURN NEXT in a non-SETOF function");
-
-						new = palloc0(sizeof(PLpgSQL_stmt_return_next));
-						new->cmd_type	= PLPGSQL_STMT_RETURN_NEXT;
-						new->lineno		= $2;
-						new->expr = NULL;
-						new->retvarno	= -1;
-
-						if (plpgsql_curr_compile->out_param_varno >= 0)
-						{
-							if (yylex() != ';')
-								yyerror("RETURN NEXT cannot have a parameter in function with OUT parameters");
-							new->retvarno = plpgsql_curr_compile->out_param_varno;
-						}
-						else if (plpgsql_curr_compile->fn_retistuple)
-						{
-							switch (yylex())
-							{
-								case T_ROW:
-									new->retvarno = yylval.row->rowno;
-									break;
-
-								case T_RECORD:
-									new->retvarno = yylval.rec->recno;
-									break;
-
-								default:
-									yyerror("RETURN NEXT must specify a record or row variable in function returning tuple");
-									break;
-							}
-							if (yylex() != ';')
-								yyerror("RETURN NEXT must specify a record or row variable in function returning tuple");
-						}
-						else
-							new->expr = plpgsql_read_expression(';', ";");
-
-						$$ = (PLpgSQL_stmt *)new;
 					}
 				;
 
@@ -1537,13 +1444,7 @@ stmt_open		: K_OPEN lno cursor_varptr
 
 stmt_fetch		: K_FETCH lno cursor_variable K_INTO
 					{
-						PLpgSQL_stmt_fetch *new;
-
-						new = (PLpgSQL_stmt_fetch *)make_fetch_stmt();
-						new->curvar = $3;
-
-						$$ = (PLpgSQL_stmt *)new;
-						$$->lineno = $2;
+						$$ = make_fetch_stmt($2, $3);
 					}
 				;
 
@@ -1991,7 +1892,7 @@ read_datatype(int tok)
 }
 
 static PLpgSQL_stmt *
-make_select_stmt(void)
+make_select_stmt(int lineno)
 {
 	PLpgSQL_dstring		ds;
 	int					nparams = 0;
@@ -2112,6 +2013,7 @@ make_select_stmt(void)
 
 		select = palloc0(sizeof(PLpgSQL_stmt_select));
 		select->cmd_type = PLPGSQL_STMT_SELECT;
+		select->lineno   = lineno;
 		select->rec		 = rec;
 		select->row		 = row;
 		select->query	 = expr;
@@ -2125,6 +2027,7 @@ make_select_stmt(void)
 
 		execsql = palloc(sizeof(PLpgSQL_stmt_execsql));
 		execsql->cmd_type = PLPGSQL_STMT_EXECSQL;
+		execsql->lineno   = lineno;
 		execsql->sqlstmt  = expr;
 
 		return (PLpgSQL_stmt *)execsql;
@@ -2133,7 +2036,7 @@ make_select_stmt(void)
 
 
 static PLpgSQL_stmt *
-make_fetch_stmt(void)
+make_fetch_stmt(int lineno, int curvar)
 {
 	int					tok;
 	PLpgSQL_row		   *row = NULL;
@@ -2172,12 +2075,124 @@ make_fetch_stmt(void)
 	if (tok != ';')
 		yyerror("syntax error");
 
-	fetch = palloc0(sizeof(PLpgSQL_stmt_select));
+	fetch = palloc0(sizeof(PLpgSQL_stmt_fetch));
 	fetch->cmd_type = PLPGSQL_STMT_FETCH;
-	fetch->rec		 = rec;
-	fetch->row		 = row;
+	fetch->lineno	= lineno;
+	fetch->rec		= rec;
+	fetch->row		= row;
+	fetch->curvar	= curvar;
+
+	return (PLpgSQL_stmt *) fetch;
+}
+
+
+static PLpgSQL_stmt *
+make_return_stmt(int lineno)
+{
+	PLpgSQL_stmt_return *new;
+
+	new = palloc0(sizeof(PLpgSQL_stmt_return));
+	new->cmd_type = PLPGSQL_STMT_RETURN;
+	new->lineno   = lineno;
+	new->expr	  = NULL;
+	new->retvarno = -1;
+
+	if (plpgsql_curr_compile->fn_retset)
+	{
+		if (yylex() != ';')
+			yyerror("RETURN cannot have a parameter in function returning set; use RETURN NEXT");
+	}
+	else if (plpgsql_curr_compile->out_param_varno >= 0)
+	{
+		if (yylex() != ';')
+			yyerror("RETURN cannot have a parameter in function with OUT parameters");
+		new->retvarno = plpgsql_curr_compile->out_param_varno;
+	}
+	else if (plpgsql_curr_compile->fn_rettype == VOIDOID)
+	{
+		if (yylex() != ';')
+			yyerror("RETURN cannot have a parameter in function returning void");
+	}
+	else if (plpgsql_curr_compile->fn_retistuple)
+	{
+		switch (yylex())
+		{
+			case K_NULL:
+				/* we allow this to support RETURN NULL in triggers */
+				break;
+
+			case T_ROW:
+				new->retvarno = yylval.row->rowno;
+				break;
+
+			case T_RECORD:
+				new->retvarno = yylval.rec->recno;
+				break;
+
+			default:
+				yyerror("RETURN must specify a record or row variable in function returning tuple");
+				break;
+		}
+		if (yylex() != ';')
+			yyerror("RETURN must specify a record or row variable in function returning tuple");
+	}
+	else
+	{
+		/*
+		 * Note that a well-formed expression is
+		 * _required_ here; anything else is a
+		 * compile-time error.
+		 */
+		new->expr = plpgsql_read_expression(';', ";");
+	}
+
+	return (PLpgSQL_stmt *) new;
+}
+
+
+static PLpgSQL_stmt *
+make_return_next_stmt(int lineno)
+{
+	PLpgSQL_stmt_return_next *new;
+
+	if (!plpgsql_curr_compile->fn_retset)
+		yyerror("cannot use RETURN NEXT in a non-SETOF function");
+
+	new = palloc0(sizeof(PLpgSQL_stmt_return_next));
+	new->cmd_type	= PLPGSQL_STMT_RETURN_NEXT;
+	new->lineno		= lineno;
+	new->expr		= NULL;
+	new->retvarno	= -1;
+
+	if (plpgsql_curr_compile->out_param_varno >= 0)
+	{
+		if (yylex() != ';')
+			yyerror("RETURN NEXT cannot have a parameter in function with OUT parameters");
+		new->retvarno = plpgsql_curr_compile->out_param_varno;
+	}
+	else if (plpgsql_curr_compile->fn_retistuple)
+	{
+		switch (yylex())
+		{
+			case T_ROW:
+				new->retvarno = yylval.row->rowno;
+				break;
+
+			case T_RECORD:
+				new->retvarno = yylval.rec->recno;
+				break;
+
+			default:
+				yyerror("RETURN NEXT must specify a record or row variable in function returning tuple");
+				break;
+		}
+		if (yylex() != ';')
+			yyerror("RETURN NEXT must specify a record or row variable in function returning tuple");
+	}
+	else
+		new->expr = plpgsql_read_expression(';', ";");
 
-	return (PLpgSQL_stmt *)fetch;
+	return (PLpgSQL_stmt *) new;
 }
 
 

src/pl/plpgsql/src/scan.l

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/scan.l,v 1.52 2006/06/16 23:29:27 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/scan.l,v 1.53 2006/08/14 00:46:53 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -36,8 +36,6 @@ static int	scanner_functype;
 static bool	scanner_typereported;
 static int	pushback_token;
 static bool have_pushback_token;
-static int	lookahead_token;	
-static bool have_lookahead_token;
 static const char *cur_line_start;
 static int	cur_line_num;
 static char    *dolqstart;      /* current $foo$ quote start string */
@@ -346,53 +344,25 @@ dump			{ return O_DUMP;			}
 
 /*
  * This is the yylex routine called from outside. It exists to provide
- * a pushback facility, as well as to allow us to parse syntax that
- * requires more than one token of lookahead.
+ * a one-token pushback facility.  Beware of trying to make it do more:
+ * for the most part, plpgsql's gram.y assumes that yytext is in step
+ * with the "current token".
  */
 int
 plpgsql_yylex(void)
 {
-	int cur_token;
-
 	if (have_pushback_token)
 	{
 		have_pushback_token = false;
-		cur_token = pushback_token;
-	}
-	else if (have_lookahead_token)
-	{
-		have_lookahead_token = false;
-		cur_token = lookahead_token;
-	}
-	else
-		cur_token = yylex();
-
-	/* Do we need to look ahead for a possible multiword token? */
-	switch (cur_token)
-	{
-		/* RETURN NEXT must be reduced to a single token */
-		case K_RETURN:
-			if (!have_lookahead_token)
-			{
-				lookahead_token = yylex();
-				have_lookahead_token = true;
-			}
-			if (lookahead_token == K_NEXT)
-			{
-				have_lookahead_token = false;
-				cur_token = K_RETURN_NEXT;
-			}
-			break;
-
-		default:
-			break;
+		return pushback_token;
 	}
-
-	return cur_token;
+	return yylex();
 }
 
 /*
  * Push back a single token to be re-read by next plpgsql_yylex() call.
+ *
+ * NOTE: this does not cause yytext to "back up".
  */
 void
 plpgsql_push_back_token(int token)
@@ -494,7 +464,6 @@ plpgsql_scanner_init(const char *str, int functype)
     scanner_typereported = false;
 
 	have_pushback_token = false;
-	have_lookahead_token = false;
 
 	cur_line_start = scanbuf;
 	cur_line_num = 1;