Skip to content

P
Postgres FD Implementation
Commit 20aea2ec authored by Tom Lane's avatar Tom Lane
Browse files
Options

Alter plpgsql's lexer so that yylineno and yymore are not used. This 

avoids 'input buffer overflow' failure on long literals, improves
performance, gives the right answer for line position in functions
containing multiline literals, suppresses annoying compiler warnings,
and generally is so much better I wonder why we didn't do it before.
parent ea7896bf
Show whitespace changes
Inline Side-by-side
Showing with 132 additions and 88 deletions
src/pl/plpgsql/src/gram.y
View file @ 20aea2ec
......@@ -4,7 +4,7 @@
* procedural language
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/pl/plpgsql/src/gram.y,v 1.42 2003/04/27 22:21:22 tgl Exp $
* $Header: /cvsroot/pgsql/src/pl/plpgsql/src/gram.y,v 1.43 2003/05/05 16:46:27 tgl Exp $
*
* This software is copyrighted by Jan Wieck - Hamburg.
*
......@@ -334,7 +334,7 @@ decl_statement : decl_varname decl_const decl_datatype decl_notnull decl_defval
/* Composite type --- treat as rowtype */
PLpgSQL_row *row;
row = build_rowtype($3->typrelid);
row = plpgsql_build_rowtype($3->typrelid);
row->dtype = PLPGSQL_DTYPE_ROW;
row->refname = $1.name;
row->lineno = $1.lineno;
......@@ -486,7 +486,7 @@ decl_cursor_arglist : decl_cursor_arg
new->dtype = PLPGSQL_DTYPE_ROW;
new->refname = strdup("*internal*");
new->lineno = yylineno;
new->lineno = plpgsql_scanner_lineno();
new->rowtypeclass = InvalidOid;
/*
* We make temporary fieldnames/varnos arrays that
......@@ -553,7 +553,7 @@ decl_aliasitem : T_WORD
nsi = plpgsql_ns_lookup(name, NULL);
if (nsi == NULL)
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "function has no parameter %s", name);
}
......@@ -578,7 +578,7 @@ decl_varname : T_WORD
plpgsql_convert_ident(yytext, &name, 1);
/* name should be malloc'd for use as varname */
$$.name = strdup(name);
$$.lineno = yylineno;
$$.lineno = plpgsql_scanner_lineno();
pfree(name);
}
;
......@@ -625,7 +625,7 @@ decl_defval : ';'
PLpgSQL_dstring ds;
PLpgSQL_expr *expr;
lno = yylineno;
lno = plpgsql_scanner_lineno();
expr = malloc(sizeof(PLpgSQL_expr));
plpgsql_dstring_init(&ds);
plpgsql_dstring_append(&ds, "SELECT ");
......@@ -1034,7 +1034,7 @@ fori_varname : T_VARIABLE
plpgsql_convert_ident(yytext, &name, 1);
/* name should be malloc'd for use as varname */
$$.name = strdup(name);
$$.lineno = yylineno;
$$.lineno = plpgsql_scanner_lineno();
pfree(name);
}
| T_WORD
......@@ -1044,7 +1044,7 @@ fori_varname : T_VARIABLE
plpgsql_convert_ident(yytext, &name, 1);
/* name should be malloc'd for use as varname */
$$.name = strdup(name);
$$.lineno = yylineno;
$$.lineno = plpgsql_scanner_lineno();
pfree(name);
}
;
......@@ -1405,7 +1405,7 @@ stmt_open : K_OPEN lno cursor_varptr
if (tok != '(')
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "cursor %s has arguments",
$3->refname);
}
......@@ -1427,7 +1427,7 @@ stmt_open : K_OPEN lno cursor_varptr
if (strncmp(cp, "SELECT", 6) != 0)
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "expected 'SELECT (', got '%s' (internal error)",
new->argquery->query);
}
......@@ -1436,7 +1436,7 @@ stmt_open : K_OPEN lno cursor_varptr
cp++;
if (*cp != '(')
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "expected 'SELECT (', got '%s' (internal error)",
new->argquery->query);
}
......@@ -1454,13 +1454,13 @@ stmt_open : K_OPEN lno cursor_varptr
if (tok == '(')
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "cursor %s has no arguments", $3->refname);
}
if (tok != ';')
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "syntax error at \"%s\"", yytext);
}
}
......@@ -1502,7 +1502,7 @@ cursor_varptr : T_VARIABLE
if (((PLpgSQL_var *) yylval.variable)->datatype->typoid != REFCURSOROID)
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "%s must be of type cursor or refcursor",
((PLpgSQL_var *) yylval.variable)->refname);
}
......@@ -1517,7 +1517,7 @@ cursor_variable : T_VARIABLE
if (((PLpgSQL_var *) yylval.variable)->datatype->typoid != REFCURSOROID)
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "%s must be of type refcursor",
((PLpgSQL_var *) yylval.variable)->refname);
}
......@@ -1583,8 +1583,7 @@ opt_lblname : T_WORD
lno :
{
plpgsql_error_lineno = yylineno;
$$ = yylineno;
$$ = plpgsql_error_lineno = plpgsql_scanner_lineno();
}
;
......@@ -1618,7 +1617,7 @@ read_sql_construct(int until,
char buf[32];
PLpgSQL_expr *expr;
lno = yylineno;
lno = plpgsql_scanner_lineno();
plpgsql_dstring_init(&ds);
plpgsql_dstring_append(&ds, (char *) sqlstart);
......@@ -1690,7 +1689,7 @@ read_datatype(int tok)
bool needspace = false;
int parenlevel = 0;
lno = yylineno;
lno = plpgsql_scanner_lineno();
/* Often there will be a lookahead token, but if not, get one */
if (tok == YYEMPTY)
......@@ -1769,14 +1768,14 @@ make_select_stmt(void)
break;
if (tok == 0)
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "unexpected end of file");
}
if (tok == K_INTO)
{
if (have_into)
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "INTO specified more than once");
}
tok = yylex();
......@@ -1814,7 +1813,7 @@ make_select_stmt(void)
break;
default:
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "plpgsql: %s is not a variable",
yytext);
}
......@@ -1824,7 +1823,7 @@ make_select_stmt(void)
row = malloc(sizeof(PLpgSQL_row));
row->dtype = PLPGSQL_DTYPE_ROW;
row->refname = strdup("*internal*");
row->lineno = yylineno;
row->lineno = plpgsql_scanner_lineno();
row->rowtypeclass = InvalidOid;
row->nfields = nfields;
row->fieldnames = malloc(sizeof(char *) * nfields);
......@@ -1945,7 +1944,7 @@ make_fetch_stmt(void)
break;
default:
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "plpgsql: %s is not a variable",
yytext);
}
......@@ -1955,7 +1954,7 @@ make_fetch_stmt(void)
row = malloc(sizeof(PLpgSQL_row));
row->dtype = PLPGSQL_DTYPE_ROW;
row->refname = strdup("*internal*");
row->lineno = yylineno;
row->lineno = plpgsql_scanner_lineno();
row->rowtypeclass = InvalidOid;
row->nfields = nfields;
row->fieldnames = malloc(sizeof(char *) * nfields);
......@@ -2028,7 +2027,7 @@ check_assignable(PLpgSQL_datum *datum)
case PLPGSQL_DTYPE_VAR:
if (((PLpgSQL_var *) datum)->isconst)
{
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "%s is declared CONSTANT",
((PLpgSQL_var *) datum)->refname);
}
......
src/pl/plpgsql/src/pl_comp.c
View file @ 20aea2ec
......@@ -3,7 +3,7 @@
* procedural language
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/pl/plpgsql/src/pl_comp.c,v 1.57 2003/04/27 22:21:22 tgl Exp $
* $Header: /cvsroot/pgsql/src/pl/plpgsql/src/pl_comp.c,v 1.58 2003/05/05 16:46:27 tgl Exp $
*
* This software is copyrighted by Jan Wieck - Hamburg.
*
......@@ -141,7 +141,8 @@ plpgsql_compile(Oid fn_oid, int functype)
procStruct = (Form_pg_proc) GETSTRUCT(procTup);
proc_source = DatumGetCString(DirectFunctionCall1(textout,
PointerGetDatum(&procStruct->prosrc)));
plpgsql_setinput(proc_source, functype);
plpgsql_scanner_init(proc_source, functype);
pfree(proc_source);
plpgsql_error_funcname = pstrdup(NameStr(procStruct->proname));
plpgsql_error_lineno = 0;
......@@ -258,7 +259,7 @@ plpgsql_compile(Oid fn_oid, int functype)
* For tuple type parameters, we set up a record of
* that type
*/
row = build_rowtype(typeStruct->typrelid);
row = plpgsql_build_rowtype(typeStruct->typrelid);
row->refname = strdup(buf);
......@@ -496,6 +497,8 @@ plpgsql_compile(Oid fn_oid, int functype)
if (parse_rc != 0)
elog(ERROR, "plpgsql: parser returned %d ???", parse_rc);
plpgsql_scanner_finish();
/*
* If that was successful, complete the functions info.
*/
......@@ -1200,7 +1203,7 @@ plpgsql_parse_wordrowtype(char *word)
/*
* Build and return the complete row definition
*/
plpgsql_yylval.row = build_rowtype(classOid);
plpgsql_yylval.row = plpgsql_build_rowtype(classOid);
pfree(cp[0]);
pfree(cp[1]);
......@@ -1241,7 +1244,7 @@ plpgsql_parse_dblwordrowtype(char *word)
/*
* Build and return the complete row definition
*/
plpgsql_yylval.row = build_rowtype(classOid);
plpgsql_yylval.row = plpgsql_build_rowtype(classOid);
pfree(cp);
......@@ -1252,7 +1255,7 @@ plpgsql_parse_dblwordrowtype(char *word)
* Build a rowtype data structure given the pg_class OID.
*/
PLpgSQL_row *
build_rowtype(Oid classOid)
plpgsql_build_rowtype(Oid classOid)
{
PLpgSQL_row *row;
HeapTuple classtup;
......@@ -1494,6 +1497,6 @@ plpgsql_add_initdatums(int **varnos)
void
plpgsql_yyerror(const char *s)
{
plpgsql_error_lineno = plpgsql_yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "%s at or near \"%s\"", s, plpgsql_yytext);
}
src/pl/plpgsql/src/plpgsql.h
View file @ 20aea2ec
......@@ -3,7 +3,7 @@
* procedural language
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/pl/plpgsql/src/plpgsql.h,v 1.35 2003/04/27 22:21:22 tgl Exp $
* $Header: /cvsroot/pgsql/src/pl/plpgsql/src/plpgsql.h,v 1.36 2003/05/05 16:46:28 tgl Exp $
*
* This software is copyrighted by Jan Wieck - Hamburg.
*
......@@ -573,13 +573,10 @@ extern PLpgSQL_datum **plpgsql_Datums;
extern int plpgsql_error_lineno;
extern char *plpgsql_error_funcname;
/* linkage to the real yytext and yylineno variables */
/* linkage to the real yytext variable */
extern char *plpgsql_base_yytext;
#define plpgsql_yytext plpgsql_base_yytext
extern int plpgsql_base_yylineno;
#define plpgsql_yylineno plpgsql_base_yylineno
extern PLpgSQL_function *plpgsql_curr_compile;
......@@ -601,7 +598,7 @@ extern int plpgsql_parse_tripwordtype(char *word);
extern int plpgsql_parse_wordrowtype(char *word);
extern int plpgsql_parse_dblwordrowtype(char *word);
extern PLpgSQL_type *plpgsql_parse_datatype(char *string);
extern PLpgSQL_row *build_rowtype(Oid classOid);
extern PLpgSQL_row *plpgsql_build_rowtype(Oid classOid);
extern void plpgsql_adddatum(PLpgSQL_datum * new);
extern int plpgsql_add_initdatums(int **varnos);
extern void plpgsql_yyerror(const char *s);
......@@ -660,6 +657,8 @@ extern int plpgsql_yyparse(void);
extern int plpgsql_base_yylex(void);
extern int plpgsql_yylex(void);
extern void plpgsql_push_back_token(int token);
extern void plpgsql_setinput(char *s, int functype);
extern int plpgsql_scanner_lineno(void);
extern void plpgsql_scanner_init(const char *str, int functype);
extern void plpgsql_scanner_finish(void);
#endif /* PLPGSQL_H */
src/pl/plpgsql/src/scan.l
View file @ 20aea2ec
......@@ -4,7 +4,7 @@
* procedural language
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/pl/plpgsql/src/Attic/scan.l,v 1.24 2002/11/07 06:06:17 tgl Exp $
* $Header: /cvsroot/pgsql/src/pl/plpgsql/src/Attic/scan.l,v 1.25 2003/05/05 16:46:28 tgl Exp $
*
* This software is copyrighted by Jan Wieck - Hamburg.
*
......@@ -39,20 +39,26 @@
#include "plpgsql.h"
static char *plpgsql_source;
static int plpgsql_bytes_left;
/* No reason to constrain amount of data slurped */
#define YY_READ_BUF_SIZE 16777216
/* Avoid exit() on fatal scanner errors (a bit ugly -- see yy_fatal_error) */
#define fprintf(file, fmt, msg) ereport(FATAL, (errmsg_internal("%s", msg)))
/* Handles to the buffer that the lexer uses internally */
static YY_BUFFER_STATE scanbufhandle;
static char *scanbuf;
static int scanner_functype;
static int scanner_typereported;
static int pushback_token;
static bool have_pushback_token;
static int lookahead_token;
static bool have_lookahead_token;
static const char *cur_line_start;
static int cur_line_num;
int plpgsql_SpaceScanned = 0;
static void plpgsql_input(char *buf, int *result, int max);
#define YY_INPUT(buf,res,max) plpgsql_input(buf, &res, max)
%}
%option 8bit
......@@ -60,7 +66,6 @@ static void plpgsql_input(char *buf, int *result, int max);
%option nounput
%option noyywrap
%option yylineno
%option case-insensitive
......@@ -78,11 +83,12 @@ space [ \t\n\r\f]
%%
/* ----------
* Local variable in scanner to remember where
* Local variables in scanner to remember where
* a string or comment started
* ----------
*/
int start_lineno = 0;
char *start_charpos = NULL;
/* ----------
* Reset the state when entering the scanner
......@@ -185,7 +191,7 @@ dump { return O_DUMP; }
{digit}+ { return T_NUMBER; }
\". {
plpgsql_error_lineno = yylineno;
plpgsql_error_lineno = plpgsql_scanner_lineno();
elog(ERROR, "unterminated quoted identifier");
}
......@@ -201,7 +207,7 @@ dump { return O_DUMP; }
*/
--[^\r\n]* ;
\/\* { start_lineno = yylineno;
\/\* { start_lineno = plpgsql_scanner_lineno();
BEGIN IN_COMMENT;
}
<IN_COMMENT>\*\/ { BEGIN INITIAL; plpgsql_SpaceScanned = 1; }
......@@ -214,22 +220,30 @@ dump { return O_DUMP; }
/* ----------
* Collect anything inside of ''s and return one STRING
*
* Hacking yytext/yyleng here lets us avoid using yymore(), which is
* a win for performance. It's safe because we know the underlying
* input buffer is not changing.
* ----------
*/
' { start_lineno = yylineno;
' {
start_lineno = plpgsql_scanner_lineno();
start_charpos = yytext;
BEGIN IN_STRING;
yymore();
}
<IN_STRING>\\. |
<IN_STRING>'' { yymore(); }
<IN_STRING>' { BEGIN INITIAL;
<IN_STRING>\\. { }
<IN_STRING>'' { }
<IN_STRING>' {
yyleng -= (yytext - start_charpos);
yytext = start_charpos;
BEGIN INITIAL;
return T_STRING;
}
<IN_STRING><<EOF>> {
plpgsql_error_lineno = start_lineno;
elog(ERROR, "unterminated string");
}
<IN_STRING>[^'\\]* { yymore(); }
<IN_STRING>[^'\\]* { }
/* ----------
* Any unmatched character is returned as is
......@@ -240,26 +254,6 @@ dump { return O_DUMP; }
%%
static void
plpgsql_input(char *buf, int *result, int max)
{
int n = max;
if (n > plpgsql_bytes_left)
n = plpgsql_bytes_left;
if (n == 0)
{
*result = YY_NULL;
return;
}
*result = n;
memcpy(buf, plpgsql_source, n);
plpgsql_source += n;
plpgsql_bytes_left -= n;
}
/*
* This is the yylex routine called from outside. It exists to provide
* a pushback facility, as well as to allow us to parse syntax that
......@@ -319,17 +313,35 @@ plpgsql_push_back_token(int token)
have_pushback_token = true;
}
/*
* Get the line number at which the current token ends. This substitutes
* for flex's very poorly implemented yylineno facility.
*
* We assume that flex has written a '\0' over the character following the
* current token in scanbuf. So, we just have to count the '\n' characters
* before that. We optimize this a little by keeping track of the last
* '\n' seen so far.
*/
int
plpgsql_scanner_lineno(void)
{
const char *c;
while ((c = strchr(cur_line_start, '\n')) != NULL)
{
cur_line_start = c + 1;
cur_line_num++;
}
return cur_line_num;
}
/*
* Initialize the scanner for new input.
* Called before any actual parsing is done
*/
void
plpgsql_setinput(char *source, int functype)
plpgsql_scanner_init(const char *str, int functype)
{
yyrestart(NULL);
yylineno = 1;
plpgsql_source = source;
Size slen;
/*----------
* Hack: skip any initial newline, so that in the common coding layout
......@@ -339,16 +351,47 @@ plpgsql_setinput(char *source, int functype)
* we will think "line 1" is what the programmer thinks of as line 1.
*----------
*/
if (*plpgsql_source == '\r')
plpgsql_source++;
if (*plpgsql_source == '\n')
plpgsql_source++;
if (*str == '\r')
str++;
if (*str == '\n')
str++;
slen = strlen(str);
plpgsql_bytes_left = strlen(plpgsql_source);
/*
* Might be left over after ereport()
*/
if (YY_CURRENT_BUFFER)
yy_delete_buffer(YY_CURRENT_BUFFER);
/*
* Make a scan buffer with special termination needed by flex.
*/
scanbuf = palloc(slen + 2);
memcpy(scanbuf, str, slen);
scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
/* Other setup */
scanner_functype = functype;
scanner_typereported = 0;
have_pushback_token = false;
have_lookahead_token = false;
cur_line_start = scanbuf;
cur_line_num = 1;
BEGIN(INITIAL);
}
/*
* Called after parsing is done to clean up after plpgsql_scanner_init()
*/
void
plpgsql_scanner_finish(void)
{
yy_delete_buffer(scanbufhandle);
pfree(scanbuf);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment