Skip to content

P
Postgres FD Implementation
Commit 32c6c99e authored by Peter Eisentraut's avatar Peter Eisentraut
Browse files
Options

Scanner performance improvements 

Use flex flags -CF.  Pass the to-be-scanned string around as StringInfo
type, to avoid querying the length repeatedly.  Clean up some code and
remove lex-compatibility cruft.  Escape backslash sequences inline.  Use
flex-provided yy_scan_buffer() function to set up input, rather than using
myinput().
parent ff428147
Hide whitespace changes
Inline Side-by-side
Showing with 144 additions and 155 deletions
src/backend/parser/Makefile
View file @ 32c6c99e
......@@ -2,7 +2,7 @@
#
# Makefile for parser
#
# $Header: /cvsroot/pgsql/src/backend/parser/Makefile,v 1.36 2002/03/08 07:12:11 tgl Exp $
# $Header: /cvsroot/pgsql/src/backend/parser/Makefile,v 1.37 2002/04/20 21:56:14 petere Exp $
#
#-------------------------------------------------------------------------
......@@ -14,6 +14,8 @@ OBJS= analyze.o gram.o keywords.o parser.o parse_agg.o parse_clause.o \
parse_expr.o parse_func.o parse_node.o parse_oper.o parse_relation.o \
parse_type.o parse_coerce.o parse_target.o scan.o scansup.o
FLEXFLAGS = -CF
all: SUBSYS.o
......@@ -42,7 +44,7 @@ endif
$(srcdir)/scan.c: scan.l
ifdef FLEX
$(FLEX) $(FLEXFLAGS) -Pbase_yy -o'$@' $<
$(FLEX) $(FLEXFLAGS) -o'$@' $<
else
@$(missing) flex $< $@
endif
......@@ -59,13 +61,3 @@ clean:
rm -f SUBSYS.o $(OBJS)
# And the garbage that might have been left behind by partial build:
@rm -f y.tab.c y.tab.h lex.yy.c
# This is unusual: We actually have to build some of the parts before
# we know what the header file dependencies are.
dep depend: gram.c scan.c
$(CC) -MM $(CFLAGS) *.c >depend
ifeq (depend,$(wildcard depend))
include depend
endif
src/backend/parser/parse_type.c
View file @ 32c6c99e
......@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/parser/parse_type.c,v 1.39 2002/03/30 01:02:41 tgl Exp $
* $Header: /cvsroot/pgsql/src/backend/parser/parse_type.c,v 1.40 2002/04/20 21:56:14 petere Exp $
*
*-------------------------------------------------------------------------
*/
......@@ -473,17 +473,17 @@ typeidTypeRelid(Oid type_id)
void
parseTypeString(const char *str, Oid *type_id, int32 *typmod)
{
char *buf;
StringInfoData buf;
List *raw_parsetree_list;
SelectStmt *stmt;
ResTarget *restarget;
A_Const *aconst;
TypeName *typename;
buf = (char *) palloc(strlen(str) + 16);
sprintf(buf, "SELECT (NULL::%s)", str);
initStringInfo(&buf);
appendStringInfo(&buf, "SELECT (NULL::%s)", str);
raw_parsetree_list = parser(buf, NULL, 0);
raw_parsetree_list = parser(&buf, NULL, 0);
/*
* Make sure we got back exactly what we expected and no more;
......@@ -528,5 +528,5 @@ parseTypeString(const char *str, Oid *type_id, int32 *typmod)
*type_id = typenameTypeId(typename);
*typmod = typename->typmod;
pfree(buf);
pfree(buf.data);
}
src/backend/parser/parser.c
View file @ 32c6c99e
......@@ -14,7 +14,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/parser/parser.c,v 1.51 2001/11/05 17:46:26 momjian Exp $
* $Header: /cvsroot/pgsql/src/backend/parser/parser.c,v 1.52 2002/04/20 21:56:14 petere Exp $
*
*-------------------------------------------------------------------------
*/
......@@ -28,12 +28,6 @@
#include "parser/parse_expr.h"
#if defined(FLEX_SCANNER)
extern void DeleteBuffer(void);
#endif /* FLEX_SCANNER */
char *parseString; /* the char* which holds the string to be
* parsed */
List *parsetree; /* result of parsing is left here */
static int lookahead_token; /* one-token lookahead */
......@@ -48,24 +42,20 @@ static bool have_lookahead; /* lookahead_token set? */
* Returns a list of raw (un-analyzed) parse trees.
*/
List *
parser(char *str, Oid *typev, int nargs)
parser(StringInfo str, Oid *typev, int nargs)
{
int yyresult;
parseString = str;
parsetree = NIL; /* in case parser forgets to set it */
have_lookahead = false;
scanner_init();
scanner_init(str);
parser_init(typev, nargs);
parse_expr_init();
yyresult = yyparse();
#if defined(FLEX_SCANNER)
DeleteBuffer();
#endif /* FLEX_SCANNER */
scanner_finish();
clearerr(stdin);
if (yyresult) /* error */
......
src/backend/parser/scan.l
View file @ 32c6c99e
......@@ -9,7 +9,7 @@
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/parser/scan.l,v 1.91 2002/03/06 06:09:56 momjian Exp $
* $Header: /cvsroot/pgsql/src/backend/parser/scan.l,v 1.92 2002/04/20 21:56:14 petere Exp $
*
*-------------------------------------------------------------------------
*/
......@@ -17,9 +17,6 @@
#include <ctype.h>
#include <unistd.h>
#ifndef __linux__
#include <math.h>
#endif
#include <errno.h>
#include "miscadmin.h"
......@@ -28,45 +25,18 @@
#include "parser/gramparse.h"
#include "parser/keywords.h"
#include "parser/parse.h"
#include "parser/scansup.h"
#include "utils/builtins.h"
#ifdef MULTIBYTE
#include "mb/pg_wchar.h"
#endif
extern char *parseString;
static char *parseCh;
/* some versions of lex define this as a macro */
#if defined(yywrap)
#undef yywrap
#endif /* yywrap */
/* set up my input handler --- need one flavor for flex, one for lex */
#if defined(FLEX_SCANNER)
#define YY_NEVER_INTERACTIVE 1
#define YY_NO_UNPUT
static int myinput(char* buf, int max);
#undef YY_INPUT
#define YY_INPUT(buf,result,max) {result = myinput(buf,max);}
/* No reason to constrain amount of data slurped per myinput() call. */
/* No reason to constrain amount of data slurped */
#define YY_READ_BUF_SIZE 16777216
/* Avoid exit() on fatal scanner errors (a bit ugly -- see yy_fatal_error) */
#define fprintf(file, fmt, msg) elog(FATAL, "%s", (msg))
#else /* !FLEX_SCANNER */
#undef input
int input();
#undef unput
void unput(char);
#endif /* FLEX_SCANNER */
extern YYSTYPE yylval;
static int xcdepth = 0; /* depth of nesting in slash-star comments */
......@@ -83,8 +53,23 @@ static int literalalloc; /* current allocated buffer size */
#define startlit() (literalbuf[0] = '\0', literallen = 0)
static void addlit(char *ytext, int yleng);
static void addlitchar(unsigned char ychar);
static char *litbufdup(void);
/* Handles to the buffer that the lexer uses internally */
static YY_BUFFER_STATE scanbufhandle;
static char *scanbuf;
unsigned char unescape_single_char(unsigned char c);
%}
%option 8bit
%option never-interactive
%option nounput
%option noyywrap
%option prefix="base_yy"
/*
* OK, here is a short description of lex/flex rules behavior.
* The longest pattern which matches an input string is always chosen.
......@@ -126,18 +111,17 @@ xhcat {quote}{whitespace_with_newline}{quote}
/* Extended quote
* xqdouble implements SQL92 embedded quote
* xqcat allows strings to cross input lines
* Note: reduction of '' and \ sequences to output text is done in scanstr(),
* not by rules here. But we do get rid of xqcat sequences here.
*/
quote '
xqstart {quote}
xqstop {quote}
xqdouble {quote}{quote}
xqinside [^\\']+
xqliteral [\\](.|\n)
xqescape [\\][^0-7]
xqoctesc [\\][0-7]{1,3}
xqcat {quote}{whitespace_with_newline}{quote}
/* Delimited quote
/* Double quote
* Allows embedded spaces and other special characters into identifiers.
*/
dquote \"
......@@ -238,10 +222,7 @@ whitespace_with_newline ({horiz_whitespace}*{newline}{whitespace}*)
other .
/* DO NOT PUT ANY COMMENTS IN THE FOLLOWING SECTION.
* AT&T lex does not properly handle C-style comments in this second lex block.
* So, put comments here. thomas - 1997-09-08
*
/*
* Quoted strings must allow some special characters such as single-quote
* and newline.
* Embedded single-quotes are implemented both in the SQL92-standard
......@@ -285,14 +266,14 @@ other .
{xbitstart} {
BEGIN(xbit);
startlit();
addlit("b", 1);
addlitchar('b');
}
<xbit>{xbitstop} {
BEGIN(INITIAL);
if (literalbuf[strspn(literalbuf + 1, "01") + 1] != '\0')
elog(ERROR, "invalid bit string input: '%s'",
literalbuf);
yylval.str = pstrdup(literalbuf);
yylval.str = litbufdup();
return BITCONST;
}
<xh>{xhinside} |
......@@ -335,14 +316,22 @@ other .
}
<xq>{xqstop} {
BEGIN(INITIAL);
yylval.str = scanstr(literalbuf);
yylval.str = litbufdup();
return SCONST;
}
<xq>{xqdouble} |
<xq>{xqinside} |
<xq>{xqliteral} {
<xq>{xqdouble} {
addlitchar('\'');
}
<xq>{xqinside} {
addlit(yytext, yyleng);
}
<xq>{xqescape} {
addlitchar(unescape_single_char(yytext[1]));
}
<xq>{xqoctesc} {
unsigned char c = strtoul(yytext+1, NULL, 8);
addlitchar(c);
}
<xq>{xqcat} {
/* ignore */
}
......@@ -371,11 +360,11 @@ other .
literalbuf[NAMEDATALEN-1] = '\0';
#endif
}
yylval.str = pstrdup(literalbuf);
yylval.str = litbufdup();
return IDENT;
}
<xd>{xddouble} {
addlit(yytext, yyleng-1);
addlitchar('"');
}
<xd>{xdinside} {
addlit(yytext, yyleng);
......@@ -540,36 +529,44 @@ yyerror(const char *message)
elog(ERROR, "parser: %s at or near \"%s\"", message, yytext);
}
int
yywrap(void)
{
return(1);
}
/*
scanner_init:
called by postgres before any actual parsing is done
*/
* Called before any actual parsing is done
*/
void
scanner_init(void)
scanner_init(StringInfo str)
{
/* it's important to set this to NULL
because input()/myinput() checks the non-nullness of parseCh
to know when to pass the string to lex/flex */
parseCh = NULL;
/*
* Might be left over after elog()
*/
if (YY_CURRENT_BUFFER)
yy_delete_buffer(YY_CURRENT_BUFFER);
scanbuf = palloc(str->len + 2);
memcpy(scanbuf, str->data, str->len);
scanbuf[str->len] = scanbuf[str->len + 1] = YY_END_OF_BUFFER_CHAR;
scanbufhandle = yy_scan_buffer(scanbuf, str->len + 2);
/* initialize literal buffer to a reasonable but expansible size */
literalalloc = 128;
literalbuf = (char *) palloc(literalalloc);
startlit();
#if defined(FLEX_SCANNER)
if (YY_CURRENT_BUFFER)
yy_flush_buffer(YY_CURRENT_BUFFER);
#endif /* FLEX_SCANNER */
BEGIN INITIAL;
BEGIN(INITIAL);
}
/*
* Called after parsing is done to clean up after scanner_init()
*/
void
scanner_finish(void)
{
yy_delete_buffer(scanbufhandle);
pfree(scanbuf);
}
static void
addlit(char *ytext, int yleng)
{
......@@ -587,54 +584,55 @@ addlit(char *ytext, int yleng)
literalbuf[literallen] = '\0';
}
#if !defined(FLEX_SCANNER)
/* get lex input from a string instead of from stdin */
int
input()
static void
addlitchar(unsigned char ychar)
{
if (parseCh == NULL)
parseCh = parseString;
if (*parseCh == '\0')
return(0);
else
return(*parseCh++);
/* enlarge buffer if needed */
if ((literallen+1) >= literalalloc)
{
literalalloc *= 2;
literalbuf = (char *) repalloc(literalbuf, literalalloc);
}
/* append new data, add trailing null */
literalbuf[literallen] = ychar;
literallen += 1;
literalbuf[literallen] = '\0';
}
/* undo lex input from a string instead of from stdin */
void
unput(char c)
/*
* One might be tempted to write pstrdup(literalbuf) instead of this,
* but for long literals this is much faster because the length is
* already known.
*/
static char *
litbufdup(void)
{
if (parseCh == NULL)
elog(FATAL, "Unput() failed.\n");
else if (c != 0)
*--parseCh = c;
}
char *new;
#endif /* !defined(FLEX_SCANNER) */
new = palloc(literallen + 1);
memcpy(new, literalbuf, literallen+1);
return new;
}
#ifdef FLEX_SCANNER
/* input routine for flex to read input from a string instead of a file */
static int
myinput(char* buf, int max)
unsigned char
unescape_single_char(unsigned char c)
{
int len;
if (parseCh == NULL)
parseCh = parseString;
len = strlen(parseCh); /* remaining data available */
/* Note: this code used to think that flex wants a null-terminated
* string. It does NOT, and returning 1 less character than it asks
* for will cause failure under the right boundary conditions. So
* shut up and fill the buffer to the limit, you hear?
*/
if (len > max)
len = max;
if (len > 0)
memcpy(buf, parseCh, len);
parseCh += len;
return len;
switch (c)
{
case 'b':
return '\b';
case 'f':
return '\f';
case 'n':
return '\n';
case 'r':
return '\r';
case 't':
return '\t';
default:
return c;
}
}
#endif /* FLEX_SCANNER */
src/backend/tcop/postgres.c
View file @ 32c6c99e
......@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/tcop/postgres.c,v 1.261 2002/04/18 20:01:09 tgl Exp $
* $Header: /cvsroot/pgsql/src/backend/tcop/postgres.c,v 1.262 2002/04/20 21:56:15 petere Exp $
*
* NOTES
* this is the "main" module of the postgres backend and
......@@ -120,7 +120,7 @@ int XfuncMode = 0;
static int InteractiveBackend(StringInfo inBuf);
static int SocketBackend(StringInfo inBuf);
static int ReadCommand(StringInfo inBuf);
static List *pg_parse_query(char *query_string, Oid *typev, int nargs);
static List *pg_parse_query(StringInfo query_string, Oid *typev, int nargs);
static List *pg_analyze_and_rewrite(Node *parsetree);
static void start_xact_command(void);
static void finish_xact_command(void);
......@@ -330,11 +330,15 @@ pg_parse_and_rewrite(char *query_string, /* string to execute */
List *raw_parsetree_list;
List *querytree_list;
List *list_item;
StringInfoData stri;
initStringInfo(&stri);
appendStringInfo(&stri, "%s", query_string);
/*
* (1) parse the request string into a list of raw parse trees.
*/
raw_parsetree_list = pg_parse_query(query_string, typev, nargs);
raw_parsetree_list = pg_parse_query(&stri, typev, nargs);
/*
* (2) Do parse analysis and rule rewrite.
......@@ -365,12 +369,12 @@ pg_parse_and_rewrite(char *query_string, /* string to execute */
* commands are not processed any further than the raw parse stage.
*/
static List *
pg_parse_query(char *query_string, Oid *typev, int nargs)
pg_parse_query(StringInfo query_string, Oid *typev, int nargs)
{
List *raw_parsetree_list;
if (Debug_print_query)
elog(LOG, "query: %s", query_string);
elog(LOG, "query: %s", query_string->data);
if (Show_parser_stats)
ResetUsage();
......@@ -549,7 +553,7 @@ pg_plan_query(Query *querytree)
*/
void
pg_exec_query_string(char *query_string, /* string to execute */
pg_exec_query_string(StringInfo query_string, /* string to execute */
CommandDest dest, /* where results should go */
MemoryContext parse_context) /* context for
* parsetrees */
......@@ -559,7 +563,7 @@ pg_exec_query_string(char *query_string, /* string to execute */
List *parsetree_list,
*parsetree_item;
debug_query_string = query_string; /* used by pgmonitor */
debug_query_string = query_string->data; /* used by pgmonitor */
/*
* Start up a transaction command. All queries generated by the
......@@ -725,7 +729,7 @@ pg_exec_query_string(char *query_string, /* string to execute */
* process utility functions (create, destroy, etc..)
*/
if (Debug_print_query)
elog(LOG, "ProcessUtility: %s", query_string);
elog(LOG, "ProcessUtility: %s", query_string->data);
else elog(DEBUG2, "ProcessUtility");
if (querytree->originalQuery)
......@@ -1688,7 +1692,7 @@ PostgresMain(int argc, char *argv[], const char *username)
if (!IsUnderPostmaster)
{
puts("\nPOSTGRES backend interactive interface ");
puts("$Revision: 1.261 $ $Date: 2002/04/18 20:01:09 $\n");
puts("$Revision: 1.262 $ $Date: 2002/04/20 21:56:15 $\n");
}
/*
......@@ -1913,7 +1917,7 @@ PostgresMain(int argc, char *argv[], const char *username)
pgstat_report_activity(parser_input->data);
pg_exec_query_string(parser_input->data,
pg_exec_query_string(parser_input,
whereToSendOutput,
QueryContext);
......
src/include/parser/gramparse.h
View file @ 32c6c99e
......@@ -7,7 +7,7 @@
* Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* $Id: gramparse.h,v 1.20 2002/04/09 20:35:55 tgl Exp $
* $Id: gramparse.h,v 1.21 2002/04/20 21:56:15 petere Exp $
*
*-------------------------------------------------------------------------
*/
......@@ -15,11 +15,14 @@
#ifndef GRAMPARSE_H
#define GRAMPARSE_H
#include "lib/stringinfo.h"
/* from parser.c */
extern int yylex(void);
/* from scan.l */
extern void scanner_init(void);
extern void scanner_init(StringInfo str);
extern void scanner_finish(void);
extern int base_yylex(void);
extern void yyerror(const char *message);
......
src/include/parser/parser.h
View file @ 32c6c99e
......@@ -7,15 +7,16 @@
* Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* $Id: parser.h,v 1.11 2001/11/05 17:46:35 momjian Exp $
* $Id: parser.h,v 1.12 2002/04/20 21:56:15 petere Exp $
*
*-------------------------------------------------------------------------
*/
#ifndef PARSER_H
#define PARSER_H
#include "lib/stringinfo.h"
#include "parser/parse_node.h"
extern List *parser(char *str, Oid *typev, int nargs);
extern List *parser(StringInfo str, Oid *typev, int nargs);
#endif /* PARSER_H */
src/include/tcop/tcopprot.h
View file @ 32c6c99e
......@@ -7,7 +7,7 @@
* Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* $Id: tcopprot.h,v 1.47 2001/11/10 23:51:14 tgl Exp $
* $Id: tcopprot.h,v 1.48 2002/04/20 21:56:15 petere Exp $
*
* OLD COMMENTS
* This file was created so that other c files could get the two
......@@ -22,6 +22,7 @@
#include <setjmp.h>
#include "executor/execdesc.h"
#include "lib/stringinfo.h"
#include "tcop/dest.h"
......@@ -37,7 +38,7 @@ extern bool ShowPortNumber;
extern List *pg_parse_and_rewrite(char *query_string,
Oid *typev, int nargs);
extern Plan *pg_plan_query(Query *querytree);
extern void pg_exec_query_string(char *query_string,
extern void pg_exec_query_string(StringInfo query_string,
CommandDest dest,
MemoryContext parse_context);
#endif /* BOOTSTRAP_INCLUDE */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment