Commit b66adb7b authored by Alvaro Herrera's avatar Alvaro Herrera

Revert "Permit dump/reload of not-too-large >1GB tuples"

This reverts commits fa2fa995 and 42f50cb8.

While the functionality that was intended to be provided by these
commits is desired, the patch didn't actually solve as many of the
problematic situations as we hoped, and it created a bunch of its own
problems.  Since we're going to require more extensive changes soon for
other reasons and users have been working around these problems for a
long time already, there is no point in spending effort in fixing this
halfway measure.

Per complaint from Tom Lane.
Discussion: https://postgr.es/m/21407.1484606922@sss.pgh.pa.us

(Commit fa2fa995 had already been reverted in branches 9.5 as
f858524ee4f and 9.6 as e9e44a0953, so this touches master only.
Commit 42f50cb8 was not present in the older branches.)
parent b83f4e4a
......@@ -741,9 +741,7 @@ heap_form_tuple(TupleDesc tupleDescriptor,
* Allocate and zero the space needed. Note that the tuple body and
* HeapTupleData management structure are allocated in one chunk.
*/
tuple = MemoryContextAllocExtended(CurrentMemoryContext,
HEAPTUPLESIZE + len,
MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO);
tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len);
tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
/*
......
......@@ -394,7 +394,7 @@ ReceiveCopyBegin(CopyState cstate)
pq_sendint(&buf, format, 2); /* per-column formats */
pq_endmessage(&buf);
cstate->copy_dest = COPY_NEW_FE;
cstate->fe_msgbuf = makeLongStringInfo();
cstate->fe_msgbuf = makeStringInfo();
}
else
{
......@@ -1954,7 +1954,7 @@ CopyTo(CopyState cstate)
cstate->null_print_client = cstate->null_print; /* default */
/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
cstate->fe_msgbuf = makeLongStringInfo();
cstate->fe_msgbuf = makeStringInfo();
/* Get info about the columns we need to process. */
cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
......@@ -2909,8 +2909,8 @@ BeginCopyFrom(ParseState *pstate,
cstate->cur_attval = NULL;
/* Set up variables to avoid per-attribute overhead. */
initLongStringInfo(&cstate->attribute_buf);
initLongStringInfo(&cstate->line_buf);
initStringInfo(&cstate->attribute_buf);
initStringInfo(&cstate->line_buf);
cstate->line_buf_converted = false;
cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
cstate->raw_buf_index = cstate->raw_buf_len = 0;
......
......@@ -4,8 +4,7 @@
*
* StringInfo provides an indefinitely-extensible string data type.
* It can be used to buffer either ordinary C strings (null-terminated text)
* or arbitrary binary data. All storage is allocated with palloc() and
* friends.
* or arbitrary binary data. All storage is allocated with palloc().
*
* Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
......@@ -37,29 +36,11 @@ makeStringInfo(void)
return res;
}
/*
* makeLongStringInfo
*
* Same as makeStringInfo, for larger strings.
*/
StringInfo
makeLongStringInfo(void)
{
StringInfo res;
res = (StringInfo) palloc(sizeof(StringInfoData));
initLongStringInfo(res);
return res;
}
/*
* initStringInfo
*
* Initialize a StringInfoData struct (with previously undefined contents)
* to describe an empty string; don't enable long strings yet.
* to describe an empty string.
*/
void
initStringInfo(StringInfo str)
......@@ -68,22 +49,9 @@ initStringInfo(StringInfo str)
str->data = (char *) palloc(size);
str->maxlen = size;
str->long_ok = false;
resetStringInfo(str);
}
/*
* initLongStringInfo
*
* Same as initStringInfo, plus enable long strings.
*/
void
initLongStringInfo(StringInfo str)
{
initStringInfo(str);
str->long_ok = true;
}
/*
* resetStringInfo
*
......@@ -174,7 +142,7 @@ appendStringInfoVA(StringInfo str, const char *fmt, va_list args)
/*
* Return pvsnprintf's estimate of the space needed. (Although this is
* given as a size_t, we know it will fit in int because it's not more
* than either MaxAllocSize or half an int's width.)
* than MaxAllocSize.)
*/
return (int) nprinted;
}
......@@ -276,17 +244,7 @@ appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
void
enlargeStringInfo(StringInfo str, int needed)
{
Size newlen;
Size limit;
/*
* Determine the upper size limit. Because of overflow concerns outside
* of this module, we limit ourselves to 4-byte signed integer range,
* even for "long_ok" strings.
*/
limit = str->long_ok ?
(((Size) 1) << (sizeof(int32) * 8 - 1)) - 1 :
MaxAllocSize;
int newlen;
/*
* Guard against out-of-range "needed" values. Without this, we can get
......@@ -294,7 +252,7 @@ enlargeStringInfo(StringInfo str, int needed)
*/
if (needed < 0) /* should not happen */
elog(ERROR, "invalid string enlargement request size: %d", needed);
if (((Size) needed) >= (limit - (Size) str->len))
if (((Size) needed) >= (MaxAllocSize - (Size) str->len))
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("out of memory"),
......@@ -303,7 +261,7 @@ enlargeStringInfo(StringInfo str, int needed)
needed += str->len + 1; /* total space required now */
/* Because of the above test, we now have needed <= limit */
/* Because of the above test, we now have needed <= MaxAllocSize */
if (needed <= str->maxlen)
return; /* got enough space already */
......@@ -313,20 +271,19 @@ enlargeStringInfo(StringInfo str, int needed)
* for efficiency, double the buffer size each time it overflows.
* Actually, we might need to more than double it if 'needed' is big...
*/
newlen = 2 * (Size) str->maxlen;
while ((Size) needed > newlen)
newlen = 2 * str->maxlen;
while (needed > newlen)
newlen = 2 * newlen;
/*
* Clamp to the limit in case we went past it. (We used to depend on
* limit <= INT32_MAX/2, to avoid overflow in the loop above; we no longer
* depend on that, but if "needed" and str->maxlen ever become wider, we
* will need similar caution here.) We will still have newlen >= needed.
* Clamp to MaxAllocSize in case we went past it. Note we are assuming
* here that MaxAllocSize <= INT_MAX/2, else the above loop could
* overflow. We will still have newlen >= needed.
*/
if (newlen > limit)
newlen = limit;
if (newlen > (int) MaxAllocSize)
newlen = (int) MaxAllocSize;
str->data = (char *) repalloc_huge(str->data, newlen);
str->data = (char *) repalloc(str->data, newlen);
str->maxlen = newlen;
}
......@@ -30,8 +30,6 @@
* cursor is initialized to zero by makeStringInfo or initStringInfo,
* but is not otherwise touched by the stringinfo.c routines.
* Some routines use it to scan through a StringInfo.
* long_ok whether this StringInfo can allocate more than MaxAllocSize
* bytes (but still up to 2GB).
*-------------------------
*/
typedef struct StringInfoData
......@@ -40,7 +38,6 @@ typedef struct StringInfoData
int len;
int maxlen;
int cursor;
bool long_ok;
} StringInfoData;
typedef StringInfoData *StringInfo;
......@@ -49,11 +46,11 @@ typedef StringInfoData *StringInfo;
/*------------------------
* There are two ways to create a StringInfo object initially:
*
* StringInfo stringptr = makeStringInfo(); // or makeLongStringInfo();
* StringInfo stringptr = makeStringInfo();
* Both the StringInfoData and the data buffer are palloc'd.
*
* StringInfoData string;
* initStringInfo(&string); // or initLongStringInfo();
* initStringInfo(&string);
* The data buffer is palloc'd but the StringInfoData is just local.
* This is the easiest approach for a StringInfo object that will
* only live as long as the current routine.
......@@ -70,26 +67,21 @@ typedef StringInfoData *StringInfo;
/*------------------------
* makeStringInfo
* makeLongStringInfo
* Create an empty 'StringInfoData' & return a pointer to it. The former
* allows up to 1 GB in size, per palloc(); the latter allows up to 2 GB.
* Create an empty 'StringInfoData' & return a pointer to it.
*/
extern StringInfo makeStringInfo(void);
extern StringInfo makeLongStringInfo(void);
/*------------------------
* initStringInfo
* initLongStringInfo
* Initialize a StringInfoData struct (with previously undefined contents)
* to describe an empty string. Size limits as above.
* to describe an empty string.
*/
extern void initStringInfo(StringInfo str);
extern void initLongStringInfo(StringInfo str);
/*------------------------
* resetStringInfo
* Clears the current content of the StringInfo, if any. The
* StringInfo remains valid. The long_ok flag is not reset.
* StringInfo remains valid.
*/
extern void resetStringInfo(StringInfo str);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment