Commit 30bb26b5 authored by Teodor Sigaev's avatar Teodor Sigaev

Allow usage of huge maintenance_work_mem for GIN build.

Currently, in-memory posting list during GIN build process is limited 1GB
because of using repalloc. The patch replaces call of repalloc to repalloc_huge.
It increases limit of posting list from 180 millions
(1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields
in GinEntryAccumulator and subsequent calls. Check added.

Also, fix accounting of allocatedMemory during build to prevent integer
overflow with maintenance_work_mem > 4GB.

Robert Abraham <robert.abraham86@googlemail.com> with additions by me
parent 075ab425
......@@ -14,6 +14,8 @@
#include "postgres.h"
#include <limits.h>
#include "access/gin_private.h"
#include "utils/datum.h"
#include "utils/memutils.h"
......@@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg)
*/
if (eo->count >= eo->maxcount)
{
if (eo->maxcount > INT_MAX)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("posting list is too long"),
errhint("Reduce maintenance_work_mem")));
accum->allocatedMemory -= GetMemoryChunkSpace(eo->list);
eo->maxcount *= 2;
eo->list = (ItemPointerData *)
repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount);
repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount);
accum->allocatedMemory += GetMemoryChunkSpace(eo->list);
}
......
......@@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,
*/
if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber ||
(GinPageHasFullRow(page) &&
(accum.allocatedMemory >= maintenance_work_mem * 1024L)))
(accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))
{
ItemPointerData *list;
uint32 nlist;
......
......@@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
&htup->t_self);
/* If we've maxed out our available memory, dump everything to the index */
if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)
{
ItemPointerData *list;
Datum key;
......
......@@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator
typedef struct
{
GinState *ginstate;
long allocatedMemory;
Size allocatedMemory;
GinEntryAccumulator *entryallocator;
uint32 eas_used;
RBTree *tree;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment