Commit 30bb26b5 authored by Teodor Sigaev's avatar Teodor Sigaev

Allow usage of huge maintenance_work_mem for GIN build.

Currently, in-memory posting list during GIN build process is limited 1GB
because of using repalloc. The patch replaces call of repalloc to repalloc_huge.
It increases limit of posting list from 180 millions
(1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields
in GinEntryAccumulator and subsequent calls. Check added.

Also, fix accounting of allocatedMemory during build to prevent integer
overflow with maintenance_work_mem > 4GB.

Robert Abraham <robert.abraham86@googlemail.com> with additions by me
parent 075ab425
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
#include "postgres.h" #include "postgres.h"
#include <limits.h>
#include "access/gin_private.h" #include "access/gin_private.h"
#include "utils/datum.h" #include "utils/datum.h"
#include "utils/memutils.h" #include "utils/memutils.h"
...@@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg) ...@@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg)
*/ */
if (eo->count >= eo->maxcount) if (eo->count >= eo->maxcount)
{ {
if (eo->maxcount > INT_MAX)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("posting list is too long"),
errhint("Reduce maintenance_work_mem")));
accum->allocatedMemory -= GetMemoryChunkSpace(eo->list); accum->allocatedMemory -= GetMemoryChunkSpace(eo->list);
eo->maxcount *= 2; eo->maxcount *= 2;
eo->list = (ItemPointerData *) eo->list = (ItemPointerData *)
repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount); repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount);
accum->allocatedMemory += GetMemoryChunkSpace(eo->list); accum->allocatedMemory += GetMemoryChunkSpace(eo->list);
} }
......
...@@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate, ...@@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,
*/ */
if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber || if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber ||
(GinPageHasFullRow(page) && (GinPageHasFullRow(page) &&
(accum.allocatedMemory >= maintenance_work_mem * 1024L))) (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))
{ {
ItemPointerData *list; ItemPointerData *list;
uint32 nlist; uint32 nlist;
......
...@@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values, ...@@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
&htup->t_self); &htup->t_self);
/* If we've maxed out our available memory, dump everything to the index */ /* If we've maxed out our available memory, dump everything to the index */
if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L) if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)
{ {
ItemPointerData *list; ItemPointerData *list;
Datum key; Datum key;
......
...@@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator ...@@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator
typedef struct typedef struct
{ {
GinState *ginstate; GinState *ginstate;
long allocatedMemory; Size allocatedMemory;
GinEntryAccumulator *entryallocator; GinEntryAccumulator *entryallocator;
uint32 eas_used; uint32 eas_used;
RBTree *tree; RBTree *tree;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment