From 27a68b43636233952bff8aefce08112194b39d40 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Sun, 26 Oct 2025 21:14:44 +0100 Subject: [PATCH v4 1/3] Allow parallel GIN builds to allocate large chunks The parallel GIN builds used palloc/repalloc to maintain TID lists, which with high maintance_work_mem values can lead to failures like ERROR: invalid memory alloc request size 1113001620 The reason is that while merging intermediate worker data, we call GinBufferStoreTuple() which coalesces the TID lists, and the result may not fit into MaxAllocSize. Fixed by allowing huge allocations when merging TID lists, including an existing palloc call in ginMergeItemPointers(). Report by Greg Smith, investigation and fix by me. Batchpatched to 18, where parallel GIN builds were introduced. Reported-by: Gregory Smith Discussion: https://postgr.es/m/CAHLJuCWDwn-PE2BMZE4Kux7x5wWt_6RoWtA0mUQffEDLeZ6sfA@mail.gmail.com Backpatch-through: 18 --- src/backend/access/gin/gininsert.c | 7 ++++--- src/backend/access/gin/ginpostinglist.c | 10 ++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c index 3d71b442aa9..2355b96b351 100644 --- a/src/backend/access/gin/gininsert.c +++ b/src/backend/access/gin/gininsert.c @@ -1496,10 +1496,11 @@ GinBufferStoreTuple(GinBuffer *buffer, GinTuple *tup) * still pass 0 as number of elements in that array though. */ if (buffer->items == NULL) - buffer->items = palloc((buffer->nitems + tup->nitems) * sizeof(ItemPointerData)); + buffer->items = palloc_extended((buffer->nitems + tup->nitems) * sizeof(ItemPointerData), + MCXT_ALLOC_HUGE); else - buffer->items = repalloc(buffer->items, - (buffer->nitems + tup->nitems) * sizeof(ItemPointerData)); + buffer->items = repalloc_huge(buffer->items, + (buffer->nitems + tup->nitems) * sizeof(ItemPointerData)); new = ginMergeItemPointers(&buffer->items[buffer->nfrozen], /* first unfrozen */ (buffer->nitems - buffer->nfrozen), /* num of unfrozen */ diff --git a/src/backend/access/gin/ginpostinglist.c b/src/backend/access/gin/ginpostinglist.c index 48eadec87b0..a60ea46204a 100644 --- a/src/backend/access/gin/ginpostinglist.c +++ b/src/backend/access/gin/ginpostinglist.c @@ -308,7 +308,8 @@ ginPostingListDecodeAllSegments(GinPostingList *segment, int len, int *ndecoded_ * Guess an initial size of the array. */ nallocated = segment->nbytes * 2 + 1; - result = palloc(nallocated * sizeof(ItemPointerData)); + result = palloc_extended(nallocated * sizeof(ItemPointerData), + MCXT_ALLOC_HUGE); ndecoded = 0; while ((char *) segment < endseg) @@ -317,7 +318,7 @@ ginPostingListDecodeAllSegments(GinPostingList *segment, int len, int *ndecoded_ if (ndecoded >= nallocated) { nallocated *= 2; - result = repalloc(result, nallocated * sizeof(ItemPointerData)); + result = repalloc_huge(result, nallocated * sizeof(ItemPointerData)); } /* copy the first item */ @@ -335,7 +336,7 @@ ginPostingListDecodeAllSegments(GinPostingList *segment, int len, int *ndecoded_ if (ndecoded >= nallocated) { nallocated *= 2; - result = repalloc(result, nallocated * sizeof(ItemPointerData)); + result = repalloc_huge(result, nallocated * sizeof(ItemPointerData)); } val += decode_varbyte(&ptr); @@ -381,7 +382,8 @@ ginMergeItemPointers(ItemPointerData *a, uint32 na, { ItemPointerData *dst; - dst = (ItemPointer) palloc((na + nb) * sizeof(ItemPointerData)); + dst = (ItemPointer) palloc_extended((na + nb) * sizeof(ItemPointerData), + MCXT_ALLOC_HUGE); /* * If the argument arrays don't overlap, we can just append them to each -- 2.51.0