*** a/src/backend/access/heap/heapam.c --- b/src/backend/access/heap/heapam.c *************** *** 70,75 **** --- 70,76 ---- #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/tqual.h" + #include "utils/pg_lzcompress.h" /* GUC variable */ *************** *** 85,90 **** static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup, --- 86,92 ---- TransactionId xid, CommandId cid, int options); static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from, Buffer newbuf, HeapTuple newtup, + HeapTuple oldtup, bool all_visible_cleared, bool new_all_visible_cleared); static bool HeapSatisfiesHOTUpdate(Relation relation, Bitmapset *hot_attrs, HeapTuple oldtup, HeapTuple newtup); *************** *** 3195,3204 **** l2: /* XLOG stuff */ if (RelationNeedsWAL(relation)) { ! XLogRecPtr recptr = log_heap_update(relation, buffer, oldtup.t_self, ! newbuf, heaptup, ! all_visible_cleared, ! all_visible_cleared_new); if (newbuf != buffer) { --- 3197,3208 ---- /* XLOG stuff */ if (RelationNeedsWAL(relation)) { ! XLogRecPtr recptr; ! ! recptr = log_heap_update(relation, buffer, oldtup.t_self, ! newbuf, heaptup, &oldtup, ! all_visible_cleared, ! all_visible_cleared_new); if (newbuf != buffer) { *************** *** 4428,4434 **** log_heap_visible(RelFileNode rnode, BlockNumber block, Buffer vm_buffer, */ static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from, ! Buffer newbuf, HeapTuple newtup, bool all_visible_cleared, bool new_all_visible_cleared) { xl_heap_update xlrec; --- 4432,4438 ---- */ static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from, ! Buffer newbuf, HeapTuple newtup, HeapTuple oldtup, bool all_visible_cleared, bool new_all_visible_cleared) { xl_heap_update xlrec; *************** *** 4437,4442 **** log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from, --- 4441,4456 ---- XLogRecPtr recptr; XLogRecData rdata[4]; Page page = BufferGetPage(newbuf); + union + { + PGLZ_Header pglzheader; + char buf[BLCKSZ]; + } buf; + char *newtupdata; + int newtuplen; + char *oldtupdata; + int oldtuplen; + bool compressed = false; /* Caller should not call me on a non-WAL-logged relation */ Assert(RelationNeedsWAL(reln)); *************** *** 4446,4456 **** log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from, else info = XLOG_HEAP_UPDATE; xlrec.target.node = reln->rd_node; xlrec.target.tid = from; ! xlrec.all_visible_cleared = all_visible_cleared; xlrec.newtid = newtup->t_self; ! xlrec.new_all_visible_cleared = new_all_visible_cleared; rdata[0].data = (char *) &xlrec; rdata[0].len = SizeOfHeapUpdate; --- 4460,4506 ---- else info = XLOG_HEAP_UPDATE; + newtupdata = ((char *) newtup->t_data) + offsetof(HeapTupleHeaderData, t_bits); + newtuplen = newtup->t_len - offsetof(HeapTupleHeaderData, t_bits); + oldtupdata = ((char *) oldtup->t_data) + offsetof(HeapTupleHeaderData, t_bits); + oldtuplen = oldtup->t_len - offsetof(HeapTupleHeaderData, t_bits); + + /* Is the update is going to the same page? */ + if (oldbuf == newbuf) + { + /* + * enable this if you only want to compress the new tuple as is, + * without taking advantage of the old tuple. + */ + #ifdef COMPRESS_ONLY + oldtuplen = 0; + #endif + + if (PGLZ_MAX_OUTPUT(newtuplen) < sizeof(buf)) + { + /* Delta-encode the new tuple using the old tuple */ + if (pglz_compress_with_history(newtupdata, newtuplen, + oldtupdata, oldtuplen, + (PGLZ_Header *) (char *) &buf.pglzheader, + NULL)) + { + compressed = true; + newtupdata = (char *) &buf.pglzheader; + newtuplen = VARSIZE(&buf.pglzheader); + } + } + } + + xlrec.flags = 0; xlrec.target.node = reln->rd_node; xlrec.target.tid = from; ! if (all_visible_cleared) ! xlrec.flags |= XL_HEAP_UPDATE_ALL_VISIBLE_CLEARED; xlrec.newtid = newtup->t_self; ! if (new_all_visible_cleared) ! xlrec.flags |= XL_HEAP_UPDATE_NEW_ALL_VISIBLE_CLEARED; ! if (compressed) ! xlrec.flags |= XL_HEAP_UPDATE_DELTA_ENCODED; rdata[0].data = (char *) &xlrec; rdata[0].len = SizeOfHeapUpdate; *************** *** 4478,4485 **** log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from, rdata[2].next = &(rdata[3]); /* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */ ! rdata[3].data = (char *) newtup->t_data + offsetof(HeapTupleHeaderData, t_bits); ! rdata[3].len = newtup->t_len - offsetof(HeapTupleHeaderData, t_bits); rdata[3].buffer = newbuf; rdata[3].buffer_std = true; rdata[3].next = NULL; --- 4528,4535 ---- rdata[2].next = &(rdata[3]); /* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */ ! rdata[3].data = newtupdata; ! rdata[3].len = newtuplen; rdata[3].buffer = newbuf; rdata[3].buffer_std = true; rdata[3].next = NULL; *************** *** 5232,5237 **** heap_xlog_update(XLogRecPtr lsn, XLogRecord *record, bool hot_update) --- 5282,5289 ---- OffsetNumber offnum; ItemId lp = NULL; HeapTupleHeader htup; + HeapTupleHeader oldtup = NULL; + uint32 old_tup_len = 0; struct { HeapTupleHeaderData hdr; *************** *** 5246,5252 **** heap_xlog_update(XLogRecPtr lsn, XLogRecord *record, bool hot_update) * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ ! if (xlrec->all_visible_cleared) { Relation reln = CreateFakeRelcacheEntry(xlrec->target.node); BlockNumber block = ItemPointerGetBlockNumber(&xlrec->target.tid); --- 5298,5304 ---- * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ ! if (xlrec->flags & XL_HEAP_UPDATE_ALL_VISIBLE_CLEARED) { Relation reln = CreateFakeRelcacheEntry(xlrec->target.node); BlockNumber block = ItemPointerGetBlockNumber(&xlrec->target.tid); *************** *** 5289,5295 **** heap_xlog_update(XLogRecPtr lsn, XLogRecord *record, bool hot_update) if (PageGetMaxOffsetNumber(page) < offnum || !ItemIdIsNormal(lp)) elog(PANIC, "heap_update_redo: invalid lp"); ! htup = (HeapTupleHeader) PageGetItem(page, lp); htup->t_infomask &= ~(HEAP_XMAX_COMMITTED | HEAP_XMAX_INVALID | --- 5341,5348 ---- if (PageGetMaxOffsetNumber(page) < offnum || !ItemIdIsNormal(lp)) elog(PANIC, "heap_update_redo: invalid lp"); ! oldtup = htup = (HeapTupleHeader) PageGetItem(page, lp); ! old_tup_len = ItemIdGetLength(lp); htup->t_infomask &= ~(HEAP_XMAX_COMMITTED | HEAP_XMAX_INVALID | *************** *** 5308,5314 **** heap_xlog_update(XLogRecPtr lsn, XLogRecord *record, bool hot_update) /* Mark the page as a candidate for pruning */ PageSetPrunable(page, record->xl_xid); ! if (xlrec->all_visible_cleared) PageClearAllVisible(page); /* --- 5361,5367 ---- /* Mark the page as a candidate for pruning */ PageSetPrunable(page, record->xl_xid); ! if (xlrec->flags & XL_HEAP_UPDATE_ALL_VISIBLE_CLEARED) PageClearAllVisible(page); /* *************** *** 5330,5336 **** newt:; * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ ! if (xlrec->new_all_visible_cleared) { Relation reln = CreateFakeRelcacheEntry(xlrec->target.node); BlockNumber block = ItemPointerGetBlockNumber(&xlrec->newtid); --- 5383,5389 ---- * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ ! if (xlrec->flags & XL_HEAP_UPDATE_NEW_ALL_VISIBLE_CLEARED) { Relation reln = CreateFakeRelcacheEntry(xlrec->target.node); BlockNumber block = ItemPointerGetBlockNumber(&xlrec->newtid); *************** *** 5380,5395 **** newsame:; hsize = SizeOfHeapUpdate + SizeOfHeapHeader; newlen = record->xl_len - hsize; Assert(newlen <= MaxHeapTupleSize); memcpy((char *) &xlhdr, (char *) xlrec + SizeOfHeapUpdate, SizeOfHeapHeader); htup = &tbuf.hdr; MemSet((char *) htup, 0, sizeof(HeapTupleHeaderData)); ! /* PG73FORMAT: get bitmap [+ padding] [+ oid] + data */ ! memcpy((char *) htup + offsetof(HeapTupleHeaderData, t_bits), ! (char *) xlrec + hsize, ! newlen); newlen += offsetof(HeapTupleHeaderData, t_bits); htup->t_infomask2 = xlhdr.t_infomask2; htup->t_infomask = xlhdr.t_infomask; --- 5433,5467 ---- hsize = SizeOfHeapUpdate + SizeOfHeapHeader; newlen = record->xl_len - hsize; + Assert(newlen <= MaxHeapTupleSize); memcpy((char *) &xlhdr, (char *) xlrec + SizeOfHeapUpdate, SizeOfHeapHeader); htup = &tbuf.hdr; MemSet((char *) htup, 0, sizeof(HeapTupleHeaderData)); ! ! /* ! * If the new tuple was delta-encoded, decode it. ! */ ! if (xlrec->flags & XL_HEAP_UPDATE_DELTA_ENCODED) ! { ! char *encoded_data = (((char *) xlrec) + hsize); ! ! /* XXX: also add some sanity checks with PGLZ_RAW_SIZE here.*/ ! pglz_decompress_with_history(encoded_data, ! ((char *) htup) + offsetof(HeapTupleHeaderData, t_bits), ! &newlen, ! ((char *) oldtup) + offsetof(HeapTupleHeaderData, t_bits), ! old_tup_len - offsetof(HeapTupleHeaderData, t_bits)); ! } ! else ! { ! /* PG73FORMAT: get bitmap [+ padding] [+ oid] + data */ ! memcpy((char *) htup + offsetof(HeapTupleHeaderData, t_bits), ! (char *) xlrec + hsize, ! newlen); ! } newlen += offsetof(HeapTupleHeaderData, t_bits); htup->t_infomask2 = xlhdr.t_infomask2; htup->t_infomask = xlhdr.t_infomask; *************** *** 5404,5410 **** newsame:; if (offnum == InvalidOffsetNumber) elog(PANIC, "heap_update_redo: failed to add tuple"); ! if (xlrec->new_all_visible_cleared) PageClearAllVisible(page); freespace = PageGetHeapFreeSpace(page); /* needed to update FSM below */ --- 5476,5482 ---- if (offnum == InvalidOffsetNumber) elog(PANIC, "heap_update_redo: failed to add tuple"); ! if (xlrec->flags & XL_HEAP_UPDATE_NEW_ALL_VISIBLE_CLEARED) PageClearAllVisible(page); freespace = PageGetHeapFreeSpace(page); /* needed to update FSM below */ *** a/src/backend/utils/adt/pg_lzcompress.c --- b/src/backend/utils/adt/pg_lzcompress.c *************** *** 373,379 **** do { \ */ static inline int pglz_find_match(PGLZ_HistEntry **hstart, const char *input, const char *end, ! int *lenp, int *offp, int good_match, int good_drop) { PGLZ_HistEntry *hent; int32 len = 0; --- 373,380 ---- */ static inline int pglz_find_match(PGLZ_HistEntry **hstart, const char *input, const char *end, ! const char *hend, int *lenp, int *offp, int good_match, ! int good_drop) { PGLZ_HistEntry *hent; int32 len = 0; *************** *** 391,399 **** pglz_find_match(PGLZ_HistEntry **hstart, const char *input, const char *end, int32 thislen; /* * Stop if the offset does not fit into our tag anymore. */ - thisoff = ip - hp; if (thisoff >= 0x0fff) break; --- 392,408 ---- int32 thislen; /* + * Check If the history presents, calculate the offset from history + * end instead of input + */ + if (NULL == hend) + thisoff = ip - hp; + else + thisoff = hend - hp; + + /* * Stop if the offset does not fit into our tag anymore. */ if (thisoff >= 0x0fff) break; *************** *** 482,487 **** bool --- 491,510 ---- pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, const PGLZ_Strategy *strategy) { + return pglz_compress_with_history(source, slen, NULL, 0, dest, strategy); + } + + /* + * Like pglz_compress, but uses another piece of data to initialize the + * history table. When decompressing, you must pass the same history data + * to pglz_decompress_with_history(). This makes it possible to do simple + * delta compression. + */ + bool + pglz_compress_with_history(const char *source, int32 slen, + const char *history, int32 hlen, + PGLZ_Header *dest, const PGLZ_Strategy *strategy) + { unsigned char *bp = ((unsigned char *) dest) + sizeof(PGLZ_Header); unsigned char *bstart = bp; int hist_next = 0; *************** *** 500,505 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, --- 523,530 ---- int32 result_size; int32 result_max; int32 need_rate; + const char *hp = NULL; + const char *hend = NULL; /* * Our fallback strategy is the default. *************** *** 560,565 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, --- 585,608 ---- * hist_entries[] array; its entries are initialized as they are used. */ memset(hist_start, 0, sizeof(hist_start)); + if (hlen > 0) + { + hp = history; + hend = history + hlen; + while (hp < hend) + { + /* + * XXX: I think this doesn't handle the last few bytes of the + * history correctly, or at least not in the most efficient way. + * Logically, we should behave like the history and the source + * strings are concatenated, but we use 'hend' here. + */ + pglz_hist_add(hist_start, hist_entries, + hist_next, hist_recycle, + hp, hend); + hp++; /* Do not do this ++ in the line above! */ + } + } /* * Compress the source directly into the output buffer. *************** *** 588,594 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, /* * Try to find a match in the history */ ! if (pglz_find_match(hist_start, dp, dend, &match_len, &match_off, good_match, good_drop)) { /* --- 631,637 ---- /* * Try to find a match in the history */ ! if (pglz_find_match(hist_start, dp, dend, hend, &match_len, &match_off, good_match, good_drop)) { /* *************** *** 596,608 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, * characters. */ pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); ! while (match_len--) { ! pglz_hist_add(hist_start, hist_entries, ! hist_next, hist_recycle, ! dp, dend); ! dp++; /* Do not do this ++ in the line above! */ ! /* The macro would do it four times - Jan. */ } found_match = true; } --- 639,668 ---- * characters. */ pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); ! ! /* ! * Incase of histor is passed as a separate buffer then don't add ! * source data further to the history. This is required as we need ! * to calculate the offset in the history buffer. ! */ ! if (NULL == hend) { ! while (match_len--) ! { ! pglz_hist_add(hist_start, hist_entries, ! hist_next, hist_recycle, ! dp, dend); ! dp++; /* Do not do this ++ in the line above! */ ! /* The macro would do it four times - Jan. */ ! } ! } ! else ! { ! /* ! * Increment the source pointer with the match len directly ! * because source data is not adding to the history. ! */ ! dp += match_len; } found_match = true; } *************** *** 612,620 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, * No match found. Copy one literal byte. */ pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); ! pglz_hist_add(hist_start, hist_entries, ! hist_next, hist_recycle, ! dp, dend); dp++; /* Do not do this ++ in the line above! */ /* The macro would do it four times - Jan. */ } --- 672,686 ---- * No match found. Copy one literal byte. */ pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); ! ! /* ! * Incase of history is passed as a separate buffer, don't add any ! * unmatched input data to the history. ! */ ! if (NULL == hend) ! pglz_hist_add(hist_start, hist_entries, ! hist_next, hist_recycle, ! dp, dend); dp++; /* Do not do this ++ in the line above! */ /* The macro would do it four times - Jan. */ } *************** *** 647,661 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, void pglz_decompress(const PGLZ_Header *source, char *dest) { const unsigned char *sp; const unsigned char *srcend; unsigned char *dp; unsigned char *destend; sp = ((const unsigned char *) source) + sizeof(PGLZ_Header); ! srcend = ((const unsigned char *) source) + VARSIZE(source); dp = (unsigned char *) dest; ! destend = dp + source->rawsize; while (sp < srcend && dp < destend) { --- 713,753 ---- void pglz_decompress(const PGLZ_Header *source, char *dest) { + pglz_decompress_with_history((char *) source, dest, NULL, NULL, 0); + } + + /* ---------- + * pglz_decompress_with_history - + * + * Decompresses source into dest by using the specified history. + * ---------- + */ + void + pglz_decompress_with_history(const char *source, char *dest, uint32 *destlen, + const char *history, int32 hlen) + { + PGLZ_Header src; const unsigned char *sp; const unsigned char *srcend; unsigned char *dp; unsigned char *destend; + unsigned char *hend = NULL; + + /* To avoid the unaligned access of PGLZ_Header */ + memcpy((char *) &src, source, sizeof(PGLZ_Header)); + + if (hlen > 0) + hend = (unsigned char *) history + hlen; sp = ((const unsigned char *) source) + sizeof(PGLZ_Header); ! srcend = ((const unsigned char *) source) + VARSIZE(&src); dp = (unsigned char *) dest; ! destend = dp + src.rawsize; ! ! if (destlen) ! { ! *destlen = src.rawsize; ! } while (sp < srcend && dp < destend) { *************** *** 679,684 **** pglz_decompress(const PGLZ_Header *source, char *dest) --- 771,777 ---- */ int32 len; int32 off; + int32 hoff; len = (sp[0] & 0x0f) + 3; off = ((sp[0] & 0xf0) << 4) | sp[1]; *************** *** 705,713 **** pglz_decompress(const PGLZ_Header *source, char *dest) * memcpy() here, because the copied areas could overlap * extremely! */ while (len--) { ! *dp = dp[-off]; dp++; } } --- 798,818 ---- * memcpy() here, because the copied areas could overlap * extremely! */ + hoff = off; while (len--) { ! if (NULL == hend) ! *dp = dp[-off]; ! else ! { ! /* ! * hoff provides the offset in the history buffer from ! * the history end ! */ ! Assert(hoff < hlen); ! *dp = hend[-hoff]; ! hoff--; ! } dp++; } } *** a/src/include/access/heapam_xlog.h --- b/src/include/access/heapam_xlog.h *************** *** 142,153 **** typedef struct xl_heap_update { xl_heaptid target; /* deleted tuple id */ ItemPointerData newtid; /* new inserted tuple id */ ! bool all_visible_cleared; /* PD_ALL_VISIBLE was cleared */ ! bool new_all_visible_cleared; /* same for the page of newtid */ /* NEW TUPLE xl_heap_header AND TUPLE DATA FOLLOWS AT END OF STRUCT */ } xl_heap_update; ! #define SizeOfHeapUpdate (offsetof(xl_heap_update, new_all_visible_cleared) + sizeof(bool)) /* * This is what we need to know about vacuum page cleanup/redirect --- 142,157 ---- { xl_heaptid target; /* deleted tuple id */ ItemPointerData newtid; /* new inserted tuple id */ ! char flags; ! /* NEW TUPLE xl_heap_header AND TUPLE DATA FOLLOWS AT END OF STRUCT */ } xl_heap_update; ! #define XL_HEAP_UPDATE_ALL_VISIBLE_CLEARED 0x01 ! #define XL_HEAP_UPDATE_NEW_ALL_VISIBLE_CLEARED 0x02 ! #define XL_HEAP_UPDATE_DELTA_ENCODED 0x04 ! ! #define SizeOfHeapUpdate (offsetof(xl_heap_update, flags) + sizeof(char)) /* * This is what we need to know about vacuum page cleanup/redirect *** a/src/include/utils/pg_lzcompress.h --- b/src/include/utils/pg_lzcompress.h *************** *** 107,112 **** extern const PGLZ_Strategy *const PGLZ_strategy_always; --- 107,118 ---- */ extern bool pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, const PGLZ_Strategy *strategy); + extern bool pglz_compress_with_history(const char *source, int32 slen, + const char *history, int32 hlen, + PGLZ_Header *dest, + const PGLZ_Strategy *strategy); extern void pglz_decompress(const PGLZ_Header *source, char *dest); + extern void pglz_decompress_with_history(const char *source, char *dest, + uint32 *destlen, const char *history, int32 hlen); #endif /* _PG_LZCOMPRESS_H_ */