From 88a2bdb6b03dd4bc0e1cc1d289485461d025f51b Mon Sep 17 00:00:00 2001 From: John Naylor Date: Tue, 6 Feb 2024 13:11:33 +0700 Subject: [PATCH v18 3/3] Speed up tail processing when hashing aligned C strings After encountering the NUL terminator, the word-at-a-time loop exits and we must hash the remaining bytes. Previously we calculated the terminator's position and re-loaded the remaining bytes from the input string. We already have all the data we need in a register, so lets's just mask off the bytes we need and hash them immediately. The mask can be cheaply computed without knowing the terminator's position. We still need that position for the length calculation, but the CPU can now do that in parallel with other work, shortening the dependency chain. Ants Aasma and John Naylor Discussion: https://postgr.es/m/CANwKhkP7pCiW_5fAswLhs71-JKGEz1c1%2BPC0a_w1fwY4iGMqUA%40mail.gmail.com --- src/include/common/hashfn_unstable.h | 44 +++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/include/common/hashfn_unstable.h b/src/include/common/hashfn_unstable.h index af80e65fef..308d1982c8 100644 --- a/src/include/common/hashfn_unstable.h +++ b/src/include/common/hashfn_unstable.h @@ -219,8 +219,9 @@ static inline int fasthash_accum_cstring_aligned(fasthash_state *hs, const char *str) { const char *const start = str; - int remainder; + uint64 chunk; uint64 zero_byte_low; + uint64 mask; Assert(PointerIsAligned(start, uint64)); @@ -239,7 +240,7 @@ fasthash_accum_cstring_aligned(fasthash_state *hs, const char *str) */ for (;;) { - uint64 chunk = *(uint64 *) str; + chunk = *(uint64 *) str; #ifdef WORDS_BIGENDIAN zero_byte_low = haszero64(pg_bswap64(chunk)); @@ -254,14 +255,37 @@ fasthash_accum_cstring_aligned(fasthash_state *hs, const char *str) str += FH_SIZEOF_ACCUM; } - /* - * The byte corresponding to the NUL will be 0x80, so the rightmost bit - * position will be in the range 7, 15, ..., 63. Turn this into byte - * position by dividing by 8. - */ - remainder = pg_rightmost_one_pos64(zero_byte_low) / BITS_PER_BYTE; - fasthash_accum(hs, str, remainder); - str += remainder; + if (zero_byte_low & 0xFF) + { + /* + * The next byte in the input is the NUL terminator, so we have + * nothing to do. + */ + } + else + { + /* + * Create a mask for the remaining bytes so we can combine them into + * the hash. The mask also covers the NUL terminator, but that's + * harmless. The mask could contain 0x80 in bytes corresponding to the + * input past the terminator, but only where the input byte is zero or + * one, so also harmless. + */ + mask = zero_byte_low | (zero_byte_low - 1); +#ifdef WORDS_BIGENDIAN + /* need to mask the upper bytes */ + mask = pg_bswap64(mask); +#endif + hs->accum = chunk & mask; + fasthash_combine(hs); + + /* + * The byte corresponding to the NUL will be 0x80, so the rightmost + * bit position will be in the range 15, 23, ..., 63. Turn this into + * byte position by dividing by 8. + */ + str += pg_rightmost_one_pos64(zero_byte_low) / BITS_PER_BYTE; + } return str - start; } -- 2.43.0