From f65964c997cf8958dd9e53a04fb7e92098c784f1 Mon Sep 17 00:00:00 2001 From: John Naylor Date: Wed, 11 Aug 2021 12:17:51 -0400 Subject: [PATCH v1 1/2] Use direct function calls for pg_popcount{32,64} on non-x86 platforms Previously, all pg_popcount{32,64} calls were indirected through a function pointer, even though we had no fast implementation for non-x86 platforms. To fix, use macros to cover both the direct and indirect cases, and define them appropriately similar to the CRC code. --- src/backend/access/heap/visibilitymap.c | 6 ++-- src/backend/nodes/bitmapset.c | 4 +-- src/include/port/pg_bitutils.h | 42 +++++++++++++++++++++- src/port/pg_bitutils.c | 46 ++++--------------------- 4 files changed, 52 insertions(+), 46 deletions(-) diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 114fbbdd30..75b0377807 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -411,14 +411,14 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro if (all_frozen == NULL) { for (i = 0; i < MAPSIZE / sizeof(uint64); i++) - nvisible += pg_popcount64(map[i] & VISIBLE_MASK64); + nvisible += PG_POPCOUNT64(map[i] & VISIBLE_MASK64); } else { for (i = 0; i < MAPSIZE / sizeof(uint64); i++) { - nvisible += pg_popcount64(map[i] & VISIBLE_MASK64); - nfrozen += pg_popcount64(map[i] & FROZEN_MASK64); + nvisible += PG_POPCOUNT64(map[i] & VISIBLE_MASK64); + nfrozen += PG_POPCOUNT64(map[i] & FROZEN_MASK64); } } diff --git a/src/backend/nodes/bitmapset.c b/src/backend/nodes/bitmapset.c index 649478b0d4..a6c4c62914 100644 --- a/src/backend/nodes/bitmapset.c +++ b/src/backend/nodes/bitmapset.c @@ -57,11 +57,11 @@ #if BITS_PER_BITMAPWORD == 32 #define bmw_leftmost_one_pos(w) pg_leftmost_one_pos32(w) #define bmw_rightmost_one_pos(w) pg_rightmost_one_pos32(w) -#define bmw_popcount(w) pg_popcount32(w) +#define bmw_popcount(w) PG_POPCOUNT32(w) #elif BITS_PER_BITMAPWORD == 64 #define bmw_leftmost_one_pos(w) pg_leftmost_one_pos64(w) #define bmw_rightmost_one_pos(w) pg_rightmost_one_pos64(w) -#define bmw_popcount(w) pg_popcount64(w) +#define bmw_popcount(w) PG_POPCOUNT64(w) #else #error "invalid BITS_PER_BITMAPWORD" #endif diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h index 086bd08132..ea97192b3a 100644 --- a/src/include/port/pg_bitutils.h +++ b/src/include/port/pg_bitutils.h @@ -253,9 +253,49 @@ pg_ceil_log2_64(uint64 num) return pg_leftmost_one_pos64(num - 1) + 1; } -/* Count the number of one-bits in a uint32 or uint64 */ +/* + * With MSVC on x86_64 builds, try using native popcnt instructions via the + * __popcnt and __popcnt64 intrinsics. These don't work the same as GCC's + * __builtin_popcount* intrinsic functions as they always emit popcnt + * instructions. + */ +#if defined(_MSC_VER) && defined(_M_AMD64) +#define HAVE_X86_64_POPCNTQ +#endif + +/* + * On x86_64, we can use the hardware popcount instruction, but only if + * we can verify that the CPU supports it via the cpuid instruction. + * + * Otherwise, we fall back to a hand-rolled implementation. + */ +#ifdef HAVE_X86_64_POPCNTQ +#if defined(HAVE__GET_CPUID) || defined(HAVE__CPUID) +#define TRY_POPCNT_FAST 1 +#endif +#endif + +#ifdef TRY_POPCNT_FAST +/* Use the POPCNT instruction, but perform a runtime check first. */ +#define PG_POPCOUNT32(word) pg_popcount32(word) +#define PG_POPCOUNT64(word) pg_popcount64(word) + +extern int pg_popcount32_slow(uint32 word); +extern int pg_popcount64_slow(uint64 word); extern int (*pg_popcount32) (uint32 word); extern int (*pg_popcount64) (uint64 word); +extern int pg_popcount32_fast(uint32 word); +extern int pg_popcount64_fast(uint64 word); + +#else +/* Use a portable implementation */ +#define PG_POPCOUNT32(word) pg_popcount32_slow(word) +#define PG_POPCOUNT64(word) pg_popcount64_slow(word) + +extern int pg_popcount32_slow(uint32 word); +extern int pg_popcount64_slow(uint64 word); + +#endif /* TRY_POPCNT_FAST */ /* Count the number of one-bits in a byte array */ extern uint64 pg_popcount(const char *buf, int bytes); diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c index 10676b285c..3e90de5249 100644 --- a/src/port/pg_bitutils.c +++ b/src/port/pg_bitutils.c @@ -103,47 +103,13 @@ const uint8 pg_number_of_ones[256] = { 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 }; -/* - * With MSVC on x86_64 builds, try using native popcnt instructions via the - * __popcnt and __popcnt64 intrinsics. These don't work the same as GCC's - * __builtin_popcount* intrinsic functions as they always emit popcnt - * instructions. - */ -#if defined(_MSC_VER) && defined(_M_AMD64) -#define HAVE_X86_64_POPCNTQ -#endif - -/* - * On x86_64, we can use the hardware popcount instruction, but only if - * we can verify that the CPU supports it via the cpuid instruction. - * - * Otherwise, we fall back to __builtin_popcount if the compiler has that, - * or a hand-rolled implementation if not. - */ -#ifdef HAVE_X86_64_POPCNTQ -#if defined(HAVE__GET_CPUID) || defined(HAVE__CPUID) -#define TRY_POPCNT_FAST 1 -#endif -#endif - -static int pg_popcount32_slow(uint32 word); -static int pg_popcount64_slow(uint64 word); - #ifdef TRY_POPCNT_FAST static bool pg_popcount_available(void); static int pg_popcount32_choose(uint32 word); static int pg_popcount64_choose(uint64 word); -static int pg_popcount32_fast(uint32 word); -static int pg_popcount64_fast(uint64 word); int (*pg_popcount32) (uint32 word) = pg_popcount32_choose; int (*pg_popcount64) (uint64 word) = pg_popcount64_choose; -#else -int (*pg_popcount32) (uint32 word) = pg_popcount32_slow; -int (*pg_popcount64) (uint64 word) = pg_popcount64_slow; -#endif /* TRY_POPCNT_FAST */ - -#ifdef TRY_POPCNT_FAST /* * Return true if CPUID indicates that the POPCNT instruction is available. @@ -208,7 +174,7 @@ pg_popcount64_choose(uint64 word) * pg_popcount32_fast * Return the number of 1 bits set in word */ -static int +int pg_popcount32_fast(uint32 word) { #ifdef _MSC_VER @@ -225,7 +191,7 @@ __asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc"); * pg_popcount64_fast * Return the number of 1 bits set in word */ -static int +int pg_popcount64_fast(uint64 word) { #ifdef _MSC_VER @@ -245,7 +211,7 @@ __asm__ __volatile__(" popcntq %1,%0\n":"=q"(res):"rm"(word):"cc"); * pg_popcount32_slow * Return the number of 1 bits set in word */ -static int +int pg_popcount32_slow(uint32 word) { #ifdef HAVE__BUILTIN_POPCOUNT @@ -267,7 +233,7 @@ pg_popcount32_slow(uint32 word) * pg_popcount64_slow * Return the number of 1 bits set in word */ -static int +int pg_popcount64_slow(uint64 word) { #ifdef HAVE__BUILTIN_POPCOUNT @@ -309,7 +275,7 @@ pg_popcount(const char *buf, int bytes) while (bytes >= 8) { - popcnt += pg_popcount64(*words++); + popcnt += PG_POPCOUNT64(*words++); bytes -= 8; } @@ -323,7 +289,7 @@ pg_popcount(const char *buf, int bytes) while (bytes >= 4) { - popcnt += pg_popcount32(*words++); + popcnt += PG_POPCOUNT32(*words++); bytes -= 4; } -- 2.31.1