From 41600458cfafb9b79dc2a74f2af274f7bff30b82 Mon Sep 17 00:00:00 2001
From: Ewan Young <kdbase.hack@gmail.com>
Date: Wed, 17 Jun 2026 01:44:59 +0800
Subject: [PATCH v1] btree_gist: Fix NaN handling in float4 and float8
 opclasses

The float4 and float8 GiST opclasses in btree_gist compared key values
with the plain C operators (>, >=, ==, <=, <).  Those follow IEEE 754
semantics, under which every comparison involving a NaN is false,
including NaN = NaN.  PostgreSQL's float types instead define a total
order in which NaN is equal to itself and sorts after every non-NaN
value, as implemented by float8_cmp_internal()/float4_cmp_internal()
and used both by the float B-tree opclasses and by sequential scans.

Because the consistent and union support functions used the raw
operators, a GiST index on a float column treated NaN inconsistently
with a sequential scan: union never widened an internal key's upper
bound to cover a NaN child, and consistent then pruned that subtree, so
queries such as "a = 'NaN'", "a >= 'NaN'" or "a > 'Infinity'" silently
omitted the NaN rows.  As these consistent functions report exact
results (recheck = false), the wrong answer was returned uncorrected.

Fix by comparing through float8_cmp_internal()/float4_cmp_internal() in
the gbt_floatNgt/ge/eq/le/lt callbacks and in gbt_floatNkey_cmp.  This
is the same NaN-aware approach commit 50354637694b used for the in-core
geometric GiST opclasses, and it matches the sortsupport comparators
already present in these files.

Indexes built before this change may be missing entries for NaN values
and should be reindexed.

Also add regression tests exercising NaN and infinities through both
sequential and index scans.
---
 contrib/btree_gist/btree_float4.c      | 23 ++++++-------
 contrib/btree_gist/btree_float8.c      | 23 ++++++-------
 contrib/btree_gist/expected/float4.out | 45 ++++++++++++++++++++++++++
 contrib/btree_gist/expected/float8.out | 45 ++++++++++++++++++++++++++
 contrib/btree_gist/sql/float4.sql      | 18 +++++++++++
 contrib/btree_gist/sql/float8.sql      | 18 +++++++++++
 6 files changed, 146 insertions(+), 26 deletions(-)

diff --git a/contrib/btree_gist/btree_float4.c b/contrib/btree_gist/btree_float4.c
index c076918fd48..38e99e0906c 100644
--- a/contrib/btree_gist/btree_float4.c
+++ b/contrib/btree_gist/btree_float4.c
@@ -29,27 +29,27 @@ PG_FUNCTION_INFO_V1(gbt_float4_sortsupport);
 static bool
 gbt_float4gt(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float4 *) a) > *((const float4 *) b));
+	return float4_cmp_internal(*((const float4 *) a), *((const float4 *) b)) > 0;
 }
 static bool
 gbt_float4ge(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float4 *) a) >= *((const float4 *) b));
+	return float4_cmp_internal(*((const float4 *) a), *((const float4 *) b)) >= 0;
 }
 static bool
 gbt_float4eq(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float4 *) a) == *((const float4 *) b));
+	return float4_cmp_internal(*((const float4 *) a), *((const float4 *) b)) == 0;
 }
 static bool
 gbt_float4le(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float4 *) a) <= *((const float4 *) b));
+	return float4_cmp_internal(*((const float4 *) a), *((const float4 *) b)) <= 0;
 }
 static bool
 gbt_float4lt(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float4 *) a) < *((const float4 *) b));
+	return float4_cmp_internal(*((const float4 *) a), *((const float4 *) b)) < 0;
 }
 
 static int
@@ -57,16 +57,13 @@ gbt_float4key_cmp(const void *a, const void *b, FmgrInfo *flinfo)
 {
 	float4KEY  *ia = (float4KEY *) (((const Nsrt *) a)->t);
 	float4KEY  *ib = (float4KEY *) (((const Nsrt *) b)->t);
+	int			res;
 
-	if (ia->lower == ib->lower)
-	{
-		if (ia->upper == ib->upper)
-			return 0;
+	res = float4_cmp_internal(ia->lower, ib->lower);
+	if (res == 0)
+		return float4_cmp_internal(ia->upper, ib->upper);
 
-		return (ia->upper > ib->upper) ? 1 : -1;
-	}
-
-	return (ia->lower > ib->lower) ? 1 : -1;
+	return res;
 }
 
 static float8
diff --git a/contrib/btree_gist/btree_float8.c b/contrib/btree_gist/btree_float8.c
index d7386e885a2..73fdb8359cf 100644
--- a/contrib/btree_gist/btree_float8.c
+++ b/contrib/btree_gist/btree_float8.c
@@ -30,27 +30,27 @@ PG_FUNCTION_INFO_V1(gbt_float8_sortsupport);
 static bool
 gbt_float8gt(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float8 *) a) > *((const float8 *) b));
+	return float8_cmp_internal(*((const float8 *) a), *((const float8 *) b)) > 0;
 }
 static bool
 gbt_float8ge(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float8 *) a) >= *((const float8 *) b));
+	return float8_cmp_internal(*((const float8 *) a), *((const float8 *) b)) >= 0;
 }
 static bool
 gbt_float8eq(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float8 *) a) == *((const float8 *) b));
+	return float8_cmp_internal(*((const float8 *) a), *((const float8 *) b)) == 0;
 }
 static bool
 gbt_float8le(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float8 *) a) <= *((const float8 *) b));
+	return float8_cmp_internal(*((const float8 *) a), *((const float8 *) b)) <= 0;
 }
 static bool
 gbt_float8lt(const void *a, const void *b, FmgrInfo *flinfo)
 {
-	return (*((const float8 *) a) < *((const float8 *) b));
+	return float8_cmp_internal(*((const float8 *) a), *((const float8 *) b)) < 0;
 }
 
 static int
@@ -58,16 +58,13 @@ gbt_float8key_cmp(const void *a, const void *b, FmgrInfo *flinfo)
 {
 	float8KEY  *ia = (float8KEY *) (((const Nsrt *) a)->t);
 	float8KEY  *ib = (float8KEY *) (((const Nsrt *) b)->t);
+	int			res;
 
-	if (ia->lower == ib->lower)
-	{
-		if (ia->upper == ib->upper)
-			return 0;
+	res = float8_cmp_internal(ia->lower, ib->lower);
+	if (res == 0)
+		return float8_cmp_internal(ia->upper, ib->upper);
 
-		return (ia->upper > ib->upper) ? 1 : -1;
-	}
-
-	return (ia->lower > ib->lower) ? 1 : -1;
+	return res;
 }
 
 static float8
diff --git a/contrib/btree_gist/expected/float4.out b/contrib/btree_gist/expected/float4.out
index dfe732049e6..84c50bbd86f 100644
--- a/contrib/btree_gist/expected/float4.out
+++ b/contrib/btree_gist/expected/float4.out
@@ -89,3 +89,48 @@ SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3;
  -158.17741 | 20.822586
 (3 rows)
 
+-- NaN and infinities must give identical results via seqscan and index scan;
+-- NaN sorts after every non-NaN value and is equal to itself.
+INSERT INTO float4tmp VALUES ('NaN'), ('Infinity'), ('-Infinity');
+SET enable_seqscan=on;
+SET enable_indexscan=off;
+SET enable_bitmapscan=off;
+SELECT count(*) FROM float4tmp WHERE a =  'NaN';
+ count 
+-------
+     1
+(1 row)
+
+SELECT count(*) FROM float4tmp WHERE a >= 'NaN';
+ count 
+-------
+     1
+(1 row)
+
+SELECT count(*) FROM float4tmp WHERE a >  'Infinity';
+ count 
+-------
+     1
+(1 row)
+
+SET enable_seqscan=off;
+SET enable_indexscan=on;
+SET enable_bitmapscan=on;
+SELECT count(*) FROM float4tmp WHERE a =  'NaN';
+ count 
+-------
+     1
+(1 row)
+
+SELECT count(*) FROM float4tmp WHERE a >= 'NaN';
+ count 
+-------
+     1
+(1 row)
+
+SELECT count(*) FROM float4tmp WHERE a >  'Infinity';
+ count 
+-------
+     1
+(1 row)
+
diff --git a/contrib/btree_gist/expected/float8.out b/contrib/btree_gist/expected/float8.out
index ebd0ef3d689..61229f28edc 100644
--- a/contrib/btree_gist/expected/float8.out
+++ b/contrib/btree_gist/expected/float8.out
@@ -89,3 +89,48 @@ SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3;
   -1769.73634 | 120.26366000000007
 (3 rows)
 
+-- NaN and infinities must give identical results via seqscan and index scan;
+-- NaN sorts after every non-NaN value and is equal to itself.
+INSERT INTO float8tmp VALUES ('NaN'), ('Infinity'), ('-Infinity');
+SET enable_seqscan=on;
+SET enable_indexscan=off;
+SET enable_bitmapscan=off;
+SELECT count(*) FROM float8tmp WHERE a =  'NaN';
+ count 
+-------
+     1
+(1 row)
+
+SELECT count(*) FROM float8tmp WHERE a >= 'NaN';
+ count 
+-------
+     1
+(1 row)
+
+SELECT count(*) FROM float8tmp WHERE a >  'Infinity';
+ count 
+-------
+     1
+(1 row)
+
+SET enable_seqscan=off;
+SET enable_indexscan=on;
+SET enable_bitmapscan=on;
+SELECT count(*) FROM float8tmp WHERE a =  'NaN';
+ count 
+-------
+     1
+(1 row)
+
+SELECT count(*) FROM float8tmp WHERE a >= 'NaN';
+ count 
+-------
+     1
+(1 row)
+
+SELECT count(*) FROM float8tmp WHERE a >  'Infinity';
+ count 
+-------
+     1
+(1 row)
+
diff --git a/contrib/btree_gist/sql/float4.sql b/contrib/btree_gist/sql/float4.sql
index 3da1ce953c8..82eab363c47 100644
--- a/contrib/btree_gist/sql/float4.sql
+++ b/contrib/btree_gist/sql/float4.sql
@@ -35,3 +35,21 @@ SELECT count(*) FROM float4tmp WHERE a >  -179.0::float4;
 EXPLAIN (COSTS OFF)
 SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3;
 SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3;
+
+-- NaN and infinities must give identical results via seqscan and index scan;
+-- NaN sorts after every non-NaN value and is equal to itself.
+INSERT INTO float4tmp VALUES ('NaN'), ('Infinity'), ('-Infinity');
+
+SET enable_seqscan=on;
+SET enable_indexscan=off;
+SET enable_bitmapscan=off;
+SELECT count(*) FROM float4tmp WHERE a =  'NaN';
+SELECT count(*) FROM float4tmp WHERE a >= 'NaN';
+SELECT count(*) FROM float4tmp WHERE a >  'Infinity';
+
+SET enable_seqscan=off;
+SET enable_indexscan=on;
+SET enable_bitmapscan=on;
+SELECT count(*) FROM float4tmp WHERE a =  'NaN';
+SELECT count(*) FROM float4tmp WHERE a >= 'NaN';
+SELECT count(*) FROM float4tmp WHERE a >  'Infinity';
diff --git a/contrib/btree_gist/sql/float8.sql b/contrib/btree_gist/sql/float8.sql
index e1e819b37f9..1d8b21e91bd 100644
--- a/contrib/btree_gist/sql/float8.sql
+++ b/contrib/btree_gist/sql/float8.sql
@@ -35,3 +35,21 @@ SELECT count(*) FROM float8tmp WHERE a >  -1890.0::float8;
 EXPLAIN (COSTS OFF)
 SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3;
 SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3;
+
+-- NaN and infinities must give identical results via seqscan and index scan;
+-- NaN sorts after every non-NaN value and is equal to itself.
+INSERT INTO float8tmp VALUES ('NaN'), ('Infinity'), ('-Infinity');
+
+SET enable_seqscan=on;
+SET enable_indexscan=off;
+SET enable_bitmapscan=off;
+SELECT count(*) FROM float8tmp WHERE a =  'NaN';
+SELECT count(*) FROM float8tmp WHERE a >= 'NaN';
+SELECT count(*) FROM float8tmp WHERE a >  'Infinity';
+
+SET enable_seqscan=off;
+SET enable_indexscan=on;
+SET enable_bitmapscan=on;
+SELECT count(*) FROM float8tmp WHERE a =  'NaN';
+SELECT count(*) FROM float8tmp WHERE a >= 'NaN';
+SELECT count(*) FROM float8tmp WHERE a >  'Infinity';
-- 
2.47.3

