From e55950585b2cb6760ec3b8504a42d145310c6ad9 Mon Sep 17 00:00:00 2001
From: Ewan Young <kdbase.hack@gmail.com>
Date: Thu, 11 Jun 2026 01:43:22 +0800
Subject: [PATCH v1] Discard ORDER BY and DISTINCT in an ANY/IN sub-select
 pulled up to a join

When convert_ANY_sublink_to_join() turns an IN/ANY (or, for a provably
null-safe NOT IN, ANTI) sublink into a semi or anti join, the sub-select's
ORDER BY and DISTINCT clauses become irrelevant: such a join only depends
on whether a matching row exists, not on the order of the inner rows or
whether they contain duplicates.  Discard those clauses once we have
committed to the conversion.

This mirrors what simplify_EXISTS_query() already does for EXISTS, and has
two benefits: the executor no longer needlessly sorts or de-duplicates the
inner side, and removing the clauses can make the sub-select "simple" so
that pull_up_subqueries() is able to flatten it into the parent query
instead of leaving a separate subquery (often with a redundant
HashAggregate); see is_simple_subquery().

The clauses are kept when the sub-select has LIMIT/OFFSET, since then
ORDER BY and DISTINCT do affect which rows are returned, or uses
DISTINCT ON, which selects particular rows in conjunction with ORDER BY.
---
 src/backend/optimizer/plan/subselect.c  |  24 +++
 src/test/regress/expected/subselect.out | 219 ++++++++++++++++++++++++
 src/test/regress/sql/subselect.sql      |  76 ++++++++
 3 files changed, 319 insertions(+)

diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c
index 6aa8971c95d..f609ecff74f 100644
--- a/src/backend/optimizer/plan/subselect.c
+++ b/src/backend/optimizer/plan/subselect.c
@@ -1405,6 +1405,30 @@ convert_ANY_sublink_to_join(PlannerInfo *root, SubLink *sublink,
 	if (contain_volatile_functions(sublink->testexpr))
 		return NULL;
 
+	/*
+	 * Now that we have decided to convert the sublink to a semi or anti join,
+	 * we can discard any ORDER BY and DISTINCT clauses in the sub-select.  A
+	 * semijoin or antijoin only cares whether a matching row exists, not about
+	 * the ordering of the sub-select's output or whether it contains duplicate
+	 * rows; so these clauses cannot change the result.  This mirrors what
+	 * simplify_EXISTS_query() does for EXISTS.  Besides saving the executor
+	 * from a useless sort or de-duplication of the inner rows, removing these
+	 * clauses may allow the sub-select to be flattened into the parent query
+	 * later on by pull_up_subqueries() (see is_simple_subquery()).
+	 *
+	 * We must leave things alone if there is a LIMIT or OFFSET, since then the
+	 * ORDER BY and DISTINCT do affect which rows are returned.  Likewise,
+	 * DISTINCT ON selects particular rows (in conjunction with ORDER BY), so
+	 * we don't touch the sub-select in that case either.
+	 */
+	if (subselect->limitOffset == NULL &&
+		subselect->limitCount == NULL &&
+		!subselect->hasDistinctOn)
+	{
+		subselect->distinctClause = NIL;
+		subselect->sortClause = NIL;
+	}
+
 	/* Create a dummy ParseState for addRangeTableEntryForSubquery */
 	pstate = make_parsestate(NULL);
 
diff --git a/src/test/regress/expected/subselect.out b/src/test/regress/expected/subselect.out
index a3778c23c34..e487f89ae69 100644
--- a/src/test/regress/expected/subselect.out
+++ b/src/test/regress/expected/subselect.out
@@ -3785,3 +3785,222 @@ WHERE id NOT IN (SELECT id FROM notnull_notvalid_tab);
 (0 rows)
 
 ROLLBACK;
+--
+-- ORDER BY and DISTINCT in an ANY/IN sublink are irrelevant once the sublink
+-- is converted to a semi/anti join, so convert_ANY_sublink_to_join() discards
+-- them.  With them gone the query plans the same as if they had not been
+-- written at all (and, absent LIMIT/OFFSET or DISTINCT ON, the sub-select can
+-- be pulled up).
+--
+-- DISTINCT removed: same plan as the variant written without DISTINCT
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select distinct hundred from tenk2);
+                        QUERY PLAN                        
+----------------------------------------------------------
+ Merge Join
+   Merge Cond: (t.unique1 = tenk2.hundred)
+   ->  Index Scan using tenk1_unique1 on tenk1 t
+   ->  Unique
+         ->  Index Only Scan using tenk2_hundred on tenk2
+(5 rows)
+
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select hundred from tenk2);
+                        QUERY PLAN                        
+----------------------------------------------------------
+ Merge Join
+   Merge Cond: (t.unique1 = tenk2.hundred)
+   ->  Index Scan using tenk1_unique1 on tenk1 t
+   ->  Unique
+         ->  Index Only Scan using tenk2_hundred on tenk2
+(5 rows)
+
+-- ORDER BY is likewise removed.  A correlated sub-select with ORDER BY would
+-- otherwise remain an un-flattened, per-outer-row re-executed Sort subplan;
+-- with the ORDER BY gone it is pulled up into a plain semi join.
+explain (costs off)
+select * from tenk1 t
+where t.hundred in (select b.ten from tenk2 b where b.unique2 = t.unique1 order by b.ten);
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Hash Semi Join
+   Hash Cond: ((t.unique1 = b.unique2) AND (t.hundred = b.ten))
+   ->  Seq Scan on tenk1 t
+   ->  Hash
+         ->  Seq Scan on tenk2 b
+(5 rows)
+
+-- a LIMIT makes ORDER BY/DISTINCT significant, so they are kept
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select distinct hundred from tenk2 limit 5);
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Nested Loop
+   ->  Limit
+         ->  Unique
+               ->  Index Only Scan using tenk2_hundred on tenk2
+   ->  Index Scan using tenk1_unique1 on tenk1 t
+         Index Cond: (unique1 = tenk2.hundred)
+(6 rows)
+
+-- DISTINCT ON is also left alone
+explain (costs off)
+select * from tenk1 t
+where t.unique1 in (select distinct on (thousand) hundred from tenk2 order by thousand);
+                     QUERY PLAN                     
+----------------------------------------------------
+ Hash Join
+   Hash Cond: (t.unique1 = tenk2.hundred)
+   ->  Seq Scan on tenk1 t
+   ->  Hash
+         ->  HashAggregate
+               Group Key: tenk2.hundred
+               ->  Unique
+                     ->  Sort
+                           Sort Key: tenk2.thousand
+                           ->  Seq Scan on tenk2
+(10 rows)
+
+-- removing the redundant inner DISTINCT must not change the result
+select count(*) from tenk1 t where t.unique1 in (select distinct hundred from tenk2);
+ count 
+-------
+   100
+(1 row)
+
+-- The simplification is also safe when the sub-select computes aggregates or
+-- window functions: a semijoin still only cares about membership, and the set
+-- of output values is unchanged by removing the outer DISTINCT.  The sub-select
+-- is not flattened in these cases (it has aggregates/window functions), but the
+-- redundant DISTINCT de-duplication on top is still removed, so the plan matches
+-- the spelling written without DISTINCT.
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select distinct count(*) from tenk2 group by hundred);
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Hash Semi Join
+   Hash Cond: (t.unique1 = unnamed_subquery.count)
+   ->  Seq Scan on tenk1 t
+   ->  Hash
+         ->  Subquery Scan on unnamed_subquery
+               ->  GroupAggregate
+                     Group Key: tenk2.hundred
+                     ->  Index Only Scan using tenk2_hundred on tenk2
+(8 rows)
+
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select count(*) from tenk2 group by hundred);
+                              QUERY PLAN                              
+----------------------------------------------------------------------
+ Hash Semi Join
+   Hash Cond: (t.unique1 = unnamed_subquery.count)
+   ->  Seq Scan on tenk1 t
+   ->  Hash
+         ->  Subquery Scan on unnamed_subquery
+               ->  GroupAggregate
+                     Group Key: tenk2.hundred
+                     ->  Index Only Scan using tenk2_hundred on tenk2
+(8 rows)
+
+select count(*) from tenk1 t where t.unique1 in (select distinct count(*) from tenk2 group by hundred);
+ count 
+-------
+     1
+(1 row)
+
+-- window function: the window's own ORDER BY (windowClause) is preserved; only
+-- the query-level DISTINCT is removed.
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select distinct rank() over (order by hundred) from tenk2);
+                                     QUERY PLAN                                      
+-------------------------------------------------------------------------------------
+ Hash Join
+   Hash Cond: (t.unique1 = (rank() OVER w1))
+   ->  Seq Scan on tenk1 t
+   ->  Hash
+         ->  HashAggregate
+               Group Key: rank() OVER w1
+               ->  WindowAgg
+                     Window: w1 AS (ORDER BY tenk2.hundred ROWS UNBOUNDED PRECEDING)
+                     ->  Index Only Scan using tenk2_hundred on tenk2
+(9 rows)
+
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select rank() over (order by hundred) from tenk2);
+                                     QUERY PLAN                                      
+-------------------------------------------------------------------------------------
+ Hash Join
+   Hash Cond: (t.unique1 = (rank() OVER w1))
+   ->  Seq Scan on tenk1 t
+   ->  Hash
+         ->  HashAggregate
+               Group Key: rank() OVER w1
+               ->  WindowAgg
+                     Window: w1 AS (ORDER BY tenk2.hundred ROWS UNBOUNDED PRECEDING)
+                     ->  Index Only Scan using tenk2_hundred on tenk2
+(9 rows)
+
+select count(*) from tenk1 t where t.unique1 in (select distinct rank() over (order by hundred) from tenk2);
+ count 
+-------
+   100
+(1 row)
+
+-- A correlated IN sublink with DISTINCT.  Removing the DISTINCT lets the
+-- (lateral) sub-select be pulled up, turning what would otherwise be a
+-- per-outer-row re-executed Sort+Unique subplan into a plain semi join.
+explain (costs off)
+select * from tenk1 t
+where t.hundred in (select distinct b.ten from tenk2 b where b.unique2 = t.unique1);
+                           QUERY PLAN                           
+----------------------------------------------------------------
+ Hash Semi Join
+   Hash Cond: ((t.unique1 = b.unique2) AND (t.hundred = b.ten))
+   ->  Seq Scan on tenk1 t
+   ->  Hash
+         ->  Seq Scan on tenk2 b
+(5 rows)
+
+select count(*) from tenk1 t
+where t.hundred in (select distinct b.ten from tenk2 b where b.unique2 = t.unique1);
+ count 
+-------
+   107
+(1 row)
+
+-- The same applies to NOT IN, which becomes an anti join when the compared
+-- columns are provably non-null.  An anti join needs no de-duplication of its
+-- inner side at all, so removing the DISTINCT simply leaves a plain anti join
+-- (with the DISTINCT kept, the inner side would carry a redundant aggregation).
+create temp table anti_o (a int not null);
+create temp table anti_i (k int not null);
+insert into anti_o select unique1 from tenk1;
+insert into anti_i select hundred from tenk2;
+analyze anti_o;
+analyze anti_i;
+explain (costs off) select * from anti_o where a not in (select distinct k from anti_i);
+             QUERY PLAN             
+------------------------------------
+ Hash Anti Join
+   Hash Cond: (anti_o.a = anti_i.k)
+   ->  Seq Scan on anti_o
+   ->  Hash
+         ->  Seq Scan on anti_i
+(5 rows)
+
+explain (costs off) select * from anti_o where a not in (select k from anti_i);
+             QUERY PLAN             
+------------------------------------
+ Hash Anti Join
+   Hash Cond: (anti_o.a = anti_i.k)
+   ->  Seq Scan on anti_o
+   ->  Hash
+         ->  Seq Scan on anti_i
+(5 rows)
+
+select count(*) from anti_o where a not in (select distinct k from anti_i);
+ count 
+-------
+  9900
+(1 row)
+
diff --git a/src/test/regress/sql/subselect.sql b/src/test/regress/sql/subselect.sql
index 1a02c3f86c0..c066a193287 100644
--- a/src/test/regress/sql/subselect.sql
+++ b/src/test/regress/sql/subselect.sql
@@ -1647,3 +1647,79 @@ SELECT * FROM not_null_tab
 WHERE id NOT IN (SELECT id FROM notnull_notvalid_tab);
 
 ROLLBACK;
+
+--
+-- ORDER BY and DISTINCT in an ANY/IN sublink are irrelevant once the sublink
+-- is converted to a semi/anti join, so convert_ANY_sublink_to_join() discards
+-- them.  With them gone the query plans the same as if they had not been
+-- written at all (and, absent LIMIT/OFFSET or DISTINCT ON, the sub-select can
+-- be pulled up).
+--
+
+-- DISTINCT removed: same plan as the variant written without DISTINCT
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select distinct hundred from tenk2);
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select hundred from tenk2);
+
+-- ORDER BY is likewise removed.  A correlated sub-select with ORDER BY would
+-- otherwise remain an un-flattened, per-outer-row re-executed Sort subplan;
+-- with the ORDER BY gone it is pulled up into a plain semi join.
+explain (costs off)
+select * from tenk1 t
+where t.hundred in (select b.ten from tenk2 b where b.unique2 = t.unique1 order by b.ten);
+
+-- a LIMIT makes ORDER BY/DISTINCT significant, so they are kept
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select distinct hundred from tenk2 limit 5);
+
+-- DISTINCT ON is also left alone
+explain (costs off)
+select * from tenk1 t
+where t.unique1 in (select distinct on (thousand) hundred from tenk2 order by thousand);
+
+-- removing the redundant inner DISTINCT must not change the result
+select count(*) from tenk1 t where t.unique1 in (select distinct hundred from tenk2);
+
+-- The simplification is also safe when the sub-select computes aggregates or
+-- window functions: a semijoin still only cares about membership, and the set
+-- of output values is unchanged by removing the outer DISTINCT.  The sub-select
+-- is not flattened in these cases (it has aggregates/window functions), but the
+-- redundant DISTINCT de-duplication on top is still removed, so the plan matches
+-- the spelling written without DISTINCT.
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select distinct count(*) from tenk2 group by hundred);
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select count(*) from tenk2 group by hundred);
+select count(*) from tenk1 t where t.unique1 in (select distinct count(*) from tenk2 group by hundred);
+
+-- window function: the window's own ORDER BY (windowClause) is preserved; only
+-- the query-level DISTINCT is removed.
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select distinct rank() over (order by hundred) from tenk2);
+explain (costs off)
+select * from tenk1 t where t.unique1 in (select rank() over (order by hundred) from tenk2);
+select count(*) from tenk1 t where t.unique1 in (select distinct rank() over (order by hundred) from tenk2);
+
+-- A correlated IN sublink with DISTINCT.  Removing the DISTINCT lets the
+-- (lateral) sub-select be pulled up, turning what would otherwise be a
+-- per-outer-row re-executed Sort+Unique subplan into a plain semi join.
+explain (costs off)
+select * from tenk1 t
+where t.hundred in (select distinct b.ten from tenk2 b where b.unique2 = t.unique1);
+select count(*) from tenk1 t
+where t.hundred in (select distinct b.ten from tenk2 b where b.unique2 = t.unique1);
+
+-- The same applies to NOT IN, which becomes an anti join when the compared
+-- columns are provably non-null.  An anti join needs no de-duplication of its
+-- inner side at all, so removing the DISTINCT simply leaves a plain anti join
+-- (with the DISTINCT kept, the inner side would carry a redundant aggregation).
+create temp table anti_o (a int not null);
+create temp table anti_i (k int not null);
+insert into anti_o select unique1 from tenk1;
+insert into anti_i select hundred from tenk2;
+analyze anti_o;
+analyze anti_i;
+explain (costs off) select * from anti_o where a not in (select distinct k from anti_i);
+explain (costs off) select * from anti_o where a not in (select k from anti_i);
+select count(*) from anti_o where a not in (select distinct k from anti_i);
-- 
2.47.3

