diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c index d9f0e79..e1a1020 100644 --- a/src/backend/executor/nodeWindowAgg.c +++ b/src/backend/executor/nodeWindowAgg.c @@ -2000,6 +2000,16 @@ WinGetCurrentPosition(WindowObject winobj) Assert(WindowObjectIsValid(winobj)); return winobj->winstate->currentpos; } +/* + * WinGetFrameOptions + * Returns the frame option flags + */ +int +WinGetFrameOptions(WindowObject winobj) +{ + Assert(WindowObjectIsValid(winobj)); + return winobj->winstate->frameOptions; +} /* * WinGetPartitionRowCount diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 5094226..0309a99 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -488,6 +488,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type window_clause window_definition_list opt_partition_clause %type window_definition over_clause window_specification opt_frame_clause frame_extent frame_bound + over_specification %type opt_existing_window_name %type opt_if_not_exists @@ -543,7 +544,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); HANDLER HAVING HEADER_P HOLD HOUR_P - IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IN_P + IDENTITY_P IF_P IGNORE ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IN_P INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION @@ -573,7 +574,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); RANGE READ REAL REASSIGN RECHECK RECURSIVE REF REFERENCES REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA - RESET RESTART RESTRICT RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK + RESET RESPECT RESTART RESTRICT RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROW ROWS RULE SAVEPOINT SCHEMA SCROLL SEARCH SECOND_P SECURITY SELECT SEQUENCE SEQUENCES @@ -607,6 +608,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); * creates these tokens when required. */ %token NULLS_FIRST NULLS_LAST WITH_TIME +%token RESPECT_NULLS IGNORE_NULLS /* Precedence: lowest to highest */ @@ -11752,7 +11754,8 @@ window_definition: } ; -over_clause: OVER window_specification +over_specification: + OVER window_specification { $$ = $2; } | OVER ColId { @@ -11767,6 +11770,18 @@ over_clause: OVER window_specification n->location = @2; $$ = n; } + ; + +over_clause: over_specification + { $$ = $1; } + | RESPECT_NULLS over_specification + { $$ = $2; } + | IGNORE_NULLS over_specification + { + if($2) + $2->frameOptions |= FRAMEOPTION_IGNORE_NULLS; + $$ = $2; + } | /*EMPTY*/ { $$ = NULL; } ; @@ -12740,6 +12755,7 @@ unreserved_keyword: | HOUR_P | IDENTITY_P | IF_P + | IGNORE | IMMEDIATE | IMMUTABLE | IMPLICIT_P @@ -12827,6 +12843,7 @@ unreserved_keyword: | REPLACE | REPLICA | RESET + | RESPECT | RESTART | RESTRICT | RETURNS diff --git a/src/backend/parser/parser.c b/src/backend/parser/parser.c index b8ec790..1b2b2de 100644 --- a/src/backend/parser/parser.c +++ b/src/backend/parser/parser.c @@ -118,15 +118,7 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner) cur_token = NULLS_LAST; break; default: - /* save the lookahead token for next time */ - yyextra->lookahead_token = next_token; - yyextra->lookahead_yylval = lvalp->core_yystype; - yyextra->lookahead_yylloc = *llocp; - yyextra->have_lookahead = true; - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; + goto restore_next_token; } break; @@ -144,15 +136,38 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner) cur_token = WITH_TIME; break; default: - /* save the lookahead token for next time */ - yyextra->lookahead_token = next_token; - yyextra->lookahead_yylval = lvalp->core_yystype; - yyextra->lookahead_yylloc = *llocp; - yyextra->have_lookahead = true; - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; + goto restore_next_token; + } + break; + + /* + * Window functions can use RESPECT NULLS or IGNORE NULLS to + * modify their behaviour + */ + case RESPECT: + cur_yylval = lvalp->core_yystype; + cur_yylloc = *llocp; + next_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); + switch (next_token) + { + case NULLS_P: + cur_token = RESPECT_NULLS; + break; + default: + goto restore_next_token; + } + break; + case IGNORE: + cur_yylval = lvalp->core_yystype; + cur_yylloc = *llocp; + next_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); + switch (next_token) + { + case NULLS_P: + cur_token = IGNORE_NULLS; break; + default: + goto restore_next_token; } break; @@ -161,4 +176,16 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner) } return cur_token; + +restore_next_token: + /* save the lookahead token for next time */ + yyextra->lookahead_token = next_token; + yyextra->lookahead_yylval = lvalp->core_yystype; + yyextra->lookahead_yylloc = *llocp; + yyextra->have_lookahead = true; + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + + return cur_token; } diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c index b7c42d3..84110ab 100644 --- a/src/backend/utils/adt/windowfuncs.c +++ b/src/backend/utils/adt/windowfuncs.c @@ -25,6 +25,15 @@ typedef struct rank_context } rank_context; /* + * structure for IGNORE NULLS / RESPECT NULLS semantics + */ +typedef struct leadlag_context +{ + int64 last; /* last non-null result, initially 0 */ + bool seen_one; /* true iff we can output the row in "last" now */ +} leadlag_context; + +/* * ntile process information */ typedef struct @@ -292,6 +301,15 @@ leadlag_common(FunctionCallInfo fcinfo, Datum result; bool isnull; bool isout; + bool ignore_nulls; + leadlag_context* context; + + /* + * We want to set the markpos (the earliest tuple we can access) as + * aggressively as possible to save memory, but we can't move the mark + * beyond the last non-null tuple! + */ + ignore_nulls = (WinGetFrameOptions(winobj) & FRAMEOPTION_IGNORE_NULLS) != 0; if (withoffset) { @@ -305,11 +323,15 @@ leadlag_common(FunctionCallInfo fcinfo, offset = 1; const_offset = true; } + if(!forward) + { + offset = -offset; + } result = WinGetFuncArgInPartition(winobj, 0, - (forward ? offset : -offset), + offset, WINDOW_SEEK_CURRENT, - const_offset, + const_offset && !ignore_nulls, &isnull, &isout); if (isout) @@ -322,6 +344,34 @@ leadlag_common(FunctionCallInfo fcinfo, result = WinGetFuncArgCurrent(winobj, 2, &isnull); } + if (ignore_nulls) + { + /* + * We'll keep the last non-null value we've seen in our per-partition chunk + * of memory, so it gets cleaned up for us. + */ + context = (leadlag_context *) + WinGetPartitionLocalMemory(winobj, sizeof(leadlag_context)); + if (isnull) + { + if (context->seen_one) + { + /* restore the datum at the stashed index */ + result = WinGetFuncArgInPartition(winobj, 0, + context->last, + WINDOW_SEEK_HEAD, + const_offset, /* drag mark up after us */ + &isnull, &isout); + } + } + else + { + /* work out which tuple we just loaded */ + context->last = WinGetCurrentPosition(winobj) + offset; + context->seen_one = true; + } + } + if (isnull) PG_RETURN_NULL(); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 6723647..59fc635 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -435,6 +435,7 @@ typedef struct WindowDef #define FRAMEOPTION_END_VALUE_PRECEDING 0x00800 /* end is V. P. */ #define FRAMEOPTION_START_VALUE_FOLLOWING 0x01000 /* start is V. F. */ #define FRAMEOPTION_END_VALUE_FOLLOWING 0x02000 /* end is V. F. */ +#define FRAMEOPTION_IGNORE_NULLS 0x04000 /* lead/lag/nth */ #define FRAMEOPTION_START_VALUE \ (FRAMEOPTION_START_VALUE_PRECEDING | FRAMEOPTION_START_VALUE_FOLLOWING) diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 68a13b7..2acf073 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -179,6 +179,7 @@ PG_KEYWORD("hold", HOLD, UNRESERVED_KEYWORD) PG_KEYWORD("hour", HOUR_P, UNRESERVED_KEYWORD) PG_KEYWORD("identity", IDENTITY_P, UNRESERVED_KEYWORD) PG_KEYWORD("if", IF_P, UNRESERVED_KEYWORD) +PG_KEYWORD("ignore", IGNORE, UNRESERVED_KEYWORD) PG_KEYWORD("ilike", ILIKE, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("immediate", IMMEDIATE, UNRESERVED_KEYWORD) PG_KEYWORD("immutable", IMMUTABLE, UNRESERVED_KEYWORD) @@ -312,6 +313,7 @@ PG_KEYWORD("repeatable", REPEATABLE, UNRESERVED_KEYWORD) PG_KEYWORD("replace", REPLACE, UNRESERVED_KEYWORD) PG_KEYWORD("replica", REPLICA, UNRESERVED_KEYWORD) PG_KEYWORD("reset", RESET, UNRESERVED_KEYWORD) +PG_KEYWORD("respect", RESPECT, UNRESERVED_KEYWORD) PG_KEYWORD("restart", RESTART, UNRESERVED_KEYWORD) PG_KEYWORD("restrict", RESTRICT, UNRESERVED_KEYWORD) PG_KEYWORD("returning", RETURNING, RESERVED_KEYWORD) diff --git a/src/include/windowapi.h b/src/include/windowapi.h index 5bbf1fa..81f5ba0 100644 --- a/src/include/windowapi.h +++ b/src/include/windowapi.h @@ -46,6 +46,8 @@ extern void *WinGetPartitionLocalMemory(WindowObject winobj, Size sz); extern int64 WinGetCurrentPosition(WindowObject winobj); extern int64 WinGetPartitionRowCount(WindowObject winobj); +extern int WinGetFrameOptions(WindowObject winobj); + extern void WinSetMarkPosition(WindowObject winobj, int64 markpos); extern bool WinRowsArePeers(WindowObject winobj, int64 pos1, int64 pos2); diff --git a/src/interfaces/ecpg/preproc/parse.pl b/src/interfaces/ecpg/preproc/parse.pl index f4b51d6..fe5dcb3 100644 --- a/src/interfaces/ecpg/preproc/parse.pl +++ b/src/interfaces/ecpg/preproc/parse.pl @@ -45,6 +45,8 @@ my %replace_string = ( 'WITH_TIME' => 'with time', 'NULLS_FIRST' => 'nulls first', 'NULLS_LAST' => 'nulls last', + 'RESPECT_NULLS' => 'respect nulls', + 'IGNORE_NULLS' => 'ignore nulls', 'TYPECAST' => '::', 'DOT_DOT' => '..', 'COLON_EQUALS' => ':=',); diff --git a/src/interfaces/ecpg/preproc/parser.c b/src/interfaces/ecpg/preproc/parser.c index 2ce9dd9..3780652 100644 --- a/src/interfaces/ecpg/preproc/parser.c +++ b/src/interfaces/ecpg/preproc/parser.c @@ -83,15 +83,7 @@ filtered_base_yylex(void) cur_token = NULLS_LAST; break; default: - /* save the lookahead token for next time */ - lookahead_token = next_token; - lookahead_yylval = base_yylval; - lookahead_yylloc = base_yylloc; - have_lookahead = true; - /* and back up the output info to cur_token */ - base_yylval = cur_yylval; - base_yylloc = cur_yylloc; - break; + goto restore_next_token; } break; @@ -109,15 +101,38 @@ filtered_base_yylex(void) cur_token = WITH_TIME; break; default: - /* save the lookahead token for next time */ - lookahead_token = next_token; - lookahead_yylval = base_yylval; - lookahead_yylloc = base_yylloc; - have_lookahead = true; - /* and back up the output info to cur_token */ - base_yylval = cur_yylval; - base_yylloc = cur_yylloc; + goto restore_next_token; + } + break; + + /* + * Window functions can use RESPECT NULLS or IGNORE NULLS to + * modify their behaviour + */ + case RESPECT: + cur_yylval = base_yylval; + cur_yylloc = base_yylloc; + next_token = base_yylex(); + switch (next_token) + { + case NULLS_P: + cur_token = RESPECT_NULLS; + break; + default: + goto restore_next_token; + } + break; + case IGNORE: + cur_yylval = base_yylval; + cur_yylloc = base_yylloc; + next_token = base_yylex(); + switch (next_token) + { + case NULLS_P: + cur_token = IGNORE_NULLS; break; + default: + goto restore_next_token; } break; @@ -126,4 +141,16 @@ filtered_base_yylex(void) } return cur_token; + +restore_next_token: + /* save the lookahead token for next time */ + lookahead_token = next_token; + lookahead_yylval = base_yylval; + lookahead_yylloc = base_yylloc; + have_lookahead = true; + /* and back up the output info to cur_token */ + base_yylval = cur_yylval; + base_yylloc = cur_yylloc; + + return cur_token; } diff --git a/src/test/regress/expected/window.out b/src/test/regress/expected/window.out index 752c7b4..47e3ba2 100644 --- a/src/test/regress/expected/window.out +++ b/src/test/regress/expected/window.out @@ -5,19 +5,21 @@ CREATE TEMPORARY TABLE empsalary ( depname varchar, empno bigint, salary int, - enroll_date date + enroll_date date, + term_date date, + respect text ); INSERT INTO empsalary VALUES -('develop', 10, 5200, '2007-08-01'), -('sales', 1, 5000, '2006-10-01'), -('personnel', 5, 3500, '2007-12-10'), -('sales', 4, 4800, '2007-08-08'), -('personnel', 2, 3900, '2006-12-23'), -('develop', 7, 4200, '2008-01-01'), -('develop', 9, 4500, '2008-01-01'), -('sales', 3, 4800, '2007-08-01'), -('develop', 8, 6000, '2006-10-01'), -('develop', 11, 5200, '2007-08-15'); +('develop', 10, 5200, '2007-08-01', null, null), +('sales', 1, 5000, '2006-10-01', null, 'frog'), +('personnel', 5, 3500, '2007-12-10', null, null), +('sales', 4, 4800, '2007-08-08', '2010-09-22', 'chicken'), +('personnel', 2, 3900, '2006-12-23', null, null), +('develop', 7, 4200, '2008-01-01', null, null), +('develop', 9, 4500, '2008-01-01', null, 'gorilla'), +('sales', 3, 4800, '2007-08-01', '2009-03-05', null), +('develop', 8, 6000, '2006-10-01', '2009-11-17', 'tiger'), +('develop', 11, 5200, '2007-08-15', null, null); SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) FROM empsalary ORDER BY depname, salary; depname | empno | salary | sum -----------+-------+--------+------- @@ -1020,5 +1022,114 @@ SELECT ntile(0) OVER (ORDER BY ten), ten, four FROM tenk1; ERROR: argument of ntile must be greater than zero SELECT nth_value(four, 0) OVER (ORDER BY ten), ten, four FROM tenk1; ERROR: argument of nth_value must be greater than zero +-- test null behaviour: (1) lags +SELECT term_date, lag(term_date) OVER (ORDER BY empno) FROM empsalary; + term_date | lag +------------+------------ + | + | + 03-05-2009 | + 09-22-2010 | 03-05-2009 + | 09-22-2010 + | + 11-17-2009 | + | 11-17-2009 + | + | +(10 rows) + +SELECT term_date, lag(term_date) RESPECT NULLS OVER (ORDER BY empno) FROM empsalary; + term_date | lag +------------+------------ + | + | + 03-05-2009 | + 09-22-2010 | 03-05-2009 + | 09-22-2010 + | + 11-17-2009 | + | 11-17-2009 + | + | +(10 rows) + +-- a numeric (date) column +SELECT term_date, lag(term_date) IGNORE NULLS OVER (ORDER BY empno) FROM empsalary; + term_date | lag +------------+------------ + | + | + 03-05-2009 | + 09-22-2010 | 03-05-2009 + | 09-22-2010 + | 09-22-2010 + 11-17-2009 | 09-22-2010 + | 11-17-2009 + | 11-17-2009 + | 11-17-2009 +(10 rows) + +-- a text column +SELECT respect, lag(respect) IGNORE NULLS OVER (ORDER BY empno) FROM empsalary; + respect | lag +---------+--------- + frog | + | frog + | frog + chicken | frog + | chicken + | chicken + tiger | chicken + gorilla | tiger + | gorilla + | gorilla +(10 rows) + +-- (2) leads +SELECT term_date, lead(term_date) OVER (ORDER BY empno) FROM empsalary; + term_date | lead +------------+------------ + | + | 03-05-2009 + 03-05-2009 | 09-22-2010 + 09-22-2010 | + | + | 11-17-2009 + 11-17-2009 | + | + | + | +(10 rows) + +SELECT term_date, lead(term_date) RESPECT NULLS OVER (ORDER BY empno) FROM empsalary; + term_date | lead +------------+------------ + | + | 03-05-2009 + 03-05-2009 | 09-22-2010 + 09-22-2010 | + | + | 11-17-2009 + 11-17-2009 | + | + | + | +(10 rows) + +SELECT term_date, lead(term_date) IGNORE NULLS OVER (ORDER BY empno) FROM empsalary; + term_date | lead +------------+------------ + | + | 03-05-2009 + 03-05-2009 | 09-22-2010 + 09-22-2010 | 09-22-2010 + | 09-22-2010 + | 11-17-2009 + 11-17-2009 | 11-17-2009 + | 11-17-2009 + | 11-17-2009 + | 11-17-2009 +(10 rows) + -- cleanup DROP TABLE empsalary; diff --git a/src/test/regress/sql/window.sql b/src/test/regress/sql/window.sql index 769be0f..eb0817b 100644 --- a/src/test/regress/sql/window.sql +++ b/src/test/regress/sql/window.sql @@ -6,20 +6,22 @@ CREATE TEMPORARY TABLE empsalary ( depname varchar, empno bigint, salary int, - enroll_date date + enroll_date date, + term_date date, + respect text ); INSERT INTO empsalary VALUES -('develop', 10, 5200, '2007-08-01'), -('sales', 1, 5000, '2006-10-01'), -('personnel', 5, 3500, '2007-12-10'), -('sales', 4, 4800, '2007-08-08'), -('personnel', 2, 3900, '2006-12-23'), -('develop', 7, 4200, '2008-01-01'), -('develop', 9, 4500, '2008-01-01'), -('sales', 3, 4800, '2007-08-01'), -('develop', 8, 6000, '2006-10-01'), -('develop', 11, 5200, '2007-08-15'); +('develop', 10, 5200, '2007-08-01', null, null), +('sales', 1, 5000, '2006-10-01', null, 'frog'), +('personnel', 5, 3500, '2007-12-10', null, null), +('sales', 4, 4800, '2007-08-08', '2010-09-22', 'chicken'), +('personnel', 2, 3900, '2006-12-23', null, null), +('develop', 7, 4200, '2008-01-01', null, null), +('develop', 9, 4500, '2008-01-01', null, 'gorilla'), +('sales', 3, 4800, '2007-08-01', '2009-03-05', null), +('develop', 8, 6000, '2006-10-01', '2009-11-17', 'tiger'), +('develop', 11, 5200, '2007-08-15', null, null); SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) FROM empsalary ORDER BY depname, salary; @@ -264,5 +266,25 @@ SELECT ntile(0) OVER (ORDER BY ten), ten, four FROM tenk1; SELECT nth_value(four, 0) OVER (ORDER BY ten), ten, four FROM tenk1; +-- test null behaviour: (1) lags + +SELECT term_date, lag(term_date) OVER (ORDER BY empno) FROM empsalary; + +SELECT term_date, lag(term_date) RESPECT NULLS OVER (ORDER BY empno) FROM empsalary; + +-- a numeric (date) column +SELECT term_date, lag(term_date) IGNORE NULLS OVER (ORDER BY empno) FROM empsalary; + +-- a text column +SELECT respect, lag(respect) IGNORE NULLS OVER (ORDER BY empno) FROM empsalary; + +-- (2) leads + +SELECT term_date, lead(term_date) OVER (ORDER BY empno) FROM empsalary; + +SELECT term_date, lead(term_date) RESPECT NULLS OVER (ORDER BY empno) FROM empsalary; + +SELECT term_date, lead(term_date) IGNORE NULLS OVER (ORDER BY empno) FROM empsalary; + -- cleanup DROP TABLE empsalary;