From ca1c8814a1026d018bdae54e64d682e32e82758e Mon Sep 17 00:00:00 2001 From: Hou Zhijie Date: Tue, 11 Jul 2023 19:55:55 +0800 Subject: [PATCH] test deparser module regress twice When running the regression test(tests in parallel_schedule), we replace the executing ddl statement with the its deparsed version and execute the deparsed statement, so that we can run all the regression with the deparsed statement and can expect the output to be the same as the existing expected/*.out. As developers typically add new regression tests to test new syntax, so we expect this test can automatically identify any new syntax changes. To achieve this, the strategy is to run the regression test twice. The first run will create event triggers to deparse ddl statements, it is intended to collect all the deparsed statements and can catch ERRORs/WARNINGs caused by the deparser. We can dump these statements from the database and then reload them in the second regression run. This allows us to utilize the deparsed statements to replace the local statements in the second regression run. This approach does not need to handle any remote messages and client variable stuff during execution, although it could take more time to finsh the test. --- src/test/modules/Makefile | 1 + src/test/modules/test_deparser/Makefile | 57 ++++ .../modules/test_deparser/exclusion_schedule | 7 + .../test_deparser/expected/copy_cmd.out | 3 + .../expected/test_deparser_1.out | 34 ++ .../expected/test_deparser_2.out | 18 + src/test/modules/test_deparser/meson.build | 24 ++ .../modules/test_deparser/sql/copy_cmd.sql | 3 + .../test_deparser/sql/test_deparser_1.sql | 40 +++ .../test_deparser/sql/test_deparser_2.sql | 16 + .../test_deparser/test_deparser--1.0.sql | 9 + .../modules/test_deparser/test_deparser.c | 310 ++++++++++++++++++ .../modules/test_deparser/test_deparser.conf | 1 + .../test_deparser/test_deparser.control | 4 + 14 files changed, 527 insertions(+) create mode 100644 src/test/modules/test_deparser/Makefile create mode 100644 src/test/modules/test_deparser/exclusion_schedule create mode 100644 src/test/modules/test_deparser/expected/copy_cmd.out create mode 100644 src/test/modules/test_deparser/expected/test_deparser_1.out create mode 100644 src/test/modules/test_deparser/expected/test_deparser_2.out create mode 100644 src/test/modules/test_deparser/meson.build create mode 100644 src/test/modules/test_deparser/sql/copy_cmd.sql create mode 100644 src/test/modules/test_deparser/sql/test_deparser_1.sql create mode 100644 src/test/modules/test_deparser/sql/test_deparser_2.sql create mode 100644 src/test/modules/test_deparser/test_deparser--1.0.sql create mode 100644 src/test/modules/test_deparser/test_deparser.c create mode 100644 src/test/modules/test_deparser/test_deparser.conf create mode 100644 src/test/modules/test_deparser/test_deparser.control diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 6331c976dc..cad68a59bc 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -17,6 +17,7 @@ SUBDIRS = \ test_bloomfilter \ test_copy_callbacks \ test_custom_rmgrs \ + test_deparser \ test_ddl_deparse \ test_extensions \ test_ginpostinglist \ diff --git a/src/test/modules/test_deparser/Makefile b/src/test/modules/test_deparser/Makefile new file mode 100644 index 0000000000..78a3ea0150 --- /dev/null +++ b/src/test/modules/test_deparser/Makefile @@ -0,0 +1,57 @@ +# src/test/modules/test_deparser/Makefile + +MODULES = test_deparser +PGFILEDESC = "test_deparser - regression testing for DDL deparsing" + +EXTENSION = test_deparser +DATA = test_deparser--1.0.sql + +EXTRA_CLEAN = $(pg_regress_clean_files) deparse_init_schedule deparse_regress_schedule + +MODULE_big = test_deparser +OBJS = \ + $(WIN32RES) \ + test_deparser.o + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_deparser +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +MINIMAL_REGRESS = test_setup \ + create_index \ + create_table \ + alter_table + +REGRESS_OPTS += --load-extension=test_deparser --dlpath=$(top_builddir)/src/test/regress \ + --temp-config $(top_srcdir)/src/test/modules/test_deparser/test_deparser.conf \ + --inputdir=$(top_srcdir)/src/test/regress + +deparse_init: all deparse_init_schedule + $(pg_regress_check) $(REGRESS_OPTS) --schedule=deparse_init_schedule + +deparse_regress: all deparse_regress_schedule + $(pg_regress_check) $(REGRESS_OPTS) --schedule=deparse_regress_schedule + +check: all deparse_init deparse_regress + +checkminimal: all + $(pg_regress_check) $(REGRESS_OPTS) test_deparser_1 $(MINIMAL_REGRESS) copy_cmd + $(pg_regress_check) $(REGRESS_OPTS) test_deparser_2 $(MINIMAL_REGRESS) + +deparse_init_schedule: + echo "test: test_deparser_1" > $@ + cat $(top_srcdir)/src/test/regress/parallel_schedule >> $@ + echo "test: copy_cmd" >> $@ + sed 's/\(.*\)/s\/\\b\1\\b\/\/g/' exclusion_schedule | sed -f - -i $@ + +deparse_regress_schedule: + echo "test: test_deparser_2" > $@ + cat $(top_srcdir)/src/test/regress/parallel_schedule >> $@ + sed 's/\(.*\)/s\/\\b\1\\b\/\/g/' exclusion_schedule | sed -f - -i $@ diff --git a/src/test/modules/test_deparser/exclusion_schedule b/src/test/modules/test_deparser/exclusion_schedule new file mode 100644 index 0000000000..243bb66d44 --- /dev/null +++ b/src/test/modules/test_deparser/exclusion_schedule @@ -0,0 +1,7 @@ + +# For tests that don't include any DDL statement or can +# give unexpected output because of the extra event trigger functions and table +# created in this test, we skipping running these tests. +opr_sanity +misc_sanity +event_trigger diff --git a/src/test/modules/test_deparser/expected/copy_cmd.out b/src/test/modules/test_deparser/expected/copy_cmd.out new file mode 100644 index 0000000000..cefc60ba4b --- /dev/null +++ b/src/test/modules/test_deparser/expected/copy_cmd.out @@ -0,0 +1,3 @@ +\getenv abs_builddir PG_ABS_BUILDDIR +\set filename :abs_builddir '/results/deparse_test_commands.data' +COPY deparse_test_commands TO :'filename'; diff --git a/src/test/modules/test_deparser/expected/test_deparser_1.out b/src/test/modules/test_deparser/expected/test_deparser_1.out new file mode 100644 index 0000000000..158ff55366 --- /dev/null +++ b/src/test/modules/test_deparser/expected/test_deparser_1.out @@ -0,0 +1,34 @@ +CREATE ROLE deparse_role SUPERUSER; +CREATE OR REPLACE FUNCTION public.deparse_test_ddl_command_end() + RETURNS event_trigger + SECURITY DEFINER + LANGUAGE plpgsql +AS $fn$ +BEGIN + BEGIN + INSERT INTO pg_catalog.deparse_test_commands + (id, test_name, deparse_time, original_command, command) + SELECT public.tdparser_get_cmdcount(), + current_setting('application_name'), clock_timestamp(), current_query(), + pg_catalog.ddl_deparse_expand_command(pg_catalog.ddl_deparse_to_json(command)) + FROM pg_event_trigger_ddl_commands() WITH ORDINALITY; + + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'state: % errm: %', sqlstate, sqlerrm; + END; +END; +$fn$; +SET allow_system_table_mods = on; +CREATE UNLOGGED TABLE pg_catalog.deparse_test_commands ( + id int, + test_name text, + deparse_time timestamptz, + original_command TEXT NOT NULL, + command TEXT +); +CREATE EVENT TRIGGER deparse_test_trg_sql_drop + ON sql_drop + EXECUTE PROCEDURE test_deparser_drop_command(); +CREATE EVENT TRIGGER deparse_test_trg_ddl_command_end + ON ddl_command_end WHEN TAG IN ('CREATE TABLE', 'ALTER TABLE') + EXECUTE PROCEDURE deparse_test_ddl_command_end(); diff --git a/src/test/modules/test_deparser/expected/test_deparser_2.out b/src/test/modules/test_deparser/expected/test_deparser_2.out new file mode 100644 index 0000000000..5f069337c5 --- /dev/null +++ b/src/test/modules/test_deparser/expected/test_deparser_2.out @@ -0,0 +1,18 @@ +\getenv abs_builddir PG_ABS_BUILDDIR +\set filename :abs_builddir '/results/deparse_test_commands.data' +SET allow_system_table_mods = on; +CREATE TABLE pg_catalog.deparse_test_commands ( + id int, + test_name text, + deparse_time timestamptz, + original_command TEXT, + command TEXT +); +COPY pg_catalog.deparse_test_commands FROM :'filename'; +ALTER SYSTEM SET test_deparser.deparse_mode = on; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + diff --git a/src/test/modules/test_deparser/meson.build b/src/test/modules/test_deparser/meson.build new file mode 100644 index 0000000000..b57553fa5a --- /dev/null +++ b/src/test/modules/test_deparser/meson.build @@ -0,0 +1,24 @@ +# Copyright (c) 2022-2023, PostgreSQL Global Development Group + +test_deparser_sources = files( + 'test_deparser.c', +) + +if host_system == 'windows' + test_deparser_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_deparser', + '--FILEDESC', 'test_deparser - allow delay between parsing and execution',]) +endif + +test_deparser = shared_module('test_deparser', + test_deparser_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_deparser + +test_install_data += files( + 'test_deparser.control', + 'test_deparser--1.0.sql', +) + +# TODO regression tests diff --git a/src/test/modules/test_deparser/sql/copy_cmd.sql b/src/test/modules/test_deparser/sql/copy_cmd.sql new file mode 100644 index 0000000000..cefc60ba4b --- /dev/null +++ b/src/test/modules/test_deparser/sql/copy_cmd.sql @@ -0,0 +1,3 @@ +\getenv abs_builddir PG_ABS_BUILDDIR +\set filename :abs_builddir '/results/deparse_test_commands.data' +COPY deparse_test_commands TO :'filename'; diff --git a/src/test/modules/test_deparser/sql/test_deparser_1.sql b/src/test/modules/test_deparser/sql/test_deparser_1.sql new file mode 100644 index 0000000000..1d289b5f92 --- /dev/null +++ b/src/test/modules/test_deparser/sql/test_deparser_1.sql @@ -0,0 +1,40 @@ + +CREATE ROLE deparse_role SUPERUSER; + +CREATE OR REPLACE FUNCTION public.deparse_test_ddl_command_end() + RETURNS event_trigger + SECURITY DEFINER + LANGUAGE plpgsql +AS $fn$ +BEGIN + BEGIN + INSERT INTO pg_catalog.deparse_test_commands + (id, test_name, deparse_time, original_command, command) + SELECT public.tdparser_get_cmdcount(), + current_setting('application_name'), clock_timestamp(), current_query(), + pg_catalog.ddl_deparse_expand_command(pg_catalog.ddl_deparse_to_json(command)) + FROM pg_event_trigger_ddl_commands() WITH ORDINALITY; + + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'state: % errm: %', sqlstate, sqlerrm; + END; +END; +$fn$; + +SET allow_system_table_mods = on; + +CREATE UNLOGGED TABLE pg_catalog.deparse_test_commands ( + id int, + test_name text, + deparse_time timestamptz, + original_command TEXT NOT NULL, + command TEXT +); + +CREATE EVENT TRIGGER deparse_test_trg_sql_drop + ON sql_drop + EXECUTE PROCEDURE test_deparser_drop_command(); + +CREATE EVENT TRIGGER deparse_test_trg_ddl_command_end + ON ddl_command_end WHEN TAG IN ('CREATE TABLE', 'ALTER TABLE') + EXECUTE PROCEDURE deparse_test_ddl_command_end(); diff --git a/src/test/modules/test_deparser/sql/test_deparser_2.sql b/src/test/modules/test_deparser/sql/test_deparser_2.sql new file mode 100644 index 0000000000..3bc3400b4f --- /dev/null +++ b/src/test/modules/test_deparser/sql/test_deparser_2.sql @@ -0,0 +1,16 @@ + +\getenv abs_builddir PG_ABS_BUILDDIR +\set filename :abs_builddir '/results/deparse_test_commands.data' + +SET allow_system_table_mods = on; +CREATE TABLE pg_catalog.deparse_test_commands ( + id int, + test_name text, + deparse_time timestamptz, + original_command TEXT, + command TEXT +); + +COPY pg_catalog.deparse_test_commands FROM :'filename'; +ALTER SYSTEM SET test_deparser.deparse_mode = on; +SELECT pg_reload_conf(); diff --git a/src/test/modules/test_deparser/test_deparser--1.0.sql b/src/test/modules/test_deparser/test_deparser--1.0.sql new file mode 100644 index 0000000000..b42f2559e6 --- /dev/null +++ b/src/test/modules/test_deparser/test_deparser--1.0.sql @@ -0,0 +1,9 @@ +\echo Use "CREATE EXTENSION test_deparser" to load this file. \quit + +CREATE FUNCTION tdparser_get_cmdcount() + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_deparser_drop_command() +RETURNS event_trigger STRICT SECURITY DEFINER +AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_deparser/test_deparser.c b/src/test/modules/test_deparser/test_deparser.c new file mode 100644 index 0000000000..ee55e4e4d5 --- /dev/null +++ b/src/test/modules/test_deparser/test_deparser.c @@ -0,0 +1,310 @@ +/*------------------------------------------------------------------------- + * + * test_deparser.c + * Test DDL deparser + * + * Copyright (c) 2020-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_deparser/test_deparser.c + * + * When running the regression test(tests in parallel_schedule), we replace the + * executing ddl statement with the its deparsed version and execute the + * deparsed statement, so that we can run all the regression with the deparsed + * statement and can expect the output to be the same as the existing + * expected *.out. As developers typically add new regression tests to test new + * syntax, so we expect this test can automatically identify any new syntax + * changes. + * + * The strategy is to run the regression test twice. The first run will create + * event triggers to deparse ddl statements, it is intended to collect all the + * deparsed statements and can catch ERRORs/WARNINGs caused by the deparser. We + * can dump these statements from the database and then reload them in the + * second regression run. This allows us to utilize the deparsed statements to + * replace the local statements in the second regression run. This approach + * does not need to handle any remote messages and client variable stuff during + * execution, although it could take more time to finsh the test. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "catalog/dependency.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_class.h" +#include "catalog/pg_database.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_proc.h" +#include "commands/event_trigger.h" +#include "commands/seclabel.h" +#include "executor/executor.h" +#include "executor/spi.h" +#include "fmgr.h" +#include "miscadmin.h" +#include "tcop/ddldeparse.h" +#include "tcop/utility.h" +#include "utils/builtins.h" +#include "utils/guc.h" +#include "utils/portal.h" +#include "utils/queryenvironment.h" + +PG_MODULE_MAGIC; + +static ProcessUtility_hook_type prev_ProcessUtility = NULL; +static ExecutorRun_hook_type prev_ExecutorRun = NULL; +extern EventTriggerQueryState *currentEventTriggerState; + +static void tdeparser_ProcessUtility(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc); + +static void tdeparser_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, + bool execute_once); + +static int nesting_level = 0; +static int cmd_count = 0; +static bool deparse_mode = false; + + +static List * +change_deparsed_stmt(PlannedStmt *pstmt, const char *queryString) +{ + List *nstmt_list = NIL; + List *parsetree_list; + ListCell *lc; + Oid save_userid = 0; + const char *deparsed_cmd = queryString; + int save_sec_context = 0; + int i; + SPITupleTable *tuptable; + StringInfoData cmd; + + if (!deparse_mode) + return list_make1(pstmt); + + initStringInfo(&cmd); + appendStringInfo(&cmd, "SELECT command, id, deparse_time from pg_catalog.deparse_test_commands " + "WHERE id = %d AND test_name = '%s' AND original_command = current_query() order by deparse_time", + cmd_count, application_name); + elog(LOG, "ID: %d", cmd_count); + + /* Change to superuser to avoid access deny. */ + GetUserIdAndSecContext(&save_userid, &save_sec_context); + SetUserIdAndSecContext(BOOTSTRAP_SUPERUSERID, + save_sec_context | SECURITY_LOCAL_USERID_CHANGE); + + SPI_connect(); + + /* Query the deparsed statement. */ + if (SPI_exec(cmd.data, 1024) != SPI_OK_SELECT) + elog(ERROR, "SPI_exec failed: %s", cmd.data); + + if (SPI_tuptable == NULL) + elog(ERROR, "could not find deparsed statement"); + + tuptable = SPI_tuptable; + + /* + * Return if the statement was not deparsed which means this statement is + * expected to fail. + */ + if (tuptable->numvals == 0) + { + SetUserIdAndSecContext(save_userid, save_sec_context); + elog(LOG, "no deparsed statement"); + SPI_finish(); + return list_make1(pstmt); + } + + resetStringInfo(&cmd); + + for (i = 0 ; i < tuptable->numvals; i++) + { + char *value = SPI_getvalue(tuptable->vals[i], tuptable->tupdesc, 1); + appendStringInfo(&cmd, "%s;", value ? value : " "); + } + + elog(LOG, "EXECQ: %s", cmd.data); + + deparsed_cmd = cmd.data; + + SPI_finish(); + + SetUserIdAndSecContext(save_userid, save_sec_context); + + parsetree_list = pg_parse_query(deparsed_cmd); + + /* + * One statment could be deparsed into mulitiple statements, for example: + * "DROP TABLE t1,t2" will be deparsed into "DROP TABLE t1" AND "DROP TABLE + * t2". So we need to maintain them in a list. + */ + foreach(lc, parsetree_list) + { + List *plans; + RawStmt *rs = lfirst_node(RawStmt, lc); + List *querytree_list = pg_analyze_and_rewrite_fixedparams(rs, deparsed_cmd, + NULL, 0, NULL); + + Assert(list_length(querytree_list) == 1); + + plans = pg_plan_queries(querytree_list, deparsed_cmd, + CURSOR_OPT_PARALLEL_OK, NULL); + + Assert(list_length(plans) == 1); + + nstmt_list = lappend(nstmt_list, linitial(plans)); + } + + return nstmt_list; +} + +/* + * ProcessUtility hook + */ +static void +tdeparser_ProcessUtility(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc) +{ + List *nplan_list; + ListCell *lc; + CommandTag tag = CreateCommandTag(pstmt->utilityStmt); + + nesting_level++; + PG_TRY(); + { + if (nesting_level == 1 && command_tag_event_trigger_ok(tag)) + nplan_list = change_deparsed_stmt(pstmt, queryString); + else + nplan_list = list_make1(pstmt); + + foreach(lc, nplan_list) + { + PlannedStmt *plan = (PlannedStmt *) lfirst(lc); + if (prev_ProcessUtility) + prev_ProcessUtility(plan, queryString, readOnlyTree, + context, params, queryEnv, + dest, qc); + else + standard_ProcessUtility(plan, queryString, readOnlyTree, + context, params, queryEnv, + dest, qc); + } + } + PG_FINALLY(); + { + if (nesting_level == 1) + cmd_count++; + + nesting_level--; + } + PG_END_TRY(); +} + +/* + * ExecutorRun hook: all we need do is track nesting depth + */ +static void +tdeparser_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, + bool execute_once) +{ + nesting_level++; + PG_TRY(); + { + if (prev_ExecutorRun) + prev_ExecutorRun(queryDesc, direction, count, execute_once); + else + standard_ExecutorRun(queryDesc, direction, count, execute_once); + } + PG_FINALLY(); + { + nesting_level--; + } + PG_END_TRY(); +} + +PG_FUNCTION_INFO_V1(tdparser_get_cmdcount); +PG_FUNCTION_INFO_V1(test_deparser_drop_command); + +Datum +tdparser_get_cmdcount(PG_FUNCTION_ARGS) +{ + PG_RETURN_INT32(cmd_count); +} + +Datum +test_deparser_drop_command(PG_FUNCTION_ARGS) +{ + slist_iter iter; + + /* Drop commands are not part commandlist but handled here as part of SQLDropList */ + slist_foreach(iter, &(currentEventTriggerState->SQLDropList)) + { + SQLDropObject *obj; + EventTriggerData *trigdata; + + trigdata = (EventTriggerData *) fcinfo->context; + + obj = slist_container(SQLDropObject, next, iter.cur); + + if (!obj->original) + continue; + + if (strcmp(obj->objecttype, "table") == 0) + { + char *command; + + command = deparse_drop_table(obj->objidentity, trigdata->parsetree); + if (command) + { + StringInfoData cmd; + + command = deparse_ddl_json_to_string(command); + initStringInfo(&cmd); + + appendStringInfo(&cmd, "INSERT INTO pg_catalog.deparse_test_commands " + "(id, test_name, deparse_time, original_command, command) " + "SELECT public.tdparser_get_cmdcount(), " + "current_setting('application_name'), " + "clock_timestamp(), current_query(), '%s'", command); + + /* insert deparsed statement into table */ + SPI_connect(); + if (SPI_exec(cmd.data, 8) != SPI_OK_INSERT) + elog(ERROR, "SPI_exec failed: %s", cmd.data); + SPI_finish(); + } + } + } + + return PointerGetDatum(NULL); +} + +/* Module load function */ +void +_PG_init(void) +{ + DefineCustomBoolVariable("test_deparser.deparse_mode", + "Change the statement to deparsed one", + NULL, + &deparse_mode, + false, + PGC_SUSET, + 0, + NULL, + NULL, + NULL); + + prev_ProcessUtility = ProcessUtility_hook; + ProcessUtility_hook = tdeparser_ProcessUtility; + + prev_ExecutorRun = ExecutorRun_hook; + ExecutorRun_hook = tdeparser_ExecutorRun; +} diff --git a/src/test/modules/test_deparser/test_deparser.conf b/src/test/modules/test_deparser/test_deparser.conf new file mode 100644 index 0000000000..35865b5c5a --- /dev/null +++ b/src/test/modules/test_deparser/test_deparser.conf @@ -0,0 +1 @@ +session_preload_libraries = 'test_deparser' diff --git a/src/test/modules/test_deparser/test_deparser.control b/src/test/modules/test_deparser/test_deparser.control new file mode 100644 index 0000000000..c3e9eaf064 --- /dev/null +++ b/src/test/modules/test_deparser/test_deparser.control @@ -0,0 +1,4 @@ +comment = 'Test code for DDL deparse feature' +default_version = '1.0' +module_pathname = '$libdir/test_deparser' +relocatable = true \ No newline at end of file -- 2.30.0.windows.2