From f31d962c1d1101b7d90ab1956e18b4c94eda63b8 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Fri, 18 Feb 2022 15:13:54 +0100 Subject: [PATCH v2] pg_regress TAP output format --- src/test/regress/pg_regress.c | 396 +++++++++++++++++++++++++++------- 1 file changed, 317 insertions(+), 79 deletions(-) diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index e6f71c7582..81f3bb7bab 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -95,6 +95,8 @@ static char *dlpath = PKGLIBDIR; static char *user = NULL; static _stringlist *extraroles = NULL; static char *config_auth_datadir = NULL; +static char *format = NULL; +static char *psql_formatting = NULL; /* internal variables */ static const char *progname; @@ -120,11 +122,73 @@ static int fail_ignore_count = 0; static bool directory_exists(const char *dir); static void make_directory(const char *dir); -static void header(const char *fmt,...) pg_attribute_printf(1, 2); +struct output_func +{ + void (*header)(const char *line); + void (*footer)(const char *difffilename, const char *logfilename); + void (*comment)(const char *comment); + + void (*test_status_preamble)(const char *testname); + + void (*test_status_ok)(const char *testname); + void (*test_status_failed)(const char *testname); + void (*test_status_ignored)(const char *testname); + + void (*test_runtime)(const char *testname, double runtime); +}; + + +void (*test_runtime)(const char *testname, double runtime); +/* Text output format */ +static void header_text(const char *line); +static void footer_text(const char *difffilename, const char *logfilename); +static void comment_text(const char *comment); +static void test_status_preamble_text(const char *testname); +static void test_status_ok_text(const char *testname); +static void test_status_failed_text(const char *testname); +static void test_runtime_text(const char *testname, double runtime); + +struct output_func output_func_text = +{ + header_text, + footer_text, + comment_text, + test_status_preamble_text, + test_status_ok_text, + test_status_failed_text, + NULL, + test_runtime_text +}; + +/* TAP output format */ +static void header_tap(const char *line); +static void footer_tap(const char *difffilename, const char *logfilename); +static void comment_tap(const char *comment); +static void test_status_ok_tap(const char *testname); +static void test_status_failed_tap(const char *testname); +static void test_status_ignored_tap(const char *testname); + +struct output_func output_func_tap = +{ + header_tap, + footer_tap, + comment_tap, + NULL, + test_status_ok_tap, + test_status_failed_tap, + test_status_ignored_tap, + NULL +}; + +struct output_func *output = &output_func_text; + +static void test_status_ok(const char *testname); + static void status(const char *fmt,...) pg_attribute_printf(1, 2); static StringInfo psql_start_command(void); static void psql_add_command(StringInfo buf, const char *query,...) pg_attribute_printf(2, 3); static void psql_end_command(StringInfo buf, const char *database); +static void status_end(void); /* * allow core files if possible. @@ -208,18 +272,214 @@ split_to_stringlist(const char *s, const char *delim, _stringlist **listhead) /* * Print a progress banner on stdout. */ +static void +header_text(const char *line) +{ + fprintf(stdout, "============== %-38s ==============\n", line); + fflush(stdout); +} + +static void +header_tap(const char *line) +{ + fprintf(stdout, "# %s\n", line); + fflush(stdout); +} + static void header(const char *fmt,...) { char tmp[64]; va_list ap; + if (!output->header) + return; + va_start(ap, fmt); vsnprintf(tmp, sizeof(tmp), fmt, ap); va_end(ap); - fprintf(stdout, "============== %-38s ==============\n", tmp); - fflush(stdout); + output->header(tmp); +} + +static void +footer_tap(const char *difffilename, const char *logfilename) +{ + status("1..%i\n", (fail_count + fail_ignore_count + success_count)); + status_end(); +} + +static void +footer(const char *difffilename, const char *logfilename) +{ + if (output->footer) + output->footer(difffilename, logfilename); +} + +static void +comment_text(const char *comment) +{ + status("%s", comment); +} + +static void +comment_tap(const char *comment) +{ + status("# %s", comment); +} + +static void +comment(const char *fmt,...) +{ + char tmp[256]; + va_list ap; + + if (!output->comment) + return; + + va_start(ap, fmt); + vsnprintf(tmp, sizeof(tmp), fmt, ap); + va_end(ap); + + output->comment(tmp); +} + +static void +test_status_preamble_text(const char *testname) +{ + status(_("test %-28s ... "), testname); +} + +static void +test_status_preamble(const char *testname) +{ + if (output->test_status_preamble) + output->test_status_preamble(testname); +} + +static void +test_status_ok_tap(const char *testname) +{ + /* There is no NLS translation here as "ok" is a protocol message */ + status("ok %i - %s", + (fail_count + fail_ignore_count + success_count), + testname); +} + +static void +test_status_ok_text(const char *testname) +{ + (void) testname; /* unused */ + status(_("ok ")); /* align with FAILED */ +} + +static void +test_status_ok(const char *testname) +{ + success_count++; + if (output->test_status_ok) + output->test_status_ok(testname); +} + +static void +test_status_failed_tap(const char *testname) +{ + status("not ok %i - %s", + (fail_count + fail_ignore_count + success_count), + testname); +} + +static void +test_status_failed_text(const char *testname) +{ + status(_("FAILED")); +} + +static void +test_status_failed(const char *testname) +{ + fail_count++; + if (output->test_status_failed) + output->test_status_failed(testname); +} + +static void +test_status_ignored(const char *testname) +{ + fail_ignore_count++; + if (output->test_status_ignored) + output->test_status_ignored(testname); +} + +static void +test_status_ignored_tap(const char *testname) +{ + status("ok %i - %s # SKIP (ignored)", + (fail_count + fail_ignore_count + success_count), + testname); +} + +static void +test_runtime_text(const char *testname, double runtime) +{ + (void)testname; + status(_(" %8.0f ms"), runtime); +} + +static void +runtime(const char *testname, double runtime) +{ + if (output->test_runtime) + output->test_runtime(testname, runtime); +} + +static void +footer_text(const char *difffilename, const char *logfilename) +{ + char buf[256]; + + /* + * Emit nice-looking summary message + */ + if (fail_count == 0 && fail_ignore_count == 0) + snprintf(buf, sizeof(buf), + _(" All %d tests passed. "), + success_count); + else if (fail_count == 0) /* fail_count=0, fail_ignore_count>0 */ + snprintf(buf, sizeof(buf), + _(" %d of %d tests passed, %d failed test(s) ignored. "), + success_count, + success_count + fail_ignore_count, + fail_ignore_count); + else if (fail_ignore_count == 0) /* fail_count>0 && fail_ignore_count=0 */ + snprintf(buf, sizeof(buf), + _(" %d of %d tests failed. "), + fail_count, + success_count + fail_count); + else + /* fail_count>0 && fail_ignore_count>0 */ + snprintf(buf, sizeof(buf), + _(" %d of %d tests failed, %d of these failures ignored. "), + fail_count + fail_ignore_count, + success_count + fail_count + fail_ignore_count, + fail_ignore_count); + + putchar('\n'); + for (int i = strlen(buf); i > 0; i--) + putchar('='); + printf("\n%s\n", buf); + for (int i = strlen(buf); i > 0; i--) + putchar('='); + putchar('\n'); + putchar('\n'); + + if (difffilename && logfilename) + { + printf(_("The differences that caused some tests to fail can be viewed in the\n" + "file \"%s\". A copy of the test summary that you see\n" + "above is saved in the file \"%s\".\n\n"), + difffilename, logfilename); + } } /* @@ -752,13 +1012,13 @@ initialize_environment(void) #endif if (pghost && pgport) - printf(_("(using postmaster on %s, port %s)\n"), pghost, pgport); + comment(_("(using postmaster on %s, port %s)\n"), pghost, pgport); if (pghost && !pgport) - printf(_("(using postmaster on %s, default port)\n"), pghost); + comment(_("(using postmaster on %s, default port)\n"), pghost); if (!pghost && pgport) - printf(_("(using postmaster on Unix socket, port %s)\n"), pgport); + comment(_("(using postmaster on Unix socket, port %s)\n"), pgport); if (!pghost && !pgport) - printf(_("(using postmaster on Unix socket, default port)\n")); + comment(_("(using postmaster on Unix socket, default port)\n")); } load_resultmap(); @@ -984,9 +1244,10 @@ psql_start_command(void) StringInfo buf = makeStringInfo(); appendStringInfo(buf, - "\"%s%spsql\" -X", + "\"%s%spsql\" -X %s", bindir ? bindir : "", - bindir ? "/" : ""); + bindir ? "/" : "", + psql_formatting ? psql_formatting : ""); return buf; } @@ -1585,6 +1846,9 @@ run_schedule(const char *schedule, test_start_function startfunc, c++; add_stringlist_item(&ignorelist, c); + test_status_ignored(c); + status_end(); + /* * Note: ignore: lines do not run the test, they just say that * failure of this test when run later on is to be ignored. A bit @@ -1643,7 +1907,7 @@ run_schedule(const char *schedule, test_start_function startfunc, if (num_tests == 1) { - status(_("test %-28s ... "), tests[0]); + test_status_preamble(tests[0]); pids[0] = (startfunc) (tests[0], &resultfiles[0], &expectfiles[0], &tags[0]); INSTR_TIME_SET_CURRENT(starttimes[0]); wait_for_tests(pids, statuses, stoptimes, NULL, 1); @@ -1659,8 +1923,8 @@ run_schedule(const char *schedule, test_start_function startfunc, { int oldest = 0; - status(_("parallel group (%d tests, in groups of %d): "), - num_tests, max_connections); + comment(_("parallel group (%d tests, in groups of %d): "), + num_tests, max_connections); for (i = 0; i < num_tests; i++) { if (i - oldest >= max_connections) @@ -1680,7 +1944,7 @@ run_schedule(const char *schedule, test_start_function startfunc, } else { - status(_("parallel group (%d tests): "), num_tests); + comment(_("parallel group (%d tests): "), num_tests); for (i = 0; i < num_tests; i++) { pids[i] = (startfunc) (tests[i], &resultfiles[i], &expectfiles[i], &tags[i]); @@ -1699,7 +1963,7 @@ run_schedule(const char *schedule, test_start_function startfunc, bool differ = false; if (num_tests > 1) - status(_(" %-28s ... "), tests[i]); + test_status_preamble(tests[i]); /* * Advance over all three lists simultaneously. @@ -1739,27 +2003,18 @@ run_schedule(const char *schedule, test_start_function startfunc, } } if (ignore) - { - status(_("failed (ignored)")); - fail_ignore_count++; - } + test_status_ignored(tests[i]); else - { - status(_("FAILED")); - fail_count++; - } + test_status_failed(tests[i]); } else - { - status(_("ok ")); /* align with FAILED */ - success_count++; - } + test_status_ok(tests[i]); if (statuses[i] != 0) log_child_failure(statuses[i]); INSTR_TIME_SUBTRACT(stoptimes[i], starttimes[i]); - status(_(" %8.0f ms"), INSTR_TIME_GET_MILLISEC(stoptimes[i])); + runtime(tests[i], INSTR_TIME_GET_MILLISEC(stoptimes[i])); status_end(); } @@ -1798,7 +2053,7 @@ run_single_test(const char *test, test_start_function startfunc, *tl; bool differ = false; - status(_("test %-28s ... "), test); + test_status_preamble(test); pid = (startfunc) (test, &resultfiles, &expectfiles, &tags); INSTR_TIME_SET_CURRENT(starttime); wait_for_tests(&pid, &exit_status, &stoptime, NULL, 1); @@ -1828,15 +2083,9 @@ run_single_test(const char *test, test_start_function startfunc, } if (differ) - { - status(_("FAILED")); - fail_count++; - } + test_status_failed(test); else - { - status(_("ok ")); /* align with FAILED */ - success_count++; - } + test_status_ok(test); if (exit_status != 0) log_child_failure(exit_status); @@ -1983,6 +2232,7 @@ help(void) printf(_(" --debug turn on debug mode in programs that are run\n")); printf(_(" --dlpath=DIR look for dynamic libraries in DIR\n")); printf(_(" --encoding=ENCODING use ENCODING as the encoding\n")); + printf(_(" --format=(regress|tap) output format to use\n")); printf(_(" -h, --help show this help, then exit\n")); printf(_(" --inputdir=DIR take input files from DIR (default \".\")\n")); printf(_(" --launcher=CMD use CMD as launcher of psql\n")); @@ -2046,6 +2296,7 @@ regression_main(int argc, char *argv[], {"load-extension", required_argument, NULL, 22}, {"config-auth", required_argument, NULL, 24}, {"max-concurrent-tests", required_argument, NULL, 25}, + {"format", required_argument, NULL, 26}, {NULL, 0, NULL, 0} }; @@ -2175,6 +2426,9 @@ regression_main(int argc, char *argv[], case 25: max_concurrent_tests = atoi(optarg); break; + case 26: + format = pg_strdup(optarg); + break; default: /* getopt_long already emitted a complaint */ fprintf(stderr, _("\nTry \"%s -h\" for more information.\n"), @@ -2201,6 +2455,24 @@ regression_main(int argc, char *argv[], exit(0); } + /* + * text (regress) is the default so we don't need to set any variables in + * that case. + */ + if (format) + { + if (strcmp(format, "tap") == 0) + { + output = &output_func_tap; + psql_formatting = pg_strdup("-q"); + } + else if (strcmp(format, "regress") != 0) + { + fprintf(stderr, _("\n%s: invalid format specified: \"%s\". Supported formats are \"tap\" and \"regress\"\n"), progname, format); + exit(2); + } + } + if (temp_instance && !port_specified_by_user) /* @@ -2523,54 +2795,20 @@ regression_main(int argc, char *argv[], fclose(logfile); - /* - * Emit nice-looking summary message - */ - if (fail_count == 0 && fail_ignore_count == 0) - snprintf(buf, sizeof(buf), - _(" All %d tests passed. "), - success_count); - else if (fail_count == 0) /* fail_count=0, fail_ignore_count>0 */ - snprintf(buf, sizeof(buf), - _(" %d of %d tests passed, %d failed test(s) ignored. "), - success_count, - success_count + fail_ignore_count, - fail_ignore_count); - else if (fail_ignore_count == 0) /* fail_count>0 && fail_ignore_count=0 */ - snprintf(buf, sizeof(buf), - _(" %d of %d tests failed. "), - fail_count, - success_count + fail_count); - else - /* fail_count>0 && fail_ignore_count>0 */ - snprintf(buf, sizeof(buf), - _(" %d of %d tests failed, %d of these failures ignored. "), - fail_count + fail_ignore_count, - success_count + fail_count + fail_ignore_count, - fail_ignore_count); - - putchar('\n'); - for (i = strlen(buf); i > 0; i--) - putchar('='); - printf("\n%s\n", buf); - for (i = strlen(buf); i > 0; i--) - putchar('='); - putchar('\n'); - putchar('\n'); - - if (file_size(difffilename) > 0) - { - printf(_("The differences that caused some tests to fail can be viewed in the\n" - "file \"%s\". A copy of the test summary that you see\n" - "above is saved in the file \"%s\".\n\n"), - difffilename, logfilename); - } - else + if (file_size(difffilename) <= 0) { unlink(difffilename); unlink(logfilename); + + free(difffilename); + difffilename = NULL; + free(logfilename); + logfilename = NULL; } + footer(difffilename, logfilename); + status_end(); + if (fail_count != 0) exit(1); -- 2.24.3 (Apple Git-128)