diff --git b/src/test/modules/warm/t/001_recovery.pl a/src/test/modules/warm/t/001_recovery.pl new file mode 100644 index 0000000..2a76830 --- /dev/null +++ a/src/test/modules/warm/t/001_recovery.pl @@ -0,0 +1,50 @@ +# Single-node test: run workload, crash, recover and run sanity check + +use strict; +use warnings; + +use TestLib; +use Test::More tests => 2; +use PostgresNode; + +my $node = get_new_node(); +$node->init; +$node->start; + +# Create a table, do some WARM updates and then restart +$node->safe_psql('postgres', + 'create table accounts (aid int unique, branch int, balance bigint) with (fillfactor=98)'); +$node->safe_psql('postgres', + 'create table history (aid int, delta int)'); +$node->safe_psql('postgres', + 'insert into accounts select generate_series(1,10000), (random()*1000)::int % 10, 0'); +$node->safe_psql('postgres', + 'create index accounts_bal_indx on accounts(balance)'); + +for( $a = 1; $a <= 1000; $a = $a + 1 ) { + my $aid1 = int(rand(10000)) + 1; + my $aid2 = int(rand(10000)) + 1; + my $balance = int(rand(99999)); + $node->safe_psql('postgres', + "begin; + update accounts set balance = balance + $balance where aid = $aid1; + update accounts set balance = balance - $balance where aid = $aid2; + insert into history values ($aid1, $balance); + insert into history values ($aid2, 0 - $balance); + end;"); +} + +# Verify that we read the same TS after crash recovery +$node->stop('immediate'); +$node->start; + +my $recovered_balance = $node->safe_psql('postgres', 'select sum(balance) from accounts'); +my $total_delta = $node->safe_psql('postgres', 'select sum(delta) from history'); + +# since delta is credited to one account and debited from the other, we expect +# the sum(balance) to stay zero. +is($recovered_balance, 0, 'balanace matches after recovery'); + +# A positive and a negative value is inserted in the history table. Hence the +# sum(delta) should remain zero. +is($total_delta, 0, 'sum(delta) matches after recovery'); diff --git b/src/test/modules/warm/t/002_warm_stress.pl a/src/test/modules/warm/t/002_warm_stress.pl new file mode 100644 index 0000000..a1a2371 --- /dev/null +++ a/src/test/modules/warm/t/002_warm_stress.pl @@ -0,0 +1,289 @@ +# Run varity of tests to check consistency of index access. +# +# These tests are primarily designed to test if WARM updates cause any +# inconsistency in the indexes. We use a pgbench-like setup with an "accounts" +# table and a "branches" table. But instead of a single "aid" column the +# pgbench_warm_accounts table has four additional columns. These columns have +# initial value as "aid * 10", "aid * 20", "aid * 30" and "aid * 40". And +# unlike the aid column, values in these columns do not remain static. The +# values are changed in a narrow change around the original value, such that +# they still remain distinct, even after updates. We also build indexes on +# these additional columns. +# +# This allows us to force WARM updates to the table, while accessing individual +# rows using these auxillary columns. If things are solid, we must not miss any +# row irrespective of which column we use to fetch the row. Also, the sum of +# balances in two tables should match at the end. +# +# We drop and recreate indexes concurrently and also run VACUUM and run +# consistency checks to ensure nothing breaks. The tests also aborts +# transactions, acquires share/update locks etc to check any negative effects +# of those things. + +use strict; +use warnings; + +use TestLib; +use Test::More tests => 10; +use PostgresNode; + +# Different kinds of queries, some committing, some aborting. Also include FOR +# SHARE, FOR UPDATE which may have implications on the visibility bits etc. +my @query_set1 = ( + + "begin; + update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid; + select abalance from pgbench_warm_accounts where aid = :aid; + update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid; + end;", + + "begin; + update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid; + select abalance from pgbench_warm_accounts where aid = :aid; + update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid; + rollback;", + + "begin; + select abalance from pgbench_warm_accounts where aid = :aid for update; + update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid; + select bbalance from pgbench_warm_branches where bid = :bid for update; + update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid; + commit;", + + "begin; + select abalance from pgbench_warm_accounts where aid = :aid for update; + update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid; + select bbalance from pgbench_warm_branches where bid = :bid for update; + update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid; + rollback;", + + "begin; + select abalance from pgbench_warm_accounts where aid = :aid for share; + update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid; + select bbalance from pgbench_warm_branches where bid = :bid for update; + update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid; + commit;", + + "begin; + select abalance from pgbench_warm_accounts where aid = :aid for update; + update pgbench_warm_accounts set abalance = abalance + :delta where aid = :aid; + select bbalance from pgbench_warm_branches where bid = :bid for update; + update pgbench_warm_branches set bbalance = bbalance + :delta where bid = :bid; + rollback;" +); + +# The following queries use user-defined functions to update rows in +# pgbench_warm_accounts table by using auxillary columns. This allows us to +# test if the updates are working fine in various scenarios. +my @query_set2 = ( + + "begin; + set enable_seqscan to false; + select pgbench_warm_update_using_aid1(:chg1, :aid, :bid, :delta); + commit;", + + "begin; + set enable_seqscan to false; + select pgbench_warm_update_using_aid2(:chg2, :aid, :bid, :delta); + commit;", + + "begin; + set enable_seqscan to false; + select pgbench_warm_update_using_aid3(:chg3, :aid, :bid, :delta); + commit;", + + "begin; + set enable_seqscan to false; + select pgbench_warm_update_using_aid4(:chg4, :aid, :bid, :delta); + commit;", + + "begin; + set enable_seqscan to false; + select pgbench_warm_update_using_aid1(:chg1, :aid, :bid, :delta); + rollback;", + + "begin; + set enable_seqscan to false; + select pgbench_warm_update_using_aid2(:chg2, :aid, :bid, :delta); + rollback;", + + "begin; + set enable_seqscan to false; + select pgbench_warm_update_using_aid3(:chg3, :aid, :bid, :delta); + rollback;", + + "begin; + set enable_seqscan to false; + select pgbench_warm_update_using_aid4(:chg4, :aid, :bid, :delta); + rollback;" +); + +# Specify concurrent DDLs that you may want to execute with the tests. +my @ddl_queries = ( + "drop index pgb_a_aid1; + create index pgb_a_aid1 on pgbench_warm_accounts(aid1);", + "drop index pgb_a_aid2; + create index pgb_a_aid2 on pgbench_warm_accounts(aid2);", + "drop index pgb_a_aid3; + create index pgb_a_aid3 on pgbench_warm_accounts(aid3) using hash (aid3);", + "drop index pgb_a_aid4; + create index pgb_a_aid4 on pgbench_warm_accounts(aid4);", + "drop index pgb_a_aid1; + create index concurrently pgb_a_aid1 on pgbench_warm_accounts(aid1);", + "drop index pgb_a_aid2; + create index concurrently pgb_a_aid2 on pgbench_warm_accounts(aid2);", + "drop index pgb_a_aid3; + create index concurrently pgb_a_aid3 on pgbench_warm_accounts(aid3) using hash(aid3);", + "drop index pgb_a_aid4; + create index concurrently pgb_a_aid4 on pgbench_warm_accounts(aid4);", + "vacuum pgbench_warm_accounts", + "vacuum pgbench_warm_branches", + "vacuum full pgbench_warm_accounts", + "vacuum full pgbench_warm_branches" +); + +# Consistency check queries. +my @check_queries = ( + "set enable_seqscan to false; select pgbench_warm_check_consistency();", + "set enable_seqscan to false; select pgbench_warm_check_row(:aid);" +); + +my $node = get_new_node(); +$node->init; +$node->start; + +# prepare the test for execution +$node->run_log([ 'psql', '-X', $node->connstr(), '-f', 't/warm_stress_init.sql']); + +my $res = $node->safe_psql('postgres', "select proname from pg_proc where proname = 'pgbench_warm_update_using_aid1'"); +is($res, 'pgbench_warm_update_using_aid1', 'dummy test passed'); + +$res = $node->safe_psql('postgres', "select count(*) from pgbench_warm_accounts"); +is($res, 10000, 'Fine match'); + +# Start as many connections as we need +sub create_connections { + my $count = shift; + my @handles; + my ($stdin, $stdout, $stderr) = ('','',''); + for (my $proc = 0; $proc < $count; $proc = $proc + 1) { + my $handle = IPC::Run::start( + [ + 'psql', '-v', '-f -', $node->connstr(), + ], + \$stdin, \$stdout, \$stderr); + push @handles, [$handle,\$stdin,\$stdout,\$stderr]; + } + return \@handles; +} + +sub check_connections { + my @handles = @_; + my $failures = 0; + print @handles; + foreach my $elem (@handles) { + my ($handle, $stdin, $stdout, $stderr) = @$elem; + # Wait for all queries to complete and psql sessions to exit, checking + # exit codes. We don't need to do the fancy interpretation safe_psql + # does. + $handle->finish; + if (!is($handle->full_result(0), 0, "psql exited normally")) + { + $failures ++; + diag "psql exit code: " . ($handle->result(0)) . " or signal: " . ($handle->full_result(0) & 127); + diag "Stdout:\n---\n$$stdout\n---\nStderr:\n----\n$$stderr\n---"; + } + } + return $failures; +} + +my $set1_handles = create_connections(3); +my $set2_handles = create_connections(3); +my $aux_handles = create_connections(1); + +# Run a few thousand transactions, using various kinds of queries +my $scale = 1; +for (my $txn = 0; $txn < 10000; $txn = $txn + 1) { + # Run a randomly chosen query from set1 + my $aid = int(rand($scale*10000)) + 1; + my $bid = int(rand(100)) + 1; + my $delta = int(rand(1000)) - 500; + + my $connindx = rand(@$set1_handles); + my $elem = @$set1_handles[$connindx]; + my ($handle, $stdin, $stdout, $stderr) = @$elem; + + my $queryindx = rand(@query_set1); + my $query = $query_set1[$queryindx]; + + $query =~ s/\:aid/$aid/g; + $query =~ s/\:bid/$bid/g; + $query =~ s/\:delta/$delta/g; + + $$stdin .= $query . "\n"; + pump $handle while length $$stdin; + + # Run a randomly chosen query from set1 + my $chg1 = int(rand(4)) - 2; + my $chg2 = int(rand(6)) - 3; + my $chg3 = int(rand(8)) - 4; + my $chg4 = int(rand(10)) - 5; + + $connindx = rand(@$set2_handles); + $elem = @$set2_handles[$connindx]; + ($handle, $stdin, $stdout, $stderr) = @$elem; + + $queryindx = rand(@query_set2); + $query = $query_set2[$queryindx]; + + $query =~ s/\:aid/$aid/g; + $query =~ s/\:bid/$bid/g; + $query =~ s/\:delta/$delta/g; + $query =~ s/\:chg1/$chg1/g; + $query =~ s/\:chg2/$chg2/g; + $query =~ s/\:chg3/$chg3/g; + $query =~ s/\:chg4/$chg4/g; + + $$stdin .= $query . "\n"; + pump $handle while length $$stdin; + + # Some randomly picked numbers to run DDLs and consistency checks + my $random = int(rand(100)); + + # Consistenct checks every 5 transactions + if ($random % 5 == 0) + { + $connindx = rand(@$aux_handles); + $elem = @$aux_handles[$connindx]; + ($handle, $stdin, $stdout, $stderr) = @$elem; + + $queryindx = rand(@check_queries); + $query = $check_queries[$queryindx]; + + $$stdin .= $query . "\n"; + pump $handle while length $$stdin; + } + + # 1% DDLs + if ($random == 17) + { + $connindx = rand(@$aux_handles); + $elem = @$aux_handles[$connindx]; + ($handle, $stdin, $stdout, $stderr) = @$elem; + + $queryindx = rand(@ddl_queries); + $query = $ddl_queries[$queryindx]; + + $$stdin .= $query . "\n"; + pump $handle while length $$stdin; + } +} + +check_connections(@$set1_handles); +check_connections(@$set2_handles); +check_connections(@$aux_handles); + +# Run final consistency checks +my $res1 = $node->safe_psql('postgres', "select sum(abalance) from pgbench_warm_accounts"); +my $res2 = $node->safe_psql('postgres', "select sum(bbalance) from pgbench_warm_branches"); +is($res1, $res2, 'Fine match'); diff --git b/src/test/modules/warm/t/warm_stress_init.sql a/src/test/modules/warm/t/warm_stress_init.sql new file mode 100644 index 0000000..4697480 --- /dev/null +++ a/src/test/modules/warm/t/warm_stress_init.sql @@ -0,0 +1,209 @@ + +drop table if exists pgbench_warm_branches; +drop table if exists pgbench_warm_accounts; + +create table pgbench_warm_branches ( + bid bigint, + bbalance bigint); + +create table pgbench_warm_accounts ( + aid bigint, + bid bigint, + abalance bigint, + aid1 bigint , + aid2 bigint , + aid3 bigint , + aid4 bigint , + aid5 text default md5(random()::text), + aid6 text default md5(random()::text), + aid7 text default md5(random()::text), + aid8 text default md5(random()::text), + aid9 text default md5(random()::text), + aid10 text default md5(random()::text), + gistcol polygon default null +); + +-- update using aid1. aid1 should stay within the range (aid * 10 - 2 <= aid1 <= aid * 10 + 2) +create or replace function pgbench_warm_update_using_aid1(chg integer, v_aid bigint, v_bid bigint, delta bigint) +returns void as $$ +declare + qry varchar; + lower varchar; + upper varchar; + range integer; + aid_updated bigint; +begin + range := 2; + update pgbench_warm_accounts p set aid1 = aid1 + chg, abalance = abalance + +delta where aid1 >= v_aid * 10 - range - chg and aid1 <= v_aid * 10 + range - chg +returning p.aid into aid_updated; + if aid_updated is not null then + update pgbench_warm_branches p set bbalance = bbalance + delta where p.bid = v_bid; + else + select aid into aid_updated from pgbench_warm_accounts p where aid1 >= +v_aid * 10 - range and aid1 <= v_aid * 10 + range; + if aid_updated is null then + raise exception 'pgbench_warm_accounts row not found'; + end if; + end if; +end +$$ language plpgsql; + +-- update using aid2. aid2 should stay within the range (aid * 20 - 4 <= aid2 <= aid * 20 + 4) +create or replace function pgbench_warm_update_using_aid2(chg integer, v_aid bigint, v_bid bigint, delta bigint) +returns void as $$ +declare + qry varchar; + lower varchar; + upper varchar; + range integer; + aid_updated bigint; +begin + range := 4; + update pgbench_warm_accounts p set aid2 = aid2 + chg, abalance = abalance + +delta where aid2 >= v_aid * 20 - range - chg and aid2 <= v_aid * 20 + range - chg +returning p.aid into aid_updated; + if aid_updated is not null then + update pgbench_warm_branches p set bbalance = bbalance + delta where p.bid = v_bid; + else + select aid into aid_updated from pgbench_warm_accounts p where aid2 >= v_aid * 20 - range and aid2 <= v_aid * 20 + range; + if aid_updated is null then + raise exception 'pgbench_warm_accounts row not found'; + end if; + end if; +end +$$ language plpgsql; + +-- update using aid3. aid3 should stay within the range (aid * 30 - 6 <= aid3 <= aid * 30 + 6) +create or replace function pgbench_warm_update_using_aid3(chg integer, v_aid bigint, v_bid bigint, delta bigint) +returns void as $$ +declare + qry varchar; + lower varchar; + upper varchar; + range integer; + aid_updated bigint; +begin + range := 6; + update pgbench_warm_accounts p set aid3 = aid3 + chg, abalance = abalance + +delta where aid3 >= v_aid * 30 - range - chg and aid3 <= v_aid * 30 + range - chg +returning p.aid into aid_updated; + if aid_updated is not null then + update pgbench_warm_branches p set bbalance = bbalance + delta where p.bid = v_bid; + else + select aid into aid_updated from pgbench_warm_accounts p where aid3 >= v_aid * 30 - range and aid3 <= v_aid * 30 + range; + if aid_updated is null then + raise exception 'pgbench_warm_accounts row not found'; + end if; + end if; +end +$$ language plpgsql; + +-- update using aid4. aid4 should stay within the range (aid * 40 - 8 <= aid4 <= aid * 40 + 8) +create or replace function pgbench_warm_update_using_aid4(chg integer, v_aid bigint, v_bid bigint, delta bigint) +returns void as $$ +declare + qry varchar; + lower varchar; + upper varchar; + range integer; + aid_updated bigint; +begin + range := 8; + update pgbench_warm_accounts p set aid4 = aid4 + chg, abalance = abalance + +delta where aid4 >= v_aid * 40 - range - chg and aid4 <= v_aid * 40 + range - chg +returning p.aid into aid_updated; + if aid_updated is not null then + update pgbench_warm_branches p set bbalance = bbalance + delta where p.bid = v_bid; + else + select aid into aid_updated from pgbench_warm_accounts p where aid4 >= v_aid * 40 - range and aid4 <= v_aid * 40 + range; + if aid_updated is null then + raise exception 'pgbench_warm_accounts row not found'; + end if; + end if; +end +$$ language plpgsql; + +-- ensure that exactly one row exists within a given range. use different +-- indexes to fetch the row +create or replace function pgbench_warm_check_row(v_aid bigint) +returns void as $$ +declare + range integer; + factor integer; + ret_aid1 bigint; + ret_aid2 bigint; + ret_aid3 bigint; + ret_aid4 bigint; +begin + range := 2; + factor := 10; + select aid into ret_aid1 from pgbench_warm_accounts p where aid1 >= v_aid * + factor - range and aid1 <= v_aid * factor + range; + + range := 4; + factor := 20; + select aid into ret_aid2 from pgbench_warm_accounts p where aid2 >= v_aid * + factor - range and aid2 <= v_aid * factor + range; + + range := 6; + factor := 30; + select aid into ret_aid3 from pgbench_warm_accounts p where aid3 >= v_aid * + factor - range and aid3 <= v_aid * factor + range; + + range := 8; + factor := 40; + select aid into ret_aid4 from pgbench_warm_accounts p where aid4 >= v_aid * + factor - range and aid4 <= v_aid * factor + range; + + if ret_aid1 is null or ret_aid1 != v_aid then + raise exception 'pgbench_warm_accounts row (%) not found via aid1', v_aid; + end if; + + if ret_aid2 is null or ret_aid2 != v_aid then + raise exception 'pgbench_warm_accounts row (%) not found via aid2', v_aid; + end if; + + if ret_aid3 is null or ret_aid3 != v_aid then + raise exception 'pgbench_warm_accounts row (%) not found via aid3', v_aid; + end if; + + if ret_aid4 is null or ret_aid4 != v_aid then + raise exception 'pgbench_warm_accounts row (%) not found via aid4', v_aid; + end if; +end +$$ language plpgsql; + +create or replace function pgbench_warm_check_consistency() +returns void as $$ +declare + sum_abalance bigint; + sum_bbalance bigint; +begin + select sum(abalance) into sum_abalance from pgbench_warm_accounts; + select sum(bbalance) into sum_bbalance from pgbench_warm_branches; + if sum_abalance != sum_bbalance then + raise exception 'found inconsitency in sum (%, %)', sum_abalance, sum_bbalance; + end if; +end +$$ language plpgsql; + +\set end 10000 +insert into pgbench_warm_branches select generate_series(1, 100), 0 ; +insert into pgbench_warm_accounts select generate_series(1, :end), + (random() * 100)::int, 0, + generate_series(1, :end) * 10, + generate_series(1, :end) * 20, + generate_series(1, :end) * 30, + generate_series(1, :end) * 40; + +create unique index pgb_a_aid on pgbench_warm_accounts(aid); +create index pgb_a_aid1 on pgbench_warm_accounts(aid1); +create index pgb_a_aid2 on pgbench_warm_accounts(aid2); +create index pgb_a_aid3 on pgbench_warm_accounts(aid3) using hash(aid3); +create index pgb_a_aid4 on pgbench_warm_accounts(aid4); + +create unique index pgb_b_bid on pgbench_warm_branches(bid); +create index pgb_b_bbalance on pgbench_warm_branches(bbalance); + +vacuum analyze;