#!/bin/bash
# Crash parallel autovacuum at the DSM segment limit: InitializeParallelDSM()
# falls back with pcxt->seg NULL and parallel_vacuum_init() then calls
# on_dsm_detach(pcxt->seg, ...).  SQL only, no injection points.
#
# 99 slots here (64 + 5 * MaxBackends).  A Gather holds one segment until its
# query ends, and creates it even when no worker can start, so one big UNION ALL
# parks all but one slot and a looping tiny query takes and releases that one.
# Meanwhile 50 all-dead tables with 3 indexes each keep an autovacuum worker in
# parallel_vacuum_init(), which needs two segments a few microseconds apart.
#
# Usage: [RUNS=3] [WAIT=120] [PGBIN=...] [WORK=...] ./repro.sh
# Logs/cores/backtraces in $WORK.  Exit 9 if anything crashed.
set -uo pipefail

TREE=${TREE:-/local/home/rupiredd/postgres}
PGBIN=${PGBIN:-$TREE/inst/bin}
WORK=${WORK:-$TREE/scratch/parallel-av-dsm-crash/run}
RUNS=${RUNS:-3}
WAIT=${WAIT:-120}
PORT=${PORT:-55444}
export PATH=$PGBIN:$PATH

PGDATA=$WORK/data
LOG=$WORK/server.log
MARK=$WORK/.mark
P="psql -h $WORK -p $PORT -d postgres -Xq"
PAR="SET parallel_setup_cost=0; SET parallel_tuple_cost=0; SET min_parallel_table_scan_size=0;
     SET max_parallel_workers_per_gather=2; SET enable_parallel_append=off;"
PIDS=()
mkdir -p "$WORK"

cleanup() {
	for p in "${PIDS[@]:-}"; do kill "$p" 2>/dev/null; wait "$p" 2>/dev/null; done
	pg_ctl -D "$PGDATA" stop -m immediate >/dev/null 2>&1
	find /dev/shm -maxdepth 1 -name 'PostgreSQL.*' -newer "$MARK" -delete 2>/dev/null
}
trap cleanup EXIT

one_run() {
	local run=$1 t errs many i
	[ -f "$PGDATA/PG_VERSION" ] && rm -rf "$PGDATA"
	rm -f "$LOG"
	initdb -D "$PGDATA" -N >"$WORK/initdb.log" 2>&1 || { tail "$WORK/initdb.log"; exit 1; }
	cat >> "$PGDATA/postgresql.conf" <<-CONF
		port = $PORT
		listen_addresses = ''
		unix_socket_directories = '$WORK'
		fsync = off
		max_connections = 4
		autovacuum_worker_slots = 1
		autovacuum_max_workers = 1
		max_worker_processes = 0
		max_wal_senders = 0
		wal_level = minimal
		dynamic_shared_memory_type = posix
		min_dynamic_shared_memory = 0
		autovacuum_max_parallel_workers = 2
		autovacuum_naptime = '1s'
		min_parallel_index_scan_size = 0
		log_line_prefix = '%m [%p] '
	CONF
	ulimit -c unlimited
	touch "$MARK"
	pg_ctl -D "$PGDATA" -l "$LOG" -w start >/dev/null || { tail -20 "$LOG"; exit 1; }

	$P -c "CREATE TABLE f AS SELECT g AS a FROM generate_series(1,1000) g" >/dev/null
	$P >/dev/null <<-SQL
		DO \$\$ BEGIN
		  FOR i IN 1..50 LOOP
		    EXECUTE format('CREATE TABLE t%s (a int, b int, c int) WITH
		      (autovacuum_parallel_workers = 2, autovacuum_enabled = false,
		       autovacuum_vacuum_threshold = 1, autovacuum_vacuum_scale_factor = 0)', i);
		    EXECUTE format('INSERT INTO t%s SELECT g,g,g FROM generate_series(1,2000) g', i);
		    EXECUTE format('CREATE INDEX ON t%s (a)', i);
		    EXECUTE format('CREATE INDEX ON t%s (b)', i);
		    EXECUTE format('CREATE INDEX ON t%s (c)', i);
		    EXECUTE format('DELETE FROM t%s', i);
		  END LOOP;
		END \$\$;
	SQL

	many="SELECT count(*) FROM ("
	for ((i = 1; i <= 150; i++)); do many="$many(SELECT a FROM f LIMIT 1000000) UNION ALL "; done
	many="$many(SELECT 1 FROM pg_sleep(100000))) s"

	# holds one slot for 8s, then gives it back
	$P -c "$PAR SELECT count(*) FROM ((SELECT a FROM f LIMIT 1000000)
	       UNION ALL (SELECT 1 FROM pg_sleep(8))) s" >/dev/null 2>&1 &
	sleep 1
	$P -c "$PAR $many" >/dev/null 2>&1 &                    # takes all the rest
	PIDS+=($!)
	sleep 8
	$P -c "$PAR DO \$\$ BEGIN FOR i IN 1..100000000 LOOP PERFORM count(*) FROM f;
	       END LOOP; END \$\$;" >/dev/null 2>&1 &           # cycles the last slot
	PIDS+=($!)
	sleep 1

	$P -c "DO \$\$ BEGIN FOR i IN 1..50 LOOP
	         EXECUTE format('ALTER TABLE t%s SET (autovacuum_enabled = true)', i);
	       END LOOP; END \$\$;" >/dev/null
	echo "run $run: slots filled, autovacuum on"

	for ((t = 1; t <= WAIT; t++)); do
		if grep -q "was terminated by signal" "$LOG"; then
			errs=$(grep -c "too many dynamic shared memory segments" "$LOG")
			echo "run $run: CRASH after ${t}s ($errs clean errors first)"
			grep "was terminated by signal" "$LOG" | tail -1 | cut -c1-110
			cp "$LOG" "$WORK/crash-run$run.log"
			core=$(ls -t "$PGDATA"/core* 2>/dev/null | head -1)
			[ -n "$core" ] && cp "$core" "$WORK/crash-run$run.core" &&
				gdb -q -batch -ex 'bt 5' "$PGBIN/postgres" "$core" 2>/dev/null |
				grep '^#' | awk '!s[$1]++' | tee "$WORK/crash-run$run.bt" | sed 's/^/  /'
			return 9
		fi
		sleep 1
	done
	errs=$(grep -c "too many dynamic shared memory segments" "$LOG")
	echo "run $run: no crash in ${WAIT}s ($errs clean errors)"
	cp "$LOG" "$WORK/nocrash-run$run.log"
	return 0
}

echo "$(postgres --version) from $PGBIN; $(git -C "$TREE" log --oneline -1 2>/dev/null)"
git -C "$TREE" diff --stat -- src/backend/commands/vacuumparallel.c 2>/dev/null | tail -1
crashes=0
for ((r = 1; r <= RUNS; r++)); do
	one_run "$r" || crashes=$((crashes + 1))
	cleanup; PIDS=(); sleep 2
done
echo "$crashes of $RUNS runs crashed; logs, cores, backtraces in $WORK"
[ "$crashes" -gt 0 ] && exit 9
exit 0