#!/bin/bash
# Checks the claim in the v4 cover letter: "Existing on-disk data is not
# affected by any of them; pglz and lz4 values keep their current
# representation, bit for bit."
#   1. The same pglz and lz4 data written by master and by v4: the raw bytes
#      of the inline compressed datums (pageinspect) and of the TOAST chunks
#      must be identical.
#   2. pg_upgrade of the master cluster to v4: every value reads back the
#      same, and amcheck (verify_heapam with check_toast) finds nothing.
#   3. zstd on v4: amcheck after ALTER ... SET COMPRESSION, VACUUM FULL, and
#      copying values between pglz, lz4 and zstd columns.
# Run through the build lane:  compila -n zstd-format -- bash format_check.sh
set -u
W=$HOME/pgzstd
T=$(mktemp -d /tmp/claude-1000/zfmt.XXXX)
DOCS=$W/src-base/doc/src/sgml
cat $(ls $DOCS/*.sgml $DOCS/ref/*.sgml | sort) > $T/docs.txt

setup="
CREATE EXTENSION pageinspect; CREATE EXTENSION amcheck;
CREATE TABLE docs_src AS SELECT pg_read_file('$T/docs.txt') AS t;
CREATE TABLE t_pglz (id int, v text COMPRESSION pglz);
CREATE TABLE t_lz4  (id int, v text COMPRESSION lz4);
-- Each table from the source text: copying compressed values between
-- tables keeps them as they are, whatever the target column says.
INSERT INTO t_pglz SELECT g, substr(t, 1 + (g * 7919) % 5000000, CASE WHEN g % 2 = 0 THEN 4000 ELSE 40000 END)
  FROM docs_src, generate_series(1, 2000) g;
INSERT INTO t_lz4 SELECT g, substr(t, 1 + (g * 7919) % 5000000, CASE WHEN g % 2 = 0 THEN 4000 ELSE 40000 END)
  FROM docs_src, generate_series(1, 2000) g;
CHECKPOINT;"

# One line per table: inline compressed datums, TOAST chunks, all values.
digest="
SELECT r, 'inline=' || md5(string_agg(md5(a.t_attrs[2]), '' ORDER BY blk, a.lp))
       || ' chunks=' || (SELECT md5(string_agg(md5(chunk_data), '' ORDER BY chunk_id, chunk_seq))
                           FROM REL)
       || ' values=' || (SELECT md5(string_agg(md5(v), '' ORDER BY id)) FROM TBL)
  FROM (SELECT 'TBL'::text r) x,
       generate_series(0, pg_relation_size('TBL') / 8192 - 1) blk,
       heap_page_item_attrs(get_raw_page('TBL', blk::int), 'TBL'::regclass) a
 WHERE a.t_attrs IS NOT NULL
 GROUP BY r"

start() { "$1/bin/pg_ctl" -D "$2" -o "-p $3 -c unix_socket_directories=$T" -l "$2.log" -w start > /dev/null; }
stop()  { "$1/bin/pg_ctl" -D "$2" -m fast -w stop > /dev/null; }
q()     { local I=$1 P=$2; shift 2; "$I/bin/psql" -X -qAt -h "$T" -p $P -U postgres -d postgres "$@"; }
digests() {  # install port
  for tbl in t_pglz t_lz4; do
    rel=$(q $1 $2 -c "SELECT reltoastrelid::regclass FROM pg_class WHERE relname = '$tbl'")
    q $1 $2 -v ON_ERROR_STOP=1 -c "$(echo "$digest" | sed "s/TBL/$tbl/g; s/REL/$rel/g")" \
      || echo "$tbl: DIGEST FAILED"
  done
}

echo "== 1. same data written by master and by v4"
for b in base v4c; do
  I=$W/i-$b
  "$I/bin/initdb" -D "$T/d-$b" -U postgres --no-sync -A trust > /dev/null 2>&1
  start $I "$T/d-$b" 55461
  q $I 55461 -c "$setup" > /dev/null
  digests $I 55461 > "$T/digest-$b.txt"
  sed "s/^/   $b: /" "$T/digest-$b.txt"
  stop $I "$T/d-$b"
done
if [ "$(grep -c 'inline=' "$T/digest-base.txt")" != 2 ]; then echo "   NO DIGESTS, check failed"
elif cmp -s "$T/digest-base.txt" "$T/digest-v4c.txt"; then echo "   identical"
else echo "   DIFFERENT"; fi

echo "== 2. pg_upgrade master -> v4"
"$W/i-v4c/bin/initdb" -D "$T/d-up" -U postgres --no-sync -A trust > /dev/null 2>&1
( cd "$T" && "$W/i-v4c/bin/pg_upgrade" -b "$W/i-base/bin" -B "$W/i-v4c/bin" -d "$T/d-base" -D "$T/d-up" \
    -U postgres -s "$T" -p 55461 -P 55462 > "$T/pg_upgrade.log" 2>&1 )
echo "   pg_upgrade rc=$?"
start $W/i-v4c "$T/d-up" 55462
digests $W/i-v4c 55462 > "$T/digest-up.txt"
cmp -s "$T/digest-base.txt" "$T/digest-up.txt" && echo "   digests after upgrade: identical to master" || { echo "   digests after upgrade: DIFFERENT"; cat "$T/digest-up.txt"; }
for tbl in t_pglz t_lz4; do
  echo "   amcheck $tbl: $(q $W/i-v4c 55462 -c "SELECT count(*) FROM verify_heapam('$tbl', check_toast => true)") problems,"\
       "methods: $(q $W/i-v4c 55462 -c "SELECT string_agg(DISTINCT coalesce(pg_column_compression(v), 'none'), ',') FROM $tbl")"
done

echo "== 3. zstd on v4"
q $W/i-v4c 55462 -v ON_ERROR_STOP=1 > /dev/null <<'SQL'
CREATE TABLE t_zstd (id int, v text COMPRESSION zstd);
INSERT INTO t_zstd SELECT id, v || '' FROM t_pglz;
-- A column that ends up holding pglz, lz4 and zstd values at once.
CREATE TABLE t_mix (id int, v text COMPRESSION pglz);
INSERT INTO t_mix SELECT id, v || '' FROM t_pglz;
ALTER TABLE t_mix ALTER COLUMN v SET COMPRESSION zstd;
INSERT INTO t_mix SELECT id + 100000, v || '' FROM t_lz4;
INSERT INTO t_mix SELECT id + 200000, v FROM t_lz4;
VACUUM FULL t_zstd;
CREATE TABLE t_back (id int, v text COMPRESSION lz4);
INSERT INTO t_back SELECT id, v || '' FROM t_zstd;
UPDATE t_zstd SET v = v || 'x' WHERE id % 10 = 0;
DELETE FROM t_zstd WHERE id % 7 = 0;
VACUUM t_zstd;
SQL
for tbl in t_zstd t_mix t_back; do
  echo "   $tbl: amcheck problems $(q $W/i-v4c 55462 -c "SELECT count(*) FROM verify_heapam('$tbl', check_toast => true)"),"\
       "methods $(q $W/i-v4c 55462 -c "SELECT string_agg(m || ':' || n, ' ') FROM (SELECT coalesce(pg_column_compression(v), 'none') m, count(*) n FROM $tbl GROUP BY 1 ORDER BY 1) s")"
done
echo "   values equal to the source: $(q $W/i-v4c 55462 -c "SELECT count(*) FROM t_pglz a JOIN t_back b USING (id) WHERE a.v = b.v") of $(q $W/i-v4c 55462 -c "SELECT count(*) FROM t_pglz")"
stop $W/i-v4c "$T/d-up"
echo "== logs in $T"