Thread: Pg11 server w/parallel backends unresponsive to cancel/terminate...

Pg11 server w/parallel backends unresponsive to cancel/terminate...

From

Jerry Sievers

Date:

11 September 2020, 16:08:20

Greetings!  One of my $power users launched big-nasty job on our ~50TB
warehouse.

Long-running jobs on that box aren't unusual, however this one we want
stopped and it's unresponsive to cancel or terminate signalling for
several hours.

Below are 10 lines captured by running strace over each of the PIDs.

What else to look for before we stop-immediate the system?

Thx!

PS: Will repost w/gdb output as soon as I can remember how to run it for
a case like this :-)



 PostgreSQL 11.7 (Ubuntu 11.7-2.pgdg16.04+1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 5.4.0-6ubuntu1~16.04.12)
5.4.020160609, 64-bit
 



# echo $#
9

# for p; do strace -p $p 2>&1 | head -10; echo; done 
strace: Process 42079 attached
strace: [ Process PID=42079 runs in x32 mode. ]
strace: [ Process PID=42079 runs in 64 bit mode. ]
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219365000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214364000
munmap(0x7f8219365000, 41947136)        = 0
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81ff856000
munmap(0x7f8214364000, 83890176)        = 0
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81eb855000
munmap(0x7f81ff856000, 167776256)       = 0

strace: Process 42372 attached
strace: [ Process PID=42372 runs in x32 mode. ]
strace: [ Process PID=42372 runs in 64 bit mode. ]
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
munmap(0x7f81fdafb000, 167776256)       = 0
munmap(0x7f81e9afa000, 335548416)       = 0
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
munmap(0x7f8219a0b000, 41947136)        = 0
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000

strace: Process 42373 attached
strace: [ Process PID=42373 runs in x32 mode. ]
strace: [ Process PID=42373 runs in 64 bit mode. ]
munmap(0x7f81e9afa000, 335548416)       = 0
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
munmap(0x7f8217a0a000, 41947136)        = 0
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
munmap(0x7f8212a09000, 83890176)        = 0
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000

strace: Process 42374 attached
strace: [ Process PID=42374 runs in x32 mode. ]
strace: [ Process PID=42374 runs in 64 bit mode. ]
munmap(0x7f81e9afa000, 335548416)       = 0
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
munmap(0x7f8219a0b000, 41947136)        = 0
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
munmap(0x7f8214a0a000, 83890176)        = 0
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000

strace: Process 42375 attached
strace: [ Process PID=42375 runs in x32 mode. ]
strace: [ Process PID=42375 runs in 64 bit mode. ]
munmap(0x7f81e9afa000, 335548416)       = 0
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
munmap(0x7f8219a0b000, 41947136)        = 0
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
munmap(0x7f8214a0a000, 83890176)        = 0
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000

strace: Process 42376 attached
strace: [ Process PID=42376 runs in x32 mode. ]
strace: [ Process PID=42376 runs in 64 bit mode. ]
munmap(0x7f81e9afa000, 335548416)       = 0
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
munmap(0x7f8217a0a000, 41947136)        = 0
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
munmap(0x7f8212a09000, 83890176)        = 0
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000

strace: Process 42377 attached
munmap(0x7f81fdafb000, 167776256)       = 0
munmap(0x7f81e9afa000, 335548416)       = 0
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
munmap(0x7f8217a0a000, 41947136)        = 0
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
munmap(0x7f8212a09000, 83890176)        = 0
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
munmap(0x7f81fdafb000, 167776256)       = 0

strace: Process 42378 attached
strace: [ Process PID=42378 runs in x32 mode. ]
strace: [ Process PID=42378 runs in 64 bit mode. ]
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
munmap(0x7f8214a0a000, 83890176)        = 0
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
munmap(0x7f81fdafb000, 167776256)       = 0
munmap(0x7f81e9afa000, 335548416)       = 0
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000

strace: Process 42379 attached
strace: [ Process PID=42379 runs in x32 mode. ]
strace: [ Process PID=42379 runs in 64 bit mode. ]
munmap(0x7f81e9afa000, 335548416)       = 0
mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
munmap(0x7f8217a0a000, 41947136)        = 0
mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
munmap(0x7f8212a09000, 83890176)        = 0
mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000




-- 
Jerry Sievers
Postgres DBA/Development Consulting
e: postgres.consulting@comcast.net

Re: Pg11 server w/parallel backends unresponsive to cancel/terminate...

From

John Scalia

Date:

11 September 2020, 16:15:48

Easiest thing I would do is find that processes PID by querying the pg_stat_activity table, then invoke “select from
pg_terminate_backend(pid#);where of course, you put the determined PID from your first query into those parenthesis. 
—
Jay

Sent from my iPad

> On Sep 11, 2020, at 9:08 AM, Jerry Sievers <gsievers19@comcast.net> wrote:
>
> Greetings!  One of my $power users launched big-nasty job on our ~50TB
> warehouse.
>
> Long-running jobs on that box aren't unusual, however this one we want
> stopped and it's unresponsive to cancel or terminate signalling for
> several hours.
>
> Below are 10 lines captured by running strace over each of the PIDs.
>
> What else to look for before we stop-immediate the system?
>
> Thx!
>
> PS: Will repost w/gdb output as soon as I can remember how to run it for
> a case like this :-)
>
>
>
> PostgreSQL 11.7 (Ubuntu 11.7-2.pgdg16.04+1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 5.4.0-6ubuntu1~16.04.12)
5.4.020160609, 64-bit 
>
>
>
> # echo $#
> 9
>
> # for p; do strace -p $p 2>&1 | head -10; echo; done
> strace: Process 42079 attached
> strace: [ Process PID=42079 runs in x32 mode. ]
> strace: [ Process PID=42079 runs in 64 bit mode. ]
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219365000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214364000
> munmap(0x7f8219365000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81ff856000
> munmap(0x7f8214364000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81eb855000
> munmap(0x7f81ff856000, 167776256)       = 0
>
> strace: Process 42372 attached
> strace: [ Process PID=42372 runs in x32 mode. ]
> strace: [ Process PID=42372 runs in 64 bit mode. ]
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
> munmap(0x7f81fdafb000, 167776256)       = 0
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
> munmap(0x7f8219a0b000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>
> strace: Process 42373 attached
> strace: [ Process PID=42373 runs in x32 mode. ]
> strace: [ Process PID=42373 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
> munmap(0x7f8217a0a000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8212a09000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
> strace: Process 42374 attached
> strace: [ Process PID=42374 runs in x32 mode. ]
> strace: [ Process PID=42374 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
> munmap(0x7f8219a0b000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8214a0a000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
> strace: Process 42375 attached
> strace: [ Process PID=42375 runs in x32 mode. ]
> strace: [ Process PID=42375 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
> munmap(0x7f8219a0b000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8214a0a000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
> strace: Process 42376 attached
> strace: [ Process PID=42376 runs in x32 mode. ]
> strace: [ Process PID=42376 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
> munmap(0x7f8217a0a000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8212a09000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
> strace: Process 42377 attached
> munmap(0x7f81fdafb000, 167776256)       = 0
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
> munmap(0x7f8217a0a000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8212a09000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
> munmap(0x7f81fdafb000, 167776256)       = 0
>
> strace: Process 42378 attached
> strace: [ Process PID=42378 runs in x32 mode. ]
> strace: [ Process PID=42378 runs in 64 bit mode. ]
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8214a0a000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
> munmap(0x7f81fdafb000, 167776256)       = 0
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
>
> strace: Process 42379 attached
> strace: [ Process PID=42379 runs in x32 mode. ]
> strace: [ Process PID=42379 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
> munmap(0x7f8217a0a000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8212a09000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
>
>
>
> --
> Jerry Sievers
> Postgres DBA/Development Consulting
> e: postgres.consulting@comcast.net
>
>

Re: Pg11 server w/parallel backends unresponsive to cancel/terminate... (here are some backtraces)

From

Jerry Sievers

Date:

11 September 2020, 16:30:46

Included a few backtraces...

Jerry Sievers <gsievers19@comcast.net> writes:

> Greetings!  One of my $power users launched big-nasty job on our ~50TB
> warehouse.
>
> Long-running jobs on that box aren't unusual, however this one we want
> stopped and it's unresponsive to cancel or terminate signalling for
> several hours.
>
> Below are 10 lines captured by running strace over each of the PIDs.
>
> What else to look for before we stop-immediate the system?
>
> Thx!
>
> PS: Will repost w/gdb output as soon as I can remember how to run it for
> a case like this :-)

cat debug.log
#0  0x00007f82bca695a0 in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#1  0x00007f82bca6b586 in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#2  0x00007f82bca739ca in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#3  0x00007f82bd0fd1a6 in llvm::SplitCriticalEdge(llvm::TerminatorInst*, unsigned int,
llvm::CriticalEdgeSplittingOptionsconst&) () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
 
#4  0x00007f82bd38e46f in llvm::GVN::splitCriticalEdges(llvm::BasicBlock*, llvm::BasicBlock*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#5  0x00007f82bd39a8af in llvm::GVN::processFoldableCondBr(llvm::BranchInst*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#6  0x00007f82bd39af97 in llvm::GVN::processInstruction(llvm::Instruction*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#7  0x00007f82bd39b188 in llvm::GVN::processBlock(llvm::BasicBlock*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#8  0x00007f82bd39c390 in llvm::GVN::iterateOnFunction(llvm::Function&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#9  0x00007f82bd39c4bb in llvm::GVN::runImpl(llvm::Function&, llvm::AssumptionCache&, llvm::DominatorTree&,
llvm::TargetLibraryInfoconst&, llvm::AAResults&, llvm::MemoryDependenceResults*, llvm::LoopInfo*,
llvm::OptimizationRemarkEmitter*)() from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
 
#10 0x00007f82bd39cc98 in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#11 0x00007f82bcaa9a22 in llvm::FPPassManager::runOnFunction(llvm::Function&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#12 0x00007f82bd6e870e in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#13 0x00007f82bcaa966f in llvm::legacy::PassManagerImpl::run(llvm::Module&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#14 0x00007f82bca365c9 in LLVMRunPassManager () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#15 0x00007f82cbd177c9 in llvm_get_function () from /usr/lib/postgresql/11/lib/llvmjit.so
#16 0x00007f82cbd205d0 in ?? () from /usr/lib/postgresql/11/lib/llvmjit.so
#17 0x00005640aabfb965 in ExecScan ()
#18 0x00005640aac09aaa in ?? ()
#19 0x00005640aac1b826 in ?? ()
#20 0x00005640aabf2d1b in standard_ExecutorRun ()
#21 0x00007fa3a05d95a5 in ?? () from /usr/lib/postgresql/11/lib/auto_explain.so
#22 0x00007fa3a03d0dfd in ?? () from /usr/lib/postgresql/11/lib/pg_stat_statements.so
#23 0x00005640aab7ec0c in ExecCreateTableAs ()
#24 0x00005640aad4b860 in ?? ()
#25 0x00005640aad4aa60 in standard_ProcessUtility ()
#26 0x00007fa3a03d316f in ?? () from /usr/lib/postgresql/11/lib/pg_stat_statements.so
#27 0x00007fa3a01b6613 in ?? () from /usr/lib/postgresql/11/lib/pglogical.so
#28 0x00005640aad47959 in ?? ()
#29 0x00005640aad48488 in ?? ()
#30 0x00005640aad491f5 in PortalRun ()
#31 0x00005640aad44cb1 in ?? ()
#32 0x00005640aad461ed in PostgresMain ()
#33 0x00005640aaa53d37 in ?? ()
#34 0x00005640aacd0882 in PostmasterMain ()
#35 0x00005640aaa550e5 in main ()
Continuing.

Program received signal SIGINT, Interrupt.
0x00007f82bca60a63 in llvm::DominatorTreeBase<llvm::BasicBlock, false>::updateDFSNumbers() const () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#0  0x00007f82bca60a63 in llvm::DominatorTreeBase<llvm::BasicBlock, false>::updateDFSNumbers() const () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#1  0x00007f82bca60b77 in llvm::DominatorTreeBase<llvm::BasicBlock,
false>::dominates(llvm::DomTreeNodeBase<llvm::BasicBlock>const*, llvm::DomTreeNodeBase<llvm::BasicBlock> const*) const
()from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
 
#2  0x00007f82bd392950 in llvm::GVN::findLeader(llvm::BasicBlock const*, unsigned int) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#3  0x00007f82bd39b00d in llvm::GVN::processInstruction(llvm::Instruction*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#4  0x00007f82bd39b188 in llvm::GVN::processBlock(llvm::BasicBlock*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#5  0x00007f82bd39c390 in llvm::GVN::iterateOnFunction(llvm::Function&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#6  0x00007f82bd39c4bb in llvm::GVN::runImpl(llvm::Function&, llvm::AssumptionCache&, llvm::DominatorTree&,
llvm::TargetLibraryInfoconst&, llvm::AAResults&, llvm::MemoryDependenceResults*, llvm::LoopInfo*,
llvm::OptimizationRemarkEmitter*)() from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
 
#7  0x00007f82bd39cc98 in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#8  0x00007f82bcaa9a22 in llvm::FPPassManager::runOnFunction(llvm::Function&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#9  0x00007f82bd6e870e in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#10 0x00007f82bcaa966f in llvm::legacy::PassManagerImpl::run(llvm::Module&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#11 0x00007f82bca365c9 in LLVMRunPassManager () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#12 0x00007f82cbd177c9 in llvm_get_function () from /usr/lib/postgresql/11/lib/llvmjit.so
#13 0x00007f82cbd205d0 in ?? () from /usr/lib/postgresql/11/lib/llvmjit.so
#14 0x00005640aabfb965 in ExecScan ()
#15 0x00005640aac09aaa in ?? ()
#16 0x00005640aac1b826 in ?? ()
#17 0x00005640aabf2d1b in standard_ExecutorRun ()
#18 0x00007fa3a05d95a5 in ?? () from /usr/lib/postgresql/11/lib/auto_explain.so
#19 0x00007fa3a03d0dfd in ?? () from /usr/lib/postgresql/11/lib/pg_stat_statements.so
#20 0x00005640aab7ec0c in ExecCreateTableAs ()
#21 0x00005640aad4b860 in ?? ()
#22 0x00005640aad4aa60 in standard_ProcessUtility ()
#23 0x00007fa3a03d316f in ?? () from /usr/lib/postgresql/11/lib/pg_stat_statements.so
#24 0x00007fa3a01b6613 in ?? () from /usr/lib/postgresql/11/lib/pglogical.so
#25 0x00005640aad47959 in ?? ()
#26 0x00005640aad48488 in ?? ()
#27 0x00005640aad491f5 in PortalRun ()
#28 0x00005640aad44cb1 in ?? ()
#29 0x00005640aad461ed in PostgresMain ()
#30 0x00005640aaa53d37 in ?? ()
#31 0x00005640aacd0882 in PostmasterMain ()
#32 0x00005640aaa550e5 in main ()
Quit
Continuing.

Program received signal SIGINT, Interrupt.
0x00007fa3a7689747 in munmap () from /lib/x86_64-linux-gnu/libc.so.6
#0  0x00007fa3a7689747 in munmap () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007fa3a760c5d1 in free () from /lib/x86_64-linux-gnu/libc.so.6
#2  0x00007f82bca6065f in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#3  0x00007f82bca6b5aa in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#4  0x00007f82bca739ca in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#5  0x00007f82bd0fd1a6 in llvm::SplitCriticalEdge(llvm::TerminatorInst*, unsigned int,
llvm::CriticalEdgeSplittingOptionsconst&) () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
 
#6  0x00007f82bd38e46f in llvm::GVN::splitCriticalEdges(llvm::BasicBlock*, llvm::BasicBlock*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#7  0x00007f82bd39a8af in llvm::GVN::processFoldableCondBr(llvm::BranchInst*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#8  0x00007f82bd39af97 in llvm::GVN::processInstruction(llvm::Instruction*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#9  0x00007f82bd39b188 in llvm::GVN::processBlock(llvm::BasicBlock*) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#10 0x00007f82bd39c390 in llvm::GVN::iterateOnFunction(llvm::Function&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#11 0x00007f82bd39c4bb in llvm::GVN::runImpl(llvm::Function&, llvm::AssumptionCache&, llvm::DominatorTree&,
llvm::TargetLibraryInfoconst&, llvm::AAResults&, llvm::MemoryDependenceResults*, llvm::LoopInfo*,
llvm::OptimizationRemarkEmitter*)() from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1 
#12 0x00007f82bd39cc98 in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#13 0x00007f82bcaa9a22 in llvm::FPPassManager::runOnFunction(llvm::Function&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#14 0x00007f82bd6e870e in ?? () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#15 0x00007f82bcaa966f in llvm::legacy::PassManagerImpl::run(llvm::Module&) () from
/usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#16 0x00007f82bca365c9 in LLVMRunPassManager () from /usr/lib/x86_64-linux-gnu/libLLVM-6.0.so.1
#17 0x00007f82cbd177c9 in llvm_get_function () from /usr/lib/postgresql/11/lib/llvmjit.so
#18 0x00007f82cbd205d0 in ?? () from /usr/lib/postgresql/11/lib/llvmjit.so
#19 0x00005640aabfb965 in ExecScan ()
#20 0x00005640aac09aaa in ?? ()
#21 0x00005640aac1b826 in ?? ()
#22 0x00005640aabf2d1b in standard_ExecutorRun ()
#23 0x00007fa3a05d95a5 in ?? () from /usr/lib/postgresql/11/lib/auto_explain.so
#24 0x00007fa3a03d0dfd in ?? () from /usr/lib/postgresql/11/lib/pg_stat_statements.so
#25 0x00005640aab7ec0c in ExecCreateTableAs ()
#26 0x00005640aad4b860 in ?? ()
#27 0x00005640aad4aa60 in standard_ProcessUtility ()
#28 0x00007fa3a03d316f in ?? () from /usr/lib/postgresql/11/lib/pg_stat_statements.so
#29 0x00007fa3a01b6613 in ?? () from /usr/lib/postgresql/11/lib/pglogical.so
#30 0x00005640aad47959 in ?? ()
#31 0x00005640aad48488 in ?? ()
#32 0x00005640aad491f5 in PortalRun ()
#33 0x00005640aad44cb1 in ?? ()
#34 0x00005640aad461ed in PostgresMain ()
#35 0x00005640aaa53d37 in ?? ()
#36 0x00005640aacd0882 in PostmasterMain ()
#37 0x00005640aaa550e5 in main ()
Detaching from program: /usr/lib/postgresql/11/bin/postgres, process 42079

Thx
>
>
>
>  PostgreSQL 11.7 (Ubuntu 11.7-2.pgdg16.04+1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 5.4.0-6ubuntu1~16.04.12)
5.4.020160609, 64-bit
 
>
>
>
> # echo $#
> 9
>
> # for p; do strace -p $p 2>&1 | head -10; echo; done 
> strace: Process 42079 attached
> strace: [ Process PID=42079 runs in x32 mode. ]
> strace: [ Process PID=42079 runs in 64 bit mode. ]
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219365000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214364000
> munmap(0x7f8219365000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81ff856000
> munmap(0x7f8214364000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81eb855000
> munmap(0x7f81ff856000, 167776256)       = 0
>
> strace: Process 42372 attached
> strace: [ Process PID=42372 runs in x32 mode. ]
> strace: [ Process PID=42372 runs in 64 bit mode. ]
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
> munmap(0x7f81fdafb000, 167776256)       = 0
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
> munmap(0x7f8219a0b000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>
> strace: Process 42373 attached
> strace: [ Process PID=42373 runs in x32 mode. ]
> strace: [ Process PID=42373 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
> munmap(0x7f8217a0a000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8212a09000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
> strace: Process 42374 attached
> strace: [ Process PID=42374 runs in x32 mode. ]
> strace: [ Process PID=42374 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
> munmap(0x7f8219a0b000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8214a0a000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
> strace: Process 42375 attached
> strace: [ Process PID=42375 runs in x32 mode. ]
> strace: [ Process PID=42375 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
> munmap(0x7f8219a0b000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8214a0a000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
> strace: Process 42376 attached
> strace: [ Process PID=42376 runs in x32 mode. ]
> strace: [ Process PID=42376 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
> munmap(0x7f8217a0a000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8212a09000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>
> strace: Process 42377 attached
> munmap(0x7f81fdafb000, 167776256)       = 0
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
> munmap(0x7f8217a0a000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8212a09000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
> munmap(0x7f81fdafb000, 167776256)       = 0
>
> strace: Process 42378 attached
> strace: [ Process PID=42378 runs in x32 mode. ]
> strace: [ Process PID=42378 runs in 64 bit mode. ]
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8214a0a000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
> munmap(0x7f81fdafb000, 167776256)       = 0
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
>
> strace: Process 42379 attached
> strace: [ Process PID=42379 runs in x32 mode. ]
> strace: [ Process PID=42379 runs in 64 bit mode. ]
> munmap(0x7f81e9afa000, 335548416)       = 0
> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
> munmap(0x7f8217a0a000, 41947136)        = 0
> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
> munmap(0x7f8212a09000, 83890176)        = 0
> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000

-- 
Jerry Sievers
Postgres DBA/Development Consulting
e: postgres.consulting@comcast.net

Re: Pg11 server w/parallel backends unresponsive to cancel/terminate...

From

Jerry Sievers

Date:

11 September 2020, 16:33:08

John Scalia <jayknowsunix@gmail.com> writes:

> Easiest thing I would do is find that processes PID by querying the
> pg_stat_activity table, then invoke “select from
> pg_terminate_backend(pid#); where of course, you put the determined
> PID from your first query into those parenthesis.

John, none of the backends respond to
pg_(cancel|terminate)_backend(PID)...  as per $subject :-)

Thx


> —
> Jay
>
> Sent from my iPad
>
>> On Sep 11, 2020, at 9:08 AM, Jerry Sievers <gsievers19@comcast.net> wrote:
>>
>> Greetings!  One of my $power users launched big-nasty job on our ~50TB
>> warehouse.
>>
>> Long-running jobs on that box aren't unusual, however this one we want
>> stopped and it's unresponsive to cancel or terminate signalling for
>> several hours.
>>
>> Below are 10 lines captured by running strace over each of the PIDs.
>>
>> What else to look for before we stop-immediate the system?
>>
>> Thx!
>>
>> PS: Will repost w/gdb output as soon as I can remember how to run it for
>> a case like this :-)
>>
>>
>>
>> PostgreSQL 11.7 (Ubuntu 11.7-2.pgdg16.04+1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 5.4.0-6ubuntu1~16.04.12)
5.4.020160609, 64-bit 
>>
>>
>>
>> # echo $#
>> 9
>>
>> # for p; do strace -p $p 2>&1 | head -10; echo; done
>> strace: Process 42079 attached
>> strace: [ Process PID=42079 runs in x32 mode. ]
>> strace: [ Process PID=42079 runs in 64 bit mode. ]
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219365000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214364000
>> munmap(0x7f8219365000, 41947136)        = 0
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81ff856000
>> munmap(0x7f8214364000, 83890176)        = 0
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81eb855000
>> munmap(0x7f81ff856000, 167776256)       = 0
>>
>> strace: Process 42372 attached
>> strace: [ Process PID=42372 runs in x32 mode. ]
>> strace: [ Process PID=42372 runs in 64 bit mode. ]
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>> munmap(0x7f81fdafb000, 167776256)       = 0
>> munmap(0x7f81e9afa000, 335548416)       = 0
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
>> munmap(0x7f8219a0b000, 41947136)        = 0
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>>
>> strace: Process 42373 attached
>> strace: [ Process PID=42373 runs in x32 mode. ]
>> strace: [ Process PID=42373 runs in 64 bit mode. ]
>> munmap(0x7f81e9afa000, 335548416)       = 0
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
>> munmap(0x7f8217a0a000, 41947136)        = 0
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>> munmap(0x7f8212a09000, 83890176)        = 0
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>>
>> strace: Process 42374 attached
>> strace: [ Process PID=42374 runs in x32 mode. ]
>> strace: [ Process PID=42374 runs in 64 bit mode. ]
>> munmap(0x7f81e9afa000, 335548416)       = 0
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
>> munmap(0x7f8219a0b000, 41947136)        = 0
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>> munmap(0x7f8214a0a000, 83890176)        = 0
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>>
>> strace: Process 42375 attached
>> strace: [ Process PID=42375 runs in x32 mode. ]
>> strace: [ Process PID=42375 runs in 64 bit mode. ]
>> munmap(0x7f81e9afa000, 335548416)       = 0
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
>> munmap(0x7f8219a0b000, 41947136)        = 0
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>> munmap(0x7f8214a0a000, 83890176)        = 0
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>>
>> strace: Process 42376 attached
>> strace: [ Process PID=42376 runs in x32 mode. ]
>> strace: [ Process PID=42376 runs in 64 bit mode. ]
>> munmap(0x7f81e9afa000, 335548416)       = 0
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
>> munmap(0x7f8217a0a000, 41947136)        = 0
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>> munmap(0x7f8212a09000, 83890176)        = 0
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>>
>> strace: Process 42377 attached
>> munmap(0x7f81fdafb000, 167776256)       = 0
>> munmap(0x7f81e9afa000, 335548416)       = 0
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
>> munmap(0x7f8217a0a000, 41947136)        = 0
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>> munmap(0x7f8212a09000, 83890176)        = 0
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>> munmap(0x7f81fdafb000, 167776256)       = 0
>>
>> strace: Process 42378 attached
>> strace: [ Process PID=42378 runs in x32 mode. ]
>> strace: [ Process PID=42378 runs in 64 bit mode. ]
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>> munmap(0x7f8214a0a000, 83890176)        = 0
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>> munmap(0x7f81fdafb000, 167776256)       = 0
>> munmap(0x7f81e9afa000, 335548416)       = 0
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8219a0b000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8214a0a000
>>
>> strace: Process 42379 attached
>> strace: [ Process PID=42379 runs in x32 mode. ]
>> strace: [ Process PID=42379 runs in 64 bit mode. ]
>> munmap(0x7f81e9afa000, 335548416)       = 0
>> mmap(NULL, 41947136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8217a0a000
>> mmap(NULL, 83890176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8212a09000
>> munmap(0x7f8217a0a000, 41947136)        = 0
>> mmap(NULL, 167776256, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81fdafb000
>> munmap(0x7f8212a09000, 83890176)        = 0
>> mmap(NULL, 335548416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f81e9afa000
>>
>>
>>
>>
>> --
>> Jerry Sievers
>> Postgres DBA/Development Consulting
>> e: postgres.consulting@comcast.net
>>
>>
>
>
>

--
Jerry Sievers
Postgres DBA/Development Consulting
e: postgres.consulting@comcast.net

Re: Pg11 server w/parallel backends unresponsive to cancel/terminate... (here are some backtraces)

From

Tom Lane

Date:

11 September 2020, 17:03:49

Jerry Sievers <gsievers19@comcast.net> writes:
> Included a few backtraces...

Hmm, seems like you've managed to hit an infinite loop in LLVM :-(

That explains why it's not responding to SIGTERM - LLVM would not
know that it's supposed to check for interrupts every so often.
I'm afraid you'll have to go the immediate-shutdown route (and it'll
still take a few seconds).

Going forward, you should try to update to newest LLVM.  Or disable JIT.

            regards, tom lane