Segmentation fault with core dump - Mailing list pgsql-general

Hi group,

I'm using PG 9.6.6 and I have a problem with seg fault from every few days to up to two week,
this server is a replica, the other servers (master, and other slaves) do not have this problem.

I could not identify the problem, so I do not know what triggers the problem, however I have the PostgreSQL log and the core-dump generated by the problem.

The server has 60 GB RAM, PG is configured:

shared_buffers = 14GB
work_mem = 192MB


Below are the relevant details.

$ cat /etc/redhat-release
CentOS Linux release 7.3.1611 (Core)


postgres=# select version();
                                                 version                                                 
----------------------------------------------------------------------------------------------------------
 PostgreSQL 9.6.6 on x86_64-pc-linux-gnu, compiled by gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-16), 64-bit
(1 row)


# cat postgresql-Mon.log | grep 'was terminated by signal 11: Segmentation fault'
2018-01-08 01:51:27.909 -03 [85039]: [102-1] user=,db=,app=,client= LOG:  server process (PID 40286) was terminated by signal 11: Segmentation fault
2018-01-08 05:09:51.929 -03 [85039]: [107-1] user=,db=,app=,client= LOG:  server process (PID 62427) was terminated by signal 11: Segmentation fault
2018-01-08 06:33:46.840 -03 [85039]: [112-1] user=,db=,app=,client= LOG:  server process (PID 72156) was terminated by signal 11: Segmentation fault
2018-01-08 13:59:37.422 -03 [119484]: [4-1] user=,db=,app=,client= LOG:  server process (PID 124190) was terminated by signal 11: Segmentation fault
2018-01-08 14:09:41.590 -03 [119484]: [9-1] user=,db=,app=,client= LOG:  checkpointer process (PID 124528) was terminated by signal 11: Segmentation fault
2018-01-08 15:18:06.379 -03 [119484]: [13-1] user=,db=,app=,client= LOG:  server process (PID 129026) was terminated by signal 11: Segmentation fault
2018-01-08 15:23:15.586 -03 [119484]: [18-1] user=,db=,app=,client= LOG:  server process (PID 6528) was terminated by signal 11: Segmentation fault
2018-01-08 15:55:32.029 -03 [119484]: [23-1] user=,db=,app=,client= LOG:  server process (PID 8762) was terminated by signal 11: Segmentation fault
2018-01-08 20:52:16.344 -03 [14804]: [5-1] user=,db=,app=,client= LOG:  checkpointer process (PID 14828) was terminated by signal 11: Segmentation fault

(gdb) bt
#0  ckpt_buforder_comparator (pa=pa@entry=0x7f6fa9ef4b2c, pb=pb@entry=0x1be06d2d06644) at bufmgr.c:4137
#1  0x0000000000801268 in med3 (a=0x7f6fa9ef4b2c "\177\006", b=0x1be06d2d06644 <Address 0x1be06d2d06644 out of bounds>, c=0x2fc9dfbb1815c <Address 0x2fc9dfbb1815c out of bounds>, cmp=0x6a4d20 <ckpt_buforder_comparator>)
    at qsort.c:107
#2  0x0000000000801621 in pg_qsort (a=0x7f6fa9ef4b2c, a@entry=0x7f6fa9ea8380, n=<optimized out>, es=es@entry=20, cmp=cmp@entry=0x6a4d20 <ckpt_buforder_comparator>) at qsort.c:157
#3  0x00000000008015e2 in pg_qsort (a=0x7f6fa9ea8380, n=<optimized out>, n@entry=111473, es=es@entry=20, cmp=cmp@entry=0x6a4d20 <ckpt_buforder_comparator>) at qsort.c:203
#4  0x00000000006a81cf in BufferSync (flags=flags@entry=128) at bufmgr.c:1863
#5  0x00000000006a8477 in CheckPointBuffers (flags=flags@entry=128) at bufmgr.c:2578
#6  0x00000000004dd781 in CheckPointGuts (checkPointRedo=<optimized out>, flags=<optimized out>) at xlog.c:8698
#7  0x00000000004e9faf in CreateRestartPoint (flags=<optimized out>) at xlog.c:8856
#8  0x000000000066977c in CheckpointerMain () at checkpointer.c:490
#9  0x00000000004f2820 in AuxiliaryProcessMain (argc=argc@entry=2, argv=argv@entry=0x7ffd8bac2b80) at bootstrap.c:429
#10 0x0000000000673330 in StartChildProcess (type=CheckpointerProcess) at postmaster.c:5252
#11 0x0000000000674b1f in sigusr1_handler (postgres_signal_arg=<optimized out>) at postmaster.c:4949
#12 <signal handler called>
#13 0x00007f6fc75a0b83 in __select_nocancel () from /lib64/libc.so.6
#14 0x000000000046ef32 in ServerLoop () at postmaster.c:1683
#15 0x0000000000675b69 in PostmasterMain (argc=argc@entry=3, argv=argv@entry=0x27ca210) at postmaster.c:1327
#16 0x000000000047053e in main (argc=3, argv=0x27ca210) at main.c:228



#0  ckpt_buforder_comparator (pa=pa@entry=0x7f6fa9ef4b2c, pb=pb@entry=0x1be06d2d06644) at bufmgr.c:4137
        a = 0x7f6fa9ef4b2c
        b = 0x1be06d2d06644
#1  0x0000000000801268 in med3 (a=0x7f6fa9ef4b2c "\177\006", b=0x1be06d2d06644 <Address 0x1be06d2d06644 out of bounds>, c=0x2fc9dfbb1815c <Address 0x2fc9dfbb1815c out of bounds>, cmp=0x6a4d20 <ckpt_buforder_comparator>)
    at qsort.c:107
No locals.
#2  0x0000000000801621 in pg_qsort (a=0x7f6fa9ef4b2c, a@entry=0x7f6fa9ea8380, n=<optimized out>, es=es@entry=20, cmp=cmp@entry=0x6a4d20 <ckpt_buforder_comparator>) at qsort.c:157
        d = 350293923535640
        pa = <optimized out>
        pb = <optimized out>
        pc = <optimized out>
        pd = <optimized out>
        pl = 0x7f6fa9ef4b2c "\177\006"
        pm = 0x7f6fa9f06174 "\177\006"
        pn = 0xa7428f0f82428 <Address 0xa7428f0f82428 out of bounds>
        d1 = <optimized out>
        d2 = <optimized out>
        r = <optimized out>
        swaptype = 2
        presorted = 0
#3  0x00000000008015e2 in pg_qsort (a=0x7f6fa9ea8380, n=<optimized out>, n@entry=111473, es=es@entry=20, cmp=cmp@entry=0x6a4d20 <ckpt_buforder_comparator>) at qsort.c:203
        pa = <optimized out>
        pb = 0x7f6fa9f3a49c "\177\006"
        pc = <optimized out>
        pd = 0x7f6faa0c8840 "\177\006"
        pl = <optimized out>
        pm = <optimized out>
        pn = 0x7f6faa0c8854 "\177\006"
        d1 = <optimized out>
        d2 = 1631160
        r = <optimized out>
        swaptype = 2
        presorted = 0
#4  0x00000000006a81cf in BufferSync (flags=flags@entry=128) at bufmgr.c:1863
        buf_state = <optimized out>
        buf_id = 1835008
        num_to_scan = 111473
        num_spaces = <optimized out>
        num_processed = <optimized out>
        num_written = <optimized out>
        per_ts_stat = 0x0
        last_tsid = <optimized out>
        ts_heap = 0x7f6c266a8340
        i = <optimized out>
        mask = -2139095040
wb_context = {max_pending = 0xc287ec <checkpoint_flush_after>, nr_pending = 0, pending_writebacks = {{tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606246}, forkNum = MAIN_FORKNUM, blockNum = 428}}, {tag = {
                rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606246}, forkNum = MAIN_FORKNUM, blockNum = 429}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606252}, forkNum = MAIN_FORKNUM,
                blockNum = 54}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606252}, forkNum = MAIN_FORKNUM, blockNum = 58}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606252},
                forkNum = MAIN_FORKNUM, blockNum = 81}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606252}, forkNum = MAIN_FORKNUM, blockNum = 98}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060,
                  relNode = 412606252}, forkNum = MAIN_FORKNUM, blockNum = 158}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606252}, forkNum = MAIN_FORKNUM, blockNum = 160}}, {tag = {rnode = {spcNode = 1663,
                  dbNode = 69060, relNode = 412606252}, forkNum = MAIN_FORKNUM, blockNum = 173}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606252}, forkNum = MAIN_FORKNUM, blockNum = 177}}, {tag = {rnode = {
                  spcNode = 1663, dbNode = 69060, relNode = 412606252}, forkNum = MAIN_FORKNUM, blockNum = 191}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606257}, forkNum = MAIN_FORKNUM, blockNum = 24}}, {
              tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606257}, forkNum = MAIN_FORKNUM, blockNum = 31}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 412606257}, forkNum = MAIN_FORKNUM,
                blockNum = 36}}, {tag = {rnode = {spcNode = 1663, dbNode = 8156905, relNode = 0}, forkNum = VISIBILITYMAP_FORKNUM, blockNum = 1}}, {tag = {rnode = {spcNode = 1663, dbNode = 69060, relNode = 20971520},
                forkNum = 14828, blockNum = 825242228}}, {tag = {rnode = {spcNode = 825240888, dbNode = 540553261, relNode = 893005874}, forkNum = 942815793, blockNum = 909194542}}, {tag = {rnode = {spcNode = 858795296,
                  dbNode = 875649824, relNode = 1563963960}, forkNum = 844832826, blockNum = 825046578}}, {tag = {rnode = {spcNode = 1937055837, dbNode = 742224485, relNode = 742220388}, forkNum = 1030778977,
                blockNum = 1768710956}}, {tag = {rnode = {spcNode = 1031040613, dbNode = 1196379168, relNode = 1914708026}, forkNum = 1635021669, blockNum = 8156905}}, {tag = {rnode = {spcNode = 0, dbNode = 2, relNode = 1},
                forkNum = 1920409658, blockNum = 543519855}}, {tag = {rnode = {spcNode = 6684672, dbNode = 14828, relNode = 825242228}, forkNum = 825240888, blockNum = 540553261}}, {tag = {rnode = {spcNode = 893005874,
                  dbNode = 943012401, relNode = 942945582}, forkNum = 858795296, blockNum = 875649824}}, {tag = {rnode = {spcNode = 1563963960, dbNode = 844832826, relNode = 825047346}, forkNum = 1937055837,
                blockNum = 742224485}}, {tag = {rnode = {spcNode = 742220388, dbNode = 1030778977, relNode = 1768710956}, forkNum = 1031040613, blockNum = 1196379168}}, {tag = {rnode = {spcNode = 1914708026, dbNode = 1635021669,
                  relNode = 1869640818}, forkNum = 544501353, blockNum = 1918989427}}, {tag = {rnode = {spcNode = 1735289204, dbNode = 1769218106, relNode = 1862952301}, forkNum = 1030513012, blockNum = 775501362}}, {tag = {
                rnode = {spcNode = 540096568, dbNode = 1931492211, relNode = 543387257}, forkNum = 1701603686, blockNum = 859323763}}, {tag = {rnode = {spcNode = 1814047799, dbNode = 1701277295, relNode = 809333875},
                forkNum = 875573294, blockNum = 539783968}}, {tag = {rnode = {spcNode = 1919252065, dbNode = 1030055777, relNode = 808463920}, forkNum = 997400624, blockNum = 1936286752}}, {tag = {rnode = {spcNode = 1668178292,
                  dbNode = 809057637, relNode = 808794162}, forkNum = 742550304, blockNum = 1953719584}}, {tag = {rnode = {spcNode = 1952542057, dbNode = 808533349, relNode = 960049456}, forkNum = 1114316856, blockNum = 8476938}},
            {tag = {rnode = {spcNode = 0, dbNode = 8477036, relNode = 0}, forkNum = -951086535, blockNum = 32623}}, {tag = {rnode = {spcNode = 6, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 4294967295, relNode = 4294967295}, forkNum = -951080648, blockNum = 32623}}, {tag = {rnode = {spcNode = 2, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 0, relNode = 0}, forkNum = -1951655728, blockNum = 32765}}, {tag = {rnode = {spcNode = 2343311504, dbNode = 32765, relNode = 32}, forkNum = MAIN_FORKNUM, blockNum = 8459196}}, {tag = {
                rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 8156905, dbNode = 0, relNode = 2}, forkNum = FSM_FORKNUM, blockNum = 0}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 17235968, relNode = 14828}, forkNum = 825242228, blockNum = 825240888}}, {tag = {rnode = {spcNode = 540553261, dbNode = 893005874, relNode = 942815793}, forkNum = 909194542,
                blockNum = 858795296}}, {tag = {rnode = {spcNode = 875649824, dbNode = 1563963960, relNode = 844832826}, forkNum = 825046834, blockNum = 1937055837}}, {tag = {rnode = {spcNode = 742224485, dbNode = 742220388,
                  relNode = 1030778977}, forkNum = 1768710956, blockNum = 1031040613}}, {tag = {rnode = {spcNode = 1196379168, dbNode = 1914708026, relNode = 1987011429}, forkNum = 544830053, blockNum = 1953719666}}, {tag = {
                rnode = {spcNode = 544502369, dbNode = 1852403568, relNode = 1952522356}, forkNum = 1110782752, blockNum = 959983411}}, {tag = {rnode = {spcNode = 825312577, dbNode = 839528498, relNode = 758657328},
                forkNum = 808268080, blockNum = 808591416}}, {tag = {rnode = {spcNode = 976303418, dbNode = 892221490, relNode = 757085745}, forkNum = 1528836912, blockNum = 842544177}}, {tag = {rnode = {spcNode = 540695864,
                  dbNode = 942813787, relNode = 542978349}, forkNum = 1919251317, blockNum = 1650732093}}, {tag = {rnode = {spcNode = 1885416509, dbNode = 1663843696, relNode = 1852139884}, forkNum = 1142963572,
                blockNum = 1229018181}}, {tag = {rnode = {spcNode = 538982988, dbNode = 1953718636, relNode = 1836016416}, forkNum = 1952803952, blockNum = 1948279909}}, {tag = {rnode = {spcNode = 1936613746, dbNode = 1769235297,
                  relNode = 1998614127}, forkNum = 1629516641, blockNum = 1869357172}}, {tag = {rnode = {spcNode = 1769218151, dbNode = 840983917, relNode = 758657328}, forkNum = 808268080, blockNum = 808591416}}, {tag = {rnode = {
                  spcNode = 976303418, dbNode = 858667058, relNode = 909523250}, forkNum = 171126829, blockNum = 10}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM,
                blockNum = 0}} <repeats 13 times>, {tag = {rnode = {spcNode = 6987421, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 2343312048}}, {tag = {rnode = {spcNode = 32765, dbNode = 0, relNode = 0},
                forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 15}, forkNum = MAIN_FORKNUM, blockNum = 3957080064}}, {tag = {rnode = {spcNode = 13747, dbNode = 6991741, relNode = 0},
                forkNum = 15, blockNum = 0}}, {tag = {rnode = {spcNode = 6985347, dbNode = 0, relNode = 707}, forkNum = MAIN_FORKNUM, blockNum = 6992616}}, {tag = {rnode = {spcNode = 0, dbNode = 15, relNode = 0}, forkNum = 15,
                blockNum = 0}}, {tag = {rnode = {spcNode = 2343313312, dbNode = 32765, relNode = 2343314560}, forkNum = 32765, blockNum = 1}}, {tag = {rnode = {spcNode = 0, dbNode = 6992908, relNode = 0}, forkNum = 2019518320,
                blockNum = 6778732}}, {tag = {rnode = {spcNode = 808464432, dbNode = 858796080, relNode = 808464432}, forkNum = 876754227, blockNum = 808464432}}, {tag = {rnode = {spcNode = 909193264, dbNode = 0, relNode = 0},
                forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0},
                forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 1}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 2343313776, relNode = 32765},
                forkNum = -949801968, blockNum = 32623}}, {tag = {rnode = {spcNode = 8449408, dbNode = 0, relNode = 3344033888}, forkNum = 32623, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0},
                forkNum = -1951653520, blockNum = 32765}}, {tag = {rnode = {spcNode = 2343313760, dbNode = 32765, relNode = 8449415}, forkNum = MAIN_FORKNUM, blockNum = 4}}, {tag = {rnode = {spcNode = 0, dbNode = 8449408,
                  relNode = 0}, forkNum = -951086535, blockNum = 32623}}, {tag = {rnode = {spcNode = 8459912, dbNode = 0, relNode = 42329571}, forkNum = MAIN_FORKNUM, blockNum = 8459902}}, {tag = {rnode = {spcNode = 0,
                  dbNode = 3343880761, relNode = 32623}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 3343886648, dbNode = 32623, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 0, relNode = 0}, forkNum = -1951654688, blockNum = 32765}}, {tag = {rnode = {spcNode = 3343886648, dbNode = 32623, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 1}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 0, relNode = 0}, forkNum = -4, blockNum = 4294967295}}, {tag = {rnode = {spcNode = 0, dbNode = 16, relNode = 6796}, forkNum = MAIN_FORKNUM, blockNum = 3344033888}}, {tag = {rnode = {
                  spcNode = 32623, dbNode = 9616944, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 32623}}, {tag = {rnode = {spcNode = 0, dbNode = 32765, relNode = 0}, forkNum = 32765, blockNum = 2343313700}}, {tag = {rnode = {
                  spcNode = 32765, dbNode = 0, relNode = 0}, forkNum = 4, blockNum = 32623}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 4294967295}}, {tag = {rnode = {
                  spcNode = 4294967295, dbNode = 3343886648, relNode = 32623}, forkNum = FSM_FORKNUM, blockNum = 32623}}, {tag = {rnode = {spcNode = 9616896, dbNode = 0, relNode = 3344033888}, forkNum = 32623, blockNum = 0}}, {
              tag = {rnode = {spcNode = 0, dbNode = 8449408, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 25, dbNode = 32765, relNode = 8449415}, forkNum = MAIN_FORKNUM, blockNum = 2}}, {tag = {
                rnode = {spcNode = 0, dbNode = 9616896, relNode = 0}, forkNum = -951086535, blockNum = 32623}}, {tag = {rnode = {spcNode = 9434791, dbNode = 0, relNode = 3343880761}, forkNum = 32623, blockNum = 28}}, {tag = {
                rnode = {spcNode = 0, dbNode = 2343312880, relNode = 32765}, forkNum = -951080648, blockNum = 32623}}, {tag = {rnode = {spcNode = 3343886648, dbNode = 32623, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {
              tag = {rnode = {spcNode = 4294967295, dbNode = 40, relNode = 48}, forkNum = -1951652928, blockNum = 32765}}, {tag = {rnode = {spcNode = 2343314176, dbNode = 32765, relNode = 2343314208}, forkNum = 32765,
                blockNum = 9434794}}, {tag = {rnode = {spcNode = 0, dbNode = 3, relNode = 0}, forkNum = 9434791, blockNum = 0}}, {tag = {rnode = {spcNode = 3343880761, dbNode = 32623, relNode = 58}, forkNum = MAIN_FORKNUM,
                blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 48, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = InvalidForkNumber,
                blockNum = 2343314320}}, {tag = {rnode = {spcNode = 32765, dbNode = 2343314304, relNode = 32765}, forkNum = 9564263, blockNum = 0}}, {tag = {rnode = {spcNode = 2343313056, dbNode = 32765, relNode = 3343886648},
                forkNum = 32623, blockNum = 3343880761}}, {tag = {rnode = {spcNode = 32623, dbNode = 3343886648, relNode = 32623}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 4294967293, dbNode = 4294967295,
                  relNode = 0}, forkNum = 10, blockNum = 229}}, {tag = {rnode = {spcNode = 0, dbNode = 3343886648, relNode = 32623}, forkNum = 32, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0},
                forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 32765, dbNode = 2343314149, relNode = 32765}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 2343314464, dbNode = 32765,
                  relNode = 2343314448}, forkNum = 32765, blockNum = 9734857}}, {tag = {rnode = {spcNode = 0, dbNode = 8448198, relNode = 0}, forkNum = 9734855, blockNum = 0}}, {tag = {rnode = {spcNode = 3343880761,
                  dbNode = 32623, relNode = 9616944}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 32623, dbNode = 0, relNode = 0}, forkNum = 9434791, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0,
                  relNode = 3}, forkNum = MAIN_FORKNUM, blockNum = 9434794}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 2343313296, dbNode = 32765,
                  relNode = 2343313232}, forkNum = 32765, blockNum = 0}}, {tag = {rnode = {spcNode = 32765, dbNode = 2343314608, relNode = 32765}, forkNum = MAIN_FORKNUM, blockNum = 32765}}, {tag = {rnode = {spcNode = 0,
                  dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 4, relNode = 32623}, forkNum = 32, blockNum = 48}}, {tag = {rnode = {spcNode = 0, dbNode = 32765,
                  relNode = 0}, forkNum = 32765, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = INIT_FORKNUM, blockNum = 4294967295}}, {tag = {rnode = {spcNode = 3343886648, dbNode = 32623,
                  relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 4294967295, relNode = 4294967295}, forkNum = -951080648, blockNum = 32623}}, {tag = {rnode = {spcNode = 0,
                  dbNode = 32623, relNode = 32}, forkNum = 48, blockNum = 2343314848}}, {tag = {rnode = {spcNode = 32765, dbNode = 0, relNode = 0}, forkNum = 9734855, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0,
                  relNode = 6}, forkNum = 32623, blockNum = 9734860}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 32765,
                  relNode = 2343314533}, forkNum = 32765, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 32765}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0,
                  relNode = 4294967295}, forkNum = InvalidForkNumber, blockNum = 3343886648}}, {tag = {rnode = {spcNode = 32623, dbNode = 0, relNode = 32623}, forkNum = 16, blockNum = 48}}, {tag = {rnode = {spcNode = 2343315152,
                  dbNode = 32765, relNode = 2343314944}, forkNum = 32765, blockNum = 9434791}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = INIT_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 9434794,
                  dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 4294967295, relNode = 4294967295}, forkNum = -1951652320, blockNum = 32765}}, {tag = {rnode = {
                  spcNode = 2343314960, dbNode = 32765, relNode = 41731220}, forkNum = MAIN_FORKNUM, blockNum = 9795134}}, {tag = {rnode = {spcNode = 0, dbNode = 41731192, relNode = 0}, forkNum = -951086535, blockNum = 32623}}, {
              tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 4}, forkNum = 10, blockNum = 16}}, {tag = {rnode = {spcNode = 48, dbNode = 2343315296, relNode = 32765}, forkNum = -1951652208, blockNum = 32765}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 3}}, {tag = {rnode = {spcNode = 32623, dbNode = 3343886648, relNode = 32623}, forkNum = -1951653488, blockNum = 32765}}, {tag = {rnode = {
                  spcNode = 2343313744, dbNode = 32765, relNode = 4294967295}, forkNum = InvalidForkNumber, blockNum = 3343886648}}, {tag = {rnode = {spcNode = 32623, dbNode = 0, relNode = 32623}, forkNum = MAIN_FORKNUM,
                blockNum = 48}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = 32765, blockNum = 9734855}}, {tag = {rnode = {spcNode = 0, dbNode = 32, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {
              tag = {rnode = {spcNode = 3343886648, dbNode = 32623, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 32765}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {
                rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 4294967295}}, {tag = {rnode = {spcNode = 4294967295, dbNode = 3343886648, relNode = 32623}, forkNum = 8, blockNum = 16}}, {tag = {
                rnode = {spcNode = 2309886240, dbNode = 1127760177, relNode = 2343313776}, forkNum = 32765, blockNum = 2343314384}}, {tag = {rnode = {spcNode = 32765, dbNode = 2343313776, relNode = 32765}, forkNum = 8449408,
                blockNum = 0}}, {tag = {rnode = {spcNode = 2343314152, dbNode = 32765, relNode = 1}, forkNum = MAIN_FORKNUM, blockNum = 1023}}, {tag = {rnode = {spcNode = 0, dbNode = 2343314384, relNode = 32765},
                forkNum = -950277931, blockNum = 32623}}, {tag = {rnode = {spcNode = 4222451713, dbNode = 0, relNode = 2343314384}, forkNum = 32765, blockNum = 2343314384}}, {tag = {rnode = {spcNode = 32765, dbNode = 2343314384,
                  relNode = 32765}, forkNum = -1951652912, blockNum = 32765}}, {tag = {rnode = {spcNode = 2343314409, dbNode = 32765, relNode = 2343315407}, forkNum = 32765, blockNum = 2343314384}}, {tag = {rnode = {
                  spcNode = 32765, dbNode = 2343315407, relNode = 32765}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 0, relNode = 0}, forkNum = 12, blockNum = 4}}, {tag = {rnode = {spcNode = 9616896, dbNode = 0, relNode = 2343305216}, forkNum = 942833661, blockNum = 0}}, {tag = {rnode = {spcNode = 0,
                  dbNode = 2343314512, relNode = 32765}, forkNum = -1951652784, blockNum = 32765}}, {tag = {rnode = {spcNode = 12, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 8}}, {tag = {rnode = {spcNode = 48,
                  dbNode = 2343315808, relNode = 32765}, forkNum = InvalidForkNumber, blockNum = 808615933}}, {tag = {rnode = {spcNode = 2343314048, dbNode = 32765, relNode = 2343314576}, forkNum = 32765, blockNum = 3347493952}}, {
              tag = {rnode = {spcNode = 32623, dbNode = 0, relNode = 0}, forkNum = -947468368, blockNum = 32623}}, {tag = {rnode = {spcNode = 1951653217, dbNode = 4294934530, relNode = 2343314080}, forkNum = 32765,
  blockNum = 2343314079}}, {tag = {rnode = {spcNode = 32765, dbNode = 2343314624, relNode = 32765}, forkNum = 12, blockNum = 0}}, {tag = {rnode = {spcNode = 9616896, dbNode = 0, relNode = 2343314408},
                forkNum = 32765, blockNum = 1}}, {tag = {rnode = {spcNode = 0, dbNode = 2343314096, relNode = 32765}, forkNum = 6796, blockNum = 0}}, {tag = {rnode = {spcNode = 155648, dbNode = 0, relNode = 526916352},
                forkNum = 32620, blockNum = 6}}, {tag = {rnode = {spcNode = 0, dbNode = 3347498848, relNode = 32623}, forkNum = -947468448, blockNum = 32623}}, {tag = {rnode = {spcNode = 1, dbNode = 0, relNode = 1951653025},
                forkNum = -32766, blockNum = 2052}}, {tag = {rnode = {spcNode = 0, dbNode = 513, relNode = 0}, forkNum = 64, blockNum = 0}}, {tag = {rnode = {spcNode = 561, dbNode = 155, relNode = 2343314272}, forkNum = 32765,
                blockNum = 8}}, {tag = {rnode = {spcNode = 0, dbNode = 1, relNode = 0}, forkNum = 118, blockNum = 120}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 2343314271}, forkNum = 32765, blockNum = 1}}, {tag = {
                rnode = {spcNode = 0, dbNode = 118, relNode = 120}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 124, dbNode = 32765, relNode = 6987421}, forkNum = MAIN_FORKNUM, blockNum = 4036731840}}, {
              tag = {rnode = {spcNode = 13747, dbNode = 2343314384, relNode = 32765}, forkNum = 124, blockNum = 0}}, {tag = {rnode = {spcNode = 384, dbNode = 0, relNode = 6}, forkNum = MAIN_FORKNUM, blockNum = 3347498848}}, {
              tag = {rnode = {spcNode = 32623, dbNode = 42324488, relNode = 0}, forkNum = 42324506, blockNum = 0}}, {tag = {rnode = {spcNode = 4, dbNode = 0, relNode = 3344354262}, forkNum = 32623, blockNum = 42324416}}, {tag = {
                rnode = {spcNode = 0, dbNode = 42324464, relNode = 0}, forkNum = -192, blockNum = 4294967295}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 3}, forkNum = MAIN_FORKNUM, blockNum = 3344353169}}, {tag = {
                rnode = {spcNode = 32623, dbNode = 128, relNode = 0}, forkNum = INIT_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 128, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 4029030200}}, {tag = {
                rnode = {spcNode = 13747, dbNode = 3, relNode = 0}, forkNum = INIT_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 3344353005, dbNode = 32623, relNode = 41743440}, forkNum = MAIN_FORKNUM, blockNum = 6985401}}, {
              tag = {rnode = {spcNode = 0, dbNode = 42324416, relNode = 0}, forkNum = 4895616, blockNum = 0}}, {tag = {rnode = {spcNode = 41743440, dbNode = 0, relNode = 6985401}, forkNum = MAIN_FORKNUM, blockNum = 42324416}}, {
              tag = {rnode = {spcNode = 0, dbNode = 6876528, relNode = 0}, forkNum = 41733255, blockNum = 0}}, {tag = {rnode = {spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 0, relNode = 0}, forkNum = MAIN_FORKNUM, blockNum = 0}}, {tag = {rnode = {spcNode = 2343314816, dbNode = 4, relNode = 3343787445}, forkNum = 32623, blockNum = 41680896}}, {tag = {rnode = {
                  spcNode = 0, dbNode = 0, relNode = 0}, forkNum = 41867936, blockNum = 0}}...}}
#5  0x00000000006a8477 in CheckPointBuffers (flags=flags@entry=128) at bufmgr.c:2578
No locals.
#6  0x00000000004dd781 in CheckPointGuts (checkPointRedo=<optimized out>, flags=<optimized out>) at xlog.c:8698
No locals.
#7  0x00000000004e9faf in CreateRestartPoint (flags=<optimized out>) at xlog.c:8856
        lastCheckPointRecPtr = <optimized out>
        lastCheckPointEndPtr = <optimized out>
        lastCheckPoint = <optimized out>
        PriorRedoPtr = <optimized out>
        xtime = <optimized out>
        __func__ = "CreateRestartPoint"
#8  0x000000000066977c in CheckpointerMain () at checkpointer.c:490
        ckpt_performed = 0 '\000'
        do_restartpoint = 1 '\001'
        flags = 128
        do_checkpoint = <optimized out>
        now = 1515455518
        elapsed_secs = 300
        cur_timeout = <optimized out>
        rc = <optimized out>
        local_sigjmp_buf = {{__jmpbuf = {140726946769872, -6702958939962280073, 2, 8447397, 0, 41868768, -6702958940010514569, 6701736465640912759}, __mask_was_saved = 1, __saved_mask = {__val = {18446744066192964103,
                140726946769792, 2, 8447397, 0, 41868768, 6968122, 0, 0, 0, 0, 0, 4, 8, 6700557, 1}}}}
        checkpointer_context = 0x27cbb88
        __func__ = "CheckpointerMain"
#9  0x00000000004f2820 in AuxiliaryProcessMain (argc=argc@entry=2, argv=argv@entry=0x7ffd8bac2b80) at bootstrap.c:429
        progname = 0x80e5a5 "postgres"
        flag = <optimized out>
        userDoption = 0x0
        __func__ = "AuxiliaryProcessMain"
#10 0x0000000000673330 in StartChildProcess (type=CheckpointerProcess) at postmaster.c:5252
        pid = <optimized out>
        av = {0x80e5a5 "postgres", 0x7ffd8bac2bd0 "-x4", 0x0, 0xffec105a240ead00 <Address 0xffec105a240ead00 out of bounds>, 0x1 <Address 0x1 out of bounds>, 0x39e6 <Address 0x39e6 out of bounds>, 0x7ffd8bac2bf4 "",
          0x27ede00 "\260\215\301", 0x7f6fc9248800 "", 0x0}
        ac = 2
        typebuf = "-x4\000\000\000\000\000vCg\000\000\000\000\000\360>N\307o\177\000\000\065\233N\307o\177\000"
#11 0x0000000000674b1f in sigusr1_handler (postgres_signal_arg=<optimized out>) at postmaster.c:4949
        save_errno = 0
        __func__ = "sigusr1_handler"
#12 <signal handler called>
No symbol table info available.
#13 0x00007f6fc75a0b83 in __select_nocancel () from /lib64/libc.so.6
No symbol table info available.
#14 0x000000000046ef32 in ServerLoop () at postmaster.c:1683
        timeout = {tv_sec = 59, tv_usec = 999708}
        rmask = {fds_bits = {120, 0 <repeats 15 times>}}
        selres = <optimized out>
        now = <optimized out>
        readmask = {fds_bits = {120, 0 <repeats 15 times>}}
        last_lockfile_recheck_time = 1515438348
        last_touch_time = 1515438348
        __func__ = "ServerLoop"
#15 0x0000000000675b69 in PostmasterMain (argc=argc@entry=3, argv=argv@entry=0x27ca210) at postmaster.c:1327
        opt = <optimized out>
        status = <optimized out>
        userDoption = <optimized out>
        listen_addr_saved = <optimized out>
        i = <optimized out>
        output_config_variable = <optimized out>
        __func__ = "PostmasterMain"
#16 0x000000000047053e in main (argc=3, argv=0x27ca210) at main.c:228


Kind Regards,
    Glauco Torres