PostgreSQL 9.2.9 segfault (backtrace included) - Mailing list pgsql-bugs

We have been seeing regular segfaults on one of our data warehouse servers
lately. I have been able get a backtrace:


Core was generated by `postgres: username dbname 10.20.0.51(32831) CREATE
TABLE AS'.
Program terminated with signal 11, Segmentation fault.
#0  ExecEvalScalarVar (exprstate=0x7f2b52b2e020, econtext=0x7f2b52b2e080,
isNull=0x7fff1df38a5f "", isDone=0x0)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execQual.c:625
625
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execQual.c:
No such file or directory.
        in
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execQual.c
(gdb) bt
#0  ExecEvalScalarVar (exprstate=0x7f2b52b2e020, econtext=0x7f2b52b2e080,
isNull=0x7fff1df38a5f "", isDone=0x0)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execQual.c:625
#1  0x00007f2b4f183d5d in ExecIndexEvalRuntimeKeys
(econtext=0x7f2b52b2e080, runtimeKeys=<optimized out>, numRuntimeKeys=1)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeIndexscan.c:234
#2  0x00007f2b4f183f85 in ExecReScanIndexScan (node=0x7f2b52b2a888) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeIndexscan.c:181
#3  0x00007f2b4f16c53d in ExecReScan (node=0x7f2b52b2a888) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execAmi.c:156
#4  0x00007f2b4f184015 in ExecIndexScan (node=0x7f2b52b2e020) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeIndexscan.c:148
#5  0x00007f2b4f170468 in ExecProcNode (node=0x7f2b52b2a888) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:403
#6  0x00007f2b4f17cc41 in ExecAppend (node=0x7f2b52b26900) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeAppend.c:209
#7  0x00007f2b4f1704a8 in ExecProcNode (node=0x7f2b52b26900) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:380
#8  0x00007f2b4f1890f0 in ExecNestLoop (node=0x7f2b52b26610) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeNestloop.c:123
#9  0x00007f2b4f1703c8 in ExecProcNode (node=0x7f2b52b26610) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:448
#10 0x00007f2b4f182a9e in ExecHashJoin (node=0x7f2b52b24790) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeHashjoin.c:153
#11 0x00007f2b4f1703a8 in ExecProcNode (node=0x7f2b52b24790) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:456
#12 0x00007f2b4f17ee20 in agg_fill_hash_table (aggstate=<optimized out>) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeAgg.c:1239
#13 ExecAgg (node=0x7f2b52b24240) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeAgg.c:1007
#14 0x00007f2b4f170368 in ExecProcNode (node=0x7f2b52b24240) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:475
#15 0x00007f2b4f18abb1 in ExecSort (node=0x7f2b52b24130) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeSort.c:103
#16 0x00007f2b4f170388 in ExecProcNode (node=0x7f2b52b24130) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:467
#17 0x00007f2b4f18b554 in CteScanNext (node=0x7f2b52fecf10) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeCtescan.c:103
#18 0x00007f2b4f177f1a in ExecScanFetch (recheckMtd=<optimized out>,
accessMtd=<optimized out>, node=<optimized out>)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execScan.c:82
#19 ExecScan (node=0x7f2b52fecf10, accessMtd=<optimized out>,
recheckMtd=<optimized out>)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execScan.c:167
#20 0x00007f2b4f1703f8 in ExecProcNode (node=0x7f2b52fecf10) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:433
#21 0x00007f2b4f184ea8 in ExecLimit (node=0x7f2b52fecc70) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeLimit.c:91
#22 0x00007f2b4f170305 in ExecProcNode (node=0x7f2b52fecc70) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:499
#23 0x00007f2b4f18b554 in CteScanNext (node=0x7f2b52ffb9a0) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeCtescan.c:103
#24 0x00007f2b4f17806e in ExecScanFetch (recheckMtd=<optimized out>,
accessMtd=<optimized out>, node=<optimized out>)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execScan.c:82
#25 ExecScan (node=0x7f2b52ffb9a0, accessMtd=<optimized out>,
recheckMtd=<optimized out>)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execScan.c:132
#26 0x00007f2b4f1703f8 in ExecProcNode (node=0x7f2b52ffb9a0) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:433
#27 0x00007f2b4f1890f0 in ExecNestLoop (node=0x7f2b52ffb6b0) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeNestloop.c:123
#28 0x00007f2b4f1703c8 in ExecProcNode (node=0x7f2b52ffb6b0) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:448
#29 0x00007f2b4f1890f0 in ExecNestLoop (node=0x7f2b52ffb340) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeNestloop.c:123
#30 0x00007f2b4f1703c8 in ExecProcNode (node=0x7f2b52ffb340) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:448
#31 0x00007f2b4f1890f0 in ExecNestLoop (node=0x7f2b52ffaf50) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeNestloop.c:123
#32 0x00007f2b4f1703c8 in ExecProcNode (node=0x7f2b52ffaf50) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:448
#33 0x00007f2b4f1890f0 in ExecNestLoop (node=0x7f2b52ffaa60) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/nodeNestloop.c:123
#34 0x00007f2b4f1703c8 in ExecProcNode (node=0x7f2b52ffaa60) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execProcnode.c:448
#35 0x00007f2b4f16f1e7 in ExecutePlan (dest=<optimized out>,
direction=<optimized out>, numberTuples=<optimized out>,
sendTuples=<optimized out>,
    operation=<optimized out>, planstate=<optimized out>, estate=<optimized
out>)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execMain.c:1396
#36 standard_ExecutorRun (queryDesc=0x7f2b52a63488, direction=1387454592,
count=0)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/executor/execMain.c:304
#37 0x00007f2b4b05c8e9 in pgss_ExecutorRun (queryDesc=<optimized out>,
direction=<optimized out>, count=<optimized out>)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../contrib/pg_stat_statements/pg_stat_statements.c:722
#38 0x00007f2b4f11a9a9 in ExecCreateTableAs (stmt=<optimized out>,
queryString=<optimized out>, params=<optimized out>,
completionTag=0x7fff1df39940 "")
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/commands/createas.c:144
#39 0x00007f2b4b05c62f in pgss_ProcessUtility (parsetree=<optimized out>,
queryString=<optimized out>, params=<optimized out>, isTopLevel=0 '\000',
    dest=<optimized out>, completionTag=<optimized out>) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../contrib/pg_stat_statements/pg_stat_statements.c:827
#40 0x00007f2b4f253ee7 in PortalRunUtility (portal=0x7f2b502a6e90,
utilityStmt=0x7f2b504ce200, isTopLevel=0 '\000', dest=0x7f2b504ced80,
    completionTag=0x7fff1df39940 "") at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/tcop/pquery.c:1187
#41 0x00007f2b4f254f36 in PortalRunMulti (portal=0x7f2b502a6e90,
isTopLevel=0 '\000', dest=<optimized out>, altdest=<optimized out>,
    completionTag=<optimized out>) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/tcop/pquery.c:1318
#42 0x00007f2b4f25588d in PortalRun (portal=<optimized out>,
count=<optimized out>, isTopLevel=-128 '\200', dest=<optimized out>,
    altdest=<optimized out>, completionTag=<optimized out>) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/tcop/pquery.c:816
#43 0x00007f2b4f2517f8 in exec_simple_query (query_string=<optimized out>)
at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/tcop/postgres.c:1046
#44 0x00007f2b4f252858 in PostgresMain (argc=<optimized out>,
argv=<optimized out>, dbname=0x7f2b502877e0 "dbname", username=<optimized
out>)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/tcop/postgres.c:3968
#45 0x00007f2b4f209999 in BackendRun (port=<optimized out>) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/postmaster/postmaster.c:3617
#46 BackendStartup (port=<optimized out>) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/postmaster/postmaster.c:3299
#47 ServerLoop () at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/postmaster/postmaster.c:1362
#48 0x00007f2b4f20c961 in PostmasterMain (argc=<optimized out>,
argv=0x7f2b502861c0)
    at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/postmaster/postmaster.c:1122
#49 0x00007f2b4f1a5acf in main (argc=5, argv=0x7f2b502861a0) at
/tmp/buildd/postgresql-9.2-9.2.9/build/../src/backend/main/main.c:229

I will try to get hold of the whole query later, if needed. It's possible
that it involved selecting from a table with 968 (!) columns because I saw
a different query causing instant segfaults earlier, before I had installed
the -dbg package and enabled core dumps.

The OS is a fully patched Debian Squeeze and PostgreSQL is installed from
the apt.postgresql.org repo.

Regards,
Per Wigren