very slow selects on a small table - Mailing list pgsql-performance

From Brian Cox
Subject very slow selects on a small table
Date
Msg-id 4A393784.9090301@ca.com
Whole thread Raw
Responses Re: very slow selects on a small table
List pgsql-performance
There are 4 threads (4 postgres processes) loading all rows from a table
with 50,018 rows. The table has a int8 PK that is incremented by 1 for
each new row and the PK is used by the threads to partition the rows so
that each loads distinct rows. As you can see below, these 4 SELECTs
have been running since 6:30am (it's now 11:30am) -- sluggish at best --
and each of the postgres processes is using 100% CPU. The table schema
is long (~160 cols), so I'm omitting it but will provide it if deemed
necessary. Any ideas about the cause of this are appreciated.

Thanks,
Brian

cemdb=# select procpid, xact_start, current_query from pg_stat_activity;
  procpid |          xact_start           |
                                current_query

---------+-------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-----------------------------------------------------------------------------
    27825 |                               | <IDLE>
    27826 |                               | <IDLE>
    27824 |                               | <IDLE>
    27828 |                               | <IDLE>
    27827 |                               | <IDLE>
    27829 |                               | <IDLE>
    27830 |                               | <IDLE>
    27831 |                               | <IDLE>
    27832 |                               | <IDLE>
    27833 |                               | <IDLE>
    14031 | 2009-06-17 05:48:02.931503-07 | autovacuum: VACUUM ANALYZE
public.ts_stats_transet_user_weekly
    16044 |                               | <IDLE>
    32169 | 2009-06-17 08:17:39.034142-07 | autovacuum: VACUUM ANALYZE
public.ts_stats_transetgroup_user_weekly
     7165 |                               | <IDLE>
    16043 |                               | <IDLE>
    22130 | 2009-06-17 11:22:05.339582-07 | select procpid, xact_start,
current_query from pg_stat_activity;
     7169 | 2009-06-17 06:31:26.997641-07 | select a.ts_id as id1_0_,
a.version_info as versionI2_1_0_, a.ts_user_id as userId1_0_, null as
tranSetId1_0_, null as tranUnitId1_0_, null as userGro10_1_0_,
a.ts_transet_group_id as tranSetG7_1_0_, a.ts_last_aggregated_row as
lastAgg12_1_0_, null as tranSetI5_1_0_, a.ts_user_incarnation_id as
userInca9_1_0_, a.ts_interval_start_time as interva11_1_0_,
a.ts_interval_wire_time as interva13_1_0_, a.ts_total_transactions as
totalTr14_1_0_, a.ts_bad_transactions as badTran15_1_0_,
a.ts_opportunities as opportu16_1_0_, a.ts_defects as defects1_0_,
a.ts_data_type as dataType1_0_, a.ts_tth_type as tthType1_0_,
a.ts_tth_lower_spec as tthLowe20_1_0_, a.ts_tth_upper_spec as
tthUppe21_1_0_, a.ts_tth_b0 as tthB22_1_0_, a.ts_tth_b1 as tthB23_1_0_,
a.ts_tth_b2 as tthB24_1_0_, a.ts_tth_b3 as tthB25_1_0_, a.ts_tth_b4 as
tthB26_1_0_, a.ts_tth_b5 as tthB27_1_0_, a.ts_tth_b6 as tthB28_1_0_,
a.ts_tth_b7 as tthB29_1_0_, a.ts_tth_b8 as tthB30_1_0_, a.ts_tth_b9 as
tthB31_1_0_, a.ts_tth_b10 as tthB32_1_0_, a.ts_tth_b11 as tthB33_1
     7171 |                               | <IDLE>
     7172 |                               | <IDLE>
    28106 |                               | <IDLE>
     7392 | 2009-06-17 06:31:26.997985-07 | select a.ts_id as id1_0_,
a.version_info as versionI2_1_0_, a.ts_user_id as userId1_0_, null as
tranSetId1_0_, null as tranUnitId1_0_, null as userGro10_1_0_,
a.ts_transet_group_id as tranSetG7_1_0_, a.ts_last_aggregated_row as
lastAgg12_1_0_, null as tranSetI5_1_0_, a.ts_user_incarnation_id as
userInca9_1_0_, a.ts_interval_start_time as interva11_1_0_,
a.ts_interval_wire_time as interva13_1_0_, a.ts_total_transactions as
totalTr14_1_0_, a.ts_bad_transactions as badTran15_1_0_,
a.ts_opportunities as opportu16_1_0_, a.ts_defects as defects1_0_,
a.ts_data_type as dataType1_0_, a.ts_tth_type as tthType1_0_,
a.ts_tth_lower_spec as tthLowe20_1_0_, a.ts_tth_upper_spec as
tthUppe21_1_0_, a.ts_tth_b0 as tthB22_1_0_, a.ts_tth_b1 as tthB23_1_0_,
a.ts_tth_b2 as tthB24_1_0_, a.ts_tth_b3 as tthB25_1_0_, a.ts_tth_b4 as
tthB26_1_0_, a.ts_tth_b5 as tthB27_1_0_, a.ts_tth_b6 as tthB28_1_0_,
a.ts_tth_b7 as tthB29_1_0_, a.ts_tth_b8 as tthB30_1_0_, a.ts_tth_b9 as
tthB31_1_0_, a.ts_tth_b10 as tthB32_1_0_, a.ts_tth_b11 as tthB33_1
     7396 |                               | <IDLE>
     7397 | 2009-06-17 06:31:26.998013-07 | select a.ts_id as id1_0_,
a.version_info as versionI2_1_0_, a.ts_user_id as userId1_0_, null as
tranSetId1_0_, null as tranUnitId1_0_, null as userGro10_1_0_,
a.ts_transet_group_id as tranSetG7_1_0_, a.ts_last_aggregated_row as
lastAgg12_1_0_, null as tranSetI5_1_0_, a.ts_user_incarnation_id as
userInca9_1_0_, a.ts_interval_start_time as interva11_1_0_,
a.ts_interval_wire_time as interva13_1_0_, a.ts_total_transactions as
totalTr14_1_0_, a.ts_bad_transactions as badTran15_1_0_,
a.ts_opportunities as opportu16_1_0_, a.ts_defects as defects1_0_,
a.ts_data_type as dataType1_0_, a.ts_tth_type as tthType1_0_,
a.ts_tth_lower_spec as tthLowe20_1_0_, a.ts_tth_upper_spec as
tthUppe21_1_0_, a.ts_tth_b0 as tthB22_1_0_, a.ts_tth_b1 as tthB23_1_0_,
a.ts_tth_b2 as tthB24_1_0_, a.ts_tth_b3 as tthB25_1_0_, a.ts_tth_b4 as
tthB26_1_0_, a.ts_tth_b5 as tthB27_1_0_, a.ts_tth_b6 as tthB28_1_0_,
a.ts_tth_b7 as tthB29_1_0_, a.ts_tth_b8 as tthB30_1_0_, a.ts_tth_b9 as
tthB31_1_0_, a.ts_tth_b10 as tthB32_1_0_, a.ts_tth_b11 as tthB33_1
     7403 | 2009-06-17 06:31:26.998273-07 | select a.ts_id as id1_0_,
a.version_info as versionI2_1_0_, a.ts_user_id as userId1_0_, null as
tranSetId1_0_, null as tranUnitId1_0_, null as userGro10_1_0_,
a.ts_transet_group_id as tranSetG7_1_0_, a.ts_last_aggregated_row as
lastAgg12_1_0_, null as tranSetI5_1_0_, a.ts_user_incarnation_id as
userInca9_1_0_, a.ts_interval_start_time as interva11_1_0_,
a.ts_interval_wire_time as interva13_1_0_, a.ts_total_transactions as
totalTr14_1_0_, a.ts_bad_transactions as badTran15_1_0_,
a.ts_opportunities as opportu16_1_0_, a.ts_defects as defects1_0_,
a.ts_data_type as dataType1_0_, a.ts_tth_type as tthType1_0_,
a.ts_tth_lower_spec as tthLowe20_1_0_, a.ts_tth_upper_spec as
tthUppe21_1_0_, a.ts_tth_b0 as tthB22_1_0_, a.ts_tth_b1 as tthB23_1_0_,
a.ts_tth_b2 as tthB24_1_0_, a.ts_tth_b3 as tthB25_1_0_, a.ts_tth_b4 as
tthB26_1_0_, a.ts_tth_b5 as tthB27_1_0_, a.ts_tth_b6 as tthB28_1_0_,
a.ts_tth_b7 as tthB29_1_0_, a.ts_tth_b8 as tthB30_1_0_, a.ts_tth_b9 as
tthB31_1_0_, a.ts_tth_b10 as tthB32_1_0_, a.ts_tth_b11 as tthB33_1
    32571 | 2009-06-16 19:03:16.645352-07 | autovacuum: VACUUM ANALYZE
public.ts_stats_transet_user_daily
(25 rows)

cemdb=# select c.oid,c.relname,l.pid,l.mode,l.granted from pg_class c
join pg_locks l on c.oid=l.relation order by l.pid;
    oid    |                         relname                          |
  pid  |           mode           | granted
----------+----------------------------------------------------------+-------+--------------------------+---------
  26612634 | ts_stats_transetgroup_user_daily_monthindex              |
  7169 | AccessShareLock          | t
  26612631 | ts_stats_transetgroup_user_daily_dayindex                |
  7169 | AccessShareLock          | t
  26612532 | ts_stats_transet_user_interval_monthindex                |
  7169 | AccessShareLock          | t
  26612322 | ts_stats_transet_user_interval_pkey                      |
  7169 | AccessShareLock          | t
  26611729 | ts_stats_transet_user_interval                           |
  7169 | AccessShareLock          | t
  26612639 | ts_stats_transetgroup_user_daily_yearindex               |
  7169 | AccessShareLock          | t
  26612638 | ts_stats_transetgroup_user_daily_weekindex               |
  7169 | AccessShareLock          | t
  26612635 | ts_stats_transetgroup_user_daily_starttimeindex          |
  7169 | AccessShareLock          | t
  26612538 | ts_stats_transet_user_interval_yearindex                 |
  7169 | AccessShareLock          | t
  26612530 | ts_stats_transet_user_interval_dayindex                  |
  7169 | AccessShareLock          | t
  26612531 | ts_stats_transet_user_interval_hourindex                 |
  7169 | AccessShareLock          | t
  26612633 | ts_stats_transetgroup_user_daily_lastaggregatedrowindex  |
  7169 | AccessShareLock          | t
  26611806 | ts_stats_transetgroup_user_daily                         |
  7169 | AccessShareLock          | t
  26612835 | ts_transetgroup_transets_map_transetgroupidindex         |
  7169 | AccessShareLock          | t
  26612534 | ts_stats_transet_user_interval_transetindex              |
  7169 | AccessShareLock          | t
  26612637 | ts_stats_transetgroup_user_daily_userindex               |
  7169 | AccessShareLock          | t
  26611350 | ts_transetgroup_transets_map                             |
  7169 | AccessShareLock          | t
  26612636 | ts_stats_transetgroup_user_daily_transetgroupindex       |
  7169 | AccessShareLock          | t
  26612836 | ts_transetgroup_transets_map_transetidindex              |
  7169 | AccessShareLock          | t
  26612632 | ts_stats_transetgroup_user_daily_hourindex               |
  7169 | AccessShareLock          | t
  26612401 | ts_transetgroup_transets_map_pkey                        |
  7169 | AccessShareLock          | t
  26612351 | ts_stats_transetgroup_user_daily_pkey                    |
  7169 | AccessShareLock          | t
  26612536 | ts_stats_transet_user_interval_userindex                 |
  7169 | AccessShareLock          | t
  26612537 | ts_stats_transet_user_interval_weekindex                 |
  7169 | AccessShareLock          | t
  26612837 | ts_transetgroup_transets_map_transetincarnationindex     |
  7169 | AccessShareLock          | t
  26612533 | ts_stats_transet_user_interval_starttime                 |
  7169 | AccessShareLock          | t
  26612535 | ts_stats_transet_user_interval_userincarnationidindex    |
  7169 | AccessShareLock          | t
  26612537 | ts_stats_transet_user_interval_weekindex                 |
  7392 | AccessShareLock          | t
  26612634 | ts_stats_transetgroup_user_daily_monthindex              |
  7392 | AccessShareLock          | t
  26612632 | ts_stats_transetgroup_user_daily_hourindex               |
  7392 | AccessShareLock          | t
  26612530 | ts_stats_transet_user_interval_dayindex                  |
  7392 | AccessShareLock          | t
  26612633 | ts_stats_transetgroup_user_daily_lastaggregatedrowindex  |
  7392 | AccessShareLock          | t
  26612533 | ts_stats_transet_user_interval_starttime                 |
  7392 | AccessShareLock          | t
  26612637 | ts_stats_transetgroup_user_daily_userindex               |
  7392 | AccessShareLock          | t
  26612351 | ts_stats_transetgroup_user_daily_pkey                    |
  7392 | AccessShareLock          | t
  26612534 | ts_stats_transet_user_interval_transetindex              |
  7392 | AccessShareLock          | t
  26612631 | ts_stats_transetgroup_user_daily_dayindex                |
  7392 | AccessShareLock          | t
  26612535 | ts_stats_transet_user_interval_userincarnationidindex    |
  7392 | AccessShareLock          | t
  26612636 | ts_stats_transetgroup_user_daily_transetgroupindex       |
  7392 | AccessShareLock          | t
  26612639 | ts_stats_transetgroup_user_daily_yearindex               |
  7392 | AccessShareLock          | t
  26611729 | ts_stats_transet_user_interval                           |
  7392 | AccessShareLock          | t
  26612322 | ts_stats_transet_user_interval_pkey                      |
  7392 | AccessShareLock          | t
  26612538 | ts_stats_transet_user_interval_yearindex                 |
  7392 | AccessShareLock          | t
  26612837 | ts_transetgroup_transets_map_transetincarnationindex     |
  7392 | AccessShareLock          | t
  26612635 | ts_stats_transetgroup_user_daily_starttimeindex          |
  7392 | AccessShareLock          | t
  26612532 | ts_stats_transet_user_interval_monthindex                |
  7392 | AccessShareLock          | t
  26612835 | ts_transetgroup_transets_map_transetgroupidindex         |
  7392 | AccessShareLock          | t
  26612638 | ts_stats_transetgroup_user_daily_weekindex               |
  7392 | AccessShareLock          | t
  26612401 | ts_transetgroup_transets_map_pkey                        |
  7392 | AccessShareLock          | t
  26612536 | ts_stats_transet_user_interval_userindex                 |
  7392 | AccessShareLock          | t
  26612836 | ts_transetgroup_transets_map_transetidindex              |
  7392 | AccessShareLock          | t
  26611806 | ts_stats_transetgroup_user_daily                         |
  7392 | AccessShareLock          | t
  26612531 | ts_stats_transet_user_interval_hourindex                 |
  7392 | AccessShareLock          | t
  26611350 | ts_transetgroup_transets_map                             |
  7392 | AccessShareLock          | t
  26612538 | ts_stats_transet_user_interval_yearindex                 |
  7397 | AccessShareLock          | t
  26612537 | ts_stats_transet_user_interval_weekindex                 |
  7397 | AccessShareLock          | t
  26612533 | ts_stats_transet_user_interval_starttime                 |
  7397 | AccessShareLock          | t
  26612639 | ts_stats_transetgroup_user_daily_yearindex               |
  7397 | AccessShareLock          | t
  26612632 | ts_stats_transetgroup_user_daily_hourindex               |
  7397 | AccessShareLock          | t
  26612531 | ts_stats_transet_user_interval_hourindex                 |
  7397 | AccessShareLock          | t
  26612835 | ts_transetgroup_transets_map_transetgroupidindex         |
  7397 | AccessShareLock          | t
  26611350 | ts_transetgroup_transets_map                             |
  7397 | AccessShareLock          | t
  26612532 | ts_stats_transet_user_interval_monthindex                |
  7397 | AccessShareLock          | t
  26612836 | ts_transetgroup_transets_map_transetidindex              |
  7397 | AccessShareLock          | t
  26612322 | ts_stats_transet_user_interval_pkey                      |
  7397 | AccessShareLock          | t
  26612535 | ts_stats_transet_user_interval_userincarnationidindex    |
  7397 | AccessShareLock          | t
  26612637 | ts_stats_transetgroup_user_daily_userindex               |
  7397 | AccessShareLock          | t
  26612631 | ts_stats_transetgroup_user_daily_dayindex                |
  7397 | AccessShareLock          | t
  26612634 | ts_stats_transetgroup_user_daily_monthindex              |
  7397 | AccessShareLock          | t
  26611729 | ts_stats_transet_user_interval                           |
  7397 | AccessShareLock          | t
  26611806 | ts_stats_transetgroup_user_daily                         |
  7397 | AccessShareLock          | t
  26612636 | ts_stats_transetgroup_user_daily_transetgroupindex       |
  7397 | AccessShareLock          | t
  26612351 | ts_stats_transetgroup_user_daily_pkey                    |
  7397 | AccessShareLock          | t
  26612536 | ts_stats_transet_user_interval_userindex                 |
  7397 | AccessShareLock          | t
  26612638 | ts_stats_transetgroup_user_daily_weekindex               |
  7397 | AccessShareLock          | t
  26612530 | ts_stats_transet_user_interval_dayindex                  |
  7397 | AccessShareLock          | t
  26612837 | ts_transetgroup_transets_map_transetincarnationindex     |
  7397 | AccessShareLock          | t
  26612633 | ts_stats_transetgroup_user_daily_lastaggregatedrowindex  |
  7397 | AccessShareLock          | t
  26612401 | ts_transetgroup_transets_map_pkey                        |
  7397 | AccessShareLock          | t
  26612534 | ts_stats_transet_user_interval_transetindex              |
  7397 | AccessShareLock          | t
  26612635 | ts_stats_transetgroup_user_daily_starttimeindex          |
  7397 | AccessShareLock          | t
  26612639 | ts_stats_transetgroup_user_daily_yearindex               |
  7403 | AccessShareLock          | t
  26612351 | ts_stats_transetgroup_user_daily_pkey                    |
  7403 | AccessShareLock          | t
  26612536 | ts_stats_transet_user_interval_userindex                 |
  7403 | AccessShareLock          | t
  26612634 | ts_stats_transetgroup_user_daily_monthindex              |
  7403 | AccessShareLock          | t
  26612635 | ts_stats_transetgroup_user_daily_starttimeindex          |
  7403 | AccessShareLock          | t
  26612538 | ts_stats_transet_user_interval_yearindex                 |
  7403 | AccessShareLock          | t
  26612633 | ts_stats_transetgroup_user_daily_lastaggregatedrowindex  |
  7403 | AccessShareLock          | t
  26612530 | ts_stats_transet_user_interval_dayindex                  |
  7403 | AccessShareLock          | t
  26612837 | ts_transetgroup_transets_map_transetincarnationindex     |
  7403 | AccessShareLock          | t
  26612532 | ts_stats_transet_user_interval_monthindex                |
  7403 | AccessShareLock          | t
  26612322 | ts_stats_transet_user_interval_pkey                      |
  7403 | AccessShareLock          | t
  26611729 | ts_stats_transet_user_interval                           |
  7403 | AccessShareLock          | t
  26612535 | ts_stats_transet_user_interval_userincarnationidindex    |
  7403 | AccessShareLock          | t
  26612637 | ts_stats_transetgroup_user_daily_userindex               |
  7403 | AccessShareLock          | t
  26612836 | ts_transetgroup_transets_map_transetidindex              |
  7403 | AccessShareLock          | t
  26611350 | ts_transetgroup_transets_map                             |
  7403 | AccessShareLock          | t
  26612401 | ts_transetgroup_transets_map_pkey                        |
  7403 | AccessShareLock          | t
  26612631 | ts_stats_transetgroup_user_daily_dayindex                |
  7403 | AccessShareLock          | t
  26612636 | ts_stats_transetgroup_user_daily_transetgroupindex       |
  7403 | AccessShareLock          | t
  26611806 | ts_stats_transetgroup_user_daily                         |
  7403 | AccessShareLock          | t
  26612534 | ts_stats_transet_user_interval_transetindex              |
  7403 | AccessShareLock          | t
  26612835 | ts_transetgroup_transets_map_transetgroupidindex         |
  7403 | AccessShareLock          | t
  26612632 | ts_stats_transetgroup_user_daily_hourindex               |
  7403 | AccessShareLock          | t
  26612531 | ts_stats_transet_user_interval_hourindex                 |
  7403 | AccessShareLock          | t
  26612537 | ts_stats_transet_user_interval_weekindex                 |
  7403 | AccessShareLock          | t
  26612533 | ts_stats_transet_user_interval_starttime                 |
  7403 | AccessShareLock          | t
  26612638 | ts_stats_transetgroup_user_daily_weekindex               |
  7403 | AccessShareLock          | t
  26611743 | ts_stats_transet_user_weekly                             |
14031 | ShareUpdateExclusiveLock | t
  26612553 | ts_stats_transet_user_weekly_starttimeindex              |
14031 | RowExclusiveLock         | t
  26612326 | ts_stats_transet_user_weekly_pkey                        |
14031 | RowExclusiveLock         | t
  26612551 | ts_stats_transet_user_weekly_lastaggregatedrowindex      |
14031 | RowExclusiveLock         | t
  26612550 | ts_stats_transet_user_weekly_hourindex                   |
14031 | RowExclusiveLock         | t
  26612552 | ts_stats_transet_user_weekly_monthindex                  |
14031 | RowExclusiveLock         | t
  26612549 | ts_stats_transet_user_weekly_dayindex                    |
14031 | RowExclusiveLock         | t
  26612558 | ts_stats_transet_user_weekly_yearindex                   |
14031 | RowExclusiveLock         | t
  26612554 | ts_stats_transet_user_weekly_transetindex                |
14031 | RowExclusiveLock         | t
  26612556 | ts_stats_transet_user_weekly_userindex                   |
14031 | RowExclusiveLock         | t
  26612557 | ts_stats_transet_user_weekly_weekindex                   |
14031 | RowExclusiveLock         | t
  26612555 | ts_stats_transet_user_weekly_userincarnationidindex      |
14031 | RowExclusiveLock         | t
     10969 | pg_locks                                                 |
22130 | AccessShareLock          | t
      1259 | pg_class                                                 |
22130 | AccessShareLock          | t
      2662 | pg_class_oid_index                                       |
22130 | AccessShareLock          | t
      2663 | pg_class_relname_nsp_index                               |
22130 | AccessShareLock          | t
  26612659 | ts_stats_transetgroup_user_weekly_hourindex              |
32169 | AccessShareLock          | t
  26612658 | ts_stats_transetgroup_user_weekly_dayindex               |
32169 | AccessShareLock          | t
  26612666 | ts_stats_transetgroup_user_weekly_yearindex              |
32169 | AccessShareLock          | t
  26612665 | ts_stats_transetgroup_user_weekly_weekindex              |
32169 | AccessShareLock          | t
  26612661 | ts_stats_transetgroup_user_weekly_monthindex             |
32169 | AccessShareLock          | t
  26612663 | ts_stats_transetgroup_user_weekly_transetgroupindex      |
32169 | AccessShareLock          | t
  26612662 | ts_stats_transetgroup_user_weekly_starttimeindex         |
32169 | AccessShareLock          | t
  26611827 | ts_stats_transetgroup_user_weekly                        |
32169 | ShareUpdateExclusiveLock | t
  26612660 | ts_stats_transetgroup_user_weekly_lastaggregatedrowindex |
32169 | AccessShareLock          | t
  26612664 | ts_stats_transetgroup_user_weekly_userindex              |
32169 | AccessShareLock          | t
  26612357 | ts_stats_transetgroup_user_weekly_pkey                   |
32169 | AccessShareLock          | t
  27208304 | ts_stats_transet_user_daily_starttimeindex               |
32571 | RowExclusiveLock         | t
  26612320 | ts_stats_transet_user_daily_pkey                         |
32571 | RowExclusiveLock         | t
  27208300 | ts_stats_transet_user_daily_dayindex                     |
32571 | RowExclusiveLock         | t
  27208305 | ts_stats_transet_user_daily_transetincarnationidindex    |
32571 | RowExclusiveLock         | t
  27208302 | ts_stats_transet_user_daily_lastaggregatedrowindex       |
32571 | RowExclusiveLock         | t
  27208310 | ts_stats_transet_user_daily_yearindex                    |
32571 | RowExclusiveLock         | t
  27208309 | ts_stats_transet_user_daily_weekindex                    |
32571 | RowExclusiveLock         | t
  27208307 | ts_stats_transet_user_daily_userincarnationidindex       |
32571 | RowExclusiveLock         | t
  26611722 | ts_stats_transet_user_daily                              |
32571 | ShareUpdateExclusiveLock | t
  27208301 | ts_stats_transet_user_daily_hourindex                    |
32571 | RowExclusiveLock         | t
  27208306 | ts_stats_transet_user_daily_transetindex                 |
32571 | RowExclusiveLock         | t
  27208303 | ts_stats_transet_user_daily_monthindex                   |
32571 | RowExclusiveLock         | t
  27208308 | ts_stats_transet_user_daily_userindex                    |
32571 | RowExclusiveLock         | t
(148 rows)

[root@rdl64xeoserv01 log]# strace -p 7397
Process 7397 attached - interrupt to quit
munmap(0x95393000, 1052672)             = 0
munmap(0x95494000, 528384)              = 0
munmap(0x95515000, 266240)              = 0
brk(0x8603000)                          = 0x8603000
brk(0x85fb000)                          = 0x85fb000
_llseek(144, 0, [292618240], SEEK_END)  = 0
brk(0x85eb000)                          = 0x85eb000
_llseek(65, 897179648, [897179648], SEEK_SET) = 0
read(65, "\276\2\0\0\320\337\275\315\1\0\0\0|\3`\22\360\37\4 \0\0"...,
8192) = 8192
_llseek(65, 471457792, [471457792], SEEK_SET) = 0
read(65, "\276\2\0\0\320\337\275\315\1\0\0\0t\6\200\6\360\37\4 \0"...,
8192) = 8192
read(65, "\276\2\0\0\354\271\355\312\1\0\0\0\374\5`\10\360\37\4 "...,
8192) = 8192
read(65, "\0\0\0\0\0\0\0\0\1\0\0\0\324\5\0\t\360\37\4 \0\0\0\0\0"...,
8192) = 8192
brk(0x8613000)                          = 0x8613000
mmap2(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x95515000
mmap2(NULL, 528384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x95494000
mmap2(NULL, 1052672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x95393000
munmap(0x95393000, 1052672)             = 0
munmap(0x95494000, 528384)              = 0
munmap(0x95515000, 266240)              = 0
brk(0x8603000)                          = 0x8603000
brk(0x85fb000)                          = 0x85fb000
_llseek(144, 0, [292618240], SEEK_END)  = 0
brk(0x85eb000)                          = 0x85eb000
_llseek(65, 86941696, [86941696], SEEK_SET) = 0
read(65, "\304\2\0\0\30\333\v\244\1\0\0\0\324\5\0\t\360\37\4 \0\0"...,
8192) = 8192
_llseek(65, 892682240, [892682240], SEEK_SET) = 0
read(65, "\276\2\0\0\fq\231\310\1\0\0\0|\3`\22\360\37\4 \0\0\0\0"...,
8192) = 8192
_llseek(65, 86949888, [86949888], SEEK_SET) = 0
read(65, "\276\2\0\0\fq\231\310\1\0\0\0t\6\200\6\360\37\4 \0\0\0"...,
8192) = 8192
read(65, "\276\2\0\0\270*\341\305\1\0\0\0\364\5\200\10\360\37\4 "...,
8192) = 8192
read(65, "\0\0\0\0\0\0\0\0\1\0\0\0\324\5\0\t\360\37\4 \0\0\0\0\0"...,
8192) = 8192
read(65, "\304\2\0\0\320`\225\242\1\0\0\0000\4\220\17\360\37\4 \0"...,
8192) = 8192
brk(0x8613000)                          = 0x8613000
mmap2(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x95515000
mmap2(NULL, 528384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x95494000
mmap2(NULL, 1052672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x95393000
munmap(0x95393000, 1052672)             = 0
munmap(0x95494000, 528384)              = 0
munmap(0x95515000, 266240)              = 0
brk(0x8603000)                          = 0x8603000
brk(0x85fb000)                          = 0x85fb000
_llseek(144, 0, [292618240], SEEK_END)  = 0
brk(0x85eb000)                          = 0x85eb000
brk(0x8613000)                          = 0x8613000
mmap2(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x95515000
mmap2(NULL, 528384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x95494000
mmap2(NULL, 1052672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x95393000
munmap(0x95393000, 1052672)             = 0
munmap(0x95494000, 528384)              = 0
munmap(0x95515000, 266240)              = 0
brk(0x8603000)                          = 0x8603000
brk(0x85fb000)                          = 0x85fb000
_llseek(144, 0, [292618240], SEEK_END)  = 0
brk(0x85eb000)                          = 0x85eb000
_llseek(65, 115359744, [115359744], SEEK_SET) = 0
read(65, "\304\2\0\0\200e\331\243\1\0\0\0`\6\320\6\360\37\4 \0\0"...,
8192) = 8192
_llseek(65, 827695104, [827695104], SEEK_SET) = 0
read(65, "\273\2\0\0\20\36uY\1\0\0\0|\3`\22\360\37\4 \0\0\0\0\340"...,
8192) = 8192
_llseek(65, 115367936, [115367936], SEEK_SET) = 0
read(65, "\273\2\0\0\20\36uY\1\0\0\0\10\0060\10\360\37\4 \0\0\0\0"...,
8192) = 8192
read(65, "\0\0\0\0\0\0\0\0\1\0\0\0\324\5\0\t\360\37\4 \0\0\0\0\0"...,
8192) = 8192
read(65, "\0\0\0\0\0\0\0\0\1\0\0\0\324\5\0\t\360\37\4 \0\0\0\0\0"...,
8192) = 8192
read(65, "\304\2\0\0\344M>\242\1\0\0\0L\3 \23\360\37\4 \0\0\0\0\340"...,
8192) = 8192
brk(0x8613000)                          = 0x8613000
mmap2(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x95515000
mmap2(NULL, 528384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x95494000
mmap2(NULL, 1052672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x95393000
munmap(0x95393000, 1052672)             = 0
munmap(0x95494000, 528384)              = 0
munmap(0x95515000, 266240)              = 0

strace output from the other 3 postgres processes looks the same.

pgsql-performance by date:

Previous
From: "Mark Steben"
Date:
Subject: Re: Performance issue - 2 linux machines, identical configs, different performance
Next
From: Bryce Ewing
Date:
Subject: Re: Index Scan taking long time