From 821a302038386afdb5cb443d20e7781cf29f9da8 Mon Sep 17 00:00:00 2001 From: Mark Kirkwood Date: Fri, 17 Jun 2011 11:49:21 +1200 Subject: [PATCH] V5 temp file limit. --- doc/src/sgml/config.sgml | 37 +++++++++++++++++++++++++ src/backend/storage/file/fd.c | 37 ++++++++++++++++++++++++- src/backend/utils/misc/guc.c | 12 ++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/utils/guc.h | 1 + src/include/utils/guc_tables.h | 1 + src/test/regress/expected/resource.out | 18 ++++++++++++ src/test/regress/serial_schedule | 1 + src/test/regress/sql/resource.sql | 17 +++++++++++ 9 files changed, 124 insertions(+), 1 deletions(-) create mode 100644 src/test/regress/expected/resource.out create mode 100644 src/test/regress/sql/resource.sql diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml new file mode 100644 index e367c29..4fc2208 *** a/doc/src/sgml/config.sgml --- b/doc/src/sgml/config.sgml *************** SET ENABLE_SEQSCAN TO OFF; *** 1025,1030 **** --- 1025,1067 ---- + + Disk + + + + temp_file_limit (integer) + + temp_file_limit configuration parameter + + + + Specifies the amount of disk space used by internal sort operations + and hash tables whist writing to temporary disk files. The value + defaults to unlimited (-1). Values larger than zero + constraint the temporary file space usage to be that number of + kilobytes. + + + A given sort or hash operation may write a number of temporary files, + the total space used by all the files produced by one backend is + constrained to be this value or less. If further bytes are written + the current query is canceled. Only superusers can change this + setting. + + + It should be noted that this parameter does not + constrain disk space used for temporary table storage. However if + the temporary table is created from a query then the any sort + and hash files used in query execution will have their space + controlled as above. + + + + + + + Kernel Resource Usage diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c new file mode 100644 index 11bab38..a9a59a9 *** a/src/backend/storage/file/fd.c --- b/src/backend/storage/file/fd.c *************** static int max_safe_fds = 32; /* default *** 132,137 **** --- 132,142 ---- */ static bool have_pending_fd_cleanup = false; + /* + * Track the total size of all temporary files + */ + static double temporary_files_size = 0.0; + typedef struct vfd { int fd; /* current FD, or VFD_CLOSED if none */ *************** typedef struct vfd *** 141,146 **** --- 146,152 ---- File lruMoreRecently; /* doubly linked recency-of-use list */ File lruLessRecently; off_t seekPos; /* current logical file position */ + off_t fileSize; /* current size of file */ char *fileName; /* name of file, or NULL for unused VFD */ /* NB: fileName is malloc'd, and must be free'd when closing the VFD */ int fileFlags; /* open(2) flags for (re)opening the file */ *************** PathNameOpenFile(FileName fileName, int *** 887,892 **** --- 893,899 ---- vfdP->fileFlags = fileFlags & ~(O_CREAT | O_TRUNC | O_EXCL); vfdP->fileMode = fileMode; vfdP->seekPos = 0; + vfdP->fileSize = 0; vfdP->fdstate = 0x0; vfdP->resowner = NULL; *************** FileClose(File file) *** 1104,1109 **** --- 1111,1123 ---- if (unlink(vfdP->fileName)) elog(LOG, "could not unlink file \"%s\": %m", vfdP->fileName); } + + if (temp_file_limit >= 0) + { + /* subtract the unlinked file size from the total */ + temporary_files_size -= (double)vfdP->fileSize; + vfdP->fileSize = 0; + } } /* Unregister it from the resource owner */ *************** retry: *** 1231,1238 **** if (returnCode != amount && errno == 0) errno = ENOSPC; ! if (returnCode >= 0) VfdCache[file].seekPos += returnCode; else { /* --- 1245,1271 ---- if (returnCode != amount && errno == 0) errno = ENOSPC; ! if (returnCode >= 0) { VfdCache[file].seekPos += returnCode; + + if (temp_file_limit >= 0 && VfdCache[file].fdstate & FD_TEMPORARY) + { + /* + * if we increase the file size, add it to the total, then check it is + * not too big + */ + if (VfdCache[file].seekPos >= VfdCache[file].fileSize ) + { + temporary_files_size += (double)(VfdCache[file].seekPos - VfdCache[file].fileSize); + VfdCache[file].fileSize = VfdCache[file].seekPos; + + if (temporary_files_size / 1024.0 > (double)temp_file_limit) + ereport(ERROR, + (errcode(ERRCODE_QUERY_CANCELED), + errmsg("aborting due to exceeding temp file limit"))); + } + } + } else { /* *************** CleanupTempFiles(bool isProcExit) *** 1848,1853 **** --- 1882,1888 ---- } have_pending_fd_cleanup = false; + temporary_files_size = 0.0; } /* Clean up "allocated" stdio files and dirs. */ diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c new file mode 100644 index 92391ed..17c062e *** a/src/backend/utils/misc/guc.c --- b/src/backend/utils/misc/guc.c *************** int log_min_messages = WARNING; *** 423,428 **** --- 423,429 ---- int client_min_messages = NOTICE; int log_min_duration_statement = -1; int log_temp_files = -1; + int temp_file_limit = -1; int trace_recovery_messages = LOG; int num_temp_buffers = 1024; *************** const char *const config_group_names[] = *** 535,540 **** --- 536,543 ---- gettext_noop("Resource Usage"), /* RESOURCES_MEM */ gettext_noop("Resource Usage / Memory"), + /* RESOURCES_DISK */ + gettext_noop("Resource Usage / Disk"), /* RESOURCES_KERNEL */ gettext_noop("Resource Usage / Kernel Resources"), /* RESOURCES_VACUUM_DELAY */ *************** static struct config_int ConfigureNamesI *** 2185,2190 **** --- 2188,2202 ---- RELSEG_SIZE, RELSEG_SIZE, RELSEG_SIZE, NULL, NULL, NULL }, + { + {"temp_file_limit", PGC_SUSET, RESOURCES_DISK, + gettext_noop("Size of all temp files."), + gettext_noop("-1 is ignored >=0 means enforce size limit."), + GUC_UNIT_KB + }, + &temp_file_limit, + -1, -1, MAX_KILOBYTES, NULL, NULL + }, { {"wal_block_size", PGC_INTERNAL, PRESET_OPTIONS, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample new file mode 100644 index 655dad4..cc468c7 *** a/src/backend/utils/misc/postgresql.conf.sample --- b/src/backend/utils/misc/postgresql.conf.sample *************** *** 118,123 **** --- 118,124 ---- #work_mem = 1MB # min 64kB #maintenance_work_mem = 16MB # min 1MB #max_stack_depth = 2MB # min 100kB + #temp_file_limit = -1 # limit backend temp file space # - Kernel Resource Usage - diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h new file mode 100644 index ee52cd7..35321ee *** a/src/include/utils/guc.h --- b/src/include/utils/guc.h *************** extern int log_min_messages; *** 208,213 **** --- 208,214 ---- extern int client_min_messages; extern int log_min_duration_statement; extern int log_temp_files; + extern int temp_file_limit; extern int num_temp_buffers; diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h new file mode 100644 index a1ca012..3d9ab37 *** a/src/include/utils/guc_tables.h --- b/src/include/utils/guc_tables.h *************** enum config_group *** 59,64 **** --- 59,65 ---- CONN_AUTH_SECURITY, RESOURCES, RESOURCES_MEM, + RESOURCES_DISK, RESOURCES_KERNEL, RESOURCES_VACUUM_DELAY, RESOURCES_BGWRITER, diff --git a/src/test/regress/expected/resource.out b/src/test/regress/expected/resource.out new file mode 100644 index ...3172f83 *** a/src/test/regress/expected/resource.out --- b/src/test/regress/expected/resource.out *************** *** 0 **** --- 1,18 ---- + -- + -- RESOURCE + -- Test resource management capabilities + -- + -- temp_file_limit + CREATE TEMPORARY TABLE resourcetemp1 AS SELECT generate_series(1,100000); + -- Should succeed + SET temp_file_limit = 10000; + SELECT count(*) FROM (select * FROM resourcetemp1 ORDER BY 1) AS a; + count + -------- + 100000 + (1 row) + + -- Should fail + SET temp_file_limit = 1000; + SELECT count(*) FROM (select * FROM resourcetemp1 ORDER BY 1) AS a; + ERROR: aborting due to exceeding temp file limit diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule new file mode 100644 index bb654f9..325cb3d *** a/src/test/regress/serial_schedule --- b/src/test/regress/serial_schedule *************** test: largeobject *** 127,129 **** --- 127,130 ---- test: with test: xml test: stats + test: resource diff --git a/src/test/regress/sql/resource.sql b/src/test/regress/sql/resource.sql new file mode 100644 index ...c593a58 *** a/src/test/regress/sql/resource.sql --- b/src/test/regress/sql/resource.sql *************** *** 0 **** --- 1,17 ---- + -- + -- RESOURCE + -- Test resource management capabilities + -- + + -- temp_file_limit + CREATE TEMPORARY TABLE resourcetemp1 AS SELECT generate_series(1,100000); + + -- Should succeed + SET temp_file_limit = 10000; + + SELECT count(*) FROM (select * FROM resourcetemp1 ORDER BY 1) AS a; + + -- Should fail + SET temp_file_limit = 1000; + + SELECT count(*) FROM (select * FROM resourcetemp1 ORDER BY 1) AS a; -- 1.7.4.1