Re: [HACKERS] loading libraries on Postmaster startup - Mailing list pgsql-patches

Tom Lane wrote:
> Joe Conway <mail@joeconway.com> writes:
>>[ what about autoloading libraries into the postmaster? ]
>
> I can see a couple possible downsides: (a) the library might have some
> weird behavior across fork boundaries; (b) the additional memory space
> that has to be duplicated into child processes will cost something per
> child launch, even if the child never uses it.  But these are only
> arguments that it might not *always* be a prudent thing to do, not that
> we shouldn't give the DBA the tool to do it if he wants.  So fire away.

Here is a patch for the above, including a documentation update. It
creates a new GUC variable "preload_libraries", that accepts a list in
the form:

   preload_libraries = '$libdir/mylib1:initfunc,$libdir/mylib2'

If ":initfunc" is omitted or not found, no initialization function is
executed, but the library is still preloaded. If "$libdir/mylib" isn't
found, the postmaster refuses to start.

In my testing with PL/R, it reduces the first call to a PL/R function
(after connecting) from almost 2 seconds, down to about 8 ms.

If there are no objections, please apply.

Thanks,

Joe
Index: doc/src/sgml/runtime.sgml
===================================================================
RCS file: /opt/src/cvs/pgsql-server/doc/src/sgml/runtime.sgml,v
retrieving revision 1.169
diff -c -r1.169 runtime.sgml
*** doc/src/sgml/runtime.sgml    19 Feb 2003 04:06:28 -0000    1.169
--- doc/src/sgml/runtime.sgml    22 Feb 2003 05:59:08 -0000
***************
*** 1781,1786 ****
--- 1781,1815 ----
       </varlistentry>

       <varlistentry>
+       <term><varname>PRELOAD_LIBRARIES</varname> (<type>string</type>)</term>
+       <indexterm><primary>preload_libraries</></>
+       <listitem>
+        <para>
+         This variable specifies one or more shared libraries that are to be
+     preloaded at Postmaster start. An initialization function can also be
+     optionally specified by adding a colon followed by the name of the
+     initialization function after the library name. For example
+     <literal>'$libdir/mylib:init_mylib'</literal> would cause <literal>mylib</>
+     to be preloaded and <literal>init_mylib</> to be executed. If more than
+     one library is to be loaded, they must be delimited with a comma.
+        </para>
+
+        <para>
+         If <literal>mylib</> is not found, the postmaster will fail to start.
+     However, if <literal>init_mylib</> is not found, <literal>mylib</> will
+     still be preloaded without executing the initialization function.
+        </para>
+
+        <para>
+         By preloading a shared library (and initializing it if applicable),
+     the library startup time is avoided when the library is used later in a
+     specific backend. However there is a cost in terms of memory duplication
+     as every backend is forked, whether or not the library is used.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry>
        <term><varname>REGEX_FLAVOR</varname> (<type>string</type>)</term>
        <indexterm><primary>regular expressions</></>
        <listitem>
Index: src/backend/postmaster/postmaster.c
===================================================================
RCS file: /opt/src/cvs/pgsql-server/src/backend/postmaster/postmaster.c,v
retrieving revision 1.306
diff -c -r1.306 postmaster.c
*** src/backend/postmaster/postmaster.c    25 Jan 2003 05:19:46 -0000    1.306
--- src/backend/postmaster/postmaster.c    22 Feb 2003 04:29:44 -0000
***************
*** 205,210 ****
--- 205,212 ----
  bool        Log_connections = false;
  bool        Db_user_namespace = false;

+ /* list of library:init-function to be preloaded */
+ char       *preload_libraries_string = NULL;

  /* Startup/shutdown state */
  static pid_t StartupPID = 0,
***************
*** 644,649 ****
--- 646,658 ----
      if (EnableSSL)
          secure_initialize();
  #endif
+
+     /*
+      * process any libraries that should be preloaded and
+      * optionally pre-initialized
+      */
+     if (preload_libraries_string)
+         process_preload_libraries(preload_libraries_string);

      /*
       * Fork away from controlling terminal, if -S specified.
Index: src/backend/utils/init/miscinit.c
===================================================================
RCS file: /opt/src/cvs/pgsql-server/src/backend/utils/init/miscinit.c,v
retrieving revision 1.100
diff -c -r1.100 miscinit.c
*** src/backend/utils/init/miscinit.c    27 Jan 2003 00:51:06 -0000    1.100
--- src/backend/utils/init/miscinit.c    22 Feb 2003 04:29:44 -0000
***************
*** 1044,1046 ****
--- 1044,1135 ----
               "which is not compatible with this version %s.",
               file_major, file_minor, version_string);
  }
+
+ /*-------------------------------------------------------------------------
+  *                Library preload support
+  *-------------------------------------------------------------------------
+  */
+
+ #if defined(__mc68000__) && defined(__ELF__)
+ typedef int32 ((*func_ptr) ());
+ #else
+ typedef char *((*func_ptr) ());
+ #endif
+
+ /*
+  * process any libraries that should be preloaded and
+  * optionally pre-initialized
+  */
+ void
+ process_preload_libraries(char *preload_libraries_string)
+ {
+     char       *rawstring;
+     List       *elemlist;
+     List       *l;
+
+     if (preload_libraries_string == NULL)
+         return;
+
+     /* Need a modifiable copy of string */
+     rawstring = pstrdup(preload_libraries_string);
+
+     /* Parse string into list of identifiers */
+     if (!SplitIdentifierString(rawstring, ',', &elemlist))
+     {
+         /* syntax error in list */
+         pfree(rawstring);
+         freeList(elemlist);
+         elog(LOG, "invalid list syntax for preload_libraries configuration option");
+     }
+
+     foreach(l, elemlist)
+     {
+         char       *tok = (char *) lfirst(l);
+         char       *sep = strstr(tok, ":");
+         char       *filename = NULL;
+         char       *funcname = NULL;
+         func_ptr    initfunc;
+
+         if (sep)
+         {
+             /*
+              * a colon separator implies there is an initialization function
+              * that we need to run in addition to loading the library
+              */
+             size_t        filename_len = sep - tok;
+             size_t        funcname_len = strlen(tok) - filename_len - 1;
+
+             filename = (char *) palloc(filename_len + 1);
+             memset(filename, '\0', filename_len + 1);
+             snprintf(filename, filename_len + 1, "%s", tok);
+
+             funcname = (char *) palloc(funcname_len + 1);
+             memset(funcname, '\0', funcname_len + 1);
+             snprintf(funcname, funcname_len + 1, "%s", sep + 1);
+         }
+         else
+         {
+             /*
+              * no separator -- just load the library
+              */
+             filename = pstrdup(tok);
+             funcname = NULL;
+         }
+
+         initfunc = (func_ptr) load_external_function(filename, funcname, false, NULL);
+         if (initfunc)
+             (*initfunc)();
+
+         elog(LOG, "preloaded library %s with initialization function %s", filename, funcname);
+
+         if (filename != NULL)
+             pfree(filename);
+
+         if (funcname != NULL)
+             pfree(funcname);
+     }
+
+     pfree(rawstring);
+     freeList(elemlist);
+ }
+
Index: src/backend/utils/misc/guc.c
===================================================================
RCS file: /opt/src/cvs/pgsql-server/src/backend/utils/misc/guc.c,v
retrieving revision 1.114
diff -c -r1.114 guc.c
*** src/backend/utils/misc/guc.c    6 Feb 2003 20:25:33 -0000    1.114
--- src/backend/utils/misc/guc.c    22 Feb 2003 06:01:33 -0000
***************
*** 61,66 ****
--- 61,67 ----
  extern int    CommitDelay;
  extern int    CommitSiblings;
  extern bool FixBTree;
+ extern char *preload_libraries_string;

  #ifdef HAVE_SYSLOG
  extern char *Syslog_facility;
***************
*** 817,822 ****
--- 818,829 ----
      {
          {"lc_time", PGC_USERSET}, &locale_time,
          "C", locale_time_assign, NULL
+     },
+
+     {
+         {"preload_libraries", PGC_POSTMASTER, GUC_LIST_INPUT | GUC_LIST_QUOTE},
+         &preload_libraries_string,
+         "", NULL, NULL
      },

      {
Index: src/backend/utils/misc/postgresql.conf.sample
===================================================================
RCS file: /opt/src/cvs/pgsql-server/src/backend/utils/misc/postgresql.conf.sample,v
retrieving revision 1.70
diff -c -r1.70 postgresql.conf.sample
*** src/backend/utils/misc/postgresql.conf.sample    6 Feb 2003 20:25:33 -0000    1.70
--- src/backend/utils/misc/postgresql.conf.sample    22 Feb 2003 04:29:44 -0000
***************
*** 213,216 ****
--- 213,217 ----
  #transform_null_equals = false
  #statement_timeout = 0        # 0 is disabled, in milliseconds
  #db_user_namespace = false
+ #preload_libraries = ''

Index: src/include/miscadmin.h
===================================================================
RCS file: /opt/src/cvs/pgsql-server/src/include/miscadmin.h,v
retrieving revision 1.115
diff -c -r1.115 miscadmin.h
*** src/include/miscadmin.h    9 Jan 2003 18:00:24 -0000    1.115
--- src/include/miscadmin.h    22 Feb 2003 04:29:44 -0000
***************
*** 292,297 ****
--- 292,298 ----
                               unsigned long id2);

  extern void ValidatePgVersion(const char *path);
+ extern void process_preload_libraries(char *preload_libraries_string);

  /* these externs do not belong here... */
  extern void IgnoreSystemIndexes(bool mode);