Generic Monitoring Framework with DTrace patch - Mailing list pgsql-patches

Hi,

I've have attached a patch along with two new files.This patch should
reflect what we discussed at the Summit. Please let me know if I miss
anything.

There are a few potential/known issues which I need help from the gurus.

1) The current logic in src/backend/Makefile will only work for Solaris
versions with DTrace, and Peter has offered to help fix this one.

2) Currently an environment variable called DTRACE_DATA_MODEL is used in
src/backend/Makefile to tell the dtrace command whether to generate a 32
or 64 bit binary.  This may not be a reliable approach since a user can
forget to set this variable. Perhaps adding a flag like DTRACEFLAGS to
the configure script is a better approach. Thoughts?

3)  When using --enable-depend, "gmake clean" removes all *.d files,
including the DTrace probe definition file which also has a .d
extension.  I tried to to use a different extension but the dtrace tool
didn't like that. Since we only plan to have one probe definition file
(pgsqlprobes.d) and one provider called "postgresql", I think
src/Makefile.global.in can be changed to not remove this file. Is this
acceptable or is there a better solution?

I have created several DTrace scripts and uploaded them to
http://pgfoundry.org/projects/dtrace/

Many thanks to Gavin and Tom for their help with creating and inserting
the current set of probes, and Peter for his help with the config files.

Regards,
-Robert
? src/backend/pgsqlprobes.d
? src/include/pg_trace.h
Index: configure
===================================================================
RCS file: /projects/cvsroot/pgsql/configure,v
retrieving revision 1.497
diff -c -r1.497 configure
*** configure    11 Jul 2006 16:14:50 -0000    1.497
--- configure    20 Jul 2006 14:14:17 -0000
***************
*** 865,870 ****
--- 865,871 ----
    --disable-rpath         do not embed shared library search path in executables
    --disable-spinlocks     do not use spinlocks
    --enable-debug          build with debugging symbols (-g)
+   --enable-dtrace         build with DTrace support
    --enable-depend         turn on automatic dependency tracking
    --enable-cassert        enable assertion checks (for debugging)
    --enable-thread-safety  make client libraries thread-safe
***************
*** 1947,1952 ****
--- 1948,1987 ----


  #
+ # DTrace
+ #
+
+
+
+ # Check whether --enable-dtrace or --disable-dtrace was given.
+ if test "${enable_dtrace+set}" = set; then
+   enableval="$enable_dtrace"
+
+   case $enableval in
+     yes)
+
+ cat >>confdefs.h <<\_ACEOF
+ #define ENABLE_DTRACE 1
+ _ACEOF
+
+       ;;
+     no)
+       :
+       ;;
+     *)
+       { { echo "$as_me:$LINENO: error: no argument expected for --enable-dtrace option" >&5
+ echo "$as_me: error: no argument expected for --enable-dtrace option" >&2;}
+    { (exit 1); exit 1; }; }
+       ;;
+   esac
+
+ else
+   enable_dtrace=no
+
+ fi;
+
+
+ #
  # C compiler
  #

***************
*** 22759,22764 ****
--- 22794,22800 ----
  enable_rpath) ;;
  enable_spinlocks) ;;
  enable_debug) ;;
+ enable_dtrace) ;;
  with_CC) ;;
  enable_depend) ;;
  enable_cassert) ;;
Index: configure.in
===================================================================
RCS file: /projects/cvsroot/pgsql/configure.in,v
retrieving revision 1.468
diff -c -r1.468 configure.in
*** configure.in    11 Jul 2006 16:14:50 -0000    1.468
--- configure.in    20 Jul 2006 14:14:18 -0000
***************
*** 206,211 ****
--- 206,219 ----
  AC_SUBST(enable_debug)

  #
+ # DTrace
+ #
+ PGAC_ARG_BOOL(enable, dtrace, no,
+               [  --enable-dtrace         build with DTrace support],
+               [AC_DEFINE([ENABLE_DTRACE], 1,
+         [Define to 1 to enable DTrace support. (--enable-dtrace)])])
+
+ #
  # C compiler
  #

Index: src/backend/Makefile
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/Makefile,v
retrieving revision 1.116
diff -c -r1.116 Makefile
*** src/backend/Makefile    22 Jun 2006 23:50:35 -0000    1.116
--- src/backend/Makefile    20 Jul 2006 14:14:19 -0000
***************
*** 21,26 ****
--- 21,28 ----

  OBJS := $(SUBSYSOBJS) $(top_builddir)/src/port/libpgport_srv.a

+ OBJ_DTRACE = pgsqlprobes.o
+
  # We put libpgport into OBJS, so remove it from LIBS
  LIBS := $(filter-out -lpgport, $(LIBS))

***************
*** 34,39 ****
--- 36,42 ----
  ifneq ($(PORTNAME), cygwin)
  ifneq ($(PORTNAME), win32)
  ifneq ($(PORTNAME), aix)
+ ifneq ($(PORTNAME), solaris)

  postgres: $(OBJS)
      $(CC) $(CFLAGS) $(LDFLAGS) $(export_dynamic) $^ $(LIBS) -o $@
***************
*** 41,46 ****
--- 44,60 ----
  endif
  endif
  endif
+ endif
+
+ ifeq ($(PORTNAME), solaris)
+
+ pgsqlprobes.o: pgsqlprobes.d tcop/SUBSYS.o access/SUBSYS.o parser/SUBSYS.o storage/SUBSYS.o storage/file/SUBSYS.o
storage/smgr/SUBSYS.ostorage/lmgr/SUBSYS.o 
+     dtrace -$(DTRACE_DATA_MODEL) -G -s $^
+
+ postgres: $(OBJS) $(OBJ_DTRACE)
+     $(CC) $(CFLAGS) $(LDFLAGS) $(export_dynamic) $^ $(LIBS) -o $@
+ endif
+

  ifeq ($(PORTNAME), cygwin)

***************
*** 223,228 ****
--- 237,247 ----
      rm -f postgres$(X) $(POSTGRES_IMP) \
          $(top_srcdir)/src/include/parser/parse.h \
          $(top_builddir)/src/include/utils/fmgroids.h
+
+ ifeq ($(PORTNAME), solaris)
+     rm -f $(top_srcdir)/src/backend/$(OBJ_DTRACE)
+ endif
+
  ifeq ($(PORTNAME), cygwin)
      rm -f postgres.dll postgres.def libpostgres.a
  endif
Index: src/backend/access/transam/xact.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/access/transam/xact.c,v
retrieving revision 1.223
diff -c -r1.223 xact.c
*** src/backend/access/transam/xact.c    14 Jul 2006 14:52:17 -0000    1.223
--- src/backend/access/transam/xact.c    20 Jul 2006 14:14:21 -0000
***************
*** 1384,1389 ****
--- 1384,1391 ----

      XactLockTableInsert(s->transactionId);

+     PG_TRACE1 (transaction__start, s->transactionId);
+
      /*
       * set transaction_timestamp() (a/k/a now()).  We want this to be the
       * same as the first command's statement_timestamp(), so don't do a
***************
*** 1535,1540 ****
--- 1537,1544 ----
          LWLockRelease(ProcArrayLock);
      }

+     PG_TRACE1 (transaction__commit, s->transactionId);
+
      /*
       * This is all post-commit cleanup.  Note that if an error is raised here,
       * it's too late to abort the transaction.  This should be just
***************
*** 1931,1936 ****
--- 1935,1942 ----
          LWLockRelease(ProcArrayLock);
      }

+     PG_TRACE1 (transaction__abort, s->transactionId);
+
      /*
       * Post-abort cleanup.    See notes in CommitTransaction() concerning
       * ordering.
Index: src/backend/storage/lmgr/lock.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/storage/lmgr/lock.c,v
retrieving revision 1.166
diff -c -r1.166 lock.c
*** src/backend/storage/lmgr/lock.c    14 Jul 2006 14:52:23 -0000    1.166
--- src/backend/storage/lmgr/lock.c    20 Jul 2006 14:14:23 -0000
***************
*** 752,759 ****
--- 752,764 ----
          /*
           * Sleep till someone wakes me up.
           */
+
+         PG_TRACE2(lock__startwait, locktag->locktag_field2, lockmode);
+
          WaitOnLock(locallock, owner);

+         PG_TRACE2(lock__endwait, locktag->locktag_field2, lockmode);
+
          /*
           * NOTE: do not do any material change of state between here and
           * return.    All required changes in locktable state must have been
***************
*** 1324,1329 ****
--- 1329,1335 ----
      LWLockRelease(partitionLock);

      RemoveLocalLock(locallock);
+
      return TRUE;
  }

Index: src/backend/storage/lmgr/lwlock.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/storage/lmgr/lwlock.c,v
retrieving revision 1.40
diff -c -r1.40 lwlock.c
*** src/backend/storage/lmgr/lwlock.c    14 Jul 2006 14:52:23 -0000    1.40
--- src/backend/storage/lmgr/lwlock.c    20 Jul 2006 14:14:23 -0000
***************
*** 424,429 ****
--- 424,431 ----
          block_counts[lockid]++;
  #endif

+         PG_TRACE2(lwlock__startwait, lockid, mode);
+
          for (;;)
          {
              /* "false" means cannot accept cancel/die interrupt here. */
***************
*** 433,438 ****
--- 435,442 ----
              extraWaits++;
          }

+         PG_TRACE2(lwlock__endwait, lockid, mode);
+
          LOG_LWDEBUG("LWLockAcquire", lockid, "awakened");

          /* Now loop back and try to acquire lock again. */
***************
*** 442,447 ****
--- 446,453 ----
      /* We are done updating shared state of the lock itself. */
      SpinLockRelease(&lock->mutex);

+     PG_TRACE2(lwlock__acquire, lockid, mode);
+
      /* Add lock to list of locks held by this backend */
      held_lwlocks[num_held_lwlocks++] = lockid;

***************
*** 511,521 ****
--- 517,529 ----
          /* Failed to get lock, so release interrupt holdoff */
          RESUME_INTERRUPTS();
          LOG_LWDEBUG("LWLockConditionalAcquire", lockid, "failed");
+         PG_TRACE2(lwlock__condacquire__fail, lockid, mode);
      }
      else
      {
          /* Add lock to list of locks held by this backend */
          held_lwlocks[num_held_lwlocks++] = lockid;
+         PG_TRACE2(lwlock__condacquire, lockid, mode);
      }

      return !mustwait;
***************
*** 600,605 ****
--- 608,615 ----
      /* We are done updating shared state of the lock itself. */
      SpinLockRelease(&lock->mutex);

+     PG_TRACE1(lwlock__release, lockid);
+
      /*
       * Awaken any waiters I removed from the queue.
       */
***************
*** 617,622 ****
--- 627,633 ----
       * Now okay to allow cancel/die interrupts.
       */
      RESUME_INTERRUPTS();
+
  }


Index: src/include/c.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/c.h,v
retrieving revision 1.206
diff -c -r1.206 c.h
*** src/include/c.h    6 Jul 2006 01:55:51 -0000    1.206
--- src/include/c.h    20 Jul 2006 14:14:24 -0000
***************
*** 56,61 ****
--- 56,62 ----
  #include "pg_config_os.h"        /* must be before any system header files */
  #endif
  #include "postgres_ext.h"
+ #include "pg_trace.h"

  #if defined(_MSC_VER) || defined(__BORLANDC__)
  #define    WIN32_ONLY_COMPILER
Index: src/include/pg_config.h.in
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/pg_config.h.in,v
retrieving revision 1.98
diff -c -r1.98 pg_config.h.in
*** src/include/pg_config.h.in    18 Jun 2006 18:30:21 -0000    1.98
--- src/include/pg_config.h.in    20 Jul 2006 14:14:24 -0000
***************
*** 36,41 ****
--- 36,44 ----
  /* Define to the default TCP port number as a string constant. */
  #undef DEF_PGPORT_STR

+ /* Define to 1 to enable DTrace support. (--enable-dtrace) */
+ #undef ENABLE_DTRACE
+
  /* Define to 1 if you want National Language Support. (--enable-nls) */
  #undef ENABLE_NLS

provider postgresql {

probe transaction__start(int);
probe transaction__commit(int);
probe transaction__abort(int);
probe lwlock__acquire(int, int);
probe lwlock__release(int);
probe lwlock__startwait(int, int);
probe lwlock__endwait(int, int);
probe lwlock__condacquire(int, int);
probe lwlock__condacquire__fail(int, int);
probe lock__startwait(int, int);
probe lock__endwait(int, int);

};

#ifndef PG_TRACE_H
#define PG_TRACE_H

#ifdef ENABLE_DTRACE

#include <sys/sdt.h>

/*
 * The PG_TRACE macros are mapped to the appropriate macros used by DTrace.
 *
 * Only one DTrace provider called "postgresql" will be used for PostgreSQL,
 * so the name is hard-coded here to avoid having to specify it in the
 * source code.
 */

#define PG_TRACE(name)                     \
    DTRACE_PROBE(postgresql, name)
#define PG_TRACE1(name, arg1)                 \
    DTRACE_PROBE1(postgresql, name, arg1)
#define PG_TRACE2(name, arg1, arg2)            \
    DTRACE_PROBE2(postgresql, name, arg1, arg2)
#define PG_TRACE3(name, arg1, arg2, arg3)        \
    DTRACE_PROBE3(postgresql, name, arg1, arg2, arg3)
#define PG_TRACE4(name, arg1, arg2, arg3, arg4)        \
    DTRACE_PROBE4(postgresql, name, arg1, arg2, arg3, arg4)
#define PG_TRACE5(name, arg1, arg2, arg3, arg4, arg5)    \
    DTRACE_PROBE5(postgresql, name, arg1, arg2, arg3, arg4, arg5)

#else

/*
 * Unless DTrace is explicitly enabled with --enable-dtrace, the PG_TRACE
 * macros will expand to no-ops.
 */

#define PG_TRACE(name)
#define PG_TRACE1(name, arg1)
#define PG_TRACE2(name, arg1, arg2)
#define PG_TRACE3(name, arg1, arg2, arg3)
#define PG_TRACE4(name, arg1, arg2, arg3, arg4)
#define PG_TRACE5(name, arg1, arg2, arg3, arg4, arg5)

#endif

#endif   /* PG_TRACE_H */