Re: pg_dump and large files - is this a problem? - Mailing list pgsql-hackers

Philip Warner wrote:
> At 10:08 PM 23/10/2002 -0400, Bruce Momjian wrote:
> >Well, that certainly changes the functionality of the code.  I thought
> >that fseeko test was done so that things that couldn't be seeked on were
> >detected.
>
> You are quite correct. It should read:
>
>          #ifdef HAVE_FSEEKO
>               ctx->hasSeek = fseeko(...,SEEK_SET);
>          #else
>               ctx->hasSeek = FALSE;
>          #endif
>
> pipes are the main case for which we are checking.

OK, I have applied the following patch to set hasSeek only if
fseek/fseeko is reliable.  This takes care of the random failure case
for large files.  Now I need to see if I can get the custom fseeko
working for more platforms.

--
  Bruce Momjian                        |  http://candle.pha.pa.us
  pgman@candle.pha.pa.us               |  (610) 359-1001
  +  If your life is a hard drive,     |  13 Roberts Road
  +  Christ can be your backup.        |  Newtown Square, Pennsylvania 19073
Index: src/bin/pg_dump/common.c
===================================================================
RCS file: /cvsroot/pgsql-server/src/bin/pg_dump/common.c,v
retrieving revision 1.71
diff -c -c -r1.71 common.c
*** src/bin/pg_dump/common.c    9 Oct 2002 16:20:25 -0000    1.71
--- src/bin/pg_dump/common.c    25 Oct 2002 01:30:51 -0000
***************
*** 290,296 ****
           * attr with the same name, then only dump it if:
           *
           * - it is NOT NULL and zero parents are NOT NULL
!          *   OR
           * - it has a default value AND the default value does not match
           *   all parent default values, or no parents specify a default.
           *
--- 290,296 ----
           * attr with the same name, then only dump it if:
           *
           * - it is NOT NULL and zero parents are NOT NULL
!          *   OR
           * - it has a default value AND the default value does not match
           *   all parent default values, or no parents specify a default.
           *
Index: src/bin/pg_dump/pg_backup_archiver.c
===================================================================
RCS file: /cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_archiver.c,v
retrieving revision 1.59
diff -c -c -r1.59 pg_backup_archiver.c
*** src/bin/pg_dump/pg_backup_archiver.c    22 Oct 2002 19:15:23 -0000    1.59
--- src/bin/pg_dump/pg_backup_archiver.c    25 Oct 2002 01:30:57 -0000
***************
*** 2338,2343 ****
--- 2338,2369 ----
  }


+ /*
+  * checkSeek
+  *      check to see if fseek can be performed.
+  */
+
+ bool
+ checkSeek(FILE *fp)
+ {
+
+     if (fseek(fp, 0, SEEK_CUR) != 0)
+         return false;
+     else if (sizeof(off_t) > sizeof(long))
+     /*
+      *    At this point, off_t is too large for long, so we return
+      *    based on whether an off_t version of fseek is available.
+      */
+ #ifdef HAVE_FSEEKO
+         return true;
+ #else
+         return false;
+ #endif
+     else
+         return true;
+ }
+
+
  static void
  _SortToc(ArchiveHandle *AH, TocSortCompareFn fn)
  {
Index: src/bin/pg_dump/pg_backup_archiver.h
===================================================================
RCS file: /cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_archiver.h,v
retrieving revision 1.48
diff -c -c -r1.48 pg_backup_archiver.h
*** src/bin/pg_dump/pg_backup_archiver.h    22 Oct 2002 19:15:23 -0000    1.48
--- src/bin/pg_dump/pg_backup_archiver.h    25 Oct 2002 01:30:58 -0000
***************
*** 27,32 ****
--- 27,33 ----

  #include "postgres_fe.h"

+ #include <stdio.h>
  #include <time.h>
  #include <errno.h>

***************
*** 284,289 ****
--- 285,291 ----
  extern void WriteDataChunks(ArchiveHandle *AH);

  extern int    TocIDRequired(ArchiveHandle *AH, int id, RestoreOptions *ropt);
+ extern bool checkSeek(FILE *fp);

  /*
   * Mandatory routines for each supported format
Index: src/bin/pg_dump/pg_backup_custom.c
===================================================================
RCS file: /cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_custom.c,v
retrieving revision 1.22
diff -c -c -r1.22 pg_backup_custom.c
*** src/bin/pg_dump/pg_backup_custom.c    22 Oct 2002 19:15:23 -0000    1.22
--- src/bin/pg_dump/pg_backup_custom.c    25 Oct 2002 01:31:01 -0000
***************
*** 179,185 ****
          if (!AH->FH)
              die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));

!         ctx->hasSeek = (fseeko(AH->FH, 0, SEEK_CUR) == 0);
      }
      else
      {
--- 179,185 ----
          if (!AH->FH)
              die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));

!         ctx->hasSeek = checkSeek(AH->FH);
      }
      else
      {
***************
*** 190,196 ****
          if (!AH->FH)
              die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));

!         ctx->hasSeek = (fseeko(AH->FH, 0, SEEK_CUR) == 0);

          ReadHead(AH);
          ReadToc(AH);
--- 190,196 ----
          if (!AH->FH)
              die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));

!         ctx->hasSeek = checkSeek(AH->FH);

          ReadHead(AH);
          ReadToc(AH);
Index: src/bin/pg_dump/pg_backup_files.c
===================================================================
RCS file: /cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_files.c,v
retrieving revision 1.20
diff -c -c -r1.20 pg_backup_files.c
*** src/bin/pg_dump/pg_backup_files.c    22 Oct 2002 19:15:23 -0000    1.20
--- src/bin/pg_dump/pg_backup_files.c    25 Oct 2002 01:31:01 -0000
***************
*** 129,135 ****
          if (AH->FH == NULL)
              die_horribly(NULL, modulename, "could not open output file: %s\n", strerror(errno));

!         ctx->hasSeek = (fseeko(AH->FH, 0, SEEK_CUR) == 0);

          if (AH->compression < 0 || AH->compression > 9)
              AH->compression = Z_DEFAULT_COMPRESSION;
--- 129,135 ----
          if (AH->FH == NULL)
              die_horribly(NULL, modulename, "could not open output file: %s\n", strerror(errno));

!         ctx->hasSeek = checkSeek(AH->FH);

          if (AH->compression < 0 || AH->compression > 9)
              AH->compression = Z_DEFAULT_COMPRESSION;
***************
*** 147,153 ****
          if (AH->FH == NULL)
              die_horribly(NULL, modulename, "could not open input file: %s\n", strerror(errno));

!         ctx->hasSeek = (fseeko(AH->FH, 0, SEEK_CUR) == 0);

          ReadHead(AH);
          ReadToc(AH);
--- 147,153 ----
          if (AH->FH == NULL)
              die_horribly(NULL, modulename, "could not open input file: %s\n", strerror(errno));

!         ctx->hasSeek = checkSeek(AH->FH);

          ReadHead(AH);
          ReadToc(AH);
Index: src/bin/pg_dump/pg_backup_tar.c
===================================================================
RCS file: /cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_tar.c,v
retrieving revision 1.31
diff -c -c -r1.31 pg_backup_tar.c
*** src/bin/pg_dump/pg_backup_tar.c    22 Oct 2002 19:15:23 -0000    1.31
--- src/bin/pg_dump/pg_backup_tar.c    25 Oct 2002 01:31:04 -0000
***************
*** 190,196 ****
           */
          /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */

!         ctx->hasSeek = (fseeko(ctx->tarFH, 0, SEEK_CUR) == 0);

          if (AH->compression < 0 || AH->compression > 9)
              AH->compression = Z_DEFAULT_COMPRESSION;
--- 190,196 ----
           */
          /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */

!         ctx->hasSeek = checkSeek(ctx->tarFH);

          if (AH->compression < 0 || AH->compression > 9)
              AH->compression = Z_DEFAULT_COMPRESSION;
***************
*** 227,233 ****

          ctx->tarFHpos = 0;

!         ctx->hasSeek = (fseeko(ctx->tarFH, 0, SEEK_CUR) == 0);

          /*
           * Forcibly unmark the header as read since we use the lookahead
--- 227,233 ----

          ctx->tarFHpos = 0;

!         ctx->hasSeek = checkSeek(ctx->tarFH);

          /*
           * Forcibly unmark the header as read since we use the lookahead