pg_dump large file support - Mailing list pgsql-hackers

I've implemented large file support for pg_dump, in what I hope is a
portable fashion.  Please review the attached patch.

This needs an additional option from autoconf, because pg_dump prints a
hex offset which needs to be either %Lx or %llx, but there is no hex
equivalent to INT64_FORMAT.  To get this working I have arbitrarily
defined OFF_T_FORMATX as "%lld", but that needs to be changed before the
patch is incorporated.

Peter E suggested that there was no need for the macro definitions in
pg_backup.h but I wasn't clear how else to achieve what was needed.


--
Oliver Elphick                                Oliver.Elphick@lfix.co.uk
Isle of Wight, UK
http://www.lfix.co.uk/oliver
GPG: 1024D/3E1D0C1C: CA12 09E0 E8D5 8870 5839  932A 614D 4C34 3E1D 0C1C
                 ========================================
     "Praying always with all prayer and supplication in the
      Spirit, and watching thereunto with all perseverance
      and supplication for all saints."       Ephesians 6:18
Index: src/bin/pg_dump/Makefile
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/Makefile,v
retrieving revision 1.36
diff -u -r1.36 Makefile
--- src/bin/pg_dump/Makefile    2002/07/27 20:10:05    1.36
+++ src/bin/pg_dump/Makefile    2002/08/15 10:31:53
@@ -18,6 +18,8 @@

 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)

+CFLAGS +=  -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+
 all: submake-libpq submake-libpgport pg_dump pg_restore pg_dumpall

 pg_dump: pg_dump.o common.o $(OBJS) $(libpq_builddir)/libpq.a
Index: src/bin/pg_dump/pg_backup.h
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup.h,v
retrieving revision 1.20
diff -u -r1.20 pg_backup.h
--- src/bin/pg_dump/pg_backup.h    2002/07/04 15:35:07    1.20
+++ src/bin/pg_dump/pg_backup.h    2002/08/15 10:31:53
@@ -27,6 +27,25 @@

 #include "libpq-fe.h"

+/*
+ * Large file support
+ */
+
+#ifdef _LARGEFILE_SOURCE
+ #define FSEEK fseeko
+ #define FTELL ftello
+ #define OFF_T_FORMAT INT64_FORMAT
+ #define OFF_T_FORMATX "%llx"            /* needs an autoconf setting */
+ typedef off_t OFF_T;
+#else
+ #define FSEEK fseek
+ #define FTELL ftell
+ #define OFF_T_FORMAT "%ld"
+ #define OFF_T_FORMATX "%lx"
+ typedef long int OFF_T;
+#endif
+
+
 #define atooid(x)  ((Oid) strtoul((x), NULL, 10))
 #define oidcmp(x,y) ( ((x) < (y) ? -1 : ((x) > (y)) ?  1 : 0) )
 #define oideq(x,y) ( (x) == (y) )
Index: src/bin/pg_dump/pg_backup_archiver.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_archiver.c,v
retrieving revision 1.53
diff -u -r1.53 pg_backup_archiver.c
--- src/bin/pg_dump/pg_backup_archiver.c    2002/08/10 16:57:31    1.53
+++ src/bin/pg_dump/pg_backup_archiver.c    2002/08/15 10:31:55
@@ -1561,7 +1561,7 @@
     }

     /* If we can't seek, then mark the header as read */
-    if (fseek(fh, 0, SEEK_SET) != 0)
+    if (FSEEK(fh, 0, SEEK_SET) != 0)
     {
         /*
          * NOTE: Formats that use the looahead buffer can unset this in
Index: src/bin/pg_dump/pg_backup_custom.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_custom.c,v
retrieving revision 1.19
diff -u -r1.19 pg_backup_custom.c
--- src/bin/pg_dump/pg_backup_custom.c    2002/05/29 01:38:56    1.19
+++ src/bin/pg_dump/pg_backup_custom.c    2002/08/15 10:31:56
@@ -81,13 +81,13 @@
     char       *zlibIn;
     int            inSize;
     int            hasSeek;
-    int            filePos;
+    OFF_T            filePos;
     int            dataStart;
 } lclContext;

 typedef struct
 {
-    int            dataPos;
+    OFF_T            dataPos;
     int            dataLen;
 } lclTocEntry;

@@ -99,7 +99,7 @@
 static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
 static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te);
 static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te);
-static int    _getFilePos(ArchiveHandle *AH, lclContext *ctx);
+static OFF_T    _getFilePos(ArchiveHandle *AH, lclContext *ctx);
 static int    _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush);

 static char *modulename = gettext_noop("custom archiver");
@@ -188,7 +188,7 @@
         if (!AH->FH)
             die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));

-        ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+        ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);

     }
     else
@@ -201,7 +201,7 @@
         if (!AH->FH)
             die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));

-        ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+        ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);

         ReadHead(AH);
         ReadToc(AH);
@@ -285,7 +285,7 @@
 {
     lclTocEntry *ctx = (lclTocEntry *) te->formatData;

-    ahprintf(AH, "-- Data Pos: %d (Length %d)\n", ctx->dataPos, ctx->dataLen);
+    ahprintf(AH, "-- Data Pos: " INT64_FORMAT " (Length %d)\n", ctx->dataPos, ctx->dataLen);
 }

 /*
@@ -486,7 +486,7 @@

         /* Grab it */

-        if (fseek(AH->FH, tctx->dataPos, SEEK_SET) != 0)
+        if (FSEEK(AH->FH, tctx->dataPos, SEEK_SET) != 0)
             die_horribly(AH, modulename, "error during file seek: %s\n", strerror(errno));

         _readBlockHeader(AH, &blkType, &id);
@@ -816,12 +816,12 @@
 _CloseArchive(ArchiveHandle *AH)
 {
     lclContext *ctx = (lclContext *) AH->formatData;
-    int            tpos;
+    OFF_T            tpos;

     if (AH->mode == archModeWrite)
     {
         WriteHead(AH);
-        tpos = ftell(AH->FH);
+        tpos = FTELL(AH->FH);
         WriteToc(AH);
         ctx->dataStart = _getFilePos(AH, ctx);
         WriteDataChunks(AH);
@@ -834,7 +834,7 @@
          */
         if (ctx->hasSeek)
         {
-            fseek(AH->FH, tpos, SEEK_SET);
+            FSEEK(AH->FH, tpos, SEEK_SET);
             WriteToc(AH);
         }
     }
@@ -853,14 +853,14 @@
 /*
  * Get the current position in the archive file.
  */
-static int
+static OFF_T
 _getFilePos(ArchiveHandle *AH, lclContext *ctx)
 {
-    int            pos;
+    OFF_T            pos;

     if (ctx->hasSeek)
     {
-        pos = ftell(AH->FH);
+        pos = FTELL(AH->FH);
         if (pos != ctx->filePos)
         {
             write_msg(modulename, "WARNING: ftell mismatch with expected position -- ftell ignored\n");
Index: src/bin/pg_dump/pg_backup_files.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_files.c,v
retrieving revision 1.16
diff -u -r1.16 pg_backup_files.c
--- src/bin/pg_dump/pg_backup_files.c    2002/05/29 01:38:56    1.16
+++ src/bin/pg_dump/pg_backup_files.c    2002/08/15 10:31:56
@@ -60,7 +60,7 @@
 typedef struct
 {
     int            hasSeek;
-    int            filePos;
+    OFF_T            filePos;
     FILE       *blobToc;
 } lclContext;

@@ -137,7 +137,7 @@
         if (AH->FH == NULL)
             die_horribly(NULL, modulename, "could not open output file: %s\n", strerror(errno));

-        ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+        ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);

         if (AH->compression < 0 || AH->compression > 9)
             AH->compression = Z_DEFAULT_COMPRESSION;
@@ -155,7 +155,7 @@
         if (AH->FH == NULL)
             die_horribly(NULL, modulename, "could not open input file: %s\n", strerror(errno));

-        ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+        ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);

         ReadHead(AH);
         ReadToc(AH);
Index: src/bin/pg_dump/pg_backup_tar.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_tar.c,v
retrieving revision 1.24
diff -u -r1.24 pg_backup_tar.c
--- src/bin/pg_dump/pg_backup_tar.c    2002/07/04 15:35:07    1.24
+++ src/bin/pg_dump/pg_backup_tar.c    2002/08/15 10:31:57
@@ -187,7 +187,7 @@
          */
         /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */

-        ctx->hasSeek = (fseek(ctx->tarFH, 0, SEEK_CUR) == 0);
+        ctx->hasSeek = (FSEEK(ctx->tarFH, 0, SEEK_CUR) == 0);

         if (AH->compression < 0 || AH->compression > 9)
             AH->compression = Z_DEFAULT_COMPRESSION;
@@ -224,7 +224,7 @@

         ctx->tarFHpos = 0;

-        ctx->hasSeek = (fseek(ctx->tarFH, 0, SEEK_CUR) == 0);
+        ctx->hasSeek = (FSEEK(ctx->tarFH, 0, SEEK_CUR) == 0);

         /*
          * Forcibly unmark the header as read since we use the lookahead
@@ -1011,9 +1011,9 @@
     /*
      * Find file len & go back to start.
      */
-    fseek(tmp, 0, SEEK_END);
-    th->fileLen = ftell(tmp);
-    fseek(tmp, 0, SEEK_SET);
+    FSEEK(tmp, 0, SEEK_END);
+    th->fileLen = FTELL(tmp);
+    FSEEK(tmp, 0, SEEK_SET);

     _tarWriteHeader(th);

@@ -1130,10 +1130,10 @@
     while (!gotBlock)
     {
 #if 0
-        if (ftell(ctx->tarFH) != ctx->tarFHpos)
+        if (FTELL(ctx->tarFH) != ctx->tarFHpos)
             die_horribly(AH, modulename,
                          "mismatch in actual vs. predicted file position (%d vs. %d)\n",
-                         ftell(ctx->tarFH), ctx->tarFHpos);
+                         FTELL(ctx->tarFH), ctx->tarFHpos);
 #endif

         /* Save the pos for reporting purposes */
@@ -1179,8 +1179,9 @@
     if (chk != sum)
         die_horribly(AH, modulename,
                      "corrupt tar header found in %s "
-        "(expected %d (%o), computed %d (%o)) file position %ld (%lx)\n",
-                     &tag[0], sum, sum, chk, chk, ftell(ctx->tarFH), ftell(ctx->tarFH));
+        "(expected %d (%o), computed %d (%o)) file position "
+        OFF_T_FORMAT " (" OFF_T_FORMATX ")\n",
+                     &tag[0], sum, sum, chk, chk, FTELL(ctx->tarFH), FTELL(ctx->tarFH));

     th->targetFile = strdup(tag);
     th->fileLen = len;