Re: Inefficient handling of LO-restore + Patch - Mailing list pgsql-hackers

Your patch has been added to the PostgreSQL unapplied patches list at:
http://candle.pha.pa.us/cgi-bin/pgpatches

I will try to apply it within the next 48 hours.

---------------------------------------------------------------------------


Mario Weilguni wrote:
> Am Donnerstag, 11. April 2002 17:44 schrieb Tom Lane:
> > "Mario Weilguni" <mario.weilguni@icomedias.com> writes:
> > > And I did not find out how I can detect the large object
> > > chunksize, either from getting it from the headers (include
> > > "storage/large_object.h" did not work)
> >
> 
> You did not answer if it's ok to post the patch, hope it's ok:
> ==================================
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.c 
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.c    Mon Feb 11 
> 01:18:20 2002
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.c    Thu Apr 11 10:41:09 
> 2002
> @@ -819,6 +819,9 @@
>          AH->createdBlobXref = 1;
>      }
>  
> +    /* Initialize the LO Buffer */
> +    AH->lo_buf_used = 0;
> +
>      /*
>       * Start long-running TXs if necessary
>       */
> @@ -848,6 +851,19 @@
>  void
>  EndRestoreBlob(ArchiveHandle *AH, Oid oid)
>  {
> +        if(AH->lo_buf_used > 0) {
> +      /* Write remaining bytes from the LO buffer */
> +        int res;
> +          res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf, 
> AH->lo_buf_used);
> +
> +      ahlog(AH, 5, "wrote remaining %d bytes of large object data (result = 
> %d)\n",
> +            (int)AH->lo_buf_used, res);
> +      if (res != AH->lo_buf_used)
> +        die_horribly(AH, modulename, "could not write to large object (result: %d, 
> expected: %d)\n",
> +             res, AH->lo_buf_used);
> +          AH->lo_buf_used = 0;
> +        }
> +
>      lo_close(AH->connection, AH->loFd);
>      AH->writingBlob = 0;
>  
> @@ -1228,14 +1244,27 @@
>  
>      if (AH->writingBlob)
>      {
> -        res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb);
> -        ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
> -              (int) (size * nmemb), res);
> -        if (res != size * nmemb)
> +            if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) {
> +          /* Split LO buffer */
> +          int remaining = AH->lo_buf_size - AH->lo_buf_used;
> +          int slack = nmemb * size - remaining;
> +
> +          memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining);
> +          res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size);
> +          ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
> +                      AH->lo_buf_size, res);
> +          if (res != AH->lo_buf_size)
>              die_horribly(AH, modulename, "could not write to large object (result: %d, 
> expected: %d)\n",
> -                         res, (int) (size * nmemb));
> +                         res, AH->lo_buf_size);
> +              memcpy(AH->lo_buf, ptr + remaining, slack);
> +          AH->lo_buf_used = slack;
> +           } else {
> +             /* LO Buffer is still large enough, buffer it */
> +         memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb);
> +         AH->lo_buf_used += size * nmemb;
> +           }
>  
> -        return res;
> +           return size * nmemb;
>      }
>      else if (AH->gzOut)
>      {
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.h 
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.h
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_archiver.h    Mon Nov  5 
> 18:46:30 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_archiver.h    Thu Apr 11 10:41:14 
> 2002
> @@ -41,6 +41,7 @@
>  #include <errno.h>
>  
>  #include "pqexpbuffer.h"
> +#define LOBBUFSIZE 32768
>  
>  #ifdef HAVE_LIBZ
>  #include <zlib.h>
> @@ -240,6 +241,9 @@
>  
>      RestoreOptions *ropt;        /* Used to check restore options in
>                                   * ahwrite etc */
> +    void                    *lo_buf;
> +    int                     lo_buf_used;
> +    int                     lo_buf_size;
>  } ArchiveHandle;
>  
>  typedef struct _tocEntry
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_custom.c 
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_custom.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_custom.c    Wed Nov 28 
> 00:48:12 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_custom.c    Thu Apr 11 10:42:45 
> 2002
> @@ -153,6 +153,12 @@
>      if (ctx->zp == NULL)
>          die_horribly(AH, modulename, "out of memory\n");
>  
> +    /* Initialize LO buffering */
> +    AH->lo_buf_size = LOBBUFSIZE;
> +    AH->lo_buf = (void *)malloc(LOBBUFSIZE);
> +    if(AH->lo_buf == NULL)
> +                die_horribly(AH, modulename, "out of memory\n");
> +
>      /*
>       * zlibOutSize is the buffer size we tell zlib it can output to.  We
>       * actually allocate one extra byte because some routines want to
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_files.c 
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_files.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_files.c    Thu Oct 25 
> 07:49:52 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_files.c    Thu Apr 11 10:43:01 
> 2002
> @@ -113,6 +113,12 @@
>      AH->formatData = (void *) ctx;
>      ctx->filePos = 0;
>  
> +    /* Initialize LO buffering */
> +    AH->lo_buf_size = LOBBUFSIZE;
> +    AH->lo_buf = (void *)malloc(LOBBUFSIZE);
> +    if(AH->lo_buf == NULL)
> +                die_horribly(AH, modulename, "out of memory\n");
> +
>      /*
>       * Now open the TOC file
>       */
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_null.c 
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_null.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_null.c    Wed Jun 27 23:21:37 
> 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_null.c    Thu Apr 11 10:44:53 2002
> @@ -64,7 +64,6 @@
>       */
>      if (AH->mode == archModeRead)
>          die_horribly(AH, NULL, "this format cannot be read\n");
> -
>  }
>  
>  /*
> diff -Nur postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_tar.c 
> postgresql-7.2.1/src/bin/pg_dump/pg_backup_tar.c
> --- postgresql-7.2.1-orig/src/bin/pg_dump/pg_backup_tar.c    Sun Oct 28 07:25:58 
> 2001
> +++ postgresql-7.2.1/src/bin/pg_dump/pg_backup_tar.c    Thu Apr 11 10:44:08 2002
> @@ -157,6 +157,12 @@
>      ctx = (lclContext *) malloc(sizeof(lclContext));
>      AH->formatData = (void *) ctx;
>      ctx->filePos = 0;
> +    
> +    /* Initialize LO buffering */
> +    AH->lo_buf_size = LOBBUFSIZE;
> +    AH->lo_buf = (void *)malloc(LOBBUFSIZE);
> +    if(AH->lo_buf == NULL)
> +                die_horribly(AH, modulename, "out of memory\n");
>  
>      /*
>       * Now open the TOC file
> ============================
> 
> >
> > ---------------------------(end of broadcast)---------------------------
> > TIP 4: Don't 'kill -9' the postmaster
> 
> ---------------------------(end of broadcast)---------------------------
> TIP 6: Have you searched our list archives?
> 
> http://archives.postgresql.org
> 

--  Bruce Momjian                        |  http://candle.pha.pa.us pgman@candle.pha.pa.us               |  (610)
853-3000+  If your life is a hard drive,     |  830 Blythe Avenue +  Christ can be your backup.        |  Drexel Hill,
Pennsylvania19026