Re: [PATCHES] Unicode characters above 0x10000 #2 - Mailing list pgsql-hackers

Bruce,

Attached patch replaces the original, applied today against CVS HEAD.
Fixes the surrogates, and limits to 4 byte utf8 as per spec.

Also extends UtfToLocal to 4 byte characters (tho, it does not add any,
just enables the code to handle them. If my interpretation of this code
is wrong, please let me know, and correct it).

... John

> -----Original Message-----
> From: Bruce Momjian [mailto:pgman@candle.pha.pa.us]
> Sent: Sunday, June 05, 2005 11:23 AM
> To: pgman@candle.pha.pa.us
> Cc: John Hansen; pgsql-hackers@postgresql.org; PostgreSQL-patches
> Subject: Re: [PATCHES] Unicode characters above 0x10000 #2
>
>
> Your patch has been added to the PostgreSQL unapplied patches list at:
>
>     http://momjian.postgresql.org/cgi-bin/pgpatches
>
> It will be applied as soon as one of the PostgreSQL
> committers reviews and approves it.
>
> --------------------------------------------------------------
> -------------
>
>
> pgman wrote:
> >
> > I have backed out this patch.  It is unclear it is a bug fix.
> >
> > It will be saved for 8.1.
> >
> >
> ----------------------------------------------------------------------
> > -----
> >
> > pgman wrote:
> > >
> > > Patch applied.  Thanks.
> > >
> > >
> --------------------------------------------------------------------
> > > -------
> > >
> > >
> > > John Hansen wrote:
> > > > 3 times lucky?
> > > >
> > > > Last one broke utf8.... Grrrr
> > > >
> > > > This one works,.... Too tired, sorry for the inconvenience..
> > > >
> > > > ... John
> > >
> > > Content-Description: cvs.diff
> > >
> > > [ Attachment, skipping... ]
> > >
> > > >
> > > > ---------------------------(end of
> > > > broadcast)---------------------------
> > > > TIP 9: the planner will ignore your desire to choose an
> index scan if your
> > > >       joining column's datatypes do not match
> > >
> > > --
> > >   Bruce Momjian                        |  http://candle.pha.pa.us
> > >   pgman@candle.pha.pa.us               |  (610) 359-1001
> > >   +  If your life is a hard drive,     |  13 Roberts Road
> > >   +  Christ can be your backup.        |  Newtown Square,
> Pennsylvania 19073
> >
> > --
> >   Bruce Momjian                        |  http://candle.pha.pa.us
> >   pgman@candle.pha.pa.us               |  (610) 359-1001
> >   +  If your life is a hard drive,     |  13 Roberts Road
> >   +  Christ can be your backup.        |  Newtown Square,
> Pennsylvania 19073
>
> > ===================================================================
> > RCS file: /projects/cvsroot/pgsql/src/backend/utils/mb/wchar.c,v
> > retrieving revision 1.38
> > diff -c -r1.38 wchar.c
> > *** src/backend/utils/mb/wchar.c    17 Sep 2004 21:59:57
> -0000    1.38
> > --- src/backend/utils/mb/wchar.c    21 Nov 2004 09:58:36 -0000
> > ***************
> > *** 343,348 ****
> > --- 343,373 ----
> >       return (pg_euc_dsplen(s));
> >   }
> >
> > + bool isLegalUTF8(const UTF8 *source, int len) {
> > +         UTF8 a;
> > +         const UTF8 *srcptr = source+len;
> > +         if(!source || (pg_utf_mblen(source) != len)) return false;
> > +         switch (len) {
> > +             default: return false;
> > +             /* Everything else falls through when "true"... */
> > +             case 6: if ((a = (*--srcptr)) < 0x80 || a >
> 0xBF) return false;
> > +             case 5: if ((a = (*--srcptr)) < 0x80 || a >
> 0xBF) return false;
> > +             case 4: if ((a = (*--srcptr)) < 0x80 || a >
> 0xBF) return false;
> > +             case 3: if ((a = (*--srcptr)) < 0x80 || a >
> 0xBF) return false;
> > +             case 2: if ((a = (*--srcptr)) > 0xBF) return false;
> > +             switch (*source) {
> > +                     /* no fall-through in this inner switch */
> > +                     case 0xE0: if (a < 0xA0) return false; break;
> > +                     case 0xF0: if (a < 0x90) return false; break;
> > +                     case 0xF4: if (a > 0x8F) return false; break;
> > +                     default:  if (a < 0x80) return false;
> > +             }
> > +             case 1: if (*source >= 0x80 && *source <
> 0xC2) return false;
> > +             if (*source > 0xFD) return false;
> > +         }
> > +         return true;
> > + }
> > +
> >   /*
> >    * convert UTF-8 string to pg_wchar (UCS-2)
> >    * caller should allocate enough space for "to"
> > ***************
> > *** 398,404 ****
> >    * returns the byte length of a UTF-8 word pointed to by s
> >    */
> >   int
> > ! pg_utf_mblen(const unsigned char *s)
> >   {
> >       int            len = 1;
> >
> > --- 423,429 ----
> >    * returns the byte length of a UTF-8 word pointed to by s
> >    */
> >   int
> > ! pg_utf_mblen(const UTF8 *s)
> >   {
> >       int            len = 1;
> >
> > ***************
> > *** 406,418 ****
> >           len = 1;
> >       else if ((*s & 0xe0) == 0xc0)
> >           len = 2;
> > !     else if ((*s & 0xe0) == 0xe0)
> > !         len = 3;
> >       return (len);
> >   }
> >
> >   static int
> > ! pg_utf_dsplen(const unsigned char *s)
> >   {
> >       return 1;                    /* XXX
> fix me! */
> >   }
> > --- 431,449 ----
> >           len = 1;
> >       else if ((*s & 0xe0) == 0xc0)
> >           len = 2;
> > !         else if ((*s & 0xf0) == 0xe0)
> > !                 len = 3;
> > !         else if ((*s & 0xf8) == 0xf0)
> > !                 len = 4;
> > !         else if ((*s & 0xfc) == 0xf8)
> > !                 len = 5;
> > !         else if ((*s & 0xfe) == 0xfc)
> > !                 len = 6;
> >       return (len);
> >   }
> >
> >   static int
> > ! pg_utf_dsplen(const UTF8 *s)
> >   {
> >       return 1;                    /* XXX
> fix me! */
> >   }
> > ***************
> > *** 721,728 ****
> >       {pg_euckr2wchar_with_len, pg_euckr_mblen,
> pg_euckr_dsplen, 3},        /* 3; PG_EUC_KR */
> >       {pg_euctw2wchar_with_len, pg_euctw_mblen,
> pg_euctw_dsplen, 3},        /* 4; PG_EUC_TW */
> >       {pg_johab2wchar_with_len, pg_johab_mblen,
> pg_johab_dsplen, 3},        /* 5; PG_JOHAB */
> > !     {pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen,
> 3},    /* 6; PG_UNICODE */
> > !     {pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen,
> 3}, /* 7; PG_MULE_INTERNAL */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 8; PG_LATIN1 */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 9; PG_LATIN2 */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 10; PG_LATIN3 */
> > --- 752,759 ----
> >       {pg_euckr2wchar_with_len, pg_euckr_mblen,
> pg_euckr_dsplen, 3},        /* 3; PG_EUC_KR */
> >       {pg_euctw2wchar_with_len, pg_euctw_mblen,
> pg_euctw_dsplen, 3},        /* 4; PG_EUC_TW */
> >       {pg_johab2wchar_with_len, pg_johab_mblen,
> pg_johab_dsplen, 3},        /* 5; PG_JOHAB */
> > !     {pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen,
> 6},        /* 6; PG_UNICODE */
> > !     {pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen,
> 3},        /* 7; PG_MULE_INTERNAL */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 8; PG_LATIN1 */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 9; PG_LATIN2 */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 10; PG_LATIN3 */
> > ***************
> > *** 744,754 ****
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 26; ISO-8859-7 */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 27; ISO-8859-8 */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 28; PG_WIN1250 */
> > !     {0, pg_sjis_mblen, pg_sjis_dsplen, 2},        /* 29;
> PG_SJIS */
> > !     {0, pg_big5_mblen, pg_big5_dsplen, 2},        /* 30;
> PG_BIG5 */
> > !     {0, pg_gbk_mblen, pg_gbk_dsplen, 2},        /* 31; PG_GBK */
> > !     {0, pg_uhc_mblen, pg_uhc_dsplen, 2},        /* 32; PG_UHC */
> > !     {0, pg_gb18030_mblen, pg_gb18030_dsplen, 2} /* 33; PG_GB18030 */
> >   };
> >
> >   /* returns the byte length of a word for mule internal code */
> > --- 775,785 ----
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 26; ISO-8859-7 */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 27; ISO-8859-8 */
> >       {pg_latin12wchar_with_len, pg_latin1_mblen,
> pg_latin1_dsplen, 1},    /* 28; PG_WIN1250 */
> > !     {0, pg_sjis_mblen, pg_sjis_dsplen, 2},
>         /* 29; PG_SJIS */
> > !     {0, pg_big5_mblen, pg_big5_dsplen, 2},
>         /* 30; PG_BIG5 */
> > !     {0, pg_gbk_mblen, pg_gbk_dsplen, 2},
>         /* 31; PG_GBK */
> > !     {0, pg_uhc_mblen, pg_uhc_dsplen, 2},
>         /* 32; PG_UHC */
> > !     {0, pg_gb18030_mblen, pg_gb18030_dsplen, 2}
>         /* 33; PG_GB18030 */
> >   };
> >
> >   /* returns the byte length of a word for mule internal code */
> > ***************
> > *** 822,872 ****
> >
> >       while (len > 0 && *mbstr)
> >       {
> > -         /* special UTF-8 check */
> > -         if (encoding == PG_UTF8 && (*mbstr & 0xf8) == 0xf0)
> > -         {
> > -             if (noError)
> > -                 return false;
> > -             ereport(ERROR,
> > -
> (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
> > -                      errmsg("Unicode
> characters greater than or equal to 0x10000 are not supported")));
> > -         }
> > -
> >           l = pg_mblen(mbstr);
> >
> > !         for (i = 1; i < l; i++)
> > !         {
> > !             /*
> > !              * we expect that every multibyte char
> consists of bytes
> > !              * having the 8th bit set
> > !              */
> > !             if (i >= len || (mbstr[i] & 0x80) == 0)
> >               {
> > !                 char        buf[8 * 2 + 1];
> > !                 char       *p = buf;
> > !                 int            j,
> >                               jlimit;
> >
> > !                 if (noError)
> > !                     return false;
> >
> > !                 jlimit = Min(l, len);
> > !                 jlimit = Min(jlimit, 8);
>     /* prevent buffer overrun */
> >
> > !                 for (j = 0; j < jlimit; j++)
> > !                     p += sprintf(p, "%02x",
> mbstr[j]);
> >
> > !                 ereport(ERROR,
> > !
> (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
> > !                 errmsg("invalid byte sequence
> for encoding \"%s\": 0x%s",
> > !
> GetDatabaseEncodingName(), buf)));
> >               }
> > -         }
> >
> >           len -= l;
> >           mbstr += l;
> >       }
> > -
> >       return true;
> >   }
> >
> > --- 853,900 ----
> >
> >       while (len > 0 && *mbstr)
> >       {
> >           l = pg_mblen(mbstr);
> >
> > !         /* special UTF-8 check */
> > !         if (encoding == PG_UTF8) {
> > !             if(!isLegalUTF8(mbstr,l)) {
> > !                 if (noError) return false;
> > !
> ereport(ERROR,(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),er
> rmsg("Invalid UNICODE byte sequence detected near character
> %c",*mbstr)));
> > !             }
> > !         } else {
> > !             for (i = 1; i < l; i++)
> >               {
> > !                 /*
> > !                  * we expect that every
> multibyte char consists of bytes
> > !                  * having the 8th bit set
> > !                  */
> > !                 if (i >= len || (mbstr[i] & 0x80) == 0)
> > !                 {
> > !                     char        buf[8 * 2 + 1];
> > !                     char       *p = buf;
> > !                     int            j,
> >                               jlimit;
> >
> > !                     if (noError)
> > !                         return false;
> >
> > !                     jlimit = Min(l, len);
> > !                     jlimit = Min(jlimit,
> 8);        /* prevent buffer overrun */
> >
> > !                     for (j = 0; j < jlimit; j++)
> > !                         p += sprintf(p,
> "%02x", mbstr[j]);
> >
> > !                     ereport(ERROR,
> > !
> (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
> > !                     errmsg("invalid byte
> sequence for encoding \"%s\": 0x%s",
> > !
> GetDatabaseEncodingName(), buf)));
> > !                 }
> >               }
> >
> > +         }
> >           len -= l;
> >           mbstr += l;
> >       }
> >       return true;
> >   }
> >
> > Index: src/include/mb/pg_wchar.h
> > ===================================================================
> > RCS file: /projects/cvsroot/pgsql/src/include/mb/pg_wchar.h,v
> > retrieving revision 1.52
> > diff -c -r1.52 pg_wchar.h
> > *** src/include/mb/pg_wchar.h    17 Sep 2004 21:59:57
> -0000    1.52
> > --- src/include/mb/pg_wchar.h    21 Nov 2004 09:58:36 -0000
> > ***************
> > *** 17,22 ****
> > --- 17,30 ----
> >    */
> >   typedef unsigned int pg_wchar;
> >
> > +
> > + /*
> > +  * The UTF types
> > +  */
> > + typedef unsigned int    UTF32;  /* at least 32 bits */
> > + typedef unsigned short    UTF16;  /* at least 16 bits */
> > + typedef unsigned char    UTF8;   /* typically 8 bits */
> > +
> >   /*
> >    * various definitions for EUC
> >    */
> > ***************
> > *** 339,342 ****
> > --- 347,352 ----
> >   extern void latin2mic_with_table(unsigned char *l,
> unsigned char *p, int len, int lc, unsigned char *tab);
> >   extern void mic2latin_with_table(unsigned char *mic,
> unsigned char
> > *p, int len, int lc, unsigned char *tab);
> >
> > + extern bool isLegalUTF8(const UTF8 *source, int len);
> > +
> >   #endif   /* PG_WCHAR_H */
> >
> >
>
> --
>   Bruce Momjian                        |  http://candle.pha.pa.us
>   pgman@candle.pha.pa.us               |  (610) 359-1001
>   +  If your life is a hard drive,     |  13 Roberts Road
>   +  Christ can be your backup.        |  Newtown Square,
> Pennsylvania 19073
>
>