Thread: libpgtcl doesn't use UTF encoding of TCL
Eugene Faukin (elf@solvo.ru) reports a bug with a severity of 2
The lower the number the more severe it is.
Short Description
libpgtcl doesn't use UTF encoding of TCL
Long Description
Modern versions of the TCL (8.2 at least) use UTF encoding to internal
storage of the text. libpgtcl uses TCL functions to insert strings directly
into TCL internal structure without any conversion.
Sample Code
I can suggest you next patch I use for myself:
diff -uNr postgresql-7.0.2.orig/src/interfaces/libpgtcl/pgtclCmds.c
postgresql-7.0.2/src/interfaces/libpgtcl/pgtclCmds.c
--- postgresql-7.0.2.orig/src/interfaces/libpgtcl/pgtclCmds.c Wed Apr 12 21:17:11 2000
+++ postgresql-7.0.2/src/interfaces/libpgtcl/pgtclCmds.c Thu Nov 16 20:26:37 2000
@@ -431,6 +431,7 @@
Pg_ConnectionId *connid;
PGconn *conn;
PGresult *result;
+ Tcl_DString putString;
if (argc != 3)
{
@@ -449,7 +450,9 @@
return TCL_ERROR;
}
- result = PQexec(conn, argv[2]);
+ Tcl_UtfToExternalDString(NULL, argv[2], -1, &putString);
+ result = PQexec(conn, Tcl_DStringValue(&putString));
+ Tcl_DStringFree(&putString);
/* Transfer any notify events from libpq to Tcl event queue. */
PgNotifyTransferEvents(connid);
@@ -535,6 +538,7 @@
char *arrVar;
char nameBuffer[256];
const char *appendstr;
+ Tcl_DString retString;
if (argc < 3 || argc > 5)
{
@@ -685,11 +689,24 @@
}
#ifdef TCL_ARRAYS
for (i = 0; i < PQnfields(result); i++)
- Tcl_AppendElement(interp, tcl_value(PQgetvalue(result, tupno, i)));
+ {
+ Tcl_ExternalToUtfDString(NULL,
+ tcl_value(PQgetvalue(result,
+ tupno, i)),
+ -1, &retString);
+ Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
+ }
#else
for (i = 0; i < PQnfields(result); i++)
- Tcl_AppendElement(interp, PQgetvalue(result, tupno, i));
+ {
+ Tcl_ExternalToUtfDString(NULL,
+ PQgetvalue(result, tupno, i),
+ -1, &retString);
+
+ Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
+ }
#endif
+ Tcl_DStringFree(&retString);
return TCL_OK;
}
else if (strcmp(opt, "-tupleArray") == 0)
@@ -707,21 +724,35 @@
}
for (i = 0; i < PQnfields(result); i++)
{
- if (Tcl_SetVar2(interp, argv[4], PQfname(result, i),
+ Tcl_ExternalToUtfDString(NULL,
#ifdef TCL_ARRAYS
- tcl_value(PQgetvalue(result, tupno, i)),
+ tcl_value(PQgetvalue(result,
+ tupno, i)),
#else
- PQgetvalue(result, tupno, i),
+ PQgetvalue(result, tupno, i),
#endif
- TCL_LEAVE_ERR_MSG) == NULL)
- return TCL_ERROR;
+ -1, &retString);
+
+ if (Tcl_SetVar2(interp, argv[4], PQfname(result, i),
+ Tcl_DStringValue(&retString),
+ TCL_LEAVE_ERR_MSG) == NULL)
+ {
+ Tcl_DStringFree(&retString);
+ return TCL_ERROR;
+ }
}
+ Tcl_DStringFree(&retString);
return TCL_OK;
}
else if (strcmp(opt, "-attributes") == 0)
{
for (i = 0; i < PQnfields(result); i++)
- Tcl_AppendElement(interp, PQfname(result, i));
+ {
+ Tcl_ExternalToUtfDString(NULL, PQfname(result, i),
+ -1, &retString);
+ Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
+ Tcl_DStringFree(&retString);
+ }
return TCL_OK;
}
else if (strcmp(opt, "-lAttributes") == 0)
@@ -1274,6 +1305,8 @@
column,
ncols;
Tcl_DString headers;
+ Tcl_DString retString;
+ Tcl_DString putString;
char buffer[2048];
struct info_s
{
@@ -1292,7 +1325,11 @@
if (conn == (PGconn *) NULL)
return TCL_ERROR;
- if ((result = PQexec(conn, argv[2])) == 0)
+ Tcl_UtfToExternalDString(NULL, argv[2], -1, &putString);
+ result = PQexec(conn, Tcl_DStringValue(&putString));
+ Tcl_DStringFree(&putString);
+
+ if (result == 0)
{
/* error occurred sending the query */
Tcl_SetResult(interp, PQerrorMessage(conn), TCL_VOLATILE);
@@ -1340,13 +1377,21 @@
Tcl_SetVar2(interp, argv[3], ".tupno", buffer, 0);
for (column = 0; column < ncols; column++)
- Tcl_SetVar2(interp, argv[3], info[column].cname,
+ {
+ Tcl_ExternalToUtfDString(NULL,
#ifdef TCL_ARRAYS
- tcl_value(PQgetvalue(result, tupno, column)),
+ tcl_value(PQgetvalue(result,
+ tupno,
+ column)),
#else
- PQgetvalue(result, tupno, column),
+ PQgetvalue(result, tupno, column),
#endif
- 0);
+ -1, &retString);
+
+ Tcl_SetVar2(interp, argv[3], info[column].cname,
+ Tcl_DStringValue(&retString), 0);
+ Tcl_DStringFree(&retString);
+ }
Tcl_SetVar2(interp, argv[3], ".command", "update", 0);
No file was uploaded with this report
Do you have any idea how this will work with earlier TCL versions? When
was Tcl_UtfToExternalDString added to TCL?
> Eugene Faukin (elf@solvo.ru) reports a bug with a severity of 2
> The lower the number the more severe it is.
>
> Short Description
> libpgtcl doesn't use UTF encoding of TCL
>
> Long Description
> Modern versions of the TCL (8.2 at least) use UTF encoding to internal
> storage of the text. libpgtcl uses TCL functions to insert strings directly
> into TCL internal structure without any conversion.
>
>
> Sample Code
> I can suggest you next patch I use for myself:
>
> diff -uNr postgresql-7.0.2.orig/src/interfaces/libpgtcl/pgtclCmds.c
postgresql-7.0.2/src/interfaces/libpgtcl/pgtclCmds.c
> --- postgresql-7.0.2.orig/src/interfaces/libpgtcl/pgtclCmds.c Wed Apr 12 21:17:11 2000
> +++ postgresql-7.0.2/src/interfaces/libpgtcl/pgtclCmds.c Thu Nov 16 20:26:37 2000
> @@ -431,6 +431,7 @@
> Pg_ConnectionId *connid;
> PGconn *conn;
> PGresult *result;
> + Tcl_DString putString;
>
> if (argc != 3)
> {
> @@ -449,7 +450,9 @@
> return TCL_ERROR;
> }
>
> - result = PQexec(conn, argv[2]);
> + Tcl_UtfToExternalDString(NULL, argv[2], -1, &putString);
> + result = PQexec(conn, Tcl_DStringValue(&putString));
> + Tcl_DStringFree(&putString);
>
> /* Transfer any notify events from libpq to Tcl event queue. */
> PgNotifyTransferEvents(connid);
> @@ -535,6 +538,7 @@
> char *arrVar;
> char nameBuffer[256];
> const char *appendstr;
> + Tcl_DString retString;
>
> if (argc < 3 || argc > 5)
> {
> @@ -685,11 +689,24 @@
> }
> #ifdef TCL_ARRAYS
> for (i = 0; i < PQnfields(result); i++)
> - Tcl_AppendElement(interp, tcl_value(PQgetvalue(result, tupno, i)));
> + {
> + Tcl_ExternalToUtfDString(NULL,
> + tcl_value(PQgetvalue(result,
> + tupno, i)),
> + -1, &retString);
> + Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
> + }
> #else
> for (i = 0; i < PQnfields(result); i++)
> - Tcl_AppendElement(interp, PQgetvalue(result, tupno, i));
> + {
> + Tcl_ExternalToUtfDString(NULL,
> + PQgetvalue(result, tupno, i),
> + -1, &retString);
> +
> + Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
> + }
> #endif
> + Tcl_DStringFree(&retString);
> return TCL_OK;
> }
> else if (strcmp(opt, "-tupleArray") == 0)
> @@ -707,21 +724,35 @@
> }
> for (i = 0; i < PQnfields(result); i++)
> {
> - if (Tcl_SetVar2(interp, argv[4], PQfname(result, i),
> + Tcl_ExternalToUtfDString(NULL,
> #ifdef TCL_ARRAYS
> - tcl_value(PQgetvalue(result, tupno, i)),
> + tcl_value(PQgetvalue(result,
> + tupno, i)),
> #else
> - PQgetvalue(result, tupno, i),
> + PQgetvalue(result, tupno, i),
> #endif
> - TCL_LEAVE_ERR_MSG) == NULL)
> - return TCL_ERROR;
> + -1, &retString);
> +
> + if (Tcl_SetVar2(interp, argv[4], PQfname(result, i),
> + Tcl_DStringValue(&retString),
> + TCL_LEAVE_ERR_MSG) == NULL)
> + {
> + Tcl_DStringFree(&retString);
> + return TCL_ERROR;
> + }
> }
> + Tcl_DStringFree(&retString);
> return TCL_OK;
> }
> else if (strcmp(opt, "-attributes") == 0)
> {
> for (i = 0; i < PQnfields(result); i++)
> - Tcl_AppendElement(interp, PQfname(result, i));
> + {
> + Tcl_ExternalToUtfDString(NULL, PQfname(result, i),
> + -1, &retString);
> + Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
> + Tcl_DStringFree(&retString);
> + }
> return TCL_OK;
> }
> else if (strcmp(opt, "-lAttributes") == 0)
> @@ -1274,6 +1305,8 @@
> column,
> ncols;
> Tcl_DString headers;
> + Tcl_DString retString;
> + Tcl_DString putString;
> char buffer[2048];
> struct info_s
> {
> @@ -1292,7 +1325,11 @@
> if (conn == (PGconn *) NULL)
> return TCL_ERROR;
>
> - if ((result = PQexec(conn, argv[2])) == 0)
> + Tcl_UtfToExternalDString(NULL, argv[2], -1, &putString);
> + result = PQexec(conn, Tcl_DStringValue(&putString));
> + Tcl_DStringFree(&putString);
> +
> + if (result == 0)
> {
> /* error occurred sending the query */
> Tcl_SetResult(interp, PQerrorMessage(conn), TCL_VOLATILE);
> @@ -1340,13 +1377,21 @@
> Tcl_SetVar2(interp, argv[3], ".tupno", buffer, 0);
>
> for (column = 0; column < ncols; column++)
> - Tcl_SetVar2(interp, argv[3], info[column].cname,
> + {
> + Tcl_ExternalToUtfDString(NULL,
> #ifdef TCL_ARRAYS
> - tcl_value(PQgetvalue(result, tupno, column)),
> + tcl_value(PQgetvalue(result,
> + tupno,
> + column)),
> #else
> - PQgetvalue(result, tupno, column),
> + PQgetvalue(result, tupno, column),
> #endif
> - 0);
> + -1, &retString);
> +
> + Tcl_SetVar2(interp, argv[3], info[column].cname,
> + Tcl_DStringValue(&retString), 0);
> + Tcl_DStringFree(&retString);
> + }
>
> Tcl_SetVar2(interp, argv[3], ".command", "update", 0);
>
>
>
> No file was uploaded with this report
>
>
> ---------------------------(end of broadcast)---------------------------
> TIP 4: Don't 'kill -9' the postmaster
>
--
Bruce Momjian | http://candle.pha.pa.us
pgman@candle.pha.pa.us | (610) 853-3000
+ If your life is a hard drive, | 830 Blythe Avenue
+ Christ can be your backup. | Drexel Hill, Pennsylvania 19026
> On Wed, 18 Jul 2001, Bruce Momjian wrote: > > > Do you have any idea how this will work with earlier TCL versions? > > It won't. If pgtcl is supposed to still be able to compile with older > versions of Tcl, the changes have to be made a compile time option. Can't we probe the TCL version in the code and handle it that way? > > When was Tcl_UtfToExternalDString added to TCL? > > According to Tcl's changelog files, it was added in 1997 for Tcl 8.1 > wich was released in 1999. Shame. I think we could have required Tcl 8.X but clearly we can't be requring >= 8.1. I have 8.0 here and I am sure many do as well. -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 853-3000 + If your life is a hard drive, | 830 Blythe Avenue + Christ can be your backup. | Drexel Hill, Pennsylvania 19026
On Wed, 18 Jul 2001, Bruce Momjian wrote:
> Do you have any idea how this will work with earlier TCL versions?
It won't. If pgtcl is supposed to still be able to compile with older
versions of Tcl, the changes have to be made a compile time option.
> When was Tcl_UtfToExternalDString added to TCL?
According to Tcl's changelog files, it was added in 1997 for Tcl 8.1
wich was released in 1999.
cu
Reinhard Max
Maintainer of the Tcl/Tk and PostgreSQL packages for SuSE Linux
Reinhard Max <max@suse.de> writes:
> On Wed, 18 Jul 2001, Bruce Momjian wrote:
>> Do you have any idea how this will work with earlier TCL versions?
> It won't. If pgtcl is supposed to still be able to compile with older
> versions of Tcl, the changes have to be made a compile time option.
Please do that and resubmit the patch. We really don't want to give up
backwards compatibility just yet.
regards, tom lane
> On Wed, Jul 18, 2001 at 02:53:22PM -0400, Tom Lane wrote: > > > > It won't. If pgtcl is supposed to still be able to compile with older > > > versions of Tcl, the changes have to be made a compile time option. > > > > Please do that and resubmit the patch. We really don't want to give up > > backwards compatibility just yet. > > > > Thank you, gentlemans :-) > I will be interest with this result too. I believe $tcl_version is what you want to use to test. -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 853-3000 + If your life is a hard drive, | 830 Blythe Avenue + Christ can be your backup. | Drexel Hill, Pennsylvania 19026
On Wed, 18 Jul 2001, Tom Lane wrote:
> Reinhard Max <max@suse.de> writes:
> > On Wed, 18 Jul 2001, Bruce Momjian wrote:
> >> Do you have any idea how this will work with earlier TCL versions?
>
> > It won't. If pgtcl is supposed to still be able to compile with older
> > versions of Tcl, the changes have to be made a compile time option.
>
> Please do that and resubmit the patch.
OK, I'll pack the new stuff inside #ifdef TCL_UTF8 and define that if
the Tcl version is 8.1 or greater.
Can anybody tell me, if that TCL_ARRAYS stuff is still good for
something? If I could remove it, TCL_UTF8 would be less complex at
some places, because I'd only have to cover two instead of four cases.
BTW, I think the proposed Patch doesn't go far enough as it assumes
the database (client) encoding is identical to the system encoding by
using NULL as the first argument for Tcl_UtfToExternalDString and
Tcl_ExternalToUtfDString. I think it should instead either use the
database's encoding for the conversion to be correct or set
PostgreSQL's client encoding to UNICODE so that no conversion would be
needed. Unfortunately, I don't have the time to do that at the moment.
> We really don't want to give up backwards compatibility just yet.
How far do you want it to be backwards compatible? If >= 8.0 is OK,
I'll possibly overwork libpq later this year to use Tcl's object
interface. I expect at least some performance gain out of this.
cu
Reinhard
On Wed, 18 Jul 2001, Tom Lane wrote:
> Reinhard Max <max@suse.de> writes:
> > On Wed, 18 Jul 2001, Bruce Momjian wrote:
> >> Do you have any idea how this will work with earlier TCL versions?
>
> > It won't. If pgtcl is supposed to still be able to compile with older
> > versions of Tcl, the changes have to be made a compile time option.
>
> Please do that and resubmit the patch.
Here it is, but I consider it still incomplete and I have not done
exhaustive testing. Some more occurrences of PQexec and PQgetvalue
need to be wrapped up with UTF8 conversion, but I'll not have the time
to do it for the next 1-2 weeks.
cu
Reinhard
> On Wed, 18 Jul 2001, Tom Lane wrote: > > > Reinhard Max <max@suse.de> writes: > > > On Wed, 18 Jul 2001, Bruce Momjian wrote: > > >> Do you have any idea how this will work with earlier TCL versions? > > > > > It won't. If pgtcl is supposed to still be able to compile with older > > > versions of Tcl, the changes have to be made a compile time option. > > > > Please do that and resubmit the patch. > > OK, I'll pack the new stuff inside #ifdef TCL_UTF8 and define that if > the Tcl version is 8.1 or greater. Is the TCL_UTF8 some variable that gets set at tcl runtime? I hope so. -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 853-3000 + If your life is a hard drive, | 830 Blythe Avenue + Christ can be your backup. | Drexel Hill, Pennsylvania 19026
Reinhard Max writes: > OK, I'll pack the new stuff inside #ifdef TCL_UTF8 and define that if > the Tcl version is 8.1 or greater. No, please add a configure check for Tcl_UtfToExternalDString or some other function representative of this interface.. -- Peter Eisentraut peter_e@gmx.net http://funkturm.homeip.net/~peter
> On Fri, 20 Jul 2001, Peter Eisentraut wrote: > > > Reinhard Max writes: > > > > > OK, I'll pack the new stuff inside #ifdef TCL_UTF8 and define that if > > > the Tcl version is 8.1 or greater. > > > > No, please add a configure check for Tcl_UtfToExternalDString or > > some other function representative of this interface.. > > Why make simple things complicated? > Tcl changed it's internal string representation starting with release > 8.1 . It is not an interface one can decide whether to use it or not. > Every extension that imports or exports strings and gets compiled for > Tcl >= 8.1 has to make sure that they are UTF8 regardless, if it uses > the Tcl_*Utf*DString functions or something else. So I consider it > sufficient to define TCL_UTF8 if Tcl's Version is >= 8.1 as I did in > the patch that was attached to my last mail. I think he is OK checking the TCL version. It is pretty common to check the TCL include file for symbols and handle things that way. Do we test any other TCL include defines from configure? -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 853-3000 + If your life is a hard drive, | 830 Blythe Avenue + Christ can be your backup. | Drexel Hill, Pennsylvania 19026
On Fri, 20 Jul 2001, Peter Eisentraut wrote:
> Reinhard Max writes:
>
> > OK, I'll pack the new stuff inside #ifdef TCL_UTF8 and define that if
> > the Tcl version is 8.1 or greater.
>
> No, please add a configure check for Tcl_UtfToExternalDString or
> some other function representative of this interface..
Why make simple things complicated?
Tcl changed it's internal string representation starting with release
8.1 . It is not an interface one can decide whether to use it or not.
Every extension that imports or exports strings and gets compiled for
Tcl >= 8.1 has to make sure that they are UTF8 regardless, if it uses
the Tcl_*Utf*DString functions or something else. So I consider it
sufficient to define TCL_UTF8 if Tcl's Version is >= 8.1 as I did in
the patch that was attached to my last mail.
cu
Reinhard
> > On Wed, 18 Jul 2001, Tom Lane wrote: > > > > > Reinhard Max <max@suse.de> writes: > > > > On Wed, 18 Jul 2001, Bruce Momjian wrote: > > > >> Do you have any idea how this will work with earlier TCL versions? > > > > > > > It won't. If pgtcl is supposed to still be able to compile with older > > > > versions of Tcl, the changes have to be made a compile time option. > > > > > > Please do that and resubmit the patch. > > > > OK, I'll pack the new stuff inside #ifdef TCL_UTF8 and define that if > > the Tcl version is 8.1 or greater. > > Is the TCL_UTF8 some variable that gets set at tcl runtime? I hope so. I now realize we can't have this configure at runtime. It has to read the tcl include file for the version it is about to be linked to. -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 853-3000 + If your life is a hard drive, | 830 Blythe Avenue + Christ can be your backup. | Drexel Hill, Pennsylvania 19026
On Wed, Jul 18, 2001 at 02:53:22PM -0400, Tom Lane wrote: > > It won't. If pgtcl is supposed to still be able to compile with older > > versions of Tcl, the changes have to be made a compile time option. > > Please do that and resubmit the patch. We really don't want to give up > backwards compatibility just yet. > Thank you, gentlemans :-) I will be interest with this result too. -- Eugene Faukin SOLVO Ltd. Company
Hum. Why don't you enable --enable-multibyte and
--enable-unicode-conversion and set client_encoding to UNICODE? That
would do a conversion from/to UTF-8 for Tcl 8.x (x > 9) clients?
--
Tatsuo Ishii
> Eugene Faukin (elf@solvo.ru) reports a bug with a severity of 2
> The lower the number the more severe it is.
>
> Short Description
> libpgtcl doesn't use UTF encoding of TCL
>
> Long Description
> Modern versions of the TCL (8.2 at least) use UTF encoding to internal
> storage of the text. libpgtcl uses TCL functions to insert strings directly
> into TCL internal structure without any conversion.
>
>
> Sample Code
> I can suggest you next patch I use for myself:
>
> diff -uNr postgresql-7.0.2.orig/src/interfaces/libpgtcl/pgtclCmds.c
postgresql-7.0.2/src/interfaces/libpgtcl/pgtclCmds.c
> --- postgresql-7.0.2.orig/src/interfaces/libpgtcl/pgtclCmds.c Wed Apr 12 21:17:11 2000
> +++ postgresql-7.0.2/src/interfaces/libpgtcl/pgtclCmds.c Thu Nov 16 20:26:37 2000
> @@ -431,6 +431,7 @@
> Pg_ConnectionId *connid;
> PGconn *conn;
> PGresult *result;
> + Tcl_DString putString;
>
> if (argc != 3)
> {
> @@ -449,7 +450,9 @@
> return TCL_ERROR;
> }
>
> - result = PQexec(conn, argv[2]);
> + Tcl_UtfToExternalDString(NULL, argv[2], -1, &putString);
> + result = PQexec(conn, Tcl_DStringValue(&putString));
> + Tcl_DStringFree(&putString);
>
> /* Transfer any notify events from libpq to Tcl event queue. */
> PgNotifyTransferEvents(connid);
> @@ -535,6 +538,7 @@
> char *arrVar;
> char nameBuffer[256];
> const char *appendstr;
> + Tcl_DString retString;
>
> if (argc < 3 || argc > 5)
> {
> @@ -685,11 +689,24 @@
> }
> #ifdef TCL_ARRAYS
> for (i = 0; i < PQnfields(result); i++)
> - Tcl_AppendElement(interp, tcl_value(PQgetvalue(result, tupno, i)));
> + {
> + Tcl_ExternalToUtfDString(NULL,
> + tcl_value(PQgetvalue(result,
> + tupno, i)),
> + -1, &retString);
> + Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
> + }
> #else
> for (i = 0; i < PQnfields(result); i++)
> - Tcl_AppendElement(interp, PQgetvalue(result, tupno, i));
> + {
> + Tcl_ExternalToUtfDString(NULL,
> + PQgetvalue(result, tupno, i),
> + -1, &retString);
> +
> + Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
> + }
> #endif
> + Tcl_DStringFree(&retString);
> return TCL_OK;
> }
> else if (strcmp(opt, "-tupleArray") == 0)
> @@ -707,21 +724,35 @@
> }
> for (i = 0; i < PQnfields(result); i++)
> {
> - if (Tcl_SetVar2(interp, argv[4], PQfname(result, i),
> + Tcl_ExternalToUtfDString(NULL,
> #ifdef TCL_ARRAYS
> - tcl_value(PQgetvalue(result, tupno, i)),
> + tcl_value(PQgetvalue(result,
> + tupno, i)),
> #else
> - PQgetvalue(result, tupno, i),
> + PQgetvalue(result, tupno, i),
> #endif
> - TCL_LEAVE_ERR_MSG) == NULL)
> - return TCL_ERROR;
> + -1, &retString);
> +
> + if (Tcl_SetVar2(interp, argv[4], PQfname(result, i),
> + Tcl_DStringValue(&retString),
> + TCL_LEAVE_ERR_MSG) == NULL)
> + {
> + Tcl_DStringFree(&retString);
> + return TCL_ERROR;
> + }
> }
> + Tcl_DStringFree(&retString);
> return TCL_OK;
> }
> else if (strcmp(opt, "-attributes") == 0)
> {
> for (i = 0; i < PQnfields(result); i++)
> - Tcl_AppendElement(interp, PQfname(result, i));
> + {
> + Tcl_ExternalToUtfDString(NULL, PQfname(result, i),
> + -1, &retString);
> + Tcl_AppendElement(interp, Tcl_DStringValue(&retString));
> + Tcl_DStringFree(&retString);
> + }
> return TCL_OK;
> }
> else if (strcmp(opt, "-lAttributes") == 0)
> @@ -1274,6 +1305,8 @@
> column,
> ncols;
> Tcl_DString headers;
> + Tcl_DString retString;
> + Tcl_DString putString;
> char buffer[2048];
> struct info_s
> {
> @@ -1292,7 +1325,11 @@
> if (conn == (PGconn *) NULL)
> return TCL_ERROR;
>
> - if ((result = PQexec(conn, argv[2])) == 0)
> + Tcl_UtfToExternalDString(NULL, argv[2], -1, &putString);
> + result = PQexec(conn, Tcl_DStringValue(&putString));
> + Tcl_DStringFree(&putString);
> +
> + if (result == 0)
> {
> /* error occurred sending the query */
> Tcl_SetResult(interp, PQerrorMessage(conn), TCL_VOLATILE);
> @@ -1340,13 +1377,21 @@
> Tcl_SetVar2(interp, argv[3], ".tupno", buffer, 0);
>
> for (column = 0; column < ncols; column++)
> - Tcl_SetVar2(interp, argv[3], info[column].cname,
> + {
> + Tcl_ExternalToUtfDString(NULL,
> #ifdef TCL_ARRAYS
> - tcl_value(PQgetvalue(result, tupno, column)),
> + tcl_value(PQgetvalue(result,
> + tupno,
> + column)),
> #else
> - PQgetvalue(result, tupno, column),
> + PQgetvalue(result, tupno, column),
> #endif
> - 0);
> + -1, &retString);
> +
> + Tcl_SetVar2(interp, argv[3], info[column].cname,
> + Tcl_DStringValue(&retString), 0);
> + Tcl_DStringFree(&retString);
> + }
>
> Tcl_SetVar2(interp, argv[3], ".command", "update", 0);
>
>
>
> No file was uploaded with this report
>
>
> ---------------------------(end of broadcast)---------------------------
> TIP 4: Don't 'kill -9' the postmaster
>
On Sun, Jul 22, 2001 at 08:10:32PM +0900, Tatsuo Ishii wrote: > Hum. Why don't you enable --enable-multibyte and > --enable-unicode-conversion and set client_encoding to UNICODE? That > would do a conversion from/to UTF-8 for Tcl 8.x (x > 9) clients? You're right. Probably, this way correct enough too :-) Thank you for suggest. But, I think, patching the libpgtcl has not to be superfluous. -- Eugene Faukin SOLVO Ltd. Company
> On Wed, 18 Jul 2001, Tom Lane wrote: > > > Reinhard Max <max@suse.de> writes: > > > On Wed, 18 Jul 2001, Bruce Momjian wrote: > > >> Do you have any idea how this will work with earlier TCL versions? > > > > > It won't. If pgtcl is supposed to still be able to compile with older > > > versions of Tcl, the changes have to be made a compile time option. > > > > Please do that and resubmit the patch. > > Here it is, but I consider it still incomplete and I have not done > exhaustive testing. Some more occurrences of PQexec and PQgetvalue > need to be wrapped up with UTF8 conversion, but I'll not have the time > to do it for the next 1-2 weeks. > > cu > Reinhard I have a patch here that handles all the TCL/UTF issues. Would you let me know if it is OK? -- Bruce Momjian | http://candle.pha.pa.us pgman@candle.pha.pa.us | (610) 853-3000 + If your life is a hard drive, | 830 Blythe Avenue + Christ can be your backup. | Drexel Hill, Pennsylvania 19026 Index: configure.in =================================================================== RCS file: /junk/pgsql/repo/pgsql/configure.in,v retrieving revision 1.132 diff -u -r1.132 configure.in --- configure.in 2001/08/01 23:52:50 1.132 +++ configure.in 2001/08/23 15:18:30 @@ -411,6 +411,21 @@ # +# If Tcl is enabled (above) then check for pltcl_utf +# +AC_MSG_CHECKING([whether to build with PL/Tcl with UTF support]) +if test "$with_tcl" = yes; then + PGAC_ARG_BOOL(enable, pltcl-utf, no, + [ --enable-pltcl-utf build PL/Tcl UTF support (if Tcl is enabled)], + [AC_DEFINE([ENABLE_PLTCL_UTF])]) +else + enable_pltcl_utf=no +fi +AC_MSG_RESULT([$enable_pltcl_utf]) +AC_SUBST([enable_pltcl_utf]) + + +# # Optionally build Perl modules (Pg.pm and PL/Perl) # AC_MSG_CHECKING([whether to build Perl modules]) Index: doc/src/sgml/installation.sgml =================================================================== RCS file: /junk/pgsql/repo/pgsql/doc/src/sgml/installation.sgml,v retrieving revision 1.50 diff -u -r1.50 installation.sgml --- doc/src/sgml/installation.sgml 2001/06/02 18:25:16 1.50 +++ doc/src/sgml/installation.sgml 2001/08/24 12:39:53 @@ -674,6 +674,17 @@ </varlistentry> <varlistentry> + <term>--enable-pltcl-utf</term> + <listitem> + <para> + Enables enables PL/Tcl Tcl_UtfToExternal and Tcl_ExternalToUtf + conversion support. These functions needed for Tcl versions 8.1 + and above for proper handling of 8-bit characters. + </para> + </listitem> + </varlistentry> + + <varlistentry> <term>--enable-odbc</term> <listitem> <para> Index: src/include/config.h.in =================================================================== RCS file: /junk/pgsql/repo/pgsql/src/include/config.h.in,v retrieving revision 1.170 diff -u -r1.170 config.h.in --- src/include/config.h.in 2001/08/01 23:52:50 1.170 +++ src/include/config.h.in 2001/08/23 15:01:41 @@ -84,6 +84,9 @@ /* --enable-pltcl-unknown */ #undef ENABLE_PLTCL_UNKNOWN +/* --enable-pltcl-utf */ +#undef ENABLE_PLTCL_UTF + /* --enable-nls */ #undef ENABLE_NLS Index: src/pl/tcl/pltcl.c =================================================================== RCS file: /junk/pgsql/repo/pgsql/src/pl/tcl/pltcl.c,v retrieving revision 1.38 diff -u -r1.38 pltcl.c --- src/pl/tcl/pltcl.c 2001/08/02 15:45:55 1.38 +++ src/pl/tcl/pltcl.c 2001/08/24 12:43:06 @@ -59,6 +59,18 @@ #include "catalog/pg_language.h" #include "catalog/pg_type.h" +#if defined(ENABLE_PLTCL_UTF) && TCL_MAJOR_VERSION == 8 \ + && TCL_MINOR_VERSION > 0 +# define UTF_BEGIN do { Tcl_DString _pltcl_ds_tmp +# define UTF_END Tcl_DStringFree(&_pltcl_ds_tmp); } while (0) +# define UTF_U2E(x) (Tcl_UtfToExternalDString(NULL,(x),-1,&_pltcl_ds_tmp)) +# define UTF_E2U(x) (Tcl_ExternalToUtfDString(NULL,(x),-1,&_pltcl_ds_tmp)) +#else /* ENABLE_PLTCL_UTF */ +# define UTF_BEGIN +# define UTF_END +# define UTF_U2E(x) (x) +# define UTF_E2U(x) (x) +#endif /* ENABLE_PLTCL_UTF */ /********************************************************************** * The information we cache about loaded procedures @@ -333,7 +345,9 @@ SPI_tuptable->tupdesc, fno); if (part != NULL) { - Tcl_DStringAppend(&unknown_src, part, -1); + UTF_BEGIN; + Tcl_DStringAppend(&unknown_src, UTF_E2U(part), -1); + UTF_END; pfree(part); } } @@ -613,7 +627,9 @@ } proc_source = DatumGetCString(DirectFunctionCall1(textout, PointerGetDatum(&procStruct->prosrc))); - Tcl_DStringAppend(&proc_internal_body, proc_source, -1); + UTF_BEGIN; + Tcl_DStringAppend(&proc_internal_body, UTF_E2U(proc_source), -1); + UTF_END; pfree(proc_source); Tcl_DStringAppendElement(&proc_internal_def, Tcl_DStringValue(&proc_internal_body)); @@ -715,7 +731,9 @@ fcinfo->arg[i], ObjectIdGetDatum(prodesc->arg_out_elem[i]), Int32GetDatum(prodesc->arg_out_len[i]))); - Tcl_DStringAppendElement(&tcl_cmd, tmp); + UTF_BEGIN; + Tcl_DStringAppendElement(&tcl_cmd, UTF_E2U(tmp)); + UTF_END; pfree(tmp); } } @@ -777,13 +795,15 @@ if (SPI_finish() != SPI_OK_FINISH) elog(ERROR, "pltcl: SPI_finish() failed"); + UTF_BEGIN; if (fcinfo->isnull) retval = (Datum) 0; else retval = FunctionCall3(&prodesc->result_in_func, - PointerGetDatum(interp->result), + PointerGetDatum(UTF_U2E(interp->result)), ObjectIdGetDatum(prodesc->result_in_elem), Int32GetDatum(-1)); + UTF_END; /************************************************************ * Finally we may restore normal error handling. @@ -929,7 +949,9 @@ proc_source = DatumGetCString(DirectFunctionCall1(textout, PointerGetDatum(&procStruct->prosrc))); - Tcl_DStringAppend(&proc_internal_body, proc_source, -1); + UTF_BEGIN; + Tcl_DStringAppend(&proc_internal_body, UTF_E2U(proc_source), -1); + UTF_END; pfree(proc_source); Tcl_DStringAppendElement(&proc_internal_def, Tcl_DStringValue(&proc_internal_body)); @@ -1230,11 +1252,13 @@ ************************************************************/ modnulls[attnum - 1] = ' '; fmgr_info(typinput, &finfo); + UTF_BEGIN; modvalues[attnum - 1] = FunctionCall3(&finfo, - CStringGetDatum(ret_values[i++]), + CStringGetDatum(UTF_U2E(ret_values[i++])), ObjectIdGetDatum(typelem), Int32GetDatum(tupdesc->attrs[attnum - 1]->atttypmod)); + UTF_END; } rettup = SPI_modifytuple(trigdata->tg_relation, rettup, tupdesc->natts, @@ -1558,7 +1582,9 @@ /************************************************************ * Execute the query and handle return codes ************************************************************/ - spi_rc = SPI_exec(argv[query_idx], count); + UTF_BEGIN; + spi_rc = SPI_exec(UTF_U2E(argv[query_idx]), count); + UTF_END; memcpy(&Warn_restart, &save_restart, sizeof(Warn_restart)); switch (spi_rc) @@ -1794,7 +1820,9 @@ /************************************************************ * Prepare the plan and check for errors ************************************************************/ - plan = SPI_prepare(argv[1], nargs, qdesc->argtypes); + UTF_BEGIN; + plan = SPI_prepare(UTF_U2E(argv[1]), nargs, qdesc->argtypes); + UTF_END; if (plan == NULL) { @@ -2078,11 +2106,13 @@ ************************************************************/ for (j = 0; j < callnargs; j++) { + UTF_BEGIN; qdesc->argvalues[j] = FunctionCall3(&qdesc->arginfuncs[j], - CStringGetDatum(callargs[j]), + CStringGetDatum(UTF_U2E(callargs[j])), ObjectIdGetDatum(qdesc->argtypelems[j]), Int32GetDatum(qdesc->arglen[j])); + UTF_END; } /************************************************************ @@ -2377,7 +2407,9 @@ attr, ObjectIdGetDatum(typelem), Int32GetDatum(tupdesc->attrs[i]->attlen))); - Tcl_SetVar2(interp, *arrptr, *nameptr, outputstr, 0); + UTF_BEGIN; + Tcl_SetVar2(interp, *arrptr, *nameptr, UTF_E2U(outputstr), 0); + UTF_END; pfree(outputstr); } else @@ -2448,7 +2480,9 @@ ObjectIdGetDatum(typelem), Int32GetDatum(tupdesc->attrs[i]->attlen))); Tcl_DStringAppendElement(retval, attname); - Tcl_DStringAppendElement(retval, outputstr); + UTF_BEGIN; + Tcl_DStringAppendElement(retval, UTF_E2U(outputstr)); + UTF_END; pfree(outputstr); } }
Hi Bruce,
On Wed, 5 Sep 2001, Bruce Momjian wrote:
> I have a patch here that handles all the TCL/UTF issues.
> Would you let me know if it is OK?
I think, it isn't really a clean fix. It only works, if your
database's encoding and Tcl's system encoding are identical. If the
database uses a different encoding than Tcl, you still end up with
wrong characters. Also, the configure switch (if needed at all)
should IMHO be a disable switch, because the conversion is mandatory
for Tcl >= 8.1 unless someone really knows that he won't have any
8-Bit characters in his database. So less people would get bitten, if
UTF conversion was enabled by default for Tcl >= 8.1 .
Besides these flaws, I think the patch could be simpler and avoid the
UTF_BEGIN and UTF_END macros if UTF_U2E and UTF_E2U were (maybe
inlined) functions and defined like this (untested):
char* UTF_U2E(CONST char * source)
{
static Tcl_DString *destPtr = NULL;
if (destPtr == NULL) {
destPtr = (Tcl_DString *) malloc(sizeof(Tcl_DString));
} else {
Tcl_DStringFree(destPtr);
}
return Tcl_UtfToExternalDString(NULL, source, -1, destPtr);
}
See also the mail, I sent to pgsql-patches last Tuesday on the same
topic.
In addition to my suggestion there to require the database to be
UNICODE for Tcl >= 8.1, I just had another Idea how it could be solved
cleanly:
What about making --enable-unicode-conversion mandatory when
PostgreSQL gets compiled with Tcl support and changing PgTcl and
PL/Tcl to set their client encoding to UNICODE at _runtime_ when they
find themselfes running with a Tcl interpreter that needs UTF-8 (i.e.
Tcl >= 8.1)?
Going this way, we could even retain binary compatibility for PgTcl
and PL/Tcl with Tcl versions prior and after Tcl's move to UTF-8.
One Question remains here: Do --enable-multibyte and
--enable-unicode-conversion have any downsides (besides a larger
executable), if they are compiled in, but not used?
cu
Reinhard