*** contrib/tsearch.orig/./expected/tsearch.out Tue Aug 13 18:43:20 2002 --- contrib/tsearch/./expected/tsearch.out Tue Aug 13 20:00:56 2002 *************** *** 689,697 **** select txt2txtidx('345 qwe@efd.r \' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf qwer jf sdjk ewr1> ewri2 /usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234 wow < jqw <> qwerty'); ! txt2txtidx ! --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ! 'ad' 'dw' 'jf' '234' '345' '4.2' '455' 'jqw' 'qwe' 'wer' 'wow' 'asdf' 'ewr1' 'qwer' 'sdjk' '5.005' 'ewri2' 'qwqwe' 'wefjn' 'gist.c' 'gist.h' 'qwerti' '234.435' ':8100/?' 'qwe-wer' 'readlin' 'www.com' '+4.0e-10' 'gist.h.c' 'rewt/ewr' 'qwe@efd.r' '/?ad=qwe&dw' '/wqe-324/ewr' 'aew.werc.ewr' '1aew.werc.ewr' '2aew.werc.ewr' '3aew.werc.ewr' '4aew.werc.ewr' '5aew.werc.ewr' '6aew.werc.ewr' '7aew.werc.ewr' '/usr/local/fff' '/awdf/dwqe/4325' ':8100/?ad=qwe&dw' 'teodor@stack.net' '5aew.werc.ewr:8100/?' ':8100/?ad=qwe&dw=%20%32' 'aew.werc.ewr/?ad=qwe&dw' '1aew.werc.ewr/?ad=qwe&dw' '3aew.werc.ewr/?ad=qwe&dw' '6aew.werc.ewr:8100/?ad=qwe&dw' '7aew.werc.ewr:8100/?ad=qwe&dw=%20%32' (1 row) select txtidxsize(txt2txtidx('345 qw')); --- 689,697 ---- select txt2txtidx('345 qwe@efd.r \' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf qwer jf sdjk ewr1> ewri2 /usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234 wow < jqw <> qwerty'); ! txt2txtidx ! ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ! 'ad' 'dw' 'jf' '234' '345' '4.2' '455' 'jqw' 'qwe' 'wer' 'wow' 'asdf' 'ewr1' 'qwer' 'sdjk' '5.005' 'ewri2' 'qwqwe' 'wefjn' 'gist.c' 'gist.h' 'qwerti' '234.435' ':8100/?' 'qwe-wer' 'readlin' 'www.com' '+4.0e-10' 'gist.h.c' 'rewt/ewr' 'qwe@efd.r' 'readline-4' '/?ad=qwe&dw' '/wqe-324/ewr' 'aew.werc.ewr' '1aew.werc.ewr' '2aew.werc.ewr' '3aew.werc.ewr' '4aew.werc.ewr' '5aew.werc.ewr' '6aew.werc.ewr' '7aew.werc.ewr' '/usr/local/fff' '/awdf/dwqe/4325' ':8100/?ad=qwe&dw' 'teodor@stack.net' '5aew.werc.ewr:8100/?' ':8100/?ad=qwe&dw=%20%32' 'aew.werc.ewr/?ad=qwe&dw' '1aew.werc.ewr/?ad=qwe&dw' '3aew.werc.ewr/?ad=qwe&dw' '6aew.werc.ewr:8100/?ad=qwe&dw' '7aew.werc.ewr:8100/?ad=qwe&dw=%20%32' (1 row) select txtidxsize(txt2txtidx('345 qw')); *************** *** 705,711 **** wow < jqw <> qwerty')); txtidxsize ------------ ! 52 (1 row) insert into test_txtidx (a) values ('345 qwerty'); --- 705,711 ---- wow < jqw <> qwerty')); txtidxsize ------------ ! 53 (1 row) insert into test_txtidx (a) values ('345 qwerty'); *** contrib/tsearch.orig/./morph.c Tue Aug 13 18:43:20 2002 --- contrib/tsearch/./morph.c Tue Aug 13 20:00:56 2002 *************** *** 75,93 **** {NODICT, NODICT}, /* EMAIL */ {NODICT, NODICT}, /* FURL */ {NODICT, NODICT}, /* HOST */ ! {NODICT, NODICT}, /* FLOAT */ ! {NODICT, NODICT}, /* FINT */ ! {BYLOCALE, DEFAULTDICT}, /* PARTWORD */ ! {BYLOCALE, NODICT}, /* NONLATINPARTWORD */ ! {DEFAULTDICT, NODICT}, /* LATPARTWORD */ {STOPLEXEM, NODICT}, /* SPACE */ ! {STOPLEXEM, NODICT}, /* SYMTAG */ {STOPLEXEM, NODICT}, /* HTTP */ ! {BYLOCALE, DEFAULTDICT}, /* DEFISWORD */ ! {DEFAULTDICT, NODICT}, /* DEFISLATWORD */ ! {BYLOCALE, NODICT}, /* DEFISNONLATINWORD */ {NODICT, NODICT}, /* URI */ ! {NODICT, NODICT} /* FILEPATH */ }; static bool inited = false; --- 75,97 ---- {NODICT, NODICT}, /* EMAIL */ {NODICT, NODICT}, /* FURL */ {NODICT, NODICT}, /* HOST */ ! {NODICT, NODICT}, /* SCIENTIFIC */ ! {NODICT, NODICT}, /* VERSIONNUMBER */ ! {BYLOCALE, DEFAULTDICT}, /* PARTHYPHENWORD */ ! {BYLOCALE, NODICT}, /* CYRPARTHYPHENWORD */ ! {DEFAULTDICT, NODICT}, /* LATPARTHYPHENWORD */ {STOPLEXEM, NODICT}, /* SPACE */ ! {STOPLEXEM, NODICT}, /* TAG */ {STOPLEXEM, NODICT}, /* HTTP */ ! {BYLOCALE, DEFAULTDICT}, /* HYPHENWORD */ ! {DEFAULTDICT, NODICT}, /* LATHYPHENWORD */ ! {BYLOCALE, NODICT}, /* CYRHYPHENWORD */ {NODICT, NODICT}, /* URI */ ! {NODICT, NODICT}, /* FILEPATH */ ! {NODICT, NODICT}, /* DECIMAL */ ! {NODICT, NODICT}, /* SIGNEDINT */ ! {NODICT, NODICT}, /* UNSIGNEDINT */ ! {STOPLEXEM, NODICT} /* HTMLENTITY */ }; static bool inited = false; *** contrib/tsearch.orig/./parser.l Tue Aug 13 18:43:20 2002 --- contrib/tsearch/./parser.l Tue Aug 13 20:00:56 2002 *************** *** 5,22 **** /* postgres allocation function */ #include "postgres.h" ! #define free pfree ! #define malloc palloc #define realloc repalloc #ifdef strdup #undef strdup #endif ! #define strdup pstrdup ! char *token = NULL; /* pointer to token */ ! char *s = NULL; /* for returning full defis-word */ YY_BUFFER_STATE buf = NULL; /* buffer to parse; it need for parse from string */ --- 5,21 ---- /* postgres allocation function */ #include "postgres.h" ! #define free pfree ! #define malloc palloc #define realloc repalloc #ifdef strdup #undef strdup #endif ! #define strdup pstrdup char *token = NULL; /* pointer to token */ ! char *s = NULL; /* to return WHOLE hyphenated-word */ YY_BUFFER_STATE buf = NULL; /* buffer to parse; it need for parse from string */ *************** *** 57,77 **** %option nounput %option noyywrap ! ! /* parser's state for parsing defis-word */ %x DELIM /* parser's state for parsing URL*/ %x URL %x SERVER ! /* parser's state for parsing filepath */ ! %x INTAG %x QINTAG ! /* NONLATIN char */ ! NONLATINALNUM [0-9\200-\377] ! NONLATINALPHA [\200-\377] ALPHA [a-zA-Z\200-\377] ALNUM [0-9a-zA-Z\200-\377] --- 56,76 ---- %option nounput %option noyywrap ! /* parser's state for parsing hyphenated-word */ %x DELIM /* parser's state for parsing URL*/ %x URL %x SERVER ! /* parser's state for parsing TAGS */ %x INTAG %x QINTAG + %x INCOMMENT + %x INSCRIPT ! /* cyrillic koi8 char */ ! CYRALNUM [0-9\200-\377] ! CYRALPHA [\200-\377] ALPHA [a-zA-Z\200-\377] ALNUM [0-9a-zA-Z\200-\377] *************** *** 81,146 **** %% ! "<"[[:alpha:]] { BEGIN INTAG; ! token = tsearch_yytext; ! tokenlen = tsearch_yyleng; ! return SYMTAG; ! } ! ! "" { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return SYMTAG; } ! "<"[^>[:alpha:]] { token = tsearch_yytext; tokenlen = tsearch_yyleng; return SPACE; } - "\"" { BEGIN QINTAG; - token = tsearch_yytext; - tokenlen = tsearch_yyleng; - return SYMTAG; - } ! "\\\"" { ! token = tsearch_yytext; ! tokenlen = tsearch_yyleng; ! return SYMTAG; ! } ! "\"" { BEGIN INTAG; ! token = tsearch_yytext; ! tokenlen = tsearch_yyleng; ! return SYMTAG; ! } ! .|\n { token = tsearch_yytext; ! tokenlen = tsearch_yyleng; ! return SYMTAG; } ! ">" { BEGIN INITIAL; token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return SYMTAG; ! } ! .|\n { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return SYMTAG; } - [-_\.[:alnum:]]+@{HOSTNAME} /* Emails */ { token = tsearch_yytext; --- 80,138 ---- %% ! "<"[Ss][Cc][Rr][Ii][Pp][Tt] { BEGIN INSCRIPT; } ! "" { ! BEGIN INITIAL; ! *tsearch_yytext=' '; *(tsearch_yytext+1) = '\0'; token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return SPACE; } ! "" { ! BEGIN INITIAL; ! *tsearch_yytext=' '; *(tsearch_yytext+1) = '\0'; token = tsearch_yytext; tokenlen = tsearch_yyleng; return SPACE; } ! "<"[\![:alpha:]] { BEGIN INTAG; } ! ""\"" { BEGIN QINTAG; } ! ! "\\\"" ; ! ! "\"" { BEGIN INTAG; } ! ! ">" { ! BEGIN INITIAL; token = tsearch_yytext; ! *tsearch_yytext=' '; ! token = tsearch_yytext; ! tokenlen = 1; ! return TAG; } ! .|\n ; ! ! \&(quot|amp|nbsp|lt|gt)\; { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return HTMLENTITY; ! } ! \&\#[0-9][0-9]?[0-9]?\; { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return HTMLENTITY; } [-_\.[:alnum:]]+@{HOSTNAME} /* Emails */ { token = tsearch_yytext; *************** *** 148,169 **** return EMAIL; } ! [0-9] /* digit's and point (might be a version) */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return FINT; } ! [0-9]+[0-9\.]*[0-9] /* digit's and point (might be a version) */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return FINT; } ! [+-]?[0-9\.]+[eE][+-]?[0-9]+ /* float */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return FLOAT; } http"://" { --- 140,173 ---- return EMAIL; } ! [+-]?[0-9]+(\.[0-9]+)?[eEdD][+-]?[0-9]+ /* float */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return SCIENTIFIC; ! } ! ! [0-9]+\.[0-9]+\.[0-9\.]*[0-9] { ! token = tsearch_yytext; ! tokenlen = tsearch_yyleng; ! return VERSIONNUMBER; ! } ! ! [+-]?[0-9]+\.[0-9]+ { ! token = tsearch_yytext; ! tokenlen = tsearch_yyleng; ! return DECIMAL; } ! [+-][0-9]+ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return SIGNEDINT; } ! [0-9]+ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return UNSIGNEDINT; } http"://" { *************** *** 208,259 **** return FILEPATH; } ! ({NONLATINALNUM}+-)+{NONLATINALPHA}+ /* composite-word */ { BEGIN DELIM; if (s) { free(s); s=NULL; } s = strdup( tsearch_yytext ); tokenlen = tsearch_yyleng; yyless( 0 ); token = s; ! return DEFISNONLATINWORD; } ! ([[:alnum:]]+-)+[[:alpha:]]+ /* composite-word */ { BEGIN DELIM; if (s) { free(s); s=NULL; } - tokenlen = tsearch_yyleng; s = strdup( tsearch_yytext ); yyless( 0 ); token = s; ! return DEFISLATWORD; } ! ({ALNUM}+-)+{ALPHA}+ /* composite-word */ { BEGIN DELIM; if (s) { free(s); s=NULL; } s = strdup( tsearch_yytext ); tokenlen = tsearch_yyleng; yyless( 0 ); token = s; ! return DEFISWORD; } ! {NONLATINALNUM}+ /* one word in composite-word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return NONLATINPARTWORD; } ! [[:alnum:]]+ /* one word in composite-word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return LATPARTWORD; } {ALNUM}+ /* one word in composite-word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return PARTWORD; } - { --- 212,269 ---- return FILEPATH; } ! ({CYRALPHA}+-)+{CYRALPHA}+ /* composite-word */ { BEGIN DELIM; if (s) { free(s); s=NULL; } s = strdup( tsearch_yytext ); tokenlen = tsearch_yyleng; yyless( 0 ); token = s; ! return CYRHYPHENWORD; } ! ([[:alpha:]]+-)+[[:alpha:]]+ /* composite-word */ { BEGIN DELIM; if (s) { free(s); s=NULL; } s = strdup( tsearch_yytext ); + tokenlen = tsearch_yyleng; yyless( 0 ); token = s; ! return LATHYPHENWORD; } ! ({ALNUM}+-)+{ALNUM}+ /* composite-word */ { BEGIN DELIM; if (s) { free(s); s=NULL; } s = strdup( tsearch_yytext ); tokenlen = tsearch_yyleng; yyless( 0 ); token = s; ! return HYPHENWORD; ! } ! ! \+?[0-9]+\.[0-9]+ { ! token = tsearch_yytext; ! tokenlen = tsearch_yyleng; ! return DECIMAL; } ! {CYRALPHA}+ /* one word in composite-word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return CYRPARTHYPHENWORD; } ! [[:alpha:]]+ /* one word in composite-word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return LATPARTHYPHENWORD; } {ALNUM}+ /* one word in composite-word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return PARTHYPHENWORD; } - { *************** *** 264,280 **** .|\n /* return in basic state */ { BEGIN INITIAL; - tokenlen = tsearch_yyleng; yyless( 0 ); } ! {NONLATINALNUM}+ /* normal word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return NONLATINWORD; } ! [[:alnum:]]+ /* normal word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; return LATWORD; --- 274,289 ---- .|\n /* return in basic state */ { BEGIN INITIAL; yyless( 0 ); } ! {CYRALPHA}+ /* normal word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; ! return CYRWORD; } ! [[:alpha:]]+ /* normal word */ { token = tsearch_yytext; tokenlen = tsearch_yyleng; return LATWORD; *************** *** 286,292 **** return UWORD; } ! .|\n { token = tsearch_yytext; tokenlen = tsearch_yyleng; return SPACE; --- 295,307 ---- return UWORD; } ! [ \r\n\t]+ { ! token = tsearch_yytext; ! tokenlen = tsearch_yyleng; ! return SPACE; ! } ! ! . { token = tsearch_yytext; tokenlen = tsearch_yyleng; return SPACE; *** contrib/tsearch.orig/./README.tsearch Tue Aug 13 18:43:20 2002 --- contrib/tsearch/./README.tsearch Tue Aug 13 20:00:56 2002 *************** *** 4,9 **** --- 4,14 ---- All work was done by Teodor Sigaev (teodor@stack.net) and Oleg Bartunov (oleg@sai.msu.su). + CHANGES: + + August 13, 2002 + Use parser of OpenFTS v0.33. + IMPORTANT NOTICE: This is a first step of our work on integration of OpenFTS *** contrib/tsearch.orig/./deflex.h Tue Aug 13 18:43:20 2002 --- contrib/tsearch/./deflex.h Tue Aug 13 20:00:56 2002 *************** *** 2,29 **** #define __DEFLEX_H__ /* rememder !!!! */ ! #define LASTNUM 19 #define LATWORD 1 ! #define NONLATINWORD 2 #define UWORD 3 #define EMAIL 4 #define FURL 5 #define HOST 6 ! #define FLOAT 7 ! #define FINT 8 ! #define PARTWORD 9 ! #define NONLATINPARTWORD 10 ! #define LATPARTWORD 11 ! #define SPACE 12 ! #define SYMTAG 13 ! #define HTTP 14 ! #define DEFISWORD 15 ! #define DEFISLATWORD 16 ! #define DEFISNONLATINWORD 17 #define URI 18 #define FILEPATH 19 extern const char *descr[]; #endif --- 2,34 ---- #define __DEFLEX_H__ /* rememder !!!! */ ! #define LASTNUM 23 #define LATWORD 1 ! #define CYRWORD 2 #define UWORD 3 #define EMAIL 4 #define FURL 5 #define HOST 6 ! #define SCIENTIFIC 7 ! #define VERSIONNUMBER 8 ! #define PARTHYPHENWORD 9 ! #define CYRPARTHYPHENWORD 10 ! #define LATPARTHYPHENWORD 11 ! #define SPACE 12 ! #define TAG 13 ! #define HTTP 14 ! #define HYPHENWORD 15 ! #define LATHYPHENWORD 16 ! #define CYRHYPHENWORD 17 #define URI 18 #define FILEPATH 19 + #define DECIMAL 20 + #define SIGNEDINT 21 + #define UNSIGNEDINT 22 + #define HTMLENTITY 23 extern const char *descr[]; #endif +