X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=src%2Fcharsets.c;h=b2470bd9be7a5bfaf7ed65abfca9788cdf9e230d;hb=67ebd94d4bd94b8ade005dfa371638e0855f84d1;hp=3f45baa3286bbdef91a7ef9ba39f386608b19a91;hpb=27cfb6d89ca9b02f63f8334b6b8e666cf7db2ff7;p=pazpar2-moved-to-github.git diff --git a/src/charsets.c b/src/charsets.c index 3f45baa..b2470bd 100644 --- a/src/charsets.c +++ b/src/charsets.c @@ -1,7 +1,5 @@ -/* $Id: charsets.c,v 1.1 2007-05-10 11:46:09 adam Exp $ - Copyright (c) 2006-2007, Index Data. - -This file is part of Pazpar2. +/* This file is part of Pazpar2. + Copyright (C) 2006-2008 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -14,47 +12,110 @@ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with Pazpar2; see the file LICENSE. If not, write to the -Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA -02111-1307, USA. - */ +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ /** \file charsets.c \brief Pazpar2 Character set facilities */ #if HAVE_CONFIG_H -#include "cconfig.h" +#include #endif #include #include +#include #include #include + #include "charsets.h" +#include "normalize7bit.h" + +#ifdef HAVE_ICU +#include "icu_I18N.h" +#endif // HAVE_ICU /* charset handle */ struct pp2_charset_s { const char *(*token_next_handler)(pp2_relevance_token_t prt); - /* other handlers will come as we see fit */ + const char *(*get_sort_handler)(pp2_relevance_token_t prt, int skip); +#ifdef HAVE_ICU + struct icu_chain * icu_chn; + UErrorCode icu_sts; +#endif // HAVE_ICU }; static const char *pp2_relevance_token_a_to_z(pp2_relevance_token_t prt); -/* in the future : */ -// static const char *pp2_relevance_token_icu(pp2_relevance_token_t prt); +static const char *pp2_get_sort_ascii(pp2_relevance_token_t prt, int skip_article); + +#ifdef HAVE_ICU +static const char *pp2_relevance_token_icu(pp2_relevance_token_t prt); +static const char *pp2_get_sort_icu(pp2_relevance_token_t prt, int skip_article); +#endif // HAVE_ICU /* tokenzier handle */ struct pp2_relevance_token_s { const char *cp; /* unnormalized buffer we're tokenizing */ + const char *last_cp; /* pointer to last token we're dealing with */ pp2_charset_t pct; /* our main charset handle (type+config) */ WRBUF norm_str; /* normized string we return (temporarily) */ + WRBUF sort_str; /* sort string we return (temporarily) */ }; -pp2_charset_t pp2_charset_create(void) + +pp2_charset_t pp2_charset_create_xml(xmlNode *xml_node) +{ +#ifdef HAVE_ICU + UErrorCode status = U_ZERO_ERROR; + struct icu_chain *chain = 0; + while (xml_node && xml_node->type != XML_ELEMENT_NODE) + xml_node = xml_node->next; + chain = icu_chain_xml_config(xml_node, &status); + if (!chain || U_FAILURE(status)){ + //xmlDocPtr icu_doc = 0; + //xmlChar *xmlstr = 0; + //int size = 0; + //xmlDocDumpMemory(icu_doc, size); + + yaz_log(YLOG_FATAL, "Could not parse ICU chain config:\n" + "<%s>\n ... \n", + xml_node->name, xml_node->name); + return 0; + } + return pp2_charset_create(chain); +#else // HAVE_ICU + yaz_log(YLOG_FATAL, "Error: ICU support requested with element:\n" + "<%s>\n ... \n", + xml_node->name, xml_node->name); + yaz_log(YLOG_FATAL, + "But no ICU support compiled into pazpar2 server."); + yaz_log(YLOG_FATAL, + "Please install libicu36-dev and icu-doc or similar, " + "re-configure and re-compile"); + return 0; +#endif // HAVE_ICU +} + + +pp2_charset_t pp2_charset_create(struct icu_chain * icu_chn) { pp2_charset_t pct = xmalloc(sizeof(*pct)); pct->token_next_handler = pp2_relevance_token_a_to_z; + pct->get_sort_handler = pp2_get_sort_ascii; +#ifdef HAVE_ICU + pct->icu_chn = 0; + if (icu_chn) + { + pct->icu_chn = icu_chn; + pct->icu_sts = U_ZERO_ERROR; + pct->token_next_handler = pp2_relevance_token_icu; + pct->get_sort_handler = pp2_get_sort_icu; + } +#endif // HAVE_ICU return pct; } @@ -69,16 +130,34 @@ pp2_relevance_token_t pp2_relevance_tokenize(pp2_charset_t pct, pp2_relevance_token_t prt = xmalloc(sizeof(*prt)); assert(pct); + prt->norm_str = wrbuf_alloc(); + prt->sort_str = wrbuf_alloc(); prt->cp = buf; + prt->last_cp = 0; prt->pct = pct; + +#ifdef HAVE_ICU + if (pct->icu_chn) + { + int ok = 0; + pct->icu_sts = U_ZERO_ERROR; + ok = icu_chain_assign_cstr(pct->icu_chn, buf, &pct->icu_sts); + //printf("\nfield ok: %d '%s'\n", ok, buf); + prt->pct = pct; + } +#endif // HAVE_ICU return prt; } + void pp2_relevance_token_destroy(pp2_relevance_token_t prt) { assert(prt); - wrbuf_destroy(prt->norm_str); + if(prt->norm_str) + wrbuf_destroy(prt->norm_str); + if(prt->sort_str) + wrbuf_destroy(prt->sort_str); xfree(prt); } @@ -88,7 +167,12 @@ const char *pp2_relevance_token_next(pp2_relevance_token_t prt) return (prt->pct->token_next_handler)(prt); } -#define raw_char(c) (((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 1 : -1) +const char *pp2_get_sort(pp2_relevance_token_t prt, int skip) +{ + return prt->pct->get_sort_handler(prt, skip); +} + +#define raw_char(c) (((c) >= 'a' && (c) <= 'z') ? (c) : -1) /* original tokenizer with our tokenize interface, but we add +1 to ensure no '\0' are in our string (except for EOF) */ @@ -103,9 +187,12 @@ static const char *pp2_relevance_token_a_to_z(pp2_relevance_token_t prt) if (*cp == '\0') { prt->cp = cp; + prt->last_cp = 0; return 0; } /* now read the term itself */ + + prt->last_cp = cp; wrbuf_rewind(prt->norm_str); while (*cp && (c = raw_char(tolower(*cp))) >= 0) { @@ -116,6 +203,48 @@ static const char *pp2_relevance_token_a_to_z(pp2_relevance_token_t prt) return wrbuf_cstr(prt->norm_str); } +static const char *pp2_get_sort_ascii(pp2_relevance_token_t prt, + int skip_article) +{ + if (prt->last_cp == 0) + return 0; + else + { + char *tmp = xstrdup(prt->last_cp); + char *result = 0; + result = normalize7bit_mergekey(tmp, skip_article); + + wrbuf_rewind(prt->sort_str); + wrbuf_puts(prt->sort_str, result); + xfree(tmp); + return wrbuf_cstr(prt->sort_str); + } +} + + +#ifdef HAVE_ICU +static const char *pp2_relevance_token_icu(pp2_relevance_token_t prt) +{ + if (icu_chain_next_token(prt->pct->icu_chn, &prt->pct->icu_sts)) + { + if (U_FAILURE(prt->pct->icu_sts)) + { + return 0; + } + return icu_chain_get_norm(prt->pct->icu_chn); + } + return 0; +} + +static const char *pp2_get_sort_icu(pp2_relevance_token_t prt, + int skip_article) +{ + return icu_chain_get_sort(prt->pct->icu_chn); +} + +#endif // HAVE_ICU + + /* * Local variables: