X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=src%2Ficu_I18N.c;h=f40b529294dd19bd505d38c7a2d853b5418bfbba;hb=f8892da7570d4e365d36d2de128cc581f5240980;hp=cc9f3434302779ddac5a1ee3e72df32c5fba2c95;hpb=6f08d24c048052306511961bb9710694f07b5528;p=pazpar2-moved-to-github.git diff --git a/src/icu_I18N.c b/src/icu_I18N.c index cc9f343..f40b529 100644 --- a/src/icu_I18N.c +++ b/src/icu_I18N.c @@ -1,26 +1,24 @@ -/* $Id: icu_I18N.c,v 1.13 2007-05-15 15:11:42 marc Exp $ - Copyright (c) 2006-2007, Index Data. +/* This file is part of Pazpar2. + Copyright (C) 2006-2008 Index Data - This file is part of Pazpar2. +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. - Pazpar2 is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2, or (at your option) any later - version. +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. - Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - You should have received a copy of the GNU General Public License - along with Pazpar2; see the file LICENSE. If not, write to the - Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. */ #if HAVE_CONFIG_H -#include "cconfig.h" +#include #endif #define USE_TIMING 0 @@ -83,7 +81,6 @@ struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity) return buf16; }; - struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16, size_t capacity) { @@ -122,6 +119,7 @@ struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16, icu_buf_utf16_resize(dest16, src16->utf16_len * 2); u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len); + dest16->utf16_len = src16->utf16_len; return dest16; }; @@ -171,8 +169,6 @@ struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8, buf8->utf8 = (uint8_t *) realloc(buf8->utf8, sizeof(uint8_t) * capacity); - buf8->utf8[0] = (uint8_t) 0; - buf8->utf8_len = 0; buf8->utf8_cap = capacity; } else { @@ -205,6 +201,16 @@ struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8, }; +const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8) +{ + if (!src8 || src8->utf8_len == 0) + return ""; + if (src8->utf8_len == src8->utf8_cap) + src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1); + src8->utf8[src8->utf8_len] = '\0'; + return (const char *) src8->utf8; +} + void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8) { @@ -240,7 +246,7 @@ UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16, //if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) - && utf16_len < dest16->utf16_cap) + && utf16_len <= dest16->utf16_cap) dest16->utf16_len = utf16_len; else { dest16->utf16[0] = (UChar) 0; @@ -278,7 +284,7 @@ UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16, // if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) - && utf16_len < dest16->utf16_cap) + && utf16_len <= dest16->utf16_cap) dest16->utf16_len = utf16_len; else { dest16->utf16[0] = (UChar) 0; @@ -315,7 +321,7 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, //if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) - && utf8_len < dest8->utf8_cap) + && utf8_len <= dest8->utf8_cap) dest8->utf8_len = utf8_len; else { dest8->utf8[0] = (uint8_t) 0; @@ -327,6 +333,51 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, +struct icu_casemap * icu_casemap_create(const char *locale, char action, + UErrorCode *status) +{ + struct icu_casemap * casemap + = (struct icu_casemap *) malloc(sizeof(struct icu_casemap)); + strcpy(casemap->locale, locale); + casemap->action = action; + + switch(casemap->action) { + case 'l': + break; + case 'u': + break; + case 't': + break; + case 'f': + break; + default: + icu_casemap_destroy(casemap); + return 0; + } + + return casemap; +}; + +void icu_casemap_destroy(struct icu_casemap * casemap) +{ + if (casemap) + free(casemap); +}; + + +int icu_casemap_casemap(struct icu_casemap * casemap, + struct icu_buf_utf16 * dest16, + struct icu_buf_utf16 * src16, + UErrorCode *status) +{ + if(!casemap) + return 0; + + return icu_utf16_casemap(dest16, src16, + casemap->locale, casemap->action, status); +}; + + int icu_utf16_casemap(struct icu_buf_utf16 * dest16, struct icu_buf_utf16 * src16, const char *locale, char action, @@ -399,7 +450,7 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, } if (U_SUCCESS(*status) - && dest16_len < dest16->utf16_cap) + && dest16_len <= dest16->utf16_cap) dest16->utf16_len = dest16_len; else { dest16->utf16[0] = (UChar) 0; @@ -621,15 +672,6 @@ int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer) -//struct icu_normalizer -//{ -// char action; -// struct icu_buf_utf16 * rules16; -// UParseError parse_error[256]; -// UTransliterator * trans; -//}; - - struct icu_normalizer * icu_normalizer_create(const char *rules, char action, UErrorCode *status) { @@ -650,6 +692,7 @@ struct icu_normalizer * icu_normalizer_create(const char *rules, char action, UTRANS_FORWARD, 0, 0, normalizer->parse_error, status); + // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans); break; case 'r': normalizer->trans @@ -658,6 +701,7 @@ struct icu_normalizer * icu_normalizer_create(const char *rules, char action, UTRANS_REVERSE , 0, 0, normalizer->parse_error, status); + // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans); break; default: *status = U_UNSUPPORTED_ERROR; @@ -679,7 +723,10 @@ void icu_normalizer_destroy(struct icu_normalizer * normalizer){ if (normalizer->rules16) icu_buf_utf16_destroy(normalizer->rules16); if (normalizer->trans) + { + // yaz_log(YLOG_LOG, "utrans_close %p", normalizer->trans); utrans_close(normalizer->trans); + } free(normalizer); } }; @@ -716,7 +763,7 @@ int icu_normalizer_normalize(struct icu_normalizer * normalizer, struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, enum icu_chain_step_type type, const uint8_t * rule, - struct icu_buf_utf16 * src16, + struct icu_buf_utf16 * buf16, UErrorCode *status) { struct icu_chain_step * step = 0; @@ -726,15 +773,21 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, step = (struct icu_chain_step *) malloc(sizeof(struct icu_chain_step)); + step->type = type; + + step->buf16 = buf16; + // create auxilary objects switch(step->type) { case ICU_chain_step_type_display: break; - case ICU_chain_step_type_norm: + case ICU_chain_step_type_index: break; - case ICU_chain_step_type_sort: + case ICU_chain_step_type_sortkey: break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + step->u.casemap = icu_casemap_create((char *) chain->locale, + (char) rule[0], status); break; case ICU_chain_step_type_normalize: step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status); @@ -747,10 +800,6 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, break; } - if (src16) - step->src16 = src16; - - return step; }; @@ -759,35 +808,32 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ if (!step) return; - - if (step->previous) - icu_chain_step_destroy(step->previous); - if (step->src16) - icu_buf_utf16_destroy(step->src16); - - // destroy last living icu_chain_step + icu_chain_step_destroy(step->previous); switch(step->type) { case ICU_chain_step_type_display: break; - case ICU_chain_step_type_norm: + case ICU_chain_step_type_index: break; - case ICU_chain_step_type_sort: + case ICU_chain_step_type_sortkey: break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + icu_casemap_destroy(step->u.casemap); + icu_buf_utf16_destroy(step->buf16); break; case ICU_chain_step_type_normalize: icu_normalizer_destroy(step->u.normalizer); + icu_buf_utf16_destroy(step->buf16); break; case ICU_chain_step_type_tokenize: icu_tokenizer_destroy(step->u.tokenizer); + icu_buf_utf16_destroy(step->buf16); break; default: break; } - - + free(step); }; @@ -820,16 +866,103 @@ struct icu_chain * icu_chain_create(const uint8_t * identifier, void icu_chain_destroy(struct icu_chain * chain) { - icu_buf_utf8_destroy(chain->display8); - icu_buf_utf8_destroy(chain->norm8); - icu_buf_utf8_destroy(chain->sort8); + if (chain){ + icu_buf_utf8_destroy(chain->display8); + icu_buf_utf8_destroy(chain->norm8); + icu_buf_utf8_destroy(chain->sort8); + + icu_buf_utf16_destroy(chain->src16); + + icu_chain_step_destroy(chain->steps); + free(chain); + } +}; - icu_buf_utf16_destroy(chain->src16); - icu_chain_step_destroy(chain->steps); + +struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, + UErrorCode * status){ + + xmlNode *node = 0; + struct icu_chain * chain = 0; + xmlChar *xml_id = 0; + xmlChar *xml_locale = 0; + + if (!xml_node + ||xml_node->type != XML_ELEMENT_NODE + || strcmp((const char *) xml_node->name, "icu_chain")) + + return 0; + + xml_id = xmlGetProp(xml_node, (xmlChar *) "id"); + xml_locale = xmlGetProp(xml_node, (xmlChar *) "locale"); + + if (!xml_id || !strlen((const char *) xml_id) + || !xml_locale || !strlen((const char *) xml_locale)) + return 0; + + chain = icu_chain_create((const uint8_t *) xml_id, + (const uint8_t *) xml_locale); + + xmlFree(xml_id); + xmlFree(xml_locale); + if (!chain) + return 0; + + for (node = xml_node->children; node; node = node->next) + { + xmlChar *xml_rule = 0; + struct icu_chain_step * step = 0; + if (node->type != XML_ELEMENT_NODE) + continue; + + xml_rule = xmlGetProp(node, (xmlChar *) "rule"); + + if (!strcmp((const char *) node->name, + (const char *) "casemap")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, + (const uint8_t *) xml_rule, status); + } + else if (!strcmp((const char *) node->name, + (const char *) "normalize")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, + (const uint8_t *) xml_rule, status); + } + else if (!strcmp((const char *) node->name, + (const char *) "tokenize")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, + (const uint8_t *) xml_rule, status); + } + else if (!strcmp((const char *) node->name, + (const char *) "display")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_display, + (const uint8_t *) "", status); + } + else if (!strcmp((const char *) node->name, + (const char *) "index")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_index, + (const uint8_t *) "", status); + } + else if (!strcmp((const char *) node->name, + (const char *) "sortkey")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, + (const uint8_t *) "", status); + } + + xmlFree(xml_rule); + if (!step || U_FAILURE(*status)){ + icu_chain_destroy(chain); + return 0; + } + + + } + + return chain; }; + struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, enum icu_chain_step_type type, const uint8_t * rule, @@ -837,33 +970,46 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, { struct icu_chain_step * step = 0; struct icu_buf_utf16 * src16 = 0; + struct icu_buf_utf16 * buf16 = 0; if (!chain || !type || !rule) return 0; - //if(chain->steps && chain->steps->src16) + // assign utf16 src buffers as needed + if (chain->steps && chain->steps->buf16) + src16 = chain->steps->buf16; + else if (chain->src16) + src16 = chain->src16; + else + return 0; + - // assign utf16 src buffers as needed - switch(step->type) { + // create utf16 destination buffers as needed, or + switch(type) { case ICU_chain_step_type_display: + buf16 = src16; break; - case ICU_chain_step_type_norm: + case ICU_chain_step_type_index: + buf16 = src16; break; - case ICU_chain_step_type_sort: + case ICU_chain_step_type_sortkey: + buf16 = src16; break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + buf16 = icu_buf_utf16_create(0); break; case ICU_chain_step_type_normalize: + buf16 = icu_buf_utf16_create(0); break; case ICU_chain_step_type_tokenize: + buf16 = icu_buf_utf16_create(0); break; default: break; } // create actual chain step with this buffer - // leave zero for implicit buffer creation - step = icu_chain_step_create(chain, type, rule, src16, status); + step = icu_chain_step_create(chain, type, rule, buf16, status); step->previous = chain->steps; chain->steps = step; @@ -872,6 +1018,194 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, }; +int icu_chain_step_next_token(struct icu_chain * chain, + struct icu_chain_step * step, + UErrorCode *status) +{ + struct icu_buf_utf16 * src16 = 0; + + //printf("icu_chain_step_next_token %d\n", (int) step); + + if (!chain || !chain->src16 || !step || !step->more_tokens) + return 0; + + // assign utf16 src buffers as neeed, advance in previous steps + // tokens until non-zero token met, and setting stop condition + if (step->previous){ + src16 = step->previous->buf16; + if (step->need_new_token) + //while (step->more_tokens && !src16->utf16_len) + step->more_tokens + = icu_chain_step_next_token(chain, step->previous, status); + } + else { // first step can only work once on chain->src16 input buffer + src16 = chain->src16; + step->more_tokens = 1; + } + + // stop if nothing to process + // i.e new token source was not properly assigned + if (!step->more_tokens || !src16) // || !src16->utf16_len + return 0; + + //printf("icu_chain_step_next_token %d working\n", (int) step); + + + // perform the work, eventually put this steps output in + // step->buf16 or the chains UTF8 output buffers + switch(step->type) { + case ICU_chain_step_type_display: + icu_utf16_to_utf8(chain->display8, src16, status); + break; + case ICU_chain_step_type_index: + icu_utf16_to_utf8(chain->norm8, src16, status); + break; + case ICU_chain_step_type_sortkey: + icu_utf16_to_utf8(chain->sort8, src16, status); + break; + case ICU_chain_step_type_casemap: + icu_casemap_casemap(step->u.casemap, + step->buf16, src16, status); + break; + case ICU_chain_step_type_normalize: + icu_normalizer_normalize(step->u.normalizer, + step->buf16, src16, status); + break; + case ICU_chain_step_type_tokenize: + // attach to new src16 token only first time during splitting + if (step->need_new_token){ + icu_tokenizer_attach(step->u.tokenizer, src16, status); + step->need_new_token = 0; + } + // splitting one src16 token into multiple buf16 tokens + step->more_tokens + = icu_tokenizer_next_token(step->u.tokenizer, + step->buf16, status); + // make sure to get new previous token if this one had been used up + if (step->previous && !step->more_tokens){ + if (icu_chain_step_next_token(chain, step->previous, status)){ + icu_tokenizer_attach(step->u.tokenizer, src16, status); + step->need_new_token = 0; + step->more_tokens + = icu_tokenizer_next_token(step->u.tokenizer, + step->buf16, status); + } + } + if (0 == step->more_tokens) + return 0; + break; + default: + return 0; + break; + } + + + + // stop further token processing if last step and + // new tokens are needed from previous (non-existing) step + if (!step->previous && step->need_new_token) + step->more_tokens = 0; + + //printf("%d %d %d\n", + // step->more_tokens, src16->utf16_len, step->buf16->utf16_len); + + + if (U_FAILURE(*status)) + return 0; + + return 1; +}; + + + +int icu_chain_assign_cstr(struct icu_chain * chain, + const char * src8cstr, + UErrorCode *status) +{ + struct icu_chain_step * stp = 0; + + if (!chain || !src8cstr) + return 0; + + stp = chain->steps; + + // clear token count + chain->token_count = 0; + + // clear all steps stop states + + while (stp){ + stp->more_tokens = 1; + stp->need_new_token = 1; + stp = stp->previous; + } + + // finally convert UTF8 to UTF16 string + icu_utf16_from_utf8_cstr(chain->src16, src8cstr, status); + + if (U_FAILURE(*status)) + return 0; + + return 1; +}; + + + +int icu_chain_next_token(struct icu_chain * chain, + UErrorCode *status) +{ + int success = 0; + + if (!chain || !chain->steps) + return 0; + + success = icu_chain_step_next_token(chain, chain->steps, status); + + if (success){ + chain->token_count++; + return chain->token_count; + } + + return 0; +}; + +int icu_chain_get_token_count(struct icu_chain * chain) +{ + if (!chain) + return 0; + + return chain->token_count; +}; + + + +const char * icu_chain_get_display(struct icu_chain * chain) +{ + if (chain->display8) + return icu_buf_utf8_to_cstr(chain->display8); + + return 0; +}; + +const char * icu_chain_get_norm(struct icu_chain * chain) +{ + if (chain->norm8) + return icu_buf_utf8_to_cstr(chain->norm8); + + return 0; +}; + +const char * icu_chain_get_sort(struct icu_chain * chain) +{ + if (chain->sort8) + return icu_buf_utf8_to_cstr(chain->sort8); + + return 0; +}; + + + + #endif // HAVE_ICU