X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=src%2Flogic.c;h=4ec42cf1a5f8fc70fd31196e932dd881d8b107ca;hb=06ec84ec5e71464bbdb286b0548483b97147be81;hp=624a4740cd23edf04bd405c8f9ec2c0802f7c9e5;hpb=e8edd24406f53d5e232203025ab9f4f820e7d837;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index 624a474..4ec42cf 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,4 +1,4 @@ -/* $Id: logic.c,v 1.22 2007-04-26 11:03:54 marc Exp $ +/* $Id: logic.c,v 1.32 2007-05-23 21:58:28 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -70,6 +70,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include "database.h" #include "client.h" #include "settings.h" +#include "normalize7bit.h" #define MAX_CHUNK 15 @@ -81,7 +82,8 @@ struct parameters global_parameters = "", "", 0, - 0, + 0, /* dump_records */ + 0, /* debug_mode */ 30, "81", "Index Data PazPar2", @@ -121,75 +123,6 @@ void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) } } -char *normalize_mergekey(char *buf, int skiparticle) -{ - char *p = buf, *pout = buf; - - if (skiparticle) - { - char firstword[64]; - char articles[] = "the den der die des an a "; // must end in space - - while (*p && !isalnum(*p)) - p++; - pout = firstword; - while (*p && *p != ' ' && pout - firstword < 62) - *(pout++) = tolower(*(p++)); - *(pout++) = ' '; - *(pout++) = '\0'; - if (!strstr(articles, firstword)) - p = buf; - pout = buf; - } - - while (*p) - { - while (*p && !isalnum(*p)) - p++; - while (isalnum(*p)) - *(pout++) = tolower(*(p++)); - if (*p) - *(pout++) = ' '; - while (*p && !isalnum(*p)) - p++; - } - if (buf != pout) - do { - *(pout--) = '\0'; - } - while (pout > buf && *pout == ' '); - - return buf; -} - -// Extract what appears to be years from buf, storing highest and -// lowest values. -static int extract_years(const char *buf, int *first, int *last) -{ - *first = -1; - *last = -1; - while (*buf) - { - const char *e; - int len; - - while (*buf && !isdigit(*buf)) - buf++; - len = 0; - for (e = buf; *e && isdigit(*e); e++) - len++; - if (len == 4) - { - int value = atoi(buf); - if (*first < 0 || value < *first) - *first = value; - if (*last < 0 || value > *last) - *last = value; - } - buf = e; - } - return *first; -} static void add_facet(struct session *s, const char *type, const char *value) @@ -277,7 +210,6 @@ xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) for (m = sdb->map; m; m = m->next){ xmlDoc *new = 0; -#if 1 { xmlNodePtr root = 0; new = xsltApplyStylesheet(m->stylesheet, rdoc, 0); @@ -291,25 +223,6 @@ xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) return 0; } } -#endif - -#if 0 - // do it another way to detect transformation errors right now - // but does not seem to work either! - { - xsltTransformContextPtr ctxt; - ctxt = xsltNewTransformContext(m->stylesheet, rdoc); - new = xsltApplyStylesheetUser(m->stylesheet, rdoc, 0, 0, 0, ctxt); - if ((ctxt->state == XSLT_STATE_ERROR) || - (ctxt->state == XSLT_STATE_STOPPED)){ - yaz_log(YLOG_WARN, "XSLT transformation failed from %s", - cl->database->database->url); - xmlFreeDoc(new); - xmlFreeDoc(rdoc); - return 0; - } - } -#endif xmlFreeDoc(rdoc); rdoc = new; @@ -581,6 +494,17 @@ static void session_init_databases_fun(void *context, struct database *db) new->database = db; new->yaz_marc = 0; + +#ifdef HAVE_ICU + if (global_parameters.server && global_parameters.server->icu_chn) + new->pct + = pp2_charset_create(global_parameters.server->icu_chn); + else + new->pct = pp2_charset_create(0); +#else // HAVE_ICU + new->pct = pp2_charset_create(0); +#endif // HAVE_ICU + new->map = 0; new->settings = nmem_malloc(se->session_nmem, sizeof(struct settings *) * num); @@ -604,6 +528,8 @@ static void session_database_destroy(struct session_database *sdb) xsltFreeStylesheet(m->stylesheet); if (sdb->yaz_marc) yaz_marc_destroy(sdb->yaz_marc); + if (sdb->pct) + pp2_charset_destroy(sdb->pct); } // Initialize session_database list -- this represents this session's view @@ -644,13 +570,19 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, { struct session_database *sdb = find_session_database(se, dbname); struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new)); - int offset = settings_offset(setting); + int offset = settings_offset_cprefix(setting); if (offset < 0) { yaz_log(YLOG_WARN, "Unknown setting %s", setting); return; } + // Jakub: This breaks the filter setting. + /*if (offset == PZ_ID) + { + yaz_log(YLOG_WARN, "No need to set pz:id setting. Ignoring"); + return; + }*/ new->precedence = 0; new->target = dbname; new->name = setting; @@ -953,7 +885,7 @@ struct record *ingest_record(struct client *cl, Z_External *rec, { xmlDoc *xdoc = normalize_record(client_get_database(cl), rec); xmlNode *root, *n; - struct record *res; + struct record *record; struct record_cluster *cluster; struct session *se = client_get_session(cl); xmlChar *mergekey, *mergekey_norm; @@ -972,21 +904,17 @@ struct record *ingest_record(struct client *cl, Z_External *rec, return 0; } - res = nmem_malloc(se->nmem, sizeof(struct record)); - res->next = 0; - res->client = cl; - res->metadata = nmem_malloc(se->nmem, - sizeof(struct record_metadata*) * service->num_metadata); - memset(res->metadata, 0, - sizeof(struct record_metadata*) * service->num_metadata); + record = record_create(se->nmem, + service->num_metadata, service->num_sortkeys); + record_assign_client(record, cl); mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey); xmlFree(mergekey); - normalize_mergekey((char *) mergekey_norm, 0); + normalize7bit_mergekey((char *) mergekey_norm, 0); cluster = reclist_insert(se->reclist, global_parameters.server->service, - res, (char *) mergekey_norm, + record, (char *) mergekey_norm, &se->total_merged); if (global_parameters.dump_records) yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid, @@ -999,6 +927,8 @@ struct record *ingest_record(struct client *cl, Z_External *rec, } relevance_newrec(se->relevance, cluster); + + // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { if (type) @@ -1013,7 +943,8 @@ struct record *ingest_record(struct client *cl, Z_External *rec, { struct conf_metadata *ser_md = 0; struct conf_sortkey *ser_sk = 0; - struct record_metadata **wheretoput, *newm; + struct record_metadata **wheretoput = 0; + struct record_metadata *rec_md = 0; int md_field_id = -1; int sk_field_id = -1; int first, last; @@ -1024,47 +955,45 @@ struct record *ingest_record(struct client *cl, Z_External *rec, if (!type || !value) continue; - // First, find out what field we're looking at - for (md_field_id = 0; md_field_id < service->num_metadata; - md_field_id++) - if (!strcmp((const char *) type, - service->metadata[md_field_id].name)) - { - ser_md = &service->metadata[md_field_id]; - if (ser_md->sortkey_offset >= 0) - ser_sk = &service->sortkeys[ser_md->sortkey_offset]; - break; - } - if (!ser_md) + md_field_id + = conf_service_metadata_field_id(service, (const char *) type); + if (md_field_id < 0) { yaz_log(YLOG_WARN, "Ignoring unknown metadata element: %s", type); continue; } - // Find out where we are putting it + ser_md = &service->metadata[md_field_id]; + + if (ser_md->sortkey_offset >= 0){ + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } + + // Find out where we are putting it - based on merge or not if (ser_md->merge == Metadata_merge_no) - wheretoput = &res->metadata[md_field_id]; + wheretoput = &record->metadata[md_field_id]; else wheretoput = &cluster->metadata[md_field_id]; - // Put it there - newm = nmem_malloc(se->nmem, sizeof(struct record_metadata)); - newm->next = 0; + // create new record_metadata + rec_md = record_metadata_create(se->nmem); + + // and polulate with data: + // type based charmapping decisions follow here if (ser_md->type == Metadata_type_generic) { - char *p, *pe; - for (p = (char *) value; *p && isspace(*p); p++) - ; - for (pe = p + strlen(p) - 1; - pe > p && strchr(" ,/.:([", *pe); pe--) - *pe = '\0'; - newm->data.text = nmem_strdup(se->nmem, p); + + char * p = (char *) value; + p = normalize7bit_generic(p, " ,/.:(["); + + rec_md->data.text = nmem_strdup(se->nmem, p); } else if (ser_md->type == Metadata_type_year) { - if (extract_years((char *) value, &first, &last) < 0) + if (extract7bit_years((char *) value, &first, &last) < 0) continue; } else @@ -1073,62 +1002,58 @@ struct record *ingest_record(struct client *cl, Z_External *rec, "Unknown type in metadata element %s", type); continue; } - if (ser_md->type == Metadata_type_year - && ser_md->merge != Metadata_merge_range) - { - yaz_log(YLOG_WARN, "Only range merging supported for years"); - continue; - } + + // and polulate with data: + // assign cluster or record based on merge action if (ser_md->merge == Metadata_merge_unique) { struct record_metadata *mnode; for (mnode = *wheretoput; mnode; mnode = mnode->next) if (!strcmp((const char *) mnode->data.text, - newm->data.text)) + rec_md->data.text)) break; if (!mnode) { - newm->next = *wheretoput; - *wheretoput = newm; + rec_md->next = *wheretoput; + *wheretoput = rec_md; } } else if (ser_md->merge == Metadata_merge_longest) { if (!*wheretoput - || strlen(newm->data.text) + || strlen(rec_md->data.text) > strlen((*wheretoput)->data.text)) { - *wheretoput = newm; + *wheretoput = rec_md; if (ser_sk) { - char *s = nmem_strdup(se->nmem, newm->data.text); - if (!cluster->sortkeys[ser_md->sortkey_offset]) - cluster->sortkeys[ser_md->sortkey_offset] = + char *s = nmem_strdup(se->nmem, rec_md->data.text); + if (!cluster->sortkeys[sk_field_id]) + cluster->sortkeys[sk_field_id] = nmem_malloc(se->nmem, sizeof(union data_types)); - normalize_mergekey(s, + normalize7bit_mergekey(s, (ser_sk->type == Metadata_sortkey_skiparticle)); - cluster->sortkeys[ser_md->sortkey_offset]->text = s; + cluster->sortkeys[sk_field_id]->text = s; } } } else if (ser_md->merge == Metadata_merge_all || ser_md->merge == Metadata_merge_no) { - newm->next = *wheretoput; - *wheretoput = newm; + rec_md->next = *wheretoput; + *wheretoput = rec_md; } else if (ser_md->merge == Metadata_merge_range) { - assert(ser_md->type == Metadata_type_year); if (!*wheretoput) { - *wheretoput = newm; + *wheretoput = rec_md; (*wheretoput)->data.number.min = first; (*wheretoput)->data.number.max = last; if (ser_sk) - cluster->sortkeys[ser_md->sortkey_offset] - = &newm->data; + cluster->sortkeys[sk_field_id] + = &rec_md->data; } else { @@ -1138,23 +1063,23 @@ struct record *ingest_record(struct client *cl, Z_External *rec, (*wheretoput)->data.number.max = last; } #ifdef GAGA - if (sk) + if (ser_sk) { union data_types *sdata - = cluster->sortkeys[md->sortkey_offset]; + = cluster->sortkeys[sk_field_id]; yaz_log(YLOG_LOG, "SK range: %d-%d", sdata->number.min, sdata->number.max); } #endif } - else - yaz_log(YLOG_WARN, - "Don't know how to merge on element name %s", - ser_md->name); + + // ranking of _all_ fields enabled ... if (ser_md->rank) relevance_countwords(se->relevance, cluster, (char *) value, ser_md->rank); + + // construct facets ... if (ser_md->termlist) { if (ser_md->type == Metadata_type_year) @@ -1171,6 +1096,8 @@ struct record *ingest_record(struct client *cl, Z_External *rec, else add_facet(se, (char *) type, (char *) value); } + + // cleaning up xmlFree(type); xmlFree(value); type = value = 0; @@ -1189,7 +1116,7 @@ struct record *ingest_record(struct client *cl, Z_External *rec, relevance_donerecord(se->relevance, cluster); se->total_records++; - return res; + return record; }