X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=src%2Flogic.c;h=5a014aac5194fa9a5f26d078f73233de255aee03;hb=1ea74f79189dfde78b3b65e7355774da84efd0dc;hp=a3fdd49eb987577874749947fed2b12226166f8b;hpb=fba8c6cb11ed2ab48f433619315fb32e1f0f5184;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index a3fdd49..5a014aa 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,4 +1,4 @@ -/* $Id: logic.c,v 1.24 2007-04-26 12:12:19 marc Exp $ +/* $Id: logic.c,v 1.26 2007-04-27 12:17:04 marc Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -70,6 +70,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include "database.h" #include "client.h" #include "settings.h" +#include "normalize7bit.h" #define MAX_CHUNK 15 @@ -121,75 +122,6 @@ void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) } } -char *normalize_mergekey(char *buf, int skiparticle) -{ - char *p = buf, *pout = buf; - - if (skiparticle) - { - char firstword[64]; - char articles[] = "the den der die des an a "; // must end in space - - while (*p && !isalnum(*p)) - p++; - pout = firstword; - while (*p && *p != ' ' && pout - firstword < 62) - *(pout++) = tolower(*(p++)); - *(pout++) = ' '; - *(pout++) = '\0'; - if (!strstr(articles, firstword)) - p = buf; - pout = buf; - } - - while (*p) - { - while (*p && !isalnum(*p)) - p++; - while (isalnum(*p)) - *(pout++) = tolower(*(p++)); - if (*p) - *(pout++) = ' '; - while (*p && !isalnum(*p)) - p++; - } - if (buf != pout) - do { - *(pout--) = '\0'; - } - while (pout > buf && *pout == ' '); - - return buf; -} - -// Extract what appears to be years from buf, storing highest and -// lowest values. -static int extract_years(const char *buf, int *first, int *last) -{ - *first = -1; - *last = -1; - while (*buf) - { - const char *e; - int len; - - while (*buf && !isdigit(*buf)) - buf++; - len = 0; - for (e = buf; *e && isdigit(*e); e++) - len++; - if (len == 4) - { - int value = atoi(buf); - if (*first < 0 || value < *first) - *first = value; - if (*last < 0 || value > *last) - *last = value; - } - buf = e; - } - return *first; -} static void add_facet(struct session *s, const char *type, const char *value) @@ -291,9 +223,7 @@ xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) return 0; } } -#endif - -#if 0 +#else // do it another way to detect transformation errors right now // but does not seem to work either! { @@ -989,7 +919,7 @@ struct record *ingest_record(struct client *cl, Z_External *rec, mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey); xmlFree(mergekey); - normalize_mergekey((char *) mergekey_norm, 0); + normalize7bit_mergekey((char *) mergekey_norm, 0); cluster = reclist_insert(se->reclist, global_parameters.server->service, @@ -1006,6 +936,8 @@ struct record *ingest_record(struct client *cl, Z_External *rec, } relevance_newrec(se->relevance, cluster); + + // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { if (type) @@ -1072,35 +1004,43 @@ struct record *ingest_record(struct client *cl, Z_External *rec, } #endif - // Find out where we are putting it + // Find out where we are putting it - based on merge or not if (ser_md->merge == Metadata_merge_no) wheretoput = &record->metadata[md_field_id]; else wheretoput = &cluster->metadata[md_field_id]; - // Put it there - -#if 1 //something wrong with constructor, debug tomorrow + // create new record_metadata +#if 0 rec_md = nmem_malloc(se->nmem, sizeof(struct record_metadata)); rec_md->next = 0; #else rec_md = record_metadata_create(se->nmem); #endif + // and polulate with data: + // type based charmapping decisions follow here if (ser_md->type == Metadata_type_generic) { + +#if 0 char *p, *pe; for (p = (char *) value; *p && isspace(*p); p++) ; for (pe = p + strlen(p) - 1; pe > p && strchr(" ,/.:([", *pe); pe--) *pe = '\0'; +#else + char * p = (char *) value; + p = normalize7bit_generic(p, " ,/.:(["); +#endif + rec_md->data.text = nmem_strdup(se->nmem, p); } else if (ser_md->type == Metadata_type_year) { - if (extract_years((char *) value, &first, &last) < 0) + if (extract7bit_years((char *) value, &first, &last) < 0) continue; } else @@ -1109,12 +1049,22 @@ struct record *ingest_record(struct client *cl, Z_External *rec, "Unknown type in metadata element %s", type); continue; } + +#if 0 // this test does not belong here. + // the condition is enforced in the constructor + // inside conf_metadata_assign() + // and will _never_ occur if (ser_md->type == Metadata_type_year && ser_md->merge != Metadata_merge_range) { yaz_log(YLOG_WARN, "Only range merging supported for years"); continue; } +# endif + + + // and polulate with data: + // assign cluster or record based on merge action if (ser_md->merge == Metadata_merge_unique) { struct record_metadata *mnode; @@ -1142,7 +1092,7 @@ struct record *ingest_record(struct client *cl, Z_External *rec, cluster->sortkeys[sk_field_id] = nmem_malloc(se->nmem, sizeof(union data_types)); - normalize_mergekey(s, + normalize7bit_mergekey(s, (ser_sk->type == Metadata_sortkey_skiparticle)); cluster->sortkeys[sk_field_id]->text = s; } @@ -1156,7 +1106,14 @@ struct record *ingest_record(struct client *cl, Z_External *rec, } else if (ser_md->merge == Metadata_merge_range) { + +#if 0 // this assert does not belong here. + // the condition is enforced in the constructor + // inside conf_metadata_assign() + // and will _never_ occur assert(ser_md->type == Metadata_type_year); +#endif + if (!*wheretoput) { *wheretoput = rec_md; @@ -1183,14 +1140,23 @@ struct record *ingest_record(struct client *cl, Z_External *rec, } #endif } + +#if 0 // this else is only entered when Metadata_merge_no + // but I believe then we should _not_ pollute with log messages else + yaz_log(YLOG_WARN, "Don't know how to merge on element name %s", ser_md->name); +#endif + + // ranking of _all_ fields enabled ... if (ser_md->rank) relevance_countwords(se->relevance, cluster, (char *) value, ser_md->rank); + + // construct facets ... if (ser_md->termlist) { if (ser_md->type == Metadata_type_year) @@ -1207,6 +1173,8 @@ struct record *ingest_record(struct client *cl, Z_External *rec, else add_facet(se, (char *) type, (char *) value); } + + // cleaning up xmlFree(type); xmlFree(value); type = value = 0;