X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=src%2Frelevance.c;h=a002a7b631eec74f076d89b28821febe9e1f8e9c;hb=446f32183265d59ee79e2859376c598fa24408e0;hp=a365ebd91a369c6f1e4336c003e784ba47a0f6d4;hpb=76e54009be46b03589a043a4433e71ba11c98338;p=pazpar2-moved-to-github.git diff --git a/src/relevance.c b/src/relevance.c index a365ebd..a002a7b 100644 --- a/src/relevance.c +++ b/src/relevance.c @@ -31,55 +31,115 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA struct relevance { int *doc_frequency_vec; + int *term_frequency_vec_tmp; int vec_len; struct word_entry *entries; pp2_charset_token_t prt; + int rank_cluster; + int follow_boost; + int lead_boost; + int length_divide; NMEM nmem; }; struct word_entry { const char *norm_str; + const char *display_str; int termno; + int follow_boost; char *ccl_field; struct word_entry *next; }; -int word_entry_match(struct word_entry *entries, const char *norm_str) +static int word_entry_match(struct relevance *r, const char *norm_str, + const char *rank, int *mult) { - for (; entries; entries = entries->next) + int i = 1; + struct word_entry *entries = r->entries; + for (; entries; entries = entries->next, i++) { - if (!strcmp(norm_str, entries->norm_str)) + if (*norm_str && !strcmp(norm_str, entries->norm_str)) + { + int extra = r->follow_boost; + struct word_entry *e_follow = entries; + const char *cp = 0; + int no_read = 0; + sscanf(rank, "%d%n", mult, &no_read); + rank += no_read; + while (*rank == ' ') + rank++; + if (no_read > 0 && (cp = strchr(rank, ' '))) + { + if ((cp - rank) == strlen(entries->ccl_field) && + memcmp(entries->ccl_field, rank, cp - rank) == 0) + *mult = atoi(cp + 1); + } + (*mult) += entries->follow_boost; + while ((e_follow = e_follow->next) != 0 && extra > 0) + { + e_follow->follow_boost = extra--; + } return entries->termno; + } + entries->follow_boost = 0; } return 0; } void relevance_countwords(struct relevance *r, struct record_cluster *cluster, - const char *words, int multiplier, const char *name) + const char *words, const char *rank, + const char *name) { - int *mult = cluster->term_frequency_vec_tmp; + int *mult = r->term_frequency_vec_tmp; const char *norm_str; int i, length = 0; + int lead_mult = r->lead_boost; + struct word_entry *e; + WRBUF w = cluster->relevance_explain1; pp2_charset_token_first(r->prt, words, 0); - for (i = 1; i < r->vec_len; i++) + for (e = r->entries, i = 1; i < r->vec_len; i++, e = e->next) + { mult[i] = 0; + e->follow_boost = 0; + } + assert(rank); while ((norm_str = pp2_charset_token_next(r->prt))) { - int res = word_entry_match(r->entries, norm_str); + int local_mult = 0; + int res = word_entry_match(r, norm_str, rank, &local_mult); if (res) { assert(res < r->vec_len); - mult[res] += multiplier; + mult[res] += local_mult + lead_mult; } + if (lead_mult > 0) + --lead_mult; length++; } - for (i = 1; i < r->vec_len; i++) + for (e = r->entries, i = 1; i < r->vec_len; i++, e = e->next) { - if (length > 0) /* only add if non-empty */ + if (length == 0 || mult[i] == 0) + continue; + wrbuf_printf(w, "%s: field=%s vecf[%d] += mult(%d)", + e->display_str, name, i, mult[i]); + switch (r->length_divide) + { + case 0: + wrbuf_printf(w, ";\n"); + cluster->term_frequency_vecf[i] += (double) mult[i]; + break; + case 1: + wrbuf_printf(w, " / log2(1+length(%d));\n", length); + cluster->term_frequency_vecf[i] += + (double) mult[i] / log2(1 + length); + break; + case 2: + wrbuf_printf(w, " / length(%d);\n", length); cluster->term_frequency_vecf[i] += (double) mult[i] / length; + } cluster->term_frequency_vec[i] += mult[i]; } @@ -107,7 +167,7 @@ static void pull_terms(struct relevance *res, struct ccl_rpn_node *n) for (i = 0; i < numwords; i++) { const char *norm_str; - + ccl_field = nmem_strdup_null(res->nmem, n->u.t.qual); pp2_charset_token_first(res->prt, words[i], 0); @@ -120,6 +180,7 @@ static void pull_terms(struct relevance *res, struct ccl_rpn_node *n) (*e)->norm_str = nmem_strdup(res->nmem, norm_str); (*e)->ccl_field = ccl_field; (*e)->termno = res->vec_len++; + (*e)->display_str = nmem_strdup(res->nmem, words[i]); (*e)->next = 0; } } @@ -130,21 +191,34 @@ static void pull_terms(struct relevance *res, struct ccl_rpn_node *n) } struct relevance *relevance_create_ccl(pp2_charset_fact_t pft, - NMEM nmem, struct ccl_rpn_node *query) + struct ccl_rpn_node *query, + int rank_cluster, + int follow_boost, int lead_boost, + int length_divide) { + NMEM nmem = nmem_create(); struct relevance *res = nmem_malloc(nmem, sizeof(*res)); int i; res->nmem = nmem; res->entries = 0; res->vec_len = 1; + res->rank_cluster = rank_cluster; + res->follow_boost = follow_boost; + res->lead_boost = lead_boost; + res->length_divide = length_divide; res->prt = pp2_charset_token_create(pft, "relevance"); - + pull_terms(res, query); res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int)); for (i = 0; i < res->vec_len; i++) - res->doc_frequency_vec[i] = 0; + res->doc_frequency_vec[i] = 0; + + // worker array + res->term_frequency_vec_tmp = + nmem_malloc(res->nmem, + res->vec_len * sizeof(*res->term_frequency_vec_tmp)); return res; } @@ -153,6 +227,7 @@ void relevance_destroy(struct relevance **rp) if (*rp) { pp2_charset_token_destroy((*rp)->prt); + nmem_destroy((*rp)->nmem); *rp = 0; } } @@ -169,18 +244,13 @@ void relevance_newrec(struct relevance *r, struct record_cluster *rec) r->vec_len * sizeof(*rec->term_frequency_vec)); for (i = 0; i < r->vec_len; i++) rec->term_frequency_vec[i] = 0; - + // term frequency divided by length of field [1,...] rec->term_frequency_vecf = nmem_malloc(r->nmem, r->vec_len * sizeof(*rec->term_frequency_vecf)); for (i = 0; i < r->vec_len; i++) rec->term_frequency_vecf[i] = 0.0; - - // for relevance_countwords (so we don't have to xmalloc/xfree) - rec->term_frequency_vec_tmp = - nmem_malloc(r->nmem, - r->vec_len * sizeof(*rec->term_frequency_vec_tmp)); } } @@ -209,29 +279,51 @@ void relevance_prepare_read(struct relevance *rel, struct reclist *reclist) idfvec[i] = 0; else { - // This conditional may be terribly wrong - // It was there to address the situation where vec[0] == vec[i] - // which leads to idfvec[i] == 0... not sure about this - // Traditional TF-IDF may assume that a word that occurs in every - // record is irrelevant, but this is actually something we will - // see a lot - if ((idfvec[i] = log((float) rel->doc_frequency_vec[0] / - rel->doc_frequency_vec[i])) < 0.0000001) - idfvec[i] = 1; + /* add one to nominator idf(t,D) to ensure a value > 0 */ + idfvec[i] = log((float) (1 + rel->doc_frequency_vec[0]) / + rel->doc_frequency_vec[i]); } } // Calculate relevance for each document while (1) { - int t; int relevance = 0; + WRBUF w; + struct word_entry *e = rel->entries; struct record_cluster *rec = reclist_read_record(reclist); if (!rec) break; - for (t = 1; t < rel->vec_len; t++) + w = rec->relevance_explain2; + wrbuf_rewind(w); + for (i = 1; i < rel->vec_len; i++) + { + float termfreq = (float) rec->term_frequency_vecf[i]; + int add = 100000 * termfreq * idfvec[i]; + + wrbuf_printf(w, "idf[%d] = log(((1 + total(%d))/termoccur(%d));\n", + i, rel->doc_frequency_vec[0], + rel->doc_frequency_vec[i]); + wrbuf_printf(w, "%s: relevance += 100000 * vecf[%d](%f) * " + "idf[%d](%f) (%d);\n", + e->display_str, i, termfreq, i, idfvec[i], add); + relevance += add; + e = e->next; + } + if (!rel->rank_cluster) + { + struct record *record; + int cluster_size = 0; + + for (record = rec->records; record; record = record->next) + cluster_size++; + + wrbuf_printf(w, "score = relevance(%d)/cluster_size(%d);\n", + relevance, cluster_size); + relevance /= cluster_size; + } + else { - float termfreq = (float) rec->term_frequency_vecf[t]; - relevance += 100000 * (termfreq * idfvec[t] + 0.0000005); + wrbuf_printf(w, "score = relevance(%d);\n", relevance); } rec->relevance_score = relevance; }