-/* $Id: rank1.c,v 1.19 2004-10-28 10:37:15 heikki Exp $
+/* $Id: rank1.c,v 1.20 2004-11-04 13:09:06 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003
Index Data Aps
#include <unistd.h>
#endif
-#define DEBUG_RANK 0
#include "index.h"
+static int log_level=0;
+static int log_initialized=0;
+
struct rank_class_info {
int dummy;
};
struct rank_class_info *ci =
(struct rank_class_info *) xmalloc (sizeof(*ci));
- yaz_log (LOG_DEBUG, "rank-1 create");
+ if (!log_initialized)
+ {
+ log_level=yaz_log_module_level("rank1");
+ log_initialized=1;
+ }
+ yaz_log (log_level, "rank-1 create");
return ci;
}
{
struct rank_class_info *ci = (struct rank_class_info *) class_handle;
- yaz_log (LOG_DEBUG, "rank-1 destroy");
+ yaz_log (log_level, "rank-1 destroy");
xfree (ci);
}
(struct rank_set_info *) nmem_malloc (nmem,sizeof(*si));
int i;
-#if DEBUG_RANK
- yaz_log (LOG_LOG, "rank-1 begin");
-#endif
+ yaz_log (log_level, "rank-1 begin");
si->no_entries = numterms;
si->no_rank_entries = 0;
si->nmem=nmem;
for (i = 0; i < numterms; i++)
{
zint g = rset_count(terms[i]->rset);
-#if DEBUG_RANK
- yaz_log(LOG_LOG, "i=%d flags=%s '%s'", i,
+ yaz_log(log_level, "i=%d flags=%s '%s'", i,
terms[i]->flags, terms[i]->name );
-#endif
if (!strncmp (terms[i]->flags, "rank,", 5))
{
const char *cp = strstr(terms[i]->flags+4, ",w=");
si->entries[i].rank_weight = atoi (cp+3);
else
si->entries[i].rank_weight = 34;
-#if DEBUG_RANK
- yaz_log (LOG_LOG, " i=%d weight=%d g="ZINT_FORMAT, i,
+ yaz_log (log_level, " i=%d weight=%d g="ZINT_FORMAT, i,
si->entries[i].rank_weight, g);
-#endif
(si->no_rank_entries)++;
}
else
si->entries[i].local_occur = 0; /* FIXME */
si->entries[i].global_occur = g;
si->entries[i].global_inv = 32 - log2_int (g);
- yaz_log (LOG_DEBUG, " global_inv = %d g = " ZINT_FORMAT,
+ yaz_log (log_level, " global_inv = %d g = " ZINT_FORMAT,
(int) (32-log2_int (g)), g);
si->entries[i].term=terms[i];
si->entries[i].term_index=i;
*/
static void end (struct zebra_register *reg, void *set_handle)
{
- yaz_log (LOG_DEBUG, "rank-1 end");
+ yaz_log (log_level, "rank-1 end");
/* no need to free anything, they are in nmems */
}
static void add (void *set_handle, int seqno, TERMID term)
{
struct rank_set_info *si = (struct rank_set_info *) set_handle;
- struct rank_term_info *ti= (struct rank_term_info *) term->rankpriv;
+ struct rank_term_info *ti;
assert(si);
- assert(term);
+ if (!term)
+ {
+ yaz_log (log_level, "rank-1 add NULL term");
+ return;
+ }
+ ti= (struct rank_term_info *) term->rankpriv;
assert(ti);
-#if DEBUG_RANK
- yaz_log (LOG_LOG, "rank-1 add seqno=%d term=%s", seqno, term->name);
-#endif
si->last_pos = seqno;
ti->local_occur++;
+ yaz_log (log_level, "rank-1 add seqno=%d term=%s count=%d",
+ seqno, term->name,ti->local_occur);
}
/*
if (!si->no_rank_entries)
return -1;
-#if DEBUG_RANK
- yaz_log(LOG_LOG, "calc");
-#endif
for (i = 0; i < si->no_entries; i++)
{
-#if DEBUG_RANK
- yaz_log(LOG_LOG, "calc: i=%d rank_flag=%d lo=%d",
+ yaz_log(log_level, "calc: i=%d rank_flag=%d lo=%d",
i, si->entries[i].rank_flag, si->entries[i].local_occur);
-#endif
if (si->entries[i].rank_flag && (lo = si->entries[i].local_occur))
score += (8+log2_int (lo)) * si->entries[i].global_inv *
si->entries[i].rank_weight;
}
divisor = si->no_rank_entries * (8+log2_int (si->last_pos/si->no_entries));
score = score / divisor;
-#if DEBUG_RANK
- yaz_log (LOG_LOG, "calc sysno=" ZINT_FORMAT " score=%d", sysno, score);
-#endif
+ yaz_log (log_level, "calc sysno=" ZINT_FORMAT " score=%d", sysno, score);
if (score > 1000)
score = 1000;
+ /* reset the counts for the next term */
for (i = 0; i < si->no_entries; i++)
si->entries[i].local_occur = 0;
return score;
-/* $Id: zvrank.c,v 1.12 2004-10-28 10:37:15 heikki Exp $
+/* $Id: zvrank.c,v 1.13 2004-11-04 13:09:06 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003
Index Data Aps
#include "index.h"
+static int log_level=0;
+static int log_initialized=0;
+
static double blog(double x) {
/* log_2, log_e or log_10 is used, best to change it here if necessary */
if (x <= 0)
typedef struct rs_info *RS;
static void prn_rs(RS rs) { /* for debugging */
- yaz_log(LOG_DEBUG, "* RS:");
- yaz_log(LOG_DEBUG, " db_docs: %d", rs->db_docs);
- yaz_log(LOG_DEBUG, " db_terms: %d", rs->db_terms);
- yaz_log(LOG_DEBUG, " f_max: %d", rs->db_f_max);
- yaz_log(LOG_DEBUG, " f_max_str: %s", rs->db_f_max_str);
- yaz_log(LOG_DEBUG, " veclen: %d", rs->veclen);
+ yaz_log(log_level, "* RS:");
+ yaz_log(log_level, " db_docs: %d", rs->db_docs);
+ yaz_log(log_level, " db_terms: %d", rs->db_terms);
+ yaz_log(log_level, " f_max: %d", rs->db_f_max);
+ yaz_log(log_level, " f_max_str: %s", rs->db_f_max_str);
+ yaz_log(log_level, " veclen: %d", rs->veclen);
/* rscheme implies functions */
- yaz_log(LOG_DEBUG, " rscheme: %s", rs->rscheme);
+ yaz_log(log_level, " rscheme: %s", rs->rscheme);
return;
}
#if 0
static void prn_ds(DS ds) { /* for debugging */
- yaz_log(LOG_DEBUG, " * DS:");
- yaz_log(LOG_DEBUG, " docid: %s", ds->docid);
- yaz_log(LOG_DEBUG, " docno: %d", ds->docno);
- yaz_log(LOG_DEBUG, " doclen: %d", ds->doclen);
- yaz_log(LOG_DEBUG, " d_f_max: %d", ds->d_f_max);
- yaz_log(LOG_DEBUG, " d_f_max_str:%s", ds->d_f_max_str);
- yaz_log(LOG_DEBUG, " veclen: %d", ds->veclen);
+ yaz_log(log_level, " * DS:");
+ yaz_log(log_level, " docid: %s", ds->docid);
+ yaz_log(log_level, " docno: %d", ds->docno);
+ yaz_log(log_level, " doclen: %d", ds->doclen);
+ yaz_log(log_level, " d_f_max: %d", ds->d_f_max);
+ yaz_log(log_level, " d_f_max_str:%s", ds->d_f_max_str);
+ yaz_log(log_level, " veclen: %d", ds->veclen);
return;
}
#endif
#if 0
static void prn_ts(TS ts) { /* for debugging */
- yaz_log(LOG_DEBUG, " * TERM:%s gocc:%d locc:%d tf:%f idf:%f wt:%f",
+ yaz_log(log_level, " * TERM:%s gocc:%d locc:%d tf:%f idf:%f wt:%f",
ts->name, ts->gocc, ts->locc, ts->tf, ts->idf, ts->wt);
return;
}
/* idf ^ 2 */
veclen=ds->veclen;
num_docs=rs->db_docs;
- yaz_log(LOG_DEBUG, "idf_squared: db_docs required");
+ yaz_log(log_level, "idf_squared: db_docs required");
for (i=0; i < veclen; i++) {
gocc=ds->terms[i].gocc;
if (gocc==0)
return;
}
-/* add: norm_pivot, ... */
+/* FIXME add norm_pivot, ... */
static double sim_cosine(void *dsi1, void *dsi2) {
DS ds1=(DS)dsi1;
return (smul/sdiv);
}
-/* add: norm_jaccard, norm_dice, ... */
+/* FIXME: add norm_jaccard, norm_dice, ... */
/* end weighting functions */
char c0, c1, c2, c3, c4, c5, c6;
const char *def_rscheme="ntc-atn"; /* a good default */
/**/
- yaz_log(LOG_DEBUG, "zv_init_scheme");
+ yaz_log(log_level, "zv_init_scheme");
slen=strlen(sname);
if (slen < 7)
- yaz_log(LOG_LOG, "zvrank: invalid weighting-scheme \"%s\"", sname);
+ yaz_log(log_level, "zvrank: invalid weighting-scheme \"%s\"", sname);
if (slen > 0) c0=sname[0]; else c0=def_rscheme[0];
if (slen > 1) c1=sname[1]; else c1=def_rscheme[1];
if (slen > 2) c2=sname[2]; else c2=def_rscheme[2];
if (slen > 4) c4=sname[4]; else c4=def_rscheme[4];
if (slen > 5) c5=sname[5]; else c5=def_rscheme[5];
if (slen > 6) c6=sname[6]; else c6=def_rscheme[6];
- /**/
- /* assign doc functions */
- switch (c0) {
- case 'b':
- rs->d_tf_fct=tf_binary;
- rs->rscheme[0]='b';
- break;
- case 'm':
- rs->d_tf_fct=tf_max_norm;
- rs->rscheme[0]='m';
- yaz_log(LOG_DEBUG, "tf_max_norm: d_f_max required");
- break;
- case 'a':
- rs->d_tf_fct=tf_aug_norm;
- rs->rscheme[0]='a';
- yaz_log(LOG_DEBUG, "tf_aug_norm: d_f_max required");
- break;
- case 's':
- rs->d_tf_fct=tf_square;
- rs->rscheme[0]='s';
- break;
- case 'l':
- rs->d_tf_fct=tf_log;
- rs->rscheme[0]='l';
- break;
- default: /* 'n' */
- rs->d_tf_fct=tf_none;
- rs->rscheme[0]='n';
- }
- switch (c1) {
- case 't':
- rs->d_idf_fct=idf_tfidf;
- rs->rscheme[1]='t';
- yaz_log(LOG_DEBUG, "idf_tfidf: db_docs required");
- break;
- case 'p':
- rs->d_idf_fct=idf_prob;
- rs->rscheme[1]='p';
- yaz_log(LOG_DEBUG, "idf_prob: db_docs required");
- break;
- case 'f':
- rs->d_idf_fct=idf_freq;
- rs->rscheme[1]='f';
- yaz_log(LOG_DEBUG, "idf_freq: db_docs required");
- break;
- case 's':
- rs->d_idf_fct=idf_squared;
- rs->rscheme[1]='s';
- yaz_log(LOG_DEBUG, "idf_squared: db_docs required");
- break;
- default: /* 'n' */
- rs->d_idf_fct=idf_none;
- rs->rscheme[1]='n';
- }
- switch (c2) {
- case 's':
- rs->d_norm_fct=norm_sum;
- rs->rscheme[2]='s';
- break;
- case 'c':
- rs->d_norm_fct=norm_cosine;
- rs->rscheme[2]='c';
- break;
- case 'f':
- rs->d_norm_fct=norm_fourth;
- rs->rscheme[2]='t';
- break;
- case 'm':
- rs->d_norm_fct=norm_max;
- rs->rscheme[2]='m';
- break;
- default: /* 'n' */
- rs->d_norm_fct=norm_none;
- rs->rscheme[2]='n';
- }
- /**/
- rs->rscheme[3]='-';
- /* assign query functions */
- switch (c4) {
- case 'b':
- rs->q_tf_fct=tf_binary;
- rs->rscheme[4]='b';
- break;
- case 'm':
- rs->q_tf_fct=tf_max_norm;
- yaz_log(LOG_DEBUG, "tf_max_norm: d_f_max required");
- rs->rscheme[4]='m';
- break;
- case 'a':
- rs->q_tf_fct=tf_aug_norm;
- rs->rscheme[4]='a';
- yaz_log(LOG_DEBUG, "tf_aug_norm: d_f_max required");
- break;
- case 's':
- rs->q_tf_fct=tf_square;
- rs->rscheme[4]='s';
- break;
- case 'l':
- rs->q_tf_fct=tf_log;
- rs->rscheme[4]='l';
- break;
- default: /* 'n' */
- rs->q_tf_fct=tf_none;
- rs->rscheme[4]='n';
- }
- switch (c5) {
- case 't':
- rs->q_idf_fct=idf_tfidf;
- rs->rscheme[5]='t';
- yaz_log(LOG_DEBUG, "idf_tfidf: db_docs required");
- break;
- case 'p':
- rs->q_idf_fct=idf_prob;
- rs->rscheme[5]='p';
- yaz_log(LOG_DEBUG, "idf_prob: db_docs required");
- break;
- case 'f':
- rs->q_idf_fct=idf_freq;
- rs->rscheme[5]='f';
- yaz_log(LOG_DEBUG, "idf_freq: db_docs required");
- break;
- case 's':
- rs->q_idf_fct=idf_squared;
- rs->rscheme[5]='s';
- yaz_log(LOG_DEBUG, "idf_squared: db_docs required");
- break;
- default: /* 'n' */
- rs->q_idf_fct=idf_none;
- rs->rscheme[5]='n';
- }
- switch (c6) {
- case 's':
- rs->q_norm_fct=norm_sum;
- rs->rscheme[6]='s';
- break;
- case 'c':
- rs->q_norm_fct=norm_cosine;
- rs->rscheme[6]='c';
- break;
- case 'f':
- rs->q_norm_fct=norm_fourth;
- rs->rscheme[6]='f';
- break;
- case 'm':
- rs->q_norm_fct=norm_max;
- rs->rscheme[6]='m';
- break;
- default: /* 'n' */
- rs->q_norm_fct=norm_none;
- rs->rscheme[6]='n';
- }
- rs->rscheme[7]='\0';
- /**/
- rs->sim_fct=sim_cosine;
- yaz_log(LOG_DEBUG, "zv_scheme %s", rs->rscheme);
- return;
+
+ /* assign doc functions */
+ switch (c0)
+ {
+ case 'b':
+ rs->d_tf_fct=tf_binary;
+ rs->rscheme[0]='b';
+ break;
+ case 'm':
+ rs->d_tf_fct=tf_max_norm;
+ rs->rscheme[0]='m';
+ yaz_log(log_level, "tf_max_norm: d_f_max required");
+ break;
+ case 'a':
+ rs->d_tf_fct=tf_aug_norm;
+ rs->rscheme[0]='a';
+ yaz_log(log_level, "tf_aug_norm: d_f_max required");
+ break;
+ case 's':
+ rs->d_tf_fct=tf_square;
+ rs->rscheme[0]='s';
+ break;
+ case 'l':
+ rs->d_tf_fct=tf_log;
+ rs->rscheme[0]='l';
+ break;
+ default: /* 'n' */
+ rs->d_tf_fct=tf_none;
+ rs->rscheme[0]='n';
+ }
+ switch (c1)
+ {
+ case 't':
+ rs->d_idf_fct=idf_tfidf;
+ rs->rscheme[1]='t';
+ yaz_log(log_level, "idf_tfidf: db_docs required");
+ break;
+ case 'p':
+ rs->d_idf_fct=idf_prob;
+ rs->rscheme[1]='p';
+ yaz_log(log_level, "idf_prob: db_docs required");
+ break;
+ case 'f':
+ rs->d_idf_fct=idf_freq;
+ rs->rscheme[1]='f';
+ yaz_log(log_level, "idf_freq: db_docs required");
+ break;
+ case 's':
+ rs->d_idf_fct=idf_squared;
+ rs->rscheme[1]='s';
+ yaz_log(log_level, "idf_squared: db_docs required");
+ break;
+ default: /* 'n' */
+ rs->d_idf_fct=idf_none;
+ rs->rscheme[1]='n';
+ }
+ switch (c2)
+ {
+ case 's':
+ rs->d_norm_fct=norm_sum;
+ rs->rscheme[2]='s';
+ break;
+ case 'c':
+ rs->d_norm_fct=norm_cosine;
+ rs->rscheme[2]='c';
+ break;
+ case 'f':
+ rs->d_norm_fct=norm_fourth;
+ rs->rscheme[2]='t';
+ break;
+ case 'm':
+ rs->d_norm_fct=norm_max;
+ rs->rscheme[2]='m';
+ break;
+ default: /* 'n' */
+ rs->d_norm_fct=norm_none;
+ rs->rscheme[2]='n';
+ }
+ rs->rscheme[3]='-';
+ /* assign query functions */
+ switch (c4)
+ {
+ case 'b':
+ rs->q_tf_fct=tf_binary;
+ rs->rscheme[4]='b';
+ break;
+ case 'm':
+ rs->q_tf_fct=tf_max_norm;
+ yaz_log(log_level, "tf_max_norm: d_f_max required");
+ rs->rscheme[4]='m';
+ break;
+ case 'a':
+ rs->q_tf_fct=tf_aug_norm;
+ rs->rscheme[4]='a';
+ yaz_log(log_level, "tf_aug_norm: d_f_max required");
+ break;
+ case 's':
+ rs->q_tf_fct=tf_square;
+ rs->rscheme[4]='s';
+ break;
+ case 'l':
+ rs->q_tf_fct=tf_log;
+ rs->rscheme[4]='l';
+ break;
+ default: /* 'n' */
+ rs->q_tf_fct=tf_none;
+ rs->rscheme[4]='n';
+ }
+ switch (c5)
+ {
+ case 't':
+ rs->q_idf_fct=idf_tfidf;
+ rs->rscheme[5]='t';
+ yaz_log(log_level, "idf_tfidf: db_docs required");
+ break;
+ case 'p':
+ rs->q_idf_fct=idf_prob;
+ rs->rscheme[5]='p';
+ yaz_log(log_level, "idf_prob: db_docs required");
+ break;
+ case 'f':
+ rs->q_idf_fct=idf_freq;
+ rs->rscheme[5]='f';
+ yaz_log(log_level, "idf_freq: db_docs required");
+ break;
+ case 's':
+ rs->q_idf_fct=idf_squared;
+ rs->rscheme[5]='s';
+ yaz_log(log_level, "idf_squared: db_docs required");
+ break;
+ default: /* 'n' */
+ rs->q_idf_fct=idf_none;
+ rs->rscheme[5]='n';
+ }
+ switch (c6)
+ {
+ case 's':
+ rs->q_norm_fct=norm_sum;
+ rs->rscheme[6]='s';
+ break;
+ case 'c':
+ rs->q_norm_fct=norm_cosine;
+ rs->rscheme[6]='c';
+ break;
+ case 'f':
+ rs->q_norm_fct=norm_fourth;
+ rs->rscheme[6]='f';
+ break;
+ case 'm':
+ rs->q_norm_fct=norm_max;
+ rs->rscheme[6]='m';
+ break;
+ default: /* 'n' */
+ rs->q_norm_fct=norm_none;
+ rs->rscheme[6]='n';
+ }
+ rs->rscheme[7]='\0';
+ rs->sim_fct=sim_cosine;
+ yaz_log(log_level, "zv_scheme %s", rs->rscheme);
}
static void zv_init(RS rs, const char *rscheme) {
- yaz_log(LOG_DEBUG, "zv_init");
+ yaz_log(log_level, "zv_init");
/**/
rs->db_docs=100000; /* assign correct value here */
rs->db_terms=500000; /* assign correct value here (for debugging) */
const char *wscheme;
struct rank_class_info *ci = (struct rank_class_info *)
xmalloc (sizeof(*ci));
- yaz_log(LOG_DEBUG, "zv_create");
+ if (!log_initialized)
+ {
+ log_level=yaz_log_module_level("zvrank");
+ log_initialized=1;
+ }
+
+ yaz_log(log_level, "zv_create");
wscheme=res_get_def(res, "zvrank.weighting-scheme", "");
for (i=0; wscheme[i] && i < 8; i++)
ci->rscheme[i]=wscheme[i];
*/
static void zv_destroy (struct zebra_register *reg, void *class_handle) {
struct rank_class_info *ci = (struct rank_class_info *) class_handle;
- yaz_log(LOG_DEBUG, "zv_destroy");
+ yaz_log(log_level, "zv_destroy");
xfree (ci);
}
int *ip;
zint gocc;
/**/
- yaz_log(LOG_DEBUG, "zv_begin");
+ yaz_log(log_level, "zv_begin");
veclen= numterms;
zv_init(rs, ci->rscheme);
rs->nmem=nmem;
rs->rdoc->veclen=veclen;
rs->rdoc->d_f_max=10; /* just a guess */
rs->rdoc->d_f_max_str="";
- /* yaz_log(LOG_DEBUG, "zv_begin_init"); */
+ /* yaz_log(log_level, "zv_begin_init"); */
for (i = 0; i < rs->veclen; i++)
{
gocc= rset_count(terms[i]->rset);
terms[i]->rankpriv=ip=nmem_malloc(nmem, sizeof(int));
*ip=i; /* save the index for add() */
- /* yaz_log(LOG_DEBUG, "zv_begin_init i=%d gocc=%d", i, gocc); */
+ /* yaz_log(log_level, "zv_begin_init i=%d gocc=%d", i, gocc); */
rs->qdoc->terms[i].gocc=gocc;
rs->qdoc->terms[i].locc=1; /* assume query has no duplicate terms */
rs->rdoc->terms[i].gocc=gocc;
*/
static void zv_end (struct zebra_register *reg, void *rsi)
{
- yaz_log(LOG_DEBUG, "zv_end");
+ yaz_log(log_level, "zv_end");
/* they all are nmem'd */
return;
}
*/
static void zv_add (void *rsi, int seqno, TERMID term) {
RS rs=(RS)rsi;
+ if (!term)
+ {
+ yaz_log(log_level, "zvrank zv_add seqno=%d NULL term",seqno );
+ return;
+ }
int *ip = term->rankpriv;
int i=*ip;
rs->rdoc->terms[i].locc++;
- yaz_log(LOG_DEBUG, "zvrank zv_add seqno=%d '%s' term_index=%d cnt=%d",
+ yaz_log(log_level, "zvrank zv_add seqno=%d '%s' term_index=%d cnt=%d",
seqno, term->name, i, rs->rdoc->terms[i].locc );
}
int score=0;
double dscore=0.0;
RS rs=(RS)rsi;
- /* yaz_log(LOG_DEBUG, "zv_calc"); */
+ /* yaz_log(log_level, "zv_calc"); */
/**/
veclen=rs->veclen;
if (veclen==0)
dscore=rs->sim_fct(rs->qdoc, rs->rdoc);
}
score = (int) (dscore * 1000 +.5);
- yaz_log (LOG_DEBUG, "zv_calc: sysno=" ZINT_FORMAT " score=%d",
+ yaz_log (log_level, "zv_calc: sysno=" ZINT_FORMAT " score=%d",
sysno, score);
if (score > 1000) /* should not happen */
score = 1000;
- return (int) score;
+ /* reset counts for the next record */
+ for (i = 0; i < rs->veclen; i++)
+ rs->rdoc->terms[i].locc=0;
+ return score;
}
/*