1 /* This file is part of Pazpar2.
2 Copyright (C) 2006-2009 Index Data
4 Pazpar2 is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
27 #include "relevance.h"
32 int *doc_frequency_vec;
34 struct word_entry *entries;
43 struct word_entry *next;
46 static void add_word_entry(NMEM nmem,
47 struct word_entry **entries,
51 struct word_entry *ne = nmem_malloc(nmem, sizeof(*ne));
52 ne->norm_str = nmem_strdup(nmem, norm_str);
60 int word_entry_match(struct word_entry *entries, const char *norm_str)
62 for (; entries; entries = entries->next)
64 if (!strcmp(norm_str, entries->norm_str))
65 return entries->termno;
70 static struct word_entry *build_word_entries(pp2_charset_t pct, NMEM nmem,
73 int termno = 1; /* >0 signals THERE is an entry */
74 struct word_entry *entries = 0;
75 const char **p = terms;
79 pp2_relevance_token_t prt = pp2_relevance_tokenize(pct, *p);
82 while ((norm_str = pp2_relevance_token_next(prt)))
83 add_word_entry(nmem, &entries, norm_str, termno);
85 pp2_relevance_token_destroy(prt);
92 void relevance_countwords(struct relevance *r, struct record_cluster *cluster,
93 const char *words, int multiplier)
95 pp2_relevance_token_t prt = pp2_relevance_tokenize(r->pct, words);
99 while ((norm_str = pp2_relevance_token_next(prt)))
101 int res = word_entry_match(r->entries, norm_str);
103 cluster->term_frequency_vec[res] += multiplier;
104 cluster->term_frequency_vec[0]++;
106 pp2_relevance_token_destroy(prt);
109 struct relevance *relevance_create(pp2_charset_t pct,
110 NMEM nmem, const char **terms)
112 struct relevance *res = nmem_malloc(nmem, sizeof(struct relevance));
116 for (p = terms, i = 0; *p; p++, i++)
119 res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int));
120 memset(res->doc_frequency_vec, 0, res->vec_len * sizeof(int));
122 res->entries = build_word_entries(pct, nmem, terms);
127 void relevance_newrec(struct relevance *r, struct record_cluster *rec)
129 if (!rec->term_frequency_vec)
131 rec->term_frequency_vec = nmem_malloc(r->nmem, r->vec_len * sizeof(int));
132 memset(rec->term_frequency_vec, 0, r->vec_len * sizeof(int));
137 void relevance_donerecord(struct relevance *r, struct record_cluster *cluster)
141 for (i = 1; i < r->vec_len; i++)
142 if (cluster->term_frequency_vec[i] > 0)
143 r->doc_frequency_vec[i]++;
145 r->doc_frequency_vec[0]++;
148 // Prepare for a relevance-sorted read
149 void relevance_prepare_read(struct relevance *rel, struct reclist *reclist)
152 float *idfvec = xmalloc(rel->vec_len * sizeof(float));
154 reclist_rewind(reclist);
155 // Calculate document frequency vector for each term.
156 for (i = 1; i < rel->vec_len; i++)
158 if (!rel->doc_frequency_vec[i])
162 // This conditional may be terribly wrong
163 // It was there to address the situation where vec[0] == vec[i]
164 // which leads to idfvec[i] == 0... not sure about this
165 // Traditional TF-IDF may assume that a word that occurs in every
166 // record is irrelevant, but this is actually something we will
168 if ((idfvec[i] = log((float) rel->doc_frequency_vec[0] /
169 rel->doc_frequency_vec[i])) < 0.0000001)
173 // Calculate relevance for each document
179 struct record_cluster *rec = reclist_read_record(reclist);
182 for (t = 1; t < rel->vec_len; t++)
185 if (!rec->term_frequency_vec[0])
187 termfreq = (float) rec->term_frequency_vec[t] / rec->term_frequency_vec[0];
188 relevance += 100000 * (termfreq * idfvec[t] + 0.0000005);
190 rec->relevance = relevance;
192 reclist_rewind(reclist);
199 * c-file-style: "Stroustrup"
200 * indent-tabs-mode: nil
202 * vim: shiftwidth=4 tabstop=8 expandtab