2 * Copyright (C) 1994-2000, Index Data
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.31 2001-11-19 23:05:22 adam
8 * Added a few prototypes.
10 * Revision 1.30 2001/10/15 19:53:43 adam
11 * POSIX thread updates. First work on term sets.
13 * Revision 1.29 2001/01/22 10:42:56 adam
14 * Added numerical sort.
16 * Revision 1.28 2000/07/07 12:49:20 adam
17 * Optimized resultSetInsert{Rank,Sort}.
19 * Revision 1.27 2000/04/05 09:49:36 adam
20 * On Unix, zebra/z'mbol uses automake.
22 * Revision 1.26 2000/03/20 19:08:36 adam
23 * Added remote record import using Z39.50 extended services and Segment
26 * Revision 1.25 2000/03/15 15:00:31 adam
27 * First work on threaded version.
29 * Revision 1.24 1999/11/04 15:00:45 adam
30 * Implemented delete result set(s).
32 * Revision 1.23 1999/05/26 07:49:13 adam
35 * Revision 1.22 1999/02/02 14:51:15 adam
36 * Updated WIN32 code specific sections. Changed header.
38 * Revision 1.21 1998/11/16 16:03:46 adam
39 * Moved loggin utilities to Yaz. Was implemented in file zlogs.c.
41 * Revision 1.20 1998/11/16 10:10:53 adam
42 * Fixed problem with zebraPosSetCreate that occurred when positions were
45 * Revision 1.19 1998/09/22 10:48:22 adam
46 * Minor changes in search API.
48 * Revision 1.18 1998/09/22 10:03:45 adam
49 * Changed result sets to be persistent in the sense that they can
50 * be re-searched if needed.
51 * Fixed memory leak in rsm_or.
53 * Revision 1.17 1998/06/23 15:33:36 adam
54 * Added feature to specify sort criteria in query (type 7 specifies
57 * Revision 1.16 1998/05/20 10:12:24 adam
58 * Implemented automatic EXPLAIN database maintenance.
59 * Modified Zebra to work with ASN.1 compiled version of YAZ.
61 * Revision 1.15 1998/03/05 08:45:14 adam
62 * New result set model and modular ranking system. Moved towards
63 * descent server API. System information stored as "SGML" records.
65 * Revision 1.14 1998/02/10 16:39:15 adam
68 * Revision 1.13 1998/02/10 12:03:06 adam
71 * Revision 1.12 1997/09/25 14:57:36 adam
74 * Revision 1.11 1996/12/23 15:30:46 adam
76 * Bug fix: result sets weren't deleted after server shut down.
78 * Revision 1.10 1995/10/30 15:08:08 adam
81 * Revision 1.9 1995/10/17 18:02:14 adam
82 * New feature: databases. Implemented as prefix to words in dictionary.
84 * Revision 1.8 1995/10/10 13:59:25 adam
85 * Function rset_open changed its wflag parameter to general flags.
87 * Revision 1.7 1995/10/06 14:38:01 adam
88 * New result set method: r_score.
89 * Local no (sysno) and score is transferred to retrieveCtrl.
91 * Revision 1.6 1995/09/28 09:19:49 adam
92 * xfree/xmalloc used everywhere.
93 * Extract/retrieve method seems to work for text records.
95 * Revision 1.5 1995/09/27 16:17:32 adam
96 * More work on retrieve.
98 * Revision 1.4 1995/09/07 13:58:36 adam
99 * New parameter: result-set file descriptor (RSFD) to support multiple
100 * positions within the same result-set.
101 * Boolean operators: and, or, not implemented.
102 * Result-set references.
104 * Revision 1.3 1995/09/06 16:11:19 adam
105 * Option: only one word key per file.
107 * Revision 1.2 1995/09/06 10:33:04 adam
108 * More work on present. Some log messages removed.
110 * Revision 1.1 1995/09/05 15:28:40 adam
111 * More work on search engine.
125 #define SORT_IDX_ENTRYSIZE 64
126 #define ZSET_SORT_MAX_LEVEL 3
128 struct zebra_set_term_entry {
143 struct zset_sort_info *sort_info;
144 struct zebra_set_term_entry *term_entries;
145 int term_entries_max;
146 struct zebra_set *next;
150 struct zset_sort_entry {
153 char buf[ZSET_SORT_MAX_LEVEL][SORT_IDX_ENTRYSIZE];
156 struct zset_sort_info {
159 struct zset_sort_entry *all_entries;
160 struct zset_sort_entry **entries;
163 ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output,
164 Z_RPNQuery *rpn, int num_bases, char **basenames,
170 zh->errString = NULL;
173 zebraSet = resultSetAdd (zh, setname, 1);
176 zebraSet->locked = 1;
178 zebraSet->num_bases = num_bases;
179 zebraSet->basenames = basenames;
180 zebraSet->nmem = odr_extract_mem (input);
182 zebraSet->rset = rpn_search (zh, output->mem, rpn,
184 zebraSet->basenames, zebraSet->name,
186 zh->hits = zebraSet->hits;
189 zebraSet->locked = 0;
193 void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type,
194 const char *db, int set,
195 int use, const char *term)
198 s->nmem = nmem_create ();
199 if (!s->term_entries)
202 s->term_entries_max = 1000;
204 nmem_malloc (s->nmem, s->term_entries_max *
205 sizeof(*s->term_entries));
206 for (i = 0; i < s->term_entries_max; i++)
207 s->term_entries[i].term = 0;
209 if (s->hits < s->term_entries_max)
211 s->term_entries[s->hits].reg_type = reg_type;
212 s->term_entries[s->hits].db = nmem_strdup (s->nmem, db);
213 s->term_entries[s->hits].set = set;
214 s->term_entries[s->hits].use = use;
215 s->term_entries[s->hits].term = nmem_strdup (s->nmem, term);
220 ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov)
225 for (s = zh->sets; s; s = s->next)
226 if (!strcmp (s->name, name))
230 logf (LOG_DEBUG, "updating result set %s", name);
231 if (!ov || s->locked)
234 rset_delete (s->rset);
236 nmem_destroy (s->nmem);
240 logf (LOG_DEBUG, "adding result set %s", name);
241 s = (ZebraSet) xmalloc (sizeof(*s));
244 s->name = (char *) xmalloc (strlen(name)+1);
245 strcpy (s->name, name);
247 s->sort_info = (struct zset_sort_info *)
248 xmalloc (sizeof(*s->sort_info));
249 s->sort_info->max_entries = 1000;
250 s->sort_info->entries = (struct zset_sort_entry **)
251 xmalloc (sizeof(*s->sort_info->entries) *
252 s->sort_info->max_entries);
253 s->sort_info->all_entries = (struct zset_sort_entry *)
254 xmalloc (sizeof(*s->sort_info->all_entries) *
255 s->sort_info->max_entries);
256 for (i = 0; i < s->sort_info->max_entries; i++)
257 s->sort_info->entries[i] = s->sort_info->all_entries + i;
268 ZebraSet resultSetGet (ZebraHandle zh, const char *name)
272 for (s = zh->sets; s; s = s->next)
273 if (!strcmp (s->name, name))
275 if (!s->term_entries && !s->rset && s->rpn)
277 NMEM nmem = nmem_create ();
279 rpn_search (zh, nmem, s->rpn, s->num_bases,
280 s->basenames, s->name, s);
288 void resultSetDestroy (ZebraHandle zh, int num, char **names,int *statuses)
290 ZebraSet *ss = &zh->sets;
294 for (i = 0; i<num; i++)
295 statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
297 zh->errString = NULL;
304 for (i = 0; i<num; i++)
305 if (!strcmp (s->name, names[i]))
308 statuses[i] = Z_DeleteStatus_success;
317 xfree (s->sort_info->all_entries);
318 xfree (s->sort_info->entries);
319 xfree (s->sort_info);
322 nmem_destroy (s->nmem);
324 rset_delete (s->rset);
333 ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name,
334 int num, int *positions)
340 struct zset_sort_info *sort_info;
342 if (!(sset = resultSetGet (zh, name)))
344 if (!(rset = sset->rset))
346 if (!sset->term_entries)
348 sr = (ZebraPosSet) xmalloc (sizeof(*sr) * num);
349 for (i = 0; i<num; i++)
351 struct zebra_set_term_entry *entry = sset->term_entries;
358 if (positions[i] <= sset->term_entries_max)
360 sr[i].term = sset->term_entries[positions[i]-1].term;
361 sr[i].db = sset->term_entries[positions[i]-1].db;
367 sr = (ZebraPosSet) xmalloc (sizeof(*sr) * num);
368 for (i = 0; i<num; i++)
375 sort_info = sset->sort_info;
380 for (i = 0; i<num; i++)
382 position = positions[i];
383 if (position > 0 && position <= sort_info->num_entries)
385 logf (LOG_DEBUG, "got pos=%d (sorted)", position);
386 sr[i].sysno = sort_info->entries[position-1]->sysno;
387 sr[i].score = sort_info->entries[position-1]->score;
391 /* did we really get all entries using sort ? */
392 for (i = 0; i<num; i++)
397 if (i < num) /* nope, get the rest, unsorted - sorry */
407 position = sort_info->num_entries;
408 while (num_i < num && positions[num_i] < position)
410 rfd = rset_open (rset, RSETF_READ);
411 while (num_i < num && rset_read (rset, rfd, &key, &term_index))
413 if (key.sysno != psysno)
418 /* determine we alreay have this in our set */
419 for (i = sort_info->num_entries; --i >= 0; )
420 if (psysno == sort_info->entries[i]->sysno)
426 assert (num_i < num);
427 if (position == positions[num_i])
429 sr[num_i].sysno = psysno;
430 logf (LOG_DEBUG, "got pos=%d (unsorted)", position);
431 sr[num_i].score = -1;
436 rset_close (rset, rfd);
442 void zebraPosSetDestroy (ZebraHandle zh, ZebraPosSet records, int num)
453 void resultSetInsertSort (ZebraHandle zh, ZebraSet sset,
454 struct sortKeyInfo *criteria, int num_criteria,
457 struct zset_sort_entry this_entry;
458 struct zset_sort_entry *new_entry = NULL;
459 struct zset_sort_info *sort_info = sset->sort_info;
462 sortIdx_sysno (zh->service->sortIdx, sysno);
463 for (i = 0; i<num_criteria; i++)
465 sortIdx_type (zh->service->sortIdx, criteria[i].attrUse);
466 sortIdx_read (zh->service->sortIdx, this_entry.buf[i]);
468 i = sort_info->num_entries;
472 for (j = 0; j<num_criteria; j++)
474 if (criteria[j].numerical)
476 double diff = atof(this_entry.buf[j]) -
477 atof(sort_info->entries[i]->buf[j]);
486 rel = memcmp (this_entry.buf[j], sort_info->entries[i]->buf[j],
494 if (criteria[j].relation == 'A')
499 else if (criteria[j].relation == 'D')
506 j = sort_info->max_entries;
510 if (sort_info->num_entries == j)
513 j = (sort_info->num_entries)++;
514 new_entry = sort_info->entries[j];
517 sort_info->entries[j] = sort_info->entries[j-1];
520 sort_info->entries[i] = new_entry;
522 for (i = 0; i<num_criteria; i++)
523 memcpy (new_entry->buf[i], this_entry.buf[i], SORT_IDX_ENTRYSIZE);
524 new_entry->sysno = sysno;
525 new_entry->score = -1;
528 void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info,
529 int sysno, int score, int relation)
531 struct zset_sort_entry *new_entry = NULL;
534 i = sort_info->num_entries;
539 rel = score - sort_info->entries[i]->score;
546 else if (relation == 'A')
553 j = sort_info->max_entries;
557 if (sort_info->num_entries == j)
560 j = (sort_info->num_entries)++;
562 new_entry = sort_info->entries[j];
565 sort_info->entries[j] = sort_info->entries[j-1];
568 sort_info->entries[i] = new_entry;
570 new_entry->sysno = sysno;
571 new_entry->score = score;
574 void resultSetSort (ZebraHandle zh, NMEM nmem,
575 int num_input_setnames, const char **input_setnames,
576 const char *output_setname,
577 Z_SortKeySpecList *sort_sequence, int *sort_status)
582 if (num_input_setnames == 0)
587 if (num_input_setnames > 1)
592 logf (LOG_DEBUG, "result set sort input=%s output=%s",
593 *input_setnames, output_setname);
594 sset = resultSetGet (zh, input_setnames[0]);
598 zh->errString = nmem_strdup (nmem, input_setnames[0]);
601 if (!(rset = sset->rset))
604 zh->errString = nmem_strdup (nmem, input_setnames[0]);
607 if (strcmp (output_setname, input_setnames[0]))
609 rset = rset_dup (rset);
610 sset = resultSetAdd (zh, output_setname, 1);
613 resultSetSortSingle (zh, nmem, sset, rset, sort_sequence, sort_status);
616 void resultSetSortSingle (ZebraHandle zh, NMEM nmem,
617 ZebraSet sset, RSET rset,
618 Z_SortKeySpecList *sort_sequence, int *sort_status)
622 struct sortKeyInfo sort_criteria[3];
627 logf (LOG_DEBUG, "resultSetSortSingle start");
628 sset->sort_info->num_entries = 0;
631 num_criteria = sort_sequence->num_specs;
632 if (num_criteria > 3)
634 for (i = 0; i < num_criteria; i++)
636 Z_SortKeySpec *sks = sort_sequence->specs[i];
639 if (*sks->sortRelation == Z_SortRelation_ascending)
640 sort_criteria[i].relation = 'A';
641 else if (*sks->sortRelation == Z_SortRelation_descending)
642 sort_criteria[i].relation = 'D';
648 if (sks->sortElement->which == Z_SortElement_databaseSpecific)
653 else if (sks->sortElement->which != Z_SortElement_generic)
658 sk = sks->sortElement->u.generic;
661 case Z_SortKey_sortField:
662 logf (LOG_DEBUG, "Sort: key %d is of type sortField", i+1);
665 case Z_SortKey_elementSpec:
666 logf (LOG_DEBUG, "Sort: key %d is of type elementSpec", i+1);
669 case Z_SortKey_sortAttributes:
670 logf (LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1);
671 sort_criteria[i].attrUse =
672 zebra_maps_sort (zh->service->zebra_maps,
673 sk->u.sortAttributes,
674 &sort_criteria[i].numerical);
675 logf (LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse);
676 if (sort_criteria[i].attrUse == -1)
681 if (sortIdx_type (zh->service->sortIdx, sort_criteria[i].attrUse))
689 rfd = rset_open (rset, RSETF_READ);
690 while (rset_read (rset, rfd, &key, &term_index))
692 if (key.sysno != psysno)
696 resultSetInsertSort (zh, sset,
697 sort_criteria, num_criteria, psysno);
700 rset_close (rset, rfd);
702 *sort_status = Z_SortStatus_success;
703 logf (LOG_DEBUG, "resultSetSortSingle end");
706 RSET resultSetRef (ZebraHandle zh, Z_ResultSetId *resultSetId)
710 if ((s = resultSetGet (zh, resultSetId)))
715 void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
721 ZebraRankClass rank_class;
722 struct rank_control *rc;
723 struct zset_sort_info *sort_info;
725 sort_info = zebraSet->sort_info;
726 sort_info->num_entries = 0;
728 rfd = rset_open (rset, RSETF_READ);
730 logf (LOG_DEBUG, "resultSetRank");
731 for (i = 0; i < rset->no_rset_terms; i++)
732 logf (LOG_DEBUG, "term=\"%s\" cnt=%d type=%s",
733 rset->rset_terms[i]->name,
734 rset->rset_terms[i]->nn,
735 rset->rset_terms[i]->flags);
737 rank_class = zebraRankLookup (zh, "rank-1");
738 rc = rank_class->control;
740 if (rset_read (rset, rfd, &key, &term_index))
742 int psysno = key.sysno;
745 (*rc->begin) (zh, rank_class->class_handle, rset);
750 if (key.sysno != psysno)
752 score = (*rc->calc) (handle, psysno);
754 resultSetInsertRank (zh, sort_info, psysno, score, 'A');
758 (*rc->add) (handle, key.seqno, term_index);
760 while (rset_read (rset, rfd, &key, &term_index));
761 score = (*rc->calc) (handle, psysno);
762 resultSetInsertRank (zh, sort_info, psysno, score, 'A');
763 (*rc->end) (zh, handle);
765 rset_close (rset, rfd);
766 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, zebraSet->hits);
769 ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name)
771 ZebraRankClass p = zh->service->rank_classes;
772 while (p && strcmp (p->control->name, name))
774 if (p && !p->init_flag)
776 if (p->control->create)
777 p->class_handle = (*p->control->create)(zh->service);
783 void zebraRankInstall (ZebraService zh, struct rank_control *ctrl)
785 ZebraRankClass p = (ZebraRankClass) xmalloc (sizeof(*p));
786 p->control = (struct rank_control *) xmalloc (sizeof(*p->control));
787 memcpy (p->control, ctrl, sizeof(*p->control));
788 p->control->name = xstrdup (ctrl->name);
790 p->next = zh->rank_classes;
791 zh->rank_classes = p;
794 void zebraRankDestroy (ZebraService zh)
796 ZebraRankClass p = zh->rank_classes;
799 ZebraRankClass p_next = p->next;
800 if (p->init_flag && p->control->destroy)
801 (*p->control->destroy)(zh, p->class_handle);
802 xfree (p->control->name);
807 zh->rank_classes = NULL;