From 60ef5f615dd758097e842067a875307df11e4f07 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Fri, 28 Oct 2005 09:22:50 +0000 Subject: [PATCH] Using zebra_rec_keys for per-record key store --- index/extract.c | 453 +++--------------------------------------------------- index/index.h | 36 +---- index/reckeys.c | 183 ++++++++++++++++++++++ index/reckeys.h | 50 ++++++ index/retrieve.c | 9 +- index/zebraapi.c | 25 +-- 6 files changed, 264 insertions(+), 492 deletions(-) create mode 100644 index/reckeys.c create mode 100644 index/reckeys.h diff --git a/index/extract.c b/index/extract.c index 836b6de..06eb1b4 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,4 +1,4 @@ -/* $Id: extract.c,v 1.196 2005-10-28 07:25:30 adam Exp $ +/* $Id: extract.c,v 1.197 2005-10-28 09:22:50 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -99,21 +99,10 @@ static void extract_init (struct recExtractCtrl *p, RecWord *w) } static void searchRecordKey(ZebraHandle zh, -#if NEW_REC_KEYS zebra_rec_keys_t reckeys, -#else - const struct recKeys *reckeys, -#endif int attrSetS, int attrUseS, const char **ws, int ws_length) { -#if NEW_REC_KEYS -#else - void *decode_handle = iscz1_start(); - int off = 0; - int startSeq = -1; - int seqno = 0; -#endif int i; int ch; @@ -125,7 +114,6 @@ static void searchRecordKey(ZebraHandle zh, if (ch < 0) return ; -#if NEW_REC_KEYS if (zebra_rec_keys_rewind(reckeys)) { int startSeq = -1; @@ -151,36 +139,6 @@ static void searchRecordKey(ZebraHandle zh, } } } -#else - while (off < reckeys->buf_used) - { - const char *src = reckeys->buf + off; - struct it_key key; - char *dst = (char*) &key; - - iscz1_decode(decode_handle, &dst, &src); - assert(key.len <= 4 && key.len > 2); - - seqno = (int) key.mem[key.len-1]; - - if (key.mem[0] == ch) - { - int woff; - - if (startSeq == -1) - startSeq = seqno; - woff = seqno - startSeq; - if (woff >= 0 && woff < ws_length) - ws[woff] = src; - } - - while (*src++) - ; - off = src - reckeys->buf; - } - iscz1_stop(decode_handle); - assert (off == reckeys->buf_used); -#endif } struct file_read_info { @@ -249,11 +207,7 @@ static void file_end (void *handle, off_t offset) } static char *fileMatchStr (ZebraHandle zh, -#if NEW_REC_KEYS zebra_rec_keys_t reckeys, -#else - struct recKeys *reckeys, -#endif const char *fname, const char *spec) { static char dstBuf[2048]; /* static here ??? */ @@ -408,15 +362,6 @@ struct recordLogInfo { struct recordGroup *rGroup; }; -#if NEW_REC_KEYS -#else -void create_rec_keys_codec(struct recKeys *keys) -{ - keys->buf_used = 0; - iscz1_reset(keys->codec_handle); -} -#endif - static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl) { int i; @@ -458,19 +403,11 @@ static int file_extract_record(ZebraHandle zh, if (fi->fd != -1) { /* we are going to read from a file, so prepare the extraction */ -#if NEW_REC_KEYS zebra_rec_keys_reset(zh->reg->keys); -#else - create_rec_keys_codec(&zh->reg->keys); -#endif #if NATTR -#if NEW_REC_KEYS zebra_rec_keys_reset(zh->reg->sortKeys); #else - create_rec_keys_codec(&zh->reg->sortKeys); -#endif -#else zh->reg->sortKeys.buf_used = 0; #endif recordOffset = fi->file_moffset; @@ -543,13 +480,8 @@ static int file_extract_record(ZebraHandle zh, if (matchStr == 0 && zh->m_record_id && *zh->m_record_id) { -#if NEW_REC_KEYS matchStr = fileMatchStr (zh, zh->reg->keys, fname, zh->m_record_id); -#else - matchStr = fileMatchStr (zh, &zh->reg->keys, fname, - zh->m_record_id); -#endif if (!matchStr) { yaz_log(YLOG_WARN, "Bad match criteria"); @@ -566,13 +498,7 @@ static int file_extract_record(ZebraHandle zh, } } } - if (! *sysno -#if NEW_REC_KEYS - && zebra_rec_keys_empty(zh->reg->keys) -#else - && zh->reg->keys.buf_used == 0 -#endif - ) + if (! *sysno && zebra_rec_keys_empty(zh->reg->keys) ) { /* the extraction process returned no information - the record is probably empty - unless flagShowRecords is in use */ @@ -610,31 +536,18 @@ static int file_extract_record(ZebraHandle zh, dict_insert (zh->reg->matchDict, matchStr, sizeof(*sysno), sysno); } extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); -#if NEW_REC_KEYS extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, recordAttr->staticrank); -#else - extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, - recordAttr->staticrank); -#endif zh->records_inserted++; } else { /* record already exists */ -#if NEW_REC_KEYS zebra_rec_keys_t delkeys = zebra_rec_keys_open(); -#else - struct recKeys delkeys; -#endif #if NATTR -#if NEW_REC_KEYS zebra_rec_keys_t sortKeys = zebra_rec_keys_open(); #else - struct recKeys sortKeys; -#endif -#else struct sortKeys sortKeys; #endif @@ -643,18 +556,12 @@ static int file_extract_record(ZebraHandle zh, recordAttr = rec_init_attr (zh->reg->zei, rec); -#if NEW_REC_KEYS zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys], rec->size[recInfo_delKeys], 0); -#else - /* flush old keys for sort&search etc. */ - delkeys.buf_used = rec->size[recInfo_delKeys]; - delkeys.buf = rec->info[recInfo_delKeys]; -#endif -#if NEW_REC_KEYS && NATTR +#if NATTR zebra_rec_keys_set_buf(sortKeys, rec->info[recInfo_sortKeys], rec->size[recInfo_sortKeys], @@ -666,23 +573,12 @@ static int file_extract_record(ZebraHandle zh, extract_flushSortKeys (zh, *sysno, 0, &sortKeys); #endif -#if NEW_REC_KEYS extract_flushRecordKeys (zh, *sysno, 0, delkeys, recordAttr->staticrank); /* old values */ -#else - extract_flushRecordKeys (zh, *sysno, 0, &delkeys, - recordAttr->staticrank); /* old values */ -#endif if (deleteFlag) { /* record going to be deleted */ - if ( -#if NEW_REC_KEYS - zebra_rec_keys_empty(delkeys) -#else - !delkeys.buf_used -#endif - ) + if (zebra_rec_keys_empty(delkeys)) { yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); @@ -705,13 +601,7 @@ static int file_extract_record(ZebraHandle zh, else { /* record going to be updated */ - if ( -#if NEW_REC_KEYS - zebra_rec_keys_empty(delkeys) -#else - !delkeys.buf_used -#endif - ) + if (zebra_rec_keys_empty(delkeys)) { yaz_log (YLOG_LOG, "update %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); @@ -725,20 +615,13 @@ static int file_extract_record(ZebraHandle zh, zh->m_record_type, fname, recordOffset); recordAttr->staticrank = extractCtrl.staticrank; extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); -#if NEW_REC_KEYS extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, recordAttr->staticrank); -#else - extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, - recordAttr->staticrank); -#endif zh->records_updated++; } } -#if NEW_REC_KEYS zebra_rec_keys_close(delkeys); -#endif -#if NATTR && NEW_REC_KWYS +#if NATTR zebra_rec_keys_close(sortKeys); #endif } @@ -754,25 +637,11 @@ static int file_extract_record(ZebraHandle zh, /* update delete keys */ xfree (rec->info[recInfo_delKeys]); - if ( -#if NEW_REC_KEYS - !zebra_rec_keys_empty(zh->reg->keys) -#else - zh->reg->keys.buf_used > 0 -#endif - && zh->m_store_keys == 1) + if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1) { -#if NEW_REC_KEYS zebra_rec_keys_get_buf(zh->reg->keys, &rec->info[recInfo_delKeys], &rec->size[recInfo_delKeys]); -#else - - rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; - rec->info[recInfo_delKeys] = zh->reg->keys.buf; - zh->reg->keys.buf = NULL; - zh->reg->keys.buf_max = 0; -#endif } else { @@ -1002,18 +871,11 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, extractCtrl.first_record = 1; extractCtrl.fh = &fc; -#if NEW_REC_KEYS zebra_rec_keys_reset(zh->reg->keys); -#else - create_rec_keys_codec(&zh->reg->keys); -#endif + #if NATTR -#if NEW_REC_KEYS zebra_rec_keys_reset(zh->reg->sortKeys); #else - create_rec_keys_codec(&zh->reg->sortKeys); -#endif -#else zh->reg->sortKeys.buf_used = 0; #endif if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0])) @@ -1088,13 +950,8 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, matchStr = match_criteria; } else { if (zh->m_record_id && *zh->m_record_id) { -#if NEW_REC_KEYS matchStr = fileMatchStr (zh, zh->reg->keys, pr_fname, zh->m_record_id); -#else - matchStr = fileMatchStr (zh, &zh->reg->keys, pr_fname, - zh->m_record_id); -#endif if (!matchStr) { yaz_log (YLOG_WARN, "Bad match criteria (recordID)"); @@ -1111,13 +968,7 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, } } } - if ( -#if NEW_REC_KEYS - zebra_rec_keys_empty(zh->reg->keys) -#else - zh->reg->keys.buf_used == 0 -#endif -) + if (zebra_rec_keys_empty(zh->reg->keys)) { /* the extraction process returned no information - the record is probably empty - unless flagShowRecords is in use */ @@ -1153,32 +1004,19 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, } extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); -#if NEW_REC_KEYS #if 0 print_rec_keys(zh, zh->reg->keys); #endif extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, recordAttr->staticrank); -#else -#if 0 - print_rec_keys(zh, &zh->reg->keys); -#endif - extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, - recordAttr->staticrank); -#endif - zh->records_inserted++; } else { /* record already exists */ -#if NEW_REC_KEYS zebra_rec_keys_t delkeys = zebra_rec_keys_open(); -#else - struct recKeys delkeys; -#endif #if NATTR - struct recKeys sortKeys; + zebra_rec_keys_t sortKeys = zebra_rec_keys_open(); #else struct sortKeys sortKeys; #endif @@ -1197,40 +1035,24 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, recordAttr = rec_init_attr (zh->reg->zei, rec); -#if NEW_REC_KEYS zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys], rec->size[recInfo_delKeys], 0); -#else - delkeys.buf_used = rec->size[recInfo_delKeys]; - delkeys.buf = rec->info[recInfo_delKeys]; -#endif sortKeys.buf_used = rec->size[recInfo_sortKeys]; sortKeys.buf = rec->info[recInfo_sortKeys]; -#if NEW_REC_KEYS && NATTR +#if NATTR extract_flushSortKeys (zh, *sysno, 0, sortKeys); #else extract_flushSortKeys (zh, *sysno, 0, &sortKeys); #endif -#if NEW_REC_KEYS extract_flushRecordKeys (zh, *sysno, 0, delkeys, recordAttr->staticrank); -#else - extract_flushRecordKeys (zh, *sysno, 0, &delkeys, - recordAttr->staticrank); -#endif if (delete_flag) { /* record going to be deleted */ - if ( -#if NEW_REC_KEYS - zebra_rec_keys_empty(delkeys) -#else - !delkeys.buf_used -#endif - ) + if (zebra_rec_keys_empty(delkeys)) { if (show_progress) { @@ -1257,13 +1079,7 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, else { /* record going to be updated */ - if ( -#if NEW_REC_KEYS - zebra_rec_keys_empty(delkeys) -#else - !delkeys.buf_used -#endif - ) + if (zebra_rec_keys_empty(delkeys)) { if (show_progress) { @@ -1279,20 +1095,13 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, pr_fname, (long) recordOffset); recordAttr->staticrank = extractCtrl.staticrank; extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); -#if NEW_REC_KEYS extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, recordAttr->staticrank); -#else - extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, - recordAttr->staticrank); -#endif zh->records_updated++; } } -#if NEW_REC_KEYS zebra_rec_keys_close(delkeys); -#endif -#if NEW_REC_KEYS && NATTR +#if NATTR zebra_rec_keys_close(sortKeys); #endif } @@ -1308,25 +1117,11 @@ ZEBRA_RES buffer_extract_record(ZebraHandle zh, /* update delete keys */ xfree (rec->info[recInfo_delKeys]); - if ( -#if NEW_REC_KEYS - !zebra_rec_keys_empty(zh->reg->keys) -#else - zh->reg->keys.buf_used > 0 -#endif - && zh->m_store_keys == 1) + if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1) { -#if NEW_REC_KEYS zebra_rec_keys_get_buf(zh->reg->keys, &rec->info[recInfo_delKeys], &rec->size[recInfo_delKeys]); -#else - - rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; - rec->info[recInfo_delKeys] = zh->reg->keys.buf; - zh->reg->keys.buf = NULL; - zh->reg->keys.buf_max = 0; -#endif } else { @@ -1408,19 +1203,11 @@ int explain_extract (void *handle, Record rec, data1_node *n) abort (); } -#if NEW_REC_KEYS zebra_rec_keys_reset(zh->reg->keys); -#else - create_rec_keys_codec(&zh->reg->keys); -#endif #if NATTR -#if NEW_REC_KEYS zebra_rec_keys_reset(zh->reg->sortKeys); #else - create_rec_keys_codec(&zh->reg->sortKeys); -#endif -#else zh->reg->sortKeys.buf_used = 0; #endif extractCtrl.init = extract_init; @@ -1443,34 +1230,20 @@ int explain_extract (void *handle, Record rec, data1_node *n) if (rec->size[recInfo_delKeys]) { -#if NEW_REC_KEYS zebra_rec_keys_t delkeys = zebra_rec_keys_open(); -#else - struct recKeys delkeys; -#endif - + #if NATTR -#if NEW_REC_KEYS zebra_rec_keys_t sortkeys = zzebra_rec_keys_open(); #else - struct recKeys sortkeys; -#endif -#else struct sortKeys sortkeys; #endif -#if NEW_REC_KEYS zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys], rec->size[recInfo_delKeys], 0); extract_flushRecordKeys (zh, rec->sysno, 0, delkeys, 0); zebra_rec_keys_close(delkeys); -#else - delkeys.buf_used = rec->size[recInfo_delKeys]; - delkeys.buf = rec->info[recInfo_delKeys]; - extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys, 0); -#endif -#if NATTR && NEW_REC_KEYS +#if NATTR zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys], rec->size[recInfo_sortKeys], 0); @@ -1483,28 +1256,17 @@ int explain_extract (void *handle, Record rec, data1_node *n) extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys); #endif } -#if NEW_REC_KEYS extract_flushRecordKeys (zh, rec->sysno, 1, zh->reg->keys, 0); -#else - extract_flushRecordKeys (zh, rec->sysno, 1, &zh->reg->keys, 0); -#endif -#if NATTR && NEW_REC_KEYS +#if NATTR extract_flushSortKeys (zh, rec->sysno, 1, zh->reg->sortKeys); #else extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys); #endif xfree (rec->info[recInfo_delKeys]); -#if NEW_REC_KEYS zebra_rec_keys_get_buf(zh->reg->keys, &rec->info[recInfo_delKeys], &rec->size[recInfo_delKeys]); -#else - rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; - rec->info[recInfo_delKeys] = zh->reg->keys.buf; - zh->reg->keys.buf = NULL; - zh->reg->keys.buf_max = 0; -#endif xfree (rec->info[recInfo_sortKeys]); rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used; @@ -1517,19 +1279,9 @@ int explain_extract (void *handle, Record rec, data1_node *n) void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, int cmd, -#if NEW_REC_KEYS zebra_rec_keys_t reckeys, -#else - struct recKeys *reckeys, -#endif zint staticrank) { -#if NEW_REC_KEYS -#else - void *decode_handle = iscz1_start(); - int off = 0; - int ch = 0; -#endif ZebraExplainInfo zei = zh->reg->zei; if (!zh->reg->key_buf) @@ -1550,7 +1302,6 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, } zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1); -#if NEW_REC_KEYS if (zebra_rec_keys_rewind(reckeys)) { size_t slen; @@ -1608,60 +1359,6 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, (zh->reg->key_buf_used) += sizeof(key_out); } } -#else - while (off < reckeys->buf_used) - { - const char *src = reckeys->buf + off; - struct it_key key_in; - struct it_key key_out; - char *dst = (char*) &key_in; - zint *keyp = key_out.mem; - - iscz1_decode(decode_handle, &dst, &src); - assert(key_in.len == 4); - - if (zh->reg->key_buf_used + 1024 > - (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*)) - extract_flushWriteKeys(zh, 0); - ++(zh->reg->ptr_i); - assert(zh->reg->ptr_i > 0); - (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] = - (char*)zh->reg->key_buf + zh->reg->key_buf_used; - - ch = (int) key_in.mem[0]; /* ordinal for field/use/attribute */ - - zh->reg->key_buf_used += - key_SU_encode(ch, ((char*)zh->reg->key_buf) + - zh->reg->key_buf_used); - while (*src) - ((char*)zh->reg->key_buf) [(zh->reg->key_buf_used)++] = *src++; - src++; - ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = '\0'; - ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = cmd; - - if (zh->m_staticrank) /* rank config enabled ? */ - { - *keyp++ = staticrank; - key_out.len = 4; - } - else - key_out.len = 3; - - if (key_in.mem[1]) /* filter specified record ID */ - *keyp++ = key_in.mem[1]; - else - *keyp++ = sysno; - *keyp++ = key_in.mem[2]; /* section_id */ - *keyp++ = key_in.mem[3]; /* sequence .. */ - - memcpy ((char*)zh->reg->key_buf + zh->reg->key_buf_used, - &key_out, sizeof(key_out)); - (zh->reg->key_buf_used) += sizeof(key_out); - off = src - reckeys->buf; - } - assert (off == reckeys->buf_used); - iscz1_stop(decode_handle); -#endif } void extract_flushWriteKeys (ZebraHandle zh, int final) @@ -1798,7 +1495,6 @@ void extract_flushWriteKeys (ZebraHandle zh, int final) zh->reg->key_buf_used = 0; } -#if NEW_REC_KEYS void extract_add_it_key (ZebraHandle zh, zebra_rec_keys_t *keys, int reg_type, @@ -1806,48 +1502,12 @@ void extract_add_it_key (ZebraHandle zh, { zebra_rec_keys_write(*keys, reg_type, str, slen, key); } -#else -void extract_add_it_key (ZebraHandle zh, - struct recKeys *keys, - int reg_type, - const char *str, int slen, struct it_key *key) -{ - char *dst; - const char *src = (char*) key; - - if (keys->buf_used+1024 > keys->buf_max) - { - char *b = (char *) xmalloc (keys->buf_max += 128000); - if (keys->buf_used > 0) - memcpy (b, keys->buf, keys->buf_used); - xfree (keys->buf); - keys->buf = b; - } - dst = keys->buf + keys->buf_used; - - iscz1_encode(keys->codec_handle, &dst, &src); - -#if REG_TYPE_PREFIX - *dst++ = reg_type; -#endif - memcpy (dst, str, slen); - dst += slen; - *dst++ = '\0'; - keys->buf_used = dst - keys->buf; -} -#endif ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, -#if NEW_REC_KEYS zebra_rec_keys_t reckeys, -#else - struct recKeys *reckeys, -#endif zebra_snippets *snippets) { NMEM nmem = nmem_create(); - -#if NEW_REC_KEYS if (zebra_rec_keys_rewind(reckeys)) { const char *str; @@ -1872,52 +1532,12 @@ ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, nmem_reset(nmem); } } -#else - int off = 0; - int seqno = 0; - void *decode_handle = iscz1_start(); - assert(reckeys->buf); - while (off < reckeys->buf_used) - { - const char *src = reckeys->buf + off; - struct it_key key; - char *dst = (char*) &key; - char dst_buf[IT_MAX_WORD]; - char *dst_term = dst_buf; - int index_type = 0, ord; - - iscz1_decode(decode_handle, &dst, &src); - assert(key.len <= 4 && key.len > 2); - - seqno = (int) key.mem[key.len-1]; - ord = key.mem[0]; - - zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, - 0/* db */, 0/* set */, 0/* use */); - assert(index_type); - zebra_term_untrans_iconv(zh, nmem, index_type, - &dst_term, src); - zebra_snippets_append(snippets, seqno, ord, dst_term); - while (*src++) - ; - off = src - reckeys->buf; - nmem_reset(nmem); - } - iscz1_stop(decode_handle); -#endif nmem_destroy(nmem); return ZEBRA_OK; } -void print_rec_keys(ZebraHandle zh, -#if NEW_REC_KEYS - zebra_rec_keys_t reckeys -#else - struct recKeys *reckeys -#endif -) +void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys) { -#if NEW_REC_KEYS yaz_log(YLOG_LOG, "print_rec_keys"); if (zebra_rec_keys_rewind(reckeys)) { @@ -1943,40 +1563,6 @@ void print_rec_keys(ZebraHandle zh, key.mem[0], seqno, dst_buf); } } -#else - int off = 0; - int seqno = 0; - void *decode_handle = iscz1_start(); - yaz_log(YLOG_LOG, "print_rec_keys buf=%p sz=%d", reckeys->buf, - reckeys->buf_used); - assert(reckeys->buf); - while (off < reckeys->buf_used) - { - const char *src = reckeys->buf + off; - struct it_key key; - char *dst = (char*) &key; - char dst_buf[IT_MAX_WORD]; - int index_type; - const char *db = 0; - - iscz1_decode(decode_handle, &dst, &src); - assert(key.len <= 4 && key.len > 2); - - seqno = (int) key.mem[key.len-1]; - - zebraExplain_lookup_ord(zh->reg->zei, - key.mem[0], &index_type, &db, 0, 0); - - zebra_term_untrans(zh, index_type, dst_buf, src); - - yaz_log(YLOG_LOG, "ord=" ZINT_FORMAT " seqno=%d term=%s", - key.mem[0], seqno, dst_buf); - while (*src++) - ; - off = src - reckeys->buf; - } - iscz1_stop(decode_handle); -#endif } void extract_add_index_string (RecWord *p, const char *str, int length) @@ -2265,6 +1851,7 @@ void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid) } #if NATTR +#error not done yet with zebra_rec_keys_t void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, int cmd, struct recKeys *reckeys) { diff --git a/index/index.h b/index/index.h index 5f15228..7bc5c3c 100644 --- a/index/index.h +++ b/index/index.h @@ -1,4 +1,4 @@ -/* $Id: index.h,v 1.153 2005-10-28 07:25:30 adam Exp $ +/* $Id: index.h,v 1.154 2005-10-28 09:22:50 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -192,18 +192,7 @@ typedef struct zebra_rank_class { struct zebra_rank_class *next; } *ZebraRankClass; -#define NEW_REC_KEYS 1 - -#if NEW_REC_KEYS #include "reckeys.h" -#else -struct recKeys { - int buf_used; - int buf_max; - char *buf; - void *codec_handle; -}; -#endif #if NATTR @@ -240,14 +229,10 @@ struct zebra_register { int stop_flag; int active; /* 0=shutdown, 1=enabled and inactive, 2=activated */ -#if NEW_REC_KEYS zebra_rec_keys_t keys; -#else - struct recKeys keys; -#endif #if NATTR - struct recKeys sortKeys; + zebra_rec_keys_t sortKeys; #else struct sortKeys sortKeys; #endif @@ -487,34 +472,19 @@ off_t zebra_record_int_tell (void *fh); int zebra_record_int_read (void *fh, char *buf, size_t count); void zebra_record_int_end (void *fh, off_t offset); -#if NEW_REC_KEYS void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys); -#else -void print_rec_keys(ZebraHandle zh, struct recKeys *reckeys); -#endif -#if NEW_REC_KEYS ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys, zebra_snippets *snippets); -#else -ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, struct recKeys *reckeys, - zebra_snippets *snippets); -#endif ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname, zint sysno, zebra_snippets *snippets); -#if NEW_REC_KEYS void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, int cmd, zebra_rec_keys_t reckeys, zint staticrank); -#else -void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, - int cmd, struct recKeys *reckeys, - zint staticrank); -#endif #if NATTR void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, - int cmd, struct recKeys *skp); + int cmd, zebra_rec_keys_t skp); #else void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, int cmd, struct sortKeys *skp); diff --git a/index/reckeys.c b/index/reckeys.c new file mode 100644 index 0000000..4e23c93 --- /dev/null +++ b/index/reckeys.c @@ -0,0 +1,183 @@ +/* $Id: reckeys.c,v 1.1 2005-10-28 09:22:50 adam Exp $ + Copyright (C) 1995-2005 + Index Data ApS + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Zebra; see the file LICENSE.zebra. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. +*/ + +#include +#include +#include +#include + +#include "index.h" +#include "reckeys.h" + +struct zebra_rec_keys_t_ { + size_t buf_used; + size_t buf_max; + size_t fetch_offset; + char *buf; + void *encode_handle; + void *decode_handle; + char owner_of_buffer; +}; + +zebra_rec_keys_t zebra_rec_keys_open() +{ + zebra_rec_keys_t p = xmalloc(sizeof(*p)); + p->buf_used = 0; + p->buf_max = 0; + p->fetch_offset = 0; + p->buf = 0; + p->owner_of_buffer = 1; + p->encode_handle = iscz1_start(); + p->decode_handle = iscz1_start(); + return p; +} + +void zebra_rec_keys_set_buf(zebra_rec_keys_t p, char *buf, size_t sz, + int owner) +{ + if (p->owner_of_buffer) + xfree(p->buf); + p->buf_used = sz; + p->buf_max = sz; + if (!owner) + { + p->buf = buf; + } + else + { + if (!sz) + p->buf = 0; + else + { + p->buf = xmalloc(sz); + memcpy(p->buf, buf, sz); + } + } + p->owner_of_buffer = owner; +} + +void zebra_rec_keys_get_buf(zebra_rec_keys_t p, char **buf, size_t *sz) +{ + *buf = p->buf; + *sz = p->buf_used; + + p->buf = 0; + p->buf_max = 0; + p->buf_used = 0; +} + +void zebra_rec_keys_close(zebra_rec_keys_t p) +{ + if (!p) + return; + + if (p->owner_of_buffer) + xfree(p->buf); + if (p->encode_handle) + iscz1_stop(p->encode_handle); + if (p->decode_handle) + iscz1_stop(p->decode_handle); + xfree(p); +} + +void zebra_rec_keys_write(zebra_rec_keys_t keys, + int reg_type, + const char *str, size_t slen, + const struct it_key *key) +{ + char *dst; + const char *src = (char*) key; + + assert(keys->owner_of_buffer); + + if (keys->buf_used+1024 > keys->buf_max) + { + char *b = (char *) xmalloc (keys->buf_max += 128000); + if (keys->buf_used > 0) + memcpy (b, keys->buf, keys->buf_used); + xfree (keys->buf); + keys->buf = b; + } + dst = keys->buf + keys->buf_used; + + iscz1_encode(keys->encode_handle, &dst, &src); + +#if REG_TYPE_PREFIX + *dst++ = reg_type; +#endif + memcpy (dst, str, slen); + dst += slen; + *dst++ = '\0'; + keys->buf_used = dst - keys->buf; +} + +void zebra_rec_keys_reset(zebra_rec_keys_t keys) +{ + assert(keys); + keys->buf_used = 0; + + iscz1_reset(keys->encode_handle); +} + +int zebra_rec_keys_rewind(zebra_rec_keys_t keys) +{ + assert(keys); + iscz1_reset(keys->decode_handle); + keys->fetch_offset = 0; + if (keys->buf_used == 0) + return 0; + return 1; +} + +int zebra_rec_keys_empty(zebra_rec_keys_t keys) +{ + if (keys->buf_used == 0) + return 1; + return 0; +} + +int zebra_rec_keys_read(zebra_rec_keys_t keys, + const char **str, size_t *slen, + struct it_key *key) +{ + assert(keys); + if (keys->fetch_offset == keys->buf_used) + return 0; + else + { + const char *src = keys->buf + keys->fetch_offset; + char *dst = (char*) key; + + assert (keys->fetch_offset < keys->buf_used); + + /* store the destination key */ + iscz1_decode(keys->decode_handle, &dst, &src); + + /* store pointer to string and length of it */ + *str = src; + *slen = strlen(src); + src += *slen + 1; + + keys->fetch_offset = src - keys->buf; + } + return 1; +} diff --git a/index/reckeys.h b/index/reckeys.h new file mode 100644 index 0000000..8f7bbee --- /dev/null +++ b/index/reckeys.h @@ -0,0 +1,50 @@ +/* $Id: reckeys.h,v 1.1 2005-10-28 09:22:50 adam Exp $ + Copyright (C) 1995-2005 + Index Data ApS + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Zebra; see the file LICENSE.zebra. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. +*/ + +#ifndef RECKEYS_H +#define RECKEYS_H + +typedef struct zebra_rec_keys_t_ *zebra_rec_keys_t; + +zebra_rec_keys_t zebra_rec_keys_open(); + +void zebra_rec_keys_close(zebra_rec_keys_t p); + +void zebra_rec_keys_write(zebra_rec_keys_t keys, + int reg_type, + const char *str, size_t slen, + const struct it_key *key); +void zebra_rec_keys_reset(zebra_rec_keys_t keys); + +int zebra_rec_keys_read(zebra_rec_keys_t keys, + const char **str, size_t *slen, + struct it_key *key); +int zebra_rec_keys_rewind(zebra_rec_keys_t keys); + +int zebra_rec_keys_empty(zebra_rec_keys_t keys); + +void zebra_rec_keys_get_buf(zebra_rec_keys_t p, char **buf, size_t *sz); + +void zebra_rec_keys_set_buf(zebra_rec_keys_t p, char *buf, size_t sz, + int owner); + +#endif diff --git a/index/retrieve.c b/index/retrieve.c index d60a989..8c661bd 100644 --- a/index/retrieve.c +++ b/index/retrieve.c @@ -1,4 +1,4 @@ -/* $Id: retrieve.c,v 1.36 2005-10-28 07:25:30 adam Exp $ +/* $Id: retrieve.c,v 1.37 2005-10-28 09:22:50 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -221,7 +221,6 @@ int zebra_record_fetch (ZebraHandle zh, SYSNO sysno, int score, /* snippets code */ zebra_snippets *snippet; -#if NEW_REC_KEYS zebra_rec_keys_t reckeys = zebra_rec_keys_open(); zebra_rec_keys_set_buf(reckeys, @@ -230,12 +229,6 @@ int zebra_record_fetch (ZebraHandle zh, SYSNO sysno, int score, 0); zebra_snippets_rec_keys(zh, reckeys, retrieveCtrl.doc_snippet); zebra_rec_keys_close(reckeys); -#else - struct recKeys reckeys; - reckeys.buf = rec->info[recInfo_delKeys]; - reckeys.buf_used = rec->size[recInfo_delKeys]; - zebra_snippets_rec_keys(zh, &reckeys, retrieveCtrl.doc_snippet); -#endif #if 0 diff --git a/index/zebraapi.c b/index/zebraapi.c index d419085..cc7fd51 100644 --- a/index/zebraapi.c +++ b/index/zebraapi.c @@ -1,4 +1,4 @@ -/* $Id: zebraapi.c,v 1.192 2005-10-28 07:25:30 adam Exp $ +/* $Id: zebraapi.c,v 1.193 2005-10-28 09:22:50 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -326,18 +326,13 @@ struct zebra_register *zebra_register_open(ZebraService zs, const char *name, reg->key_buf = 0; -#if NEW_REC_KEYS reg->keys = zebra_rec_keys_open(); -#else - reg->keys.buf_max = 0; - reg->keys.buf = 0; - reg->keys.codec_handle = iscz1_start(); -#endif +#if NATTR + reg->sortKeys = zebra_rec_keys_open(); +#else reg->sortKeys.buf = 0; reg->sortKeys.buf_max = 0; -#if NATTR - reg->sortKeys.codec_handle = iscz1_start(); #endif reg->records = 0; @@ -500,17 +495,11 @@ static void zebra_register_close (ZebraService zs, struct zebra_register *reg) bfs_destroy (reg->bfs); data1_destroy (reg->dh); - xfree(reg->sortKeys.buf); -#if NEW_REC_KEYS zebra_rec_keys_close(reg->keys); -#else - xfree(reg->keys.buf); - if (reg->keys.codec_handle) - iscz1_stop(reg->keys.codec_handle); #if NATTR - if (reg->sortKeys.codec_handle) - iscz1_stop(reg->sortKeys.codec_handle); -#endif + zebra_rec_keys_close(rec->sortKeys); +#else + xfree(reg->sortKeys.buf); #endif xfree(reg->key_buf); -- 1.7.10.4