Fix leak for rset_key_control

diff --git a/index/Makefile.am b/index/Makefile.am
index 3e1d7b6..5f78bd8 100644 (file)
--- a/index/Makefile.am
+++ b/index/Makefile.am
@@ -1,4 +1,4 @@
-## $Id: Makefile.am,v 1.33 2005-09-15 09:27:18 pop Exp $
+## $Id: Makefile.am,v 1.34 2005-10-28 07:25:30 adam Exp $
 
 noinst_PROGRAMS = apitest kdump
 
@@ -9,7 +9,7 @@ libidzebra_api_la_SOURCES = dir.c dirs.c trav.c kinput.c kcompare.c \
  zebraapi.c api_swig.c \
  zinfo.c invstat.c sortidx.c compact.c zsets.c zrpn.c \
  rank1.c trunc.c retrieve.c extract.c rankstatic.c \
- index.h recindex.h recindxp.h \
+ index.h recindex.h recindxp.h reckeys.c reckeys.h \
  zinfo.h zserver.h zvrank.c limit.c kcontrol.c
 
 bin_PROGRAMS = zebraidx zebrasrv zebrash

diff --git a/index/extract.c b/index/extract.c
index dcd4206..836b6de 100644 (file)
--- a/index/extract.c
+++ b/index/extract.c
@@ -1,4 +1,4 @@
-/* $Id: extract.c,v 1.195 2005-09-16 09:58:38 adam Exp $
+/* $Id: extract.c,v 1.196 2005-10-28 07:25:30 adam Exp $
    Copyright (C) 1995-2005
    Index Data ApS
 
@@ -43,8 +43,6 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 
 #define USE_SHELLSORT 0
 
-#define REC_MULTI_SKIP 0
-
 #if USE_SHELLSORT
 static void shellsort(void *ar, int r, size_t s,
                       int (*cmp)(const void *a, const void *b))
@@ -101,14 +99,21 @@ static void extract_init (struct recExtractCtrl *p, RecWord *w)
 }
 
 static void searchRecordKey(ZebraHandle zh,
+#if NEW_REC_KEYS
+                           zebra_rec_keys_t reckeys,
+#else
                            const struct recKeys *reckeys,
+#endif
                            int attrSetS, int attrUseS,
                            const char **ws, int ws_length)
 {
+#if NEW_REC_KEYS
+#else
     void *decode_handle = iscz1_start();
     int off = 0;
     int startSeq = -1;
     int seqno = 0;
+#endif
     int i;
     int ch;
 
@@ -120,6 +125,33 @@ static void searchRecordKey(ZebraHandle zh,
     if (ch < 0)
        return ;
 
+#if NEW_REC_KEYS
+    if (zebra_rec_keys_rewind(reckeys))
+    {
+       int startSeq = -1;
+       const char *str;
+       size_t slen;
+       struct it_key key;
+       zint seqno;
+       while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
+       {
+           assert(key.len <= 4 && key.len > 2);
+
+           seqno = key.mem[key.len-1];
+           
+           if (key.mem[0] == ch)
+           {
+               int woff;
+               
+               if (startSeq == -1)
+                   startSeq = seqno;
+               woff = seqno - startSeq;
+               if (woff >= 0 && woff < ws_length)
+                   ws[woff] = str;
+           }
+       }
+    }
+#else
     while (off < reckeys->buf_used)
     {
         const char *src = reckeys->buf + off;
@@ -148,6 +180,7 @@ static void searchRecordKey(ZebraHandle zh,
     }
     iscz1_stop(decode_handle);
     assert (off == reckeys->buf_used);
+#endif
 }
 
 struct file_read_info {
@@ -216,7 +249,11 @@ static void file_end (void *handle, off_t offset)
 }
 
 static char *fileMatchStr (ZebraHandle zh,
+#if NEW_REC_KEYS
+                          zebra_rec_keys_t reckeys,
+#else
                            struct recKeys *reckeys,
+#endif
                            const char *fname, const char *spec)
 {
     static char dstBuf[2048];      /* static here ??? */
@@ -371,11 +408,14 @@ struct recordLogInfo {
     struct recordGroup *rGroup;
 };
 
+#if NEW_REC_KEYS
+#else
 void create_rec_keys_codec(struct recKeys *keys)
 {
     keys->buf_used = 0;
     iscz1_reset(keys->codec_handle);
 }
+#endif
 
 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
 {
@@ -418,9 +458,18 @@ static int file_extract_record(ZebraHandle zh,
     if (fi->fd != -1)
     {
         /* we are going to read from a file, so prepare the extraction */
+#if NEW_REC_KEYS
+       zebra_rec_keys_reset(zh->reg->keys);
+#else
        create_rec_keys_codec(&zh->reg->keys);
+#endif
+
 #if NATTR
+#if NEW_REC_KEYS
+       zebra_rec_keys_reset(zh->reg->sortKeys);
+#else
        create_rec_keys_codec(&zh->reg->sortKeys);
+#endif
 #else
        zh->reg->sortKeys.buf_used = 0;
 #endif
@@ -494,9 +543,13 @@ static int file_extract_record(ZebraHandle zh,
 
         if (matchStr == 0 && zh->m_record_id && *zh->m_record_id)
         {
-        
+#if NEW_REC_KEYS
+            matchStr = fileMatchStr (zh, zh->reg->keys, fname, 
+                                    zh->m_record_id);
+#else        
             matchStr = fileMatchStr (zh, &zh->reg->keys, fname, 
                                     zh->m_record_id);
+#endif
            if (!matchStr)
            {
                yaz_log(YLOG_WARN, "Bad match criteria");
@@ -513,7 +566,13 @@ static int file_extract_record(ZebraHandle zh,
            }
        }
     }
-    if (! *sysno && zh->reg->keys.buf_used == 0)
+    if (! *sysno 
+#if NEW_REC_KEYS
+       && zebra_rec_keys_empty(zh->reg->keys)
+#else
+       && zh->reg->keys.buf_used == 0
+#endif
+       )
     {
          /* the extraction process returned no information - the record
             is probably empty - unless flagShowRecords is in use */
@@ -551,17 +610,30 @@ static int file_extract_record(ZebraHandle zh,
             dict_insert (zh->reg->matchDict, matchStr, sizeof(*sysno), sysno);
         }
        extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
+#if NEW_REC_KEYS
+        extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys,
+                                recordAttr->staticrank);
+#else
         extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys,
                                 recordAttr->staticrank);
-
+#endif
         zh->records_inserted++;
     }
     else
     {
         /* record already exists */
+#if NEW_REC_KEYS
+       zebra_rec_keys_t delkeys = zebra_rec_keys_open();
+
+#else
         struct recKeys delkeys;
+#endif
 #if NATTR
+#if NEW_REC_KEYS
+       zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
+#else
        struct recKeys sortKeys;
+#endif
 #else
         struct sortKeys sortKeys;
 #endif
@@ -571,44 +643,56 @@ static int file_extract_record(ZebraHandle zh,
        
        recordAttr = rec_init_attr (zh->reg->zei, rec);
 
-#if REC_MULTI_SKIP
-       if (!force_update && recordAttr->runNumber ==
-            zebraExplain_runNumberIncrement (zh->reg->zei, 0))
-       {
-            yaz_log (YLOG_LOG, "run number = " ZINT_FORMAT,
-                           recordAttr->runNumber);
-           yaz_log (YLOG_LOG, "skipped %s %s " PRINTF_OFF_T,
-                     zh->m_record_type, fname, recordOffset);
-           extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys);
-           rec_rm (&rec);
-           logRecord (zh);
-           return 1;
-       }
-#endif
+#if NEW_REC_KEYS
+       zebra_rec_keys_set_buf(delkeys,
+                              rec->info[recInfo_delKeys],
+                              rec->size[recInfo_delKeys],
+                              0);
+#else
        /* flush old keys for sort&search etc. */
         delkeys.buf_used = rec->size[recInfo_delKeys];
        delkeys.buf = rec->info[recInfo_delKeys];
+#endif
 
+#if NEW_REC_KEYS && NATTR
+       zebra_rec_keys_set_buf(sortKeys,
+                              rec->info[recInfo_sortKeys],
+                              rec->size[recInfo_sortKeys],
+                              0);
+       extract_flushSortKeys (zh, *sysno, 0, sortKeys);
+#else
         sortKeys.buf_used = rec->size[recInfo_sortKeys];
         sortKeys.buf = rec->info[recInfo_sortKeys];
-
        extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
+#endif
+
+#if NEW_REC_KEYS
+        extract_flushRecordKeys (zh, *sysno, 0, delkeys,
+                                recordAttr->staticrank); /* old values */  
+#else
         extract_flushRecordKeys (zh, *sysno, 0, &delkeys,
                                 recordAttr->staticrank); /* old values */  
+#endif
         if (deleteFlag)
         {
             /* record going to be deleted */
-            if (!delkeys.buf_used)
+            if (
+#if NEW_REC_KEYS
+               zebra_rec_keys_empty(delkeys)
+#else
+               !delkeys.buf_used
+#endif
+               )
             {
                 yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T,
-                      zh->m_record_type, fname, recordOffset);
+                        zh->m_record_type, fname, recordOffset);
                 yaz_log (YLOG_WARN, "cannot delete file above, storeKeys false");
             }
             else
             {
                 if (zh->records_processed < zh->m_file_verbose_limit)
                     yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T,
-                         zh->m_record_type, fname, recordOffset);
+                            zh->m_record_type, fname, recordOffset);
                 zh->records_deleted++;
                 if (matchStr)
                     dict_delete (zh->reg->matchDict, matchStr);
@@ -621,7 +705,13 @@ static int file_extract_record(ZebraHandle zh,
         else
         {
             /* record going to be updated */
-            if (!delkeys.buf_used)
+            if (
+#if NEW_REC_KEYS
+               zebra_rec_keys_empty(delkeys)
+#else
+               !delkeys.buf_used
+#endif
+               )
             {
                 yaz_log (YLOG_LOG, "update %s %s " PRINTF_OFF_T,
                       zh->m_record_type, fname, recordOffset);
@@ -635,11 +725,22 @@ static int file_extract_record(ZebraHandle zh,
                         zh->m_record_type, fname, recordOffset);
                recordAttr->staticrank = extractCtrl.staticrank;
                 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
+#if NEW_REC_KEYS
+                extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys,
+                                        recordAttr->staticrank);
+#else
                 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys,
                                         recordAttr->staticrank);
+#endif
                 zh->records_updated++;
             }
         }
+#if NEW_REC_KEYS
+       zebra_rec_keys_close(delkeys);
+#endif
+#if NATTR && NEW_REC_KWYS
+       zebra_rec_keys_close(sortKeys);
+#endif
     }
     /* update file type */
     xfree (rec->info[recInfo_fileType]);
@@ -653,12 +754,25 @@ static int file_extract_record(ZebraHandle zh,
 
     /* update delete keys */
     xfree (rec->info[recInfo_delKeys]);
-    if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1)
+    if (
+#if NEW_REC_KEYS
+       !zebra_rec_keys_empty(zh->reg->keys)
+#else
+       zh->reg->keys.buf_used > 0
+#endif
+       && zh->m_store_keys == 1)
     {
+#if NEW_REC_KEYS
+       zebra_rec_keys_get_buf(zh->reg->keys,
+                              &rec->info[recInfo_delKeys],
+                              &rec->size[recInfo_delKeys]);
+#else
+
         rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
         rec->info[recInfo_delKeys] = zh->reg->keys.buf;
         zh->reg->keys.buf = NULL;
         zh->reg->keys.buf_max = 0;
+#endif
     }
     else
     {
@@ -847,16 +961,16 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
   If not, and a record is provided, then sysno is got from there
   
  */
-ZEBRA_RES buffer_extract_record (ZebraHandle zh, 
-                                const char *buf, size_t buf_size,
-                                int delete_flag,
-                                int test_mode, 
-                                const char *recordType,
-                                SYSNO *sysno,
-                                const char *match_criteria,
-                                const char *fname,
-                                int force_update,
-                                int allow_update)
+ZEBRA_RES buffer_extract_record(ZebraHandle zh, 
+                               const char *buf, size_t buf_size,
+                               int delete_flag,
+                               int test_mode, 
+                               const char *recordType,
+                               SYSNO *sysno,
+                               const char *match_criteria,
+                               const char *fname,
+                               int force_update,
+                               int allow_update)
 {
     RecordAttr *recordAttr;
     struct recExtractCtrl extractCtrl;
@@ -888,9 +1002,17 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh,
     extractCtrl.first_record = 1;
     extractCtrl.fh = &fc;
 
+#if NEW_REC_KEYS
+    zebra_rec_keys_reset(zh->reg->keys);
+#else
     create_rec_keys_codec(&zh->reg->keys);
+#endif
 #if NATTR
+#if NEW_REC_KEYS
+    zebra_rec_keys_reset(zh->reg->sortKeys);
+#else
     create_rec_keys_codec(&zh->reg->sortKeys);
+#endif
 #else
     zh->reg->sortKeys.buf_used = 0;
 #endif
@@ -966,8 +1088,13 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh,
             matchStr = match_criteria;
         } else {
             if (zh->m_record_id && *zh->m_record_id) {
+#if NEW_REC_KEYS
+                matchStr = fileMatchStr (zh, zh->reg->keys, pr_fname, 
+                                         zh->m_record_id);
+#else
                 matchStr = fileMatchStr (zh, &zh->reg->keys, pr_fname, 
                                          zh->m_record_id);
+#endif
                if (!matchStr)
                 {
                     yaz_log (YLOG_WARN, "Bad match criteria (recordID)");
@@ -984,7 +1111,13 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh,
            }
         }
     }
-    if (zh->reg->keys.buf_used == 0)
+    if (
+#if NEW_REC_KEYS
+       zebra_rec_keys_empty(zh->reg->keys)
+#else
+       zh->reg->keys.buf_used == 0
+#endif
+)
     {
        /* the extraction process returned no information - the record
           is probably empty - unless flagShowRecords is in use */
@@ -1019,15 +1152,31 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh,
                          sizeof(*sysno), sysno);
         }
        extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
+       
+#if NEW_REC_KEYS
+#if 0
+       print_rec_keys(zh, zh->reg->keys);
+#endif
+        extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys,
+                        recordAttr->staticrank);
+#else
+#if 0
+       print_rec_keys(zh, &zh->reg->keys);
+#endif
         extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys,
-                                recordAttr->staticrank);
+                        recordAttr->staticrank);
+#endif
 
         zh->records_inserted++;
     } 
     else
     {
         /* record already exists */
+#if NEW_REC_KEYS
+       zebra_rec_keys_t delkeys = zebra_rec_keys_open();
+#else
         struct recKeys delkeys;
+#endif
 #if NATTR
        struct recKeys sortKeys;
 #else
@@ -1047,36 +1196,41 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh,
         assert (rec);
        
        recordAttr = rec_init_attr (zh->reg->zei, rec);
-       
-#if REC_MULTI_SKIP
-       if (!force_update) {
-           if (recordAttr->runNumber ==
-               zebraExplain_runNumberIncrement (zh->reg->zei, 0))
-           {
-               if (show_progress)
-                   yaz_log (YLOG_LOG, "skipped %s %s %ld", recordType,
-                            pr_fname, (long) recordOffset);
-               extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys);
-               rec_rm (&rec);
-               logRecord(zh);
-               return ZEBRA_FAIL;
-           }
-       }
-#endif
 
+#if NEW_REC_KEYS
+       zebra_rec_keys_set_buf(delkeys,
+                              rec->info[recInfo_delKeys],
+                              rec->size[recInfo_delKeys],
+                              0);
+#else
         delkeys.buf_used = rec->size[recInfo_delKeys];
        delkeys.buf = rec->info[recInfo_delKeys];
-
+#endif
         sortKeys.buf_used = rec->size[recInfo_sortKeys];
         sortKeys.buf = rec->info[recInfo_sortKeys];
 
+#if NEW_REC_KEYS && NATTR
+       extract_flushSortKeys (zh, *sysno, 0, sortKeys);
+#else
        extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
+#endif
+#if NEW_REC_KEYS
+        extract_flushRecordKeys (zh, *sysno, 0, delkeys,
+                                recordAttr->staticrank);
+#else
         extract_flushRecordKeys (zh, *sysno, 0, &delkeys,
                                 recordAttr->staticrank);
+#endif
         if (delete_flag)
         {
             /* record going to be deleted */
-            if (!delkeys.buf_used)
+            if (
+#if NEW_REC_KEYS
+               zebra_rec_keys_empty(delkeys)
+#else
+               !delkeys.buf_used
+#endif
+               )
             {
                if (show_progress)
                {
@@ -1103,7 +1257,13 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh,
         else
         {
             /* record going to be updated */
-            if (!delkeys.buf_used)
+            if (
+#if NEW_REC_KEYS
+               zebra_rec_keys_empty(delkeys)
+#else
+               !delkeys.buf_used
+#endif
+               )
             {
                if (show_progress)
                {
@@ -1119,11 +1279,22 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh,
                             pr_fname, (long) recordOffset);
                recordAttr->staticrank = extractCtrl.staticrank;
                 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
+#if NEW_REC_KEYS
+                extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, 
+                                        recordAttr->staticrank);
+#else
                 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, 
                                         recordAttr->staticrank);
+#endif
                 zh->records_updated++;
             }
         }
+#if NEW_REC_KEYS
+       zebra_rec_keys_close(delkeys);
+#endif
+#if NEW_REC_KEYS && NATTR
+       zebra_rec_keys_close(sortKeys);
+#endif
     }
     /* update file type */
     xfree (rec->info[recInfo_fileType]);
@@ -1137,19 +1308,31 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh,
 
     /* update delete keys */
     xfree (rec->info[recInfo_delKeys]);
-    if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1)
+    if (
+#if NEW_REC_KEYS
+       !zebra_rec_keys_empty(zh->reg->keys)
+#else
+       zh->reg->keys.buf_used > 0
+#endif
+       && zh->m_store_keys == 1)
     {
+#if NEW_REC_KEYS
+       zebra_rec_keys_get_buf(zh->reg->keys,
+                              &rec->info[recInfo_delKeys],
+                              &rec->size[recInfo_delKeys]);
+#else
+
         rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
         rec->info[recInfo_delKeys] = zh->reg->keys.buf;
         zh->reg->keys.buf = NULL;
         zh->reg->keys.buf_max = 0;
+#endif
     }
     else
     {
         rec->info[recInfo_delKeys] = NULL;
         rec->size[recInfo_delKeys] = 0;
     }
-
     /* update sort keys */
     xfree (rec->info[recInfo_sortKeys]);
 
@@ -1225,9 +1408,18 @@ int explain_extract (void *handle, Record rec, data1_node *n)
             abort ();
     }
 
+#if NEW_REC_KEYS
+    zebra_rec_keys_reset(zh->reg->keys);
+#else
     create_rec_keys_codec(&zh->reg->keys);
+#endif
+    
 #if NATTR
+#if NEW_REC_KEYS
+    zebra_rec_keys_reset(zh->reg->sortKeys);
+#else
     create_rec_keys_codec(&zh->reg->sortKeys);
+#endif
 #else
     zh->reg->sortKeys.buf_used = 0;
 #endif
@@ -1251,30 +1443,68 @@ int explain_extract (void *handle, Record rec, data1_node *n)
 
     if (rec->size[recInfo_delKeys])
     {
+#if NEW_REC_KEYS
+       zebra_rec_keys_t delkeys = zebra_rec_keys_open();
+#else
        struct recKeys delkeys;
+#endif
+
 #if NATTR
+#if NEW_REC_KEYS
+       zebra_rec_keys_t sortkeys = zzebra_rec_keys_open();
+#else
        struct recKeys sortkeys;
+#endif
 #else
        struct sortKeys sortkeys;
 #endif
 
+#if NEW_REC_KEYS
+       zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
+                              rec->size[recInfo_delKeys],
+                              0);
+       extract_flushRecordKeys (zh, rec->sysno, 0, delkeys, 0);
+       zebra_rec_keys_close(delkeys);
+#else
        delkeys.buf_used = rec->size[recInfo_delKeys];
        delkeys.buf = rec->info[recInfo_delKeys];
+       extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys, 0);
+#endif
+#if NATTR && NEW_REC_KEYS
+       zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
+                              rec->size[recInfo_sortKeys],
+                              0);
 
+       extract_flushSortKeys (zh, rec->sysno, 0, sortkeys);
+       zebra_rec_keys_close(sortkeys);
+#else
        sortkeys.buf_used = rec->size[recInfo_sortKeys];
        sortkeys.buf = rec->info[recInfo_sortKeys];
-
        extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys);
-       extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys, 0);
+#endif
     }
+#if NEW_REC_KEYS
+    extract_flushRecordKeys (zh, rec->sysno, 1, zh->reg->keys, 0);
+#else
     extract_flushRecordKeys (zh, rec->sysno, 1, &zh->reg->keys, 0);
+#endif
+#if NATTR && NEW_REC_KEYS
+    extract_flushSortKeys (zh, rec->sysno, 1, zh->reg->sortKeys);
+#else
     extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys);
+#endif
 
     xfree (rec->info[recInfo_delKeys]);
+#if NEW_REC_KEYS
+    zebra_rec_keys_get_buf(zh->reg->keys,
+                          &rec->info[recInfo_delKeys], 
+                          &rec->size[recInfo_delKeys]);
+#else
     rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
     rec->info[recInfo_delKeys] = zh->reg->keys.buf;
     zh->reg->keys.buf = NULL;
     zh->reg->keys.buf_max = 0;
+#endif
 
     xfree (rec->info[recInfo_sortKeys]);
     rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
@@ -1286,12 +1516,20 @@ int explain_extract (void *handle, Record rec, data1_node *n)
 }
 
 void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
-                              int cmd, struct recKeys *reckeys,
+                              int cmd,
+#if NEW_REC_KEYS
+                             zebra_rec_keys_t reckeys,
+#else
+                             struct recKeys *reckeys,
+#endif
                              zint staticrank)
 {
+#if NEW_REC_KEYS
+#else
     void *decode_handle = iscz1_start();
     int off = 0;
     int ch = 0;
+#endif
     ZebraExplainInfo zei = zh->reg->zei;
 
     if (!zh->reg->key_buf)
@@ -1312,6 +1550,65 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
     }
     zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
 
+#if NEW_REC_KEYS
+    if (zebra_rec_keys_rewind(reckeys))
+    {
+       size_t slen;
+       const char *str;
+       struct it_key key_in;
+       while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
+       {
+           int ch = 0;
+           struct it_key key_out;
+           zint *keyp = key_out.mem;
+
+           assert(key_in.len == 4);
+           
+           /* check for buffer overflow */
+           if (zh->reg->key_buf_used + 1024 > 
+               (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*))
+               extract_flushWriteKeys (zh, 0);
+           
+           ++(zh->reg->ptr_i);
+           assert(zh->reg->ptr_i > 0);
+           (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] =
+               (char*)zh->reg->key_buf + zh->reg->key_buf_used;
+
+           /* encode the ordinal value (field/use/attribute) .. */
+           ch = (int) key_in.mem[0];
+           zh->reg->key_buf_used +=
+               key_SU_encode(ch, (char*)zh->reg->key_buf +
+                             zh->reg->key_buf_used);
+           
+           /* copy the 0-terminated stuff from str to output */
+           memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, str, slen);
+           zh->reg->key_buf_used += slen;
+           ((char*)zh->reg->key_buf)[(zh->reg->key_buf_used)++] = '\0';
+
+           /* the delete/insert indicator */
+           ((char*)zh->reg->key_buf)[(zh->reg->key_buf_used)++] = cmd;
+
+           if (zh->m_staticrank) /* rank config enabled ? */
+           {
+               *keyp++ = staticrank;
+               key_out.len = 4;
+           }
+           else
+               key_out.len = 3;
+           
+           if (key_in.mem[1]) /* filter specified record ID */
+               *keyp++ = key_in.mem[1];
+           else
+               *keyp++ = sysno;
+           *keyp++ = key_in.mem[2];  /* section_id */
+           *keyp++ = key_in.mem[3];  /* sequence .. */
+           
+           memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used,
+                  &key_out, sizeof(key_out));
+           (zh->reg->key_buf_used) += sizeof(key_out);
+       }
+    }
+#else
     while (off < reckeys->buf_used)
     {
         const char *src = reckeys->buf + off;
@@ -1325,7 +1622,7 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
 
         if (zh->reg->key_buf_used + 1024 > 
             (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*))
-            extract_flushWriteKeys (zh,0);
+            extract_flushWriteKeys(zh, 0);
         ++(zh->reg->ptr_i);
         assert(zh->reg->ptr_i > 0);
         (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] =
@@ -1334,8 +1631,8 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
        ch = (int) key_in.mem[0];  /* ordinal for field/use/attribute */
 
        zh->reg->key_buf_used +=
-           key_SU_encode (ch,((char*)zh->reg->key_buf) +
-                           zh->reg->key_buf_used);
+           key_SU_encode(ch, ((char*)zh->reg->key_buf) +
+                          zh->reg->key_buf_used);
         while (*src)
             ((char*)zh->reg->key_buf) [(zh->reg->key_buf_used)++] = *src++;
         src++;
@@ -1364,6 +1661,7 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
     }
     assert (off == reckeys->buf_used);
     iscz1_stop(decode_handle);
+#endif
 }
 
 void extract_flushWriteKeys (ZebraHandle zh, int final)
@@ -1500,6 +1798,15 @@ void extract_flushWriteKeys (ZebraHandle zh, int final)
     zh->reg->key_buf_used = 0;
 }
 
+#if NEW_REC_KEYS
+void extract_add_it_key (ZebraHandle zh,
+                        zebra_rec_keys_t *keys,
+                        int reg_type,
+                        const char *str, int slen, struct it_key *key)
+{
+    zebra_rec_keys_write(*keys, reg_type, str, slen, key);
+}
+#else
 void extract_add_it_key (ZebraHandle zh,
                         struct recKeys *keys,
                         int reg_type,
@@ -1528,15 +1835,47 @@ void extract_add_it_key (ZebraHandle zh,
     *dst++ = '\0';
     keys->buf_used = dst - keys->buf;
 }
+#endif
 
-ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, struct recKeys *reckeys,
+ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh,
+#if NEW_REC_KEYS
+                                 zebra_rec_keys_t reckeys,
+#else
+                                 struct recKeys *reckeys,
+#endif
                                  zebra_snippets *snippets)
 {
-    void *decode_handle = iscz1_start();
-    int off = 0;
-    int seqno = 0;
     NMEM nmem = nmem_create();
 
+#if NEW_REC_KEYS
+    if (zebra_rec_keys_rewind(reckeys)) 
+    {
+       const char *str;
+       size_t slen;
+       struct it_key key;
+       while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
+       {
+           char dst_buf[IT_MAX_WORD];
+           char *dst_term = dst_buf;
+           int ord, seqno;
+           int index_type;
+           assert(key.len <= 4 && key.len > 2);
+           seqno = (int) key.mem[key.len-1];
+           ord = key.mem[0];
+           
+           zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
+                                   0/* db */, 0/* set */, 0/* use */);
+           assert(index_type);
+           zebra_term_untrans_iconv(zh, nmem, index_type,
+                                    &dst_term, str);
+           zebra_snippets_append(snippets, seqno, ord, dst_term);
+           nmem_reset(nmem);
+       }
+    }
+#else
+    int off = 0;
+    int seqno = 0;
+    void *decode_handle = iscz1_start();
     assert(reckeys->buf);
     while (off < reckeys->buf_used)
     {
@@ -1564,18 +1903,50 @@ ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, struct recKeys *reckeys,
         off = src - reckeys->buf;
        nmem_reset(nmem);
     }
-    nmem_destroy(nmem);
     iscz1_stop(decode_handle);
+#endif
+    nmem_destroy(nmem);
     return ZEBRA_OK;
 }
 
-void print_rec_keys(ZebraHandle zh, struct recKeys *reckeys)
+void print_rec_keys(ZebraHandle zh,
+#if NEW_REC_KEYS
+                   zebra_rec_keys_t reckeys
+#else
+                   struct recKeys *reckeys
+#endif
+)
 {
-    void *decode_handle = iscz1_start();
+#if NEW_REC_KEYS
+    yaz_log(YLOG_LOG, "print_rec_keys");
+    if (zebra_rec_keys_rewind(reckeys))
+    {
+       const char *str;
+       size_t slen;
+       struct it_key key;
+       while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
+       {
+           char dst_buf[IT_MAX_WORD];
+           int seqno;
+           int index_type;
+           const char *db = 0;
+           assert(key.len <= 4 && key.len > 2);
+
+           zebraExplain_lookup_ord(zh->reg->zei,
+                                   key.mem[0], &index_type, &db, 0, 0);
+           
+           seqno = (int) key.mem[key.len-1];
+           
+           zebra_term_untrans(zh, index_type, dst_buf, str);
+           
+           yaz_log(YLOG_LOG, "ord=" ZINT_FORMAT " seqno=%d term=%s",
+                   key.mem[0], seqno, dst_buf); 
+       }
+    }
+#else
     int off = 0;
     int seqno = 0;
-    NMEM nmem = nmem_create();
-
+    void *decode_handle = iscz1_start();
     yaz_log(YLOG_LOG, "print_rec_keys buf=%p sz=%d", reckeys->buf,
            reckeys->buf_used);
     assert(reckeys->buf);
@@ -1584,28 +1955,28 @@ void print_rec_keys(ZebraHandle zh, struct recKeys *reckeys)
         const char *src = reckeys->buf + off;
         struct it_key key;
        char *dst = (char*) &key;
-       int attrSet, attrUse;
        char dst_buf[IT_MAX_WORD];
-       char *dst_term = dst_buf;
+       int index_type;
+       const char *db = 0;
 
        iscz1_decode(decode_handle, &dst, &src);
        assert(key.len <= 4 && key.len > 2);
 
-       attrSet = (int) key.mem[0] >> 16;
-       attrUse = (int) key.mem[0] & 65535;
        seqno = (int) key.mem[key.len-1];
+       
+       zebraExplain_lookup_ord(zh->reg->zei,
+                               key.mem[0], &index_type, &db, 0, 0);
 
-       zebra_term_untrans_iconv(zh, nmem, src[0], &dst_term, src+1);
+       zebra_term_untrans(zh, index_type, dst_buf, src);
        
        yaz_log(YLOG_LOG, "ord=" ZINT_FORMAT " seqno=%d term=%s",
-               key.mem[0], seqno, dst_term); 
+               key.mem[0], seqno, dst_buf); 
         while (*src++)
            ;
         off = src - reckeys->buf;
-       nmem_reset(nmem);
     }
-    nmem_destroy(nmem);
     iscz1_stop(decode_handle);
+#endif
 }
 
 void extract_add_index_string (RecWord *p, const char *str, int length)

diff --git a/index/index.h b/index/index.h
index fe7002c..5f15228 100644 (file)
--- a/index/index.h
+++ b/index/index.h
@@ -1,4 +1,4 @@
-/* $Id: index.h,v 1.152 2005-10-20 18:28:10 quinn Exp $
+/* $Id: index.h,v 1.153 2005-10-28 07:25:30 adam Exp $
    Copyright (C) 1995-2005
    Index Data ApS
 
@@ -192,12 +192,18 @@ typedef struct zebra_rank_class {
     struct zebra_rank_class *next;
 } *ZebraRankClass;
 
+#define NEW_REC_KEYS 1
+
+#if NEW_REC_KEYS
+#include "reckeys.h"
+#else
 struct recKeys {
     int buf_used;
     int buf_max;
     char *buf;
     void *codec_handle;
 };
+#endif
 
 #if NATTR
 
@@ -234,7 +240,12 @@ struct zebra_register {
     int stop_flag;
     int active; /* 0=shutdown, 1=enabled and inactive, 2=activated */
 
+#if NEW_REC_KEYS
+    zebra_rec_keys_t keys;
+#else
     struct recKeys keys;
+#endif
+
 #if NATTR
     struct recKeys sortKeys;
 #else
@@ -476,15 +487,31 @@ off_t zebra_record_int_tell (void *fh);
 int zebra_record_int_read (void *fh, char *buf, size_t count);
 void zebra_record_int_end (void *fh, off_t offset);
 
+#if NEW_REC_KEYS
+void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys);
+#else
 void print_rec_keys(ZebraHandle zh, struct recKeys *reckeys);
+#endif
+
+#if NEW_REC_KEYS
+ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys,
+                                 zebra_snippets *snippets);
+#else
 ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, struct recKeys *reckeys,
                                  zebra_snippets *snippets);
+#endif
 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
                                    zint sysno, zebra_snippets *snippets);
 
+#if NEW_REC_KEYS
+void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
+                              int cmd, zebra_rec_keys_t reckeys,
+                             zint staticrank);
+#else
 void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
                               int cmd, struct recKeys *reckeys,
                              zint staticrank);
+#endif
 #if NATTR
 void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno,
                             int cmd, struct recKeys *skp);

diff --git a/index/kinput.c b/index/kinput.c
index 748ced9..79e9f04 100644 (file)
--- a/index/kinput.c
+++ b/index/kinput.c
@@ -1,4 +1,4 @@
-/* $Id: kinput.c,v 1.67 2005-09-16 09:58:39 adam Exp $
+/* $Id: kinput.c,v 1.68 2005-10-28 07:25:30 adam Exp $
    Copyright (C) 1995-2005
    Index Data ApS
 
@@ -397,18 +397,20 @@ static int heap_read_one (struct heap_info *hi, char *name, char *key)
 #define PR_KEY_LOW 0
 #define PR_KEY_TOP 0
 
+#if 0
 static void pkey(const char *b, int mode)
 {
     key_logdump_txt(YLOG_LOG, b, mode ? "i" : "d");
 }
+#endif
 
 #if 1
 /* for debugging only */
-static void print_dict_item (ZebraHandle zh, const char *s)
+static void print_dict_item(ZebraHandle zh, const char *s)
 {
     char dst[IT_MAX_WORD+1];
     int ord;
-    int len = key_SU_decode (&ord, (const unsigned char *) s);
+    int len = key_SU_decode(&ord, (const unsigned char *) s);
     int index_type;
     const char *db = 0;
 
@@ -416,7 +418,8 @@ static void print_dict_item (ZebraHandle zh, const char *s)
                             ord, &index_type, &db, 0, 0);
 
     zebra_term_untrans(zh, index_type, dst, s + len);
-    yaz_log (YLOG_LOG, "%s", dst);
+
+    yaz_log(YLOG_LOG, "ord=%d term=%s", ord, dst);
 }
 #endif
 
@@ -656,7 +659,7 @@ int heap_inpb(struct heap_cread_info *hci, struct heap_info *hi)
         hi->no_diffs++;
 
 #if 0
-        print_dict_item (hi->zh, hci->cur_name);
+        print_dict_item(hi->zh, hci->cur_name);
 #endif
         if ((dict_info = dict_lookup (hi->reg->dict, hci->cur_name)))
         {

diff --git a/index/retrieve.c b/index/retrieve.c
index 08257c4..d60a989 100644 (file)
--- a/index/retrieve.c
+++ b/index/retrieve.c
@@ -1,4 +1,4 @@
-/* $Id: retrieve.c,v 1.35 2005-08-26 10:13:31 adam Exp $
+/* $Id: retrieve.c,v 1.36 2005-10-28 07:25:30 adam Exp $
    Copyright (C) 1995-2005
    Index Data ApS
 
@@ -219,12 +219,24 @@ int zebra_record_fetch (ZebraHandle zh, SYSNO sysno, int score,
     if (1)
     {
        /* snippets code */
-       struct recKeys reckeys;
        zebra_snippets *snippet;
+
+#if NEW_REC_KEYS
+       zebra_rec_keys_t reckeys = zebra_rec_keys_open();
+       
+       zebra_rec_keys_set_buf(reckeys,
+                              rec->info[recInfo_delKeys],
+                              rec->size[recInfo_delKeys], 
+                              0);
+       zebra_snippets_rec_keys(zh, reckeys, retrieveCtrl.doc_snippet);
+       zebra_rec_keys_close(reckeys);
+#else
+       struct recKeys reckeys;
        reckeys.buf = rec->info[recInfo_delKeys];
        reckeys.buf_used = rec->size[recInfo_delKeys];
-
        zebra_snippets_rec_keys(zh, &reckeys, retrieveCtrl.doc_snippet);
+#endif
+
 
 #if 0
        /* for debugging purposes */

diff --git a/index/zebraapi.c b/index/zebraapi.c
index 5782aeb..d419085 100644 (file)
--- a/index/zebraapi.c
+++ b/index/zebraapi.c
@@ -1,4 +1,4 @@
-/* $Id: zebraapi.c,v 1.191 2005-10-21 18:29:04 adam Exp $
+/* $Id: zebraapi.c,v 1.192 2005-10-28 07:25:30 adam Exp $
    Copyright (C) 1995-2005
    Index Data ApS
 
@@ -326,9 +326,13 @@ struct zebra_register *zebra_register_open(ZebraService zs, const char *name,
 
     reg->key_buf = 0;
 
+#if NEW_REC_KEYS
+    reg->keys = zebra_rec_keys_open();
+#else
     reg->keys.buf_max = 0;
     reg->keys.buf = 0;
     reg->keys.codec_handle = iscz1_start();
+#endif
 
     reg->sortKeys.buf = 0;
     reg->sortKeys.buf_max = 0;
@@ -497,6 +501,9 @@ static void zebra_register_close (ZebraService zs, struct zebra_register *reg)
     data1_destroy (reg->dh);
 
     xfree(reg->sortKeys.buf);
+#if NEW_REC_KEYS
+    zebra_rec_keys_close(reg->keys);
+#else
     xfree(reg->keys.buf);
     if (reg->keys.codec_handle)
        iscz1_stop(reg->keys.codec_handle);
@@ -504,6 +511,8 @@ static void zebra_register_close (ZebraService zs, struct zebra_register *reg)
     if (reg->sortKeys.codec_handle)
        iscz1_stop(reg->sortKeys.codec_handle);
 #endif
+#endif
+
     xfree(reg->key_buf);
     xfree(reg->name);
     xfree(reg);

diff --git a/index/zsets.c b/index/zsets.c
index 1b76097..949ace4 100644 (file)
--- a/index/zsets.c
+++ b/index/zsets.c
@@ -1,4 +1,4 @@
-/* $Id: zsets.c,v 1.95 2005-08-26 10:13:31 adam Exp $
+/* $Id: zsets.c,v 1.96 2005-10-28 07:25:30 adam Exp $
    Copyright (C) 1995-2005
    Index Data ApS
 
@@ -1119,6 +1119,7 @@ ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
        
        rset_delete(rset_comb);
        nmem_destroy(nmem);
+       kc->dec(kc);
     }
     return ZEBRA_OK;
 }