X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=611ce7f3d7f6545792a995498a21deffb90f1be9;hb=5a4211916c8b14d4b0ec9cc717e613b88394affb;hp=53dc8965c0f07c72065c728948b8442762df935d;hpb=0b5d38bc84d5261aaed3bce3be748c5d7008f2ff;p=idzebra-moved-to-github.git

diff --git a/index/extract.c b/index/extract.c
index 53dc896..611ce7f 100644
--- a/index/extract.c
+++ b/index/extract.c
@@ -4,7 +4,21 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: extract.c,v $
- * Revision 1.64  1996-11-08 11:10:16  adam
+ * Revision 1.68  1997-02-12 20:39:45  adam
+ * Implemented options -f <n> that limits the log to the first <n>
+ * records.
+ * Changed some log messages also.
+ *
+ * Revision 1.67  1996/11/15 15:02:14  adam
+ * Minor changes regarding logging.
+ *
+ * Revision 1.66  1996/11/14  09:52:21  adam
+ * Strings in record keys bound by IT_MAX_WORD.
+ *
+ * Revision 1.65  1996/11/14  08:57:56  adam
+ * Reduction of storeKeys area.
+ *
+ * Revision 1.64  1996/11/08 11:10:16  adam
  * Buffers used during file match got bigger.
  * Compressed ISAM support everywhere.
  * Bug fixes regarding masking characters in queries.
@@ -355,6 +369,9 @@ void encode_key_write (char *k, struct encode_info *i, FILE *outf)
     }
 }
 
+#define SORT_EXTRA 0
+
+#if SORT_EXTRA
 static int key_y_len;
 
 static int key_y_compare (const void *p1, const void *p2)
@@ -371,6 +388,7 @@ static int key_x_compare (const void *p1, const void *p2)
 {
     return strcmp (*(char**) p1, *(char**) p2);
 }
+#endif
 
 void key_flush (void)
 {
@@ -378,14 +396,16 @@ void key_flush (void)
     char out_fname[200];
     char *prevcp, *cp;
     struct encode_info encode_info;
+#if SORT_EXTRA
     int i;
+#endif
     
     if (ptr_i <= 0)
         return;
 
     key_file_no++;
     logf (LOG_LOG, "sorting section %d", key_file_no);
-#if 1
+#if !SORT_EXTRA
     qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_qsort_compare);
     getFnameTmp (out_fname, key_file_no);
 
@@ -484,6 +504,7 @@ struct recKeys {
     char *buf;
     char prevAttrSet;
     short prevAttrUse;
+    int prevSeqNo;
 } reckeys;
 
 static void addRecordKey (const RecWord *p)
@@ -493,6 +514,7 @@ static void addRecordKey (const RecWord *p)
     short attrUse;
     size_t i;
     int lead = 0;
+    int diff = 0;
 
     if (reckeys.buf_used+1024 > reckeys.buf_max)
     {
@@ -516,6 +538,14 @@ static void addRecordKey (const RecWord *p)
         lead |= 2;
     else
         reckeys.prevAttrUse = attrUse;
+#if 1
+    diff = 1 + p->seqno - reckeys.prevSeqNo;
+    if (diff >= 1 && diff <= 15)
+        lead |= (diff << 2);
+    else
+        diff = 0;
+#endif
+    reckeys.prevSeqNo = p->seqno;
 
     *dst++ = lead;
 
@@ -540,12 +570,15 @@ static void addRecordKey (const RecWord *p)
         case Word_Numeric:
             *dst++ = 'n';
     }
-    for (i = 0; p->u.string[i]; i++)
+    for (i = 0; p->u.string[i] && i < IT_MAX_WORD-3; i++)
         *dst++ = p->u.string[i];
     *dst++ = '\0';
 
-    memcpy (dst, &p->seqno, sizeof(p->seqno));
-    dst += sizeof(p->seqno);
+    if (!diff)
+    {
+        memcpy (dst, &p->seqno, sizeof(p->seqno));
+        dst += sizeof(p->seqno);
+    }
     reckeys.buf_used = dst - reckeys.buf;
 }
 
@@ -554,6 +587,7 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys,
 {
     char attrSet = -1;
     short attrUse = -1;
+    int seqno = 0;
     int off = 0;
 
     if (zebTargetInfo_curDatabase (zti, databaseName))
@@ -565,7 +599,7 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys,
     {
         const char *src = reckeys->buf + off;
         struct it_key key;
-        int lead;
+        int lead, ch;
     
         lead = *src++;
 
@@ -584,20 +618,25 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys,
         ++ptr_i;
         key_buf[ptr_top-ptr_i] = (char*)key_buf + key_buf_used;
 
-        lead = zebTargetInfo_lookupSU (zti, attrSet, attrUse);
-        if (lead < 0)
-            lead = zebTargetInfo_addSU (zti, attrSet, attrUse);
-        assert (lead > 0);
-        ((char*) key_buf) [key_buf_used++] = lead;
+        ch = zebTargetInfo_lookupSU (zti, attrSet, attrUse);
+        if (ch < 0)
+            ch = zebTargetInfo_addSU (zti, attrSet, attrUse);
+        assert (ch > 0);
+        ((char*) key_buf) [key_buf_used++] = ch;
         while (*src)
             ((char*)key_buf) [key_buf_used++] = *src++;
         src++;
         ((char*)key_buf) [key_buf_used++] = '\0';
-        
         ((char*) key_buf)[key_buf_used++] = cmd;
 
-        memcpy (&key.seqno, src, sizeof(key.seqno));
-        src += sizeof(key.seqno);
+        if (lead & 60)
+            seqno += ((lead>>2) & 15)-1;
+        else
+        {
+            memcpy (&seqno, src, sizeof(seqno));
+            src += sizeof(seqno);
+        }
+        key.seqno = seqno;
         key.sysno = sysno;
         memcpy ((char*)key_buf + key_buf_used, &key, sizeof(key));
         key_buf_used += sizeof(key);
@@ -869,7 +908,7 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
 
 struct recordLogInfo {
     const char *fname;
-    char *op;
+    int recordOffset;
     struct recordGroup *rGroup;
 };
      
@@ -880,9 +919,8 @@ static void recordLogPreamble (int level, const char *msg, void *info)
 
     if (level & LOG_LOG)
         return ;
-    if (p->op) 
-        fprintf (outf, "%s of ", p->op);
-    fprintf (outf, "%s type %s\n", p->rGroup->recordType, p->fname);
+    fprintf (outf, "File %s, offset %d, type %s\n",
+             p->rGroup->recordType, p->recordOffset, p->fname);
     log_event_start (NULL, NULL);
 }
 
@@ -899,11 +937,6 @@ static int recordExtract (SYSNO *sysno, const char *fname,
     Record rec;
     struct recordLogInfo logInfo;
 
-    logInfo.fname = fname;
-    logInfo.op = NULL;
-    logInfo.rGroup = rGroup;
-    log_event_start (recordLogPreamble, &logInfo);
-    
     if (fi->fd != -1)
     {
         /* we are going to read from a file, so prepare the extraction */
@@ -915,6 +948,7 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         reckeys.buf_used = 0;
         reckeys.prevAttrUse = -1;
         reckeys.prevAttrSet = -1;
+        reckeys.prevSeqNo = 0;
 
         recordOffset = fi->file_moffset;
         extractCtrl.offset = recordOffset;
@@ -923,19 +957,37 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         extractCtrl.endf = file_end;
         extractCtrl.map_chrs_input = map_chrs_input;
         extractCtrl.flagShowRecords = rGroup->flagShowRecords;
+        if (rGroup->flagShowRecords)
+            printf ("File: %s %ld\n", fname, (long) recordOffset);
+
+        logInfo.fname = fname;
+        logInfo.recordOffset = recordOffset;
+        logInfo.rGroup = rGroup;
+        log_event_start (recordLogPreamble, &logInfo);
+
         r = (*recType->extract)(&extractCtrl);
 
+        log_event_start (NULL, NULL);
+
         if (r)      
         {
             /* error occured during extraction ... */
-            logf (LOG_WARN, "Couldn't extract file %s, code %d", fname, r);
+            if (!rGroup->flagShowRecords &&
+                    records_processed < rGroup->fileVerboseLimit)
+            {
+                logf (LOG_WARN, "fail %s %s %ld code = %d", rGroup->recordType,
+                      fname, (long) recordOffset, r);
+            }
             return 0;
         }
         if (reckeys.buf_used == 0)
         {
             /* the extraction process returned no information - the record
-               is probably empty */
-            logf (LOG_WARN, "Empty file %s", fname);
+               is probably empty - unless flagShowRecords is in use */
+            if (rGroup->flagShowRecords)
+                return 1;
+            logf (LOG_WARN, "No keys generated for file %s", fname);
+            logf (LOG_WARN, " The file is probably empty");
             return 0;
         }
     }
@@ -975,9 +1027,8 @@ static int recordExtract (SYSNO *sysno, const char *fname,
             logf (LOG_LOG, "Cannot delete new record");
             return 1;
         }
-        logInfo.op = "add";
-        if (rGroup->fileVerboseFlag)
-            logf (LOG_LOG, "add %s %s+%ld", rGroup->recordType,
+        if (records_processed < rGroup->fileVerboseLimit)
+            logf (LOG_LOG, "add %s %s %ld", rGroup->recordType,
                   fname, (long) recordOffset);
         rec = rec_new (records);
         *sysno = rec->sysno;
@@ -1003,14 +1054,15 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         if (deleteFlag)
         {
             /* record going to be deleted */
-            logInfo.op = "delete";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot delete; storeKeys false");
+                logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
+                      fname, (long) recordOffset);
+                logf (LOG_WARN, "cannot delete file above, storeKeys false");
             }
             else
             {
-                if (rGroup->fileVerboseFlag)
+                if (records_processed < rGroup->fileVerboseLimit)
                     logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
                           fname, (long) recordOffset);
                 records_deleted++;
@@ -1024,14 +1076,15 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         else
         {
             /* record going to be updated */
-            logInfo.op = "update";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot update; storeKeys false");
+                logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
+                      fname, (long) recordOffset);
+                logf (LOG_WARN, "cannot update file above, storeKeys false");
             }
             else
             {
-                if (rGroup->fileVerboseFlag)
+                if (records_processed < rGroup->fileVerboseLimit)
                     logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
                           fname, (long) recordOffset);
                 flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); 
@@ -1157,7 +1210,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
             sprintf (ext_res, "%srecordType", gprefix);
             if (!(rGroup->recordType = res_get (common_resource, ext_res)))
             {
-                if (rGroup->fileVerboseFlag)
+                if (records_processed < rGroup->fileVerboseLimit)
                     logf (LOG_LOG, "? %s", fname);
                 return 0;
             }
@@ -1165,7 +1218,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
     }
     if (!rGroup->recordType)
     {
-        if (rGroup->fileVerboseFlag)
+        if (records_processed < rGroup->fileVerboseLimit)
             logf (LOG_LOG, "? record %s", fname);
         return 0;
     }
@@ -1243,7 +1296,6 @@ int fileExtract (SYSNO *sysno, const char *fname,
         r = recordExtract (sysno, fname, rGroup, deleteFlag, fi,
                            recType, subType);
     } while (r && !sysno && fi->file_more);
-    log_event_start (NULL, NULL);
     file_read_stop (fi);
     if (fd != -1)
         close (fd);