X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=45e636382ca2b9da082f3d91453385b51357824d;hb=f847e42bde3d8c0ac862fc9cbc65c2f8fc6aa1d3;hp=4426e4afd4678ce3728bb16e8aa50b06048ff1fb;hpb=97416b96f6407c49f583a3aa579c9857b3d3d108;p=idzebra-moved-to-github.git

diff --git a/index/extract.c b/index/extract.c
index 4426e4a..45e6363 100644
--- a/index/extract.c
+++ b/index/extract.c
@@ -4,7 +4,54 @@
  * Sebastian Hammer, Adam Dickmeiss
  *
  * $Log: extract.c,v $
- * Revision 1.41  1995-12-06 16:06:42  adam
+ * Revision 1.53  1996-04-26 12:09:43  adam
+ * Added a few comments.
+ *
+ * Revision 1.52  1996/04/25  13:27:57  adam
+ * Function recordExtract modified so that files with no keys (possibly empty)
+ * are ignored.
+ *
+ * Revision 1.51  1996/03/19  11:08:42  adam
+ * Bug fix: Log preamble wasn't always turned off after recordExtract.
+ *
+ * Revision 1.50  1996/02/12  18:45:36  adam
+ * New fileVerboseFlag in record group control.
+ *
+ * Revision 1.49  1996/02/05  12:29:57  adam
+ * Logging reduced a bit.
+ * The remaining running time is estimated during register merge.
+ *
+ * Revision 1.48  1996/02/01  20:53:26  adam
+ * The temporary per-record keys are compacted a little, and duplication
+ * of the per-records keys are avoided when they are saved in the record
+ * information buffer.
+ *
+ * Revision 1.47  1996/01/17  14:57:48  adam
+ * Prototype changed for reader functions in extract/retrieve. File
+ *  is identified by 'void *' instead of 'int.
+ *
+ * Revision 1.46  1995/12/15  14:57:16  adam
+ * Bug fix.
+ *
+ * Revision 1.45  1995/12/15  12:37:41  adam
+ * In addRecordKeyAny: Writes key only when attrSet != -1.
+ *
+ * Revision 1.44  1995/12/12  16:00:54  adam
+ * System call sync(2) used after update/commit.
+ * Locking (based on fcntl) uses F_EXLCK and F_SHLCK instead of F_WRLCK
+ * and F_RDLCK.
+ *
+ * Revision 1.43  1995/12/11  09:12:46  adam
+ * The rec_get function returns NULL if record doesn't exist - will
+ * happen in the server if the result set records have been deleted since
+ * the creation of the set (i.e. the search).
+ * The server saves a result temporarily if it is 'volatile', i.e. the
+ * set is register dependent.
+ *
+ * Revision 1.42  1995/12/07  17:38:46  adam
+ * Work locking mechanisms for concurrent updates/commit.
+ *
+ * Revision 1.41  1995/12/06  16:06:42  adam
  * Better diagnostics. Work on 'real' dictionary deletion.
  *
  * Revision 1.40  1995/12/05  16:57:40  adam
@@ -170,8 +217,6 @@ static int records_inserted = 0;
 static int records_updated = 0;
 static int records_deleted = 0;
 
-#define MATCH_DICT "match"
-
 void key_open (int mem)
 {
     if (mem < 50000)
@@ -183,9 +228,9 @@ void key_open (int mem)
     key_buf_used = 0;
     key_file_no = 0;
 
-    if (!(matchDict = dict_open (MATCH_DICT, 20, 1)))
+    if (!(matchDict = dict_open (GMATCH_DICT, 50, 1)))
     {
-        logf (LOG_FATAL, "dict_open fail of %s", MATCH_DICT);
+        logf (LOG_FATAL, "dict_open fail of %s", GMATCH_DICT);
         exit (1);
     }
     assert (!records);
@@ -323,6 +368,8 @@ struct recKeys {
     int buf_used;
     int buf_max;
     char *buf;
+    char prevAttrSet;
+    short prevAttrUse;
 } reckeys;
 
 static void addRecordKey (const RecWord *p)
@@ -331,29 +378,46 @@ static void addRecordKey (const RecWord *p)
     char attrSet;
     short attrUse;
     size_t i;
+    int lead = 0;
 
     if (reckeys.buf_used+1024 > reckeys.buf_max)
     {
         char *b;
 
-        b = xmalloc (reckeys.buf_max += 65000);
+        b = xmalloc (reckeys.buf_max += 128000);
         if (reckeys.buf_used > 0)
             memcpy (b, reckeys.buf, reckeys.buf_used);
         xfree (reckeys.buf);
         reckeys.buf = b;
     }
     dst = reckeys.buf + reckeys.buf_used;
+
+    attrSet = p->attrSet;
+    if (reckeys.buf_used > 0 && reckeys.prevAttrSet == attrSet)
+        lead |= 1;
+    else
+        reckeys.prevAttrSet = attrSet;
+    attrUse = p->attrUse;
+    if (reckeys.buf_used > 0 && reckeys.prevAttrUse == attrUse)
+        lead |= 2;
+    else
+        reckeys.prevAttrUse = attrUse;
+
     switch (p->which)
     {
     case Word_String:
-        attrSet = p->attrSet;
-        memcpy (dst, &attrSet, sizeof(attrSet));
-        dst += sizeof(attrSet);
+        *dst++ = lead;
 
-        attrUse = p->attrUse;
-        memcpy (dst, &attrUse, sizeof(attrUse));
-        dst += sizeof(attrUse);
-        
+        if (!(lead & 1))
+        {
+            memcpy (dst, &attrSet, sizeof(attrSet));
+            dst += sizeof(attrSet);
+        }
+        if (!(lead & 2))
+        {
+            memcpy (dst, &attrUse, sizeof(attrUse));
+            dst += sizeof(attrUse);
+        }
         for (i = 0; p->u.string[i]; i++)
             *dst++ = p->u.string[i];
         *dst++ = '\0';
@@ -371,20 +435,27 @@ static void addRecordKey (const RecWord *p)
 static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys, 
                              const char *databaseName)
 {
+    char attrSet = -1;
+    short attrUse = -1;
     int off = 0;
     while (off < reckeys->buf_used)
     {
         const char *src = reckeys->buf + off;
-        char attrSet;
-        short attrUse;
         struct it_key key;
-        
-        memcpy (&attrSet, src, sizeof(attrSet));
-        src += sizeof(attrSet);
-
-        memcpy (&attrUse, src, sizeof(attrUse));
-        src += sizeof(attrUse);
+        int lead;
+    
+        lead = *src++;
 
+        if (!(lead & 1))
+        {
+            memcpy (&attrSet, src, sizeof(attrSet));
+            src += sizeof(attrSet);
+        }
+        if (!(lead & 2))
+        {
+            memcpy (&attrUse, src, sizeof(attrUse));
+            src += sizeof(attrUse);
+        }
         if (key_buf_used + 1024 > (ptr_top-ptr_i)*sizeof(char*))
             key_flush ();
         ++ptr_i;
@@ -472,47 +543,57 @@ static void addRecordKeyAny (const RecWord *p)
         w.attrUse = 1016;
         addRecordKey (&w);
     }
-    addRecordKey (p);
+    if (p->attrSet != -1)
+        addRecordKey (p);
 }
 
-
 #define FILE_READ_BUFSIZE 4096
-
-static int file_noread;
+struct file_read_info {
+    int file_noread;
+    int fd;
 #if FILE_READ_BUFSIZE
-static char *file_buf;
-static int file_offset;
-static int file_bufsize;
+    char *file_buf;
+    int file_offset;
+    int file_bufsize;
 #endif
+};
 
-static void file_read_start (int fd)
+static struct file_read_info *file_read_start (int fd)
 {
-    file_noread = 0;
+    struct file_read_info *fi = xmalloc (sizeof(*fi));
+
+    fi->fd = fd;
+    fi->file_noread = 0;
 #if FILE_READ_BUFSIZE
-    file_offset = 0;
-    file_buf = xmalloc (FILE_READ_BUFSIZE);
-    file_bufsize = read (fd, file_buf, FILE_READ_BUFSIZE);
+    fi->file_offset = 0;
+    fi->file_buf = xmalloc (FILE_READ_BUFSIZE);
+    fi->file_bufsize = read (fd, fi->file_buf, FILE_READ_BUFSIZE);
 #endif
+    return fi;
 }
 
-static void file_read_stop (int fd)
+static void file_read_stop (struct file_read_info *fi)
 {
+    assert (fi);
 #if FILE_READ_BUFSIZE
-    xfree (file_buf);
-    file_buf = NULL;
+    xfree (fi->file_buf);
+    fi->file_buf = NULL;
 #endif
+    xfree (fi);
 }
 
-static int file_read (int fd, char *buf, size_t count)
+static int file_read (void *handle, char *buf, size_t count)
 {
+    struct file_read_info *p = handle;
+    int fd = p->fd;
 #if FILE_READ_BUFSIZE
-    int l = file_bufsize - file_offset;
+    int l = p->file_bufsize - p->file_offset;
 
     if (count > l)
     {
         int r;
         if (l > 0)
-            memcpy (buf, file_buf + file_offset, l);
+            memcpy (buf, p->file_buf + p->file_offset, l);
         count = count-l;
         if (count > FILE_READ_BUFSIZE)
         {
@@ -521,12 +602,12 @@ static int file_read (int fd, char *buf, size_t count)
                 logf (LOG_FATAL|LOG_ERRNO, "read");
                 exit (1);
             }
-            file_bufsize = 0;
-            file_offset = 0;
-            file_noread += l+r;
+            p->file_bufsize = 0;
+            p->file_offset = 0;
+            p->file_noread += l+r;
             return l+r;
         }
-        file_bufsize = r = read (fd, file_buf, FILE_READ_BUFSIZE);
+        p->file_bufsize = r = read (fd, p->file_buf, FILE_READ_BUFSIZE);
         if (r == -1)
         {
             logf (LOG_FATAL|LOG_ERRNO, "read");
@@ -534,28 +615,28 @@ static int file_read (int fd, char *buf, size_t count)
         }
         else if (r <= count)
         {
-            file_offset = r;
-            memcpy (buf + l, file_buf, r);
-            file_noread += l+r;
+            p->file_offset = r;
+            memcpy (buf + l, p->file_buf, r);
+            p->file_noread += l+r;
             return l+r;
         }
         else
         {
-            file_offset = count;
-            memcpy (buf + l, file_buf, count - l);
-            file_noread += count;
+            p->file_offset = count;
+            memcpy (buf + l, p->file_buf, count - l);
+            p->file_noread += count;
             return count;
         }
     }
-    memcpy (buf, file_buf + file_offset, count);
-    file_offset += count;
-    file_noread += count;
+    memcpy (buf, p->file_buf + p->file_offset, count);
+    p->file_offset += count;
+    p->file_noread += count;
     return count;
 #else
     int r;
     r = read (fd, buf, count);
     if (r > 0)
-        file_noread += r;
+        p->file_noread += r;
     return r;
 #endif
 }
@@ -632,8 +713,8 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
                 }
             if (first)
             {
-                logf (LOG_WARN, "Record in file %s didn't contain match"
-                      " fields in (%d,%d)", fname, attrSet, attrUse);
+                logf (LOG_WARN, "Record didn't contain match"
+                      " fields in (%d,%d)", attrSet, attrUse);
                 return NULL;
             }
         }
@@ -703,33 +784,69 @@ static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
     return dstBuf;
 }
 
+struct recordLogInfo {
+    const char *fname;
+    char *op;
+    struct recordGroup *rGroup;
+};
+     
+static void recordLogPreamble (int level, const char *msg, void *info)
+{
+    struct recordLogInfo *p = info;
+    FILE *outf = log_file ();
+
+    if (level & LOG_LOG)
+        return ;
+    if (p->op) 
+        fprintf (outf, "%s of ", p->op);
+    fprintf (outf, "%s type %s\n", p->rGroup->recordType, p->fname);
+    log_event_start (NULL, NULL);
+}
+
 static int recordExtract (SYSNO *sysno, const char *fname,
                           struct recordGroup *rGroup, int deleteFlag,
-                          int fd, RecType recType, char *subType)
+                          struct file_read_info *fi, RecType recType,
+                          char *subType)
 {
     struct recExtractCtrl extractCtrl;
     int r;
     char *matchStr;
     SYSNO sysnotmp;
     Record rec;
+    struct recordLogInfo logInfo;
 
-    if (fd != -1)
+    logInfo.fname = fname;
+    logInfo.op = NULL;
+    logInfo.rGroup = rGroup;
+    log_event_start (recordLogPreamble, &logInfo);
+    
+    if (fi->fd != -1)
     {
-        extractCtrl.fd = fd;
-        /* extract keys */
+        /* we are going to read from a file, so prepare the extraction */
+        extractCtrl.fh = fi;
         extractCtrl.subType = subType;
         extractCtrl.init = wordInit;
         extractCtrl.add = addRecordKeyAny;
 
         reckeys.buf_used = 0;
+        reckeys.prevAttrUse = -1;
+        reckeys.prevAttrSet = -1;
         extractCtrl.readf = file_read;
         r = (*recType->extract)(&extractCtrl);
-  
+
         if (r)      
         {
+            /* error occured during extraction ... */
             logf (LOG_WARN, "Couldn't extract file %s, code %d", fname, r);
             return 0;
         }
+        if (reckeys.buf_used == 0)
+        {
+            /* the extraction process returned no information - the record
+               is probably empty */
+            logf (LOG_WARN, "Empty file %s", fname);
+            return 0;
+        }
     }
 
     /* perform match if sysno not known and if match criteria is specified */
@@ -753,21 +870,23 @@ static int recordExtract (SYSNO *sysno, const char *fname,
             }
             else
             {
-                logf (LOG_WARN, "Record not inserted");
+                logf (LOG_WARN, "Bad match criteria");
                 return 0;
             }
         }
     }
 
-    /* new record ? */
     if (! *sysno)
     {
+        /* new record */
         if (deleteFlag)
         {
-            logf (LOG_LOG, "? %s", fname);
+            logf (LOG_LOG, "Cannot delete new record");
             return 1;
         }
-        logf (LOG_LOG, "add %s %s", rGroup->recordType, fname);
+        logInfo.op = "add";
+        if (rGroup->fileVerboseFlag)
+            logf (LOG_LOG, "add %s %s", rGroup->recordType, fname);
         rec = rec_new (records);
         *sysno = rec->sysno;
 
@@ -781,62 +900,75 @@ static int recordExtract (SYSNO *sysno, const char *fname,
     }
     else
     {
+        /* record already exists */
         struct recKeys delkeys;
 
         rec = rec_get (records, *sysno);
-
+        assert (rec);
         delkeys.buf_used = rec->size[recInfo_delKeys];
 	delkeys.buf = rec->info[recInfo_delKeys];
         flushRecordKeys (*sysno, 0, &delkeys, rec->info[recInfo_databaseName]);
         if (deleteFlag)
         {
+            /* record going to be deleted */
+            logInfo.op = "delete";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot delete %s: storeKeys false",
-                      fname);
+                logf (LOG_WARN, "cannot delete; storeKeys false");
             }
             else
             {
-                SYSNO sysnoz = 0;
-                logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname);
+                if (rGroup->fileVerboseFlag)
+                    logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname);
                 records_deleted++;
                 if (matchStr)
-                    dict_insert (matchDict, matchStr, sizeof(sysnoz), &sysnoz);
+                    dict_delete (matchDict, matchStr);
                 rec_del (records, &rec);
             }
             return 1;
         }
         else
         {
+            /* record going to be updated */
+            logInfo.op = "update";
             if (!delkeys.buf_used)
             {
-                logf (LOG_WARN, "cannot update %s: storeKeys false",
-                      fname);
+                logf (LOG_WARN, "cannot update; storeKeys false");
             }
             else
             {
-                logf (LOG_LOG, "update %s %s", rGroup->recordType,
-                      fname);
+                if (rGroup->fileVerboseFlag)
+                    logf (LOG_LOG, "update %s %s", rGroup->recordType, fname);
                 flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); 
                 records_updated++;
             }
         }
     }
+    /* update file type */
     xfree (rec->info[recInfo_fileType]);
     rec->info[recInfo_fileType] =
         rec_strdup (rGroup->recordType, &rec->size[recInfo_fileType]);
 
+    /* update filename */
     xfree (rec->info[recInfo_filename]);
     rec->info[recInfo_filename] =
         rec_strdup (fname, &rec->size[recInfo_filename]);
 
+    /* update delete keys */
     xfree (rec->info[recInfo_delKeys]);
     if (reckeys.buf_used > 0 && rGroup->flagStoreKeys == 1)
     {
+#if 1
+        rec->size[recInfo_delKeys] = reckeys.buf_used;
+        rec->info[recInfo_delKeys] = reckeys.buf;
+        reckeys.buf = NULL;
+        reckeys.buf_max = 0;
+#else
         rec->info[recInfo_delKeys] = xmalloc (reckeys.buf_used);
         rec->size[recInfo_delKeys] = reckeys.buf_used;
         memcpy (rec->info[recInfo_delKeys], reckeys.buf,
                 rec->size[recInfo_delKeys]);
+#endif
     }
     else
     {
@@ -844,27 +976,29 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         rec->size[recInfo_delKeys] = 0;
     }
 
+    /* update store data */
     xfree (rec->info[recInfo_storeData]);
     if (rGroup->flagStoreData == 1)
     {
-        rec->size[recInfo_storeData] = file_noread;
-        rec->info[recInfo_storeData] = xmalloc (file_noread);
+        rec->size[recInfo_storeData] = fi->file_noread;
+        rec->info[recInfo_storeData] = xmalloc (fi->file_noread);
 #if FILE_READ_BUFSIZE
-        if (file_noread < FILE_READ_BUFSIZE)
-	    memcpy (rec->info[recInfo_storeData], file_buf, file_noread);
+        if (fi->file_noread < FILE_READ_BUFSIZE)
+	    memcpy (rec->info[recInfo_storeData], fi->file_buf,
+                    fi->file_noread);
         else
 #endif
         {
-            if (lseek (fd, 0L, SEEK_SET) < 0)
+            if (lseek (fi->fd, 0L, SEEK_SET) < 0)
             {
                 logf (LOG_ERRNO|LOG_FATAL, "seek to 0 in %s", fname);
                 exit (1);
             }
-            if (read (fd, rec->info[recInfo_storeData], file_noread) 
-                < file_noread)
+            if (read (fi->fd, rec->info[recInfo_storeData], fi->file_noread) 
+                < fi->file_noread)
             {
                 logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
-                      file_noread, fname);
+                      fi->file_noread, fname);
                 exit (1);
             }
         }
@@ -874,10 +1008,12 @@ static int recordExtract (SYSNO *sysno, const char *fname,
         rec->info[recInfo_storeData] = NULL;
         rec->size[recInfo_storeData] = 0;
     }
+    /* update database name */
     xfree (rec->info[recInfo_databaseName]);
     rec->info[recInfo_databaseName] =
         rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]); 
 
+    /* commit this record */
     rec_put (records, &rec);
     return 1;
 }
@@ -885,7 +1021,7 @@ static int recordExtract (SYSNO *sysno, const char *fname,
 int fileExtract (SYSNO *sysno, const char *fname, 
                  const struct recordGroup *rGroupP, int deleteFlag)
 {
-    int i, fd;
+    int r, i, fd;
     char gprefix[128];
     char ext[128];
     char ext_res[128];
@@ -893,6 +1029,7 @@ int fileExtract (SYSNO *sysno, const char *fname,
     RecType recType;
     struct recordGroup rGroupM;
     struct recordGroup *rGroup = &rGroupM;
+    struct file_read_info *fi;
 
     memcpy (rGroup, rGroupP, sizeof(*rGroupP));
    
@@ -924,14 +1061,16 @@ int fileExtract (SYSNO *sysno, const char *fname,
             sprintf (ext_res, "%srecordType", gprefix);
             if (!(rGroup->recordType = res_get (common_resource, ext_res)))
             {
-                logf (LOG_LOG, "? %s", fname);
+                if (rGroup->fileVerboseFlag)
+                    logf (LOG_LOG, "? %s", fname);
                 return 0;
             }
         }
     }
     if (!rGroup->recordType)
     {
-        logf (LOG_LOG, "? record %s", fname);
+        if (rGroup->fileVerboseFlag)
+            logf (LOG_LOG, "? record %s", fname);
         return 0;
     }
     if (!(recType = recType_byName (rGroup->recordType, subType)))
@@ -1001,11 +1140,12 @@ int fileExtract (SYSNO *sysno, const char *fname,
             return 0;
         }
     }
-    file_read_start (fd);
-    recordExtract (sysno, fname, rGroup, deleteFlag, fd, recType, subType);
-    file_read_stop (fd);
+    fi = file_read_start (fd);
+    r = recordExtract (sysno, fname, rGroup, deleteFlag, fi, recType, subType);
+    log_event_start (NULL, NULL);
+    file_read_stop (fi);
     if (fd != -1)
         close (fd);
-    return 1;
+    return r;
 }