X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=86de3b25cc75b51d67a97e9d1f7de3daf7675ade;hb=3d9f3bffb46938b54ea2aaf3738d8579fdcfdebd;hp=7c325cafb2e346358c09cb45493b17c7973319f9;hpb=c41c84a497ae744aa825a90f144c85b54f1cd4bb;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 7c325ca..86de3b2 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,10 +1,39 @@ /* - * Copyright (C) 1994-1999, Index Data + * Copyright (C) 1994-2001, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.100 2000-03-20 19:08:36 adam + * Revision 1.109 2001-10-15 19:53:43 adam + * POSIX thread updates. First work on term sets. + * + * Revision 1.108 2001/06/14 11:44:56 adam + * Bug fix: default storeKeys setting wasn't read when group was specified. + * + * Revision 1.107 2001/05/28 13:58:48 adam + * Call flushSortKeys when record is skipped to fix bad re-use of + * sort keys to whatever next record that comes in. + * + * Revision 1.106 2000/12/05 12:22:53 adam + * Termlist source implemented (so that we can index values of XML/SGML + * attributes). + * + * Revision 1.105 2000/12/05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.104 2000/09/05 14:04:05 adam + * Updates for prefix 'yaz_' for YAZ log functions. + * + * Revision 1.103 2000/05/18 12:01:36 adam + * System call times(2) used again. More 64-bit fixes. + * + * Revision 1.102 2000/05/15 15:32:33 adam + * Added 64 bit file input. + * + * Revision 1.101 2000/05/15 13:02:39 adam + * Minor change. + * + * Revision 1.100 2000/03/20 19:08:36 adam * Added remote record import using Z39.50 extended services and Segment * Requests. * @@ -375,6 +404,12 @@ #include "zinfo.h" +#if _FILE_OFFSET_BITS == 64 +#define PRINTF_OFF_T "%Ld" +#else +#define PRINTF_OFF_T "%ld" +#endif + #ifndef ZEBRASDR #define ZEBRASDR 0 #endif @@ -401,6 +436,7 @@ static int records_processed = 0; static ZebraExplainInfo zti = NULL; + static void logRecord (int showFlag) { if (!showFlag) @@ -423,7 +459,7 @@ int key_open (struct recordGroup *rGroup, int mem) char *recordCompression; int record_compression = REC_COMPRESS_NONE; if (!mem) - mem = atoi(res_get_def (common_resource, "memMax", "4"))*1024*1024; + mem = atoi(res_get_def (common_resource, "memMax", "16"))*1024*1024; if (mem < 50000) mem = 50000; key_buf = (char **) xmalloc (mem); @@ -721,6 +757,19 @@ static void addIndexString (RecWord *p, const char *string, int length) *dst++ = lead; +#if SU_SCHEME + if ((lead & 3) < 3) + { + int ch = zebraExplain_lookupSU (zti, attrSet, attrUse); + if (ch < 0) + { + ch = zebraExplain_addSU (zti, attrSet, attrUse); + } + assert (ch > 0); + memcpy (dst, &ch, sizeof(ch)); + dst += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (dst, &attrSet, sizeof(attrSet)); @@ -731,6 +780,7 @@ static void addIndexString (RecWord *p, const char *string, int length) memcpy (dst, &attrUse, sizeof(attrUse)); dst += sizeof(attrUse); } +#endif *dst++ = p->reg_type; memcpy (dst, string, length); dst += length; @@ -889,8 +939,11 @@ static void flushSortKeys (SYSNO sysno, int cmd) while (sk) { struct sortKey *sk_next = sk->next; - sortIdx_type (sortIdx, sk->attrUse); - sortIdx_add (sortIdx, sk->string, sk->length); + if (cmd >= 0) + { + sortIdx_type (sortIdx, sk->attrUse); + sortIdx_add (sortIdx, sk->string, sk->length); + } xfree (sk->string); xfree (sk); sk = sk_next; @@ -900,20 +953,31 @@ static void flushSortKeys (SYSNO sysno, int cmd) static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys) { +#if SU_SCHEME +#else unsigned char attrSet = (unsigned char) -1; unsigned short attrUse = (unsigned short) -1; +#endif int seqno = 0; int off = 0; + int ch = 0; zebraExplain_recordCountIncrement (zti, cmd ? 1 : -1); while (off < reckeys->buf_used) { const char *src = reckeys->buf + off; struct it_key key; - int lead, ch; + int lead; lead = *src++; +#if SU_SCHEME + if ((lead & 3) < 3) + { + memcpy (&ch, src, sizeof(ch)); + src += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (&attrSet, src, sizeof(attrSet)); @@ -924,16 +988,25 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys) memcpy (&attrUse, src, sizeof(attrUse)); src += sizeof(attrUse); } +#endif if (key_buf_used + 1024 > (ptr_top-ptr_i)*sizeof(char*)) key_flush (); ++ptr_i; + key_buf[ptr_top-ptr_i] = (char*)key_buf + key_buf_used; +#if SU_SCHEME +#else ch = zebraExplain_lookupSU (zti, attrSet, attrUse); if (ch < 0) + { ch = zebraExplain_addSU (zti, attrSet, attrUse); + yaz_log (LOG_LOG, "addSU cmd=%d set=%d use=%d SU=%d", + cmd, attrSet, attrUse, ch); + } +#endif assert (ch > 0); - key_buf_used += key_SU_code (ch, ((char*)key_buf) + key_buf_used); + key_buf_used += key_SU_encode (ch, ((char*)key_buf) + key_buf_used); while (*src) ((char*)key_buf) [key_buf_used++] = *src++; @@ -964,13 +1037,22 @@ static const char **searchRecordKey (struct recKeys *reckeys, int off = 0; int startSeq = -1; int i; + int seqno = 0; +#if SU_SCHEME + int chS, ch; +#else short attrUse; char attrSet; - int seqno = 0; +#endif for (i = 0; i<32; i++) ws[i] = NULL; - + +#if SU_SCHEME + chS = zebraExplain_lookupSU (zti, attrSetS, attrUseS); + if (chS < 0) + return ws; +#endif while (off < reckeys->buf_used) { @@ -979,7 +1061,13 @@ static const char **searchRecordKey (struct recKeys *reckeys, int lead; lead = *src++; - +#if SU_SCHEME + if ((lead & 3)<3) + { + memcpy (&ch, src, sizeof(ch)); + src += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (&attrSet, src, sizeof(attrSet)); @@ -990,6 +1078,7 @@ static const char **searchRecordKey (struct recKeys *reckeys, memcpy (&attrUse, src, sizeof(attrUse)); src += sizeof(attrUse); } +#endif wstart = src; while (*src++) ; @@ -1000,7 +1089,13 @@ static const char **searchRecordKey (struct recKeys *reckeys, memcpy (&seqno, src, sizeof(seqno)); src += sizeof(seqno); } - if (attrUseS == attrUse && attrSetS == attrSet) + if ( +#if SU_SCHEME + ch == chS +#else + attrUseS == attrUse && attrSetS == attrSet +#endif + ) { int woff; @@ -1262,7 +1357,7 @@ struct recordLogInfo { static void recordLogPreamble (int level, const char *msg, void *info) { struct recordLogInfo *p = (struct recordLogInfo *) info; - FILE *outf = log_file (); + FILE *outf = yaz_log_file (); if (level & LOG_LOG) return ; @@ -1324,7 +1419,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, extractCtrl.flagShowRecords = !rGroup->flagRw; if (!rGroup->flagRw) - printf ("File: %s %ld\n", fname, (long) recordOffset); + printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset); logInfo.fname = fname; logInfo.recordOffset = recordOffset; @@ -1343,8 +1438,8 @@ static int recordExtract (SYSNO *sysno, const char *fname, if (rGroup->flagRw && records_processed < rGroup->fileVerboseLimit) { - logf (LOG_WARN, "fail %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + logf (LOG_WARN, "fail %s %s " PRINTF_OFF_T, rGroup->recordType, + fname, recordOffset); } return 0; } @@ -1354,8 +1449,9 @@ static int recordExtract (SYSNO *sysno, const char *fname, is probably empty - unless flagShowRecords is in use */ if (!rGroup->flagRw) return 1; - logf (LOG_WARN, "No keys generated for file %s", fname); - logf (LOG_WARN, " The file is probably empty"); + + logf (LOG_WARN, "empty %s %s " PRINTF_OFF_T, rGroup->recordType, + fname, recordOffset); return 1; } } @@ -1392,14 +1488,14 @@ static int recordExtract (SYSNO *sysno, const char *fname, /* new record */ if (deleteFlag) { - logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, rGroup->recordType, + fname, recordOffset); logf (LOG_WARN, "cannot delete record above (seems new)"); return 1; } if (records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "add %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + logf (LOG_LOG, "add %s %s " PRINTF_OFF_T, rGroup->recordType, + fname, recordOffset); rec = rec_new (records); *sysno = rec->sysno; @@ -1427,8 +1523,9 @@ static int recordExtract (SYSNO *sysno, const char *fname, if (recordAttr->runNumber == zebraExplain_runNumberIncrement (zti, 0)) { - logf (LOG_LOG, "skipped %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + logf (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, rGroup->recordType, + fname, recordOffset); + flushSortKeys (*sysno, -1); rec_rm (&rec); logRecord (0); return 1; @@ -1442,15 +1539,15 @@ static int recordExtract (SYSNO *sysno, const char *fname, /* record going to be deleted */ if (!delkeys.buf_used) { - logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, + rGroup->recordType, fname, recordOffset); logf (LOG_WARN, "cannot delete file above, storeKeys false"); } else { if (records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, + rGroup->recordType, fname, recordOffset); records_deleted++; if (matchStr) dict_delete (matchDict, matchStr); @@ -1465,15 +1562,15 @@ static int recordExtract (SYSNO *sysno, const char *fname, /* record going to be updated */ if (!delkeys.buf_used) { - logf (LOG_LOG, "update %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + logf (LOG_LOG, "update %s %s " PRINTF_OFF_T, + rGroup->recordType, fname, recordOffset); logf (LOG_WARN, "cannot update file above, storeKeys false"); } else { if (records_processed < rGroup->fileVerboseLimit) - logf (LOG_LOG, "update %s %s %ld", rGroup->recordType, - fname, (long) recordOffset); + logf (LOG_LOG, "update %s %s " PRINTF_OFF_T, + rGroup->recordType, fname, recordOffset); flushRecordKeys (*sysno, 1, &reckeys); records_updated++; } @@ -1530,8 +1627,8 @@ static int recordExtract (SYSNO *sysno, const char *fname, xmalloc (recordAttr->recordSize); if (lseek (fi->fd, recordOffset, SEEK_SET) < 0) { - logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", - (long) recordOffset, fname); + logf (LOG_ERRNO|LOG_FATAL, "seek to " PRINTF_OFF_T " in %s", + recordOffset, fname); exit (1); } if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize) @@ -1674,11 +1771,14 @@ int fileExtract (SYSNO *sysno, const char *fname, const char *sval; sprintf (ext_res, "%sstoreKeys.%s", gprefix, ext); - if (!(sval = res_get (common_resource, ext_res))) + sval = res_get (common_resource, ext_res); + if (!sval) { sprintf (ext_res, "%sstoreKeys", gprefix); sval = res_get (common_resource, ext_res); } + if (!sval) + sval = res_get (common_resource, "storeKeys"); if (sval) rGroup->flagStoreKeys = atoi (sval); }