X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=4808868b1f07a5fbfb7a8522ee6d2ff36a674fde;hb=2d8e1f86f73086330f4e9fd181d98542529ee1f9;hp=34cb68b2424e2ece028a38c5c20f70d269e34d5a;hpb=51a52e1b014d5237581f74379916b8ce92cafef7;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 34cb68b..4808868 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,10 +1,33 @@ /* - * Copyright (C) 1995, Index Data I/S + * Copyright (C) 1994-1995, Index Data I/S * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.1 1995-09-01 14:06:35 adam + * Revision 1.8 1995-09-14 07:48:22 adam + * Record control management. + * + * Revision 1.7 1995/09/11 13:09:32 adam + * More work on relevance feedback. + * + * Revision 1.6 1995/09/08 14:52:27 adam + * Minor changes. Dictionary is lower case now. + * + * Revision 1.5 1995/09/06 16:11:16 adam + * Option: only one word key per file. + * + * Revision 1.4 1995/09/05 15:28:39 adam + * More work on search engine. + * + * Revision 1.3 1995/09/04 12:33:41 adam + * Various cleanup. YAZ util used instead. + * + * Revision 1.2 1995/09/04 09:10:34 adam + * More work on index add/del/update. + * Merge sort implemented. + * Initial work on z39 server. + * + * Revision 1.1 1995/09/01 14:06:35 adam * Split of work into more files. * */ @@ -14,16 +37,18 @@ #include #include -#include +#include +#include #include "index.h" -#define KEY_BUF_SIZE 100000 - static Dict file_idx; static SYSNO sysno_next; static int key_fd = -1; +static int sys_idx_fd = -1; static char *key_buf; -static int key_offset; +static int key_offset, key_buf_size; +static int key_cmd; +static int key_sysno; void key_open (const char *fname) { @@ -32,18 +57,16 @@ void key_open (const char *fname) return; if ((key_fd = open (fname, O_RDWR|O_CREAT, 0666)) == -1) { - log (LOG_FATAL|LOG_ERRNO, "Creat %s", fname); - exit (1); - } - if (!(key_buf = malloc (KEY_BUF_SIZE))) - { - log (LOG_FATAL|LOG_ERRNO, "malloc"); + logf (LOG_FATAL|LOG_ERRNO, "open %s", fname); exit (1); } + logf (LOG_DEBUG, "key_open of %s", fname); + key_buf_size = 49100; + key_buf = xmalloc (key_buf_size); key_offset = 0; - if (!(file_idx = dict_open ("fileidx", 10, 1))) + if (!(file_idx = dict_open (FNAME_FILE_DICT, 40, 1))) { - log (LOG_FATAL, "dict_open fail of %s", "fileidx"); + logf (LOG_FATAL, "dict_open fail of %s", "fileidx"); exit (1); } file_key = dict_lookup (file_idx, "."); @@ -51,29 +74,42 @@ void key_open (const char *fname) memcpy (&sysno_next, (char*)file_key+1, sizeof(sysno_next)); else sysno_next = 1; + if ((sys_idx_fd = open (FNAME_SYS_IDX, O_RDWR|O_CREAT, 0666)) == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "open %s", FNAME_SYS_IDX); + exit (1); + } } -void key_close (void) +int key_close (void) { if (key_fd == -1) - return; + { + logf (LOG_DEBUG, "key_close - but no file"); + return 0; + } close (key_fd); + close (sys_idx_fd); dict_insert (file_idx, ".", sizeof(sysno_next), &sysno_next); dict_close (file_idx); key_fd = -1; + xfree (key_buf); + return 1; } -void key_flush (void) +void wordFlush (int sysno) { size_t i = 0; int w; - + + if (key_fd == -1) + return; while (i < key_offset) { w = write (key_fd, key_buf + i, key_offset - i); if (w == -1) { - log (LOG_FATAL|LOG_ERRNO, "Write key fail"); + logf (LOG_FATAL|LOG_ERRNO, "Write key fail"); exit (1); } i += w; @@ -81,57 +117,49 @@ void key_flush (void) key_offset = 0; } -void key_write (int cmd, struct it_key *k, const char *str) +static void wordInit (RecWord *p) { - char x = cmd; - size_t slen = strlen(str); - - if (key_offset + sizeof(*k) + slen >= KEY_BUF_SIZE - 2) - key_flush (); - memcpy (key_buf + key_offset, &x, 1); - key_offset++; - memcpy (key_buf + key_offset, k, sizeof(*k)); - key_offset += sizeof(*k); - memcpy (key_buf + key_offset, str, slen+1); - key_offset += slen+1; + p->attrSet = 1; + p->attrUse = 1; + p->which = Word_String; } -void text_extract (SYSNO sysno, int cmd, const char *fname) +static void wordAdd (const RecWord *p) { - FILE *inf; - struct it_key k; - int seqno = 1; - int c; - char w[256]; - - log (LOG_DEBUG, "Text extract of %d", sysno); - k.sysno = sysno; - inf = fopen (fname, "r"); - if (!inf) + struct it_key key; + char x; + size_t i; + + if (key_offset + 1000 > key_buf_size) { - log (LOG_WARN|LOG_ERRNO, "open %s", fname); - return; + char *new_key_buf; + + key_buf_size *= 2; + new_key_buf = xmalloc (2*key_buf_size); + memcpy (new_key_buf, key_buf, key_offset); + xfree (key_buf); + key_buf = new_key_buf; } - while ((c=getc (inf)) != EOF) + switch (p->which) { - int i = 0; - while (i < 254 && c != EOF && isalnum(c)) - { - w[i++] = c; - c = getc (inf); - } - if (i) - { - w[i] = 0; - - k.seqno = seqno++; - k.field = 0; - key_write (cmd, &k, w); - } - if (c == EOF) - break; + case Word_String: + for (i = 0; p->u.string[i]; i++) + key_buf[key_offset++] = index_char_cvt (p->u.string[i]); + key_buf[key_offset++] = '\0'; + break; + default: + return ; } - fclose (inf); + x = (key_cmd == 'a') ? 1 : 0; + memcpy (key_buf + key_offset, &x, 1); + key_offset++; + + key.sysno = key_sysno; + key.attrSet = p->attrSet; + key.attrUse = p->attrUse; + key.seqno = p->seqno; + memcpy (key_buf + key_offset, &key, sizeof(key)); + key_offset += sizeof(key); } void file_extract (int cmd, const char *fname, const char *kname) @@ -142,8 +170,11 @@ void file_extract (int cmd, const char *fname, const char *kname) char ext_res[128]; const char *file_type; void *file_info; + FILE *inf; + struct recExtractCtrl extractCtrl; + RecType rt; - log (LOG_DEBUG, "%c %s k=%s", cmd, fname, kname); + logf (LOG_DEBUG, "%c %s k=%s", cmd, fname, kname); for (i = strlen(fname); --i >= 0; ) if (fname[i] == '/') { @@ -164,11 +195,30 @@ void file_extract (int cmd, const char *fname, const char *kname) { sysno = sysno_next++; dict_insert (file_idx, kname, sizeof(sysno), &sysno); + lseek (sys_idx_fd, sysno * SYS_IDX_ENTRY_LEN, SEEK_SET); + write (sys_idx_fd, kname, strlen(kname)+1); } else memcpy (&sysno, (char*) file_info+1, sizeof(sysno)); + + if (!(inf = fopen (fname, "r"))) + { + logf (LOG_WARN|LOG_ERRNO, "open %s", fname); + return; + } if (!strcmp (file_type, "text")) - text_extract (sysno, cmd, fname); + rt = recTypeText; + else + return; + extractCtrl.inf = inf; + extractCtrl.subType = ""; + extractCtrl.init = wordInit; + extractCtrl.add = wordAdd; + key_sysno = sysno; + key_cmd = cmd; + (*rt->extract)(&extractCtrl); + fclose (inf); + wordFlush (sysno); }