From bceca936710690858ea162e7bc22de819f0e280f Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Wed, 12 May 1999 13:08:05 +0000 Subject: [PATCH] First version of ISAMS. --- bfile/bfile.c | 17 ++-- bfile/cfile.c | 21 +++-- bfile/cfile.h | 41 +--------- bfile/mfile.c | 17 ++-- dict/dcompact.c | 6 +- include/bfile.h | 55 ++----------- include/isams.h | 55 +++++++++++++ include/mfile.h | 39 +-------- include/rsisamc.h | 15 +--- index/Makefile.in | 4 +- index/index.h | 10 ++- index/invstat.c | 52 ++++++++++-- index/kcompare.c | 26 +++++- index/kinput.c | 74 +++++++++++++++-- index/trunc.c | 77 +++++++++++++++++- index/zebraapi.c | 32 ++++++-- index/zserver.h | 8 +- isamc/Makefile.in | 4 +- isamc/isams.c | 235 +++++++++++++++++++++++++++++++++++++++++++++++++++++ rset/Makefile.in | 4 +- test/gils/test.sh | 2 +- 21 files changed, 591 insertions(+), 203 deletions(-) create mode 100644 include/isams.h create mode 100644 isamc/isams.c diff --git a/bfile/bfile.c b/bfile/bfile.c index 6700679..5c0dec2 100644 --- a/bfile/bfile.c +++ b/bfile/bfile.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: bfile.c,v $ - * Revision 1.27 1999-02-02 14:50:01 adam + * Revision 1.28 1999-05-12 13:08:05 adam + * First version of ISAMS. + * + * Revision 1.27 1999/02/02 14:50:01 adam * Updated WIN32 code specific sections. Changed header. * * Revision 1.26 1998/02/17 10:32:52 adam @@ -219,20 +222,20 @@ BFile bf_open (BFiles bfs, const char *name, int block_size, int wflag) return(tmp); } -int bf_read (BFile bf, int no, int offset, int num, void *buf) +int bf_read (BFile bf, int no, int offset, int nbytes, void *buf) { int r; - if (bf->cf && (r=cf_read (bf->cf, no, offset, num, buf)) != -1) + if (bf->cf && (r=cf_read (bf->cf, no, offset, nbytes, buf)) != -1) return r; - return mf_read (bf->mf, no, offset, num, buf); + return mf_read (bf->mf, no, offset, nbytes, buf); } -int bf_write (BFile bf, int no, int offset, int num, const void *buf) +int bf_write (BFile bf, int no, int offset, int nbytes, const void *buf) { if (bf->cf) - return cf_write (bf->cf, no, offset, num, buf); - return mf_write (bf->mf, no, offset, num, buf); + return cf_write (bf->cf, no, offset, nbytes, buf); + return mf_write (bf->mf, no, offset, nbytes, buf); } int bf_commitExists (BFiles bfs) diff --git a/bfile/cfile.c b/bfile/cfile.c index 5a6209b..542b096 100644 --- a/bfile/cfile.c +++ b/bfile/cfile.c @@ -1,10 +1,13 @@ /* - * Copyright (C) 1995-1998, Index Data ApS + * Copyright (C) 1995-1999, Index Data ApS * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: cfile.c,v $ - * Revision 1.23 1998-10-15 13:09:29 adam + * Revision 1.24 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.23 1998/10/15 13:09:29 adam * Minor changes. * * Revision 1.22 1998/10/13 20:07:22 adam @@ -503,14 +506,14 @@ int cf_new (CFile cf, int no) } -int cf_read (CFile cf, int no, int offset, int num, void *buf) +int cf_read (CFile cf, int no, int offset, int nbytes, void *buf) { int block; assert (cf); if (!(block = cf_lookup (cf, no))) return -1; - if (!mf_read (cf->block_mf, block, offset, num, buf)) + if (!mf_read (cf->block_mf, block, offset, nbytes, buf)) { logf (LOG_FATAL|LOG_ERRNO, "cf_read no=%d, block=%d", no, block); exit (1); @@ -518,7 +521,7 @@ int cf_read (CFile cf, int no, int offset, int num, void *buf) return 1; } -int cf_write (CFile cf, int no, int offset, int num, const void *buf) +int cf_write (CFile cf, int no, int offset, int nbytes, const void *buf) { int block; @@ -526,16 +529,16 @@ int cf_write (CFile cf, int no, int offset, int num, const void *buf) if (!(block = cf_lookup (cf, no))) { block = cf_new (cf, no); - if (offset || num) + if (offset || nbytes) { mf_read (cf->rmf, no, 0, 0, cf->iobuf); - memcpy (cf->iobuf + offset, buf, num); + memcpy (cf->iobuf + offset, buf, nbytes); buf = cf->iobuf; offset = 0; - num = 0; + nbytes = 0; } } - if (mf_write (cf->block_mf, block, offset, num, buf)) + if (mf_write (cf->block_mf, block, offset, nbytes, buf)) { logf (LOG_FATAL|LOG_ERRNO, "cf_write no=%d, block=%d", no, block); exit (1); diff --git a/bfile/cfile.h b/bfile/cfile.h index 8006599..68ee552 100644 --- a/bfile/cfile.h +++ b/bfile/cfile.h @@ -1,42 +1,9 @@ /* - * Copyright (C) 1995-1998, Index Data + * Copyright (C) 1995-1999, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * - * $Log: cfile.h,v $ - * Revision 1.10 1998-10-15 13:09:31 adam - * Minor changes. - * - * Revision 1.9 1998/08/07 15:07:15 adam - * Fixed but in cf_commit_flat. - * - * Revision 1.8 1996/04/18 16:02:56 adam - * Changed logging a bit. - * Removed warning message when commiting flat shadow files. - * - * Revision 1.7 1996/02/07 14:03:48 adam - * Work on flat indexed shadow files. - * - * Revision 1.6 1996/02/07 10:08:45 adam - * Work on flat shadow (not finished yet). - * - * Revision 1.5 1995/12/15 12:36:52 adam - * Moved hash file information to union. - * Renamed commit files. - * - * Revision 1.4 1995/12/11 09:03:54 adam - * New function: cf_unlink. - * New member of commit file head: state (0) deleted, (1) hash file. - * - * Revision 1.3 1995/12/01 16:24:29 adam - * Commit files use separate meta file area. - * - * Revision 1.2 1995/12/01 11:37:23 adam - * Cached/commit files implemented as meta-files. - * - * Revision 1.1 1995/11/30 08:33:12 adam - * Started work on commit facility. - * + * $Id: cfile.h,v 1.11 1999-05-12 13:08:06 adam Exp $ */ #ifndef CFILE_H @@ -90,8 +57,8 @@ typedef struct CFile_struct int cf_close (CFile cf); CFile cf_open (MFile mf, MFile_area area, const char *fname, int block_size, int wflag, int *firstp); -int cf_read (CFile cf, int no, int offset, int num, void *buf); -int cf_write (CFile cf, int no, int offset, int num, const void *buf); +int cf_read (CFile cf, int no, int offset, int nbytes, void *buf); +int cf_write (CFile cf, int no, int offset, int nbytes, const void *buf); void cf_unlink (CFile cf); void cf_commit (CFile cf); diff --git a/bfile/mfile.c b/bfile/mfile.c index 051c8fc..7720bf9 100644 --- a/bfile/mfile.c +++ b/bfile/mfile.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: mfile.c,v $ - * Revision 1.32 1999-04-28 14:53:07 adam + * Revision 1.33 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.32 1999/04/28 14:53:07 adam * Fixed stupid bug regarding split-files. * * Revision 1.31 1999/02/18 12:49:33 adam @@ -390,10 +393,12 @@ MFile mf_open(MFile_area ma, const char *name, int block_size, int wflag) assert (ma); for (mnew = ma->mfiles; mnew; mnew = mnew->next) if (!strcmp(name, mnew->name)) + { if (mnew->open) abort(); else break; + } if (!mnew) { mnew = xmalloc(sizeof(*mnew)); @@ -471,16 +476,18 @@ int mf_close(MFile mf) /* * Read one block from a metafile. Interface mirrors bfile. */ -int mf_read(MFile mf, int no, int offset, int num, void *buf) +int mf_read(MFile mf, int no, int offset, int nbytes, void *buf) { int rd, toread; if ((rd = file_position(mf, no, offset)) < 0) + { if (rd == -2) return 0; else exit(1); - toread = num ? num : mf->blocksize; + } + toread = nbytes ? nbytes : mf->blocksize; if ((rd = read(mf->files[mf->cur_file].fd, buf, toread)) < 0) { logf (LOG_FATAL|LOG_ERRNO, "mf_read: Read failed (%s)", @@ -496,7 +503,7 @@ int mf_read(MFile mf, int no, int offset, int num, void *buf) /* * Write. */ -int mf_write(MFile mf, int no, int offset, int num, const void *buf) +int mf_write(MFile mf, int no, int offset, int nbytes, const void *buf) { int ps, nblocks, towrite; mf_dir *dp; @@ -572,7 +579,7 @@ int mf_write(MFile mf, int no, int offset, int num, const void *buf) nblocks * mf->blocksize; } } - towrite = num ? num : mf->blocksize; + towrite = nbytes ? nbytes : mf->blocksize; if (write(mf->files[mf->cur_file].fd, buf, towrite) < towrite) { logf (LOG_FATAL|LOG_ERRNO, "Write failed for file %s part %d", diff --git a/dict/dcompact.c b/dict/dcompact.c index 88fd6ba..f2d5fcd 100644 --- a/dict/dcompact.c +++ b/dict/dcompact.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: dcompact.c,v $ - * Revision 1.2 1999-03-09 16:27:49 adam + * Revision 1.3 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.2 1999/03/09 16:27:49 adam * More work on SDRKit integration. * * Revision 1.1 1999/03/09 13:07:06 adam @@ -24,7 +27,6 @@ int dict_compact (BFiles bfs, const char *from_name, const char *to_name) int no_dir = 0; Dict from, to; int *map, i; - map = xmalloc (100); from = dict_open (bfs, from_name, 0, 0); if (!from) return -1; diff --git a/include/bfile.h b/include/bfile.h index ec86cd9..af8c209 100644 --- a/include/bfile.h +++ b/include/bfile.h @@ -3,52 +3,7 @@ * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * - * $Log: bfile.h,v $ - * Revision 1.16 1999-02-02 14:50:29 adam - * Updated WIN32 code specific sections. Changed header. - * - * Revision 1.15 1997/09/17 12:19:07 adam - * Zebra version corresponds to YAZ version 1.4. - * Changed Zebra server so that it doesn't depend on global common_resource. - * - * Revision 1.14 1997/09/05 15:29:58 adam - * Changed prototype for chr_map_input - added const. - * Added support for C++, headers uses extern "C" for public definitions. - * - * Revision 1.13 1996/10/29 13:43:07 adam - * Added a few comments. - * - * Revision 1.12 1996/03/26 16:00:44 adam - * The directory of the shadow table can be specified by the new - * bf_lockDir call. - * - * Revision 1.11 1995/12/08 16:20:39 adam - * New commit utilities - used for 'save' update. - * - * Revision 1.10 1995/12/01 16:24:33 adam - * Commit files use separate meta file area. - * - * Revision 1.9 1995/12/01 11:37:46 adam - * Cached/commit files implemented as meta-files. - * - * Revision 1.8 1995/11/30 08:33:29 adam - * Started work on commit facility. - * - * Revision 1.7 1995/09/04 12:33:35 adam - * Various cleanup. YAZ util used instead. - * - * Revision 1.6 1994/09/14 13:10:35 quinn - * Small changes - * - * Revision 1.5 1994/08/24 08:45:52 quinn - * Using mfile. - * - * Revision 1.4 1994/08/17 15:38:28 adam - * Include of util.h. - * - * Revision 1.3 1994/08/17 14:09:47 quinn - * Small changes - * + * $Id: bfile.h,v 1.17 1999-05-12 13:08:06 adam Exp $ */ #ifndef BFILE_H @@ -85,18 +40,18 @@ int bf_close (BFile); BFile bf_open (BFiles bfs, const char *name, int block_size, int wflag); /* bf_read: reads bytes from bfile 'bf'. - reads 'num' bytes (or whole block if 0) from offset 'offset' from + reads 'nbytes' bytes (or whole block if 0) from offset 'offset' from block 'no'. stores contents in buffer 'buf'. returns 1 if whole block could be read; 0 otherwise. */ -int bf_read (BFile bf, int no, int offset, int num, void *buf); +int bf_read (BFile bf, int no, int offset, int nbytes, void *buf); /* bf_write: writes bytes to bfile 'bf'. - writes 'num' bytes (or whole block if 0) at offset 'offset' to + writes 'nbytes' bytes (or whole block if 0) at offset 'offset' to block 'no'. retrieves contents from buffer 'buf'. returns 0 if successful; non-zero otherwise. */ -int bf_write (BFile bf, int no, int offset, int num, const void *buf); +int bf_write (BFile bf, int no, int offset, int nbytes, const void *buf); /* bf_cache: enables bfile cache if spec is not NULL */ void bf_cache (BFiles bfs, const char *spec); diff --git a/include/isams.h b/include/isams.h new file mode 100644 index 0000000..a853b57 --- /dev/null +++ b/include/isams.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 1995-1999, Index Data. + * See the file LICENSE for details. + * Sebastian Hammer, Adam Dickmeiss + * + * $Id: isams.h,v 1.1 1999-05-12 13:08:06 adam Exp $ + */ +#ifndef ISAMS_H +#define ISAMS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ISAMS_s *ISAMS; +typedef int ISAMS_P; +typedef struct ISAMS_PP_s *ISAMS_PP; + +typedef struct ISAMS_M_s { + int (*compare_item)(const void *a, const void *b); + +#define ISAMC_DECODE 0 +#define ISAMC_ENCODE 1 + void *(*code_start)(int mode); + void (*code_stop)(int mode, void *p); + void (*code_item)(int mode, void *p, char **dst, char **src); + + int debug; + int block_size; +} *ISAMS_M; + +typedef struct ISAMS_I_s { + int (*read_item)(void *clientData, char **dst, int *insertMode); + void *clientData; +} *ISAMS_I; + +ISAMS_M isams_getmethod (void); + +ISAMS isams_open (BFiles bfs, const char *name, int writeflag, + ISAMS_M method); +int isams_close (ISAMS is); +ISAMS_P isams_merge (ISAMS is, ISAMS_I data); +ISAMS_PP isams_pp_open (ISAMS is, ISAMS_P pos); +void isams_pp_close (ISAMS_PP pp); +int isams_read_item (ISAMS_PP pp, char **dst); +int isams_pp_read (ISAMS_PP pp, void *buf); +int isams_pp_num (ISAMS_PP pp); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/mfile.h b/include/mfile.h index 82cce3d..69b89c0 100644 --- a/include/mfile.h +++ b/include/mfile.h @@ -3,40 +3,7 @@ * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * - * $Log: mfile.h,v $ - * Revision 1.10 1999-02-02 14:50:35 adam - * Updated WIN32 code specific sections. Changed header. - * - * Revision 1.9 1997/09/18 08:59:19 adam - * Extra generic handle for the character mapping routines. - * - * Revision 1.8 1997/09/17 12:19:10 adam - * Zebra version corresponds to YAZ version 1.4. - * Changed Zebra server so that it doesn't depend on global common_resource. - * - * Revision 1.7 1997/09/05 15:30:01 adam - * Changed prototype for chr_map_input - added const. - * Added support for C++, headers uses extern "C" for public definitions. - * - * Revision 1.6 1996/10/29 13:46:10 adam - * Removed obsolete headers alexpath, alexutil. Created zebrautl.h as - * a replacement. - * - * Revision 1.5 1995/12/05 11:15:03 quinn - * Fixed FILENAME_MAX for some Sun systems, hopefully. - * - * Revision 1.4 1995/11/30 08:33:30 adam - * Started work on commit facility. - * - * Revision 1.3 1995/09/04 12:33:35 adam - * Various cleanup. YAZ util used instead. - * - * Revision 1.2 1994/09/14 13:10:36 quinn - * Small changes - * - * Revision 1.1 1994/08/23 14:41:46 quinn - * First functional version of mfile. - * + * $Id: mfile.h,v 1.11 1999-05-12 13:08:06 adam Exp $ */ #ifndef MFILE_H @@ -126,12 +93,12 @@ int mf_close(MFile mf); /* * Read one block from a metafile. Interface mirrors bfile. */ -int mf_read(MFile mf, int no, int offset, int num, void *buf); +int mf_read(MFile mf, int no, int offset, int nbytes, void *buf); /* * Same. */ -int mf_write(MFile mf, int no, int offset, int num, const void *buf); +int mf_write(MFile mf, int no, int offset, int nbytes, const void *buf); /* * Destroy a metafile, unlinking component files. File must be open. diff --git a/include/rsisamc.h b/include/rsisamc.h index ba15bc8..430a11f 100644 --- a/include/rsisamc.h +++ b/include/rsisamc.h @@ -3,20 +3,7 @@ * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * - * $Log: rsisamc.h,v $ - * Revision 1.4 1999-02-02 14:50:40 adam - * Updated WIN32 code specific sections. Changed header. - * - * Revision 1.3 1998/03/05 08:37:44 adam - * New result set model. - * - * Revision 1.2 1997/09/05 15:30:04 adam - * Changed prototype for chr_map_input - added const. - * Added support for C++, headers uses extern "C" for public definitions. - * - * Revision 1.1 1996/10/29 13:41:33 adam - * First use of isamc. - * + * $Id: rsisamc.h,v 1.5 1999-05-12 13:08:06 adam Exp $ */ #ifndef RSET_ISAMC_H diff --git a/index/Makefile.in b/index/Makefile.in index d21698d..38b5d84 100644 --- a/index/Makefile.in +++ b/index/Makefile.in @@ -1,7 +1,7 @@ # Copyright (C) 1995-1998, Index Data # All rights reserved. # Sebastian Hammer, Adam Dickmeiss -# $Id: Makefile.in,v 1.5 1999-03-09 10:16:35 adam Exp $ +# $Id: Makefile.in,v 1.6 1999-05-12 13:08:06 adam Exp $ SHELL=/bin/sh @@ -12,7 +12,7 @@ CDEFS=@DEFS@ @ODEFS@ LIBS=@LIBS@ YAZLIB=../../yaz/lib/libyaz.a -YAZINC=-I../../z39.50 -I../../yaz/include +YAZINC=-I../../yaz/z39.50 -I../../yaz/include #OSILIB=../../xtimosi/src/libmosi.a ../../yaz/lib/librfc.a #OSILIB=../../xtimosi/src/libmosi.a -lrfc diff --git a/index/index.h b/index/index.h index f4b59b7..9608973 100644 --- a/index/index.h +++ b/index/index.h @@ -1,10 +1,13 @@ /* - * Copyright (C) 1995-1998, Index Data + * Copyright (C) 1995-1999, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: index.h,v $ - * Revision 1.61 1999-03-09 16:27:49 adam + * Revision 1.62 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.61 1999/03/09 16:27:49 adam * More work on SDRKit integration. * * Revision 1.60 1998/10/16 08:14:31 adam @@ -228,6 +231,7 @@ #include #include #include +#include #include #include @@ -310,12 +314,14 @@ void inv_prstat (BFiles bfs); void inv_compact (BFiles bfs); void key_input (BFiles bfs, int nkeys, int cache); ISAMC_M key_isamc_m (Res res); +ISAMS_M key_isams_m (Res res); int merge_sort (char **buf, int from, int to); int key_SU_code (int ch, char *out); #define FNAME_DICT "dict" #define FNAME_ISAM "isam" #define FNAME_ISAMC "isamc" +#define FNAME_ISAMS "isams" #define FNAME_CONFIG "zebra.cfg" #define GMATCH_DICT "gmatch" diff --git a/index/invstat.c b/index/invstat.c index 3c3c55a..e08685a 100644 --- a/index/invstat.c +++ b/index/invstat.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: invstat.c,v $ - * Revision 1.9 1999-02-12 13:29:23 adam + * Revision 1.10 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.9 1999/02/12 13:29:23 adam * Implemented position-flag for registers. * * Revision 1.8 1999/02/02 14:50:53 adam @@ -48,6 +51,7 @@ struct inv_stat_info { ISAM isam; ISAMC isamc; + ISAMS isams; int no_isam_entries[8]; int no_dict_entries; int no_dict_bytes; @@ -96,6 +100,23 @@ static int inv_stat_handle (char *name, const char *info, int pos, stat_info->no_isam_entries[isc_type(isam_p)] += occur; isc_pp_close (pp); } + if (stat_info->isams) + { + ISAMS_PP pp; + int occurx = 0; + struct it_key key; + + pp = isams_pp_open (stat_info->isams, isam_p); + occur = isams_pp_num (pp); + while (isams_pp_read(pp, &key)) + { + printf ("sysno=%d seqno=%d\n", key.sysno, key.seqno); + occurx++; + } + assert (occurx == occur); + stat_info->no_isam_entries[isc_type(isam_p)] += occur; + isams_pp_close (pp); + } while (occur > stat_info->isam_bounds[i] && stat_info->isam_bounds[i]) i++; @@ -108,6 +129,7 @@ void inv_prstat (BFiles bfs) Dict dict; ISAM isam = NULL; ISAMC isamc = NULL; + ISAMS isams = NULL; Records records; int i, prev; int before = 0; @@ -124,22 +146,31 @@ void inv_prstat (BFiles bfs) logf (LOG_FATAL, "dict_open fail"); exit (1); } - if (!res_get_match (common_resource, "isam", "i", NULL)) + if (res_get_match (common_resource, "isam", "i", NULL)) { - isamc = isc_open (bfs, FNAME_ISAMC, 0, key_isamc_m (common_resource)); - if (!isamc) + isam = is_open (bfs, FNAME_ISAM, key_compare, 0, + sizeof(struct it_key), common_resource); + if (!isam) { - logf (LOG_FATAL, "isc_open fail"); + logf (LOG_FATAL, "is_open fail"); + exit (1); + } + } + else if (res_get_match (common_resource, "isam", "s", NULL)) + { + isams = isams_open (bfs, FNAME_ISAMS, 0, key_isams_m(common_resource)); + if (!isams) + { + logf (LOG_FATAL, "isams_open fail"); exit (1); } } else { - isam = is_open (bfs, FNAME_ISAM, key_compare, 0, - sizeof(struct it_key), common_resource); - if (!isam) + isamc = isc_open (bfs, FNAME_ISAMC, 0, key_isamc_m (common_resource)); + if (!isamc) { - logf (LOG_FATAL, "is_open fail"); + logf (LOG_FATAL, "isc_open fail"); exit (1); } } @@ -151,6 +182,7 @@ void inv_prstat (BFiles bfs) stat_info.no_dict_bytes = 0; stat_info.isam = isam; stat_info.isamc = isamc; + stat_info.isams = isams; stat_info.isam_bounds[0] = 1; stat_info.isam_bounds[1] = 2; stat_info.isam_bounds[2] = 3; @@ -216,5 +248,7 @@ void inv_prstat (BFiles bfs) is_close (isam); if (isamc) isc_close (isamc); + if (isams) + isams_close (isams); } diff --git a/index/kcompare.c b/index/kcompare.c index d6c405a..ba8a6ef 100644 --- a/index/kcompare.c +++ b/index/kcompare.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: kcompare.c,v $ - * Revision 1.26 1999-02-02 14:50:54 adam + * Revision 1.27 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.26 1999/02/02 14:50:54 adam * Updated WIN32 code specific sections. Changed header. * * Revision 1.25 1998/06/08 15:26:06 adam @@ -297,6 +300,27 @@ ISAMC_M key_isamc_m (Res res) return me; } +ISAMS_M key_isams_m (Res res) +{ + static ISAMS_M me = NULL; + + if (me) + return me; + + me = isams_getmethod (); + + me->compare_item = key_compare; + + me->code_start = iscz1_code_start; + me->code_item = iscz1_code_item; + me->code_stop = iscz1_code_stop; + + me->debug = atoi(res_get_def (res, "isamsDebug", "0")); + + return me; +} + + int key_SU_code (int ch, char *out) { int i; diff --git a/index/kinput.c b/index/kinput.c index 9053738..07addbe 100644 --- a/index/kinput.c +++ b/index/kinput.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: kinput.c,v $ - * Revision 1.31 1999-02-02 14:50:56 adam + * Revision 1.32 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.31 1999/02/02 14:50:56 adam * Updated WIN32 code specific sections. Changed header. * * Revision 1.30 1998/10/28 10:53:57 adam @@ -312,6 +315,7 @@ struct heap_info { Dict dict; ISAM isam; ISAMC isamc; + ISAMS isams; }; struct heap_info *key_heap_init (int nkeys, @@ -495,6 +499,42 @@ int heap_inpc (struct heap_info *hi) return 0; } +int heap_inps (struct heap_info *hi) +{ + struct heap_cread_info hci; + ISAMS_I isams_i = xmalloc (sizeof(*isams_i)); + + hci.key = xmalloc (KEY_SIZE); + hci.mode = 1; + hci.hi = hi; + hci.more = heap_read_one (hi, hci.cur_name, hci.key); + + isams_i->clientData = &hci; + isams_i->read_item = heap_cread_item; + + while (hci.more) + { + char this_name[INP_NAME_MAX]; + ISAMS_P isams_p; + char *dict_info; + + strcpy (this_name, hci.cur_name); + assert (hci.cur_name[1]); + no_diffs++; + if (!(dict_info = dict_lookup (hi->dict, hci.cur_name))) + { + isams_p = isams_merge (hi->isams, isams_i); + no_insertions++; + dict_insert (hi->dict, this_name, sizeof(ISAMS_P), &isams_p); + } + else + abort(); + } + xfree (isams_i); + return 0; +} + + int heap_inp (struct heap_info *hi) { char *info; @@ -606,6 +646,7 @@ void key_input (BFiles bfs, int nkeys, int cache) Dict dict; ISAM isam = NULL; ISAMC isamc = NULL; + ISAMS isams = NULL; struct key_file **kf; char rbuf[1024]; int i, r; @@ -632,17 +673,17 @@ void key_input (BFiles bfs, int nkeys, int cache) logf (LOG_FATAL, "dict_open fail"); exit (1); } - if (!res_get_match (common_resource, "isam", "i", NULL)) + if (res_get_match (common_resource, "isam", "s", NULL)) { - isamc = isc_open (bfs, - FNAME_ISAMC, 1, key_isamc_m (common_resource)); - if (!isamc) + isams = isams_open (bfs, FNAME_ISAMS, 1, + key_isams_m (common_resource)); + if (!isams) { - logf (LOG_FATAL, "isc_open fail"); + logf (LOG_FATAL, "isams_open fail"); exit (1); } } - else + else if (res_get_match (common_resource, "isam", "i", NULL)) { isam = is_open (bfs, FNAME_ISAM, key_compare, 1, sizeof(struct it_key), common_resource); @@ -652,6 +693,16 @@ void key_input (BFiles bfs, int nkeys, int cache) exit (1); } } + else + { + isamc = isc_open (bfs, FNAME_ISAMC, 1, + key_isamc_m (common_resource)); + if (!isamc) + { + logf (LOG_FATAL, "isc_open fail"); + exit (1); + } + } kf = xmalloc ((1+nkeys) * sizeof(*kf)); progressInfo.totalBytes = 0; progressInfo.totalOffset = 0; @@ -669,19 +720,24 @@ void key_input (BFiles bfs, int nkeys, int cache) hi->dict = dict; hi->isam = isam; hi->isamc = isamc; + hi->isams = isams; for (i = 1; i<=nkeys; i++) if ((r = key_file_read (kf[i], rbuf))) key_heap_insert (hi, rbuf, r, kf[i]); if (isamc) heap_inpc (hi); - else - heap_inp (hi); + else if (isams) + heap_inps (hi); + else if (isam) + heap_inp (hi); dict_close (dict); if (isam) is_close (isam); if (isamc) isc_close (isamc); + if (isams) + isams_close (isams); for (i = 1; i<=nkeys; i++) { diff --git a/index/trunc.c b/index/trunc.c index f9ecdc6..c5113ac 100644 --- a/index/trunc.c +++ b/index/trunc.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: trunc.c,v $ - * Revision 1.12 1999-02-02 14:51:10 adam + * Revision 1.13 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.12 1999/02/02 14:51:10 adam * Updated WIN32 code specific sections. Changed header. * * Revision 1.11 1998/03/25 13:48:02 adam @@ -54,6 +57,7 @@ #include #include #include +#include #include #define NEW_TRUNC 1 @@ -286,7 +290,7 @@ static RSET rset_trunc_r (ZebraHandle zi, const char *term, int length, heap_close (ti); xfree (ispt); } - else + else if (zi->isamc) { ISAMC_PP *ispt; int i; @@ -312,7 +316,7 @@ static RSET rset_trunc_r (ZebraHandle zi, const char *term, int length, #if 0 /* section that preserve all keys */ heap_delete (ti); - if (is_readkey (ispt[n], ti->tmpbuf)) + if (isc_pp_read (ispt[n], ti->tmpbuf)) heap_insert (ti, ti->tmpbuf, n); else isc_pp_close (ispt[n]); @@ -338,6 +342,48 @@ static RSET rset_trunc_r (ZebraHandle zi, const char *term, int length, heap_close (ti); xfree (ispt); } + else if (zi->isams) + { + ISAMS_PP *ispt; + int i; + struct trunc_info *ti; + + ispt = xmalloc (sizeof(*ispt) * (to-from)); + + ti = heap_init (to-from, sizeof(struct it_key), + key_compare_it); + for (i = to-from; --i >= 0; ) + { + ispt[i] = isams_pp_open (zi->isams, isam_p[from+i]); + if (isams_pp_read (ispt[i], ti->tmpbuf)) + heap_insert (ti, ti->tmpbuf, i); + else + isams_pp_close (ispt[i]); + } + while (ti->heapnum) + { + int n = ti->indx[ti->ptr[1]]; + + rset_write (result, result_rsfd, ti->heap[ti->ptr[1]]); + while (1) + { + if (!isams_pp_read (ispt[n], ti->tmpbuf)) + { + heap_delete (ti); + isams_pp_close (ispt[n]); + break; + } + if ((*ti->cmp)(ti->tmpbuf, ti->heap[ti->ptr[1]]) > 1) + { + heap_delete (ti); + heap_insert (ti, ti->tmpbuf, n); + break; + } + } + } + heap_close (ti); + xfree (ispt); + } rset_close (result, result_rsfd); return result; } @@ -366,6 +412,14 @@ static int isamc_trunc_cmp (const void *p1, const void *p2) return isc_block (i1) - isc_block (i2); } +static int isams_trunc_cmp (const void *p1, const void *p2) +{ + ISAMS_P i1 = *(ISAMS_P*) p1; + ISAMS_P i2 = *(ISAMS_P*) p2; + + return i1 - i2; +} + RSET rset_trunc (ZebraHandle zi, ISAM_P *isam_p, int no, const char *term, int length, const char *flags) { @@ -415,9 +469,24 @@ RSET rset_trunc (ZebraHandle zi, ISAM_P *isam_p, int no, #endif qsort (isam_p, no, sizeof(*isam_p), isamc_trunc_cmp); } + else if (zi->isams) + { + if (no < 1) + return rset_create (rset_kind_null, NULL); + else if (no == 1) + { + rset_isams_parms parms; + + parms.pos = *isam_p; + parms.is = zi->isams; + parms.rset_term = rset_term_create (term, length, flags); + return rset_create (rset_kind_isams, &parms); + } + qsort (isam_p, no, sizeof(*isam_p), isams_trunc_cmp); + } else { - logf (LOG_WARN, "Neither isam nor isamc set in rset_trunc"); + logf (LOG_WARN, "Neither isam / isamc / isams set in rset_trunc"); return rset_create (rset_kind_null, NULL); } return rset_trunc_r (zi, term, length, flags, isam_p, 0, no, 100); diff --git a/index/zebraapi.c b/index/zebraapi.c index 16e5b06..ee30db0 100644 --- a/index/zebraapi.c +++ b/index/zebraapi.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zebraapi.c,v $ - * Revision 1.16 1999-02-19 10:38:30 adam + * Revision 1.17 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.16 1999/02/19 10:38:30 adam * Implemented chdir-setting. * * Revision 1.15 1999/02/17 12:18:12 adam @@ -145,22 +148,31 @@ static int zebra_register_lock (ZebraHandle zh) } zh->isam = NULL; zh->isamc = NULL; - if (!res_get_match (zh->res, "isam", "i", NULL)) + zh->isams = NULL; + if (res_get_match (zh->res, "isam", "i", NULL)) { - if (!(zh->isamc = isc_open (zh->bfs, FNAME_ISAMC, - 0, key_isamc_m(zh->res)))) + if (!(zh->isam = is_open (zh->bfs, FNAME_ISAM, key_compare, 0, + sizeof (struct it_key), zh->res))) { - logf (LOG_WARN, "isc_open"); + logf (LOG_WARN, "is_open"); + return -1; + } + } + else if (res_get_match (zh->res, "isam", "s", NULL)) + { + if (!(zh->isams = isams_open (zh->bfs, FNAME_ISAMS, 0, + key_isams_m(zh->res)))) + { + logf (LOG_WARN, "isams_open"); return -1; } - } else { - if (!(zh->isam = is_open (zh->bfs, FNAME_ISAM, key_compare, 0, - sizeof (struct it_key), zh->res))) + if (!(zh->isamc = isc_open (zh->bfs, FNAME_ISAMC, + 0, key_isamc_m(zh->res)))) { - logf (LOG_WARN, "is_open"); + logf (LOG_WARN, "isc_open"); return -1; } } @@ -253,6 +265,8 @@ void zebra_close (ZebraHandle zh) is_close (zh->isam); if (zh->isamc) isc_close (zh->isamc); + if (zh->isams) + isams_close (zh->isams); rec_close (&zh->records); zebra_register_unlock (zh); } diff --git a/index/zserver.h b/index/zserver.h index 0c2f050..b7a452f 100644 --- a/index/zserver.h +++ b/index/zserver.h @@ -1,10 +1,13 @@ /* - * Copyright (C) 1994-1998, Index Data + * Copyright (C) 1994-1999, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: zserver.h,v $ - * Revision 1.40 1998-11-16 16:03:45 adam + * Revision 1.41 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.40 1998/11/16 16:03:45 adam * Moved loggin utilities to Yaz. Was implemented in file zlogs.c. * * Revision 1.39 1998/10/16 08:14:35 adam @@ -187,6 +190,7 @@ struct zebra_info { SortIdx sortIdx; ISAM isam; ISAMC isamc; + ISAMS isams; Records records; int errCode; int hits; diff --git a/isamc/Makefile.in b/isamc/Makefile.in index d8ec6b5..67deb46 100644 --- a/isamc/Makefile.in +++ b/isamc/Makefile.in @@ -1,7 +1,7 @@ # Copyright (C) 1996-1998, Index Data # All rights reserved. # Sebastian Hammer, Adam Dickmeiss -# $Id: Makefile.in,v 1.2 1998-10-28 15:20:49 adam Exp $ +# $Id: Makefile.in,v 1.3 1999-05-12 13:08:06 adam Exp $ SHELL=/bin/sh @@ -18,7 +18,7 @@ INCLUDE=-I../include $(YAZINC) TPROG1=isamctst DEFS=$(CDEFS) $(INCLUDE) LIB=../lib/isamc.a -PO = isamc.o merge.o +PO = isamc.o merge.o isams.o all: $(LIB) diff --git a/isamc/isams.c b/isamc/isams.c new file mode 100644 index 0000000..9a178c4 --- /dev/null +++ b/isamc/isams.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) 1995-1999, Index Data. + * See the file LICENSE for details. + * Sebastian Hammer, Adam Dickmeiss + * + * $Log: isams.c,v $ + * Revision 1.1 1999-05-12 13:08:06 adam + * First version of ISAMS. + * + */ +#include +#include +#include +#include + +#include +#include + +typedef struct { + int last_offset; + int last_block; +} ISAMS_head; + +typedef unsigned ISAMS_BLOCK_SIZE; + +struct ISAMS_s { + ISAMS_M method; + ISAMS_head head; + ISAMS_head head_old; + char *merge_buf; + + int block_size; + BFile bf; +}; + +struct ISAMS_PP_s { + ISAMS is; + char *buf; + int block_offset; + int block_no; + void *decodeClientData; + int numKeys; + int numRead; +}; + +ISAMS_M isams_getmethod (void) +{ + ISAMS_M m = xmalloc (sizeof(*m)); + + m->code_start = NULL; + m->code_item = NULL; + m->code_stop = NULL; + + m->compare_item = NULL; + + m->debug = 1; + m->block_size = 128; + + return m; +} + +ISAMS isams_open (BFiles bfs, const char *name, int writeflag, + ISAMS_M method) +{ + ISAMS is = xmalloc (sizeof(*is)); + + is->method = xmalloc (sizeof(*is->method)); + memcpy (is->method, method, sizeof(*method)); + is->block_size = is->method->block_size; + + is->bf = bf_open (bfs, name, is->block_size, writeflag); + + if (!bf_read (is->bf, 0, 0, sizeof(ISAMS_head), &is->head)) + { + is->head.last_block = 1; + is->head.last_offset = 0; + } + memcpy (&is->head_old, &is->head, sizeof(is->head)); + is->merge_buf = xmalloc(2*is->block_size); + memset(is->merge_buf, 0, 2*is->block_size); + return is; +} + +int isams_close (ISAMS is) +{ + if (memcmp(&is->head, &is->head_old, sizeof(is->head))) + { + if (is->head.last_offset) + bf_write(is->bf, is->head.last_block, 0, is->head.last_offset, + is->merge_buf); + bf_write (is->bf, 0, 0, sizeof(is->head), &is->head); + } + bf_close (is->bf); + xfree (is->merge_buf); + xfree (is->method); + xfree (is); + return 0; +} + +ISAMS_P isams_merge (ISAMS is, ISAMS_I data) +{ + char i_item[128], *i_item_ptr; + int i_more, i_mode; + void *r_clientData; + int first_block = is->head.last_block; + int first_offset = is->head.last_offset; + int count = 0; + + r_clientData = (*is->method->code_start)(ISAMC_ENCODE); + + is->head.last_offset += sizeof(int); + if (is->head.last_offset > is->block_size) + { + logf (LOG_LOG, "first_block=%d", first_block); + bf_write(is->bf, is->head.last_block, 0, 0, is->merge_buf); + (is->head.last_block)++; + is->head.last_offset -= is->block_size; + memcpy (is->merge_buf, is->merge_buf + is->block_size, + is->head.last_offset); + } + while (1) + { + i_item_ptr = i_item; + i_more = (*data->read_item)(data->clientData, &i_item_ptr, &i_mode); + assert (i_mode); + + if (!i_more) + break; + else + { + char *r_out_ptr = is->merge_buf + is->head.last_offset; + + i_item_ptr = i_item; + (*is->method->code_item)(ISAMC_ENCODE, r_clientData, + &r_out_ptr, &i_item_ptr); + is->head.last_offset = r_out_ptr - is->merge_buf; + if (is->head.last_offset > is->block_size) + { + bf_write(is->bf, is->head.last_block, 0, 0, is->merge_buf); + (is->head.last_block)++; + is->head.last_offset -= is->block_size; + memcpy (is->merge_buf, is->merge_buf + is->block_size, + is->head.last_offset); + } + count++; + } + } + (*is->method->code_stop)(ISAMC_ENCODE, r_clientData); + if (first_block == is->head.last_block) + memcpy(is->merge_buf + first_offset, &count, sizeof(int)); + else if (first_block == is->head.last_block-1) + { + int gap = first_offset + sizeof(int) - is->block_size; + assert (gap <= (int) sizeof(int)); + if (gap > 0) + { + if (gap < sizeof(int)) + bf_write(is->bf, first_block, first_offset, sizeof(int)-gap, + &count); + memcpy (is->merge_buf, ((char*)&count)+(sizeof(int)-gap), gap); + } + else + bf_write(is->bf, first_block, first_offset, sizeof(int), &count); + } + else + { + bf_write(is->bf, first_block, first_offset, sizeof(int), &count); + } + return first_block * is->block_size + first_offset; +} + +ISAMS_PP isams_pp_open (ISAMS is, ISAMS_P pos) +{ + ISAMS_PP pp = xmalloc (sizeof(*pp)); + + logf (LOG_LOG, "isams: isams_pp_open pos=%ld", (long) pos); + pp->is = is; + pp->decodeClientData = (*is->method->code_start)(ISAMC_DECODE); + pp->numKeys = 0; + pp->numRead = 0; + pp->buf = xmalloc(is->block_size*2); + pp->block_no = pos/is->block_size; + pp->block_offset = pos - pp->block_no * is->block_size; + logf (LOG_LOG, "isams: isams_pp_open off=%d no=%d", + pp->block_offset, pp->block_no); + if (pos) + { + bf_read (is->bf, pp->block_no, 0, 0, pp->buf); + bf_read (is->bf, pp->block_no+1, 0, 0, pp->buf + is->block_size); + memcpy(&pp->numKeys, pp->buf + pp->block_offset, sizeof(int)); + logf (LOG_LOG, "isams: isams_pp_open numKeys=%d", pp->numKeys); + pp->block_offset += sizeof(int); + } + return pp; +} + +void isams_pp_close (ISAMS_PP pp) +{ + (*pp->is->method->code_stop)(ISAMC_DECODE, pp->decodeClientData); + xfree(pp->buf); + xfree(pp); +} + +int isams_pp_num (ISAMS_PP pp) +{ + return pp->numKeys; +} + +int isams_pp_read (ISAMS_PP pp, void *buf) +{ + return isams_read_item (pp, (char **) &buf); +} + +int isams_read_item (ISAMS_PP pp, char **dst) +{ + char *src; + if (pp->numRead >= pp->numKeys) + return 0; + (pp->numRead)++; + if (pp->block_offset > pp->is->block_size) + { + pp->block_offset -= pp->is->block_size; + (pp->block_no)++; + memcpy (pp->buf, pp->buf + pp->is->block_size, pp->is->block_size); + bf_read (pp->is->bf, pp->block_no+1, 0, 0, + pp->buf + pp->is->block_size); + } + src = pp->buf + pp->block_offset; + (*pp->is->method->code_item)(ISAMC_DECODE, pp->decodeClientData, + dst, &src); + pp->block_offset = src - pp->buf; + return 1; +} + + diff --git a/rset/Makefile.in b/rset/Makefile.in index 0a44d96..d40d478 100644 --- a/rset/Makefile.in +++ b/rset/Makefile.in @@ -1,7 +1,7 @@ # Copyright (C) 1994-1998, Index Data # All rights reserved. # Sebastian Hammer, Adam Dickmeiss -# $Id: Makefile.in,v 1.2 1998-10-28 15:20:51 adam Exp $ +# $Id: Makefile.in,v 1.3 1999-05-12 13:08:06 adam Exp $ SHELL=/bin/sh @@ -18,7 +18,7 @@ INCLUDE=-I../include $(YAZINC) DEFS=$(CDEFS) $(INCLUDE) LIB=../lib/rset.a PROG= -PO=rset.o rstemp.o rsisam.o rsnull.o rsbool.o rsisamc.o rsm_or.o +PO=rset.o rstemp.o rsisam.o rsnull.o rsbool.o rsisamc.o rsm_or.o rsisams.o all: $(LIB) diff --git a/test/gils/test.sh b/test/gils/test.sh index 8f0292b..7abd7cb 100755 --- a/test/gils/test.sh +++ b/test/gils/test.sh @@ -1,5 +1,5 @@ : echo Loading Records -../../index/zebraidx update records +../../index/zebraidx -t grs.sgml update records echo Starting Server ../../index/zebrasrv -- 1.7.10.4