From 795af4e3c7346eff351ff387228ec548956eada8 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Fri, 25 Jun 1999 13:48:02 +0000 Subject: [PATCH] Updated MSVC project files. Added BZIP2 record compression (not very well tested). --- apitest/apitest.dsp | 16 +-- configure | 103 +++++++++++++++-- configure.in | 8 +- index/apitest.c | 2 +- index/main.c | 17 +-- index/recindex.c | 304 +++++++++++++++++++++++++++++++++++++++++++++------ index/recindex.h | 8 +- zebra.dsp | 24 ++-- zebra.dsw | 2 +- 9 files changed, 416 insertions(+), 68 deletions(-) diff --git a/apitest/apitest.dsp b/apitest/apitest.dsp index a03b9a1..85ea90d 100755 --- a/apitest/apitest.dsp +++ b/apitest/apitest.dsp @@ -1,5 +1,5 @@ # Microsoft Developer Studio Project File - Name="apitest" - Package Owner=<4> -# Microsoft Developer Studio Generated Build File, Format Version 5.00 +# Microsoft Developer Studio Generated Build File, Format Version 6.00 # ** DO NOT EDIT ** # TARGTYPE "Win32 (x86) Console Application" 0x0103 @@ -22,6 +22,7 @@ CFG=apitest - Win32 Debug !MESSAGE # Begin Project +# PROP AllowPerConfigDependencies 0 # PROP Scc_ProjName "" # PROP Scc_LocalPath "" CPP=cl.exe @@ -41,7 +42,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /MD /W3 /GX /O2 /I "..\include" /I "..\..\yaz\include" /I "..\index" /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c +# ADD CPP /nologo /MD /W3 /GX /O2 /I "..\include" /I "..\..\yaz\include" /I "..\index" /D "_CONSOLE" /D "_MBCS" /D "WIN32" /D "NDEBUG" /D HAVE_BZLIB_H=1 /YX /FD /c # ADD BASE RSC /l 0x406 /d "NDEBUG" # ADD RSC /l 0x406 /d "NDEBUG" BSC32=bscmake.exe @@ -49,12 +50,12 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib zebra.lib yaz.lib server.lib /nologo /subsystem:console /machine:I386 /libpath:"..\release" /libpath:"..\..\yaz\release" /libpath:"..\..\yaz\server\release" +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib zebra.lib yaz.lib server.lib libbz2.lib /nologo /subsystem:console /machine:I386 /libpath:"..\release" /libpath:"..\..\yaz\lib" /libpath:"..\..\bzip2-0.9.0c\debug" # Begin Special Build Tool OutDir=.\Release ProjDir=. TargetName=apitest -SOURCE=$(InputPath) +SOURCE="$(InputPath)" PostBuild_Cmds=copy $(OutDir)\$(TargetName).exe $(ProjDir)\..\..\bin # End Special Build Tool @@ -69,9 +70,10 @@ PostBuild_Cmds=copy $(OutDir)\$(TargetName).exe $(ProjDir)\..\..\bin # PROP Use_Debug_Libraries 1 # PROP Output_Dir "Debug" # PROP Intermediate_Dir "Debug" +# PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /Zi /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /MDd /W3 /Gm /GX /Zi /Od /I "..\include" /I "..\..\yaz\include" /I "..\index" /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c +# ADD CPP /nologo /MDd /W3 /Gm /GX /ZI /Od /I "..\include" /I "..\..\yaz\include" /I "..\index" /D "_CONSOLE" /D "_MBCS" /D "WIN32" /D "_DEBUG" /D HAVE_BZLIB_H=1 /YX /FD /c # ADD BASE RSC /l 0x406 /d "_DEBUG" # ADD RSC /l 0x406 /d "_DEBUG" BSC32=bscmake.exe @@ -79,12 +81,12 @@ BSC32=bscmake.exe # ADD BSC32 /nologo LINK32=link.exe # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib zebra.lib yaz.lib server.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept /libpath:"..\debug" /libpath:"..\..\yaz\debug" /libpath:"..\..\yaz\server\debug" +# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib zebra.lib yaz.lib server.lib libbz2.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept /libpath:"..\debug" /libpath:"..\..\yaz\lib" /libpath:"..\..\bzip2-0.9.0c\debug" # Begin Special Build Tool OutDir=.\Debug ProjDir=. TargetName=apitest -SOURCE=$(InputPath) +SOURCE="$(InputPath)" PostBuild_Cmds=copy $(OutDir)\$(TargetName).exe $(ProjDir)\..\..\bin # End Special Build Tool diff --git a/configure b/configure index 8b0399f..5c85f88 100755 --- a/configure +++ b/configure @@ -1279,13 +1279,102 @@ else echo "$ac_t""no" 1>&6 fi +echo $ac_n "checking for bzCompressInit in -lbz2""... $ac_c" 1>&6 +echo "configure:1284: checking for bzCompressInit in -lbz2" >&5 +ac_lib_var=`echo bz2'_'bzCompressInit | sed 'y%./+-%__p_%'` +if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then + echo $ac_n "(cached) $ac_c" 1>&6 +else + ac_save_LIBS="$LIBS" +LIBS="-lbz2 $LIBS" +cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then + rm -rf conftest* + eval "ac_cv_lib_$ac_lib_var=yes" +else + echo "configure: failed program was:" >&5 + cat conftest.$ac_ext >&5 + rm -rf conftest* + eval "ac_cv_lib_$ac_lib_var=no" +fi +rm -f conftest* +LIBS="$ac_save_LIBS" + +fi +if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then + echo "$ac_t""yes" 1>&6 + ac_tr_lib=HAVE_LIB`echo bz2 | sed -e 's/[^a-zA-Z0-9_]/_/g' \ + -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'` + cat >> confdefs.h <&6 +fi + +if test "$ac_cv_lib_bz2_bzCompressInit" = "yes"; then + for ac_hdr in bzlib.h +do +ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'` +echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6 +echo "configure:1335: checking for $ac_hdr" >&5 +if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then + echo $ac_n "(cached) $ac_c" 1>&6 +else + cat > conftest.$ac_ext < +EOF +ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" +{ (eval echo configure:1345: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"` +if test -z "$ac_err"; then + rm -rf conftest* + eval "ac_cv_header_$ac_safe=yes" +else + echo "$ac_err" >&5 + echo "configure: failed program was:" >&5 + cat conftest.$ac_ext >&5 + rm -rf conftest* + eval "ac_cv_header_$ac_safe=no" +fi +rm -f conftest* +fi +if eval "test \"`echo '$ac_cv_header_'$ac_safe`\" = yes"; then + echo "$ac_t""yes" 1>&6 + ac_tr_hdr=HAVE_`echo $ac_hdr | sed 'y%abcdefghijklmnopqrstuvwxyz./-%ABCDEFGHIJKLMNOPQRSTUVWXYZ___%'` + cat >> confdefs.h <&6 +fi +done + +fi echo $ac_n "checking for ANSI C header files""... $ac_c" 1>&6 -echo "configure:1284: checking for ANSI C header files" >&5 +echo "configure:1373: checking for ANSI C header files" >&5 if eval "test \"`echo '$''{'ac_cv_header_stdc'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #include @@ -1293,7 +1382,7 @@ else #include EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1297: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1386: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"` if test -z "$ac_err"; then rm -rf conftest* @@ -1310,7 +1399,7 @@ rm -f conftest* if test $ac_cv_header_stdc = yes; then # SunOS 4.x string.h does not declare mem*, contrary to ANSI. cat > conftest.$ac_ext < EOF @@ -1328,7 +1417,7 @@ fi if test $ac_cv_header_stdc = yes; then # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. cat > conftest.$ac_ext < EOF @@ -1349,7 +1438,7 @@ if test "$cross_compiling" = yes; then : else cat > conftest.$ac_ext < #define ISLOWER(c) ('a' <= (c) && (c) <= 'z') @@ -1360,7 +1449,7 @@ if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) exit(2); exit (0); } EOF -if { (eval echo configure:1364: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1453: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null then : else diff --git a/configure.in b/configure.in index 0121ac6..61bf5ea 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ dnl Zebra, Index Data Aps, 1994-1999 -dnl $Id: configure.in,v 1.8 1999-05-26 07:49:12 adam Exp $ +dnl $Id: configure.in,v 1.9 1999-06-25 13:48:02 adam Exp $ dnl See the file LICENSE.2 for details. dnl AC_INIT(include/zebraver.h) @@ -62,6 +62,12 @@ dnl dnl ------ TCP wrapper (for Linux) AC_CHECK_LIB(wrap, main, [LIBS="$LIBS -lwrap"]) dnl +dnl ------- BZIP2 +AC_CHECK_LIB(bz2,bzCompressInit) +if test "$ac_cv_lib_bz2_bzCompressInit" = "yes"; then + AC_CHECK_HEADERS(bzlib.h) +fi +dnl dnl ------ ANSI C Header files AC_STDC_HEADERS if test "$ac_cv_header_stdc" = "no"; then diff --git a/index/apitest.c b/index/apitest.c index 7207b97..9ed3508 100644 --- a/index/apitest.c +++ b/index/apitest.c @@ -149,7 +149,7 @@ int main (int argc, char **argv) } /* result set name will be called 1,2, etc */ - sprintf (setname, "%d", i); + sprintf (setname, "%d", argno); /* fire up the search */ zebra_search_rpn (zh, odr_input, odr_output, query, 1, &base, setname); diff --git a/index/main.c b/index/main.c index e97ae7c..0c9cad8 100644 --- a/index/main.c +++ b/index/main.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: main.c,v $ - * Revision 1.65 1999-05-20 12:57:18 adam + * Revision 1.66 1999-06-25 13:48:02 adam + * Updated MSVC project files. + * Added BZIP2 record compression (not very well tested). + * + * Revision 1.65 1999/05/20 12:57:18 adam * Implemented TCL filter. Updated recctrl system. * * Revision 1.64 1999/05/15 14:36:38 adam @@ -517,13 +521,12 @@ int main (int argc, char **argv) } else if (ret == 'V') { - fprintf (stderr, "Zebra %s %s\n", -#if ZEBRASDR - "SDR", -#else - ZEBRAVER, + fprintf (stderr, "Zebra %s %s\n", ZEBRAVER, ZEBRADATE); + fprintf (stderr, " (C) 1994-1999, Index Data ApS\n"); +#if HAVE_BZLIB_H + fprintf (stderr, "libbzip2\n" + " (C) 1996-1998 Julian R Seward. All rights reserved.\n"); #endif - ZEBRADATE); } else if (ret == 'v') log_init_level (log_mask_str(arg)); diff --git a/index/recindex.c b/index/recindex.c index 17e6734..2fe16b8 100644 --- a/index/recindex.c +++ b/index/recindex.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recindex.c,v $ - * Revision 1.23 1999-05-26 07:49:13 adam + * Revision 1.24 1999-06-25 13:48:02 adam + * Updated MSVC project files. + * Added BZIP2 record compression (not very well tested). + * + * Revision 1.23 1999/05/26 07:49:13 adam * C++ compilation. * * Revision 1.22 1999/02/18 12:49:34 adam @@ -97,12 +101,32 @@ * Started work on better record management system. * */ + + +/* + * Format of first block + * next (4 bytes) + * ref_count (4 bytes) + * block (504 bytes) + * + * Format of subsequent blocks + * next (4 bytes) + * block (508 bytes) + * + * Format of each record + * sysno + * (length, data) - pairs + * length = 0 if same as previous + */ #include #include #include #include "recindxp.h" +#if HAVE_BZLIB_H +#include +#endif static void rec_write_head (Records p) { int r; @@ -118,13 +142,13 @@ static void rec_write_head (Records p) } } -static void rec_tmp_expand (Records p, int size, int dst_type) +static void rec_tmp_expand (Records p, int size) { if (p->tmp_size < size + 2048 || - p->tmp_size < p->head.block_size[dst_type]*2) + p->tmp_size < p->head.block_size[REC_BLOCK_TYPES-1]*2) { xfree (p->tmp_buf); - p->tmp_size = size + p->head.block_size[dst_type]*2 + 2048; + p->tmp_size = size + p->head.block_size[REC_BLOCK_TYPES-1]*2 + 2048; p->tmp_buf = (char *) xmalloc (p->tmp_size); } } @@ -155,12 +179,15 @@ static void write_indx (Records p, int sysno, void *buf, int itemsize) static void rec_release_blocks (Records p, int sysno) { struct record_index_entry entry; - int freeblock, freenext; + int freeblock; + int block_and_ref[2]; int dst_type; + int first = 1; + logf (LOG_LOG, "release_blocks for sysno=%d", sysno); if (read_indx (p, sysno, &entry, sizeof(entry), 1) != 1) return ; - p->head.total_bytes -= entry.size; + freeblock = entry.next; assert (freeblock > 0); dst_type = freeblock & 7; @@ -168,22 +195,40 @@ static void rec_release_blocks (Records p, int sysno) freeblock = freeblock / 8; while (freeblock) { - if (bf_read (p->data_BFile[dst_type], freeblock, 0, sizeof(freenext), - &freenext) != 1) + if (bf_read (p->data_BFile[dst_type], freeblock, 0, + sizeof(block_and_ref), block_and_ref) != 1) { logf (LOG_FATAL|LOG_ERRNO, "read in rec_del_single"); exit (1); } - if (bf_write (p->data_BFile[dst_type], freeblock, 0, sizeof(freenext), + if (first) + { + block_and_ref[1]--; + if (block_and_ref[1]) + { + if (bf_write (p->data_BFile[dst_type], freeblock, 0, + sizeof(block_and_ref), block_and_ref)) + { + logf (LOG_FATAL|LOG_ERRNO, "write in rec_del_single"); + exit (1); + } + return; + } + first = 0; + } + + if (bf_write (p->data_BFile[dst_type], freeblock, 0, sizeof(freeblock), &p->head.block_free[dst_type])) { logf (LOG_FATAL|LOG_ERRNO, "write in rec_del_single"); exit (1); } p->head.block_free[dst_type] = freeblock; - freeblock = freenext; + freeblock = block_and_ref[0]; + p->head.block_used[dst_type]--; } + p->head.total_bytes -= entry.size; } static void rec_delete_single (Records p, Record rec) @@ -198,27 +243,23 @@ static void rec_delete_single (Records p, Record rec) write_indx (p, rec->sysno, &entry, sizeof(entry)); } +static void rec_write_tmp_buf (Records p, int size, int *sysnos); static void rec_write_single (Records p, Record rec) { + + int sysnos[2]; int i, size = 0; char *cptr; - int dst_type = 0; - int no_written = 0; - int block_prev = -1, block_free; - struct record_index_entry entry; + logf (LOG_LOG, " rec_write_single !!!!!!!!!!!!!!!!!!!!!!!!!!!!"); for (i = 0; i < REC_NO_INFO; i++) if (!rec->info[i]) size += sizeof(*rec->size); else size += sizeof(*rec->size) + rec->size[i]; - for (i = 1; i= p->head.block_move[i]) - dst_type = i; - - rec_tmp_expand (p, size, dst_type); + rec_tmp_expand (p, size); cptr = p->tmp_buf + sizeof(int); /* a hack! */ for (i = 0; i < REC_NO_INFO; i++) @@ -231,7 +272,23 @@ static void rec_write_single (Records p, Record rec) cptr += rec->size[i]; } } - cptr = p->tmp_buf; + sysnos[0] = rec->sysno; + sysnos[1] = -1; + rec_write_tmp_buf (p, size, sysnos); +} + +static void rec_write_tmp_buf (Records p, int size, int *sysnos) +{ + struct record_index_entry entry; + int no_written = 0; + char *cptr = p->tmp_buf; + int block_prev = -1, block_free; + int dst_type = 0; + int i; + + for (i = 1; i= p->head.block_move[i]) + dst_type = i; while (no_written < size) { block_free = p->head.block_free[dst_type]; @@ -253,7 +310,11 @@ static void rec_write_single (Records p, Record rec) entry.next = block_free*8 + dst_type; entry.size = size; p->head.total_bytes += size; - write_indx (p, rec->sysno, &entry, sizeof(entry)); + while (*sysnos > 0) + { + write_indx (p, *sysnos, &entry, sizeof(entry)); + sysnos++; + } } else { @@ -346,19 +407,158 @@ Records rec_open (BFiles bfs, int rw) exit (1); } } - p->cache_max = 10; + p->cache_max = 400; p->cache_cur = 0; p->record_cache = (struct record_cache_entry *) xmalloc (sizeof(*p->record_cache)*p->cache_max); return p; } +static void rec_encode_unsigned (unsigned n, unsigned char *buf, int *len) +{ + (*len) = 0; + while (n > 127) + { + buf[*len] = 128 + (n & 127); + n = n >> 7; + (*len)++; + } + buf[*len] = n; + (*len)++; +} + +static void rec_decode_unsigned(unsigned *np, unsigned char *buf, int *len) +{ + unsigned n = 0; + unsigned w = 1; + (*len) = 0; + + while (buf[*len] > 127) + { + n += w*(buf[*len] & 127); + w = w << 7; + (*len)++; + } + n += w * buf[*len]; + (*len)++; + *np = n; +} +static void rec_cache_flush_block1 (Records p, Record rec, Record last_rec, + char **out_buf, int *out_size, + int *out_offset) +{ + int i; + int len; + + for (i = 0; isize[i] + 20 > *out_size) + { + int new_size = *out_offset + rec->size[i] + 65536; + char *np = (char *) xmalloc (new_size); + if (*out_offset) + memcpy (np, *out_buf, *out_offset); + xfree (*out_buf); + *out_size = new_size; + *out_buf = np; + } + if (i == 0) + { + rec_encode_unsigned (rec->sysno, *out_buf + *out_offset, &len); + (*out_offset) += len; + } + if (rec->size[i] == 0) + { + rec_encode_unsigned (1, *out_buf + *out_offset, &len); + (*out_offset) += len; + } + else if (last_rec && rec->size[i] == last_rec->size[i] && + !memcmp (rec->info[i], last_rec->info[i], rec->size[i])) + { + rec_encode_unsigned (0, *out_buf + *out_offset, &len); + (*out_offset) += len; + } + else + { + rec_encode_unsigned (rec->size[i]+1, *out_buf + *out_offset, &len); + (*out_offset) += len; + memcpy (*out_buf + *out_offset, rec->info[i], rec->size[i]); + (*out_offset) += rec->size[i]; + } + } +} + +static void rec_write_multiple (Records p, int saveCount) +{ + int i; + int ref_count = 0; + Record last_rec = 0; + int out_size = 1000; + int out_offset = 0; + char *out_buf = (char *) xmalloc (out_size); + int *sysnos = (int *) xmalloc (sizeof(*sysnos) * (p->cache_cur + 1)); + int *sysnop = sysnos; + + for (i = 0; icache_cur - saveCount; i++) + { + struct record_cache_entry *e = p->record_cache + i; + switch (e->flag) + { + case recordFlagNew: + rec_cache_flush_block1 (p, e->rec, last_rec, &out_buf, + &out_size, &out_offset); + *sysnop++ = e->rec->sysno; + ref_count++; + e->flag = recordFlagNop; + last_rec = e->rec; + break; + case recordFlagWrite: + rec_release_blocks (p, e->rec->sysno); + rec_cache_flush_block1 (p, e->rec, last_rec, &out_buf, + &out_size, &out_offset); + *sysnop++ = e->rec->sysno; + ref_count++; + e->flag = recordFlagNop; + last_rec = e->rec; + break; + default: + break; + } + } + *sysnop = -1; + if (ref_count) + { + int csize = out_offset + (out_offset >> 6) + 620; + + rec_tmp_expand (p, csize); +#if HAVE_BZLIB_H + i = bzBuffToBuffCompress (p->tmp_buf+2*sizeof(int), &csize, + out_buf, out_offset, 9, 0, 30); + if (i != BZ_OK) + { + logf (LOG_FATAL, "bzBuffToCompress error code=%d", i); + exit (1); + } +#else + memcpy (p->tmp_buf + 2*sizeof(int), out_buf, out_offset); + csize = out_offset; +#endif + memcpy (p->tmp_buf + sizeof(int), &ref_count, sizeof(ref_count)); + /* -------- compression */ + rec_write_tmp_buf (p, csize + sizeof(int), sysnos); + } + xfree (out_buf); + xfree (sysnos); +} + static void rec_cache_flush (Records p, int saveCount) { int i, j; if (saveCount >= p->cache_cur) saveCount = 0; + + rec_write_multiple (p, saveCount); for (i = 0; icache_cur - saveCount; i++) { struct record_cache_entry *e = p->record_cache + i; @@ -457,11 +657,12 @@ void rec_close (Records *pp) Record rec_get (Records p, int sysno) { - int i; + int i, in_size; Record rec, *recp; struct record_index_entry entry; int freeblock, dst_type; char *nptr, *cptr; + char *in_buf = 0; assert (sysno > 0); assert (p); @@ -482,7 +683,7 @@ Record rec_get (Records p, int sysno) assert (freeblock > 0); rec = (Record) xmalloc (sizeof(*rec)); - rec_tmp_expand (p, entry.size, dst_type); + rec_tmp_expand (p, entry.size); cptr = p->tmp_buf; bf_read (p->data_BFile[dst_type], freeblock, 0, 0, cptr); @@ -501,20 +702,51 @@ Record rec_get (Records p, int sysno) } rec->sysno = sysno; - nptr = p->tmp_buf + sizeof(freeblock); - for (i = 0; i < REC_NO_INFO; i++) +#if HAVE_BZLIB_H + in_size = entry.size * 30+100; + in_buf = (char *) xmalloc (in_size); + i = bzBuffToBuffDecompress (in_buf, &in_size, p->tmp_buf+2*sizeof(int), + entry.size-sizeof(int), 0, 4); + if (i != BZ_OK) { - memcpy (&rec->size[i], nptr, sizeof(*rec->size)); - nptr += sizeof(*rec->size); - if (rec->size[i]) - { - rec->info[i] = (char *) xmalloc (rec->size[i]); - memcpy (rec->info[i], nptr, rec->size[i]); - nptr += rec->size[i]; - } - else - rec->info[i] = NULL; + logf (LOG_FATAL, "bzBuffToDecompress error code=%d", i); + exit (1); + } +#else + in_buf = p->tmp_buf + 2*sizeof(int); + in_size = entry.size - sizeof(int); +#endif + nptr = in_buf; /* skip ref count */ + while (nptr < in_buf + in_size) + { + int this_sysno; + int len; + rec_decode_unsigned (&this_sysno, nptr, &len); + nptr += len; + + for (i = 0; i < REC_NO_INFO; i++) + { + int this_size; + rec_decode_unsigned (&this_size, nptr, &len); + nptr += len; + + if (this_size == 0) + continue; + rec->size[i] = this_size-1; + + if (rec->size[i]) + { + rec->info[i] = (char *) xmalloc (rec->size[i]); + memcpy (rec->info[i], nptr, rec->size[i]); + nptr += rec->size[i]; + } + else + rec->info[i] = NULL; + } + if (this_sysno == sysno) + break; } + xfree (in_buf); rec_cache_insert (p, rec, recordFlagNop); return rec; } diff --git a/index/recindex.h b/index/recindex.h index 2bc17e6..4c061fa 100644 --- a/index/recindex.h +++ b/index/recindex.h @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recindex.h,v $ - * Revision 1.15 1999-05-26 07:49:13 adam + * Revision 1.16 1999-06-25 13:48:02 adam + * Updated MSVC project files. + * Added BZIP2 record compression (not very well tested). + * + * Revision 1.15 1999/05/26 07:49:13 adam * C++ compilation. * * Revision 1.14 1999/02/02 14:51:04 adam @@ -76,6 +80,8 @@ typedef struct record_info { int newFlag; char *info[REC_NO_INFO]; size_t size[REC_NO_INFO]; + char buf_size[REC_NO_INFO][6]; + size_t size_size[REC_NO_INFO]; } *Record; typedef struct records_info *Records; diff --git a/zebra.dsp b/zebra.dsp index 97963b5..7d0d342 100644 --- a/zebra.dsp +++ b/zebra.dsp @@ -1,5 +1,5 @@ # Microsoft Developer Studio Project File - Name="zebra" - Package Owner=<4> -# Microsoft Developer Studio Generated Build File, Format Version 5.00 +# Microsoft Developer Studio Generated Build File, Format Version 6.00 # ** DO NOT EDIT ** # TARGTYPE "Win32 (x86) Static Library" 0x0104 @@ -22,9 +22,11 @@ CFG=zebra - Win32 Debug !MESSAGE # Begin Project +# PROP AllowPerConfigDependencies 0 # PROP Scc_ProjName "" # PROP Scc_LocalPath "" CPP=cl.exe +RSC=rc.exe !IF "$(CFG)" == "zebra - Win32 Release" @@ -39,7 +41,9 @@ CPP=cl.exe # PROP Intermediate_Dir "Release" # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /FD /c -# ADD CPP /nologo /MD /W3 /GX /O2 /I "include" /I "..\yaz\include" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /FD /c +# ADD CPP /nologo /MD /W3 /GX /O2 /I "include" /I "..\yaz\include" /I "..\bzip2-0.9.0c" /D "_WINDOWS" /D "WIN32" /D "NDEBUG" /D HAVE_BZLIB_H=1 /YX /FD /c +# ADD BASE RSC /l 0x409 +# ADD RSC /l 0x409 BSC32=bscmake.exe # ADD BASE BSC32 /nologo # ADD BSC32 /nologo @@ -60,7 +64,9 @@ LIB32=link.exe -lib # PROP Intermediate_Dir "Debug" # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /Z7 /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /YX /FD /c -# ADD CPP /nologo /MDd /W3 /GX /Zi /Od /I "include" /I "..\yaz\include" /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /FR /YX /FD /c +# ADD CPP /nologo /MDd /W3 /GX /ZI /Od /I "include" /I "..\yaz\include" /I "..\bzip2-0.9.0c" /D "_WINDOWS" /D "WIN32" /D "_DEBUG" /D HAVE_BZLIB_H=1 /FR /YX /FD /c +# ADD BASE RSC /l 0x409 +# ADD RSC /l 0x409 BSC32=bscmake.exe # ADD BASE BSC32 /nologo # ADD BSC32 /nologo @@ -156,6 +162,10 @@ SOURCE=.\isamc\isamc.c # End Source File # Begin Source File +SOURCE=.\Isamc\Isams.c +# End Source File +# Begin Source File + SOURCE=.\isam\isutil.c # End Source File # Begin Source File @@ -220,10 +230,6 @@ SOURCE=.\index\rank1.c # End Source File # Begin Source File -SOURCE=.\dfa\readfile.c -# End Source File -# Begin Source File - SOURCE=.\recctrl\recctrl.c # End Source File # Begin Source File @@ -276,6 +282,10 @@ SOURCE=.\rset\rsisamc.c # End Source File # Begin Source File +SOURCE=.\rset\rsisams.c +# End Source File +# Begin Source File + SOURCE=.\rset\rsm_or.c # End Source File # Begin Source File diff --git a/zebra.dsw b/zebra.dsw index 91c07ca..056a37b 100644 --- a/zebra.dsw +++ b/zebra.dsw @@ -1,4 +1,4 @@ -Microsoft Developer Studio Workspace File, Format Version 5.00 +Microsoft Developer Studio Workspace File, Format Version 6.00 # WARNING: DO NOT EDIT OR DELETE THIS WORKSPACE FILE! ############################################################################### -- 1.7.10.4