From ca39c711e9398043092cceeb3b4a7d8a0db1c0d8 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Tue, 21 Nov 2006 14:32:38 +0000 Subject: [PATCH] Split it_key stuff into it_key.c. For indexing, sort key chunks in a separate thread (POSIX threads only). --- include/Makefile.am | 7 +- include/it_key.h | 68 +++++++++ include/su_codec.h | 44 ++++++ index/Makefile.am | 8 +- index/extract.c | 277 ++--------------------------------- index/index.h | 174 ++++++++-------------- index/kcompare.c | 355 --------------------------------------------- index/key_block.c | 398 +++++++++++++++++++++++++++++++++++++++++++++++++++ index/key_block.h | 48 +++++++ index/kinput.c | 174 ++++++++-------------- index/reckeys.h | 6 +- index/zebraapi.c | 16 +-- util/Makefile.am | 4 +- util/it_key.c | 281 ++++++++++++++++++++++++++++++++++++ util/su_codec.c | 77 ++++++++++ 15 files changed, 1069 insertions(+), 868 deletions(-) create mode 100644 include/it_key.h create mode 100644 include/su_codec.h delete mode 100644 index/kcompare.c create mode 100644 index/key_block.c create mode 100644 index/key_block.h create mode 100644 util/it_key.c create mode 100644 util/su_codec.c diff --git a/include/Makefile.am b/include/Makefile.am index c3de7dc..c8a4834 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -1,6 +1,7 @@ -# $Id: Makefile.am,v 1.24 2006-08-15 14:28:32 adam Exp $ +# $Id: Makefile.am,v 1.25 2006-11-21 14:32:38 adam Exp $ noinst_HEADERS = bset.h charmap.h \ -direntz.h passwddb.h dfa.h zebra_xpath.h d1_absyn.h \ -rset.h dfaset.h sortidx.h zebra-lock.h attrfind.h zebramap.h + direntz.h passwddb.h dfa.h zebra_xpath.h d1_absyn.h \ + rset.h dfaset.h sortidx.h zebra-lock.h attrfind.h zebramap.h \ + it_key.h su_codec.h SUBDIRS = idzebra diff --git a/include/it_key.h b/include/it_key.h new file mode 100644 index 0000000..e2b5fb9 --- /dev/null +++ b/include/it_key.h @@ -0,0 +1,68 @@ +/* $Id: it_key.h,v 1.1 2006-11-21 14:32:38 adam Exp $ + Copyright (C) 1995-2006 + Index Data ApS + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef ZEBRA_IT_KEY_H +#define ZEBRA_IT_KEY_H + +#include +#include + +YAZ_BEGIN_CDECL + +#define IT_MAX_WORD 256 + +#define IT_KEY_LEVEL_MAX 5 +struct it_key { + int len; + zint mem[IT_KEY_LEVEL_MAX]; +}; + +void *iscz1_start(void); +void iscz1_reset(void *vp); +void iscz1_stop(void *p); +void iscz1_decode(void *vp, char **dst, const char **src); +void iscz1_encode(void *vp, char **dst, const char **src); + +int key_compare(const void *p1, const void *p2); +void key_init(struct it_key *k); +zint key_get_seq(const void *p); +zint key_get_segment(const void *p); +int key_compare_it(const void *p1, const void *p2); +int key_qsort_compare(const void *p1, const void *p2); +char *key_print_it(const void *p, char *buf); +void key_logdump(int mask, const void *p); +void key_logdump_txt(int logmask, const void *p, const char *txt); + +int key_SU_decode(int *ch, const unsigned char *out); +int key_SU_encode(int ch, char *out); + +YAZ_END_CDECL + +#endif +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/include/su_codec.h b/include/su_codec.h new file mode 100644 index 0000000..b58fdd9 --- /dev/null +++ b/include/su_codec.h @@ -0,0 +1,44 @@ +/* $Id: su_codec.h,v 1.1 2006-11-21 14:32:38 adam Exp $ + Copyright (C) 1995-2006 + Index Data ApS + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef ZEBRA_SU_CODEC_H +#define ZEBRA_SU_CODEC_H + +#include +#include + +YAZ_BEGIN_CDECL + +int key_SU_decode(int *ch, const unsigned char *out); +int key_SU_encode(int ch, char *out); + +YAZ_END_CDECL + +#endif +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/Makefile.am b/index/Makefile.am index 2430a4b..5f7907f 100644 --- a/index/Makefile.am +++ b/index/Makefile.am @@ -1,4 +1,4 @@ -## $Id: Makefile.am,v 1.58 2006-11-14 08:12:08 adam Exp $ +## $Id: Makefile.am,v 1.59 2006-11-21 14:32:38 adam Exp $ aux_libs = \ ../rset/libidzebra-rset.la \ @@ -81,14 +81,14 @@ libidzebra_2_0_la_SOURCES = \ dir.c dirs.c \ extract.c \ index.h invstat.c \ - kinput.c kcompare.c kcontrol.c \ + kinput.c kcontrol.c \ limit.c \ orddict.c orddict.h \ rank.h rank1.c ranksimilarity.c rankstatic.c \ recindex.c recindex.h recindxp.h reckeys.c reckeys.h recstat.c retrieve.c \ rpnscan.c rpnsearch.c sortidx.c symtab.c stream.c \ - update_path.c update_file.c trunc.c untrans.c \ - zaptterm.c zebraapi.c zinfo.c zinfo.h zsets.c + update_path.c update_file.c trunc.c untrans.c isam_methods.c \ + zaptterm.c zebraapi.c zinfo.c zinfo.h zsets.c key_block.c key_block.h bin_PROGRAMS = zebraidx zebrasrv diff --git a/index/extract.c b/index/extract.c index 33ee2e5..0a1d8c9 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,4 +1,4 @@ -/* $Id: extract.c,v 1.238 2006-11-20 13:59:13 adam Exp $ +/* $Id: extract.c,v 1.239 2006-11-21 14:32:38 adam Exp $ Copyright (C) 1995-2006 Index Data ApS @@ -36,13 +36,6 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include -#define ENCODE_BUFLEN 768 -struct encode_info { - void *encode_handle; - void *decode_handle; - char buf[ENCODE_BUFLEN]; -}; - static int log_level_extract = 0; static int log_level_details = 0; static int log_level_initialized = 0; @@ -66,37 +59,6 @@ static void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid); static void extract_token_add (RecWord *p); -static void encode_key_init (struct encode_info *i); -static void encode_key_write (char *k, struct encode_info *i, FILE *outf); -static void encode_key_flush (struct encode_info *i, FILE *outf); - -#define USE_SHELLSORT 0 - -#if USE_SHELLSORT -static void shellsort(void *ar, int r, size_t s, - int (*cmp)(const void *a, const void *b)) -{ - char *a = ar; - char v[100]; - int h, i, j, k; - static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936, - 13776, 4592, 1968, 861, 336, - 112, 48, 21, 7, 3, 1 }; - for ( k = 0; k < 16; k++) - for (h = incs[k], i = h; i < r; i++) - { - memcpy (v, a+s*i, s); - j = i; - while (j > h && (*cmp)(a + s*(j-h), v) > 0) - { - memcpy (a + s*j, a + s*(j-h), s); - j -= h; - } - memcpy (a+s*j, v, s); - } -} -#endif - static void logRecord (ZebraHandle zh) { ++zh->records_processed; @@ -1042,10 +1004,9 @@ void extract_rec_keys_adjust(ZebraHandle zh, int is_insert, } } -void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, - int cmd, - zebra_rec_keys_t reckeys, - zint staticrank) +void extract_flushRecordKeys(ZebraHandle zh, SYSNO sysno, int cmd, + zebra_rec_keys_t reckeys, + zint staticrank) { ZebraExplainInfo zei = zh->reg->zei; @@ -1058,21 +1019,11 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, extract_rec_keys_log(zh, cmd, reckeys, log_level_details); } - if (!zh->reg->key_buf) + if (!zh->reg->key_block) { int mem= 1024*1024* atoi( res_get_def( zh->res, "memmax", "8")); - if (mem <= 0) - { - yaz_log(YLOG_WARN, "Invalid memory setting, using default 8 MB"); - mem= 1024*1024*8; - } - /* FIXME: That "8" should be in a default settings include */ - /* not hard-coded here! -H */ - zh->reg->key_buf = (char**) xmalloc (mem); - zh->reg->ptr_top = mem/sizeof(char*); - zh->reg->ptr_i = 0; - zh->reg->key_buf_used = 0; - zh->reg->key_file_no = 0; + const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", "."); + zh->reg->key_block = key_block_create(mem, key_tmp_dir); } zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1); @@ -1083,159 +1034,13 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, struct it_key key_in; while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in)) { - int ch = 0; - int i, j = 0; - struct it_key key_out; - - assert(key_in.len >= 2); - assert(key_in.len <= IT_KEY_LEVEL_MAX); - - /* check for buffer overflow */ - if (zh->reg->key_buf_used + 1024 > - (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*)) - extract_flushWriteKeys (zh, 0); - - ++(zh->reg->ptr_i); - assert(zh->reg->ptr_i > 0); - (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] = - (char*)zh->reg->key_buf + zh->reg->key_buf_used; - - /* key_in.mem[0] ord/ch */ - /* key_in.mem[1] filter specified record ID */ - - /* encode the ordinal value (field/use/attribute) .. */ - ch = CAST_ZINT_TO_INT(key_in.mem[0]); - zh->reg->key_buf_used += - key_SU_encode(ch, (char*)zh->reg->key_buf + - zh->reg->key_buf_used); - - /* copy the 0-terminated stuff from str to output */ - memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, str, slen); - zh->reg->key_buf_used += slen; - ((char*)zh->reg->key_buf)[(zh->reg->key_buf_used)++] = '\0'; - - /* the delete/insert indicator */ - ((char*)zh->reg->key_buf)[(zh->reg->key_buf_used)++] = cmd; - - if (zh->m_staticrank) /* rank config enabled ? */ - { - if (staticrank < 0) - { - yaz_log(YLOG_WARN, "staticrank = %ld. Setting to 0", - (long) staticrank); - staticrank = 0; - } - key_out.mem[j++] = staticrank; - } - - if (key_in.mem[1]) /* filter specified record ID */ - key_out.mem[j++] = key_in.mem[1]; - else - key_out.mem[j++] = sysno; - for (i = 2; i < key_in.len; i++) - key_out.mem[j++] = key_in.mem[i]; - key_out.len = j; - - memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, - &key_out, sizeof(key_out)); - (zh->reg->key_buf_used) += sizeof(key_out); + key_block_write(zh->reg->key_block, sysno, + &key_in, cmd, str, slen, + staticrank, zh->m_staticrank); } } } -void extract_flushWriteKeys (ZebraHandle zh, int final) - /* optimizing: if final=1, and no files written yet */ - /* push the keys directly to merge, sidestepping the */ - /* temp file altogether. Speeds small updates */ -{ - FILE *outf; - char out_fname[200]; - char *prevcp, *cp; - struct encode_info encode_info; - int ptr_i = zh->reg->ptr_i; - int temp_policy; - if (!zh->reg->key_buf || ptr_i <= 0) - { - yaz_log(log_level_extract, " nothing to flush section=%d buf=%p i=%d", - zh->reg->key_file_no, zh->reg->key_buf, ptr_i); - return; - } - - (zh->reg->key_file_no)++; - yaz_log (YLOG_LOG, "sorting section %d", (zh->reg->key_file_no)); - yaz_log(log_level_extract, " sort_buff at %p n=%d", - zh->reg->key_buf + zh->reg->ptr_top - ptr_i,ptr_i); - - -#if USE_SHELLSORT - shellsort(zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, - sizeof(char*), key_qsort_compare); -#else - qsort(zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i, - sizeof(char*), key_qsort_compare); -#endif - /* zebra.cfg: tempfiles: - Y: always use temp files (old way) - A: use temp files, if more than one (auto) - = if this is both the last and the first - N: never bother with temp files (new) */ - - temp_policy=toupper(res_get_def(zh->res,"tempfiles","auto")[0]); - if (temp_policy != 'Y' && temp_policy != 'N' && temp_policy != 'A') { - yaz_log (YLOG_WARN, "Illegal tempfiles setting '%c'. using 'Auto' ", - temp_policy); - temp_policy='A'; - } - - if ( ( temp_policy =='N' ) || /* always from memory */ - ( ( temp_policy =='A' ) && /* automatic */ - (zh->reg->key_file_no == 1) && /* this is first time */ - (final) ) ) /* and last (=only) time */ - { /* go directly from memory */ - zh->reg->key_file_no =0; /* signal not to read files */ - zebra_index_merge(zh); - zh->reg->ptr_i = 0; - zh->reg->key_buf_used = 0; - return; - } - - /* Not doing directly from memory, write into a temp file */ - extract_get_fname_tmp (zh, out_fname, zh->reg->key_file_no); - - if (!(outf = fopen (out_fname, "wb"))) - { - yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname); - zebra_exit("extract_flushWriteKeys"); - } - yaz_log (YLOG_LOG, "writing section %d", zh->reg->key_file_no); - prevcp = cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i]; - - encode_key_init (&encode_info); - encode_key_write (cp, &encode_info, outf); - - while (--ptr_i > 0) - { - cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i]; - if (strcmp (cp, prevcp)) - { - encode_key_flush ( &encode_info, outf); - encode_key_init (&encode_info); - encode_key_write (cp, &encode_info, outf); - prevcp = cp; - } - else - encode_key_write (cp + strlen(cp), &encode_info, outf); - } - encode_key_flush ( &encode_info, outf); - if (fclose (outf)) - { - yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname); - zebra_exit("extract_flushWriteKeys"); - } - yaz_log (YLOG_LOG, "finished section %d", zh->reg->key_file_no); - zh->reg->ptr_i = 0; - zh->reg->key_buf_used = 0; -} ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh, zebra_rec_keys_t reckeys, @@ -1535,14 +1340,14 @@ static void extract_set_store_data_prepare(struct recExtractCtrl *p) p->setStoreData = extract_set_store_data_cb; } -static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid) +static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid) { ZebraHandle zh = (ZebraHandle) p->handle; zebraExplain_addSchema (zh->reg->zei, oid); } -void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, - int cmd, zebra_rec_keys_t reckeys) +void extract_flushSortKeys(ZebraHandle zh, SYSNO sysno, + int cmd, zebra_rec_keys_t reckeys) { if (zebra_rec_keys_rewind(reckeys)) { @@ -1566,62 +1371,6 @@ void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, } } -static void encode_key_init(struct encode_info *i) -{ - i->encode_handle = iscz1_start(); - i->decode_handle = iscz1_start(); -} - -static void encode_key_write (char *k, struct encode_info *i, FILE *outf) -{ - struct it_key key; - char *bp = i->buf, *bp0; - const char *src = (char *) &key; - - /* copy term to output buf */ - while ((*bp++ = *k++)) - ; - /* and copy & align key so we can mangle */ - memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */ - -#if 0 - /* debugging */ - key_logdump_txt(YLOG_LOG, &key, *k ? "i" : "d"); -#endif - assert(key.mem[0] >= 0); - - bp0 = bp++; - iscz1_encode(i->encode_handle, &bp, &src); - - *bp0 = (*k * 128) + bp - bp0 - 1; /* length and insert/delete combined */ - if (fwrite (i->buf, bp - i->buf, 1, outf) != 1) - { - yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite"); - zebra_exit("encode_key_write"); - } - -#if 0 - /* debugging */ - if (1) - { - struct it_key key2; - const char *src = bp0+1; - char *dst = (char*) &key2; - iscz1_decode(i->decode_handle, &dst, &src); - - key_logdump_txt(YLOG_LOG, &key2, *k ? "i" : "d"); - - assert(key2.mem[1]); - } -#endif -} - -static void encode_key_flush (struct encode_info *i, FILE *outf) -{ - iscz1_stop(i->encode_handle); - iscz1_stop(i->decode_handle); -} - /* * Local variables: * c-basic-offset: 4 diff --git a/index/index.h b/index/index.h index d323286..cf13c2d 100644 --- a/index/index.h +++ b/index/index.h @@ -1,4 +1,4 @@ -/* $Id: index.h,v 1.181 2006-11-09 14:39:24 adam Exp $ +/* $Id: index.h,v 1.182 2006-11-21 14:32:38 adam Exp $ Copyright (C) 1995-2006 Index Data ApS @@ -20,8 +20,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ -#ifndef INDEX_H -#define INDEX_H +#ifndef ZEBRA_INDEX_H +#define ZEBRA_INDEX_H #include #include @@ -48,20 +48,13 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include +#include +#include + YAZ_BEGIN_CDECL #define ISAM_DEFAULT "b" -#define SU_SCHEME 1 - -#define IT_MAX_WORD 256 - -#define IT_KEY_LEVEL_MAX 5 -struct it_key { - int len; - zint mem[IT_KEY_LEVEL_MAX]; -}; - enum dirsKind { dirs_dir, dirs_file }; struct dir_entry { @@ -77,79 +70,61 @@ struct dirs_entry { time_t mtime; }; -void getFnameTmp (Res res, char *fname, int no); +void getFnameTmp(Res res, char *fname, int no); -struct dirs_info *dirs_open (Dict dict, const char *rep, int rw); -struct dirs_info *dirs_fopen (Dict dict, const char *path, int rw); -struct dirs_entry *dirs_read (struct dirs_info *p); -struct dirs_entry *dirs_last (struct dirs_info *p); -void dirs_mkdir (struct dirs_info *p, const char *src, time_t mtime); -void dirs_rmdir (struct dirs_info *p, const char *src); -void dirs_add (struct dirs_info *p, const char *src, SYSNO sysno, time_t mtime); -void dirs_del (struct dirs_info *p, const char *src); -void dirs_free (struct dirs_info **pp); - -struct dir_entry *dir_open (const char *rep, const char *base, +struct dirs_info *dirs_open(Dict dict, const char *rep, int rw); +struct dirs_info *dirs_fopen(Dict dict, const char *path, int rw); +struct dirs_entry *dirs_read(struct dirs_info *p); +struct dirs_entry *dirs_last(struct dirs_info *p); +void dirs_mkdir(struct dirs_info *p, const char *src, time_t mtime); +void dirs_rmdir(struct dirs_info *p, const char *src); +void dirs_add(struct dirs_info *p, const char *src, SYSNO sysno, time_t mtime); +void dirs_del(struct dirs_info *p, const char *src); +void dirs_free(struct dirs_info **pp); + +struct dir_entry *dir_open(const char *rep, const char *base, int follow_links); -void dir_sort (struct dir_entry *e); -void dir_free (struct dir_entry **e_p); - -void repositoryUpdate (ZebraHandle zh, const char *path); -void repositoryAdd (ZebraHandle zh, const char *path); -void repositoryDelete (ZebraHandle zh, const char *path); -void repositoryShow (ZebraHandle zh, const char *path); - -int key_open (ZebraHandle zh, int mem); -int key_close (ZebraHandle zh); -int key_compare (const void *p1, const void *p2); -void key_init(struct it_key *k); -char *key_print_it (const void *p, char *buf); -zint key_get_seq (const void *p); -zint key_get_segment (const void *p); -int key_compare_it (const void *p1, const void *p2); -int key_qsort_compare (const void *p1, const void *p2); -void key_logdump (int mask, const void *p); -void key_logdump_txt (int logmask, const void *p, const char *txt); -void inv_prstat (ZebraHandle zh); -void inv_compact (BFiles bfs); -void key_input (ZebraHandle zh, int nkeys, int cache, Res res); -ISAMS_M *key_isams_m (Res res, ISAMS_M *me); -ISAMC_M *key_isamc_m (Res res, ISAMC_M *me); -int merge_sort (char **buf, int from, int to); -int key_SU_code (int ch, char *out); +void dir_sort(struct dir_entry *e); +void dir_free(struct dir_entry **e_p); + +void repositoryUpdate(ZebraHandle zh, const char *path); +void repositoryAdd(ZebraHandle zh, const char *path); +void repositoryDelete(ZebraHandle zh, const char *path); +void repositoryShow(ZebraHandle zh, const char *path); + +void inv_prstat(ZebraHandle zh); +void inv_compact(BFiles bfs); +void key_input(ZebraHandle zh, int nkeys, int cache, Res res); +ISAMS_M *key_isams_m(Res res, ISAMS_M *me); +ISAMC_M *key_isamc_m(Res res, ISAMC_M *me); #define FNAME_DICT "dict" #define FNAME_ISAM "isam" #define FNAME_ISAMC "isamc" #define FNAME_ISAMS "isams" -#define FNAME_ISAMH "isamh" -#define FNAME_ISAMD "isamd" #define FNAME_CONFIG "zebra.cfg" #define GMATCH_DICT "gmatch" #define FMATCH_DICT "fmatch%d" -struct strtab *strtab_mk (void); -int strtab_src (struct strtab *t, const char *name, void ***infop); -void strtab_del (struct strtab *t, - void (*func)(const char *name, void *info, void *data), - void *data); +struct strtab *strtab_mk(void); +int strtab_src(struct strtab *t, const char *name, void ***infop); +void strtab_del(struct strtab *t, + void (*func)(const char *name, void *info, void *data), + void *data); -void zebraIndexLockMsg (ZebraHandle zh, const char *str); -void zebraIndexUnlock (ZebraHandle zh); -int zebraIndexLock (BFiles bfs, ZebraHandle zh, int commitNow, const char *rval); -int zebraIndexWait (ZebraHandle zh, int commitPhase); +void zebraIndexLockMsg(ZebraHandle zh, const char *str); +void zebraIndexUnlock(ZebraHandle zh); +int zebraIndexLock(BFiles bfs, ZebraHandle zh, int commitNow, const char *rval); +int zebraIndexWait(ZebraHandle zh, int commitPhase); -void zebra_lock_prefix (Res res, char *dst); +void zebra_lock_prefix(Res res, char *dst); #define FNAME_MAIN_LOCK "zebraidx.LCK" #define FNAME_COMMIT_LOCK "zebracmt.LCK" #define FNAME_ORG_LOCK "zebraorg.LCK" #define FNAME_TOUCH_TIME "zebraidx.time" -int key_SU_decode (int *ch, const unsigned char *out); -int key_SU_encode (int ch, char *out); - typedef struct zebra_set *ZebraSet; typedef struct zebra_rank_class { @@ -160,6 +135,7 @@ typedef struct zebra_rank_class { } *ZebraRankClass; #include "reckeys.h" +#include "key_block.h" struct zebra_register { char *name; @@ -187,11 +163,8 @@ struct zebra_register { zebra_rec_keys_t keys; zebra_rec_keys_t sortKeys; - char **key_buf; - size_t ptr_top; - size_t ptr_i; - size_t key_buf_used; - int key_file_no; + + zebra_key_block_t key_block; }; struct zebra_service { @@ -315,42 +288,36 @@ RSET rset_trunc(ZebraHandle zh, ISAM_P *isam_p, int no, void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type, const char *db, const char *index_name, const char *term); -ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov); -ZebraSet resultSetGet (ZebraHandle zh, const char *name); -ZEBRA_RES resultSetAddRPN (ZebraHandle zh, NMEM m, Z_RPNQuery *rpn, +ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov); +ZebraSet resultSetGet(ZebraHandle zh, const char *name); +ZEBRA_RES resultSetAddRPN(ZebraHandle zh, NMEM m, Z_RPNQuery *rpn, int num_bases, char **basenames, const char *setname); -RSET resultSetRef (ZebraHandle zh, const char *resultSetId); -void resultSetDestroy (ZebraHandle zh, int num_names, char **names, +RSET resultSetRef(ZebraHandle zh, const char *resultSetId); +void resultSetDestroy(ZebraHandle zh, int num_names, char **names, int *statuses); -ZEBRA_RES resultSetSort (ZebraHandle zh, NMEM nmem, +ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem, int num_input_setnames, const char **input_setnames, const char *output_setname, Z_SortKeySpecList *sort_sequence, int *sort_status); -ZEBRA_RES resultSetSortSingle (ZebraHandle zh, NMEM nmem, +ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, ZebraSet sset, RSET rset, Z_SortKeySpecList *sort_sequence, int *sort_status); -ZEBRA_RES resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, +ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem); -void resultSetInvalidate (ZebraHandle zh); - -int zebra_server_lock_init (ZebraService zh); -int zebra_server_lock_destroy (ZebraService zh); -int zebra_server_lock (ZebraService zh, int lockCommit); -void zebra_server_unlock (ZebraService zh, int commitPhase); -int zebra_server_lock_get_state (ZebraService zh, time_t *timep); +void resultSetInvalidate(ZebraHandle zh); -int zebra_record_fetch (ZebraHandle zh, SYSNO sysno, int score, +int zebra_record_fetch(ZebraHandle zh, SYSNO sysno, int score, zebra_snippets *hit_snippet, ODR stream, oid_value input_format, Z_RecordComposition *comp, oid_value *output_format, char **rec_bufp, int *rec_lenp, char **basenamep, char **addinfo); -void extract_get_fname_tmp (ZebraHandle zh, char *fname, int no); +void extract_get_fname_tmp(ZebraHandle zh, char *fname, int no); -void zebra_index_merge (ZebraHandle zh); +void zebra_index_merge(ZebraHandle zh); ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh, const char *buf, size_t buf_size, @@ -376,16 +343,6 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, RecType recType, void *recTypeClientData); -#if 0 -int extract_rec_in_mem (ZebraHandle zh, const char *recordType, - const char *buf, size_t buf_size, - const char *databaseName, int delete_flag, - int test_mode, int *sysno, - int store_keys, int store_data, - const char *match_criteria); -#endif -void extract_flushWriteKeys (ZebraHandle zh, int final); - YAZ_EXPORT void zebra_create_stream_mem(struct ZebraRecStream *stream, const char *buf, size_t sz); YAZ_EXPORT void zebra_create_stream_fd(struct ZebraRecStream *stream, @@ -402,22 +359,14 @@ ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n); ZEBRA_RES zebra_extract_file(ZebraHandle zh, SYSNO *sysno, const char *fname, int deleteFlag); -ZEBRA_RES zebra_begin_read (ZebraHandle zh); -ZEBRA_RES zebra_end_read (ZebraHandle zh); +ZEBRA_RES zebra_begin_read(ZebraHandle zh); +ZEBRA_RES zebra_end_read(ZebraHandle zh); -int zebra_file_stat (const char *file_name, struct stat *buf, +int zebra_file_stat(const char *file_name, struct stat *buf, int follow_links); -void zebra_livcode_transform(ZebraHandle zh, Z_RPNQuery *query); - -void *iscz1_start (void); -void iscz1_reset (void *vp); -void iscz1_stop (void *p); -void iscz1_decode (void *vp, char **dst, const char **src); -void iscz1_encode (void *vp, char **dst, const char **src); - -Dict dict_open_res (BFiles bfs, const char *name, int cache, int rw, - int compact_flag, Res res); +Dict dict_open_res(BFiles bfs, const char *name, int cache, int rw, + int compact_flag, Res res); void zebra_setError(ZebraHandle zh, int code, const char *addinfo); void zebra_setError_zint(ZebraHandle zh, int code, zint i); @@ -454,7 +403,6 @@ ZEBRA_RES zebra_update_from_path(ZebraHandle zh, const char *path); ZEBRA_RES zebra_delete_from_path(ZebraHandle zh, const char *path); ZEBRA_RES zebra_remove_file_match(ZebraHandle zh); - struct rpn_char_map_info { ZebraMaps zm; diff --git a/index/kcompare.c b/index/kcompare.c deleted file mode 100644 index 00050c7..0000000 --- a/index/kcompare.c +++ /dev/null @@ -1,355 +0,0 @@ -/* $Id: kcompare.c,v 1.61 2006-08-16 13:16:36 adam Exp $ - Copyright (C) 1995-2006 - Index Data ApS - -This file is part of the Zebra server. - -Zebra is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Zebra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#include -#include -#include -#include - -#include "index.h" - -#ifdef __GNUC__ -#define CODEC_INLINE inline -#else -#define CODEC_INLINE -#endif - -void key_logdump_txt(int logmask, const void *p, const char *txt) -{ - struct it_key key; - if (!txt) - txt = "(none)"; - if (p) - { - char formstr[128]; - int i; - - memcpy (&key, p, sizeof(key)); - assert(key.len > 0 && key.len <= IT_KEY_LEVEL_MAX); - *formstr = '\0'; - for (i = 0; ilen; - if (((struct it_key *) p2)->len > l) - l = ((struct it_key *) p2)->len; - assert (l <= IT_KEY_LEVEL_MAX && l > 0); - for (i = 0; i < l; i++) - { - if (((struct it_key *) p1)->mem[i] != ((struct it_key *) p2)->mem[i]) - { - if (((struct it_key *) p1)->mem[i] > ((struct it_key *) p2)->mem[i]) - return l-i; - else - return i-l; - } - } - return 0; -} - -char *key_print_it (const void *p, char *buf) -{ - strcpy(buf, ""); - return buf; -} - -int key_compare (const void *p1, const void *p2) -{ - struct it_key i1, i2; - int i, l; - memcpy (&i1, p1, sizeof(i1)); - memcpy (&i2, p2, sizeof(i2)); - l = i1.len; - if (i2.len > l) - l = i2.len; - assert (l <= IT_KEY_LEVEL_MAX && l > 0); - for (i = 0; i < l; i++) - { - if (i1.mem[i] != i2.mem[i]) - { - if (i1.mem[i] > i2.mem[i]) - return l-i; - else - return i-l; - } - } - return 0; -} - -zint key_get_seq(const void *p) -{ - struct it_key k; - memcpy (&k, p, sizeof(k)); - return k.mem[k.len-1]; -} - -zint key_get_segment(const void *p) -{ - struct it_key k; - memcpy (&k, p, sizeof(k)); - return k.mem[k.len-2]; -} - -int key_qsort_compare (const void *p1, const void *p2) -{ - int r; - size_t l; - char *cp1 = *(char **) p1; - char *cp2 = *(char **) p2; - - if ((r = strcmp (cp1, cp2))) - return r; - l = strlen(cp1)+1; - if ((r = key_compare (cp1+l+1, cp2+l+1))) - return r; - return cp1[l] - cp2[l]; -} - -struct iscz1_code_info { - struct it_key key; -}; - -void *iscz1_start (void) -{ - struct iscz1_code_info *p = (struct iscz1_code_info *) - xmalloc (sizeof(*p)); - iscz1_reset(p); - return p; -} - -void key_init(struct it_key *key) -{ - int i; - key->len = 0; - for (i = 0; i < IT_KEY_LEVEL_MAX; i++) - key->mem[i] = 0; -} - -void iscz1_reset (void *vp) -{ - struct iscz1_code_info *p = (struct iscz1_code_info *) vp; - int i; - p->key.len = 0; - for (i = 0; i < IT_KEY_LEVEL_MAX; i++) - p->key.mem[i] = 0; -} - -void iscz1_stop (void *p) -{ - xfree (p); -} - -/* small encoder that works with unsigneds of any length */ -static CODEC_INLINE void iscz1_encode_int (zint d, char **dst) -{ - unsigned char *bp = (unsigned char*) *dst; - - while (d > 127) - { - *bp++ = (unsigned) (128 | (d & 127)); - d = d >> 7; - } - *bp++ = (unsigned) d; - *dst = (char *) bp; -} - -/* small decoder that works with unsigneds of any length */ -static CODEC_INLINE zint iscz1_decode_int (unsigned char **src) -{ - zint d = 0; - unsigned char c; - unsigned r = 0; - - while (((c = *(*src)++) & 128)) - { - d += ((zint) (c&127) << r); - r += 7; - } - d += ((zint) c << r); - return d; -} - -void iscz1_encode (void *vp, char **dst, const char **src) -{ - struct iscz1_code_info *p = (struct iscz1_code_info *) vp; - struct it_key tkey; - zint d; - int i; - - /* 1 - 3, 2, 9, 12 - 3, 2, 10, 2 - 4, 1 - - if diff is 0, then there is more ... - if diff is non-zero, then _may_ be more - */ - memcpy (&tkey, *src, sizeof(struct it_key)); - - /* deal with leader + delta encoding .. */ - d = 0; - assert(tkey.len > 0 && tkey.len <= IT_KEY_LEVEL_MAX); - for (i = 0; i < tkey.len; i++) - { - d = tkey.mem[i] - p->key.mem[i]; - if (d || i == tkey.len-1) - { /* all have been equal until now, now make delta .. */ - p->key.mem[i] = tkey.mem[i]; - if (d > 0) - { - iscz1_encode_int (i + (tkey.len << 3) + 64, dst); - i++; - iscz1_encode_int (d, dst); - } - else - { - iscz1_encode_int (i + (tkey.len << 3), dst); - } - break; - } - } - /* rest uses absolute encoding ... */ - for (; i < tkey.len; i++) - { - iscz1_encode_int (tkey.mem[i], dst); - p->key.mem[i] = tkey.mem[i]; - } - (*src) += sizeof(struct it_key); -} - -void iscz1_decode (void *vp, char **dst, const char **src) -{ - struct iscz1_code_info *p = (struct iscz1_code_info *) vp; - int i; - - int leader = (int) iscz1_decode_int ((unsigned char **) src); - i = leader & 7; - if (leader & 64) - p->key.mem[i] += iscz1_decode_int ((unsigned char **) src); - else - p->key.mem[i] = iscz1_decode_int ((unsigned char **) src); - p->key.len = (leader >> 3) & 7; - while (++i < p->key.len) - p->key.mem[i] = iscz1_decode_int ((unsigned char **) src); - memcpy (*dst, &p->key, sizeof(struct it_key)); - (*dst) += sizeof(struct it_key); -} - -ISAMS_M *key_isams_m (Res res, ISAMS_M *me) -{ - isams_getmethod (me); - - me->compare_item = key_compare; - me->log_item = key_logdump_txt; - - me->codec.start = iscz1_start; - me->codec.decode = iscz1_decode; - me->codec.encode = iscz1_encode; - me->codec.stop = iscz1_stop; - me->codec.reset = iscz1_reset; - - me->debug = atoi(res_get_def (res, "isamsDebug", "0")); - - return me; -} - -ISAMC_M *key_isamc_m (Res res, ISAMC_M *me) -{ - isamc_getmethod (me); - - me->compare_item = key_compare; - me->log_item = key_logdump_txt; - - me->codec.start = iscz1_start; - me->codec.decode = iscz1_decode; - me->codec.encode = iscz1_encode; - me->codec.stop = iscz1_stop; - me->codec.reset = iscz1_reset; - - me->debug = atoi(res_get_def (res, "isamcDebug", "0")); - - return me; -} - -int key_SU_encode (int ch, char *out) -{ - int i; - for (i = 0; ch; i++) - { - if (ch >= 64) - out[i] = 65 + (ch & 63); - else - out[i] = 1 + ch; - ch = ch >> 6; - } - return i; - /* in out - 0 1 - 1 2 - 63 64 - 64 65, 2 - 65 66, 2 - 127 128, 2 - 128 65, 3 - 191 128, 3 - 192 65, 4 - */ -} - -int key_SU_decode (int *ch, const unsigned char *out) -{ - int len = 1; - int fact = 1; - *ch = 0; - for (len = 1; *out >= 65; len++, out++) - { - *ch += (*out - 65) * fact; - fact <<= 6; - } - *ch += (*out - 1) * fact; - return len; -} - -/* - * Local variables: - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/index/key_block.c b/index/key_block.c new file mode 100644 index 0000000..7888fd8 --- /dev/null +++ b/index/key_block.c @@ -0,0 +1,398 @@ +/* $Id: key_block.c,v 1.1 2006-11-21 14:32:38 adam Exp $ + Copyright (C) 1995-2006 + Index Data ApS + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include +#include +#include +#include + +#if YAZ_POSIX_THREADS +#include +#endif + +#include +#include "index.h" +#include "key_block.h" + +struct zebra_key_block { + char **key_buf; + size_t ptr_top; + size_t ptr_i; + size_t key_buf_used; + int key_file_no; + char *key_tmp_dir; +#if YAZ_POSIX_THREADS + char **alt_buf; + char **thread_key_buf; + size_t thread_ptr_top; + size_t thread_ptr_i; + int exit_flag; + pthread_t thread_id; + pthread_mutex_t mutex; + + pthread_cond_t work_available; + + pthread_cond_t cond_sorting; + int is_sorting; +#endif +}; + +#define ENCODE_BUFLEN 768 +struct encode_info { + void *encode_handle; + void *decode_handle; + char buf[ENCODE_BUFLEN]; +}; + +static int log_level = 0; + +#define USE_SHELLSORT 0 + +#if USE_SHELLSORT +static void shellsort(void *ar, int r, size_t s, + int (*cmp)(const void *a, const void *b)) +{ + char *a = ar; + char v[100]; + int h, i, j, k; + static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936, + 13776, 4592, 1968, 861, 336, + 112, 48, 21, 7, 3, 1 }; + for ( k = 0; k < 16; k++) + for (h = incs[k], i = h; i < r; i++) + { + memcpy (v, a+s*i, s); + j = i; + while (j > h && (*cmp)(a + s*(j-h), v) > 0) + { + memcpy (a + s*j, a + s*(j-h), s); + j -= h; + } + memcpy (a+s*j, v, s); + } +} +#endif + + +static void encode_key_init(struct encode_info *i) +{ + i->encode_handle = iscz1_start(); + i->decode_handle = iscz1_start(); +} + +static void encode_key_write (char *k, struct encode_info *i, FILE *outf) +{ + struct it_key key; + char *bp = i->buf, *bp0; + const char *src = (char *) &key; + + /* copy term to output buf */ + while ((*bp++ = *k++)) + ; + /* and copy & align key so we can mangle */ + memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */ + +#if 0 + /* debugging */ + key_logdump_txt(YLOG_LOG, &key, *k ? "i" : "d"); +#endif + assert(key.mem[0] >= 0); + + bp0 = bp++; + iscz1_encode(i->encode_handle, &bp, &src); + + *bp0 = (*k * 128) + bp - bp0 - 1; /* length and insert/delete combined */ + if (fwrite (i->buf, bp - i->buf, 1, outf) != 1) + { + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite"); + zebra_exit("encode_key_write"); + } + +#if 0 + /* debugging */ + if (1) + { + struct it_key key2; + const char *src = bp0+1; + char *dst = (char*) &key2; + iscz1_decode(i->decode_handle, &dst, &src); + + key_logdump_txt(YLOG_LOG, &key2, *k ? "i" : "d"); + + assert(key2.mem[1]); + } +#endif +} + +static void encode_key_flush (struct encode_info *i, FILE *outf) +{ + iscz1_stop(i->encode_handle); + iscz1_stop(i->decode_handle); +} + +void key_block_flush_int(zebra_key_block_t p, + char **key_buf, size_t ptr_top, size_t ptr_i); + +#if YAZ_POSIX_THREADS +static void *thread_func(void *vp) +{ + zebra_key_block_t p = (zebra_key_block_t) vp; + while (1) + { + pthread_mutex_lock(&p->mutex); + + while (!p->is_sorting && !p->exit_flag) + pthread_cond_wait(&p->work_available, &p->mutex); + + if (p->exit_flag) + break; + + pthread_mutex_unlock(&p->mutex); + + key_block_flush_int(p, p->thread_key_buf, + p->thread_ptr_top, p->thread_ptr_i); + + pthread_mutex_lock(&p->mutex); + p->is_sorting = 0; + pthread_cond_signal(&p->cond_sorting); + pthread_mutex_unlock(&p->mutex); + } + pthread_mutex_unlock(&p->mutex); + return 0; +} +#endif + +zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir) +{ + zebra_key_block_t p = xmalloc(sizeof(*p)); + + p->key_buf = (char**) xmalloc (mem); + p->ptr_top = mem/sizeof(char*); + p->ptr_i = 0; + p->key_buf_used = 0; + p->key_tmp_dir = xstrdup(key_tmp_dir); + p->key_file_no = 0; +#if YAZ_POSIX_THREADS + p->alt_buf = (char**) xmalloc (mem); + p->is_sorting = 0; + p->exit_flag = 0; + pthread_mutex_init(&p->mutex, 0); + pthread_cond_init(&p->work_available, 0); + pthread_cond_init(&p->cond_sorting, 0); + pthread_create(&p->thread_id, 0, thread_func, p); +#endif + return p; +} + +void key_block_destroy(zebra_key_block_t *pp) +{ + zebra_key_block_t p = *pp; + if (p) + { +#if YAZ_POSIX_THREADS + pthread_mutex_lock(&p->mutex); + + while (p->is_sorting) + pthread_cond_wait(&p->cond_sorting, &p->mutex); + + p->exit_flag = 1; + + pthread_cond_broadcast(&p->work_available); + + pthread_mutex_unlock(&p->mutex); + pthread_join(p->thread_id, 0); + pthread_cond_destroy(&p->work_available); + pthread_cond_destroy(&p->cond_sorting); + pthread_mutex_destroy(&p->mutex); + + xfree(p->alt_buf); +#endif + xfree(p->key_buf); + xfree(p->key_tmp_dir); + xfree(p); + *pp = 0; + } +} + +void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, + int cmd, const char *str_buf, size_t str_len, + zint staticrank, int static_rank_enable) +{ + int ch; + int i, j = 0; + struct it_key key_out; + + if (p->key_buf_used + 1024 > (p->ptr_top -p->ptr_i)*sizeof(char*)) + key_block_flush(p, 0); + ++(p->ptr_i); + assert(p->ptr_i > 0); + (p->key_buf)[p->ptr_top - p->ptr_i] = + (char*)p->key_buf + p->key_buf_used; + + /* key_in->mem[0] ord/ch */ + /* key_in->mem[1] filter specified record ID */ + + /* encode the ordinal value (field/use/attribute) .. */ + ch = CAST_ZINT_TO_INT(key_in->mem[0]); + p->key_buf_used += + key_SU_encode(ch, (char*)p->key_buf + + p->key_buf_used); + + /* copy the 0-terminated stuff from str to output */ + memcpy((char*)p->key_buf + p->key_buf_used, str_buf, str_len); + p->key_buf_used += str_len; + ((char*)p->key_buf)[(p->key_buf_used)++] = '\0'; + + /* the delete/insert indicator */ + ((char*)p->key_buf)[(p->key_buf_used)++] = cmd; + + if (static_rank_enable) + key_out.mem[j++] = staticrank; + + if (key_in->mem[1]) /* filter specified record ID */ + key_out.mem[j++] = key_in->mem[1]; + else + key_out.mem[j++] = sysno; + for (i = 2; i < key_in->len; i++) + key_out.mem[j++] = key_in->mem[i]; + key_out.len = j; + + memcpy((char*)p->key_buf + p->key_buf_used, + &key_out, sizeof(key_out)); + (p->key_buf_used) += sizeof(key_out); +} + + +void key_block_flush_int(zebra_key_block_t p, + char **key_buf, size_t ptr_top, size_t ptr_i) +{ + FILE *outf; + char out_fname[200]; + char *prevcp, *cp; + struct encode_info encode_info; + + (p->key_file_no)++; + yaz_log(YLOG_LOG, "sorting section %d", (p->key_file_no)); + yaz_log(log_level, " sort_buff at %p n=%d", + key_buf + ptr_top - ptr_i,ptr_i); + + +#if USE_SHELLSORT + shellsort(key_buf + ptr_top - ptr_i, ptr_i, + sizeof(char*), key_qsort_compare); +#else + qsort(key_buf + ptr_top - ptr_i, ptr_i, + sizeof(char*), key_qsort_compare); +#endif + sprintf(out_fname, "%s/key%d.tmp", p->key_tmp_dir, p->key_file_no); + + if (!(outf = fopen (out_fname, "wb"))) + { + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname); + zebra_exit("key_block_flush"); + } + yaz_log(YLOG_LOG, "writing section %d", p->key_file_no); + prevcp = cp = (key_buf)[ptr_top - ptr_i]; + + encode_key_init (&encode_info); + encode_key_write (cp, &encode_info, outf); + + while (--ptr_i > 0) + { + cp = (key_buf)[ptr_top - ptr_i]; + if (strcmp (cp, prevcp)) + { + encode_key_flush ( &encode_info, outf); + encode_key_init (&encode_info); + encode_key_write (cp, &encode_info, outf); + prevcp = cp; + } + else + encode_key_write (cp + strlen(cp), &encode_info, outf); + } + encode_key_flush ( &encode_info, outf); + if (fclose (outf)) + { + yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname); + zebra_exit("key_block_flush"); + } + yaz_log(YLOG_LOG, "finished section %d", p->key_file_no); +} + +void key_block_flush(zebra_key_block_t p, int is_final) + /* optimizing: if final=1, and no files written yet */ + /* push the keys directly to merge, sidestepping the */ + /* temp file altogether. Speeds small updates */ +{ +#if YAZ_POSIX_THREADS + char **tmp; +#endif + if (!p) + return; + +#if YAZ_POSIX_THREADS + pthread_mutex_lock(&p->mutex); + + while (p->is_sorting) + pthread_cond_wait(&p->cond_sorting, &p->mutex); + + p->is_sorting = 1; + + p->thread_ptr_top = p->ptr_top; + p->thread_ptr_i = p->ptr_i; + p->thread_key_buf = p->key_buf; + + tmp = p->key_buf; + p->key_buf = p->alt_buf; + p->alt_buf = tmp; + + yaz_log(YLOG_LOG, "key_block_flush 1 p=%p", p); + pthread_cond_signal(&p->work_available); + + if (is_final) + { + while (p->is_sorting) + pthread_cond_wait(&p->cond_sorting, &p->mutex); + } + pthread_mutex_unlock(&p->mutex); +#else + key_block_flush_int(p, p->key_buf, p->ptr_top, p->ptr_i); +#endif + p->ptr_i = 0; + p->key_buf_used = 0; +} + +int key_block_get_no_files(zebra_key_block_t p) +{ + if (p) + return p->key_file_no; + return 0; +} + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/key_block.h b/index/key_block.h new file mode 100644 index 0000000..8ed9e42 --- /dev/null +++ b/index/key_block.h @@ -0,0 +1,48 @@ +/* $Id: key_block.h,v 1.1 2006-11-21 14:32:38 adam Exp $ + Copyright (C) 1995-2006 + Index Data ApS + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef ZEBRA_KEY_BLOCK_H +#define ZEBRA_KEY_BLOCK_H + +YAZ_BEGIN_CDECL + +typedef struct zebra_key_block *zebra_key_block_t; + +zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir); +void key_block_destroy(zebra_key_block_t *pp); +void key_block_flush(zebra_key_block_t p, int is_final); +void key_block_write(zebra_key_block_t p, SYSNO sysno, struct it_key *key_in, + int cmd, const char *str_buf, size_t str_len, + zint staticrank, int static_rank_enable); +int key_block_get_no_files(zebra_key_block_t p); + +YAZ_END_CDECL + +#endif +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/kinput.c b/index/kinput.c index 5469b50..8f0a52b 100644 --- a/index/kinput.c +++ b/index/kinput.c @@ -1,4 +1,4 @@ -/* $Id: kinput.c,v 1.77 2006-10-29 17:20:01 adam Exp $ +/* $Id: kinput.c,v 1.78 2006-11-21 14:32:38 adam Exp $ Copyright (C) 1995-2006 Index Data ApS @@ -90,21 +90,21 @@ void key_file_chunk_read (struct key_file *f) f->buf_size = 0; if (fd == -1) { - yaz_log (YLOG_WARN|YLOG_ERRNO, "cannot open %s", fname); + yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot open %s", fname); return ; } if (!f->length) { if ((f->length = lseek (fd, 0L, SEEK_END)) == (off_t) -1) { - yaz_log (YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname); + yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname); close (fd); return ; } } if (lseek (fd, f->offset, SEEK_SET) == -1) { - yaz_log (YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname); + yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname); close(fd); return ; } @@ -117,7 +117,7 @@ void key_file_chunk_read (struct key_file *f) } if (r == -1) { - yaz_log (YLOG_WARN|YLOG_ERRNO, "read of %s", fname); + yaz_log(YLOG_WARN|YLOG_ERRNO, "read of %s", fname); close (fd); return; } @@ -168,32 +168,6 @@ int key_file_getc (struct key_file *f) return EOF; } -int key_file_decode (struct key_file *f) -{ - int c, d; - - c = key_file_getc (f); - switch (c & 192) - { - case 0: - d = c; - break; - case 64: - d = ((c&63) << 8) + (key_file_getc (f) & 0xff); - break; - case 128: - d = ((c&63) << 8) + (key_file_getc (f) & 0xff); - d = (d << 8) + (key_file_getc (f) & 0xff); - break; - default: /* 192 */ - d = ((c&63) << 8) + (key_file_getc (f) & 0xff); - d = (d << 8) + (key_file_getc (f) & 0xff); - d = (d << 8) + (key_file_getc (f) & 0xff); - break; - } - return d; -} - int key_file_read (struct key_file *f, char *key) { int i, c; @@ -375,30 +349,12 @@ static void key_heap_insert (struct heap_info *hi, const char *buf, int nbytes, } } -static int heap_read_one_raw(struct heap_info *hi, char *name, char *key) -{ - ZebraHandle zh = hi->zh; - size_t ptr_i = zh->reg->ptr_i; - char *cp; - if (!ptr_i) - return 0; - --(zh->reg->ptr_i); - cp=(zh->reg->key_buf)[zh->reg->ptr_top - ptr_i]; - strcpy(name, cp); - memcpy(key, cp+strlen(name)+1, KEY_SIZE); - hi->no_iterations++; - return 1; -} - static int heap_read_one (struct heap_info *hi, char *name, char *key) { int n, r; char rbuf[INP_NAME_MAX]; struct key_file *kf; - if (hi->raw_reading) - return heap_read_one_raw(hi, name, key); - if (!hi->heapnum) return 0; n = hi->ptr[1]; @@ -736,7 +692,7 @@ int heap_inps (struct heap_cread_info *hci, struct heap_info *hi) } else { - yaz_log (YLOG_FATAL, "isams doesn't support this kind of update"); + yaz_log(YLOG_FATAL, "isams doesn't support this kind of update"); break; } } @@ -766,10 +722,10 @@ void progressFunc (struct key_file *keyp, void *info) remaining = (time_t) ((now - p->startTime)* ((double) p->totalBytes/p->totalOffset - 1.0)); if (remaining <= 130) - yaz_log (YLOG_LOG, "Merge %2.1f%% completed; %ld seconds remaining", + yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld seconds remaining", (100.0*p->totalOffset) / p->totalBytes, (long) remaining); else - yaz_log (YLOG_LOG, "Merge %2.1f%% completed; %ld minutes remaining", + yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld minutes remaining", (100.0*p->totalOffset) / p->totalBytes, (long) remaining/60); } p->totalOffset += keyp->buf_size; @@ -786,62 +742,50 @@ void zebra_index_merge (ZebraHandle zh) int i, r; struct heap_info *hi; struct progressInfo progressInfo; - int nkeys = zh->reg->key_file_no; - int usefile; - yaz_log (YLOG_DEBUG, " index_merge called with nk=%d b=%p", - nkeys, zh->reg->key_buf); - if ( (nkeys==0) && (zh->reg->key_buf==0) ) - return; /* nothing to merge - probably flush after end-trans */ - - usefile = (nkeys!=0); + int nkeys = key_block_get_no_files(zh->reg->key_block); - if (usefile) + if (nkeys == 0) + return; + + if (nkeys < 0) { - if (nkeys < 0) - { - char fname[1024]; - nkeys = 0; - while (1) - { - extract_get_fname_tmp (zh, fname, nkeys+1); - if (access (fname, R_OK) == -1) - break; - nkeys++; - } - if (!nkeys) - return ; - } - kf = (struct key_file **) xmalloc ((1+nkeys) * sizeof(*kf)); - progressInfo.totalBytes = 0; - progressInfo.totalOffset = 0; - time (&progressInfo.startTime); - time (&progressInfo.lastTime); - for (i = 1; i<=nkeys; i++) + char fname[1024]; + nkeys = 0; + while (1) { - kf[i] = key_file_init (i, 8192, zh->res); - kf[i]->readHandler = progressFunc; - kf[i]->readInfo = &progressInfo; - progressInfo.totalBytes += kf[i]->length; - progressInfo.totalOffset += kf[i]->buf_size; + extract_get_fname_tmp (zh, fname, nkeys+1); + if (access (fname, R_OK) == -1) + break; + nkeys++; } - hi = key_heap_init_file(zh, nkeys, key_qsort_compare); - hi->reg = zh->reg; - - for (i = 1; i<=nkeys; i++) - if ((r = key_file_read (kf[i], rbuf))) - key_heap_insert (hi, rbuf, r, kf[i]); - } /* use file */ - else - { /* do not use file, read straight from buffer */ - hi = key_heap_init_raw(zh, key_qsort_compare); - hi->reg = zh->reg; + if (!nkeys) + return ; } + kf = (struct key_file **) xmalloc ((1+nkeys) * sizeof(*kf)); + progressInfo.totalBytes = 0; + progressInfo.totalOffset = 0; + time (&progressInfo.startTime); + time (&progressInfo.lastTime); + for (i = 1; i<=nkeys; i++) + { + kf[i] = key_file_init (i, 8192, zh->res); + kf[i]->readHandler = progressFunc; + kf[i]->readInfo = &progressInfo; + progressInfo.totalBytes += kf[i]->length; + progressInfo.totalOffset += kf[i]->buf_size; + } + hi = key_heap_init_file(zh, nkeys, key_qsort_compare); + hi->reg = zh->reg; + + for (i = 1; i<=nkeys; i++) + if ((r = key_file_read (kf[i], rbuf))) + key_heap_insert (hi, rbuf, r, kf[i]); if (1) { - struct heap_cread_info hci; - - hci.key = (char *) xmalloc (KEY_SIZE); + struct heap_cread_info hci; + + hci.key = (char *) xmalloc (KEY_SIZE); hci.key_1 = (char *) xmalloc (KEY_SIZE); hci.key_2 = (char *) xmalloc (KEY_SIZE); hci.ret = -1; @@ -862,28 +806,24 @@ void zebra_index_merge (ZebraHandle zh) xfree (hci.key_2); } - if (usefile) + for (i = 1; i<=nkeys; i++) { - for (i = 1; i<=nkeys; i++) - { - extract_get_fname_tmp (zh, rbuf, i); - unlink (rbuf); - } - for (i = 1; i<=nkeys; i++) - key_file_destroy (kf[i]); - xfree (kf); + extract_get_fname_tmp (zh, rbuf, i); + unlink (rbuf); } + for (i = 1; i<=nkeys; i++) + key_file_destroy (kf[i]); + xfree (kf); if (hi->no_iterations) { /* do not log if nothing happened */ - yaz_log (YLOG_LOG, "Iterations . . .%7d", hi->no_iterations); - yaz_log (YLOG_LOG, "Distinct words .%7d", hi->no_diffs); - yaz_log (YLOG_LOG, "Updates. . . . .%7d", hi->no_updates); - yaz_log (YLOG_LOG, "Deletions. . . .%7d", hi->no_deletions); - yaz_log (YLOG_LOG, "Insertions . . .%7d", hi->no_insertions); + yaz_log(YLOG_LOG, "Iterations . . .%7d", hi->no_iterations); + yaz_log(YLOG_LOG, "Distinct words .%7d", hi->no_diffs); + yaz_log(YLOG_LOG, "Updates. . . . .%7d", hi->no_updates); + yaz_log(YLOG_LOG, "Deletions. . . .%7d", hi->no_deletions); + yaz_log(YLOG_LOG, "Insertions . . .%7d", hi->no_insertions); } - zh->reg->key_file_no = 0; - - key_heap_destroy (hi, nkeys); + key_block_destroy(&zh->reg->key_block); + key_heap_destroy(hi, nkeys); } /* * Local variables: diff --git a/index/reckeys.h b/index/reckeys.h index dd8e649..ed906da 100644 --- a/index/reckeys.h +++ b/index/reckeys.h @@ -1,4 +1,4 @@ -/* $Id: reckeys.h,v 1.6 2006-10-29 17:20:01 adam Exp $ +/* $Id: reckeys.h,v 1.7 2006-11-21 14:32:38 adam Exp $ Copyright (C) 1995-2006 Index Data ApS @@ -23,6 +23,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #ifndef RECKEYS_H #define RECKEYS_H +YAZ_BEGIN_CDECL + typedef struct zebra_rec_keys_t_ *zebra_rec_keys_t; zebra_rec_keys_t zebra_rec_keys_open(void); @@ -46,6 +48,8 @@ void zebra_rec_keys_get_buf(zebra_rec_keys_t p, char **buf, size_t *sz); void zebra_rec_keys_set_buf(zebra_rec_keys_t p, char *buf, size_t sz, int copy_buf); +YAZ_END_CDECL + #endif /* * Local variables: diff --git a/index/zebraapi.c b/index/zebraapi.c index 7788602..12d28ac 100644 --- a/index/zebraapi.c +++ b/index/zebraapi.c @@ -1,4 +1,4 @@ -/* $Id: zebraapi.c,v 1.232 2006-11-17 13:47:22 marc Exp $ +/* $Id: zebraapi.c,v 1.233 2006-11-21 14:32:38 adam Exp $ Copyright (C) 1995-2006 Index Data ApS @@ -84,9 +84,10 @@ static ZEBRA_RES zebra_flush_reg (ZebraHandle zh) ZEBRA_CHECK_HANDLE(zh); yaz_log(log_level, "zebra_flush_reg"); zebraExplain_flush (zh->reg->zei, zh); - - extract_flushWriteKeys (zh, 1 /* final */); - zebra_index_merge (zh ); + + key_block_flush(zh->reg->key_block, 1); + + zebra_index_merge(zh); return ZEBRA_OK; } @@ -367,8 +368,7 @@ struct zebra_register *zebra_register_open(ZebraService zs, const char *name, } reg->rank_classes = NULL; - reg->key_buf = 0; - + reg->key_block = 0; reg->keys = zebra_rec_keys_open(); reg->sortKeys = zebra_rec_keys_open(); @@ -381,8 +381,6 @@ struct zebra_register *zebra_register_open(ZebraService zs, const char *name, reg->isamc = 0; reg->isamb = 0; reg->zei = 0; - reg->key_file_no = 0; - reg->ptr_i = 0; /* installing rank classes */ zebraRankInstall (reg, rank_1_class); @@ -550,7 +548,7 @@ static void zebra_register_close(ZebraService zs, struct zebra_register *reg) zebra_rec_keys_close(reg->keys); zebra_rec_keys_close(reg->sortKeys); - xfree(reg->key_buf); + key_block_destroy(®->key_block); xfree(reg->name); xfree(reg); } diff --git a/util/Makefile.am b/util/Makefile.am index 11c3b96..a336959 100644 --- a/util/Makefile.am +++ b/util/Makefile.am @@ -1,4 +1,4 @@ -## $Id: Makefile.am,v 1.28 2006-11-14 08:12:10 adam Exp $ +## $Id: Makefile.am,v 1.29 2006-11-21 14:32:38 adam Exp $ noinst_LTLIBRARIES = libidzebra-util.la @@ -17,7 +17,7 @@ LDADD = libidzebra-util.la $(YAZLALIB) libidzebra_util_la_SOURCES = zint.c res.c charmap.c zebramap.c passwddb.c \ zebra-lock.c dirent.c xpath.c atoi_zn.c snippet.c flock.c attrfind.c \ - exit.c + exit.c it_key.c su_codec.c tstpass_SOURCES = tstpass.c diff --git a/util/it_key.c b/util/it_key.c new file mode 100644 index 0000000..7e4417c --- /dev/null +++ b/util/it_key.c @@ -0,0 +1,281 @@ +/* $Id: it_key.c,v 1.1 2006-11-21 14:32:38 adam Exp $ + Copyright (C) 1995-2006 + Index Data ApS + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include +#include +#include +#include + +#include +#include + +#ifdef __GNUC__ +#define CODEC_INLINE inline +#else +#define CODEC_INLINE +#endif + +void key_logdump_txt(int logmask, const void *p, const char *txt) +{ + struct it_key key; + if (!txt) + txt = "(none)"; + if (p) + { + char formstr[128]; + int i; + + memcpy (&key, p, sizeof(key)); + assert(key.len > 0 && key.len <= IT_KEY_LEVEL_MAX); + *formstr = '\0'; + for (i = 0; ilen; + if (((struct it_key *) p2)->len > l) + l = ((struct it_key *) p2)->len; + assert (l <= IT_KEY_LEVEL_MAX && l > 0); + for (i = 0; i < l; i++) + { + if (((struct it_key *) p1)->mem[i] != ((struct it_key *) p2)->mem[i]) + { + if (((struct it_key *) p1)->mem[i] > ((struct it_key *) p2)->mem[i]) + return l-i; + else + return i-l; + } + } + return 0; +} + +char *key_print_it (const void *p, char *buf) +{ + strcpy(buf, ""); + return buf; +} + +int key_compare (const void *p1, const void *p2) +{ + struct it_key i1, i2; + int i, l; + memcpy (&i1, p1, sizeof(i1)); + memcpy (&i2, p2, sizeof(i2)); + l = i1.len; + if (i2.len > l) + l = i2.len; + assert (l <= IT_KEY_LEVEL_MAX && l > 0); + for (i = 0; i < l; i++) + { + if (i1.mem[i] != i2.mem[i]) + { + if (i1.mem[i] > i2.mem[i]) + return l-i; + else + return i-l; + } + } + return 0; +} + +zint key_get_seq(const void *p) +{ + struct it_key k; + memcpy (&k, p, sizeof(k)); + return k.mem[k.len-1]; +} + +zint key_get_segment(const void *p) +{ + struct it_key k; + memcpy (&k, p, sizeof(k)); + return k.mem[k.len-2]; +} + +int key_qsort_compare (const void *p1, const void *p2) +{ + int r; + size_t l; + char *cp1 = *(char **) p1; + char *cp2 = *(char **) p2; + + if ((r = strcmp (cp1, cp2))) + return r; + l = strlen(cp1)+1; + if ((r = key_compare (cp1+l+1, cp2+l+1))) + return r; + return cp1[l] - cp2[l]; +} + +struct iscz1_code_info { + struct it_key key; +}; + +void *iscz1_start (void) +{ + struct iscz1_code_info *p = (struct iscz1_code_info *) + xmalloc (sizeof(*p)); + iscz1_reset(p); + return p; +} + +void key_init(struct it_key *key) +{ + int i; + key->len = 0; + for (i = 0; i < IT_KEY_LEVEL_MAX; i++) + key->mem[i] = 0; +} + +void iscz1_reset (void *vp) +{ + struct iscz1_code_info *p = (struct iscz1_code_info *) vp; + int i; + p->key.len = 0; + for (i = 0; i < IT_KEY_LEVEL_MAX; i++) + p->key.mem[i] = 0; +} + +void iscz1_stop (void *p) +{ + xfree (p); +} + +/* small encoder that works with unsigneds of any length */ +static CODEC_INLINE void iscz1_encode_int (zint d, char **dst) +{ + unsigned char *bp = (unsigned char*) *dst; + + while (d > 127) + { + *bp++ = (unsigned) (128 | (d & 127)); + d = d >> 7; + } + *bp++ = (unsigned) d; + *dst = (char *) bp; +} + +/* small decoder that works with unsigneds of any length */ +static CODEC_INLINE zint iscz1_decode_int (unsigned char **src) +{ + zint d = 0; + unsigned char c; + unsigned r = 0; + + while (((c = *(*src)++) & 128)) + { + d += ((zint) (c&127) << r); + r += 7; + } + d += ((zint) c << r); + return d; +} + +void iscz1_encode (void *vp, char **dst, const char **src) +{ + struct iscz1_code_info *p = (struct iscz1_code_info *) vp; + struct it_key tkey; + zint d; + int i; + + /* 1 + 3, 2, 9, 12 + 3, 2, 10, 2 + 4, 1 + + if diff is 0, then there is more ... + if diff is non-zero, then _may_ be more + */ + memcpy (&tkey, *src, sizeof(struct it_key)); + + /* deal with leader + delta encoding .. */ + d = 0; + assert(tkey.len > 0 && tkey.len <= IT_KEY_LEVEL_MAX); + for (i = 0; i < tkey.len; i++) + { + d = tkey.mem[i] - p->key.mem[i]; + if (d || i == tkey.len-1) + { /* all have been equal until now, now make delta .. */ + p->key.mem[i] = tkey.mem[i]; + if (d > 0) + { + iscz1_encode_int (i + (tkey.len << 3) + 64, dst); + i++; + iscz1_encode_int (d, dst); + } + else + { + iscz1_encode_int (i + (tkey.len << 3), dst); + } + break; + } + } + /* rest uses absolute encoding ... */ + for (; i < tkey.len; i++) + { + iscz1_encode_int (tkey.mem[i], dst); + p->key.mem[i] = tkey.mem[i]; + } + (*src) += sizeof(struct it_key); +} + +void iscz1_decode (void *vp, char **dst, const char **src) +{ + struct iscz1_code_info *p = (struct iscz1_code_info *) vp; + int i; + + int leader = (int) iscz1_decode_int ((unsigned char **) src); + i = leader & 7; + if (leader & 64) + p->key.mem[i] += iscz1_decode_int ((unsigned char **) src); + else + p->key.mem[i] = iscz1_decode_int ((unsigned char **) src); + p->key.len = (leader >> 3) & 7; + while (++i < p->key.len) + p->key.mem[i] = iscz1_decode_int ((unsigned char **) src); + memcpy (*dst, &p->key, sizeof(struct it_key)); + (*dst) += sizeof(struct it_key); +} + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/util/su_codec.c b/util/su_codec.c new file mode 100644 index 0000000..745d938 --- /dev/null +++ b/util/su_codec.c @@ -0,0 +1,77 @@ +/* $Id: su_codec.c,v 1.1 2006-11-21 14:32:38 adam Exp $ + Copyright (C) 1995-2006 + Index Data ApS + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include +#include +#include +#include + +#include +#include + +int key_SU_encode (int ch, char *out) +{ + int i; + for (i = 0; ch; i++) + { + if (ch >= 64) + out[i] = 65 + (ch & 63); + else + out[i] = 1 + ch; + ch = ch >> 6; + } + return i; + /* in out + 0 1 + 1 2 + 63 64 + 64 65, 2 + 65 66, 2 + 127 128, 2 + 128 65, 3 + 191 128, 3 + 192 65, 4 + */ +} + +int key_SU_decode (int *ch, const unsigned char *out) +{ + int len = 1; + int fact = 1; + *ch = 0; + for (len = 1; *out >= 65; len++, out++) + { + *ch += (*out - 65) * fact; + fact <<= 6; + } + *ch += (*out - 1) * fact; + return len; +} + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + -- 1.7.10.4