1 /* This file is part of the Zebra server.
2 Copyright (C) 1994-2009 Index Data
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 * Format of first block
26 * Format of subsequent blocks
30 * Format of each record
32 * (length, data) - pairs
33 * length = 0 if same as previous
40 #include <yaz/yaz-util.h>
41 #include <idzebra/bfile.h>
48 #define REC_BLOCK_TYPES 2
49 #define REC_HEAD_MAGIC "recindex"
54 int compression_method;
58 char *data_fname[REC_BLOCK_TYPES];
59 BFile data_BFile[REC_BLOCK_TYPES];
64 struct record_cache_entry *record_cache;
69 int compression_chunk_size;
76 zint block_size[REC_BLOCK_TYPES];
77 zint block_free[REC_BLOCK_TYPES];
78 zint block_last[REC_BLOCK_TYPES];
79 zint block_used[REC_BLOCK_TYPES];
80 zint block_move[REC_BLOCK_TYPES];
90 enum recordCacheFlag { recordFlagNop, recordFlagWrite, recordFlagNew,
93 struct record_cache_entry {
95 enum recordCacheFlag flag;
98 struct record_index_entry {
99 zint next; /* first block of record info / next free entry */
100 int size; /* size of record or 0 if free entry */
103 Record rec_cp(Record rec);
105 /* Modify argument to if below: 1=normal, 0=sysno testing */
107 /* If this is used sysno are not converted (no testing) */
108 #define FAKE_OFFSET 0
109 #define USUAL_RANGE 6000000000LL
112 /* Use a fake > 2^32 offset so we can test for proper 64-bit handling */
113 #define FAKE_OFFSET 6000000000LL
114 #define USUAL_RANGE 2000000000LL
117 static zint rec_sysno_to_ext(zint sysno)
119 assert(sysno >= 0 && sysno <= USUAL_RANGE);
120 return sysno + FAKE_OFFSET;
123 zint rec_sysno_to_int(zint sysno)
125 assert(sysno >= FAKE_OFFSET && sysno <= FAKE_OFFSET + USUAL_RANGE);
126 return sysno - FAKE_OFFSET;
129 static void rec_tmp_expand(Records p, int size)
131 if (p->tmp_size < size + 2048 ||
132 p->tmp_size < p->head.block_size[REC_BLOCK_TYPES-1]*2)
135 p->tmp_size = size + (int)
136 (p->head.block_size[REC_BLOCK_TYPES-1])*2 + 2048;
137 p->tmp_buf = (char *) xmalloc(p->tmp_size);
141 static ZEBRA_RES rec_release_blocks(Records p, zint sysno)
143 struct record_index_entry entry;
145 char block_and_ref[sizeof(zint) + sizeof(short)];
149 if (recindex_read_indx(p->recindex, sysno, &entry, sizeof(entry), 1) != 1)
152 freeblock = entry.next;
153 assert(freeblock > 0);
154 dst_type = CAST_ZINT_TO_INT(freeblock & 7);
155 assert(dst_type < REC_BLOCK_TYPES);
156 freeblock = freeblock / 8;
159 if (bf_read(p->data_BFile[dst_type], freeblock, 0,
160 first ? sizeof(block_and_ref) : sizeof(zint),
163 yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in rec_del_single");
169 memcpy(&ref, block_and_ref + sizeof(freeblock), sizeof(ref));
171 memcpy(block_and_ref + sizeof(freeblock), &ref, sizeof(ref));
174 /* there is still a reference to this block.. */
175 if (bf_write(p->data_BFile[dst_type], freeblock, 0,
176 sizeof(block_and_ref), block_and_ref))
178 yaz_log(YLOG_FATAL|YLOG_ERRNO, "write in rec_del_single");
183 /* the list of blocks can all be removed (ref == 0) */
187 if (bf_write(p->data_BFile[dst_type], freeblock, 0, sizeof(freeblock),
188 &p->head.block_free[dst_type]))
190 yaz_log(YLOG_FATAL|YLOG_ERRNO, "write in rec_del_single");
193 p->head.block_free[dst_type] = freeblock;
194 memcpy(&freeblock, block_and_ref, sizeof(freeblock));
196 p->head.block_used[dst_type]--;
198 p->head.total_bytes -= entry.size;
202 static ZEBRA_RES rec_delete_single(Records p, Record rec)
204 struct record_index_entry entry;
206 /* all data in entry must be reset, since it's written verbatim */
207 memset(&entry, '\0', sizeof(entry));
208 if (rec_release_blocks(p, rec_sysno_to_int(rec->sysno)) != ZEBRA_OK)
211 entry.next = p->head.index_free;
213 p->head.index_free = rec_sysno_to_int(rec->sysno);
214 recindex_write_indx(p->recindex, rec_sysno_to_int(rec->sysno), &entry, sizeof(entry));
218 static ZEBRA_RES rec_write_tmp_buf(Records p, int size, zint *sysnos)
220 struct record_index_entry entry;
222 char *cptr = p->tmp_buf;
223 zint block_prev = -1, block_free;
227 /* all data in entry must be reset, since it's written verbatim */
228 memset(&entry, '\0', sizeof(entry));
230 for (i = 1; i<REC_BLOCK_TYPES; i++)
231 if (size >= p->head.block_move[i])
233 while (no_written < size)
235 block_free = p->head.block_free[dst_type];
238 if (bf_read(p->data_BFile[dst_type],
239 block_free, 0, sizeof(*p->head.block_free),
240 &p->head.block_free[dst_type]) != 1)
242 yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in %s at free block "
244 p->data_fname[dst_type], block_free);
249 block_free = p->head.block_last[dst_type]++;
250 if (block_prev == -1)
252 entry.next = block_free*8 + dst_type;
254 p->head.total_bytes += size;
257 recindex_write_indx(p->recindex, *sysnos, &entry, sizeof(entry));
263 memcpy(cptr, &block_free, sizeof(block_free));
264 bf_write(p->data_BFile[dst_type], block_prev, 0, 0, cptr);
265 cptr = p->tmp_buf + no_written;
267 block_prev = block_free;
268 no_written += CAST_ZINT_TO_INT(p->head.block_size[dst_type])
270 p->head.block_used[dst_type]++;
272 assert(block_prev != -1);
274 memcpy(cptr, &block_free, sizeof(block_free));
275 bf_write(p->data_BFile[dst_type], block_prev, 0,
276 sizeof(block_free) + (p->tmp_buf+size) - cptr, cptr);
280 Records rec_open(BFiles bfs, int rw, int compression_method)
285 ZEBRA_RES ret = ZEBRA_OK;
287 p = (Records) xmalloc(sizeof(*p));
288 memset(&p->head, '\0', sizeof(p->head));
289 p->compression_method = compression_method;
292 p->tmp_buf = (char *) xmalloc(p->tmp_size);
293 p->compression_chunk_size = 0;
294 if (compression_method == REC_COMPRESS_BZIP2)
295 p->compression_chunk_size = 90000;
296 p->recindex = recindex_open(bfs, rw, 0 /* 1=isamb for recindex */);
297 r = recindex_read_head(p->recindex, p->tmp_buf);
301 memcpy(p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic));
302 sprintf(p->head.version, "%3d", REC_VERSION);
303 p->head.index_free = 0;
304 p->head.index_last = 1;
305 p->head.no_records = 0;
306 p->head.total_bytes = 0;
307 for (i = 0; i<REC_BLOCK_TYPES; i++)
309 p->head.block_free[i] = 0;
310 p->head.block_last[i] = 1;
311 p->head.block_used[i] = 0;
313 p->head.block_size[0] = 128;
314 p->head.block_move[0] = 0;
315 for (i = 1; i<REC_BLOCK_TYPES; i++)
317 p->head.block_size[i] = p->head.block_size[i-1] * 4;
318 p->head.block_move[i] = p->head.block_size[i] * 12;
322 if (recindex_write_head(p->recindex,
323 &p->head, sizeof(p->head)) != ZEBRA_OK)
328 memcpy(&p->head, p->tmp_buf, sizeof(p->head));
329 if (memcmp(p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)))
331 yaz_log(YLOG_FATAL, "file %s has bad format",
332 recindex_get_fname(p->recindex));
335 version = atoi(p->head.version);
336 if (version != REC_VERSION)
338 yaz_log(YLOG_FATAL, "file %s is version %d, but version"
340 recindex_get_fname(p->recindex), version, REC_VERSION);
345 for (i = 0; i<REC_BLOCK_TYPES; i++)
348 sprintf(str, "recd%c", i + 'A');
349 p->data_fname[i] = (char *) xmalloc(strlen(str)+1);
350 strcpy(p->data_fname[i], str);
351 p->data_BFile[i] = NULL;
353 for (i = 0; i<REC_BLOCK_TYPES; i++)
355 if (!(p->data_BFile[i] =
356 bf_open(bfs, p->data_fname[i],
357 CAST_ZINT_TO_INT(p->head.block_size[i]), rw)))
359 yaz_log(YLOG_FATAL|YLOG_ERRNO, "bf_open %s", p->data_fname[i]);
366 p->record_cache = (struct record_cache_entry *)
367 xmalloc(sizeof(*p->record_cache)*p->cache_max);
368 zebra_mutex_init(&p->mutex);
369 if (ret == ZEBRA_FAIL)
374 static void rec_encode_unsigned(unsigned n, unsigned char *buf, int *len)
379 buf[*len] = 128 + (n & 127);
387 static void rec_decode_unsigned(unsigned *np, unsigned char *buf, int *len)
393 while (buf[*len] > 127)
395 n += w*(buf[*len] & 127);
404 static void rec_encode_zint(zint n, unsigned char *buf, int *len)
409 buf[*len] = (unsigned) (128 + (n & 127));
413 buf[*len] = (unsigned) n;
417 static void rec_decode_zint(zint *np, unsigned char *buf, int *len)
423 while (buf[*len] > 127)
425 n += w*(buf[*len] & 127);
434 static void rec_cache_flush_block1(Records p, Record rec, Record last_rec,
435 char **out_buf, int *out_size,
441 for (i = 0; i<REC_NO_INFO; i++)
443 if (*out_offset + CAST_ZINT_TO_INT(rec->size[i]) + 20 > *out_size)
445 int new_size = *out_offset + rec->size[i] + 65536;
446 char *np = (char *) xmalloc(new_size);
448 memcpy(np, *out_buf, *out_offset);
450 *out_size = new_size;
455 rec_encode_zint(rec_sysno_to_int(rec->sysno),
456 (unsigned char *) *out_buf + *out_offset, &len);
457 (*out_offset) += len;
459 if (rec->size[i] == 0)
461 rec_encode_unsigned(1, (unsigned char *) *out_buf + *out_offset,
463 (*out_offset) += len;
465 else if (last_rec && rec->size[i] == last_rec->size[i] &&
466 !memcmp(rec->info[i], last_rec->info[i], rec->size[i]))
468 rec_encode_unsigned(0, (unsigned char *) *out_buf + *out_offset,
470 (*out_offset) += len;
474 rec_encode_unsigned(rec->size[i]+1,
475 (unsigned char *) *out_buf + *out_offset,
477 (*out_offset) += len;
478 memcpy(*out_buf + *out_offset, rec->info[i], rec->size[i]);
479 (*out_offset) += rec->size[i];
484 static ZEBRA_RES rec_flush_shared(Records p, short ref_count, zint *sysnos,
485 char *out_buf, int out_offset)
487 ZEBRA_RES ret = ZEBRA_OK;
491 unsigned int csize = 0; /* indicate compression "not performed yet" */
492 char compression_method = p->compression_method;
493 switch (compression_method)
495 case REC_COMPRESS_BZIP2:
497 csize = out_offset + (out_offset >> 6) + 620;
498 rec_tmp_expand(p, csize);
499 #ifdef BZ_CONFIG_ERROR
500 i = BZ2_bzBuffToBuffCompress
502 i = bzBuffToBuffCompress
504 (p->tmp_buf+sizeof(zint)+sizeof(short)+
506 &csize, out_buf, out_offset, 1, 0, 30);
509 yaz_log(YLOG_WARN, "bzBuffToBuffCompress error code=%d", i);
512 yaz_log(YLOG_LOG, "compress %4d %5d %5d", ref_count, out_offset,
516 case REC_COMPRESS_NONE:
521 /* either no compression or compression not supported ... */
523 rec_tmp_expand(p, csize);
524 memcpy(p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char),
525 out_buf, out_offset);
527 compression_method = REC_COMPRESS_NONE;
529 memcpy(p->tmp_buf + sizeof(zint), &ref_count, sizeof(ref_count));
530 memcpy(p->tmp_buf + sizeof(zint)+sizeof(short),
531 &compression_method, sizeof(compression_method));
533 /* -------- compression */
534 if (rec_write_tmp_buf(p, csize + sizeof(short) + sizeof(char), sysnos)
541 static ZEBRA_RES rec_write_multiple(Records p, int saveCount)
548 char *out_buf = (char *) xmalloc(out_size);
549 zint *sysnos = (zint *) xmalloc(sizeof(*sysnos) * (p->cache_cur + 1));
550 zint *sysnop = sysnos;
551 ZEBRA_RES ret = ZEBRA_OK;
553 for (i = 0; i<p->cache_cur - saveCount; i++)
555 struct record_cache_entry *e = p->record_cache + i;
559 rec_cache_flush_block1(p, e->rec, last_rec, &out_buf,
560 &out_size, &out_offset);
561 *sysnop++ = rec_sysno_to_int(e->rec->sysno);
563 e->flag = recordFlagNop;
566 case recordFlagWrite:
567 if (rec_release_blocks(p, rec_sysno_to_int(e->rec->sysno))
571 rec_cache_flush_block1(p, e->rec, last_rec, &out_buf,
572 &out_size, &out_offset);
573 *sysnop++ = rec_sysno_to_int(e->rec->sysno);
575 e->flag = recordFlagNop;
578 case recordFlagDelete:
579 if (rec_delete_single(p, e->rec) != ZEBRA_OK)
582 e->flag = recordFlagNop;
592 rec_flush_shared(p, ref_count, sysnos, out_buf, out_offset);
598 static ZEBRA_RES rec_cache_flush(Records p, int saveCount)
603 if (saveCount >= p->cache_cur)
606 ret = rec_write_multiple(p, saveCount);
608 for (i = 0; i<p->cache_cur - saveCount; i++)
610 struct record_cache_entry *e = p->record_cache + i;
613 /* i still being used ... */
614 for (j = 0; j<saveCount; j++, i++)
615 memcpy(p->record_cache+j, p->record_cache+i,
616 sizeof(*p->record_cache));
617 p->cache_cur = saveCount;
621 static Record *rec_cache_lookup(Records p, zint sysno,
622 enum recordCacheFlag flag)
625 for (i = 0; i<p->cache_cur; i++)
627 struct record_cache_entry *e = p->record_cache + i;
628 if (e->rec->sysno == sysno)
630 if (flag != recordFlagNop && e->flag == recordFlagNop)
638 static ZEBRA_RES rec_cache_insert(Records p, Record rec, enum recordCacheFlag flag)
640 struct record_cache_entry *e;
641 ZEBRA_RES ret = ZEBRA_OK;
643 if (p->cache_cur == p->cache_max)
644 ret = rec_cache_flush(p, 1);
645 else if (p->cache_cur > 0)
649 for (i = 0; i<p->cache_cur; i++)
651 Record r = (p->record_cache + i)->rec;
652 for (j = 0; j<REC_NO_INFO; j++)
655 if (used > p->compression_chunk_size)
656 ret = rec_cache_flush(p, 1);
658 assert(p->cache_cur < p->cache_max);
660 e = p->record_cache + (p->cache_cur)++;
662 e->rec = rec_cp(rec);
666 ZEBRA_RES rec_close(Records *pp)
670 ZEBRA_RES ret = ZEBRA_OK;
675 zebra_mutex_destroy(&p->mutex);
676 if (rec_cache_flush(p, 0) != ZEBRA_OK)
679 xfree(p->record_cache);
683 if (recindex_write_head(p->recindex, &p->head, sizeof(p->head)) != ZEBRA_OK)
687 recindex_close(p->recindex);
689 for (i = 0; i<REC_BLOCK_TYPES; i++)
691 if (p->data_BFile[i])
692 bf_close(p->data_BFile[i]);
693 xfree(p->data_fname[i]);
701 static Record rec_get_int(Records p, zint sysno)
705 struct record_index_entry entry;
712 unsigned int bz_size;
714 char compression_method;
719 if ((recp = rec_cache_lookup(p, sysno, recordFlagNop)))
720 return rec_cp(*recp);
722 if (recindex_read_indx(p->recindex, rec_sysno_to_int(sysno), &entry, sizeof(entry), 1) < 1)
723 return NULL; /* record is not there! */
726 return NULL; /* record is deleted */
728 dst_type = (int) (entry.next & 7);
729 assert(dst_type < REC_BLOCK_TYPES);
730 freeblock = entry.next / 8;
732 assert(freeblock > 0);
734 rec_tmp_expand(p, entry.size);
737 r = bf_read(p->data_BFile[dst_type], freeblock, 0, 0, cptr);
740 memcpy(&freeblock, cptr, sizeof(freeblock));
746 cptr += p->head.block_size[dst_type] - sizeof(freeblock);
748 memcpy(&tmp, cptr, sizeof(tmp));
749 r = bf_read(p->data_BFile[dst_type], freeblock, 0, 0, cptr);
752 memcpy(&freeblock, cptr, sizeof(freeblock));
753 memcpy(cptr, &tmp, sizeof(tmp));
756 rec = (Record) xmalloc(sizeof(*rec));
758 memcpy(&compression_method, p->tmp_buf + sizeof(zint) + sizeof(short),
759 sizeof(compression_method));
760 in_buf = p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char);
761 in_size = entry.size - sizeof(short) - sizeof(char);
762 switch (compression_method)
764 case REC_COMPRESS_BZIP2:
766 bz_size = entry.size * 20 + 100;
769 bz_buf = (char *) xmalloc(bz_size);
770 #ifdef BZ_CONFIG_ERROR
771 i = BZ2_bzBuffToBuffDecompress
773 i = bzBuffToBuffDecompress
775 (bz_buf, &bz_size, in_buf, in_size, 0, 0);
776 yaz_log(YLOG_LOG, "decompress %5d %5d", in_size, bz_size);
779 yaz_log(YLOG_LOG, "failed");
786 yaz_log(YLOG_FATAL, "cannot decompress record(s) in BZIP2 format");
790 case REC_COMPRESS_NONE:
793 for (i = 0; i<REC_NO_INFO; i++)
796 nptr = in_buf; /* skip ref count */
797 while (nptr < in_buf + in_size)
801 rec_decode_zint(&this_sysno, (unsigned char *) nptr, &len);
804 for (i = 0; i < REC_NO_INFO; i++)
806 unsigned int this_size;
807 rec_decode_unsigned(&this_size, (unsigned char *) nptr, &len);
812 rec->size[i] = this_size-1;
817 nptr += rec->size[i];
822 if (this_sysno == rec_sysno_to_int(sysno))
825 for (i = 0; i<REC_NO_INFO; i++)
827 if (rec->info[i] && rec->size[i])
829 char *np = xmalloc(rec->size[i]+1);
830 memcpy(np, rec->info[i], rec->size[i]);
831 np[rec->size[i]] = '\0';
836 assert(rec->info[i] == 0);
837 assert(rec->size[i] == 0);
841 if (rec_cache_insert(p, rec, recordFlagNop) != ZEBRA_OK)
846 Record rec_get(Records p, zint sysno)
849 zebra_mutex_lock(&p->mutex);
851 rec = rec_get_int(p, sysno);
852 zebra_mutex_unlock(&p->mutex);
856 Record rec_get_root(Records p)
858 return rec_get(p, rec_sysno_to_ext(1));
861 static Record rec_new_int(Records p)
868 rec = (Record) xmalloc(sizeof(*rec));
869 if (1 || p->head.index_free == 0)
870 sysno = (p->head.index_last)++;
873 struct record_index_entry entry;
875 if (recindex_read_indx(p->recindex, p->head.index_free, &entry, sizeof(entry), 0) < 1)
880 sysno = p->head.index_free;
881 p->head.index_free = entry.next;
883 (p->head.no_records)++;
884 rec->sysno = rec_sysno_to_ext(sysno);
885 for (i = 0; i < REC_NO_INFO; i++)
890 rec_cache_insert(p, rec, recordFlagNew);
894 Record rec_new(Records p)
897 zebra_mutex_lock(&p->mutex);
899 rec = rec_new_int(p);
900 zebra_mutex_unlock(&p->mutex);
904 ZEBRA_RES rec_del(Records p, Record *recpp)
907 ZEBRA_RES ret = ZEBRA_OK;
909 zebra_mutex_lock(&p->mutex);
910 (p->head.no_records)--;
911 if ((recp = rec_cache_lookup(p, (*recpp)->sysno, recordFlagDelete)))
918 ret = rec_cache_insert(p, *recpp, recordFlagDelete);
921 zebra_mutex_unlock(&p->mutex);
926 ZEBRA_RES rec_put(Records p, Record *recpp)
929 ZEBRA_RES ret = ZEBRA_OK;
931 zebra_mutex_lock(&p->mutex);
932 if ((recp = rec_cache_lookup(p, (*recpp)->sysno, recordFlagWrite)))
939 ret = rec_cache_insert(p, *recpp, recordFlagWrite);
942 zebra_mutex_unlock(&p->mutex);
947 void rec_free(Record *recpp)
953 for (i = 0; i < REC_NO_INFO; i++)
954 xfree((*recpp)->info[i]);
959 Record rec_cp(Record rec)
964 n = (Record) xmalloc(sizeof(*n));
965 n->sysno = rec->sysno;
966 for (i = 0; i < REC_NO_INFO; i++)
974 n->size[i] = rec->size[i];
975 n->info[i] = (char *) xmalloc(rec->size[i]+1);
976 memcpy(n->info[i], rec->info[i], rec->size[i]);
977 n->info[i][rec->size[i]] = '\0';
983 char *rec_strdup(const char *s, size_t *len)
993 p = (char *) xmalloc(*len);
998 void rec_prstat(Records records, int verbose)
1001 zint total_bytes = 0;
1004 "Total records %8" ZINT_FORMAT0,
1005 records->head.no_records);
1007 for (i = 0; i< REC_BLOCK_TYPES; i++)
1009 yaz_log (YLOG_LOG, "Record blocks of size "ZINT_FORMAT,
1010 records->head.block_size[i]);
1012 " Used/Total/Bytes used "
1013 ZINT_FORMAT "/" ZINT_FORMAT "/" ZINT_FORMAT,
1014 records->head.block_used[i], records->head.block_last[i]-1,
1015 records->head.block_used[i] * records->head.block_size[i]);
1017 records->head.block_used[i] * records->head.block_size[i];
1019 yaz_log(YLOG_LOG, " Block Last " ZINT_FORMAT, records->head.block_last[i]);
1021 { /* analyse free lists */
1023 zint block_free = records->head.block_free[i];
1024 WRBUF w = wrbuf_alloc();
1029 wrbuf_printf(w, " " ZINT_FORMAT, block_free);
1030 if (bf_read(records->data_BFile[i],
1031 block_free, 0, sizeof(nblock), &nblock) != 1)
1033 yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in %s at free block "
1035 records->data_fname[i], block_free);
1038 block_free = nblock;
1041 " Number in free list %8" ZINT_FORMAT0, no_free);
1043 yaz_log(YLOG_LOG, "%s", wrbuf_cstr(w));
1048 "Total size of record index in bytes %8" ZINT_FORMAT0,
1049 records->head.total_bytes);
1051 "Total size with overhead %8" ZINT_FORMAT0,
1058 * c-file-style: "Stroustrup"
1059 * indent-tabs-mode: nil
1061 * vim: shiftwidth=4 tabstop=8 expandtab