1 /* $Id: charmap.c,v 1.32 2004-12-10 11:56:23 heikki Exp $
2 Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
26 * Support module to handle character-conversions into and out of the
34 typedef unsigned ucs4_t;
38 /* Old yaz-util includes */
39 #include <yaz/yconfig.h>
40 #include <yaz/yaz-version.h>
41 #include <yaz/xmalloc.h>
43 #include <yaz/tpath.h>
44 #include <yaz/options.h>
45 #include <yaz/wrbuf.h>
47 #include <yaz/readconf.h>
48 #include <yaz/marcdisp.h>
49 #include <yaz/yaz-iconv.h>
52 #define CHR_MAXSTR 1024
53 #define CHR_MAXEQUIV 32
55 const unsigned char CHR_FIELD_BEGIN = '^';
57 const char *CHR_UNKNOWN = "\001";
58 const char *CHR_SPACE = "\002";
59 const char *CHR_CUT = "\003";
60 const char *CHR_BASE = "\005";
64 chr_t_entry *input; /* mapping table for input data */
65 chr_t_entry *q_input; /* mapping table for queries */
66 unsigned char *output[256]; /* return mapping - for display of registers */
67 int base_uppercase; /* Start of upper-case ordinals */
72 * Character map trie node.
76 chr_t_entry **children; /* array of children */
77 unsigned char **target; /* target for this node, if any */
81 * General argument structure for callback functions (internal use only)
83 typedef struct chrwork
86 char string[CHR_MAXSTR+1];
90 * Add an entry to the character map.
92 static chr_t_entry *set_map_string(chr_t_entry *root, NMEM nmem,
93 const char *from, int len, char *to,
100 root = (chr_t_entry *) nmem_malloc(nmem, sizeof(*root));
106 if (!root->target || !root->target[0] || strcmp(root->target[0], to))
109 root->target && root->target[0] && root->target[0][0] &&
110 strcmp (root->target[0], CHR_UNKNOWN))
112 yaz_log (YLOG_WARN, "duplicate entry for charmap from '%s'",
115 root->target = (unsigned char **)
116 nmem_malloc(nmem, sizeof(*root->target)*2);
117 root->target[0] = (unsigned char *) nmem_strdup(nmem, to);
127 root->children = (chr_t_entry **)
128 nmem_malloc(nmem, sizeof(chr_t_entry*) * 256);
129 for (i = 0; i < 256; i++)
130 root->children[i] = 0;
132 if (!(root->children[(unsigned char) *from] =
133 set_map_string(root->children[(unsigned char) *from], nmem,
134 from + 1, len - 1, to, from_0)))
140 static chr_t_entry *find_entry(chr_t_entry *t, const char **from, int len)
144 if (len && t->children && t->children[(unsigned char) **from])
146 const char *pos = *from;
149 if ((res = find_entry(t->children[(unsigned char) *pos],
155 /* no children match. use ourselves, if we have a target */
156 return t->target ? t : 0;
159 static chr_t_entry *find_entry_x(chr_t_entry *t, const char **from, int *len, int first)
164 { /* switch to next buffer */
170 if (*len > 0 && t->children)
172 const char *old_from = *from;
177 if (first && t->children[CHR_FIELD_BEGIN])
179 if ((res = find_entry_x(t->children[CHR_FIELD_BEGIN], from, len, 0)) && res != t->children[CHR_FIELD_BEGIN])
183 /* otherwhise there was no match on beginning of field, move on */
186 if (!res && t->children[(unsigned char) **from])
190 if ((res = find_entry_x(t->children[(unsigned char) *old_from],
198 /* no children match. use ourselves, if we have a target */
199 return t->target ? t : 0;
202 const char **chr_map_input_x(chrmaptab maptab, const char **from, int *len, int first)
204 chr_t_entry *t = maptab->input;
207 if (!(res = find_entry_x(t, from, len, first)))
209 return (const char **) (res->target);
212 const char **chr_map_input(chrmaptab maptab, const char **from, int len, int first)
214 chr_t_entry *t = maptab->input;
220 if (!(res = find_entry_x(t, from, len_tmp, first)))
222 return (const char **) (res->target);
225 const char *chr_map_output(chrmaptab maptab, const char **from, int len)
227 unsigned char c = ** (unsigned char **) from;
229 return (const char*) maptab->output[c];
232 unsigned char zebra_prim(char **s)
237 yaz_log (YLOG_DEBUG, "prim %.3s", *s);
244 case '\\': c = '\\'; (*s)++; break;
245 case 'r': c = '\r'; (*s)++; break;
246 case 'n': c = '\n'; (*s)++; break;
247 case 't': c = '\t'; (*s)++; break;
248 case 's': c = ' '; (*s)++; break;
249 case 'x': sscanf(*s, "x%2x", &i); c = i; *s += 3; break;
260 sscanf(*s, "%3o", &i);
276 static int zebra_ucs4_strlen(ucs4_t *s)
284 ucs4_t zebra_prim_w(ucs4_t **s)
290 yaz_log (YLOG_DEBUG, "prim_w %.3s", (char *) *s);
297 case '\\': c = '\\'; (*s)++; break;
298 case 'r': c = '\r'; (*s)++; break;
299 case 'n': c = '\n'; (*s)++; break;
300 case 't': c = '\t'; (*s)++; break;
301 case 's': c = ' '; (*s)++; break;
303 if (zebra_ucs4_strlen(*s) >= 3)
308 sscanf(fmtstr, "%x", &i);
323 if (zebra_ucs4_strlen(*s) >= 3)
329 sscanf(fmtstr, "%o", &i);
335 if (zebra_ucs4_strlen(*s) >= 5)
342 sscanf(fmtstr, "%x", &i);
356 yaz_log (YLOG_DEBUG, "out %d", c);
362 * Add an entry to the value space.
364 static void fun_addentry(const char *s, void *data, int num)
366 chrmaptab tab = (chrmaptab) data;
369 tmp[0] = num; tmp[1] = '\0';
370 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s), tmp, 0);
371 tab->output[num + tab->base_uppercase] =
372 (unsigned char *) nmem_strdup(tab->nmem, s);
377 * Add a space-entry to the value space.
379 static void fun_addspace(const char *s, void *data, int num)
381 chrmaptab tab = (chrmaptab) data;
382 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s),
383 (char*) CHR_SPACE, 0);
388 * Add a space-entry to the value space.
390 static void fun_addcut(const char *s, void *data, int num)
392 chrmaptab tab = (chrmaptab) data;
393 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s),
398 * Create a string containing the mapped characters provided.
400 static void fun_mkstring(const char *s, void *data, int num)
402 chrwork *arg = (chrwork *) data;
403 const char **res, *p = s;
405 res = chr_map_input(arg->map, &s, strlen(s), 0);
406 if (*res == (char*) CHR_UNKNOWN)
407 yaz_log(YLOG_WARN, "Map: '%s' has no mapping", p);
408 strncat(arg->string, *res, CHR_MAXSTR - strlen(arg->string));
409 arg->string[CHR_MAXSTR] = '\0';
413 * Add a map to the string contained in the argument.
415 static void fun_add_map(const char *s, void *data, int num)
417 chrwork *arg = (chrwork *) data;
419 assert(arg->map->input);
420 yaz_log (YLOG_DEBUG, "set map %.*s", (int) strlen(s), s);
421 set_map_string(arg->map->input, arg->map->nmem, s, strlen(s), arg->string,
423 for (s = arg->string; *s; s++)
424 yaz_log (YLOG_DEBUG, " %3d", (unsigned char) *s);
428 * Add a query map to the string contained in the argument.
430 static void fun_add_qmap(const char *s, void *data, int num)
432 chrwork *arg = (chrwork *) data;
434 assert(arg->map->q_input);
435 yaz_log (YLOG_DEBUG, "set qmap %.*s", (int) strlen(s), s);
436 set_map_string(arg->map->q_input, arg->map->nmem, s,
437 strlen(s), arg->string, 0);
438 for (s = arg->string; *s; s++)
439 yaz_log (YLOG_DEBUG, " %3d", (unsigned char) *s);
442 static int scan_to_utf8 (yaz_iconv_t t, ucs4_t *from, size_t inlen,
443 char *outbuf, size_t outbytesleft)
445 size_t inbytesleft = inlen * sizeof(ucs4_t);
446 char *inbuf = (char*) from;
450 *outbuf++ = *from; /* ISO-8859-1 is OK here */
453 ret = yaz_iconv (t, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
454 if (ret == (size_t) (-1))
456 yaz_log(YLOG_LOG, "from: %2X %2X %2X %2X",
457 from[0], from[1], from[2], from[3]);
458 yaz_log (YLOG_WARN|YLOG_ERRNO, "bad unicode sequence");
466 static int scan_string(char *s_native,
467 yaz_iconv_t t_unicode, yaz_iconv_t t_utf8,
468 void (*fun)(const char *c, void *data, int num),
469 void *data, int *num)
474 ucs4_t arg_prim[512];
475 ucs4_t *s0, *s = arg;
476 ucs4_t c, begin, end;
481 char *outbuf = (char *) arg;
482 char *inbuf = s_native;
483 size_t outbytesleft = sizeof(arg)-4;
484 size_t inbytesleft = strlen(s_native);
486 ret = yaz_iconv(t_unicode, &inbuf, &inbytesleft,
487 &outbuf, &outbytesleft);
488 if (ret == (size_t)(-1))
490 i = (outbuf - (char*) arg)/sizeof(ucs4_t);
494 for (i = 0; s_native[i]; i++)
495 arg[i] = s_native[i] & 255; /* ISO-8859-1 conversion */
497 arg[i] = 0; /* terminate */
498 if (s[0] == 0xfeff || s[0] == 0xfeff) /* skip byte Order Mark */
506 begin = zebra_prim_w(&s);
509 yaz_log(YLOG_FATAL, "Bad range in char-map");
513 end = zebra_prim_w(&s);
516 yaz_log(YLOG_FATAL, "Bad range in char-map");
520 for (c = begin; c <= end; c++)
522 if (scan_to_utf8 (t_utf8, &c, 1, str, sizeof(str)-1))
524 (*fun)(str, data, num ? (*num)++ : 0);
527 case '[': s++; abort(); break;
531 while (*s != ')' || s[-1] == '\\')
532 arg_prim[i++] = zebra_prim_w(&s);
534 if (scan_to_utf8 (t_utf8, arg_prim, zebra_ucs4_strlen(arg_prim), str, sizeof(str)-1))
536 (*fun)(str, data, num ? (*num)++ : 0);
540 c = zebra_prim_w(&s);
541 if (scan_to_utf8 (t_utf8, &c, 1, str, sizeof(str)-1))
543 (*fun)(str, data, num ? (*num)++ : 0);
549 chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
553 char line[512], *argv[50];
557 int argc, num = (int) *CHR_BASE, i;
559 yaz_iconv_t t_unicode = 0;
560 yaz_iconv_t t_utf8 = 0;
561 unsigned endian = 31;
562 const char *ucs4_native = "UCS-4";
564 if (*(char*) &endian == 31) /* little endian? */
565 ucs4_native = "UCS-4LE";
567 t_utf8 = yaz_iconv_open ("UTF-8", ucs4_native);
569 yaz_log (YLOG_DEBUG, "maptab %s open", name);
570 if (!(f = yaz_fopen(tabpath, name, "r", tabroot)))
572 yaz_log(YLOG_WARN|YLOG_ERRNO, "%s", name);
575 nmem = nmem_create ();
576 res = (chrmaptab) nmem_malloc(nmem, sizeof(*res));
578 res->input = (chr_t_entry *) nmem_malloc(res->nmem, sizeof(*res->input));
579 res->input->target = (unsigned char **)
580 nmem_malloc(res->nmem, sizeof(*res->input->target) * 2);
581 res->input->target[0] = (unsigned char*) CHR_UNKNOWN;
582 res->input->target[1] = 0;
583 res->input->children = (chr_t_entry **)
584 nmem_malloc(res->nmem, sizeof(res->input) * 256);
585 for (i = 0; i < 256; i++)
587 res->input->children[i] = (chr_t_entry *)
588 nmem_malloc(res->nmem, sizeof(*res->input));
589 res->input->children[i]->children = 0;
590 res->input->children[i]->target = (unsigned char **)
591 nmem_malloc (res->nmem, 2 * sizeof(unsigned char *));
592 res->input->children[i]->target[1] = 0;
595 res->input->children[i]->target[0] = (unsigned char *)
596 nmem_malloc (res->nmem, 2 * sizeof(unsigned char));
597 res->input->children[i]->target[0][0] = i;
598 res->input->children[i]->target[0][1] = 0;
601 res->input->children[i]->target[0] = (unsigned char*) CHR_UNKNOWN;
603 res->q_input = (chr_t_entry *)
604 nmem_malloc(res->nmem, sizeof(*res->q_input));
605 res->q_input->target = 0;
606 res->q_input->children = 0;
608 for (i = *CHR_BASE; i < 256; i++)
610 res->output[(int) *CHR_SPACE] = (unsigned char *) " ";
611 res->output[(int) *CHR_UNKNOWN] = (unsigned char*) "@";
612 res->base_uppercase = 0;
614 while (!errors && (argc = readconf_line(f, &lineno, line, 512, argv, 50)))
615 if (!map_only && !yaz_matchstr(argv[0], "lowercase"))
619 yaz_log(YLOG_FATAL, "Syntax error in charmap");
622 if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
625 yaz_log(YLOG_FATAL, "Bad value-set specification");
628 res->base_uppercase = num;
629 res->output[(int) *CHR_SPACE + num] = (unsigned char *) " ";
630 res->output[(int) *CHR_UNKNOWN + num] = (unsigned char*) "@";
631 num = (int) *CHR_BASE;
633 else if (!map_only && !yaz_matchstr(argv[0], "uppercase"))
635 if (!res->base_uppercase)
637 yaz_log(YLOG_FATAL, "Uppercase directive with no lowercase set");
642 yaz_log(YLOG_FATAL, "Missing arg for uppercase directive");
645 if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
648 yaz_log(YLOG_FATAL, "Bad value-set specification");
652 else if (!map_only && !yaz_matchstr(argv[0], "space"))
656 yaz_log(YLOG_FATAL, "Syntax error in charmap for space");
659 if (scan_string(argv[1], t_unicode, t_utf8,
660 fun_addspace, res, 0) < 0)
662 yaz_log(YLOG_FATAL, "Bad space specification");
666 else if (!map_only && !yaz_matchstr(argv[0], "cut"))
670 yaz_log(YLOG_FATAL, "Syntax error in charmap for cut");
673 if (scan_string(argv[1], t_unicode, t_utf8,
674 fun_addcut, res, 0) < 0)
676 yaz_log(YLOG_FATAL, "Bad cut specification");
680 else if (!yaz_matchstr(argv[0], "map"))
686 yaz_log(YLOG_FATAL, "charmap directive map requires 2 args");
690 buf.string[0] = '\0';
691 if (scan_string(argv[2], t_unicode, t_utf8,
692 fun_mkstring, &buf, 0) < 0)
694 yaz_log(YLOG_FATAL, "Bad map target");
697 if (scan_string(argv[1], t_unicode, t_utf8,
698 fun_add_map, &buf, 0) < 0)
700 yaz_log(YLOG_FATAL, "Bad map source");
704 else if (!yaz_matchstr(argv[0], "qmap"))
710 yaz_log(YLOG_FATAL, "charmap directive qmap requires 2 args");
714 buf.string[0] = '\0';
715 if (scan_string(argv[2], t_unicode, t_utf8,
716 fun_mkstring, &buf, 0) < 0)
718 yaz_log(YLOG_FATAL, "Bad qmap target");
721 if (scan_string(argv[1], t_unicode, t_utf8,
722 fun_add_qmap, &buf, 0) < 0)
724 yaz_log(YLOG_FATAL, "Bad qmap source");
728 else if (!yaz_matchstr(argv[0], "encoding"))
731 * Fix me. When t_unicode==0 and use encoding directive in *.chr file the beheviour of the
732 * zebra need to comment next part of code.
738 yaz_iconv_close (t_unicode);
739 t_unicode = yaz_iconv_open (ucs4_native, argv[1]);
742 * Fix me. It is additional staff for conversion of characters from local encoding
743 * of *.chr file to UTF-8 (internal encoding).
744 * NOTE: The derective encoding must be first directive in *.chr file.
746 /* For whatever reason Oleg enabled this.. */
749 yaz_iconv_close(t_utf8);
750 t_utf8 = yaz_iconv_open ("UTF-8", argv[1]);
755 yaz_log(YLOG_WARN, "Syntax error at '%s' in %s", line, name);
761 chrmaptab_destroy(res);
764 yaz_log (YLOG_DEBUG, "maptab %s close %d errors", name, errors);
766 yaz_iconv_close(t_utf8);
768 yaz_iconv_close(t_unicode);
772 void chrmaptab_destroy(chrmaptab tab)
775 nmem_destroy (tab->nmem);