util/marcdump.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 #define _FILE_OFFSET_BITS 64
   7
   8 #if HAVE_CONFIG_H
   9 #include <config.h>
  10 #endif
  11
  12 #if YAZ_HAVE_XML2
  13 #include <libxml/parser.h>
  14 #include <libxml/tree.h>
  15 #include <libxml/xpath.h>
  16 #include <libxml/xpathInternals.h>
  17
  18 /* Libxml2 version < 2.6.15. xmlreader not reliable/present */
  19 #if LIBXML_VERSION < 20615
  20 #define USE_XMLREADER 0
  21 #else
  22 #define USE_XMLREADER 1
  23 #endif
  24
  25 #if USE_XMLREADER
  26 #include <libxml/xmlreader.h>
  27 #endif
  28
  29 #endif
  30
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <errno.h>
  35 #include <assert.h>
  36
  37 #if HAVE_LOCALE_H
  38 #include <locale.h>
  39 #endif
  40 #if HAVE_LANGINFO_H
  41 #include <langinfo.h>
  42 #endif
  43
  44 #include <yaz/marcdisp.h>
  45 #include <yaz/yaz-util.h>
  46 #include <yaz/xmalloc.h>
  47 #include <yaz/options.h>
  48
  49 #ifndef SEEK_SET
  50 #define SEEK_SET 0
  51 #endif
  52 #ifndef SEEK_END
  53 #define SEEK_END 2
  54 #endif
  55
  56
  57 static char *prog;
  58
  59 static void usage(const char *prog)
  60 {
  61     fprintf (stderr, "Usage: %s [-i format] [-o format] [-f from] [-t to] "
  62              "[-l pos=value] [-c cfile] [-s prefix] [-C size] [-n] "
  63              "[-p] [-v] [-V] file...\n",
  64              prog);
  65 }
  66
  67 static void show_version(void)
  68 {
  69     char vstr[20], sha1_str[41];
  70
  71     yaz_version(vstr, sha1_str);
  72     printf("YAZ version: %s %s\n", YAZ_VERSION, YAZ_VERSION_SHA1);
  73     if (strcmp(sha1_str, YAZ_VERSION_SHA1))
  74         printf("YAZ DLL/SO: %s %s\n", vstr, sha1_str);
  75     exit(0);
  76 }
  77
  78 static int getbyte_stream(void *client_data)
  79 {
  80     FILE *f = (FILE*) client_data;
  81
  82     int c = fgetc(f);
  83     if (c == EOF)
  84         return 0;
  85     return c;
  86 }
  87
  88 static void ungetbyte_stream(int c, void *client_data)
  89 {
  90     FILE *f = (FILE*) client_data;
  91
  92     if (c == 0)
  93         c = EOF;
  94     ungetc(c, f);
  95 }
  96
  97 static void marcdump_read_line(yaz_marc_t mt, const char *fname)
  98 {
  99     FILE *inf = fopen(fname, "rb");
 100     if (!inf)
 101     {
 102         fprintf (stderr, "%s: cannot open %s:%s\n",
 103                  prog, fname, strerror (errno));
 104         exit(1);
 105     }
 106
 107     while (yaz_marc_read_line(mt, getbyte_stream,
 108                               ungetbyte_stream, inf) == 0)
 109     {
 110         WRBUF wrbuf = wrbuf_alloc();
 111         yaz_marc_write_mode(mt, wrbuf);
 112         fputs(wrbuf_cstr(wrbuf), stdout);
 113         wrbuf_destroy(wrbuf);
 114     }
 115     fclose(inf);
 116 }
 117
 118 #if YAZ_HAVE_XML2
 119 static void marcdump_read_xml(yaz_marc_t mt, const char *fname)
 120 {
 121     WRBUF wrbuf = wrbuf_alloc();
 122 #if USE_XMLREADER
 123     xmlTextReaderPtr reader = xmlReaderForFile(fname, 0 /* encoding */,
 124                                                0 /* options */);
 125
 126     if (reader)
 127     {
 128         int ret;
 129         while ((ret = xmlTextReaderRead(reader)) == 1)
 130         {
 131             int type = xmlTextReaderNodeType(reader);
 132             if (type == XML_READER_TYPE_ELEMENT)
 133             {
 134                 const char *name = (const char *)
 135                     xmlTextReaderLocalName(reader);
 136                 if (!strcmp(name, "record"))
 137                 {
 138                     xmlNodePtr ptr = xmlTextReaderExpand(reader);
 139
 140                     int r = yaz_marc_read_xml(mt, ptr);
 141                     if (r)
 142                         fprintf(stderr, "yaz_marc_read_xml failed\n");
 143                     else
 144                     {
 145                         int write_rc = yaz_marc_write_mode(mt, wrbuf);
 146                                                 if (write_rc)
 147                                                         yaz_log(YLOG_WARN, "yaz_marc_write_mode: write error: %d", write_rc);
 148
 149                         fputs(wrbuf_cstr(wrbuf), stdout);
 150                         wrbuf_rewind(wrbuf);
 151                     }
 152                 }
 153             }
 154         }
 155     }
 156 #else
 157     xmlDocPtr doc = xmlParseFile(fname);
 158     if (doc)
 159     {
 160         xmlNodePtr ptr = xmlDocGetRootElement(doc);
 161         for (; ptr; ptr = ptr->next)
 162         {
 163             if (ptr->type == XML_ELEMENT_NODE)
 164             {
 165                 if (!strcmp((const char *) ptr->name, "collection"))
 166                 {
 167                     ptr = ptr->children;
 168                     continue;
 169                 }
 170                 if (!strcmp((const char *) ptr->name, "record"))
 171                 {
 172                     int r = yaz_marc_read_xml(mt, ptr);
 173                     if (r)
 174                         fprintf(stderr, "yaz_marc_read_xml failed\n");
 175                     else
 176                     {
 177                         yaz_marc_write_mode(mt, wrbuf);
 178
 179                         fputs(wrbuf_cstr(wrbuf), stdout);
 180                         wrbuf_rewind(wrbuf);
 181                     }
 182                 }
 183             }
 184         }
 185         xmlFreeDoc(doc);
 186     }
 187 #endif
 188     fputs(wrbuf_cstr(wrbuf), stdout);
 189     wrbuf_destroy(wrbuf);
 190 }
 191 #endif
 192
 193 static void dump(const char *fname, const char *from, const char *to,
 194                  int input_format, int output_format,
 195                  int write_using_libxml2,
 196                  int print_offset, const char *split_fname, int split_chunk,
 197                  int verbose, FILE *cfile, const char *leader_spec)
 198 {
 199     yaz_marc_t mt = yaz_marc_create();
 200     yaz_iconv_t cd = 0;
 201
 202     if (yaz_marc_leader_spec(mt, leader_spec))
 203     {
 204         fprintf(stderr, "bad leader spec: %s\n", leader_spec);
 205         yaz_marc_destroy(mt);
 206         exit(2);
 207     }
 208     if (from && to)
 209     {
 210         cd = yaz_iconv_open(to, from);
 211         if (!cd)
 212         {
 213             fprintf(stderr, "conversion from %s to %s "
 214                     "unsupported\n", from, to);
 215             yaz_marc_destroy(mt);
 216             exit(2);
 217         }
 218         yaz_marc_iconv(mt, cd);
 219     }
 220     yaz_marc_enable_collection(mt);
 221     yaz_marc_set_read_format(mt, input_format);
 222     yaz_marc_set_write_format(mt, output_format);
 223     yaz_marc_write_using_libxml2(mt, write_using_libxml2);
 224     yaz_marc_write_turbo_format(mt, output_format == YAZ_MARC_TMARCXML);
 225     yaz_marc_debug(mt, verbose);
 226
 227     if (input_format == YAZ_MARC_MARCXML || input_format == YAZ_MARC_TMARCXML || input_format == YAZ_MARC_XCHANGE)
 228     {
 229 #if YAZ_HAVE_XML2
 230         marcdump_read_xml(mt, fname);
 231 #endif
 232     }
 233     else if (input_format == YAZ_MARC_LINE)
 234     {
 235         marcdump_read_line(mt, fname);
 236     }
 237     else if (input_format == YAZ_MARC_ISO2709)
 238     {
 239         FILE *inf = fopen(fname, "rb");
 240         int num = 1;
 241         int marc_no = 0;
 242         int split_file_no = -1;
 243         if (!inf)
 244         {
 245             fprintf (stderr, "%s: cannot open %s:%s\n",
 246                      prog, fname, strerror (errno));
 247             exit(1);
 248         }
 249         if (cfile)
 250             fprintf (cfile, "char *marc_records[] = {\n");
 251         for(;; marc_no++)
 252         {
 253             const char *result = 0;
 254             size_t len;
 255             size_t rlen;
 256             size_t len_result;
 257             size_t r;
 258             char buf[100001];
 259
 260             r = fread (buf, 1, 5, inf);
 261             if (r < 5)
 262             {
 263                 if (r && print_offset && verbose)
 264                     printf ("<!-- Extra %ld bytes at end of file -->\n",
 265                             (long) r);
 266                 break;
 267             }
 268             while (*buf < '0' || *buf > '9')
 269             {
 270                 int i;
 271                 long off = ftell(inf) - 5;
 272                 if (verbose || print_offset)
 273                     printf("<!-- Skipping bad byte %d (0x%02X) at offset "
 274                            "%ld (0x%lx) -->\n",
 275                            *buf & 0xff, *buf & 0xff,
 276                            off, off);
 277                 for (i = 0; i<4; i++)
 278                     buf[i] = buf[i+1];
 279                 r = fread(buf+4, 1, 1, inf);
 280                 if (r < 1)
 281                     break;
 282             }
 283             if (r < 1)
 284             {
 285                 if (verbose || print_offset)
 286                     printf ("<!-- End of file with data -->\n");
 287                 break;
 288             }
 289             if (print_offset)
 290             {
 291                 long off = ftell(inf) - 5;
 292                 printf ("<!-- Record %d offset %ld (0x%lx) -->\n",
 293                         num, off, off);
 294             }
 295             len = atoi_n(buf, 5);
 296             if (len < 25 || len > 100000)
 297             {
 298                 long off = ftell(inf) - 5;
 299                 printf("Bad Length %ld read at offset %ld (%lx)\n",
 300                        (long)len, (long) off, (long) off);
 301                 break;
 302             }
 303             rlen = len - 5;
 304             r = fread (buf + 5, 1, rlen, inf);
 305             if (r < rlen)
 306                 break;
 307             while (buf[len-1] != ISO2709_RS)
 308             {
 309                 if (len > sizeof(buf)-2)
 310                     break;
 311                 r = fread (buf + len, 1, 1, inf);
 312                 if (r != 1)
 313                     break;
 314                 len++;
 315             }
 316             if (split_fname)
 317             {
 318                 char fname[256];
 319                 const char *mode = 0;
 320                 FILE *sf;
 321                 if ((marc_no % split_chunk) == 0)
 322                 {
 323                     mode = "wb";
 324                     split_file_no++;
 325                 }
 326                 else
 327                     mode = "ab";
 328                 sprintf(fname, "%.200s%07d", split_fname, split_file_no);
 329                 sf = fopen(fname, mode);
 330                 if (!sf)
 331                 {
 332                     fprintf(stderr, "Could not open %s\n", fname);
 333                     split_fname = 0;
 334                 }
 335                 else
 336                 {
 337                     if (fwrite(buf, 1, len, sf) != len)
 338                     {
 339                         fprintf(stderr, "Could write content to %s\n",
 340                                 fname);
 341                         split_fname = 0;
 342                     }
 343                     fclose(sf);
 344                 }
 345             }
 346             len_result = rlen;
 347             r = yaz_marc_decode_buf(mt, buf, -1, &result, &len_result);
 348             if (r > 0 && result && len_result)
 349             {
 350                 if (fwrite(result, len_result, 1, stdout) != 1)
 351                 {
 352                     fprintf(stderr, "Write to stdout failed\n");
 353                     break;
 354                 }
 355             }
 356             if (r > 0 && cfile)
 357             {
 358                 char *p = buf;
 359                 size_t i;
 360                 if (marc_no)
 361                     fprintf (cfile, ",");
 362                 fprintf (cfile, "\n");
 363                 for (i = 0; i < r; i++)
 364                 {
 365                     if ((i & 15) == 0)
 366                         fprintf (cfile, "  \"");
 367                     fprintf (cfile, "\\x%02X", p[i] & 255);
 368
 369                     if (i < r - 1 && (i & 15) == 15)
 370                         fprintf (cfile, "\"\n");
 371
 372                 }
 373                 fprintf (cfile, "\"\n");
 374             }
 375             num++;
 376             if (verbose)
 377                 printf("\n");
 378         }
 379         if (cfile)
 380             fprintf (cfile, "};\n");
 381         fclose(inf);
 382     }
 383     {
 384         WRBUF wrbuf = wrbuf_alloc();
 385         yaz_marc_write_trailer(mt, wrbuf);
 386         fputs(wrbuf_cstr(wrbuf), stdout);
 387         wrbuf_destroy(wrbuf);
 388     }
 389     if (cd)
 390         yaz_iconv_close(cd);
 391     yaz_marc_destroy(mt);
 392 }
 393
 394 int main (int argc, char **argv)
 395 {
 396     int r;
 397     int print_offset = 0;
 398     char *arg;
 399     int verbose = 0;
 400     int no = 0;
 401     int output_format = YAZ_MARC_LINE;
 402     FILE *cfile = 0;
 403     char *from = 0, *to = 0;
 404     int input_format = YAZ_MARC_ISO2709;
 405     int split_chunk = 1;
 406     const char *split_fname = 0;
 407     const char *leader_spec = 0;
 408     int write_using_libxml2 = 0;
 409
 410 #if HAVE_LOCALE_H
 411     setlocale(LC_CTYPE, "");
 412 #endif
 413 #if HAVE_LANGINFO_H
 414 #ifdef CODESET
 415     to = nl_langinfo(CODESET);
 416 #endif
 417 #endif
 418
 419     prog = *argv;
 420     while ((r = options("i:o:C:npc:xOeXIf:t:s:l:Vv", argv, argc, &arg)) != -2)
 421     {
 422         no++;
 423         switch (r)
 424         {
 425         case 'i':
 426             input_format = yaz_marc_decode_formatstr(arg);
 427             if (input_format == -1)
 428             {
 429                 fprintf(stderr, "%s: bad input format: %s\n", prog, arg);
 430                 exit(1);
 431             }
 432 #if YAZ_HAVE_XML2
 433 #else
 434             if (input_format == YAZ_MARC_MARCXML
 435                 || input_format == YAZ_MARC_XCHANGE)
 436             {
 437                 fprintf(stderr, "%s: Libxml2 support not enabled\n", prog);
 438                 exit(3);
 439             }
 440 #endif
 441             break;
 442         case 'o':
 443             /* dirty hack so we can make Libxml2 do the writing ..
 444                rather than WRBUF */
 445             if (strlen(arg) > 4 && strncmp(arg, "xml,", 4) == 0)
 446             {
 447                 arg = arg + 4;
 448                 write_using_libxml2 = 1;
 449             }
 450             output_format = yaz_marc_decode_formatstr(arg);
 451             if (output_format == -1)
 452             {
 453                 fprintf(stderr, "%s: bad output format: %s\n", prog, arg);
 454                 exit(1);
 455             }
 456             break;
 457         case 'l':
 458             leader_spec = arg;
 459             break;
 460         case 'f':
 461             from = arg;
 462             break;
 463         case 't':
 464             to = arg;
 465             break;
 466         case 'c':
 467             if (cfile)
 468                 fclose (cfile);
 469             cfile = fopen(arg, "w");
 470             break;
 471         case 'x':
 472             fprintf(stderr, "%s: -x no longer supported. "
 473                     "Use -i marcxml instead\n", prog);
 474             exit(1);
 475             break;
 476         case 'O':
 477             fprintf(stderr, "%s: OAI MARC no longer supported."
 478                     " Use MARCXML instead.\n", prog);
 479             exit(1);
 480             break;
 481         case 'e':
 482             fprintf(stderr, "%s: -e no longer supported. "
 483                     "Use -o marcxchange instead\n", prog);
 484             exit(1);
 485             break;
 486         case 'X':
 487             fprintf(stderr, "%s: -X no longer supported. "
 488                     "Use -o marcxml instead\n", prog);
 489             exit(1);
 490             break;
 491         case 'I':
 492             fprintf(stderr, "%s: -I no longer supported. "
 493                     "Use -o marc instead\n", prog);
 494             exit(1);
 495             break;
 496         case 'n':
 497             output_format = YAZ_MARC_CHECK;
 498             break;
 499         case 'p':
 500             print_offset = 1;
 501             break;
 502         case 's':
 503             split_fname = arg;
 504             break;
 505         case 'C':
 506             split_chunk = atoi(arg);
 507             break;
 508         case 0:
 509             dump(arg, from, to, input_format, output_format,
 510                  write_using_libxml2,
 511                  print_offset, split_fname, split_chunk,
 512                  verbose, cfile, leader_spec);
 513             break;
 514         case 'v':
 515             verbose++;
 516             break;
 517         case 'V':
 518             show_version();
 519             break;
 520         default:
 521             usage(prog);
 522             exit(1);
 523         }
 524     }
 525     if (cfile)
 526         fclose (cfile);
 527     if (!no)
 528     {
 529         usage(prog);
 530         exit (1);
 531     }
 532     exit (0);
 533 }
 534 /*
 535  * Local variables:
 536  * c-basic-offset: 4
 537  * c-file-style: "Stroustrup"
 538  * indent-tabs-mode: nil
 539  * End:
 540  * vim: shiftwidth=4 tabstop=8 expandtab
 541  */
 542