index/main.c

   1 /*
   2  * Copyright (C) 1994-2001, Index Data
   3  * All rights reserved.
   4  *
   5  * $Id: main.c,v 1.81 2001-11-19 23:29:09 adam Exp $
   6  */
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <assert.h>
  10 #ifdef WIN32
  11 #include <io.h>
  12 #else
  13 #include <unistd.h>
  14 #endif
  15
  16 #include <yaz/data1.h>
  17 #include "index.h"
  18 #include "recindex.h"
  19
  20 #ifndef ZEBRASDR
  21 #define ZEBRASDR 0
  22 #endif
  23
  24 #if ZEBRASDR
  25 #include "zebrasdr.h"
  26 #endif
  27
  28 char *prog;
  29
  30 Res common_resource = 0;
  31
  32
  33 int main (int argc, char **argv)
  34 {
  35     int ret;
  36     int cmd = 0;
  37     char *arg;
  38     char *configName = FNAME_CONFIG;
  39     int nsections = 0;
  40     int disableCommit = 0;
  41     size_t mem_max = 0;
  42 #ifndef WIN32
  43     char nbuf[100];
  44 #endif
  45     struct recordGroup rGroupDef;
  46
  47     nmem_init ();
  48
  49 #ifdef WIN32
  50 #else
  51     sprintf(nbuf, "%.40s(%d)", *argv, getpid());
  52     yaz_log_init_prefix (nbuf);
  53 #endif
  54
  55 #if ZEBRASDR
  56     zebraSdr_std ();
  57     rGroupDef.useSDR = 0;
  58 #endif
  59     rGroupDef.groupName = NULL;
  60     rGroupDef.databaseName = NULL;
  61     rGroupDef.path = NULL;
  62     rGroupDef.recordId = NULL;
  63     rGroupDef.recordType = NULL;
  64     rGroupDef.flagStoreData = -1;
  65     rGroupDef.flagStoreKeys = -1;
  66     rGroupDef.flagRw = 1;
  67     rGroupDef.databaseNamePath = 0;
  68     rGroupDef.explainDatabase = 0;
  69     rGroupDef.fileVerboseLimit = 100000;
  70     rGroupDef.zebra_maps = NULL;
  71     rGroupDef.dh = data1_create ();
  72     rGroupDef.recTypes = recTypes_init (rGroupDef.dh);
  73     recTypes_default_handlers (rGroupDef.recTypes);
  74
  75     prog = *argv;
  76     if (argc < 2)
  77     {
  78         fprintf (stderr, "%s [options] command <dir> ...\n"
  79         "Commands:\n"
  80         " update <dir>  Update index with files below <dir>.\n"
  81         "               If <dir> is empty filenames are read from stdin.\n"
  82         " delete <dir>  Delete index with files below <dir>.\n"
  83         " commit        Commit changes\n"
  84         " clean         Clean shadow files\n"
  85         "Options:\n"
  86         " -t <type>     Index files as <type> (grs or text).\n"
  87         " -c <config>   Read configuration file <config>.\n"
  88         " -g <group>    Index files according to group settings.\n"
  89         " -d <database> Records belong to Z39.50 database <database>.\n"
  90         " -m <mbytes>   Use <mbytes> before flushing keys to disk.\n"
  91         " -n            Don't use shadow system.\n"
  92         " -s            Show analysis on stdout, but do no work.\n"
  93         " -v <level>    Set logging to <level>.\n"
  94         " -l <file>     Write log to <file>.\n"
  95         " -f <n>        Display information for the first <n> records.\n"
  96 #if ZEBRASDR
  97         " -S            Use SDRKit\n"
  98 #endif
  99         " -V            Show version.\n", *argv
 100                  );
 101         exit (1);
 102     }
 103     while ((ret = options ("sVt:c:g:d:m:v:nf:l:"
 104 #if ZEBRASDR
 105                            "S"
 106 #endif
 107                            , argv, argc, &arg)) != -2)
 108     {
 109         if (ret == 0)
 110         {
 111             const char *rval;
 112             if(cmd == 0) /* command */
 113             {
 114                 if (!common_resource)
 115                 {
 116 #if ZMBOL
 117                     logf (LOG_LOG, "zmbol version %s %s",
 118                           ZEBRAVER, ZEBRADATE);
 119 #else
 120                     logf (LOG_LOG, "zebra version %s %s",
 121                           ZEBRAVER, ZEBRADATE);
 122 #endif
 123                     common_resource = res_open (configName ?
 124                                                 configName : FNAME_CONFIG);
 125                     if (!common_resource)
 126                     {
 127                         logf (LOG_FATAL, "cannot read file `%s'", configName);
 128                         exit (1);
 129                     }
 130                     data1_set_tabpath (rGroupDef.dh, res_get (common_resource,
 131                                                               "profilePath"));
 132
 133                     rGroupDef.bfs =
 134                         bfs_create (res_get (common_resource, "register"));
 135                     if (!rGroupDef.bfs)
 136                     {
 137                         logf (LOG_FATAL, "Cannot access register");
 138                         exit(1);
 139                     }
 140
 141                     bf_lockDir (rGroupDef.bfs,
 142                                 res_get (common_resource, "lockDir"));
 143                     rGroupDef.zebra_maps = zebra_maps_open (common_resource);
 144                 }
 145                 if (!strcmp (arg, "update"))
 146                     cmd = 'u';
 147                 else if (!strcmp (arg, "update1"))
 148                     cmd = 'U';
 149                 else if (!strcmp (arg, "update2"))
 150                     cmd = 'm';
 151                 else if (!strcmp (arg, "dump"))
 152                     cmd = 's';
 153                 else if (!strcmp (arg, "del") || !strcmp(arg, "delete"))
 154                     cmd = 'd';
 155                 else if (!strcmp (arg, "init"))
 156                 {
 157                     zebraIndexUnlock();
 158                     rval = res_get (common_resource, "shadow");
 159                     zebraIndexLock (rGroupDef.bfs, 0, rval);
 160                     if (rval && *rval)
 161                         bf_cache (rGroupDef.bfs, rval);
 162                     zebraIndexLockMsg ("w");
 163                     bf_reset (rGroupDef.bfs);
 164                 }
 165                 else if (!strcmp (arg, "commit"))
 166                 {
 167                     rval = res_get (common_resource, "shadow");
 168                     zebraIndexLock (rGroupDef.bfs, 1, rval);
 169                     if (rval && *rval)
 170                         bf_cache (rGroupDef.bfs, rval);
 171                     else
 172                     {
 173                         logf (LOG_FATAL, "Cannot perform commit");
 174                         logf (LOG_FATAL, "No shadow area defined");
 175                         exit (1);
 176                     }
 177                     if (bf_commitExists (rGroupDef.bfs))
 178                     {
 179                         logf (LOG_LOG, "commit start");
 180                         zebraIndexLockMsg ("c");
 181                         zebraIndexWait (1);
 182                         logf (LOG_LOG, "commit execute");
 183                         bf_commitExec (rGroupDef.bfs);
 184 #ifndef WIN32
 185                         sync ();
 186 #endif
 187                         zebraIndexLockMsg ("d");
 188                         zebraIndexWait (0);
 189                         logf (LOG_LOG, "commit clean");
 190                         bf_commitClean (rGroupDef.bfs, rval);
 191                     }
 192                     else
 193                         logf (LOG_LOG, "nothing to commit");
 194                 }
 195                 else if (!strcmp (arg, "clean"))
 196                 {
 197                     rval = res_get (common_resource, "shadow");
 198                     zebraIndexLock (rGroupDef.bfs, 1, rval);
 199                     if (bf_commitExists (rGroupDef.bfs))
 200                     {
 201                         zebraIndexLockMsg ("d");
 202                         zebraIndexWait (0);
 203                         logf (LOG_LOG, "commit clean");
 204                         bf_commitClean (rGroupDef.bfs, rval);
 205                     }
 206                     else
 207                         logf (LOG_LOG, "nothing to clean");
 208                 }
 209                 else if (!strcmp (arg, "stat") || !strcmp (arg, "status"))
 210                 {
 211                     Records records;
 212                     rval = res_get (common_resource, "shadow");
 213                     zebraIndexLock (rGroupDef.bfs, 0, rval);
 214                     if (rval && *rval)
 215                     {
 216                         bf_cache (rGroupDef.bfs, rval);
 217                         zebraIndexLockMsg ("r");
 218                     }
 219                     records = rec_open (rGroupDef.bfs, 0, 0);
 220                     rec_prstat (records);
 221                     rec_close (&records);
 222                     inv_prstat (rGroupDef.bfs);
 223                 }
 224                 else if (!strcmp (arg, "compact"))
 225                 {
 226                     rval = res_get (common_resource, "shadow");
 227                     zebraIndexLock (rGroupDef.bfs, 0, rval);
 228                     if (rval && *rval)
 229                     {
 230                         bf_cache (rGroupDef.bfs, rval);
 231                         zebraIndexLockMsg ("r");
 232                     }
 233                     inv_compact(rGroupDef.bfs);
 234                 }
 235                 else
 236                 {
 237                     logf (LOG_FATAL, "unknown command: %s", arg);
 238                     exit (1);
 239                 }
 240             }
 241             else
 242             {
 243                 struct recordGroup rGroup;
 244 #if ZMBOL
 245 #else
 246                 /* For zebra, delete lock file and reset register */
 247                 if (rGroupDef.flagRw)
 248                 {
 249                     zebraIndexUnlock();
 250                     bf_reset (rGroupDef.bfs);
 251                 }
 252 #endif
 253                 rval = res_get (common_resource, "shadow");
 254                 zebraIndexLock (rGroupDef.bfs, 0, rval);
 255                 if (rGroupDef.flagRw)
 256                 {
 257                     if (rval && *rval && !disableCommit)
 258                     {
 259                         bf_cache (rGroupDef.bfs, rval);
 260                         zebraIndexLockMsg ("r");
 261                     }
 262                     else
 263                     {
 264                         bf_cache (rGroupDef.bfs, 0);
 265                         zebraIndexLockMsg ("w");
 266                     }
 267                     zebraIndexWait (0);
 268                 }
 269                 memcpy (&rGroup, &rGroupDef, sizeof(rGroup));
 270                 rGroup.path = arg;
 271                 switch (cmd)
 272                 {
 273                 case 'u':
 274                     if (!key_open (&rGroup, mem_max))
 275                     {
 276                         logf (LOG_LOG, "updating %s", rGroup.path);
 277                         repositoryUpdate (&rGroup);
 278                         nsections = key_close (&rGroup);
 279                     }
 280                     break;
 281                 case 'U':
 282                     if (!key_open (&rGroup, mem_max))
 283                     {
 284                         logf (LOG_LOG, "updating (pass 1) %s", rGroup.path);
 285                         repositoryUpdate (&rGroup);
 286                         key_close (&rGroup);
 287                     }
 288                     nsections = 0;
 289                     break;
 290                 case 'd':
 291                     if (!key_open (&rGroup,mem_max))
 292                     {
 293                         logf (LOG_LOG, "deleting %s", rGroup.path);
 294                         repositoryDelete (&rGroup);
 295                         nsections = key_close (&rGroup);
 296                     }
 297                     break;
 298                 case 's':
 299                     logf (LOG_LOG, "dumping %s", rGroup.path);
 300                     repositoryShow (&rGroup);
 301                     nsections = 0;
 302                     break;
 303                 case 'm':
 304                     nsections = -1;
 305                     break;
 306                 default:
 307                     nsections = 0;
 308                 }
 309                 cmd = 0;
 310                 if (nsections)
 311                 {
 312                     logf (LOG_LOG, "merging with index");
 313                     key_input (rGroup.bfs, nsections, 60, common_resource);
 314 #ifndef WIN32
 315                     sync ();
 316 #endif
 317                 }
 318                 log_event_end (NULL, NULL);
 319             }
 320         }
 321         else if (ret == 'V')
 322         {
 323 #if ZMBOL
 324             fprintf (stderr, "Z'mbol %s %s\n", ZEBRAVER, ZEBRADATE);
 325 #else
 326             fprintf (stderr, "Zebra %s %s\n", ZEBRAVER, ZEBRADATE);
 327 #endif
 328             fprintf (stderr, " (C) 1994-2001, Index Data ApS\n");
 329 #ifdef WIN32
 330 #ifdef _DEBUG
 331             fprintf (stderr, " WIN32 Debug\n");
 332 #else
 333             fprintf (stderr, " WIN32 Release\n");
 334 #endif
 335 #endif
 336 #if HAVE_BZLIB_H
 337             fprintf (stderr, "libbzip2\n"
 338                      " (C) 1996-1999 Julian R Seward.  All rights reserved.\n");
 339 #endif
 340         }
 341         else if (ret == 'v')
 342             yaz_log_init_level (yaz_log_mask_str(arg));
 343         else if (ret == 'l')
 344             yaz_log_init_file (arg);
 345         else if (ret == 'm')
 346             mem_max = 1024*1024*atoi(arg);
 347         else if (ret == 'd')
 348             rGroupDef.databaseName = arg;
 349         else if (ret == 's')
 350             rGroupDef.flagRw = 0;
 351         else if (ret == 'g')
 352             rGroupDef.groupName = arg;
 353         else if (ret == 'f')
 354             rGroupDef.fileVerboseLimit = atoi(arg);
 355         else if (ret == 'c')
 356             configName = arg;
 357         else if (ret == 't')
 358             rGroupDef.recordType = arg;
 359         else if (ret == 'n')
 360             disableCommit = 1;
 361 #if ZEBRASDR
 362         else if (ret == 'S')
 363             rGroupDef.useSDR = 1;
 364 #endif
 365         else
 366             logf (LOG_WARN, "unknown option '-%s'", arg);
 367     }
 368     recTypes_destroy (rGroupDef.recTypes);
 369     if (common_resource)
 370     {
 371         zebraIndexUnlock ();
 372         bfs_destroy (rGroupDef.bfs);
 373     }
 374     data1_destroy (rGroupDef.dh);
 375     exit (0);
 376     return 0;
 377 }
 378