X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=src%2Flogic.c;h=2a0e9617a11cb9d1ffd08255ea2ff9b3748fe760;hb=315af5c601a268e2d0da60477c3470a6c12654c9;hp=1f61db6bd74dc2c626de6a7ad8da56b3fe6350bf;hpb=5cc5bfa026237076ecb44ae016b78069edfcc492;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index 1f61db6..2a0e961 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,4 +1,4 @@ -/* $Id: logic.c,v 1.2 2007-04-16 13:22:17 marc Exp $ +/* $Id: logic.c,v 1.19 2007-04-23 08:15:22 marc Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -19,6 +19,9 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ +// This file contains the primary business logic. Several parts of it should +// Eventually be factored into separate modules. + #include #include #include @@ -74,8 +77,6 @@ static int client_prep_connection(struct client *cl); static void ingest_records(struct client *cl, Z_Records *r); void session_alert_watch(struct session *s, int what); -IOCHAN channel_list = 0; // Master list of connections we're handling events to - static struct connection *connection_freelist = 0; static struct client *client_freelist = 0; @@ -103,7 +104,7 @@ struct parameters global_parameters = 0, 30, "81", - "Index Data PazPar2 (MasterKey)", + "Index Data PazPar2", VERSION, 600, // 10 minutes 60, @@ -185,10 +186,9 @@ static void send_init(IOCHAN i) && 0 < strlen(cl->database->database->url)) { #if YAZ_VERSIONL >= 0x020163 - const int *oid_proxy = yaz_string_to_oid(yaz_oid_std(), - CLASS_USERINFO, OID_STR_PROXY); yaz_oi_set_string_oid(&a->u.initRequest->otherInfo, - global_parameters.odr_out, oid_proxy, + global_parameters.odr_out, + yaz_oid_userinfo_proxy, 1, cl->database->database->url); #else yaz_oi_set_string_oidval(&a->u.initRequest->otherInfo, @@ -207,8 +207,13 @@ static void send_init(IOCHAN i) odr_reset(global_parameters.odr_out); } +// Recursively traverse query structure to extract terms. static void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) { + char **words; + int numwords; + int i; + switch (n->kind) { case CCL_RPN_AND: @@ -219,7 +224,9 @@ static void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int * pull_terms(nmem, n->u.p[1], termlist, num); break; case CCL_RPN_TERM: - termlist[(*num)++] = nmem_strdup(nmem, n->u.t.term); + nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords); + for (i = 0; i < numwords; i++) + termlist[(*num)++] = words[i]; break; default: // NOOP break; @@ -242,10 +249,9 @@ static void send_search(IOCHAN i) struct session *se = cl->session; struct session_database *sdb = cl->database; Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest); - int ndb, cerror, cpos; + int ndb; char **databaselist; Z_Query *zquery; - struct ccl_rpn_node *cn; int ssub = 0, lslb = 100000, mspn = 10; char *recsyn = 0; char *piggyback = 0; @@ -254,25 +260,11 @@ static void send_search(IOCHAN i) yaz_log(YLOG_DEBUG, "Sending search to %s", cl->database->database->url); - cn = ccl_find_str(sdb->database->ccl_map, se->query, &cerror, &cpos); - if (!cn) - return; - - if (!se->relevance) - { - // Initialize relevance structure with query terms - char *p[512]; - extract_terms(se->nmem, cn, p); - se->relevance = relevance_create(se->nmem, (const char **) p, - se->expected_maxrecs); - } - // constructing RPN query a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out, sizeof(Z_Query)); zquery->which = Z_Query_type_1; - zquery->u.type_1 = ccl_rpn_query(global_parameters.odr_out, cn); - ccl_rpn_delete(cn); + zquery->u.type_1 = p_query_rpn(global_parameters.odr_out, cl->pquery); // converting to target encoding if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){ @@ -517,8 +509,9 @@ static void add_facet(struct session *s, const char *type, const char *value) if (i == SESSION_MAX_TERMLISTS) { yaz_log(YLOG_FATAL, "Too many termlists"); - exit(1); + return; } + s->termlists[i].name = nmem_strdup(s->nmem, type); s->termlists[i].termlist = termlist_create(s->nmem, s->expected_maxrecs, 15); s->num_termlists = i + 1; @@ -529,36 +522,36 @@ static void add_facet(struct session *s, const char *type, const char *value) static xmlDoc *normalize_record(struct client *cl, Z_External *rec) { struct database_retrievalmap *m; - struct database *db = cl->database->database; + struct session_database *sdb = cl->database; + struct database *db = sdb->database; xmlNode *res; xmlDoc *rdoc; // First normalize to XML - if (db->yaz_marc) + if (sdb->yaz_marc) { char *buf; int len; if (rec->which != Z_External_octet) { yaz_log(YLOG_WARN, "Unexpected external branch, probably BER %s", - cl->database->database->url); + db->url); return 0; } buf = (char*) rec->u.octet_aligned->buf; len = rec->u.octet_aligned->len; - if (yaz_marc_read_iso2709(db->yaz_marc, buf, len) < 0) + if (yaz_marc_read_iso2709(sdb->yaz_marc, buf, len) < 0) { - yaz_log(YLOG_WARN, "Failed to decode MARC %s", - cl->database->database->url); + yaz_log(YLOG_WARN, "Failed to decode MARC %s", db->url); return 0; } - yaz_marc_write_using_libxml2(db->yaz_marc, 1); - if (yaz_marc_write_xml(db->yaz_marc, &res, + yaz_marc_write_using_libxml2(sdb->yaz_marc, 1); + if (yaz_marc_write_xml(sdb->yaz_marc, &res, "http://www.loc.gov/MARC21/slim", 0, 0) < 0) { yaz_log(YLOG_WARN, "Failed to encode as XML %s", - cl->database->database->url); + db->url); return 0; } rdoc = xmlNewDoc((xmlChar *) "1.0"); @@ -569,14 +562,14 @@ static xmlDoc *normalize_record(struct client *cl, Z_External *rec) { yaz_log(YLOG_FATAL, "Unknown native_syntax in normalize_record from %s", - cl->database->database->url); - exit(1); + db->url); + return 0; } if (global_parameters.dump_records){ fprintf(stderr, "Input Record (normalized) from %s\n----------------\n", - cl->database->database->url); + db->url); #if LIBXML_VERSION >= 20600 xmlDocFormatDump(stderr, rdoc, 1); #else @@ -584,7 +577,7 @@ static xmlDoc *normalize_record(struct client *cl, Z_External *rec) #endif } - for (m = db->map; m; m = m->next){ + for (m = sdb->map; m; m = m->next){ xmlDoc *new = 0; #if 1 @@ -700,7 +693,9 @@ static struct record *ingest_record(struct client *cl, Z_External *rec) xmlFree(mergekey); normalize_mergekey((char *) mergekey_norm, 0); - cluster = reclist_insert(global_parameters.server->service, se->reclist, res, (char *) mergekey_norm, + cluster = reclist_insert(se->reclist, + global_parameters.server->service, + res, (char *) mergekey_norm, &se->total_merged); if (global_parameters.dump_records) yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid, @@ -973,7 +968,7 @@ static void do_presentResponse(IOCHAN i, Z_APDU *a) } } -static void handler(IOCHAN i, int event) +void connection_handler(IOCHAN i, int event) { struct connection *co = iochan_getdata(i); struct client *cl = co->client; @@ -1080,7 +1075,7 @@ static void handler(IOCHAN i, int event) if (cl->state == Client_Idle) { - if (cl->requestid != se->requestid && *se->query) { + if (cl->requestid != se->requestid && cl->pquery) { send_search(i); } else if (cl->hits > 0 && cl->records < global_parameters.toget && @@ -1106,8 +1101,12 @@ static void connection_release(struct connection *co) static void connection_destroy(struct connection *co) { struct host *h = co->host; - cs_close(co->link); - iochan_destroy(co->iochan); + + if (co->link) + { + cs_close(co->link); + iochan_destroy(co->iochan); + } yaz_log(YLOG_DEBUG, "Connection destroy %s", co->host->hostport); if (h->connections == co) @@ -1132,54 +1131,103 @@ static void connection_destroy(struct connection *co) connection_freelist = co; } -// Creates a new connection for client, associated with the host of -// client's database -static struct connection *connection_create(struct client *cl) +static int connection_connect(struct connection *con) { - struct connection *new; - COMSTACK link; - int res; + COMSTACK link = 0; + struct client *cl = con->client; + struct host *host = con->host; void *addr; + int res; + assert(host->ipport); + assert(cl); if (!(link = cs_create(tcpip_type, 0, PROTO_Z3950))) - { - yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to create comstack"); - exit(1); - } + { + yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to create comstack"); + return -1; + } if (0 == strlen(global_parameters.zproxy_override)){ /* no Z39.50 proxy needed - direct connect */ yaz_log(YLOG_DEBUG, "Connection create %s", cl->database->database->url); - if (!(addr = cs_straddr(link, cl->database->database->host->ipport))) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, - "Lookup of IP address %s failed", - cl->database->database->host->ipport); - return 0; - } - + if (!(addr = cs_straddr(link, host->ipport))) + { + yaz_log(YLOG_WARN|YLOG_ERRNO, + "Lookup of IP address %s failed", host->ipport); + return -1; + } + } else { /* Z39.50 proxy connect */ yaz_log(YLOG_DEBUG, "Connection create %s proxy %s", cl->database->database->url, global_parameters.zproxy_override); - + if (!(addr = cs_straddr(link, global_parameters.zproxy_override))) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, - "Lookup of IP address %s failed", - global_parameters.zproxy_override); - return 0; - } + { + yaz_log(YLOG_WARN|YLOG_ERRNO, + "Lookup of IP address %s failed", + global_parameters.zproxy_override); + return -1; + } } - + res = cs_connect(link, addr); if (res < 0) { yaz_log(YLOG_WARN|YLOG_ERRNO, "cs_connect %s", cl->database->database->url); - return 0; + return -1; + } + con->link = link; + con->state = Conn_Connecting; + con->iochan = iochan_create(cs_fileno(link), connection_handler, 0); + iochan_setdata(con->iochan, con); + pazpar2_add_channel(con->iochan); + + /* this fragment is bad DRY: from client_prep_connection */ + cl->state = Client_Connecting; + iochan_setflag(con->iochan, EVENT_OUTPUT); + return 0; +} + +void connect_resolver_host(struct host *host) +{ + struct connection *con = host->connections; + + while (con) + { + if (con->state == Conn_Resolving) + { + if (!host->ipport) /* unresolved */ + { + connection_destroy(con); + /* start all over .. at some point it will be NULL */ + con = host->connections; + } + else if (!con->client) + { + yaz_log(YLOG_WARN, "connect_unresolved_host : ophan client"); + connection_destroy(con); + /* start all over .. at some point it will be NULL */ + con = host->connections; + } + else + { + connection_connect(con); + con = con->next; + } + } } +} + + +// Creates a new connection for client, associated with the host of +// client's database +static struct connection *connection_create(struct client *cl) +{ + struct connection *new; + struct host *host = cl->database->database->host; if ((new = connection_freelist)) connection_freelist = new->next; @@ -1189,18 +1237,15 @@ static struct connection *connection_create(struct client *cl) new->ibuf = 0; new->ibufsize = 0; } - new->state = Conn_Connecting; - new->host = cl->database->database->host; + new->host = host; new->next = new->host->connections; new->host->connections = new; new->client = cl; cl->connection = new; - new->link = link; - - new->iochan = iochan_create(cs_fileno(link), handler, 0); - iochan_setdata(new->iochan, new); - new->iochan->next = channel_list; - channel_list = new->iochan; + new->link = 0; + new->state = Conn_Resolving; + if (host->ipport) + connection_connect(new); return new; } @@ -1258,6 +1303,167 @@ static int client_prep_connection(struct client *cl) return 0; } +// Initialize YAZ Map structures for MARC-based targets +static int prepare_yazmarc(struct session_database *sdb) +{ + char *s; + + if (!sdb->settings) + { + yaz_log(YLOG_WARN, "No settings for %s", sdb->database->url); + return -1; + } + if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) && !strncmp(s, "iso2709", 7)) + { + char *encoding = "marc-8s", *e; + yaz_iconv_t cm; + + // See if a native encoding is specified + if ((e = strchr(s, ';'))) + encoding = e + 1; + + sdb->yaz_marc = yaz_marc_create(); + yaz_marc_subfield_str(sdb->yaz_marc, "\t"); + + cm = yaz_iconv_open("utf-8", encoding); + if (!cm) + { + yaz_log(YLOG_FATAL, + "Unable to map from %s to UTF-8 for target %s", + encoding, sdb->database->url); + return -1; + } + yaz_marc_iconv(sdb->yaz_marc, cm); + } + return 0; +} + +// Prepare XSLT stylesheets for record normalization +// Structures are allocated on the session_wide nmem to avoid having +// to recompute this for every search. This would lead +// to leaking if a single session was to repeatedly change the PZ_XSLT +// setting. However, this is not a realistic use scenario. +static int prepare_map(struct session *se, struct session_database *sdb) +{ + char *s; + + if (!sdb->settings) + { + yaz_log(YLOG_WARN, "No settings on %s", sdb->database->url); + return -1; + } + if ((s = session_setting_oneval(sdb, PZ_XSLT))) + { + char **stylesheets; + struct database_retrievalmap **m = &sdb->map; + int num, i; + + nmem_strsplit(se->session_nmem, ",", s, &stylesheets, &num); + for (i = 0; i < num; i++) + { + (*m) = nmem_malloc(se->session_nmem, sizeof(**m)); + (*m)->next = 0; + if (!((*m)->stylesheet = conf_load_stylesheet(stylesheets[i]))) + { + yaz_log(YLOG_FATAL, "Unable to load stylesheet: %s", + stylesheets[i]); + return -1; + } + m = &(*m)->next; + } + } + if (!sdb->map) + yaz_log(YLOG_WARN, "No Normalization stylesheet for target %s", + sdb->database->url); + return 0; +} + +// This analyzes settings and recomputes any supporting data structures +// if necessary. +static int prepare_session_database(struct session *se, struct session_database *sdb) +{ + if (!sdb->settings) + { + yaz_log(YLOG_WARN, "No settings associated with %s", sdb->database->url); + return -1; + } + if (sdb->settings[PZ_NATIVESYNTAX] && !sdb->yaz_marc) + { + if (prepare_yazmarc(sdb) < 0) + return -1; + } + if (sdb->settings[PZ_XSLT] && !sdb->map) + { + if (prepare_map(se, sdb) < 0) + return -1; + } + return 0; +} + +// Initialize CCL map for a target +static CCL_bibset prepare_cclmap(struct client *cl) +{ + struct session_database *sdb = cl->database; + struct setting *s; + CCL_bibset res; + + if (!sdb->settings) + return 0; + res = ccl_qual_mk(); + for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next) + { + char *p = strchr(s->name + 3, ':'); + if (!p) + { + yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name); + ccl_qual_rm(&res); + return 0; + } + p++; + ccl_qual_fitem(res, s->value, p); + } + return res; +} + +// Parse the query given the settings specific to this client +static int client_parse_query(struct client *cl, const char *query) +{ + struct session *se = cl->session; + struct ccl_rpn_node *cn; + int cerror, cpos; + CCL_bibset ccl_map = prepare_cclmap(cl); + + if (!ccl_map) + return -1; + cn = ccl_find_str(ccl_map, query, &cerror, &cpos); + ccl_qual_rm(&ccl_map); + if (!cn) + { + cl->state = Client_Error; + yaz_log(YLOG_WARN, "Failed to parse query for %s", + cl->database->database->url); + return -1; + } + wrbuf_rewind(se->wrbuf); + ccl_pquery(se->wrbuf, cn); + wrbuf_putc(se->wrbuf, '\0'); + if (cl->pquery) + xfree(cl->pquery); + cl->pquery = xstrdup(wrbuf_buf(se->wrbuf)); + + if (!se->relevance) + { + // Initialize relevance structure with query terms + char *p[512]; + extract_terms(se->nmem, cn, p); + se->relevance = relevance_create(se->nmem, (const char **) p, + se->expected_maxrecs); + } + + ccl_rpn_delete(cn); + return 0; +} + static struct client *client_create(void) { struct client *r; @@ -1268,6 +1474,7 @@ static struct client *client_create(void) } else r = xmalloc(sizeof(struct client)); + r->pquery = 0; r->database = 0; r->connection = 0; r->session = 0; @@ -1401,59 +1608,40 @@ char *search(struct session *se, char *query, char *filter) nmem_reset(se->nmem); criteria = parse_filter(se->nmem, filter); - strcpy(se->query, query); se->requestid++; - select_targets(se, criteria); - for (cl = se->clients; cl; cl = cl->next) - { - if (client_prep_connection(cl)) - live_channels++; - } + live_channels = select_targets(se, criteria); if (live_channels) { int maxrecs = live_channels * global_parameters.toget; se->num_termlists = 0; se->reclist = reclist_create(se->nmem, maxrecs); // This will be initialized in send_search() - se->relevance = 0; se->total_records = se->total_hits = se->total_merged = 0; se->expected_maxrecs = maxrecs; } else return "NOTARGETS"; - return 0; -} + se->relevance = 0; -// Apply a session override to a database -void session_apply_setting(struct session *se, char *dbname, char *setting, char *value) -{ - struct session_database *sdb; + for (cl = se->clients; cl; cl = cl->next) + { + if (prepare_session_database(se, cl->database) < 0) + return "CONFIG_ERROR"; + if (client_parse_query(cl, query) < 0) // Query must parse for all targets + return "QUERY"; + } - for (sdb = se->databases; sdb; sdb = sdb->next) - if (!strcmp(dbname, sdb->database->url)) - { - struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new)); - int offset = settings_offset(setting); + for (cl = se->clients; cl; cl = cl->next) + { + client_prep_connection(cl); + } - if (offset < 0) - { - yaz_log(YLOG_WARN, "Unknown setting %s", setting); - return; - } - new->precedence = 0; - new->target = dbname; - new->name = setting; - new->value = value; - new->next = sdb->settings[offset]; - sdb->settings[offset] = new; - break; - } - if (!sdb) - yaz_log(YLOG_WARN, "Unknown database in setting override: %s", dbname); + return 0; } -void session_init_databases_fun(void *context, struct database *db) +// Creates a new session_database object for a database +static void session_init_databases_fun(void *context, struct database *db) { struct session *se = (struct session *) context; struct session_database *new = nmem_malloc(se->session_nmem, sizeof(*new)); @@ -1461,13 +1649,30 @@ void session_init_databases_fun(void *context, struct database *db) int i; new->database = db; + new->yaz_marc = 0; + new->map = 0; new->settings = nmem_malloc(se->session_nmem, sizeof(struct settings *) * num); - for (i = 0; i < num; i++) - new->settings[i] = db->settings[i]; + memset(new->settings, 0, sizeof(struct settings*) * num); + if (db->settings) + { + for (i = 0; i < num; i++) + new->settings[i] = db->settings[i]; + } new->next = se->databases; se->databases = new; } +// Doesn't free memory associated with sdb -- nmem takes care of that +static void session_database_destroy(struct session_database *sdb) +{ + struct database_retrievalmap *m; + + for (m = sdb->map; m; m = m->next) + xsltFreeStylesheet(m->stylesheet); + if (sdb->yaz_marc) + yaz_marc_destroy(sdb->yaz_marc); +} + // Initialize session_database list -- this represents this session's view // of the database list -- subject to modification by the settings ws command void session_init_databases(struct session *se) @@ -1476,11 +1681,80 @@ void session_init_databases(struct session *se) grep_databases(se, 0, session_init_databases_fun); } +// Probably session_init_databases_fun should be refactored instead of +// called here. +static struct session_database *load_session_database(struct session *se, char *id) +{ + struct database *db = find_database(id, 0); + + session_init_databases_fun((void*) se, db); + // New sdb is head of se->databases list + return se->databases; +} + +// Find an existing session database. If not found, load it +static struct session_database *find_session_database(struct session *se, char *id) +{ + struct session_database *sdb; + + for (sdb = se->databases; sdb; sdb = sdb->next) + if (!strcmp(sdb->database->url, id)) + return sdb; + return load_session_database(se, id); +} + +// Apply a session override to a database +void session_apply_setting(struct session *se, char *dbname, char *setting, char *value) +{ + struct session_database *sdb = find_session_database(se, dbname); + struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new)); + int offset = settings_offset(setting); + + if (offset < 0) + { + yaz_log(YLOG_WARN, "Unknown setting %s", setting); + return; + } + new->precedence = 0; + new->target = dbname; + new->name = setting; + new->value = value; + new->next = sdb->settings[offset]; + sdb->settings[offset] = new; + + // Force later recompute of settings-driven data structures + // (happens when a search starts and client connections are prepared) + switch (offset) + { + case PZ_NATIVESYNTAX: + if (sdb->yaz_marc) + { + yaz_marc_destroy(sdb->yaz_marc); + sdb->yaz_marc = 0; + } + break; + case PZ_XSLT: + if (sdb->map) + { + struct database_retrievalmap *m; + // We don't worry about the map structure -- it's in nmem + for (m = sdb->map; m; m = m->next) + xsltFreeStylesheet(m->stylesheet); + sdb->map = 0; + } + break; + } +} + void destroy_session(struct session *s) { + struct session_database *sdb; + yaz_log(YLOG_LOG, "Destroying session"); while (s->clients) client_destroy(s->clients); + for (sdb = s->databases; sdb; sdb = sdb->next) + session_database_destroy(sdb); nmem_destroy(s->nmem); wrbuf_destroy(s->wrbuf); } @@ -1499,7 +1773,6 @@ struct session *new_session(NMEM nmem) session->requestid = -1; session->clients = 0; session->expected_maxrecs = 0; - session->query[0] = '\0'; session->session_nmem = nmem; session->nmem = nmem_create(); session->wrbuf = wrbuf_alloc(); @@ -1725,7 +1998,18 @@ void start_zproxy(void) return; } +// Master list of connections we're handling events to +static IOCHAN channel_list = 0; +void pazpar2_add_channel(IOCHAN chan) +{ + chan->next = channel_list; + channel_list = chan; +} +void pazpar2_event_loop() +{ + event_loop(&channel_list); +} /* * Local variables: