- }
- return rdoc;
-}
-
-// Extract what appears to be years from buf, storing highest and
-// lowest values.
-static int extract_years(const char *buf, int *first, int *last)
-{
- *first = -1;
- *last = -1;
- while (*buf)
- {
- const char *e;
- int len;
-
- while (*buf && !isdigit(*buf))
- buf++;
- len = 0;
- for (e = buf; *e && isdigit(*e); e++)
- len++;
- if (len == 4)
- {
- int value = atoi(buf);
- if (*first < 0 || value < *first)
- *first = value;
- if (*last < 0 || value > *last)
- *last = value;
- }
- buf = e;
- }
- return *first;
-}
-
-static struct record *ingest_record(struct client *cl, Z_External *rec)
-{
- xmlDoc *xdoc = normalize_record(cl, rec);
- xmlNode *root, *n;
- struct record *res;
- struct record_cluster *cluster;
- struct session *se = cl->session;
- xmlChar *mergekey, *mergekey_norm;
- xmlChar *type = 0;
- xmlChar *value = 0;
- struct conf_service *service = global_parameters.server->service;
-
- if (!xdoc)
- return 0;
-
- root = xmlDocGetRootElement(xdoc);
- if (!(mergekey = xmlGetProp(root, (xmlChar *) "mergekey")))
- {
- yaz_log(YLOG_WARN, "No mergekey found in record");
- xmlFreeDoc(xdoc);
- return 0;
- }
-
- res = nmem_malloc(se->nmem, sizeof(struct record));
- res->next = 0;
- res->client = cl;
- res->metadata = nmem_malloc(se->nmem,
- sizeof(struct record_metadata*) * service->num_metadata);
- memset(res->metadata, 0, sizeof(struct record_metadata*) * service->num_metadata);
-
- mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey);
- xmlFree(mergekey);
- normalize_mergekey((char *) mergekey_norm, 0);
-
- cluster = reclist_insert(se->reclist,
- global_parameters.server->service,
- res, (char *) mergekey_norm,
- &se->total_merged);
- if (global_parameters.dump_records)
- yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid,
- cl->database->database->url, cl->records);
- if (!cluster)
- {
- /* no room for record */
- xmlFreeDoc(xdoc);
- return 0;
- }
- relevance_newrec(se->relevance, cluster);
-
- for (n = root->children; n; n = n->next)
- {
- if (type)
- xmlFree(type);
- if (value)
- xmlFree(value);
- type = value = 0;
-
- if (n->type != XML_ELEMENT_NODE)
- continue;
- if (!strcmp((const char *) n->name, "metadata"))
- {
- struct conf_metadata *md = 0;
- struct conf_sortkey *sk = 0;
- struct record_metadata **wheretoput, *newm;
- int imeta;
- int first, last;
-
- type = xmlGetProp(n, (xmlChar *) "type");
- value = xmlNodeListGetString(xdoc, n->children, 0);
-
- if (!type || !value)
- continue;
-
- // First, find out what field we're looking at
- for (imeta = 0; imeta < service->num_metadata; imeta++)
- if (!strcmp((const char *) type, service->metadata[imeta].name))
- {
- md = &service->metadata[imeta];
- if (md->sortkey_offset >= 0)
- sk = &service->sortkeys[md->sortkey_offset];
- break;
- }
- if (!md)
- {
- yaz_log(YLOG_WARN, "Ignoring unknown metadata element: %s", type);
- continue;
- }
-
- // Find out where we are putting it
- if (md->merge == Metadata_merge_no)
- wheretoput = &res->metadata[imeta];
- else
- wheretoput = &cluster->metadata[imeta];
-
- // Put it there
- newm = nmem_malloc(se->nmem, sizeof(struct record_metadata));
- newm->next = 0;
- if (md->type == Metadata_type_generic)
- {
- char *p, *pe;
- for (p = (char *) value; *p && isspace(*p); p++)
- ;
- for (pe = p + strlen(p) - 1;
- pe > p && strchr(" ,/.:([", *pe); pe--)
- *pe = '\0';
- newm->data.text = nmem_strdup(se->nmem, p);
-
- }
- else if (md->type == Metadata_type_year)
- {
- if (extract_years((char *) value, &first, &last) < 0)
- continue;
- }
- else
- {
- yaz_log(YLOG_WARN, "Unknown type in metadata element %s", type);
- continue;
- }
- if (md->type == Metadata_type_year && md->merge != Metadata_merge_range)
- {
- yaz_log(YLOG_WARN, "Only range merging supported for years");
- continue;
- }
- if (md->merge == Metadata_merge_unique)
- {
- struct record_metadata *mnode;
- for (mnode = *wheretoput; mnode; mnode = mnode->next)
- if (!strcmp((const char *) mnode->data.text, newm->data.text))
- break;
- if (!mnode)
- {
- newm->next = *wheretoput;
- *wheretoput = newm;
- }
- }
- else if (md->merge == Metadata_merge_longest)
- {
- if (!*wheretoput ||
- strlen(newm->data.text) > strlen((*wheretoput)->data.text))
- {
- *wheretoput = newm;
- if (sk)
- {
- char *s = nmem_strdup(se->nmem, newm->data.text);
- if (!cluster->sortkeys[md->sortkey_offset])
- cluster->sortkeys[md->sortkey_offset] =
- nmem_malloc(se->nmem, sizeof(union data_types));
- normalize_mergekey(s,
- (sk->type == Metadata_sortkey_skiparticle));
- cluster->sortkeys[md->sortkey_offset]->text = s;
- }
- }
- }
- else if (md->merge == Metadata_merge_all || md->merge == Metadata_merge_no)
- {
- newm->next = *wheretoput;
- *wheretoput = newm;
- }
- else if (md->merge == Metadata_merge_range)
- {
- assert(md->type == Metadata_type_year);
- if (!*wheretoput)
- {
- *wheretoput = newm;
- (*wheretoput)->data.number.min = first;
- (*wheretoput)->data.number.max = last;
- if (sk)
- cluster->sortkeys[md->sortkey_offset] = &newm->data;
- }
- else
- {
- if (first < (*wheretoput)->data.number.min)
- (*wheretoput)->data.number.min = first;
- if (last > (*wheretoput)->data.number.max)
- (*wheretoput)->data.number.max = last;
- }
-#ifdef GAGA
- if (sk)
- {
- union data_types *sdata = cluster->sortkeys[md->sortkey_offset];
- yaz_log(YLOG_LOG, "SK range: %d-%d", sdata->number.min, sdata->number.max);
- }
-#endif
- }
- else
- yaz_log(YLOG_WARN, "Don't know how to merge on element name %s", md->name);
-
- if (md->rank)
- relevance_countwords(se->relevance, cluster,
- (char *) value, md->rank);
- if (md->termlist)
- {
- if (md->type == Metadata_type_year)
- {
- char year[64];
- sprintf(year, "%d", last);
- add_facet(se, (char *) type, year);
- if (first != last)
- {
- sprintf(year, "%d", first);
- add_facet(se, (char *) type, year);
- }
- }
- else
- add_facet(se, (char *) type, (char *) value);
- }
- xmlFree(type);
- xmlFree(value);
- type = value = 0;
- }
- else
- yaz_log(YLOG_WARN, "Unexpected element %s in internal record", n->name);
- }
- if (type)
- xmlFree(type);
- if (value)
- xmlFree(value);
-
- xmlFreeDoc(xdoc);
-
- relevance_donerecord(se->relevance, cluster);
- se->total_records++;
-
- return res;
-}
-
-// Retrieve first defined value for 'name' for given database.
-// Will be extended to take into account user associated with session
-char *session_setting_oneval(struct session_database *db, int offset)
-{
- if (!db->settings[offset])
- return "";
- return db->settings[offset]->value;
-}
-
-static void ingest_records(struct client *cl, Z_Records *r)
-{
-#if USE_TIMING
- yaz_timing_t t = yaz_timing_create();
-#endif
- struct record *rec;
- struct session *s = cl->session;
- Z_NamePlusRecordList *rlist;
- int i;
-
- if (r->which != Z_Records_DBOSD)
- return;
- rlist = r->u.databaseOrSurDiagnostics;
- for (i = 0; i < rlist->num_records; i++)
- {
- Z_NamePlusRecord *npr = rlist->records[i];
-
- cl->records++;
- if (npr->which != Z_NamePlusRecord_databaseRecord)
- {
- yaz_log(YLOG_WARN,
- "Unexpected record type, probably diagnostic %s",
- cl->database->database->url);
- continue;
- }
-
- rec = ingest_record(cl, npr->u.databaseRecord);
- if (!rec)
- continue;
- }
- if (s->watchlist[SESSION_WATCH_RECORDS].fun && rlist->num_records)
- session_alert_watch(s, SESSION_WATCH_RECORDS);
-
-#if USE_TIMING
- yaz_timing_stop(t);
- yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f",
- yaz_timing_get_real(t), yaz_timing_get_user(t),
- yaz_timing_get_sys(t));
- yaz_timing_destroy(&t);
-#endif
-}
-
-static void do_presentResponse(IOCHAN i, Z_APDU *a)
-{
- struct connection *co = iochan_getdata(i);
- struct client *cl = co->client;
- Z_PresentResponse *r = a->u.presentResponse;
-
- if (r->records) {
- Z_Records *recs = r->records;
- if (recs->which == Z_Records_NSD)
- {
- yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s",
- cl->database->database->url);
- cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
- cl->state = Client_Error;
- }
- }
-
- if (!*r->presentStatus && cl->state != Client_Error)
- {
- yaz_log(YLOG_DEBUG, "Good Present response %s",
- cl->database->database->url);
- ingest_records(cl, r->records);
- cl->state = Client_Idle;
- }
- else if (*r->presentStatus)
- {
- yaz_log(YLOG_WARN, "Bad Present response %s",
- cl->database->database->url);
- cl->state = Client_Error;
- }
-}
-
-void connection_handler(IOCHAN i, int event)
-{
- struct connection *co = iochan_getdata(i);
- struct client *cl = co->client;
- struct session *se = 0;
-
- if (cl)
- se = cl->session;
- else
- {
- yaz_log(YLOG_WARN, "Destroying orphan connection");
- connection_destroy(co);
- return;
- }
-
- if (co->state == Conn_Connecting && event & EVENT_OUTPUT)
- {
- int errcode;
- socklen_t errlen = sizeof(errcode);
-
- if (getsockopt(cs_fileno(co->link), SOL_SOCKET, SO_ERROR, &errcode,
- &errlen) < 0 || errcode != 0)
- {
- client_fatal(cl);
- return;
- }
- else
- {
- yaz_log(YLOG_DEBUG, "Connect OK");
- co->state = Conn_Open;
- if (cl)
- cl->state = Client_Connected;
- }
- }
-
- else if (event & EVENT_INPUT)
- {
- int len = cs_get(co->link, &co->ibuf, &co->ibufsize);
-
- if (len < 0)
- {
- yaz_log(YLOG_WARN|YLOG_ERRNO, "Error reading from %s",
- cl->database->database->url);
- connection_destroy(co);
- return;
- }
- else if (len == 0)
- {
- yaz_log(YLOG_WARN, "EOF reading from %s", cl->database->database->url);
- connection_destroy(co);
- return;
- }
- else if (len > 1) // We discard input if we have no connection
- {
- co->state = Conn_Open;
-
- if (cl && (cl->requestid == se->requestid || cl->state == Client_Initializing))
- {
- Z_APDU *a;
-
- odr_reset(global_parameters.odr_in);
- odr_setbuf(global_parameters.odr_in, co->ibuf, len, 0);
- if (!z_APDU(global_parameters.odr_in, &a, 0, 0))
- {
- client_fatal(cl);
- return;
- }
- switch (a->which)
- {
- case Z_APDU_initResponse:
- do_initResponse(i, a);
- break;
- case Z_APDU_searchResponse:
- do_searchResponse(i, a);
- break;
- case Z_APDU_presentResponse:
- do_presentResponse(i, a);
- break;
- case Z_APDU_close:
- do_closeResponse(i, a);
- break;
- default:
- yaz_log(YLOG_WARN,
- "Unexpected Z39.50 response from %s",
- cl->database->database->url);
- client_fatal(cl);
- return;
- }
- // We aren't expecting staggered output from target
- // if (cs_more(t->link))
- // iochan_setevent(i, EVENT_INPUT);
- }
- else // we throw away response and go to idle mode
- {
- yaz_log(YLOG_DEBUG, "Ignoring result of expired operation");
- cl->state = Client_Idle;
- }
- }
- /* if len==1 we do nothing but wait for more input */
- }
-
- if (cl->state == Client_Connected) {
- send_init(i);
- }
-
- if (cl->state == Client_Idle)
- {
- if (cl->requestid != se->requestid && cl->pquery) {
- send_search(i);
- }
- else if (cl->hits > 0 && cl->records < global_parameters.toget &&
- cl->records < cl->hits) {
- send_present(i);
- }
- }
-}
-
-// Disassociate connection from client
-static void connection_release(struct connection *co)
-{
- struct client *cl = co->client;
-
- yaz_log(YLOG_DEBUG, "Connection release %s", co->host->hostport);
- if (!cl)
- return;
- cl->connection = 0;
- co->client = 0;
-}
-
-// Close connection and recycle structure
-static void connection_destroy(struct connection *co)
-{
- struct host *h = co->host;
- cs_close(co->link);
- iochan_destroy(co->iochan);
-
- yaz_log(YLOG_DEBUG, "Connection destroy %s", co->host->hostport);
- if (h->connections == co)
- h->connections = co->next;
- else
- {
- struct connection *pco;
- for (pco = h->connections; pco && pco->next != co; pco = pco->next)
- ;
- if (pco)
- pco->next = co->next;
- else
- abort();
- }
- if (co->client)
- {
- if (co->client->state != Client_Idle)
- co->client->state = Client_Disconnected;
- co->client->connection = 0;
- }
- co->next = connection_freelist;
- connection_freelist = co;
-}
-
-
-// Creates a new connection for client, associated with the host of
-// client's database
-static struct connection *connection_create(struct client *cl)
-{
- struct connection *new;
- COMSTACK link;
- int res;
- void *addr;
- struct host *host = cl->database->database->host;
-
- if (!host->ipport)
- {
- yaz_log(YLOG_WARN, "Not yet resolved: %s",
- cl->database->database->url);
- return 0;
- }
-
- if (!(link = cs_create(tcpip_type, 0, PROTO_Z3950)))
- {
- yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to create comstack");
- exit(1);
- }
-
- if (0 == strlen(global_parameters.zproxy_override)){
- /* no Z39.50 proxy needed - direct connect */
- yaz_log(YLOG_DEBUG, "Connection create %s", cl->database->database->url);
-
- if (!(addr = cs_straddr(link, host->ipport)))
- {
- yaz_log(YLOG_WARN|YLOG_ERRNO,
- "Lookup of IP address %s failed", host->ipport);
- return 0;
- }
-
- } else {
- /* Z39.50 proxy connect */
- yaz_log(YLOG_DEBUG, "Connection create %s proxy %s",
- cl->database->database->url, global_parameters.zproxy_override);
-
- if (!(addr = cs_straddr(link, global_parameters.zproxy_override)))
- {
- yaz_log(YLOG_WARN|YLOG_ERRNO,
- "Lookup of IP address %s failed",
- global_parameters.zproxy_override);
- return 0;
- }
- }
-
- res = cs_connect(link, addr);
- if (res < 0)
- {
- yaz_log(YLOG_WARN|YLOG_ERRNO, "cs_connect %s", cl->database->database->url);
- return 0;
- }
-
- if ((new = connection_freelist))
- connection_freelist = new->next;
- else
- {
- new = xmalloc(sizeof (struct connection));
- new->ibuf = 0;
- new->ibufsize = 0;
- }
- new->state = Conn_Connecting;
- new->host = host;
- new->next = new->host->connections;
- new->host->connections = new;
- new->client = cl;
- cl->connection = new;
- new->link = link;
-
- new->iochan = iochan_create(cs_fileno(link), connection_handler, 0);
- iochan_setdata(new->iochan, new);
- pazpar2_add_channel(new->iochan);
- return new;
-}
-
-// Close connection and set state to error
-static void client_fatal(struct client *cl)
-{
- yaz_log(YLOG_WARN, "Fatal error from %s", cl->database->database->url);
- connection_destroy(cl->connection);
- cl->state = Client_Error;
-}
-
-// Ensure that client has a connection associated
-static int client_prep_connection(struct client *cl)
-{
- struct connection *co;
- struct session *se = cl->session;
- struct host *host = cl->database->database->host;
-
- co = cl->connection;
-
- yaz_log(YLOG_DEBUG, "Client prep %s", cl->database->database->url);
-
- if (!co)
- {
- // See if someone else has an idle connection
- // We should look at timestamps here to select the longest-idle connection
- for (co = host->connections; co; co = co->next)
- if (co->state == Conn_Open && (!co->client || co->client->session != se))
- break;
- if (co)
- {
- connection_release(co);
- cl->connection = co;
- co->client = cl;
- }
- else
- co = connection_create(cl);
- }
- if (co)
- {
- if (co->state == Conn_Connecting)
- {
- cl->state = Client_Connecting;
- iochan_setflag(co->iochan, EVENT_OUTPUT);
- }
- else if (co->state == Conn_Open)
- {
- if (cl->state == Client_Error || cl->state == Client_Disconnected)
- cl->state = Client_Idle;
- iochan_setflag(co->iochan, EVENT_OUTPUT);
- }
- return 1;
- }
- else
- return 0;