Use CONNECT for SSL backends and for Z39.50 thru HTTP proxy YAZ-825
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri);
54
55 void ZOOM_set_dset_error(ZOOM_connection c, int error,
56                          const char *dset,
57                          const char *addinfo, const char *addinfo2)
58 {
59     char *cp;
60
61     xfree(c->addinfo);
62     c->addinfo = 0;
63     c->error = error;
64     if (!c->diagset || strcmp(dset, c->diagset))
65     {
66         xfree(c->diagset);
67         c->diagset = xstrdup(dset);
68         /* remove integer part from SRW diagset .. */
69         if ((cp = strrchr(c->diagset, '/')))
70             *cp = '\0';
71     }
72     if (addinfo && addinfo2)
73     {
74         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
75         strcpy(c->addinfo, addinfo);
76         strcat(c->addinfo, ": ");
77         strcat(c->addinfo, addinfo2);
78     }
79     else if (addinfo)
80         c->addinfo = xstrdup(addinfo);
81     if (error != ZOOM_ERROR_NONE)
82     {
83         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
84                 c, c->host_port ? c->host_port : "<>", dset, error,
85                 addinfo ? addinfo : "",
86                 addinfo2 ? addinfo2 : "");
87         ZOOM_connection_remove_tasks(c);
88     }
89 }
90
91 int ZOOM_uri_to_code(const char *uri)
92 {
93     int code = 0;
94     const char *cp;
95     if ((cp = strrchr(uri, '/')))
96         code = atoi(cp+1);
97     return code;
98 }
99
100 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
101 {
102     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
103 }
104
105 static void clear_error(ZOOM_connection c)
106 {
107     /*
108      * If an error is tied to an operation then it's ok to clear: for
109      * example, a diagnostic returned from a search is cleared by a
110      * subsequent search.  However, problems such as Connection Lost
111      * or Init Refused are not cleared, because they are not
112      * recoverable: doing another search doesn't help.
113      */
114
115     ZOOM_connection_remove_events(c);
116     switch (c->error)
117     {
118     case ZOOM_ERROR_CONNECT:
119     case ZOOM_ERROR_MEMORY:
120     case ZOOM_ERROR_DECODE:
121     case ZOOM_ERROR_CONNECTION_LOST:
122     case ZOOM_ERROR_INIT:
123     case ZOOM_ERROR_INTERNAL:
124     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
125         break;
126     default:
127         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
128     }
129 }
130
131 void ZOOM_connection_show_task(ZOOM_task task)
132 {
133     switch(task->which)
134     {
135     case ZOOM_TASK_SEARCH:
136         yaz_log(YLOG_LOG, "search p=%p", task);
137         break;
138     case ZOOM_TASK_CONNECT:
139         yaz_log(YLOG_LOG, "connect p=%p", task);
140         break;
141     case ZOOM_TASK_SCAN:
142         yaz_log(YLOG_LOG, "scan p=%p", task);
143         break;
144     }
145 }
146
147 void ZOOM_connection_show_tasks(ZOOM_connection c)
148 {
149     ZOOM_task task;
150     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
151     for (task = c->tasks; task; task = task->next)
152         ZOOM_connection_show_task(task);
153 }
154
155 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
156 {
157     ZOOM_task *taskp = &c->tasks;
158     while (*taskp)
159         taskp = &(*taskp)->next;
160     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
161     (*taskp)->running = 0;
162     (*taskp)->which = which;
163     (*taskp)->next = 0;
164     clear_error(c);
165     return *taskp;
166 }
167
168 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
169 {
170     return c->tasks ? 0 : 1;
171 }
172
173 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
174 {
175     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
176
177     task->next = c->tasks;
178     c->tasks = task;
179
180     task->running = 0;
181     task->which = which;
182     return task;
183 }
184
185 void ZOOM_connection_remove_task(ZOOM_connection c)
186 {
187     ZOOM_task task = c->tasks;
188
189     if (task)
190     {
191         c->tasks = task->next;
192         switch (task->which)
193         {
194         case ZOOM_TASK_SEARCH:
195             resultset_destroy(task->u.search.resultset);
196             xfree(task->u.search.syntax);
197             xfree(task->u.search.elementSetName);
198             xfree(task->u.search.schema);
199             break;
200         case ZOOM_TASK_CONNECT:
201             break;
202         case ZOOM_TASK_SCAN:
203             ZOOM_scanset_destroy(task->u.scan.scan);
204             break;
205         case ZOOM_TASK_PACKAGE:
206             ZOOM_package_destroy(task->u.package);
207             break;
208         case ZOOM_TASK_SORT:
209             resultset_destroy(task->u.sort.resultset);
210             ZOOM_query_destroy(task->u.sort.q);
211             break;
212         default:
213             assert(0);
214         }
215         xfree(task);
216
217         if (!c->tasks)
218         {
219             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
220             ZOOM_connection_put_event(c, event);
221         }
222     }
223 }
224
225 void ZOOM_connection_remove_tasks(ZOOM_connection c)
226 {
227     while (c->tasks)
228         ZOOM_connection_remove_task(c);
229 }
230
231 static void odr_wrbuf_write(ODR o, void *handle, int type,
232                             const char *buf, int len)
233 {
234     WRBUF w = (WRBUF) handle;
235     wrbuf_write(w, buf, len);
236 }
237
238 ZOOM_API(ZOOM_connection)
239     ZOOM_connection_create(ZOOM_options options)
240 {
241     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
242
243     initlog();
244
245     c->log_api = log_api0;
246     c->log_details = log_details0;
247
248     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
249
250     c->proto = PROTO_Z3950;
251     c->cs = 0;
252     ZOOM_connection_set_mask(c, 0);
253     c->reconnect_ok = 0;
254     c->state = STATE_IDLE;
255     c->addinfo = 0;
256     c->diagset = 0;
257     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
258     c->buf_in = 0;
259     c->len_in = 0;
260     c->buf_out = 0;
261     c->len_out = 0;
262     c->resultsets = 0;
263
264     c->options = ZOOM_options_create_with_parent(options);
265
266     c->host_port = 0;
267     c->proxy = 0;
268     c->tproxy = 0;
269     c->proxy_mode = 0;
270
271     c->charset = c->lang = 0;
272
273     c->cookie_out = 0;
274     c->cookie_in = 0;
275     c->client_IP = 0;
276     c->tasks = 0;
277
278     c->user = 0;
279     c->group = 0;
280     c->password = 0;
281     c->url_authentication = 0;
282
283     c->maximum_record_size = 0;
284     c->preferred_message_size = 0;
285
286     c->odr_in = odr_createmem(ODR_DECODE);
287     c->odr_out = odr_createmem(ODR_ENCODE);
288     c->odr_print = 0;
289     c->odr_save = 0;
290
291     c->async = 0;
292     c->support_named_resultsets = 0;
293     c->last_event = ZOOM_EVENT_NONE;
294
295     c->m_queue_front = 0;
296     c->m_queue_back = 0;
297
298     c->sru_version = 0;
299     c->no_redirects = 0;
300     c->cookies = 0;
301     c->location = 0;
302     c->saveAPDU_wrbuf = 0;
303
304     ZOOM_memcached_init(c);
305     return c;
306 }
307
308 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
309 {
310     if (c->odr_save)
311     {
312         odr_destroy(c->odr_save);
313         c->odr_save = 0;
314     }
315     if (w)
316     {
317         c->odr_save = odr_createmem(ODR_PRINT);
318         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
319     }
320 }
321
322 /* set database names. Take local databases (if set); otherwise
323    take databases given in ZURL (if set); otherwise use Default */
324 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
325                                      int *num, ODR odr)
326 {
327     char **databaseNames;
328     const char *cp = ZOOM_options_get(options, "databaseName");
329
330     if ((!cp || !*cp) && con->host_port)
331         cs_get_host_args(con->host_port, &cp);
332     if (!cp || !*cp)
333         cp = "Default";
334     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
335     return databaseNames;
336 }
337
338 ZOOM_API(ZOOM_connection)
339     ZOOM_connection_new(const char *host, int portnum)
340 {
341     ZOOM_connection c = ZOOM_connection_create(0);
342
343     ZOOM_connection_connect(c, host, portnum);
344     return c;
345 }
346
347 static zoom_sru_mode get_sru_mode_from_string(const char *s)
348 {
349     if (!s || !*s)
350         return zoom_sru_soap;
351     if (!yaz_matchstr(s, "soap"))
352         return zoom_sru_soap;
353     else if (!yaz_matchstr(s, "get"))
354         return zoom_sru_get;
355     else if (!yaz_matchstr(s, "post"))
356         return zoom_sru_post;
357     else if (!yaz_matchstr(s, "solr"))
358         return zoom_sru_solr;
359     return zoom_sru_error;
360 }
361
362 ZOOM_API(void)
363     ZOOM_connection_connect(ZOOM_connection c,
364                             const char *host, int portnum)
365 {
366     const char *val;
367
368     initlog();
369
370     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
371             c, host ? host : "null", portnum);
372
373     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
374     ZOOM_connection_remove_tasks(c);
375
376     if (c->cs)
377     {
378         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
379         c->reconnect_ok = 1;
380         return;
381     }
382     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
383     xfree(c->proxy);
384     c->proxy = 0;
385     val = ZOOM_options_get(c->options, "proxy");
386     if (val && *val)
387     {
388         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
389         c->proxy = xstrdup(val);
390     }
391
392     xfree(c->tproxy);
393     c->tproxy = 0;
394     val = ZOOM_options_get(c->options, "tproxy");
395     if (val && *val)
396     {
397         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
398         c->tproxy = xstrdup(val);
399     }
400
401     xfree(c->charset);
402     c->charset = 0;
403     val = ZOOM_options_get(c->options, "charset");
404     if (val && *val)
405     {
406         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
407         c->charset = xstrdup(val);
408     }
409
410     xfree(c->lang);
411     val = ZOOM_options_get(c->options, "lang");
412     if (val && *val)
413     {
414         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
415         c->lang = xstrdup(val);
416     }
417     else
418         c->lang = 0;
419
420     if (host)
421     {
422         char hostn[128];
423         const char *http_lead;
424
425         val = ZOOM_options_get(c->options, "sru");
426         if (val && *val && !strstr(host, "://"))
427             http_lead = "http://";
428         else
429             http_lead = "";
430         c->sru_mode = get_sru_mode_from_string(val);
431
432         xfree(c->host_port);
433         if (portnum)
434         {
435             sprintf(hostn, "%.80s:%d", host, portnum);
436             host = hostn;
437         }
438         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
439         strcpy(c->host_port, http_lead);
440         strcat(c->host_port, host);
441     }
442
443     {
444         /*
445          * If the "<scheme>:" part of the host string is preceded by one
446          * or more comma-separated <name>=<value> pairs, these are taken
447          * to be options to be set on the connection object.  Among other
448          * applications, this facility can be used to embed authentication
449          * in a host string:
450          *          user=admin,password=secret,tcp:localhost:9999
451          */
452         char *remainder = c->host_port;
453         char *pcolon = strchr(remainder, ':');
454         char *pcomma;
455         char *pequals;
456         while ((pcomma = strchr(remainder, ',')) != 0 &&
457                (pcolon == 0 || pcomma < pcolon))
458         {
459             *pcomma = '\0';
460             if ((pequals = strchr(remainder, '=')) != 0)
461             {
462                 *pequals = '\0';
463                 ZOOM_connection_option_set(c, remainder, pequals+1);
464             }
465             remainder = pcomma+1;
466         }
467
468         if (remainder != c->host_port)
469         {
470             remainder = xstrdup(remainder);
471             xfree(c->host_port);
472             c->host_port = remainder;
473         }
474     }
475
476     xfree(c->sru_version);
477     val = ZOOM_options_get(c->options, "sru_version");
478     c->sru_version = xstrdup(val ? val : "1.2");
479
480     ZOOM_options_set(c->options, "host", c->host_port);
481
482     xfree(c->cookie_out);
483     c->cookie_out = 0;
484     val = ZOOM_options_get(c->options, "cookie");
485     if (val && *val)
486     {
487         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
488         c->cookie_out = xstrdup(val);
489     }
490
491     xfree(c->client_IP);
492     c->client_IP = 0;
493     val = ZOOM_options_get(c->options, "clientIP");
494     if (val && *val)
495     {
496         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
497                 c, val);
498         c->client_IP = xstrdup(val);
499     }
500
501     xfree(c->group);
502     c->group = 0;
503     val = ZOOM_options_get(c->options, "group");
504     if (val && *val)
505         c->group = xstrdup(val);
506
507     xfree(c->user);
508     c->user = 0;
509     val = ZOOM_options_get(c->options, "user");
510     if (val && *val)
511         c->user = xstrdup(val);
512
513     xfree(c->password);
514     c->password = 0;
515     val = ZOOM_options_get(c->options, "password");
516     if (!val)
517         val = ZOOM_options_get(c->options, "pass");
518     if (val && *val)
519         c->password = xstrdup(val);
520
521     val = ZOOM_options_get(c->options, "authenticationMode");
522     if (val && !strcmp(val, "url"))
523         c->url_authentication = 1;
524     else
525         c->url_authentication = 0;
526
527     c->maximum_record_size =
528         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
529     c->preferred_message_size =
530         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
531
532     c->async = ZOOM_options_get_bool(c->options, "async", 0);
533
534     yaz_cookies_destroy(c->cookies);
535     c->cookies = yaz_cookies_create();
536
537     if (ZOOM_memcached_configure(c))
538     {
539         ZOOM_connection_remove_tasks(c);
540         return;
541     }
542     if (c->sru_mode == zoom_sru_error)
543     {
544         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
545         ZOOM_connection_remove_tasks(c);
546         return;
547     }
548
549     if (c->odr_print)
550         odr_destroy(c->odr_print);
551     c->odr_print = 0;
552     val = ZOOM_options_get(c->options, "apdufile");
553     if (val)
554     {
555         c->odr_print = odr_createmem(ODR_PRINT);
556         if (strcmp(val, "-"))
557         {
558             FILE *f = fopen(val, "a");
559             if (!f)
560             {
561                 WRBUF w = wrbuf_alloc();
562                 wrbuf_printf(w, "fopen: %s", val);
563                 ZOOM_set_error(c, ZOOM_ERROR_INTERNAL, wrbuf_cstr(w));
564                 wrbuf_destroy(w);
565                 return;
566             }
567             odr_setprint(c->odr_print, f);
568         }
569     }
570     else if (ZOOM_options_get_bool(c->options, "apdulog", 0))
571     {
572         c->odr_print = odr_createmem(ODR_PRINT);
573         odr_setprint_noclose(c->odr_print, yaz_log_file());
574     }
575
576     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
577     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
578
579     if (!c->async)
580     {
581         while (ZOOM_event(1, &c))
582             ;
583     }
584 }
585
586 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
587 {
588     if (r->connection)
589     {
590         /* remove ourselves from the resultsets in connection */
591         ZOOM_resultset *rp = &r->connection->resultsets;
592         while (1)
593         {
594             assert(*rp);   /* we must be in this list!! */
595             if (*rp == r)
596             {   /* OK, we're here - take us out of it */
597                 *rp = (*rp)->next;
598                 break;
599             }
600             rp = &(*rp)->next;
601         }
602         r->connection = 0;
603     }
604 }
605
606 ZOOM_API(void)
607     ZOOM_connection_destroy(ZOOM_connection c)
608 {
609     ZOOM_resultset r;
610     if (!c)
611         return;
612     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
613
614     ZOOM_memcached_destroy(c);
615     if (c->cs)
616         cs_close(c->cs);
617
618     for (r = c->resultsets; r; r = r->next)
619         r->connection = 0;
620
621     xfree(c->buf_in);
622     xfree(c->addinfo);
623     xfree(c->diagset);
624     odr_destroy(c->odr_in);
625     odr_destroy(c->odr_out);
626     if (c->odr_save)
627         odr_destroy(c->odr_save);
628     if (c->odr_print)
629         odr_destroy(c->odr_print);
630     ZOOM_options_destroy(c->options);
631     ZOOM_connection_remove_tasks(c);
632     ZOOM_connection_remove_events(c);
633     xfree(c->host_port);
634     xfree(c->proxy);
635     xfree(c->tproxy);
636     xfree(c->charset);
637     xfree(c->lang);
638     xfree(c->cookie_out);
639     xfree(c->cookie_in);
640     xfree(c->client_IP);
641     xfree(c->user);
642     xfree(c->group);
643     xfree(c->password);
644     xfree(c->sru_version);
645     xfree(c->location);
646     yaz_cookies_destroy(c->cookies);
647     wrbuf_destroy(c->saveAPDU_wrbuf);
648     xfree(c);
649 }
650
651 void ZOOM_resultset_addref(ZOOM_resultset r)
652 {
653     if (r)
654     {
655         yaz_mutex_enter(r->mutex);
656         (r->refcount)++;
657         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
658                 r, r->refcount);
659         yaz_mutex_leave(r->mutex);
660     }
661 }
662
663 static int g_resultsets = 0;
664 static YAZ_MUTEX g_resultset_mutex = 0;
665
666 /* TODO We need to initialize this before running threaded:
667  * call resultset_use(0)  */
668
669 static int resultset_use(int delta) {
670     int resultset_count;
671     if (g_resultset_mutex == 0)
672         yaz_mutex_create(&g_resultset_mutex);
673     yaz_mutex_enter(g_resultset_mutex);
674     g_resultsets += delta;
675     resultset_count = g_resultsets;
676     yaz_mutex_leave(g_resultset_mutex);
677     return resultset_count;
678 }
679
680 int resultsets_count(void) {
681     return resultset_use(0);
682 }
683
684 ZOOM_resultset ZOOM_resultset_create(void)
685 {
686     int i;
687     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
688
689     initlog();
690
691     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
692     r->refcount = 1;
693     r->size = 0;
694     r->odr = odr_createmem(ODR_DECODE);
695     r->piggyback = 1;
696     r->setname = 0;
697     r->step = 0;
698     for (i = 0; i<RECORD_HASH_SIZE; i++)
699         r->record_hash[i] = 0;
700     r->r_sort_spec = 0;
701     r->query = 0;
702     r->connection = 0;
703     r->databaseNames = 0;
704     r->num_databaseNames = 0;
705     r->req_facets = 0;
706     r->res_facets = 0;
707     r->num_res_facets = 0;
708     r->facets_names = 0;
709     r->mutex = 0;
710     yaz_mutex_create(&r->mutex);
711 #if SHPTR
712     {
713         WRBUF w = wrbuf_alloc();
714         YAZ_SHPTR_INIT(r->record_wrbuf, w);
715     }
716 #endif
717     resultset_use(1);
718     r->mc_key = 0;
719     r->live_set = 0;
720     return r;
721 }
722
723 ZOOM_API(ZOOM_resultset)
724     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
725 {
726     ZOOM_resultset r;
727     ZOOM_query s = ZOOM_query_create();
728
729     ZOOM_query_prefix(s, q);
730
731     r = ZOOM_connection_search(c, s);
732     ZOOM_query_destroy(s);
733     return r;
734 }
735
736 ZOOM_API(ZOOM_resultset)
737     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
738 {
739     ZOOM_resultset r = ZOOM_resultset_create();
740     ZOOM_task task;
741     int start, count;
742     const char *syntax, *elementSetName, *schema;
743     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
744     r->r_sort_spec = ZOOM_query_get_sortspec(q);
745     r->query = q;
746     ZOOM_query_addref(q);
747
748     r->options = ZOOM_options_create_with_parent(c->options);
749
750     r->req_facets = odr_strdup_null(r->odr,
751                                     ZOOM_options_get(r->options, "facets"));
752     start = ZOOM_options_get_int(r->options, "start", 0);
753     count = ZOOM_options_get_int(r->options, "count", 0);
754     {
755         /* If "presentChunk" is defined use that; otherwise "step" */
756         const char *cp = ZOOM_options_get(r->options, "presentChunk");
757         r->step = ZOOM_options_get_int(r->options,
758                                        (cp != 0 ? "presentChunk": "step"), 0);
759     }
760     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
761     r->setname = odr_strdup_null(r->odr,
762                                  ZOOM_options_get(r->options, "setname"));
763     r->databaseNames = ZOOM_connection_get_databases(c, c->options,
764                                                      &r->num_databaseNames,
765                                                      r->odr);
766     r->connection = c;
767     r->next = c->resultsets;
768     c->resultsets = r;
769
770     ZOOM_memcached_resultset(r, q);
771
772     if (c->host_port && c->proto == PROTO_HTTP)
773     {
774         if (!c->cs)
775         {
776             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
777             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
778         }
779         else
780         {
781             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
782             c->reconnect_ok = 1;
783         }
784     }
785
786     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
787     task->u.search.resultset = r;
788     task->u.search.start = start;
789     task->u.search.count = count;
790
791     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
792     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
793     elementSetName = ZOOM_options_get(r->options, "elementSetName");
794     task->u.search.elementSetName = elementSetName ?
795         xstrdup(elementSetName) : 0;
796     schema = ZOOM_options_get(r->options, "schema");
797     task->u.search.schema = schema ? xstrdup(schema) : 0;
798
799     ZOOM_resultset_addref(r);
800
801     if (!c->async)
802     {
803         while (ZOOM_event(1, &c))
804             ;
805     }
806     return r;
807 }
808
809 ZOOM_API(void)
810     ZOOM_resultset_sort(ZOOM_resultset r,
811                          const char *sort_type, const char *sort_spec)
812 {
813     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
814 }
815
816 ZOOM_API(int)
817     ZOOM_resultset_sort1(ZOOM_resultset r,
818                          const char *sort_type, const char *sort_spec)
819 {
820     ZOOM_connection c = r->connection;
821     ZOOM_task task;
822     ZOOM_query newq;
823
824     newq = ZOOM_query_create();
825     if (ZOOM_query_sortby(newq, sort_spec) < 0)
826         return -1;
827
828     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
829             r, r, sort_type, sort_spec);
830     if (!c)
831         return 0;
832
833     if (c->host_port && c->proto == PROTO_HTTP)
834     {
835         if (!c->cs)
836         {
837             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
838             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
839         }
840         else
841         {
842             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
843                     r);
844             c->reconnect_ok = 1;
845         }
846     }
847
848     ZOOM_resultset_cache_reset(r);
849     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
850     task->u.sort.resultset = r;
851     task->u.sort.q = newq;
852
853     ZOOM_resultset_addref(r);
854
855     if (!c->async)
856     {
857         while (ZOOM_event(1, &c))
858             ;
859     }
860
861     return 0;
862 }
863
864 ZOOM_API(void)
865     ZOOM_resultset_destroy(ZOOM_resultset r)
866 {
867     resultset_destroy(r);
868 }
869
870 static void resultset_destroy(ZOOM_resultset r)
871 {
872     if (!r)
873         return;
874     yaz_mutex_enter(r->mutex);
875     (r->refcount)--;
876     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
877             r, r, r->refcount);
878     if (r->refcount == 0)
879     {
880         yaz_mutex_leave(r->mutex);
881
882         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
883         ZOOM_resultset_cache_reset(r);
884         ZOOM_resultset_release(r);
885         ZOOM_query_destroy(r->query);
886         ZOOM_options_destroy(r->options);
887         odr_destroy(r->odr);
888         yaz_mutex_destroy(&r->mutex);
889 #if SHPTR
890         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
891 #endif
892         wrbuf_destroy(r->mc_key);
893         resultset_use(-1);
894         xfree(r);
895     }
896     else
897         yaz_mutex_leave(r->mutex);
898 }
899
900 ZOOM_API(size_t)
901     ZOOM_resultset_size(ZOOM_resultset r)
902 {
903     return r->size;
904 }
905
906 int ZOOM_test_reconnect(ZOOM_connection c)
907 {
908     ZOOM_Event event;
909
910     if (!c->reconnect_ok)
911         return 0;
912     ZOOM_connection_close(c);
913     c->reconnect_ok = 0;
914     c->tasks->running = 0;
915     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
916
917     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
918     ZOOM_connection_put_event(c, event);
919
920     return 1;
921 }
922
923 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
924                                     int force_sync, int start, int count)
925 {
926     ZOOM_task task;
927     ZOOM_connection c;
928     const char *cp;
929     const char *syntax, *elementSetName;
930
931     if (!r)
932         return;
933     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
934             " count=%d", r, force_sync, start, count);
935     c = r->connection;
936     if (!c)
937         return;
938
939     if (c->host_port && c->proto == PROTO_HTTP)
940     {
941         if (!c->cs)
942         {
943             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
944             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
945         }
946         else
947         {
948             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
949                     "reconnect", r);
950             c->reconnect_ok = 1;
951         }
952     }
953     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
954     task->u.search.resultset = r;
955     task->u.search.start = start;
956     task->u.search.count = count;
957
958     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
959     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
960     elementSetName = ZOOM_options_get(r->options, "elementSetName");
961     task->u.search.elementSetName = elementSetName
962         ? xstrdup(elementSetName) : 0;
963
964     cp = ZOOM_options_get(r->options, "schema");
965     task->u.search.schema = cp ? xstrdup(cp) : 0;
966
967     ZOOM_resultset_addref(r);
968
969     if (!r->connection->async || force_sync)
970         while (r->connection && ZOOM_event(1, &r->connection))
971             ;
972 }
973
974 ZOOM_API(void)
975     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
976                            size_t start, size_t count)
977 {
978     int force_present = 0;
979
980     if (!r)
981         return ;
982     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
983             r, r, (long) start, (long) count);
984     if (count && recs)
985         force_present = 1;
986     ZOOM_resultset_retrieve(r, force_present, start, count);
987     if (force_present)
988     {
989         size_t i;
990         for (i = 0; i< count; i++)
991             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
992     }
993 }
994
995 ZOOM_API(size_t)
996     ZOOM_resultset_facets_size(ZOOM_resultset r)
997 {
998     return r->num_res_facets;
999 }
1000
1001 ZOOM_API(ZOOM_facet_field)
1002     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
1003 {
1004     int num = r->num_res_facets;
1005     ZOOM_facet_field *facets = r->res_facets;
1006     int i;
1007     for (i = 0; i < num; i++)
1008         if (!strcmp(facets[i]->facet_name, name))
1009             return facets[i];
1010     return 0;
1011 }
1012
1013 ZOOM_API(ZOOM_facet_field)
1014     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
1015 {
1016     int num = r->num_res_facets;
1017     ZOOM_facet_field *facets = r->res_facets;
1018     if (idx >= 0 && idx < num)
1019         return facets[idx];
1020     return 0;
1021 }
1022
1023 ZOOM_API(ZOOM_facet_field *)
1024     ZOOM_resultset_facets(ZOOM_resultset r)
1025 {
1026     return r->res_facets;
1027 }
1028
1029 ZOOM_API(const char**)
1030     ZOOM_resultset_facets_names(ZOOM_resultset r)
1031 {
1032     return (const char **) r->facets_names;
1033 }
1034
1035 ZOOM_API(const char*)
1036     ZOOM_facet_field_name(ZOOM_facet_field field)
1037 {
1038     return field->facet_name;
1039 }
1040
1041 ZOOM_API(size_t)
1042     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1043 {
1044     return field->num_terms;
1045 }
1046
1047 ZOOM_API(const char*)
1048     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
1049 {
1050     *freq = field->facet_terms[idx].frequency;
1051     return field->facet_terms[idx].term;
1052 }
1053
1054
1055 static void get_cert(ZOOM_connection c)
1056 {
1057     char *cert_buf;
1058     int cert_len;
1059
1060     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1061     {
1062         ZOOM_connection_option_setl(c, "sslPeerCert",
1063                                     cert_buf, cert_len);
1064         xfree(cert_buf);
1065     }
1066 }
1067
1068 static zoom_ret do_connect_host(ZOOM_connection c,
1069                                 const char *logical_url);
1070
1071 static zoom_ret do_connect(ZOOM_connection c)
1072 {
1073     return do_connect_host(c, c->host_port);
1074 }
1075
1076 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1077 {
1078     void *add;
1079
1080     if (c->cs)
1081         cs_close(c->cs);
1082     c->cs = cs_create_host2(logical_url, CS_FLAGS_DNS_NO_BLOCK, &add,
1083                             c->tproxy ? c->tproxy : c->proxy,
1084                             &c->proxy_mode);
1085
1086     if (c->cs && c->cs->protocol == PROTO_HTTP)
1087     {
1088 #if YAZ_HAVE_XML2
1089         c->proto = PROTO_HTTP;
1090 #else
1091         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1092         ZOOM_connection_close(c);
1093         return zoom_complete;
1094 #endif
1095     }
1096     if (c->cs)
1097     {
1098         int ret = cs_connect(c->cs, add);
1099         if (ret == 0)
1100         {
1101             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1102             ZOOM_connection_put_event(c, event);
1103             get_cert(c);
1104             if (c->proto == PROTO_Z3950)
1105                 ZOOM_connection_Z3950_send_init(c);
1106             else
1107             {
1108                 /* no init request for SRW .. */
1109                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1110                 ZOOM_connection_remove_task(c);
1111                 ZOOM_connection_set_mask(c, 0);
1112
1113                 if (c->cs && c->location)
1114                     send_HTTP_redirect(c, c->location);
1115                 else
1116                     ZOOM_connection_exec_task(c);
1117             }
1118             c->state = STATE_ESTABLISHED;
1119             return zoom_pending;
1120         }
1121         else if (ret > 0)
1122         {
1123             int mask = ZOOM_SELECT_EXCEPT;
1124             if (c->cs->io_pending & CS_WANT_WRITE)
1125                 mask += ZOOM_SELECT_WRITE;
1126             if (c->cs->io_pending & CS_WANT_READ)
1127                 mask += ZOOM_SELECT_READ;
1128             ZOOM_connection_set_mask(c, mask);
1129             c->state = STATE_CONNECTING;
1130             return zoom_pending;
1131         }
1132     }
1133     c->state = STATE_IDLE;
1134     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1135     return zoom_complete;
1136 }
1137
1138 /* returns 1 if PDU was sent OK (still pending )
1139    0 if PDU was not sent OK (nothing to wait for)
1140 */
1141
1142 ZOOM_API(ZOOM_record)
1143     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1144 {
1145     const char *syntax =
1146         ZOOM_options_get(s->options, "preferredRecordSyntax");
1147     const char *elementSetName =
1148         ZOOM_options_get(s->options, "elementSetName");
1149     const char *schema =
1150         ZOOM_options_get(s->options, "schema");
1151
1152     return ZOOM_record_cache_lookup_i(s, pos, syntax, elementSetName, schema);
1153 }
1154
1155 ZOOM_API(ZOOM_record)
1156     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1157 {
1158     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1159
1160     if (!rec)
1161     {
1162         /*
1163          * MIKE: I think force_sync should always be zero, but I don't
1164          * want to make this change until I get the go-ahead from
1165          * Adam, in case something depends on the old synchronous
1166          * behaviour.
1167          */
1168         int force_sync = 1;
1169         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1170         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1171         rec = ZOOM_resultset_record_immediate(r, pos);
1172     }
1173     return rec;
1174 }
1175
1176 ZOOM_API(ZOOM_scanset)
1177     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1178 {
1179     ZOOM_scanset s;
1180     ZOOM_query q = ZOOM_query_create();
1181
1182     ZOOM_query_prefix(q, start);
1183
1184     s = ZOOM_connection_scan1(c, q);
1185     ZOOM_query_destroy(q);
1186     return s;
1187
1188 }
1189
1190 ZOOM_API(ZOOM_scanset)
1191     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1192 {
1193     ZOOM_scanset scan = 0;
1194     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1195
1196     if (!z_query)
1197         return 0;
1198     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1199     scan->connection = c;
1200     scan->odr = odr_createmem(ODR_DECODE);
1201     scan->options = ZOOM_options_create_with_parent(c->options);
1202     scan->refcount = 1;
1203     scan->scan_response = 0;
1204     scan->srw_scan_response = 0;
1205
1206     scan->query = q;
1207     ZOOM_query_addref(q);
1208     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1209                                             &scan->num_databaseNames,
1210                                             scan->odr);
1211
1212     if (1)
1213     {
1214         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1215         task->u.scan.scan = scan;
1216
1217         (scan->refcount)++;
1218         if (!c->async)
1219         {
1220             while (ZOOM_event(1, &c))
1221                 ;
1222         }
1223     }
1224     return scan;
1225 }
1226
1227 ZOOM_API(void)
1228     ZOOM_scanset_destroy(ZOOM_scanset scan)
1229 {
1230     if (!scan)
1231         return;
1232     (scan->refcount)--;
1233     if (scan->refcount == 0)
1234     {
1235         ZOOM_query_destroy(scan->query);
1236
1237         odr_destroy(scan->odr);
1238
1239         ZOOM_options_destroy(scan->options);
1240         xfree(scan);
1241     }
1242 }
1243
1244 static zoom_ret send_package(ZOOM_connection c)
1245 {
1246     ZOOM_Event event;
1247
1248     yaz_log(c->log_details, "%p send_package", c);
1249     if (!c->tasks)
1250         return zoom_complete;
1251     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1252
1253     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1254     ZOOM_connection_put_event(c, event);
1255
1256     c->buf_out = c->tasks->u.package->buf_out;
1257     c->len_out = c->tasks->u.package->len_out;
1258
1259     return ZOOM_send_buf(c);
1260 }
1261
1262 ZOOM_API(size_t)
1263     ZOOM_scanset_size(ZOOM_scanset scan)
1264 {
1265     if (!scan)
1266         return 0;
1267
1268     if (scan->scan_response && scan->scan_response->entries)
1269         return scan->scan_response->entries->num_entries;
1270     else if (scan->srw_scan_response)
1271         return scan->srw_scan_response->num_terms;
1272     return 0;
1273 }
1274
1275 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1276                                 size_t *occ,
1277                                 const char **value_term, size_t *value_len,
1278                                 const char **disp_term, size_t *disp_len)
1279 {
1280     size_t noent = ZOOM_scanset_size(scan);
1281
1282     *value_term = 0;
1283     *value_len = 0;
1284
1285     *disp_term = 0;
1286     *disp_len = 0;
1287
1288     *occ = 0;
1289     if (pos >= noent)
1290         return;
1291     if (scan->scan_response)
1292     {
1293         Z_ScanResponse *res = scan->scan_response;
1294         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1295         {
1296             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1297
1298             *value_term = (const char *) t->term->u.general->buf;
1299             *value_len = t->term->u.general->len;
1300             if (t->displayTerm)
1301             {
1302                 *disp_term = t->displayTerm;
1303                 *disp_len = strlen(*disp_term);
1304             }
1305             else if (t->term->which == Z_Term_general)
1306             {
1307                 *disp_term = (const char *) t->term->u.general->buf;
1308                 *disp_len = t->term->u.general->len;
1309             }
1310             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1311         }
1312     }
1313     if (scan->srw_scan_response)
1314     {
1315         Z_SRW_scanResponse *res = scan->srw_scan_response;
1316         Z_SRW_scanTerm *t = res->terms + pos;
1317         if (t)
1318         {
1319             *value_term = t->value;
1320             *value_len = strlen(*value_term);
1321
1322             if (t->displayTerm)
1323                 *disp_term = t->displayTerm;
1324             else
1325                 *disp_term = t->value;
1326             *disp_len = strlen(*disp_term);
1327             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1328         }
1329     }
1330 }
1331
1332 ZOOM_API(const char *)
1333     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1334                       size_t *occ, size_t *len)
1335 {
1336     const char *value_term = 0;
1337     size_t value_len = 0;
1338     const char *disp_term = 0;
1339     size_t disp_len = 0;
1340
1341     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1342                         &disp_term, &disp_len);
1343
1344     *len = value_len;
1345     return value_term;
1346 }
1347
1348 ZOOM_API(const char *)
1349     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1350                               size_t *occ, size_t *len)
1351 {
1352     const char *value_term = 0;
1353     size_t value_len = 0;
1354     const char *disp_term = 0;
1355     size_t disp_len = 0;
1356
1357     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1358                         &disp_term, &disp_len);
1359
1360     *len = disp_len;
1361     return disp_term;
1362 }
1363
1364 ZOOM_API(const char *)
1365     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1366 {
1367     return ZOOM_options_get(scan->options, key);
1368 }
1369
1370 ZOOM_API(void)
1371     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1372                             const char *val)
1373 {
1374     ZOOM_options_set(scan->options, key, val);
1375 }
1376
1377
1378 ZOOM_API(ZOOM_package)
1379     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1380 {
1381     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1382
1383     p->connection = c;
1384     p->odr_out = odr_createmem(ODR_ENCODE);
1385     p->options = ZOOM_options_create_with_parent2(options, c->options);
1386     p->refcount = 1;
1387     p->buf_out = 0;
1388     p->len_out = 0;
1389     return p;
1390 }
1391
1392 ZOOM_API(void)
1393     ZOOM_package_destroy(ZOOM_package p)
1394 {
1395     if (!p)
1396         return;
1397     (p->refcount)--;
1398     if (p->refcount == 0)
1399     {
1400         odr_destroy(p->odr_out);
1401         xfree(p->buf_out);
1402
1403         ZOOM_options_destroy(p->options);
1404         xfree(p);
1405     }
1406 }
1407
1408 ZOOM_API(const char *)
1409     ZOOM_package_option_get(ZOOM_package p, const char *key)
1410 {
1411     return ZOOM_options_get(p->options, key);
1412 }
1413
1414 ZOOM_API(const char *)
1415     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1416 {
1417     return ZOOM_options_getl(p->options, key, lenp);
1418 }
1419
1420 ZOOM_API(void)
1421     ZOOM_package_option_set(ZOOM_package p, const char *key,
1422                             const char *val)
1423 {
1424     ZOOM_options_set(p->options, key, val);
1425 }
1426
1427 ZOOM_API(void)
1428     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1429                              const char *val, int len)
1430 {
1431     ZOOM_options_setl(p->options, key, val, len);
1432 }
1433
1434 ZOOM_API(int)
1435     ZOOM_connection_exec_task(ZOOM_connection c)
1436 {
1437     ZOOM_task task = c->tasks;
1438     zoom_ret ret = zoom_complete;
1439
1440     if (!task)
1441         return 0;
1442     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1443             c, task->which, task->running);
1444     if (c->error != ZOOM_ERROR_NONE)
1445     {
1446         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1447                 "removing tasks because of error = %d", c, c->error);
1448         ZOOM_connection_remove_tasks(c);
1449         return 0;
1450     }
1451     if (task->running)
1452     {
1453         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1454                 "task already running", c);
1455         return 0;
1456     }
1457     task->running = 1;
1458     ret = zoom_complete;
1459     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1460     {
1461         switch (task->which)
1462         {
1463         case ZOOM_TASK_SEARCH:
1464             if (c->proto == PROTO_HTTP)
1465                 ret = ZOOM_connection_srw_send_search(c);
1466             else
1467                 ret = ZOOM_connection_Z3950_search(c);
1468             break;
1469         case ZOOM_TASK_CONNECT:
1470             ret = do_connect(c);
1471             break;
1472         case ZOOM_TASK_SCAN:
1473             if (c->proto == PROTO_HTTP)
1474                 ret = ZOOM_connection_srw_send_scan(c);
1475             else
1476                 ret = ZOOM_connection_Z3950_send_scan(c);
1477             break;
1478         case ZOOM_TASK_PACKAGE:
1479             ret = send_package(c);
1480             break;
1481         case ZOOM_TASK_SORT:
1482             c->tasks->u.sort.resultset->r_sort_spec =
1483                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1484             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1485             break;
1486         }
1487     }
1488     else
1489     {
1490         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1491                 "remove tasks because no connection exist", c);
1492         ZOOM_connection_remove_tasks(c);
1493     }
1494     if (ret == zoom_complete)
1495     {
1496         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1497                 "task removed (complete)", c);
1498         ZOOM_connection_remove_task(c);
1499         return 0;
1500     }
1501     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1502             "task pending", c);
1503     return 1;
1504 }
1505
1506
1507 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1508 {
1509     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1510
1511     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1512     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1513                       "text/xml");
1514     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1515     if (c->user && c->password)
1516     {
1517         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1518                                      c->user, c->password);
1519     }
1520     xfree(c->location);
1521     c->location = 0;
1522     return ZOOM_send_GDU(c, gdu);
1523 }
1524
1525
1526 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1527 {
1528     ZOOM_Event event;
1529
1530     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1531     if (!r)
1532         return zoom_complete;
1533     if (c->odr_print)
1534         z_GDU(c->odr_print, &gdu, 0, 0);
1535     if (c->odr_save)
1536         z_GDU(c->odr_save, &gdu, 0, 0);
1537     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1538     odr_reset(c->odr_out);
1539
1540     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1541     ZOOM_connection_put_event(c, event);
1542
1543     return ZOOM_send_buf(c);
1544 }
1545
1546 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1547                          const char *addinfo, const char *addinfo2)
1548 {
1549     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1550 }
1551
1552 #if YAZ_HAVE_XML2
1553 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1554 {
1555     zoom_ret cret = zoom_complete;
1556     int ret = -1;
1557     char *addinfo = 0;
1558     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1559                                                        "Connection");
1560     const char *location;
1561
1562     ZOOM_connection_set_mask(c, 0);
1563     yaz_log(c->log_details, "%p handle_http", c);
1564
1565     yaz_cookies_response(c->cookies, hres);
1566     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1567         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1568     {
1569         c->no_redirects++;
1570         if (c->no_redirects > 10)
1571         {
1572             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1573             c->no_redirects = 0;
1574             ZOOM_connection_close(c);
1575         }
1576         else
1577         {
1578             /* since redirect may change host we just reconnect. A smarter
1579                implementation might check whether it's the same server */
1580
1581             int host_change = 0;
1582             location = yaz_check_location(c->odr_in, c->host_port,
1583                                           location, &host_change);
1584             xfree(c->location);
1585             c->location = xstrdup(location);
1586             do_connect_host(c, location);
1587             return;
1588         }
1589     }
1590     else
1591     {
1592         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1593         if (ret == 0)
1594         {
1595             if (c->no_redirects) /* end of redirect. change hosts again */
1596                 ZOOM_connection_close(c);
1597         }
1598         c->no_redirects = 0;
1599     }
1600     if (ret)
1601     {
1602         if (hres->code != 200)
1603             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1604         else
1605         {
1606             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1607                     addinfo : "NULL");
1608             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1609         }
1610         ZOOM_connection_close(c);
1611     }
1612     if (cret == zoom_complete)
1613     {
1614         yaz_log(c->log_details, "removing tasks in handle_http");
1615         ZOOM_connection_remove_task(c);
1616     }
1617     {
1618         int must_close = 0;
1619         if (!strcmp(hres->version, "1.0"))
1620         {
1621             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1622             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1623                 must_close = 1;
1624         }
1625         else
1626         {
1627             /* HTTP 1.1: only if no close we stay alive.. */
1628             if (connection_head && !strcmp(connection_head, "close"))
1629                 must_close = 1;
1630         }
1631         if (must_close)
1632         {
1633             ZOOM_connection_close(c);
1634             if (c->tasks)
1635             {
1636                 c->tasks->running = 0;
1637                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1638                 c->reconnect_ok = 0;
1639             }
1640         }
1641         else
1642             c->reconnect_ok = 1; /* if the server closes anyway */
1643     }
1644 }
1645 #endif
1646
1647 static int do_read(ZOOM_connection c)
1648 {
1649     int r, more;
1650     ZOOM_Event event;
1651
1652     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1653     ZOOM_connection_put_event(c, event);
1654
1655     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1656     more = cs_more(c->cs);
1657     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1658     if (r == 1)
1659         return 0;
1660     if (r <= 0)
1661     {
1662         if (!ZOOM_test_reconnect(c))
1663         {
1664             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1665             ZOOM_connection_close(c);
1666         }
1667     }
1668     else
1669     {
1670         Z_GDU *gdu;
1671         ZOOM_Event event;
1672
1673         odr_reset(c->odr_in);
1674         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1675         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1676         ZOOM_connection_put_event(c, event);
1677
1678         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1679         {
1680             int x;
1681             int err = odr_geterrorx(c->odr_in, &x);
1682             char msg[100];
1683             const char *element = odr_getelement(c->odr_in);
1684             yaz_snprintf(msg, sizeof(msg),
1685                     "ODR code %d:%d element=%s offset=%d",
1686                     err, x, element ? element : "<unknown>",
1687                     odr_offset(c->odr_in));
1688             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1689             if (c->log_api)
1690             {
1691                 FILE *ber_file = yaz_log_file();
1692                 if (ber_file)
1693                     odr_dumpBER(ber_file, c->buf_in, r);
1694             }
1695             ZOOM_connection_close(c);
1696         }
1697         else
1698         {
1699             if (c->odr_print)
1700                 z_GDU(c->odr_print, &gdu, 0, 0);
1701             if (c->odr_save)
1702                 z_GDU(c->odr_save, &gdu, 0, 0);
1703             if (gdu->which == Z_GDU_Z3950)
1704                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1705             else if (gdu->which == Z_GDU_HTTP_Response)
1706             {
1707 #if YAZ_HAVE_XML2
1708                 handle_http(c, gdu->u.HTTP_Response);
1709 #else
1710                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1711                 ZOOM_connection_close(c);
1712 #endif
1713             }
1714         }
1715     }
1716     return 1;
1717 }
1718
1719 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1720 {
1721     int r;
1722     ZOOM_Event event;
1723
1724     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1725     ZOOM_connection_put_event(c, event);
1726
1727     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1728     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1729     {
1730         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1731         if (ZOOM_test_reconnect(c))
1732         {
1733             return zoom_pending;
1734         }
1735         if (c->state == STATE_CONNECTING)
1736             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1737         else
1738             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1739         ZOOM_connection_close(c);
1740         return zoom_complete;
1741     }
1742     else if (r == 1)
1743     {
1744         int mask = ZOOM_SELECT_EXCEPT;
1745         if (c->cs->io_pending & CS_WANT_WRITE)
1746             mask += ZOOM_SELECT_WRITE;
1747         if (c->cs->io_pending & CS_WANT_READ)
1748             mask += ZOOM_SELECT_READ;
1749         ZOOM_connection_set_mask(c, mask);
1750         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1751                 c, c->mask);
1752     }
1753     else
1754     {
1755         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1756         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1757                 c, c->mask);
1758     }
1759     return zoom_pending;
1760 }
1761
1762 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1763 {
1764     return do_write_ex(c, c->buf_out, c->len_out);
1765 }
1766
1767
1768 ZOOM_API(const char *)
1769     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1770 {
1771     if (!strcmp(key, "APDU"))
1772     {
1773         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1774     }
1775     else
1776         return ZOOM_options_get(c->options, key);
1777 }
1778
1779 ZOOM_API(const char *)
1780     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1781 {
1782     if (!strcmp(key, "APDU"))
1783     {
1784         if (c->saveAPDU_wrbuf)
1785         {
1786             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1787             return wrbuf_cstr(c->saveAPDU_wrbuf);
1788         }
1789         else
1790         {
1791             *lenp = 0;
1792             return "";
1793         }
1794     }
1795     else
1796         return ZOOM_options_getl(c->options, key, lenp);
1797 }
1798
1799 ZOOM_API(void)
1800     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1801                                const char *val)
1802 {
1803     if (!strcmp(key, "saveAPDU"))
1804     {
1805         if (val && strcmp(val, "0"))
1806         {
1807             if (!c->saveAPDU_wrbuf)
1808                 c->saveAPDU_wrbuf = wrbuf_alloc();
1809             else
1810                 wrbuf_rewind(c->saveAPDU_wrbuf);
1811         }
1812         else
1813         {
1814             wrbuf_destroy(c->saveAPDU_wrbuf);
1815             c->saveAPDU_wrbuf = 0;
1816         }
1817         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1818     }
1819     else
1820         ZOOM_options_set(c->options, key, val);
1821 }
1822
1823 ZOOM_API(void)
1824     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1825                                 const char *val, int len)
1826 {
1827     ZOOM_options_setl(c->options, key, val, len);
1828 }
1829
1830 ZOOM_API(const char *)
1831     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1832 {
1833     return ZOOM_options_get(r->options, key);
1834 }
1835
1836 ZOOM_API(void)
1837     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1838                               const char *val)
1839 {
1840     ZOOM_options_set(r->options, key, val);
1841 }
1842
1843
1844 ZOOM_API(int)
1845     ZOOM_connection_errcode(ZOOM_connection c)
1846 {
1847     return ZOOM_connection_error(c, 0, 0);
1848 }
1849
1850 ZOOM_API(const char *)
1851     ZOOM_connection_errmsg(ZOOM_connection c)
1852 {
1853     const char *msg;
1854     ZOOM_connection_error(c, &msg, 0);
1855     return msg;
1856 }
1857
1858 ZOOM_API(const char *)
1859     ZOOM_connection_addinfo(ZOOM_connection c)
1860 {
1861     const char *addinfo;
1862     ZOOM_connection_error(c, 0, &addinfo);
1863     return addinfo;
1864 }
1865
1866 ZOOM_API(const char *)
1867     ZOOM_connection_diagset(ZOOM_connection c)
1868 {
1869     const char *diagset;
1870     ZOOM_connection_error_x(c, 0, 0, &diagset);
1871     return diagset;
1872 }
1873
1874 ZOOM_API(const char *)
1875     ZOOM_diag_str(int error)
1876 {
1877     switch (error)
1878     {
1879     case ZOOM_ERROR_NONE:
1880         return "No error";
1881     case ZOOM_ERROR_CONNECT:
1882         return "Connect failed";
1883     case ZOOM_ERROR_MEMORY:
1884         return "Out of memory";
1885     case ZOOM_ERROR_ENCODE:
1886         return "Encoding failed";
1887     case ZOOM_ERROR_DECODE:
1888         return "Decoding failed";
1889     case ZOOM_ERROR_CONNECTION_LOST:
1890         return "Connection lost";
1891     case ZOOM_ERROR_INIT:
1892         return "Init rejected";
1893     case ZOOM_ERROR_INTERNAL:
1894         return "Internal failure";
1895     case ZOOM_ERROR_TIMEOUT:
1896         return "Timeout";
1897     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1898         return "Unsupported protocol";
1899     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1900         return "Unsupported query type";
1901     case ZOOM_ERROR_INVALID_QUERY:
1902         return "Invalid query";
1903     case ZOOM_ERROR_CQL_PARSE:
1904         return "CQL parsing error";
1905     case ZOOM_ERROR_CQL_TRANSFORM:
1906         return "CQL transformation error";
1907     case ZOOM_ERROR_CCL_CONFIG:
1908         return "CCL configuration error";
1909     case ZOOM_ERROR_CCL_PARSE:
1910         return "CCL parsing error";
1911     case ZOOM_ERROR_ES_INVALID_ACTION:
1912         return "Extended Service. invalid action";
1913     case ZOOM_ERROR_ES_INVALID_VERSION:
1914         return "Extended Service. invalid version";
1915     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1916         return "Extended Service. invalid syntax";
1917     case ZOOM_ERROR_MEMCACHED:
1918         return "Memcached";
1919     default:
1920         return diagbib1_str(error);
1921     }
1922 }
1923
1924 ZOOM_API(int)
1925     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1926                             const char **addinfo, const char **diagset)
1927 {
1928     int error = c->error;
1929     if (cp)
1930     {
1931         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1932             *cp = ZOOM_diag_str(error);
1933         else if (!strcmp(c->diagset, "HTTP"))
1934             *cp = z_HTTP_errmsg(c->error);
1935         else if (!strcmp(c->diagset, "Bib-1"))
1936             *cp = ZOOM_diag_str(error);
1937         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1938             *cp = yaz_diag_srw_str(c->error);
1939         else
1940             *cp = "Unknown error and diagnostic set";
1941     }
1942     if (addinfo)
1943         *addinfo = c->addinfo ? c->addinfo : "";
1944     if (diagset)
1945         *diagset = c->diagset ? c->diagset : "";
1946     return c->error;
1947 }
1948
1949 ZOOM_API(int)
1950     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1951                           const char **addinfo)
1952 {
1953     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1954 }
1955
1956 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1957 {
1958     ZOOM_Event event = 0;
1959     int r = cs_look(c->cs);
1960     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1961             c, mask, r);
1962
1963     if (r == CS_NONE)
1964     {
1965         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1966         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1967         ZOOM_connection_close(c);
1968         ZOOM_connection_put_event(c, event);
1969     }
1970     else if (r == CS_CONNECT)
1971     {
1972         int ret = ret = cs_rcvconnect(c->cs);
1973         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1974                 "cs_rcvconnect returned %d", c, ret);
1975         if (ret == 1)
1976         {
1977             int mask = ZOOM_SELECT_EXCEPT;
1978             if (c->cs->io_pending & CS_WANT_WRITE)
1979                 mask += ZOOM_SELECT_WRITE;
1980             if (c->cs->io_pending & CS_WANT_READ)
1981                 mask += ZOOM_SELECT_READ;
1982             ZOOM_connection_set_mask(c, mask);
1983             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1984             ZOOM_connection_put_event(c, event);
1985         }
1986         else if (ret == 0)
1987         {
1988             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1989             ZOOM_connection_put_event(c, event);
1990             get_cert(c);
1991             if (c->proto == PROTO_Z3950)
1992                 ZOOM_connection_Z3950_send_init(c);
1993             else
1994             {
1995                 /* no init request for SRW .. */
1996                 if (c->tasks->which == ZOOM_TASK_CONNECT)
1997                 {
1998                     ZOOM_connection_remove_task(c);
1999                     ZOOM_connection_set_mask(c, 0);
2000                 }
2001                 if (c->cs && c->location)
2002                     send_HTTP_redirect(c, c->location);
2003                 else
2004                     ZOOM_connection_exec_task(c);
2005             }
2006             c->state = STATE_ESTABLISHED;
2007         }
2008         else
2009         {
2010             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
2011             ZOOM_connection_close(c);
2012         }
2013     }
2014     else
2015     {
2016         if (mask & ZOOM_SELECT_EXCEPT)
2017         {
2018             if (!ZOOM_test_reconnect(c))
2019             {
2020                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
2021                 ZOOM_connection_close(c);
2022             }
2023             return;
2024         }
2025         if (mask & ZOOM_SELECT_READ)
2026             do_read(c);
2027         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2028             ZOOM_send_buf(c);
2029     }
2030 }
2031
2032 ZOOM_API(int)
2033     ZOOM_connection_last_event(ZOOM_connection cs)
2034 {
2035     if (!cs)
2036         return ZOOM_EVENT_NONE;
2037     return cs->last_event;
2038 }
2039
2040
2041 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2042 {
2043     if (c->mask)
2044     {
2045         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2046         /* timeout and this connection was waiting */
2047         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2048         ZOOM_connection_close(c);
2049         ZOOM_connection_put_event(c, event);
2050     }
2051     return 0;
2052 }
2053
2054 ZOOM_API(int)
2055     ZOOM_connection_process(ZOOM_connection c)
2056 {
2057     ZOOM_Event event;
2058     if (!c)
2059         return 0;
2060
2061     event = ZOOM_connection_get_event(c);
2062     if (event)
2063     {
2064         ZOOM_Event_destroy(event);
2065         return 1;
2066     }
2067     ZOOM_connection_exec_task(c);
2068     event = ZOOM_connection_get_event(c);
2069     if (event)
2070     {
2071         ZOOM_Event_destroy(event);
2072         return 1;
2073     }
2074     return 0;
2075 }
2076
2077 ZOOM_API(int)
2078     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2079 {
2080     int i;
2081
2082     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2083
2084     for (i = 0; i<no; i++)
2085     {
2086         ZOOM_connection c = cs[i];
2087
2088         if (c && ZOOM_connection_process(c))
2089             return i+1;
2090     }
2091     return 0;
2092 }
2093
2094 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2095 {
2096     if (c->mask && mask)
2097         ZOOM_connection_do_io(c, mask);
2098     return 0;
2099 }
2100
2101 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2102 {
2103     if (c->cs)
2104         return cs_fileno(c->cs);
2105     return -1;
2106 }
2107
2108 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2109 {
2110     c->mask = mask;
2111     if (!c->cs)
2112         return -1;
2113     return 0;
2114 }
2115
2116 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2117 {
2118     if (c->cs)
2119         return c->mask;
2120     return 0;
2121 }
2122
2123 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2124 {
2125     return ZOOM_options_get_int(c->options, "timeout", 30);
2126 }
2127
2128 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2129 {
2130     if (c->cs)
2131         cs_close(c->cs);
2132     c->cs = 0;
2133     ZOOM_connection_set_mask(c, 0);
2134     c->state = STATE_IDLE;
2135 }
2136
2137 /*
2138  * Local variables:
2139  * c-basic-offset: 4
2140  * c-file-style: "Stroustrup"
2141  * indent-tabs-mode: nil
2142  * End:
2143  * vim: shiftwidth=4 tabstop=8 expandtab
2144  */
2145