+ type = zebra_index_type_get(rules, "any:w");
+ YAZ_CHECK(type);
+ if (type)
+ {
+ const char *buf = " How are you?";
+ size_t len = strlen(buf);
+ int r = 1;
+
+ if (r)
+ {
+ const char *result_buf = 0;
+ size_t result_len = 0;
+ r = zebra_index_type_tokenize(type, buf, len,
+ &result_buf, &result_len);
+ YAZ_CHECK_EQ(r, 1);
+ YAZ_CHECK(result_len == 3 &&
+ !memcmp(result_buf, "how", result_len));
+ }
+
+ if (r)
+ {
+ const char *result_buf = 0;
+ size_t result_len = 0;
+ r = zebra_index_type_tokenize(type, 0, 0,
+ &result_buf, &result_len);
+ YAZ_CHECK_EQ(r, 1);
+ YAZ_CHECK(result_len == 3 &&
+ !memcmp(result_buf, "are", result_len));
+ }
+
+ if (r)
+ {
+ const char *result_buf = 0;
+ size_t result_len = 0;
+ r = zebra_index_type_tokenize(type, 0, 0,
+ &result_buf, &result_len);
+ YAZ_CHECK_EQ(r, 1);
+ YAZ_CHECK(result_len == 3 &&
+ !memcmp(result_buf, "you", result_len));
+ }
+
+ if (r)
+ {
+ const char *result_buf = 0;
+ size_t result_len = 0;
+ r = zebra_index_type_tokenize(type, 0, 0,
+ &result_buf, &result_len);
+ YAZ_CHECK_EQ(r, 0);
+ }
+ }