+
+// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8
+
+int test_icu_tokenizer(const char * locale, char action,
+ const char * src8cstr, int count)
+{
+ int success = 1;
+
+ UErrorCode status = U_ZERO_ERROR;
+ struct icu_buf_utf16 * src16 = icu_buf_utf16_create(0);
+ struct icu_buf_utf16 * tkn16 = icu_buf_utf16_create(0);
+ struct icu_buf_utf8 * tkn8 = icu_buf_utf8_create(0);
+
+ //printf("Input: '%s'\n", src8cstr);
+
+ // transforming to UTF16
+ icu_utf16_from_utf8_cstr(src16, src8cstr, &status);
+ icu_check_status(status);
+
+ // set up tokenizer
+ struct icu_tokenizer * tokenizer
+ = icu_tokenizer_create(locale, action, &status);
+ icu_check_status(status);
+ YAZ_CHECK(tokenizer);
+
+ // attach text buffer to tokenizer
+ icu_tokenizer_attach(tokenizer, src16, &status);
+ icu_check_status(status);
+ YAZ_CHECK(tokenizer->bi);
+
+ // perform work on tokens
+ //printf("Tokens: ");
+ while(icu_tokenizer_next_token(tokenizer, tkn16, &status)){
+ icu_check_status(status);
+
+ // converting to UTF8
+ icu_utf16_to_utf8(tkn8, tkn16, &status);
+
+ //printf("token %d %d %d %d '%s'\n",
+ //
+ // icu_tokenizer_token_start(tokenizer),
+ // icu_tokenizer_token_end(tokenizer),
+ // icu_tokenizer_token_length(tokenizer),
+ // tkn8->utf8);
+ }
+
+ if (count != icu_tokenizer_token_count(tokenizer)){
+ success = 0;
+ printf("\nTokenizer '%s:%c' Error: \n", locale, action);
+ printf("Input: '%s'\n", src8cstr);
+ printf("Tokens: %d", icu_tokenizer_token_count(tokenizer));
+ printf(", expected: %d\n", count);
+ }
+
+ icu_tokenizer_destroy(tokenizer);
+ icu_buf_utf16_destroy(src16);
+ icu_buf_utf16_destroy(tkn16);
+ icu_buf_utf8_destroy(tkn8);
+
+ return success;
+}
+
+
+// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8
+
+void test_icu_I18N_tokenizer(int argc, char **argv)