X-Git-Url: http://jsfdemo.indexdata.com/?a=blobdiff_plain;f=src%2Ficu_I18N.c;h=d791abadd5f4b6e7e4c439439dc185327e82f83f;hb=d293bd2470d6d93f44c89ca292ae3dfbb22e9b09;hp=8d67c76635eaa0a7065943b6f5ae2f3d6a39d441;hpb=dc45cca6d0b58f964f92c9e1412512530eb58acf;p=pazpar2-moved-to-github.git

diff --git a/src/icu_I18N.c b/src/icu_I18N.c
index 8d67c76..d791aba 100644
--- a/src/icu_I18N.c
+++ b/src/icu_I18N.c
@@ -1,4 +1,4 @@
-/* $Id: icu_I18N.c,v 1.15 2007-05-16 19:12:00 marc Exp $
+/* $Id: icu_I18N.c,v 1.16 2007-05-16 19:50:01 marc Exp $
    Copyright (c) 2006-2007, Index Data.
 
    This file is part of Pazpar2.
@@ -938,9 +938,10 @@ int icu_chain_step_next_token(struct icu_chain * chain,
                                  step->buf16, src16, status);
         break;
     case ICU_chain_step_type_tokenize:
-        // step->more_tokens
-        //       = icu_tokenizer_next_token(step->u.tokenizer,
-        //                               step->buf16, status);
+        icu_tokenizer_attach(step->u.tokenizer, src16, status);
+        step->more_tokens
+            = icu_tokenizer_next_token(step->u.tokenizer,
+                                       step->buf16, status);
         break;
     default:
         return 0;