1 // $Id: CQLLexer.java,v 1.14 2007-07-03 13:30:42 mike Exp $
3 package org.z3950.zing.cql;
4 import java.io.InputStream;
6 import java.io.StreamTokenizer;
7 import java.io.StringReader;
10 // This is a semi-trivial subclass for java.io.StreamTokenizer that:
11 // * Has a halfDecentPushBack() method that actually works
12 // * Includes a render() method
13 // * Knows about the multi-character tokens "<=", ">=" and "<>"
14 // * Recognises a set of keywords as tokens in their own right
15 // * Includes some primitive debugging-output facilities
16 // It's used only by CQLParser.
18 class CQLLexer extends StreamTokenizer {
19 // New publicly visible token-types
20 public final static int TT_LE = 1000; // The "<=" relation
21 public final static int TT_GE = 1001; // The ">=" relation
22 public final static int TT_NE = 1002; // The "<>" relation
23 public final static int TT_EQEQ = 1003; // The "==" relation
24 public final static int TT_AND = 1004; // The "and" boolean
25 public final static int TT_OR = 1005; // The "or" boolean
26 public final static int TT_NOT = 1006; // The "not" boolean
27 public final static int TT_PROX = 1007; // The "prox" boolean
28 public final static int TT_SORTBY = 1008; // The "sortby" operator
30 // Support for keywords. It would be nice to compile this linear
31 // list into a Hashtable, but it's hard to store ints as hash
32 // values, and next to impossible to use them as hash keys. So
33 // we'll just scan the (very short) list every time we need to do
35 private class Keyword {
38 Keyword(int token, String keyword) {
40 this.keyword = keyword;
43 // This should logically be static, but Java won't allow it :-P
44 private Keyword[] keywords = {
45 new Keyword(TT_AND, "and"),
46 new Keyword(TT_OR, "or"),
47 new Keyword(TT_NOT, "not"),
48 new Keyword(TT_PROX, "prox"),
49 new Keyword(TT_SORTBY, "sortby"),
52 // For halfDecentPushBack() and the code at the top of nextToken()
53 private static int TT_UNDEFINED = -1000;
54 private int saved_ttype = TT_UNDEFINED;
55 private double saved_nval;
56 private String saved_sval;
58 // Controls debugging output
59 private static boolean DEBUG;
61 CQLLexer(String cql, boolean lexdebug) {
62 this(new StringReader(cql), lexdebug);
65 CQLLexer(Reader cql, boolean lexdebug) {
67 wordChars('!', '?'); // ASCII-dependency!
68 wordChars('[', '`'); // ASCII-dependency!
76 wordChars('\'', '\''); // prevent this from introducing strings
80 ordinaryChars('0', '9');
85 private static void debug(String str) {
87 System.err.println("LEXDEBUG: " + str);
90 // I don't honestly understand why we need this, but the
91 // documentation for java.io.StreamTokenizer.pushBack() is pretty
92 // vague about its semantics, and it seems to me that they could
93 // be summed up as "it doesn't work". This version has the very
94 // clear semantics "pretend I didn't call nextToken() just then".
96 private void halfDecentPushBack() {
103 public int nextToken() throws java.io.IOException {
104 if (saved_ttype != TT_UNDEFINED) {
108 saved_ttype = TT_UNDEFINED;
109 debug("using saved ttype=" + ttype + ", " +
110 "nval=" + nval + ", sval='" + sval + "'");
114 underlyingNextToken();
116 debug("token starts with '<' ...");
117 underlyingNextToken();
119 debug("token continues with '=' - it's '<='");
121 } else if (ttype == '>') {
122 debug("token continues with '>' - it's '<>'");
125 debug("next token is " + render() + " (pushed back)");
126 halfDecentPushBack();
128 debug("AFTER: ttype is now " + ttype + " - " + render());
130 } else if (ttype == '>') {
131 debug("token starts with '>' ...");
132 underlyingNextToken();
134 debug("token continues with '=' - it's '>='");
137 debug("next token is " + render() + " (pushed back)");
138 halfDecentPushBack();
140 debug("AFTER: ttype is now " + ttype + " - " + render());
142 } else if (ttype == '=') {
143 debug("token starts with '=' ...");
144 underlyingNextToken();
146 debug("token continues with '=' - it's '=='");
149 debug("next token is " + render() + " (pushed back)");
150 halfDecentPushBack();
152 debug("AFTER: ttype is now " + ttype + " - " + render());
156 debug("done nextToken(): ttype=" + ttype + ", " +
157 "nval=" + nval + ", " + "sval='" + sval + "'" +
158 " (" + render() + ")");
163 // It's important to do keyword recognition here at the lowest
164 // level, otherwise when one of these words follows "<" or ">"
165 // (which can be the beginning of multi-character tokens) it gets
166 // pushed back as a string, and its keywordiness is not
169 public int underlyingNextToken() throws java.io.IOException {
171 if (ttype == TT_WORD)
172 for (int i = 0; i < keywords.length; i++)
173 if (sval.equalsIgnoreCase(keywords[i].keyword))
174 ttype = keywords[i].token;
179 // Simpler interface for the usual case: current token with quoting
181 return render(ttype, true);
184 String render(int token, boolean quoteChars) {
185 if (token == TT_EOF) {
187 } else if (token == TT_NUMBER) {
188 if ((double) nval == (int) nval) {
189 return new Integer((int) nval).toString();
191 return new Double((double) nval).toString();
193 } else if (token == TT_WORD) {
194 return "word: " + sval;
195 } else if (token == '"') {
196 return "string: \"" + sval + "\"";
197 } else if (token == TT_LE) {
199 } else if (token == TT_GE) {
201 } else if (token == TT_NE) {
203 } else if (token == TT_EQEQ) {
207 // Check whether its associated with one of the keywords
208 for (int i = 0; i < keywords.length; i++)
209 if (token == keywords[i].token)
210 return keywords[i].keyword;
212 // Otherwise it must be a single character, such as '(' or '/'.
213 String res = String.valueOf((char) token);
214 if (quoteChars) res = "'" + res + "'";
218 public static void main(String[] args) throws Exception {
219 if (args.length > 1) {
220 System.err.println("Usage: CQLLexer [<CQL-query>]");
221 System.err.println("If unspecified, query is read from stdin");
226 if (args.length == 1) {
229 byte[] bytes = new byte[10000];
231 // Read in the whole of standard input in one go
232 int nbytes = System.in.read(bytes);
233 } catch (java.io.IOException ex) {
234 System.err.println("Can't read query: " + ex.getMessage());
237 cql = new String(bytes);
240 CQLLexer lexer = new CQLLexer(cql, true);
242 while ((token = lexer.nextToken()) != TT_EOF) {
243 // Nothing to do: debug() statements render tokens for us