content: package org.apache.lucene.queryparser.flexible.aqp;
  
  /**
   * Licensed to the Apache Software Foundation (ASF) under one or more
   * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0
   * (the "License"); you may not use this file except in compliance with
   * the License.  You may obtain a copy of the License at
   * <p>
   * http://www.apache.org/licenses/LICENSE-2.0
   * <p>
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  import org.apache.lucene.analysis.*;
  import org.apache.lucene.analysis.core.StopAnalyzer;
  import org.apache.lucene.analysis.core.StopFilter;
  import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
  import org.apache.lucene.analysis.en.EnglishAnalyzer;
  import org.apache.lucene.analysis.standard.StandardAnalyzer;
  import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
  import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
  import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
  import org.apache.lucene.document.DateTools;
  import org.apache.lucene.document.Document;
  import org.apache.lucene.document.TextField;
  import org.apache.lucene.index.DirectoryReader;
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.index.IndexWriter;
  import org.apache.lucene.index.Term;
  import org.apache.lucene.queryparser.flexible.aqp.parser.AqpStandardLuceneParser;
  import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
  import org.apache.lucene.queryparser.flexible.core.builders.QueryTreeBuilder;
  import org.apache.lucene.queryparser.flexible.core.config.QueryConfigHandler;
  import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages;
  import org.apache.lucene.queryparser.flexible.core.nodes.FuzzyQueryNode;
  import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
  import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl;
  import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorPipeline;
  import org.apache.lucene.queryparser.flexible.messages.MessageImpl;
  import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.Operator;
  import org.apache.lucene.queryparser.flexible.standard.nodes.WildcardQueryNode;
  import org.apache.lucene.queryparser.flexible.standard.processors.BooleanQuery2ModifierNodeProcessor;
  import org.apache.lucene.search.*;
  import org.apache.lucene.search.BooleanQuery.Builder;
  import org.apache.lucene.store.ByteBuffersDirectory;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.tests.analysis.MockAnalyzer;
  import org.apache.lucene.tests.analysis.MockTokenizer;
  import org.apache.lucene.util.automaton.Automata;
  import org.apache.lucene.util.automaton.CharacterRunAutomaton;
  
  import java.io.IOException;
  import java.text.DateFormat;
  import java.text.ParseException;
  import java.text.SimpleDateFormat;
  import java.util.*;
  
  /**
   * This test case is a copy of the core Lucene query parser test, it was adapted
   * to use new QueryParserHelper instead of the old query parser.
   *
   * TODO: modify the QueryParserHelper so that we can extend it (it is not
   * flexible in getting the parser, otherwise we could use the test methods there
   * for most part)
   *
   * Tests QueryParser.
   */
  public class TestAqpSLGStandardTest extends AqpTestAbstractCase {
  
      public static Analyzer qpAnalyzer = new QPTestAnalyzer();
  
      public static final class QPTestFilter extends TokenFilter {
          private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
          private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
  
          /**
           * Filter which discards the token 'stop' and which expands the token
           * 'phrase' into 'phrase1 phrase2'
           */
          public QPTestFilter(TokenStream in) {
              super(in);
          }
  
          boolean inPhrase = false;
          int savedStart = 0, savedEnd = 0;
  
          @Override
          public boolean incrementToken() throws IOException {
              if (inPhrase) {
                  inPhrase = false;
                  clearAttributes();
                  termAtt.setEmpty().append("phrase2");
                  offsetAtt.setOffset(savedStart, savedEnd);
                  return true;
              } else
                  while (input.incrementToken()) {
                      if (termAtt.toString().equals("phrase")) {
                          inPhrase = true;
                          savedStart = offsetAtt.startOffset();
                          savedEnd = offsetAtt.endOffset();
                          termAtt.setEmpty().append("phrase1");
                          offsetAtt.setOffset(savedStart, savedEnd);
                          return true;
                      } else if (!termAtt.toString().equals("stop"))
                          return true;
                  }
              return false;
          }
      }
  
      public static final class QPTestAnalyzer extends Analyzer {
  
          /** Filters MockTokenizer with StopFilter. */
          @Override
          public TokenStreamComponents createComponents(String fieldName) {
              Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE,
                      true);
              return new TokenStreamComponents(tokenizer, new QPTestFilter(tokenizer));
          }
      }
  
      public static class QPTestParser extends AqpQueryParser {
          public QPTestParser(QueryConfigHandler config, AqpSyntaxParser parser,
                              QueryNodeProcessorPipeline processor, QueryTreeBuilder builder) {
              super(config, parser, processor, builder);
              // TODO Auto-generated constructor stub
          }
  
          public static AqpQueryParser init(Analyzer a) throws Exception {
              AqpQueryParser p = AqpStandardLuceneParser.init();
  
              ((QueryNodeProcessorPipeline) p.getQueryNodeProcessor())
                      .add(new QPTestParserQueryNodeProcessor());
              p.setAnalyzer(a);
              return p;
          }
  
          private static class QPTestParserQueryNodeProcessor extends
                  QueryNodeProcessorImpl {
  
              @Override
              protected QueryNode postProcessNode(QueryNode node)
                      throws QueryNodeException {
  
                  return node;
  
              }
  
              @Override
              protected QueryNode preProcessNode(QueryNode node)
                      throws QueryNodeException {
  
                  if (node instanceof WildcardQueryNode || node instanceof FuzzyQueryNode) {
  
                      throw new QueryNodeException(new MessageImpl(
                              QueryParserMessages.EMPTY_MESSAGE));
  
                  }
  
                  return node;
  
              }
  
              @Override
              protected List<QueryNode> setChildrenOrder(List<QueryNode> children)
                      throws QueryNodeException {
  
                  return children;
  
              }
  
          }
  
      }
  
      @Override
      public void setUp() throws Exception {
          super.setUp();
          originalMaxClauses = BooleanQuery.getMaxClauseCount();
      }
  
      public void testConstantScoreAutoRewrite() throws Exception {
          AqpQueryParser qp = getParser();
          qp.setAnalyzer(new WhitespaceAnalyzer());
  
          Query q = qp.parse("foo*bar", "field");
          assertTrue(q instanceof WildcardQuery);
          assertEquals(MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE,
                  ((MultiTermQuery) q).getRewriteMethod());
  
          q = qp.parse("foo*", "field");
          assertTrue(q instanceof PrefixQuery);
          assertEquals(MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE,
                  ((MultiTermQuery) q).getRewriteMethod());
  
          q = qp.parse("[a TO z]", "field");
          assertTrue(q instanceof TermRangeQuery);
          assertEquals(MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE,
                  ((MultiTermQuery) q).getRewriteMethod());
      }
  
      public void testCJK() throws Exception {
          // Test Ideographic Space - As wide as a CJK character cell (fullwidth)
          // used google to translate the word "term" to japanese -> ??
          assertQueryEquals("term\u3000term\u3000term", null,
                  "term\u0020term\u0020term");
          assertQueryEqualsAllowLeadingWildcard("??\u3000\u30bf\u3000\u30e0", null,
                  "??\u0020\u30bf\u0020\u30e0");
      }
  
      public void testCJKTerm() throws Exception {
          // individual CJK chars as terms
          StandardAnalyzer analyzer = new StandardAnalyzer();
  
          Builder expected = new BooleanQuery.Builder();
          expected.add(new TermQuery(new Term("field", "中")),
                  BooleanClause.Occur.SHOULD);
          expected.add(new TermQuery(new Term("field", "国")),
                  BooleanClause.Occur.SHOULD);
  
          assertEquals(expected.build(), getQuery("中国", analyzer));
      }
  
      public void testCJKBoostedTerm() throws Exception {
          // individual CJK chars as terms
          StandardAnalyzer analyzer = new StandardAnalyzer();
  
          Builder expected = new BooleanQuery.Builder();
          expected.add(new TermQuery(new Term("field", "中")),
                  BooleanClause.Occur.SHOULD);
          expected.add(new TermQuery(new Term("field", "国")),
                  BooleanClause.Occur.SHOULD);
  
          assertEquals(new BoostQuery(expected.build(), 0.5f), getQuery("中国^0.5", analyzer));
      }
  
      public void testCJKPhrase() throws Exception {
          // individual CJK chars as terms
          StandardAnalyzer analyzer = new StandardAnalyzer();
  
          PhraseQuery.Builder expected = new PhraseQuery.Builder();
          expected.add(new Term("field", "中"));
          expected.add(new Term("field", "国"));
  
          assertEquals(expected.build(), getQuery("\"中国\"", analyzer));
      }
  
      public void testCJKBoostedPhrase() throws Exception {
          // individual CJK chars as terms
          StandardAnalyzer analyzer = new StandardAnalyzer();
  
          PhraseQuery.Builder expected = new PhraseQuery.Builder();
          expected.add(new Term("field", "中"));
          expected.add(new Term("field", "国"));
  
          assertEquals(new BoostQuery(expected.build(), 0.5f), getQuery("\"中国\"^0.5", analyzer));
      }
  
      public void testCJKSloppyPhrase() throws Exception {
          // individual CJK chars as terms
          StandardAnalyzer analyzer = new StandardAnalyzer();
  
          PhraseQuery.Builder expected = new PhraseQuery.Builder();
          expected.add(new Term("field", "中"));
          expected.add(new Term("field", "国"));
          expected.setSlop(3);
  
          assertEquals(expected.build(), getQuery("\"中国\"~3", analyzer));
      }
  
      public void testSimple() throws Exception {
          assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
          assertQueryEquals("term term term", null, "term term term");
          assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(), "t�rm term term");
          assertQueryEquals("�mlaut", new WhitespaceAnalyzer(),
                  "�mlaut");
  
          // XXX: not allowed, TODO???
          // assertQueryEquals("\"\"", new KeywordAnalyzer(), "");
          // assertQueryEquals("foo:\"\"", new KeywordAnalyzer(), "foo:");
  
          assertQueryEquals("a AND b", null, "+a +b");
          assertQueryEquals("(a AND b)", null, "+a +b");
          assertQueryEquals("c OR (a AND b)", null, "c (+a +b)");
  
          assertQueryEquals("a AND NOT b", null, "+a -b");
          assertQueryEquals("a NOT b", null, "+a -b");
  
          assertQueryEquals("a AND -b", null, "+a -b");
  
          assertQueryEquals("a AND !b", null, "+a -b");
  
          assertQueryEquals("a && b", null, "+a +b");
  
          assertQueryEquals("a && ! b", null, "+a -b");
  
          assertQueryEquals("a OR b", null, "a b");
          assertQueryEquals("a || b", null, "a b");
  
          assertQueryEquals("a OR !b", null, "a -b");
  
          assertQueryEquals("a OR ! b", null, "a -b");
  
          assertQueryEquals("a OR -b", null, "a -b");
  
          assertQueryEquals("+term -term term", null, "+term -term term");
          assertQueryEquals("foo:term AND field:anotherTerm", null,
                  "+foo:term +anotherterm");
          assertQueryEquals("term AND \"phrase phrase\"", null,
                  "+term +\"phrase phrase\"");
          assertQueryEquals("\"hello there\"", null, "\"hello there\"");
          assertTrue(getQuery("a AND b", null) instanceof BooleanQuery);
          assertTrue(getQuery("hello", null) instanceof TermQuery);
          assertTrue(getQuery("\"hello there\"", null) instanceof PhraseQuery);
  
          assertQueryEquals("germ term^2.0", null, "germ (term)^2.0");
          assertQueryEquals("(term)^2.0", null, "(term)^2.0");
          assertQueryEquals("(germ term)^2.0", null, "(germ term)^2.0");
          assertQueryEquals("term^2.0", null, "(term)^2.0");
          assertQueryEquals("term^2", null, "(term)^2.0");
          assertQueryEquals("\"germ term\"^2.0", null, "(\"germ term\")^2.0");
          assertQueryEquals("\"term germ\"^2", null, "(\"term germ\")^2.0");
  
          assertQueryEquals("(foo OR bar) AND (baz OR boo)", null,
                  "+(foo bar) +(baz boo)");
  
          assertQueryEquals("((a OR b) AND NOT c) OR d", null, "(+(a b) -c) d");
          assertQueryEquals("((a OR b) NOT c) OR d", null, "(+(a b) -c) d");
  
          assertQueryEquals("+(apple \"steve jobs\") -(foo bar baz)", null,
                  "+(apple \"steve jobs\") -(foo bar baz)");
          assertQueryEquals("+title:(dog OR cat) -author:\"bob dole\"", null,
                  "+(title:dog title:cat) -author:\"bob dole\"");
  
          AqpQueryParser qp = getParser();
          qp.setDefaultOperator(Operator.OR);
          assertQueryMatch(qp, "title:(+a -b c)", "text", "+title:a -title:b title:c");
  
          qp.setDefaultOperator(Operator.AND);
          assertQueryMatch(qp, "title:(+a -b c)", "text",
                  "+title:a -title:b +title:c");
  
      }
  
      public void testPunct() throws Exception {
          Analyzer a = new WhitespaceAnalyzer();
          assertQueryEquals("a&b", a, "a&b");
          assertQueryEquals("a&&b", a, "a&&b");
          assertQueryEquals(".NET", a, ".NET");
      }
  
      public void testSlop() throws Exception {
  
          assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
          assertQueryEquals("\"term germ\"~2 flork", null, "\"term germ\"~2 flork");
          assertQueryEquals("\"term\"~2", null, "term");
          assertQueryEquals("\" \"~2 germ", null, "germ");
          assertQueryEquals("\"term germ\"~2^2", null, "(\"term germ\"~2)^2.0");
      }
  
      public void testNumber() throws Exception {
          // The numbers go away because SimpleAnalzyer ignores them
          assertQueryEquals("3", null, "MatchNoDocsQuery(\"\")");
          assertQueryEquals("term 1.0 1 2", null, "term");
          assertQueryEquals("term term1 term2", null, "term term term");
  
          Analyzer a = new StandardAnalyzer();
          assertQueryEquals("3", a, "3");
          assertQueryEquals("term 1.0 1 2", a, "term 1.0 1 2");
          assertQueryEquals("term term1 term2", a, "term term1 term2");
      }
  
      public void testWildcard() throws Exception {
          assertQueryEquals("term*", null, "term*");
          assertQueryEquals("term*^2", null, "(term*)^2.0");
          assertQueryEquals("term~", null, "term~2");
          assertQueryEquals("term~0.7", null, "term~1");
  
          assertQueryEquals("term~^2", null, "(term~2)^2.0");
  
          assertQueryEquals("term^2~", null, "(term~2)^2.0");
          assertQueryEquals("term*germ", null, "term*germ");
          assertQueryEquals("term*germ^3", null, "(term*germ)^3.0");
  
          assertTrue(getQuery("term*", null) instanceof PrefixQuery);
          assertTrue(((BoostQuery) getQuery("term*^2", null)).getQuery() instanceof PrefixQuery);
          assertTrue(getQuery("term~", null) instanceof FuzzyQuery);
          assertTrue(getQuery("term~0.7", null) instanceof FuzzyQuery);
  
          FuzzyQuery fq = (FuzzyQuery) getQuery("term~0.7", null);
          assertEquals(1, fq.getMaxEdits());
          assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
          fq = (FuzzyQuery) getQuery("term~", null);
          assertEquals(2, fq.getMaxEdits());
          assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
  
          assertTrue(getQuery("term*germ", null) instanceof WildcardQuery);
  
          /*
           * Tests to see that wild card terms are (or are not) properly lower-cased
           * with propery parser configuration
           */
          // First prefix queries:
          // by default, convert to lowercase:
          assertWildcardQueryEquals("Term*", "term*");
          // explicitly set lowercase:
          assertWildcardQueryEquals("term*", "term*");
          assertWildcardQueryEquals("Term*", "term*");
          assertWildcardQueryEquals("TERM*", "term*");
          // Then 'full' wildcard queries:
          // by default, convert to lowercase:
          assertWildcardQueryEquals("Te?m", "te?m");
          // Fuzzy queries:
          assertWildcardQueryEquals("Term~", "term~2");
          // Range queries:
  
          // TODO: implement this on QueryParser
          // Q0002E_INVALID_SYNTAX_CANNOT_PARSE: Syntax Error, cannot parse '[A TO
          // C]': Lexical error at line 1, column 1. Encountered: "[" (91), after
          // : ""
          assertWildcardQueryEquals("[A TO C]", "[a TO c]");
          assertWildcardQueryEquals("[A TO C]", "[a TO c]");
          // Test suffix queries: first disallow
          try {
              assertWildcardQueryEquals("*Term", "*term");
              fail();
          } catch (QueryNodeException pe) {
              // expected exception
          }
          try {
              assertWildcardQueryEquals("?Term", "?term");
              fail();
          } catch (QueryNodeException pe) {
              // expected exception
          }
          // Test suffix queries: then allow
          assertWildcardQueryEquals("*Term", "*term", true);
          assertWildcardQueryEquals("?Term", "?term", true);
      }
  
      public void testLeadingWildcardType() throws Exception {
          AqpQueryParser qp = getParser(null);
          qp.setAllowLeadingWildcard(true);
          assertEquals(WildcardQuery.class, qp.parse("t*erm*", "field").getClass());
          assertEquals(WildcardQuery.class, qp.parse("?term*", "field").getClass());
          assertEquals(WildcardQuery.class, qp.parse("*term*", "field").getClass());
      }
  
      public void testQPA() throws Exception {
          assertQueryEquals("term term^3.0 term", qpAnalyzer, "term (term)^3.0 term");
          assertQueryEquals("term stop^3.0 term", qpAnalyzer, "term term");
  
          assertQueryEquals("term term term", qpAnalyzer, "term term term");
          assertQueryEquals("term +stop term", qpAnalyzer, "term term");
          assertQueryEquals("term -stop term", qpAnalyzer, "term term");
  
          assertQueryEquals("drop AND (stop) AND roll", qpAnalyzer, "+drop +roll");
          assertQueryEquals("term +(stop) term", qpAnalyzer, "term term");
          assertQueryEquals("term -(stop) term", qpAnalyzer, "term term");
  
          assertQueryEquals("drop AND stop AND roll", qpAnalyzer, "+drop +roll");
          // rca TODO: plug the modifier GroupQueryNodeProcessor
          // expected: term phrase1 phrase2 term
          assertQueryEquals("term phrase term", qpAnalyzer,
                  "term (phrase1 phrase2) term");
  
          // TODO: plug the modifier GroupQueryNodeProcessor
          // expected: term phrase1 phrase2 term
          assertQueryEquals("term AND NOT phrase term", qpAnalyzer,
                  "(+term -(phrase1 phrase2)) term");
  
          assertQueryEquals("stop^3", qpAnalyzer, "MatchNoDocsQuery(\"\")");
          assertQueryEquals("stop", qpAnalyzer, "MatchNoDocsQuery(\"\")");
          assertQueryEquals("(stop)^3", qpAnalyzer, "MatchNoDocsQuery(\"\")");
          assertQueryEquals("((stop))^3", qpAnalyzer, "MatchNoDocsQuery(\"\")");
          assertQueryEquals("(stop^3)", qpAnalyzer, "MatchNoDocsQuery(\"\")");
          assertQueryEquals("((stop)^3)", qpAnalyzer, "MatchNoDocsQuery(\"\")");
          assertQueryEquals("(stop)", qpAnalyzer, "MatchNoDocsQuery(\"\")");
          assertQueryEquals("((stop))", qpAnalyzer, "MatchNoDocsQuery(\"\")");
          assertTrue(getQuery("term term term", qpAnalyzer) instanceof BooleanQuery);
          assertTrue(getQuery("term +stop", qpAnalyzer) instanceof TermQuery);
      }
  
      public void testRange() throws Exception {
          assertQueryEquals("[ a TO z]", null, "[a TO z]");
          assertEquals(MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE,
                  ((TermRangeQuery) getQuery("[ a TO z]", null)).getRewriteMethod());
  
          AqpQueryParser qp = getParser();
  
          qp.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE);
          assertEquals(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE,
                  ((TermRangeQuery) qp.parse("[ a TO z]", "field")).getRewriteMethod());
  
          assertQueryEquals("[ a TO z ]", null, "[a TO z]");
          assertQueryEquals("{ a TO z}", null, "{a TO z}");
          assertQueryEquals("{ a TO z }", null, "{a TO z}");
          assertQueryEquals("{ a TO z }^2.0", null, "({a TO z})^2.0");
          assertQueryEquals("[ a TO z] OR bar", null, "[a TO z] bar");
          assertQueryEquals("[ a TO z] AND bar", null, "+[a TO z] +bar");
          assertQueryEquals("( bar blar { a TO z}) ", null, "bar blar {a TO z}");
  
          // the original expected value was: gack (bar blar {a TO z})
          assertQueryEquals("gack ( bar blar { a TO z}) ", null,
                  "gack bar blar {a TO z}");
      }
  
      /**
       * removed in lucene-4.0 public void testFarsiRangeCollating() throws
       * Exception { Directory ramDir = newDirectory(); IndexWriter iw = new
       * IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new
       * WhitespaceAnalyzer(TEST_VERSION_CURRENT))); Document doc = new Document();
       * doc.add(newField("content", "\u0633\u0627\u0628", Field.Store.YES,
       * Field.Index.NOT_ANALYZED)); iw.addDocument(doc); iw.close(); IndexSearcher
       * is = new IndexSearcher(ramDir, true);
       *
       * AqpQueryParser qp = getParser(); qp.setAnalyzer(new
       * WhitespaceAnalyzer(TEST_VERSION_CURRENT));
       *
       * // Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in //
       * RuleBasedCollator. However, the Arabic Locale seems to order the // Farsi
       * // characters properly. Collator c = Collator.getInstance(new
       * Locale("ar")); qp.setRangeCollator(c);
       *
       * // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi //
       * orders the U+0698 character before the U+0633 character, so the // single
       * // index Term below should NOT be returned by a ConstantScoreRangeQuery //
       * with a Farsi Collator (or an Arabic one for the case when Farsi is // not
       * // supported).
       *
       * // Test ConstantScoreRangeQuery
       * qp.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
       * ScoreDoc[] result = is.search(qp.parse("[ \u062F TO \u0698 ]", "content"),
       * null, 1000).scoreDocs;
       * assertEquals("The index Term should not be included.", 0, result.length);
       *
       * result = is.search(qp.parse("[ \u0633 TO \u0638 ]", "content"), null,
       * 1000).scoreDocs; assertEquals("The index Term should be included.", 1,
       * result.length);
       *
       * // Test RangeQuery
       * qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
       * result = is.search(qp.parse("[ \u062F TO \u0698 ]", "content"), null,
       * 1000).scoreDocs; assertEquals("The index Term should not be included.", 0,
       * result.length);
       *
       * result = is.search(qp.parse("[ \u0633 TO \u0638 ]", "content"), null,
       * 1000).scoreDocs; assertEquals("The index Term should be included.", 1,
       * result.length);
       *
       * is.close(); ramDir.close(); }
       */
  
      public void testDateRange() throws Exception {
          String startDate = getLocalizedDate(2002, 1, 1);
          String endDate = getLocalizedDate(2002, 1, 4);
          Calendar endDateExpected = new GregorianCalendar();
          endDateExpected.clear();
          endDateExpected.set(2002, 1, 4, 23, 59, 59);
          endDateExpected.set(Calendar.MILLISECOND, 999);
          final String defaultField = "default";
          final String monthField = "month";
          final String hourField = "hour";
          AqpQueryParser qp = getParser();
  
          Map<CharSequence, DateTools.Resolution> dateRes = new HashMap<CharSequence, DateTools.Resolution>();
  
          // set a field specific date resolution
          dateRes.put(monthField, DateTools.Resolution.MONTH);
          qp.setDateResolution(dateRes);
  
          // set default date resolution to MILLISECOND
          qp.setDateResolution(DateTools.Resolution.MILLISECOND);
  
          // set second field specific date resolution
          dateRes.put(hourField, DateTools.Resolution.HOUR);
          qp.setDateResolution(dateRes);
  
          // for this field no field specific date resolution has been set,
          // so verify if the default resolution is used
          assertDateRangeQueryEquals(qp, defaultField, startDate, endDate,
                  endDateExpected.getTime(), DateTools.Resolution.MILLISECOND);
  
          // verify if field specific date resolutions are used for these two
          // fields
          assertDateRangeQueryEquals(qp, monthField, startDate, endDate,
                  endDateExpected.getTime(), DateTools.Resolution.MONTH);
  
          assertDateRangeQueryEquals(qp, hourField, startDate, endDate,
                  endDateExpected.getTime(), DateTools.Resolution.HOUR);
      }
  
      public void testEscaped() throws Exception {
          Analyzer a = new WhitespaceAnalyzer();
  
          /*
           * assertQueryEquals("\\[brackets", a, "\\[brackets");
           * assertQueryEquals("\\[brackets", null, "brackets");
           * assertQueryEquals("\\\\", a, "\\\\"); assertQueryEquals("\\+blah", a,
           * "\\+blah"); assertQueryEquals("\\(blah", a, "\\(blah");
           *
           * assertQueryEquals("\\-blah", a, "\\-blah"); assertQueryEquals("\\!blah",
           * a, "\\!blah"); assertQueryEquals("\\{blah", a, "\\{blah");
           * assertQueryEquals("\\}blah", a, "\\}blah"); assertQueryEquals("\\:blah",
           * a, "\\:blah"); assertQueryEquals("\\^blah", a, "\\^blah");
           * assertQueryEquals("\\[blah", a, "\\[blah"); assertQueryEquals("\\]blah",
           * a, "\\]blah"); assertQueryEquals("\\\"blah", a, "\\\"blah");
           * assertQueryEquals("\\(blah", a, "\\(blah"); assertQueryEquals("\\)blah",
           * a, "\\)blah"); assertQueryEquals("\\~blah", a, "\\~blah");
           * assertQueryEquals("\\*blah", a, "\\*blah"); assertQueryEquals("\\?blah",
           * a, "\\?blah"); //assertQueryEquals("foo \\&\\& bar", a,
           * "foo \\&\\& bar"); //assertQueryEquals("foo \\|| bar", a,
           * "foo \\|| bar"); //assertQueryEquals("foo \\AND bar", a,
           * "foo \\AND bar");
           */
  
          assertQueryEquals("\\*", a, "*");
  
          assertQueryEquals("\\a", a, "a");
  
          assertQueryEquals("a\\-b:c", a, "a-b:c");
          assertQueryEquals("a\\+b:c", a, "a+b:c");
          assertQueryEquals("a\\:b:c", a, "a:b:c");
          assertQueryEquals("a\\\\b:c", a, "a\\b:c");
  
          assertQueryEquals("a:b\\-c", a, "a:b-c");
          assertQueryEquals("a:b\\+c", a, "a:b+c");
          assertQueryEquals("a:b\\:c", a, "a:b:c");
          assertQueryEquals("a:b\\\\c", a, "a:b\\c");
  
          assertQueryEquals("a:b\\-c*", a, "a:b-c*");
          assertQueryEquals("a:b\\+c*", a, "a:b+c*");
          assertQueryEquals("a:b\\:c*", a, "a:b:c*");
  
          assertQueryEquals("a:b\\\\c*", a, "a:b\\c*");
  
          assertQueryEquals("a:b\\-?c", a, "a:b-?c");
          assertQueryEquals("a:b\\+?c", a, "a:b+?c");
          assertQueryEquals("a:b\\:?c", a, "a:b:?c");
  
          assertQueryEquals("a:b\\\\?c", a, "a:b\\?c");
  
          assertQueryEquals("a:b\\-c~", a, "a:b-c~1");
          assertQueryEquals("a:b\\+c~", a, "a:b+c~1");
          assertQueryEquals("a:b\\:c~", a, "a:b:c~1");
          assertQueryEquals("a:b\\\\c~", a, "a:b\\c~1");
  
          // TODO: implement Range queries on QueryParser
          assertQueryEquals("[ a\\- TO a\\+ ]", null, "[a- TO a+]");
          assertQueryEquals("[ a\\: TO a\\~ ]", null, "[a: TO a~]");
          assertQueryEquals("[ a\\\\ TO a\\* ]", null, "[a\\ TO a*]");
  
          assertQueryEquals(
                  "[\"c\\:\\\\temp\\\\\\~foo0.txt\" TO \"c\\:\\\\temp\\\\\\~foo9.txt\"]",
                  a, "[c:\\temp\\~foo0.txt TO c:\\temp\\~foo9.txt]");
  
          assertQueryEquals("a\\\\\\+b", a, "a\\+b");
  
          assertQueryEquals("a \\\"b c\\\" d", a, "a \"b c\" d");
          assertQueryEquals("\"a \\\"b c\\\" d\"", a, "\"a \"b c\" d\"");
          assertQueryEquals("\"a \\+b c d\"", a, "\"a +b c d\"");
  
          assertQueryEquals("c\\:\\\\temp\\\\\\~foo.txt", a, "c:\\temp\\~foo.txt");
  
          assertQueryNodeException("XY\\"); // there must be a character after the
          // escape char
  
          // test unicode escaping
          assertQueryEquals("a\\u0062c", a, "abc");
          assertQueryEquals("XY\\u005a", a, "XYZ");
          assertQueryEquals("XY\\u005A", a, "XYZ");
          assertQueryEquals("\"a \\\\\\u0028\\u0062\\\" c\"", a, "\"a \\(b\" c\"");
  
          assertQueryNodeException("XY\\u005G"); // test non-hex character in escaped
          // unicode sequence
          assertQueryNodeException("XY\\u005"); // test incomplete escaped unicode
          // sequence
  
          // Tests bug LUCENE-800
          assertQueryEquals("(item:\\\\ item:ABCD\\\\)", a, "item:\\ item:ABCD\\");
          assertQueryNodeException("(item:\\\\ item:ABCD\\\\))"); // unmatched closing
          assertQueryNodeException("(item:\\\\ item:ABCD\\\\)))"); // unmatched closing
          assertQueryNodeException("(item:\\\\ item:ABCD\\\\) foo)"); // unmatched closing
  
          // paranthesis
          assertQueryEquals("\\*", a, "*");
          assertQueryEquals("\\\\", a, "\\"); // escaped backslash
  
          assertQueryNodeException("\\"); // a backslash must always be escaped
  
          // LUCENE-1189
          // the original result was set to be:
          // assertQueryEquals("(\"a\\\\\") or (\"b\")", a, "a\\ or b");
          // however, this is wrong, because the query is: ("a\\") or ("b")
          // the AQP parser is handling escaped chars (even escaped escapes)
          // correctly
          assertQueryEquals("(\"a\\\\\") or (\"b\")", a, "a\\ b");
          assertQueryEquals("(foo:\"a\\\\\") or (bar:\"b\")", a, "foo:a\\ bar:b");
      }
  
      public void testQueryStringEscaping() throws Exception {
          Analyzer a = new WhitespaceAnalyzer();
  
          assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c");
          assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c");
          assertEscapedQueryEquals("a:b:c", a, "a\\:b\\:c");
          assertEscapedQueryEquals("a\\b:c", a, "a\\\\b\\:c");
  
          assertEscapedQueryEquals("a:b-c", a, "a\\:b\\-c");
          assertEscapedQueryEquals("a:b+c", a, "a\\:b\\+c");
          assertEscapedQueryEquals("a:b:c", a, "a\\:b\\:c");
          assertEscapedQueryEquals("a:b\\c", a, "a\\:b\\\\c");
  
          assertEscapedQueryEquals("a:b-c*", a, "a\\:b\\-c\\*");
          assertEscapedQueryEquals("a:b+c*", a, "a\\:b\\+c\\*");
          assertEscapedQueryEquals("a:b:c*", a, "a\\:b\\:c\\*");
  
          assertEscapedQueryEquals("a:b\\\\c*", a, "a\\:b\\\\\\\\c\\*");
  
          assertEscapedQueryEquals("a:b-?c", a, "a\\:b\\-\\?c");
          assertEscapedQueryEquals("a:b+?c", a, "a\\:b\\+\\?c");
          assertEscapedQueryEquals("a:b:?c", a, "a\\:b\\:\\?c");
  
          assertEscapedQueryEquals("a:b?c", a, "a\\:b\\?c");
  
          assertEscapedQueryEquals("a:b-c~", a, "a\\:b\\-c\\~");
          assertEscapedQueryEquals("a:b+c~", a, "a\\:b\\+c\\~");
          assertEscapedQueryEquals("a:b:c~", a, "a\\:b\\:c\\~");
          assertEscapedQueryEquals("a:b\\c~", a, "a\\:b\\\\c\\~");
  
          assertEscapedQueryEquals("[ a - TO a+ ]", null, "\\[ a \\- TO a\\+ \\]");
          assertEscapedQueryEquals("[ a : TO a~ ]", null, "\\[ a \\: TO a\\~ \\]");
          assertEscapedQueryEquals("[ a\\ TO a* ]", null, "\\[ a\\\\ TO a\\* \\]");
  
          // LUCENE-881
          assertEscapedQueryEquals("|| abc ||", a, "\\|\\| abc \\|\\|");
          assertEscapedQueryEquals("&& abc &&", a, "\\&\\& abc \\&\\&");
      }
  
      public void testTabNewlineCarriageReturn() throws Exception {
          assertQueryEqualsDOA("+weltbank +worlbank", null, "+weltbank +worlbank");
  
          assertQueryEqualsDOA("+weltbank\n+worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \n+worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \n +worlbank", null, "+weltbank +worlbank");
  
          assertQueryEqualsDOA("+weltbank\r+worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \r+worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \r +worlbank", null, "+weltbank +worlbank");
  
          assertQueryEqualsDOA("+weltbank\r\n+worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \r\n+worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \r\n +worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \r \n +worlbank", null,
                  "+weltbank +worlbank");
  
          assertQueryEqualsDOA("+weltbank\t+worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \t+worlbank", null, "+weltbank +worlbank");
          assertQueryEqualsDOA("weltbank \t +worlbank", null, "+weltbank +worlbank");
      }
  
      public void testSimpleDAO() throws Exception {
          assertQueryEqualsDOA("term term term", null, "+term +term +term");
          assertQueryEqualsDOA("term +term term", null, "+term +term +term");
          assertQueryEqualsDOA("term term +term", null, "+term +term +term");
          assertQueryEqualsDOA("term +term +term", null, "+term +term +term");
          assertQueryEqualsDOA("-term term term", null, "-term +term +term");
      }
  
      public void testBoost() throws Exception {
          CharacterRunAutomaton stopSet = new CharacterRunAutomaton(
                  Automata.makeString("on"));
          Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
          AqpQueryParser qp = getParser();
          qp.setAnalyzer(oneStopAnalyzer);
  
          Query q = qp.parse("on^1.0", "field");
          assertNotNull(q);
          q = qp.parse("\"hello\"^2.0", "field");
          assertNotNull(q);
          assertEquals(((BoostQuery) q).getBoost(), (float) 2.0, (float) 0.5);
          q = qp.parse("hello^2.0", "field");
          assertNotNull(q);
          assertEquals(((BoostQuery) q).getBoost(), (float) 2.0, (float) 0.5);
          q = qp.parse("\"on\"^1.0", "field");
          assertNotNull(q);
  
          AqpQueryParser qp2 = getParser();
          qp2.setAnalyzer(new StandardAnalyzer(EnglishAnalyzer.getDefaultStopSet()));
  
          q = qp2.parse("the^3", "field");
          // "the" is a stop word so the result is an empty query:
          assertNotNull(q);
          assertEquals("MatchNoDocsQuery(\"\")", q.toString());
          assertEquals(q.getClass(), MatchNoDocsQuery.class);
      }
  
      public void testException() throws Exception {
          assertQueryNodeException("*leadingWildcard"); // disallowed by default
          assertQueryNodeException("(foo bar");
  
          assertQueryNodeException("\"some phrase");
          assertQueryNodeException("foo bar))");
          assertQueryNodeException("field:term:with:colon some more terms");
          assertQueryNodeException("(sub query)^5.0^2.0 plus more");
          assertQueryNodeException("secret AND illegal) AND access:confidential");
      }
  
      public void testCustomQueryParserWildcard() throws Exception {
          try {
              QPTestParser.init(new WhitespaceAnalyzer()).parse(
                      "a?t", "contents");
              fail("Wildcard queries should not be allowed");
          } catch (QueryNodeException expected) {
              // expected exception
          }
      }
  
      public void testCustomQueryParserFuzzy() throws Exception {
          try {
              QPTestParser.init(new WhitespaceAnalyzer()).parse(
                      "xunit~", "contents");
              fail("Fuzzy queries should not be allowed");
          } catch (QueryNodeException expected) {
              // expected exception
          }
      }
  
      public void testBooleanQuery() throws Exception {
          BooleanQuery.setMaxClauseCount(2);
          try {
              AqpQueryParser qp = getParser();
              qp.setAnalyzer(new WhitespaceAnalyzer());
  
              qp.parse("one two three", "field");
              fail("ParseException expected due to too many boolean clauses");
          } catch (QueryNodeException expected) {
              // too many boolean clauses, so ParseException is expected
          }
      }
  
      /**
       * This test differs from TestPrecedenceQueryParser
       */
      public void testPrecedence() throws Exception {
          AqpQueryParser qp1 = getParser();
          qp1.setAnalyzer(new WhitespaceAnalyzer());
  
          AqpQueryParser qp2 = getParser();
          qp2.setAnalyzer(new WhitespaceAnalyzer());
  
          // TODO: to achieve standard lucene behaviour (no operator precedence)
          // modify the GroupQueryNodeProcessor to recognize our new BooleanQN classes
          // then do:
          QueryNodeProcessorPipeline processor = (QueryNodeProcessorPipeline) qp1
                  .getQueryNodeProcessor();
          processor.add(new BooleanQuery2ModifierNodeProcessor());
  
          Query query1 = qp1.parse("A AND B OR C AND D", "field");
          Query query2 = qp2.parse("+A +B +C +D", "field");
  
          assertEquals(query1, query2);
      }
  
      public void testLocalDateFormat() throws IOException, QueryNodeException,
              ParseException {
          Directory ramDir = new ByteBuffersDirectory();
          IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(new WhitespaceAnalyzer()));
          addDateDoc("a", 2005, 12, 2, 10, 15, 33, iw);
          addDateDoc("b", 2005, 12, 4, 22, 15, 00, iw);
          iw.close();
          IndexSearcher is = new IndexSearcher(DirectoryReader.open(ramDir));
  
          SimpleDateFormat format = new SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH);
          Date d1_12 = format.parse("1/12/2005");
          Date d3_12 = format.parse("3/12/2005");
          Date d4_12 = format.parse("4/12/2005");
          Date d28_12 = format.parse("28/12/2005");
  
          DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT,
                  Locale.getDefault());
          String dec1 = df.format(d1_12);
          String dec2 = df.format(format.parse("2/12/2005"));
          String dec3 = df.format(d3_12);
          String dec4 = df.format(d4_12);
          String dec28 = df.format(d28_12);
          // some locales translate date into string /w spaces, so we must use "phrase"
          assertHits(2, String.format("[\"%s\" TO \"%s\"]", dec1, dec28), is);
          assertHits(2, String.format("[\"%s\" TO \"%s\"]", dec1, dec4), is);
  
          assertHits(2, String.format("{\"%s\" TO \"%s\"}", dec1, dec28), is);
          assertHits(1, String.format("{\"%s\" TO \"%s\"}", dec1, dec4), is);
          assertHits(0, String.format("{\"%s\" TO \"%s\"}", dec3, dec4), is);
  
          ramDir.close();
      }
  
      public void testStopwords() throws Exception {
          AqpQueryParser qp = getParser();
          qp.setAnalyzer(new StopAnalyzer(StopFilter
                  .makeStopSet("the", "foo")));
  
          Query result = qp.parse("a:the OR a:foo", "a");
          assertNotNull("result is null and it shouldn't be", result);
          assertTrue("result is not a MatchNoDocsQuery", result instanceof MatchNoDocsQuery);
  
          result = qp.parse("a:woo OR a:the", "a");
          assertNotNull("result is null and it shouldn't be", result);
          assertTrue("result is not a TermQuery", result instanceof TermQuery);
  
          result = qp.parse(
                  "(fieldX:xxxxx OR fieldy:xxxxxxxx)^2 AND (fieldx:the OR fieldy:foo)",
                  "a");
  
          assertNotNull("result is null and it shouldn't be", result);
          assertEquals(result.toString(), "(fieldX:xxxxx fieldy:xxxxxxxx)^2.0");
      }
  
      public void testPositionIncrement() throws Exception {
          AqpQueryParser qp = getParser();
          qp.setAnalyzer(new StopAnalyzer(StopFilter
                  .makeStopSet("the", "in", "are", "this")));
  
          qp.setEnablePositionIncrements(true);
  
          String qtxt = "\"the words in poisitions pos02578 are stopped in this phrasequery\"";
          // 0 2 5 7 8
          int[] expectedPositions = {1, 3, 4, 6, 9};
          PhraseQuery pq = (PhraseQuery) qp.parse(qtxt, "a");
          // System.out.println("Query text: "+qtxt);
          // System.out.println("Result: "+pq);
          Term[] t = pq.getTerms();
          int[] pos = pq.getPositions();
          for (int i = 0; i < t.length; i++) {
              // System.out.println(i+". "+t[i]+"  pos: "+pos[i]);
              assertEquals("term " + i + " = " + t[i] + " has wrong term-position!",
                      expectedPositions[i], pos[i]);
          }
      }
  
      public void testMatchAllDocs() throws Exception {
          AqpQueryParser qp = getParser();
          qp.setAnalyzer(new WhitespaceAnalyzer());
  
          assertEquals(new MatchAllDocsQuery(), qp.parse("*:*", "field"));
          assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)", "field"));
          BooleanQuery bq = (BooleanQuery) qp.parse("+*:* -*:*", "field");
          assertTrue(bq.clauses().get(0).getQuery() instanceof MatchAllDocsQuery);
          assertTrue(bq.clauses().get(1).getQuery() instanceof MatchAllDocsQuery);
      }
  
      private class CannedTokenizer extends Tokenizer {
          private int upto = 0;
          private final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
          private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
  
          public CannedTokenizer() {
              super();
          }
  
          @Override
          public boolean incrementToken() {
              clearAttributes();
              if (upto == 4) {
                  return false;
              }
              if (upto == 0) {
                  posIncr.setPositionIncrement(1);
                  term.setEmpty().append("a");
              } else if (upto == 1) {
                  posIncr.setPositionIncrement(1);
                  term.setEmpty().append("b");
              } else if (upto == 2) {
                  posIncr.setPositionIncrement(0);
                  term.setEmpty().append("c");
              } else {
                  posIncr.setPositionIncrement(0);
                  term.setEmpty().append("d");
              }
              upto++;
              return true;
          }
  
          @Override
          public void reset() throws IOException {
              super.reset();
              this.upto = 0;
          }
      }
  
      private class CannedAnalyzer extends Analyzer {
          @Override
          public TokenStreamComponents createComponents(String ignored) {
              return new TokenStreamComponents(new CannedTokenizer());
          }
      }
  
      public void testMultiPhraseQuery() throws Exception {
          Directory dir = newDirectory();
          IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
                  new CannedAnalyzer()));
          Document doc = new Document();
          doc.add(newField("field", "", TextField.TYPE_NOT_STORED));
          w.addDocument(doc);
          w.commit();
          IndexReader r = DirectoryReader.open(w.getDirectory());
          IndexSearcher s = newSearcher(r);
  
          Query q = QPTestParser.init(new CannedAnalyzer()).parse("\"a\"", "field");
          assertTrue(q instanceof MultiPhraseQuery);
          assertEquals(1, s.search(q, 10).totalHits.value);
          r.close();
          w.close();
          dir.close();
      }
  
  }
  