<commit_before>/**   
 * License Agreement for OpenSearchServer
 *
 * Copyright (C) 2013 <NAME> / Jaeksoft
 * 
 * http://www.open-search-server.com
 * 
 * This file is part of OpenSearchServer.
 *
 * OpenSearchServer is free software: you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 * OpenSearchServer is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with OpenSearchServer. 
 *  If not, see <http://www.gnu.org/licenses/>.
 **/
package com.jaeksoft.searchlib.query;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.BailErrorStrategy;
import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;

import com.jaeksoft.searchlib.analysis.CompiledAnalyzer;
import com.jaeksoft.searchlib.query.parser.BooleanQueryBaseListener;
import com.jaeksoft.searchlib.query.parser.BooleanQueryLexer;
import com.jaeksoft.searchlib.query.parser.BooleanQueryParser;
import com.jaeksoft.searchlib.util.StringUtils;

public class QueryParser extends BooleanQueryBaseListener {

	private final String field;
	private final int defaultOperator;
	private final CompiledAnalyzer analyzer;
	private final int phraseSlop;
	private final Double termBoost;
	private final Double phraseBoost;

	private int currentOperator;
	private Query holdQuery;
	private BooleanQuery booleanQuery;

	private IOException ioError;

	public QueryParser(final String field, final Occur occur,
			final CompiledAnalyzer analyzer, final int phraseSlop,
			final Double termBoost, final Double phraseBoost) {
		this.field = field;
		this.defaultOperator = getOperator(occur);
		this.analyzer = analyzer;
		this.phraseSlop = phraseSlop;
		this.termBoost = termBoost;
		this.phraseBoost = phraseBoost;
	}

	final private static int getOperator(final Occur occur) {
		if (occur == null)
			return BooleanQueryLexer.AND;
		switch (occur) {
		default:
		case MUST:
			return BooleanQueryLexer.AND;
		case MUST_NOT:
			return BooleanQueryLexer.NOT;
		case SHOULD:
			return BooleanQueryLexer.OR;
		}
	}

	final private void addBooleanClause(final Query query, final int operator) {
		Occur occur = null;
		switch (operator) {
		case BooleanQueryLexer.AND:
			occur = Occur.MUST;
			break;
		case BooleanQueryLexer.OR:
			occur = Occur.SHOULD;
			break;
		case BooleanQueryLexer.NOT:
			occur = Occur.MUST_NOT;
			break;
		}
		booleanQuery.add(new BooleanClause(query, occur));
	}

	final private void addBooleanClause(final Query query) {
		if (currentOperator == -1) {
			holdQuery = query;
		} else {
			if (holdQuery != null) {
				addBooleanClause(holdQuery, currentOperator);
				holdQuery = null;
			}
			addBooleanClause(query, currentOperator);
		}
		currentOperator = defaultOperator;
	}

	final private List<String> getWords(String text) throws IOException {
		List<String> words = new ArrayList<String>(1);
		if (analyzer != null)
			analyzer.extractTerms(text, words);
		else
			words.add(text);
		return words;
	}

	final private void addTermQuery(String text) throws IOException {
		for (String word : getWords(text)) {
			Term term = new Term(field, word);
			TermQuery termQuery = new TermQuery(term);<commit_msg>Fix pattern issue when using only one NOT operator<commit_after>/**   
 * License Agreement for OpenSearchServer
 *
 * Copyright (C) 2013 <NAME> / Jaeksoft
 * 
 * http://www.open-search-server.com
 * 
 * This file is part of OpenSearchServer.
 *
 * OpenSearchServer is free software: you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 * OpenSearchServer is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with OpenSearchServer. 
 *  If not, see <http://www.gnu.org/licenses/>.
 **/
package com.jaeksoft.searchlib.query;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.BailErrorStrategy;
import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;

import com.jaeksoft.searchlib.analysis.CompiledAnalyzer;
import com.jaeksoft.searchlib.query.parser.BooleanQueryBaseListener;
import com.jaeksoft.searchlib.query.parser.BooleanQueryLexer;
import com.jaeksoft.searchlib.query.parser.BooleanQueryParser;
import com.jaeksoft.searchlib.util.StringUtils;

public class QueryParser extends BooleanQueryBaseListener {

	private final String field;
	private final int defaultOperator;
	private final CompiledAnalyzer analyzer;
	private final int phraseSlop;
	private final Double termBoost;
	private final Double phraseBoost;

	private int currentOperator;
	private Query holdQuery;
	private BooleanQuery booleanQuery;

	private IOException ioError;

	public QueryParser(final String field, final Occur occur,
			final CompiledAnalyzer analyzer, final int phraseSlop,
			final Double termBoost, final Double phraseBoost) {
		this.field = field;
		this.defaultOperator = getOperator(occur);
		this.analyzer = analyzer;
		this.phraseSlop = phraseSlop;
		this.termBoost = termBoost;
		this.phraseBoost = phraseBoost;
	}

	final private static int getOperator(final Occur occur) {
		if (occur == null)
			return BooleanQueryLexer.AND;
		switch (occur) {
		default:
		case MUST:
			return BooleanQueryLexer.AND;
		case MUST_NOT:
			return BooleanQueryLexer.NOT;
		case SHOULD:
			return BooleanQueryLexer.OR;
		}
	}

	final private void addBooleanClause(final Query query, final int operator) {
		Occur occur = null;
		switch (operator) {
		case BooleanQueryLexer.AND:
			occur = Occur.MUST;
			break;
		case BooleanQueryLexer.OR:
			occur = Occur.SHOULD;
			break;
		case BooleanQueryLexer.NOT:
			occur = Occur.MUST_NOT;
			break;
		}
		booleanQuery.add(new BooleanClause(query, occur));
	}

	final private void addBooleanClause(final Query query) {
		if (currentOperator == -1) {
			holdQuery = query;
		} else {
			if (holdQuery != null) {
				addBooleanClause(
						holdQuery,
						currentOperator == BooleanQueryLexer.NOT ? defaultOperator
								: currentOperator);
				holdQuery = null;
			}
			addBooleanClause(query, currentOperator);
		}
		currentOperator = defaultOperator;
	}

	final private List<String> getWords(String text) throws IOException {
		List<String> words = new ArrayList<String>(1);
		if (analyzer != null)
			analyzer.extractTerms(text, words);
		else
			words.add(text);
		return words;
	}

	final private void addTermQuery(String text) throws IOException {
		for (String word : getWords(text)) {
			Term term = new Term(field, word);
			TermQuery termQuery = new TermQuery(term);
{'fraction_non_alphanumeric': 0.08668991222796682, 'fraction_numerical': 0.003366598533124925, 'mean_word_length': 4.454426229508197, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5010073', 'n_tokens_mistral': 2750, 'n_tokens_neox': 2487, 'n_words': 717}