<commit_before>package com.bananity.util;


import com.bananity.nlp.AnalyzerModule;
import com.bananity.util.HashBag;

import java.util.ArrayList;


/**
 *  @author <NAME>
 */
public class SearchesTokenizer
{
	private final static int MIN_WORD_LENGTH = 2;

	public static ArrayList<String> getSubTokensList (String searchTerm) throws Exception {		
		ArrayList<String> tokens = AnalyzerModule.analyze(searchTerm, true);

		int maxLength = 0;
		for (String token : tokens) {
			if (token.length() > maxLength) maxLength = token.length();
		}

		if (maxLength == 1) return tokens; // Hack to handle cases such as "A A A"

		ArrayList<String> subTokens = new ArrayList<String>();
		for (String token : tokens) {
			subTokens.addAll(AnalyzerModule.getAllSubstrings(token, MIN_WORD_LENGTH));
		}

		return subTokens;
	}

	public static HashBag<String> getSubTokensBag (String searchTerm) throws Exception {
		ArrayList<String> tokens = AnalyzerModule.analyze(searchTerm, true);

		int maxLength = 0;
		for (String token : tokens) {
			if (token.length() > maxLength) maxLength = token.length();
		}

		if (maxLength == 1) return new HashBag(tokens); // Hack to handle cases such as "A A A"

		HashBag<String> subTokens = new HashBag<String>();
		for (String token : tokens) {
			subTokens.addAll(AnalyzerModule.getSubstringsBag(token, MIN_WORD_LENGTH));
		}

		return subTokens;
	}
}<commit_msg>Added fix to increase searches precision (this increases the index size too)<commit_after>package com.bananity.util;


import com.bananity.nlp.AnalyzerModule;
import com.bananity.util.HashBag;

import java.util.ArrayList;


/**
 *  @author <NAME>
 */
public class SearchesTokenizer
{
	private final static int MIN_WORD_LENGTH = 2;

	public static ArrayList<String> getSubTokensList (String searchTerm) throws Exception {		
		ArrayList<String> tokens = AnalyzerModule.analyze(searchTerm, true);

		int maxLength = 0;
		for (String token : tokens) {
			if (token.length() > maxLength) maxLength = token.length();
		}

		if (maxLength == 1) return tokens; // Hack to handle cases such as "A A A"

		ArrayList<String> subTokens = new ArrayList<String>();
		for (String token : tokens) {
			subTokens.addAll(AnalyzerModule.getAllSubstrings(token, MIN_WORD_LENGTH));
		}

		// This block increments the index size, but allows more precise searches
		for (int i=0, n=tokens.size()-1; i<n; i++) {
			subTokens.add(new StringBuilder(tokens.get(i)).append(" ").append(tokens.get(i+1)).toString());
		}

		return subTokens;
	}

	public static HashBag<String> getSubTokensBag (String searchTerm) throws Exception {
		ArrayList<String> tokens = AnalyzerModule.analyze(searchTerm, true);

		int maxLength = 0;
		for (String token : tokens) {
			if (token.length() > maxLength) maxLength = token.length();
		}

		if (maxLength == 1) return new HashBag(tokens); // Hack to handle cases such as "A A A"

		HashBag<String> subTokens = new HashBag<String>();
		for (String token : tokens) {
			subTokens.addAll(AnalyzerModule.getSubstringsBag(token, MIN_WORD_LENGTH));
		}

		// This block increments the index size, but allows more precise searches
		for (int i=0, n=tokens.size()-1; i<n; i++) {
			subTokens.put(new StringBuilder(tokens.get(i)).append(" ").append(tokens.get(i+1)).toString());
		}

		return subTokens;
	}
}
{'fraction_non_alphanumeric': 0.12261001517450683, 'fraction_numerical': 0.004855842185128983, 'mean_word_length': 4.438943894389439, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7146550', 'n_tokens_mistral': 1141, 'n_tokens_neox': 1035, 'n_words': 291}