<commit_before>package com.rewayaat.core;

import org.apache.commons.lang.StringUtils;

import java.util.ArrayList;
import java.util.List;

/**
 * Uses Elastic Search Query Syntax to Modify user queries to improve search
 * results. This is done mostly by removing weird characters, adding <a href=
 * "https://www.elastic.co/guide/en/elasticsearch/guide/current/fuzziness.html">fuzziness</a>,
 * and adding <a href=
 * "https://www.elastic.co/guide/en/elasticsearch/guide/current/slop.html">slop</a>
 * to phrases.
 */
public class RewayaatQuery {

    private String query;

    private String[] docFields = new String[]{"_id:", "source:", "book:", "number:", "part:", "edition:", "chapter:", "publisher:", "section:", "tags:", "volume:", "notes:", "arabic:", "gradings:"};
    public RewayaatQuery(String query) {
        this.query = query;
    }

    public String query() {
        // splits query by all spaces that are not enclosed by double quotes or brackets
        List<String> splitted = new ArrayList<>();
        List<String> allFieldItems = new ArrayList<>();
        int nextingLevel = 0;
        StringBuilder result = new StringBuilder();
        for (char c : query.toCharArray()) {
            if (c == ' ' && nextingLevel == 0) {
                splitted.add(result.toString());
                result.setLength(0);
            } else {
                if (c == '(' | c == '[' | c == '\"') {
                    nextingLevel++;
                } else if (c == ')' | c == ']' | c == '\"') {
                    nextingLevel--;
                }
                result.append(c);
            }
        }
        splitted.add(result.toString());
        for (String s : splitted) {
            if (!s.contains("~") && !s.contains("(")) {
                if (query.length() > 6) {
                    s += "~2";
                } else {
                    s += "~1";
                }
            }
            if (!StringUtils.startsWithAny(s, docFields)) {
                allFieldItems.add(s);
            }
        }
        query = String.join(" ", splitted);
        if (!allFieldItems.isEmpty()) {<commit_msg>Improved underlying bool ES query.<commit_after>package com.rewayaat.core;

import org.apache.commons.lang.StringUtils;

import java.util.ArrayList;
import java.util.List;

/**
 * Uses Elastic Search Query Syntax to Modify user queries to improve search
 * results. This is done mostly by removing weird characters, adding <a href=
 * "https://www.elastic.co/guide/en/elasticsearch/guide/current/fuzziness.html">fuzziness</a>,
 * and adding <a href=
 * "https://www.elastic.co/guide/en/elasticsearch/guide/current/slop.html">slop</a>
 * to phrases.
 */
public class RewayaatQuery {

    private String query;

    private String[] docFields = new String[]{"_id:", "source:", "book:", "number:", "part:", "edition:", "chapter:", "publisher:", "section:", "tags:", "volume:", "notes:", "arabic:", "gradings:"};
    public RewayaatQuery(String query) {
        this.query = query;
    }

    public String query() {
        // splits query by all spaces that are not enclosed by double quotes or brackets
        List<String> splitted = new ArrayList<>();
        List<String> allFieldItems = new ArrayList<>();
        int nextingLevel = 0;
        StringBuilder result = new StringBuilder();
        for (char c : query.toCharArray()) {
            if (c == ' ' && nextingLevel == 0) {
                splitted.add(result.toString());
                result.setLength(0);
            } else {
                if (c == '(' | c == '[' | c == '\"') {
                    nextingLevel++;
                } else if (c == ')' | c == ']' | c == '\"') {
                    nextingLevel--;
                }
                result.append(c);
            }
        }
        splitted.add(result.toString());
        for (String s : splitted) {
            if (!s.contains("~") && !s.contains("(")) {
                if (s.length() > 4) {
                    if (s.length() > 6) {
                        s += "~2";
                    } else {
                        s += "~1";
                    }
                }
            }
            if (!StringUtils.startsWithAny(s, docFields)) {
                allFieldItems.add(s);
            }
        }
        query = String.join(" ", splitted);
        if (!allFieldItems.isEmpty()) {
{'fraction_non_alphanumeric': 0.15791886316754528, 'fraction_numerical': 0.002979601191840477, 'mean_word_length': 1.8356075373619234, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 22, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1215860', 'n_tokens_mistral': 1275, 'n_tokens_neox': 1206, 'n_words': 305}