<commit_before>package de.uni_tuebingen.ub.ixTheo.canonesRangeSearch;


import java.util.Set;
import java.util.TreeSet;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class CanonesRangeParser extends QParser {
    private final static String QUERY_SEPARATOR = " ";
    private final static String DB_FIELD_SEPARATOR = ",";
    private static Logger logger = LoggerFactory.getLogger(CanonesRangeParser.class);

    /**
     * @param qstr        The part of the query string specific to this parser
     * @param localParams The set of parameters that are specific to this QParser.
     *                    See http://wiki.apache.org/solr/LocalParams
     * @param params      The rest of the {@link SolrParams}
     * @param req         The original {@link SolrQueryRequest}.
     */
    public CanonesRangeParser(final String qstr, final SolrParams localParams, final SolrParams params, final SolrQueryRequest req) {
        super(qstr, localParams, params, req);
    }

    static CanonesRange[] getRangesFromDatabaseField(final String db_field) {
        return CanonesRange.getRanges(db_field, DB_FIELD_SEPARATOR);
    }

    private CanonesRange[] getRangesFromQuery() {
        return CanonesRange.getRanges(getFieldsFromQuery());
    }

    private String[] getFieldsFromQuery() {
        return qstr.split(QUERY_SEPARATOR);
    }

    // @return true if "queryString" is of the form 07000000_08999999 o/w we return false.
    private boolean isCanonesRange(final String queryString) {
        if (queryString.length() != CanonesRange.CANON_LAW_CODE_LENGTH + 1 + CanonesRange.CANON_LAW_CODE_LENGTH
            || queryString.charAt(CanonesRange.CANON_LAW_CODE_LENGTH) != '_')
            return false;
        if (!queryString.substring(1, CanonesRange.CANON_LAW_CODE_LENGTH).equals("00000000")
            || !queryString.substring(CanonesRange.CANON_LAW_CODE_LENGTH + 1 + 1,
                                      CanonesRange.CANON_LAW_CODE_LENGTH + 1 + CanonesRange.CANON_LAW_CODE_LENGTH).equals("99999999"))
            return false;
        int firstCodexDigit, secondCodexDigit;
        try {
            firstCodexDigit  = Integer.parseInt(queryString.substring(0, 1));
            secondCodexDigit = Integer.parseInt(queryString.substring(CanonesRange.CANON_LAW_CODE_LENGTH + 1,
                                                                      CanonesRange.CANON_LAW_CODE_LENGTH + 1 + 1));
        } catch (NumberFormatException e) {
            return false;
        }

        return (firstCodexDigit == 1 || firstCodexDigit == 2 || firstCodexDigit == 3) && secondCodexDigit == firstCodexDigit;
    }

    /**
     * Tries to extract the codex index of a search query.
     * Then creates a query string only matching canon law references starting with the codex index.
     * If no codex index is found, only '*' will be returned.
     *
     * The first digit of a range is the codex index.
     *
     * @param queryString The search string from user
     * @return e.g.  ".*(1|2)[0-9]{8}.*" (NB. the Solr query parser anchors regular expressions at the
     * beginning and at the end) or "*"
     */
    private String getCanonesPrefixQueryString(final String queryString) {
        if (queryString == null || queryString.length() < 1) {
            return "*";
        }
        if (isCanonesRange(queryString)) {
            return "/.*" + queryString + ".*/";
        }
        final String[] ranges = getFieldsFromQuery();
        final Set<String> alreadySeenCodexDigits = new TreeSet<String>();
        // Capacity of buffer: (number of ranges) times (two digits of book and one delimiter)
        StringBuilder buffer = new StringBuilder(ranges.length * 2);
        for (String range : ranges) {
            final String firstCodexDigit = range.substring(0, 1);
            if (!alreadySeenCodexDigits.contains(firstCodexDigit)) {
                buffer.append("|" + firstCodexDigit);
                alreadySeenCodexDigits.add(firstCodexDigit);
            }
            final String secondCodexDigit = range.substring(CanonesRange.CANON_LAW_CODE_LENGTH + 1,
                                                            CanonesRange.CANON_LAW_CODE_LENGTH + 1 + 1);
            if (!alreadySeenCodexDigits.contains(secondCodexDigit)) {
                buffer.append("|" + secondCodexDigit);
                alreadySeenCodexDigits.add(secondCodexDigit);
            }
        }
        return "/.*(" + buffer.toString().substring(1) + ")[0-9]{8}.*/";
    }
<commit_msg>Fixed a comment.<commit_after>package de.uni_tuebingen.ub.ixTheo.canonesRangeSearch;


import java.util.Set;
import java.util.TreeSet;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class CanonesRangeParser extends QParser {
    private final static String QUERY_SEPARATOR = " ";
    private final static String DB_FIELD_SEPARATOR = ",";
    private static Logger logger = LoggerFactory.getLogger(CanonesRangeParser.class);

    /**
     * @param qstr        The part of the query string specific to this parser
     * @param localParams The set of parameters that are specific to this QParser.
     *                    See http://wiki.apache.org/solr/LocalParams
     * @param params      The rest of the {@link SolrParams}
     * @param req         The original {@link SolrQueryRequest}.
     */
    public CanonesRangeParser(final String qstr, final SolrParams localParams, final SolrParams params, final SolrQueryRequest req) {
        super(qstr, localParams, params, req);
    }

    static CanonesRange[] getRangesFromDatabaseField(final String db_field) {
        return CanonesRange.getRanges(db_field, DB_FIELD_SEPARATOR);
    }

    private CanonesRange[] getRangesFromQuery() {
        return CanonesRange.getRanges(getFieldsFromQuery());
    }

    private String[] getFieldsFromQuery() {
        return qstr.split(QUERY_SEPARATOR);
    }

    // @return true if "queryString" is of the form 07000000_08999999 o/w we return false.
    private boolean isCanonesRange(final String queryString) {
        if (queryString.length() != CanonesRange.CANON_LAW_CODE_LENGTH + 1 + CanonesRange.CANON_LAW_CODE_LENGTH
            || queryString.charAt(CanonesRange.CANON_LAW_CODE_LENGTH) != '_')
            return false;
        if (!queryString.substring(1, CanonesRange.CANON_LAW_CODE_LENGTH).equals("00000000")
            || !queryString.substring(CanonesRange.CANON_LAW_CODE_LENGTH + 1 + 1,
                                      CanonesRange.CANON_LAW_CODE_LENGTH + 1 + CanonesRange.CANON_LAW_CODE_LENGTH).equals("99999999"))
            return false;
        int firstCodexDigit, secondCodexDigit;
        try {
            firstCodexDigit  = Integer.parseInt(queryString.substring(0, 1));
            secondCodexDigit = Integer.parseInt(queryString.substring(CanonesRange.CANON_LAW_CODE_LENGTH + 1,
                                                                      CanonesRange.CANON_LAW_CODE_LENGTH + 1 + 1));
        } catch (NumberFormatException e) {
            return false;
        }

        return (firstCodexDigit == 1 || firstCodexDigit == 2 || firstCodexDigit == 3) && secondCodexDigit == firstCodexDigit;
    }

    /**
     * Tries to extract the codex index of a search query.
     * Then creates a query string only matching canon law references starting with the codex index.
     * If no codex index is found, only '*' will be returned.
     *
     * The first digit of a range is the codex index.
     *
     * @param queryString The search string from user
     * @return e.g.  ".*(1|2)[0-9]{8}.*" (NB. the Solr query parser anchors regular expressions at the
     * beginning and at the end) or "*"
     */
    private String getCanonesPrefixQueryString(final String queryString) {
        if (queryString == null || queryString.length() < 1) {
            return "*";
        }
        if (isCanonesRange(queryString)) {
            return "/.*" + queryString + ".*/";
        }
        final String[] ranges = getFieldsFromQuery();
        final Set<String> alreadySeenCodexDigits = new TreeSet<String>();
        // Capacity of buffer: (number of ranges) times (one digit for the codex and one for the delimiter)
        StringBuilder buffer = new StringBuilder(ranges.length * 2);
        for (String range : ranges) {
            final String firstCodexDigit = range.substring(0, 1);
            if (!alreadySeenCodexDigits.contains(firstCodexDigit)) {
                buffer.append("|" + firstCodexDigit);
                alreadySeenCodexDigits.add(firstCodexDigit);
            }
            final String secondCodexDigit = range.substring(CanonesRange.CANON_LAW_CODE_LENGTH + 1,
                                                            CanonesRange.CANON_LAW_CODE_LENGTH + 1 + 1);
            if (!alreadySeenCodexDigits.contains(secondCodexDigit)) {
                buffer.append("|" + secondCodexDigit);
                alreadySeenCodexDigits.add(secondCodexDigit);
            }
        }
        return "/.*(" + buffer.toString().substring(1) + ")[0-9]{8}.*/";
    }

{'fraction_non_alphanumeric': 0.09532336212344639, 'fraction_numerical': 0.013271539919949442, 'mean_word_length': 2.5762711864406778, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4561097', 'n_tokens_mistral': 2818, 'n_tokens_neox': 2692, 'n_words': 651}