<commit_before>package com.psddev.dari.db;

import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MoreLikeThisParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.psddev.dari.util.Lazy;
import com.psddev.dari.util.ObjectUtils;
import com.psddev.dari.util.PaginatedResult;
import com.psddev.dari.util.Profiler;
import com.psddev.dari.util.Settings;
import com.psddev.dari.util.SettingsException;
import com.psddev.dari.util.Stats;
import com.psddev.dari.util.UuidUtils;

/**
 * Database backed by a
 * <a href="http://lucene.apache.org/solr/">Apache Solr</a> server.
 */
public class SolrDatabase extends AbstractDatabase<SolrServer> {

    public static final String DEFAULT_COMMIT_WITHIN_SETTING = "dari/solrDefaultCommitWithin";

    public static final String SERVER_URL_SUB_SETTING = "serverUrl";
    public static final String READ_SERVER_URL_SUB_SETTING = "readServerUrl";
    public static final String TENANT_SUB_SETTING = "tenant";
    public static final String COMMIT_WITHIN_SUB_SETTING = "commitWithin";
    public static final String VERSION_SUB_SETTING = "version";
    public static final String SAVE_DATA_SUB_SETTING = "saveData";

    public static final double DEFAULT_COMMIT_WITHIN = 0.0;

    public static final String TENANT_FIELD = "_s__tenant";
    public static final String ID_FIELD = "id";
    public static final String TYPE_ID_FIELD = "typeId";
    public static final String DATA_FIELD = "data";
    public static final String FIELDS_FIELD = "_t__fields";
    public static final String ALL_FIELD = "all";
    public static final String SCORE_FIELD = "score";

    public static final String SUGGESTION_FIELD = "_e_suggestField";

    public static final String SCORE_EXTRA = "solr.score";
    public static final String NORMALIZED_SCORE_EXTRA = "solr.normalizedScore";

    private static final int INITIAL_FETCH_SIZE = 100;
    private static final Logger LOGGER = LoggerFactory.getLogger(SolrDatabase.class);
    private static final Pattern UUID_PATTERN = Pattern.compile("([A-Fa-f0-9]{8})-([A-Fa-f0-9]{4})-([A-Fa-f0-9]{4})-([A-Fa-f0-9]{4})-([A-Fa-f0-9]{12})");

    private static final String SHORT_NAME = "Solr";
    private static final Stats STATS = new Stats(SHORT_NAME);
    private static final String ADD_STATS_OPERATION = "Add";
    private static final String COMMIT_STATS_OPERATION = "Commit";
    private static final String DELETE_STATS_OPERATION = "Delete";
    private static final String QUERY_STATS_OPERATION = "Query";
    private static final String ADD_PROFILER_EVENT = SHORT_NAME + " " + ADD_STATS_OPERATION;
    private static final String COMMIT_PROFILER_EVENT = SHORT_NAME + " " + COMMIT_STATS_OPERATION;
    private static final String DELETE_PROFILER_EVENT = SHORT_NAME + " " + DELETE_STATS_OPERATION;
    private static final String QUERY_PROFILER_EVENT = SHORT_NAME + " " + QUERY_STATS_OPERATION;
    private static final int MAX_BINARY_FIELD_LENGTH = 32766;
    private static final Set<String> TRUNCATE_FIELD_PREFIXES = new HashSet<>();

    static {
        TRUNCATE_FIELD_PREFIXES.add("_sl_");
        TRUNCATE_FIELD_PREFIXES.add("_ss_");
    }

    private volatile SolrServer server;
    private volatile SolrServer readServer;
    private volatile String tenant;
    private volatile Double commitWithin;
    private volatile String version;
    private volatile boolean saveData = true;
<commit_msg>Changes MAX_BINARY_FIELD_LENGTH in SolrDatabase to match SqlDatabase behavior.<commit_after>package com.psddev.dari.db;

import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MoreLikeThisParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.psddev.dari.util.Lazy;
import com.psddev.dari.util.ObjectUtils;
import com.psddev.dari.util.PaginatedResult;
import com.psddev.dari.util.Profiler;
import com.psddev.dari.util.Settings;
import com.psddev.dari.util.SettingsException;
import com.psddev.dari.util.Stats;
import com.psddev.dari.util.UuidUtils;

/**
 * Database backed by a
 * <a href="http://lucene.apache.org/solr/">Apache Solr</a> server.
 */
public class SolrDatabase extends AbstractDatabase<SolrServer> {

    public static final String DEFAULT_COMMIT_WITHIN_SETTING = "dari/solrDefaultCommitWithin";

    public static final String SERVER_URL_SUB_SETTING = "serverUrl";
    public static final String READ_SERVER_URL_SUB_SETTING = "readServerUrl";
    public static final String TENANT_SUB_SETTING = "tenant";
    public static final String COMMIT_WITHIN_SUB_SETTING = "commitWithin";
    public static final String VERSION_SUB_SETTING = "version";
    public static final String SAVE_DATA_SUB_SETTING = "saveData";

    public static final double DEFAULT_COMMIT_WITHIN = 0.0;

    public static final String TENANT_FIELD = "_s__tenant";
    public static final String ID_FIELD = "id";
    public static final String TYPE_ID_FIELD = "typeId";
    public static final String DATA_FIELD = "data";
    public static final String FIELDS_FIELD = "_t__fields";
    public static final String ALL_FIELD = "all";
    public static final String SCORE_FIELD = "score";

    public static final String SUGGESTION_FIELD = "_e_suggestField";

    public static final String SCORE_EXTRA = "solr.score";
    public static final String NORMALIZED_SCORE_EXTRA = "solr.normalizedScore";

    private static final int INITIAL_FETCH_SIZE = 100;
    private static final Logger LOGGER = LoggerFactory.getLogger(SolrDatabase.class);
    private static final Pattern UUID_PATTERN = Pattern.compile("([A-Fa-f0-9]{8})-([A-Fa-f0-9]{4})-([A-Fa-f0-9]{4})-([A-Fa-f0-9]{4})-([A-Fa-f0-9]{12})");

    private static final String SHORT_NAME = "Solr";
    private static final Stats STATS = new Stats(SHORT_NAME);
    private static final String ADD_STATS_OPERATION = "Add";
    private static final String COMMIT_STATS_OPERATION = "Commit";
    private static final String DELETE_STATS_OPERATION = "Delete";
    private static final String QUERY_STATS_OPERATION = "Query";
    private static final String ADD_PROFILER_EVENT = SHORT_NAME + " " + ADD_STATS_OPERATION;
    private static final String COMMIT_PROFILER_EVENT = SHORT_NAME + " " + COMMIT_STATS_OPERATION;
    private static final String DELETE_PROFILER_EVENT = SHORT_NAME + " " + DELETE_STATS_OPERATION;
    private static final String QUERY_PROFILER_EVENT = SHORT_NAME + " " + QUERY_STATS_OPERATION;
    private static final int MAX_BINARY_FIELD_LENGTH = 500;
    private static final Set<String> TRUNCATE_FIELD_PREFIXES = new HashSet<>();

    static {
        TRUNCATE_FIELD_PREFIXES.add("_sl_");
        TRUNCATE_FIELD_PREFIXES.add("_ss_");
    }

    private volatile SolrServer server;
    private volatile SolrServer readServer;
    private volatile String tenant;
    private volatile Double commitWithin;
    private volatile String version;
    private volatile boolean saveData = true;

{'fraction_non_alphanumeric': 0.09590207557211283, 'fraction_numerical': 0.005747738158594997, 'mean_word_length': 4.943074003795067, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3014177', 'n_tokens_mistral': 3070, 'n_tokens_neox': 3010, 'n_words': 677}