<commit_before>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.mapper;

import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.PrefixCodedTerms;
import org.apache.lucene.index.PrefixCodedTerms.TermIterator;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.lucene.all.AllTermQuery;
import org.elasticsearch.common.time.DateMathParser;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.mapper.CqlMapper.CqlCollection;
import org.elasticsearch.index.mapper.CqlMapper.CqlStruct;
import org.elasticsearch.index.query.QueryRewriteContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.search.DocValueFormat;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.util.List;
import java.util.Objects;

import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.db.marshal.AbstractType;
/**
 * This defines the core properties and functions to operate on a field.
 */
public abstract class MappedFieldType extends FieldType {

    private String name;
    private float boost;
    // TODO: remove this docvalues flag and use docValuesType
    private boolean docValues;
    private NamedAnalyzer indexAnalyzer;
    private NamedAnalyzer searchAnalyzer;
    private NamedAnalyzer searchQuoteAnalyzer;
    private SimilarityProvider similarity;
    private Object nullValue;
    private String nullValueAsString; // for sending null value to _all field
    private boolean eagerGlobalOrdinals;

    private CqlCollection cqlCollection = CqlCollection.LIST;
    private CqlStruct cqlStruct = CqlStruct.UDT;
    private CQL3Type cql3Type = CQL3Type.Native.TEXT;
    private boolean cqlPartialUpdate = true;
    private boolean cqlPartitionKey = false;
    private boolean cqlStaticColumn = false;
    private boolean cqlClusteringKeyDesc = false;
    private int cqlPrimaryKeyOrder = -1;

    protected MappedFieldType(MappedFieldType ref) {
        super(ref);
        this.name = ref.name();
        this.boost = ref.boost();
        this.docValues = ref.hasDocValues();
        this.indexAnalyzer = ref.indexAnalyzer();
        this.searchAnalyzer = ref.searchAnalyzer();
        this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer();
        this.similarity = ref.similarity();
        this.nullValue = ref.nullValue();
        this.nullValueAsString = ref.nullValueAsString();
        this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals;

        this.cqlCollection = ref.cqlCollection;
        this.cqlStruct = ref.cqlStruct;
        this.cql3Type = ref.cql3Type;
        this.cqlPartialUpdate = ref.cqlPartialUpdate;
        this.cqlPartitionKey = ref.cqlPartitionKey;
        this.cqlStaticColumn = ref.cqlStaticColumn;
        this.cqlPrimaryKeyOrder = ref.cqlPrimaryKeyOrder;
        this.cqlClusteringKeyDesc = ref.cqlClusteringKeyDesc;
    }

    public MappedFieldType() {
        setTokenized(true);
        setStored(false);
        setStoreTermVectors(false);
        setOmitNorms(false);
        setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
        setBoost(1.0f);
    }

    @Override
    public abstract MappedFieldType clone();

    /** Return a fielddata builder for this field
     *  @throws IllegalArgumentException if the fielddata is not supported on this type.
     *  An IllegalArgumentException is needed in order to return an http error 400
     *  when this error occurs in a request. see: {@link org.elasticsearch.ExceptionsHelper#status}
     *
     * @param fullyQualifiedIndexName the name of the index this field-data is build for
     * */
    public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
        throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]");
    }

    @Override
    public boolean equals(Object o) {
        if (!super.equals(o)) return false;
        MappedFieldType fieldType = (MappedFieldType) o;

        return boost == fieldType.boost &&
            docValues == fieldType.docValues &&
            Objects.equals(name, fieldType.name) &&
            Objects.equals(indexAnalyzer, fieldType.indexAnalyzer) &&
            Objects.equals(searchAnalyzer, fieldType.searchAnalyzer) &&
            Objects.equals(searchQuoteAnalyzer(), fieldType.searchQuoteAnalyzer()) &&
            Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) &&
            Objects.equals(nullValue, fieldType.nullValue) &&
            Objects.equals(nullValueAsString, fieldType.nullValueAsString) &&
            Objects.equals(cqlCollection, fieldType.cqlCollection) &&
            Objects.equals(cqlStruct, fieldType.cqlStruct) &&
            Objects.equals(cql3Type, fieldType.cql3Type) &&
            Objects.equals(cqlPartialUpdate, fieldType.cqlPartialUpdate) &&
            Objects.equals(cqlPartitionKey, fieldType.cqlPartitionKey) &&
            Objects.equals(cqlStaticColumn, fieldType.cqlStaticColumn) &&
            Objects.equals(cqlClusteringKeyDesc, fieldType.cqlClusteringKeyDesc) &&
            Objects.equals(cqlPrimaryKeyOrder, fieldType.cqlPrimaryKeyOrder);
    }

    public CqlCollection cqlCollection() {
        return this.cqlCollection;
    }
<commit_msg>Fix equals on similarity<commit_after>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.mapper;

import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.PrefixCodedTerms;
import org.apache.lucene.index.PrefixCodedTerms.TermIterator;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.lucene.all.AllTermQuery;
import org.elasticsearch.common.time.DateMathParser;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.mapper.CqlMapper.CqlCollection;
import org.elasticsearch.index.mapper.CqlMapper.CqlStruct;
import org.elasticsearch.index.query.QueryRewriteContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.search.DocValueFormat;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.util.List;
import java.util.Objects;

import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.db.marshal.AbstractType;
/**
 * This defines the core properties and functions to operate on a field.
 */
public abstract class MappedFieldType extends FieldType {

    private String name;
    private float boost;
    // TODO: remove this docvalues flag and use docValuesType
    private boolean docValues;
    private NamedAnalyzer indexAnalyzer;
    private NamedAnalyzer searchAnalyzer;
    private NamedAnalyzer searchQuoteAnalyzer;
    private SimilarityProvider similarity;
    private Object nullValue;
    private String nullValueAsString; // for sending null value to _all field
    private boolean eagerGlobalOrdinals;

    private CqlCollection cqlCollection = CqlCollection.LIST;
    private CqlStruct cqlStruct = CqlStruct.UDT;
    private CQL3Type cql3Type = CQL3Type.Native.TEXT;
    private boolean cqlPartialUpdate = true;
    private boolean cqlPartitionKey = false;
    private boolean cqlStaticColumn = false;
    private boolean cqlClusteringKeyDesc = false;
    private int cqlPrimaryKeyOrder = -1;

    protected MappedFieldType(MappedFieldType ref) {
        super(ref);
        this.name = ref.name();
        this.boost = ref.boost();
        this.docValues = ref.hasDocValues();
        this.indexAnalyzer = ref.indexAnalyzer();
        this.searchAnalyzer = ref.searchAnalyzer();
        this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer();
        this.similarity = ref.similarity();
        this.nullValue = ref.nullValue();
        this.nullValueAsString = ref.nullValueAsString();
        this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals;

        this.cqlCollection = ref.cqlCollection;
        this.cqlStruct = ref.cqlStruct;
        this.cql3Type = ref.cql3Type;
        this.cqlPartialUpdate = ref.cqlPartialUpdate;
        this.cqlPartitionKey = ref.cqlPartitionKey;
        this.cqlStaticColumn = ref.cqlStaticColumn;
        this.cqlPrimaryKeyOrder = ref.cqlPrimaryKeyOrder;
        this.cqlClusteringKeyDesc = ref.cqlClusteringKeyDesc;
    }

    public MappedFieldType() {
        setTokenized(true);
        setStored(false);
        setStoreTermVectors(false);
        setOmitNorms(false);
        setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
        setBoost(1.0f);
    }

    @Override
    public abstract MappedFieldType clone();

    /** Return a fielddata builder for this field
     *  @throws IllegalArgumentException if the fielddata is not supported on this type.
     *  An IllegalArgumentException is needed in order to return an http error 400
     *  when this error occurs in a request. see: {@link org.elasticsearch.ExceptionsHelper#status}
     *
     * @param fullyQualifiedIndexName the name of the index this field-data is build for
     * */
    public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
        throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]");
    }

    @Override
    public boolean equals(Object o) {
        if (!super.equals(o)) return false;
        MappedFieldType fieldType = (MappedFieldType) o;

        return boost == fieldType.boost &&
            docValues == fieldType.docValues &&
            Objects.equals(name, fieldType.name) &&
            Objects.equals(indexAnalyzer, fieldType.indexAnalyzer) &&
            Objects.equals(searchAnalyzer, fieldType.searchAnalyzer) &&
            Objects.equals(searchQuoteAnalyzer(), fieldType.searchQuoteAnalyzer()) &&
            Objects.equals(similarity(), fieldType.similarity()) &&
            Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) &&
            Objects.equals(nullValue, fieldType.nullValue) &&
            Objects.equals(nullValueAsString, fieldType.nullValueAsString) &&
            Objects.equals(cqlCollection, fieldType.cqlCollection) &&
            Objects.equals(cqlStruct, fieldType.cqlStruct) &&
            Objects.equals(cql3Type, fieldType.cql3Type) &&
            Objects.equals(cqlPartialUpdate, fieldType.cqlPartialUpdate) &&
            Objects.equals(cqlPartitionKey, fieldType.cqlPartitionKey) &&
            Objects.equals(cqlStaticColumn, fieldType.cqlStaticColumn) &&
            Objects.equals(cqlClusteringKeyDesc, fieldType.cqlClusteringKeyDesc) &&
            Objects.equals(cqlPrimaryKeyOrder, fieldType.cqlPrimaryKeyOrder);
    }

    public CqlCollection cqlCollection() {
        return this.cqlCollection;
    }

{'fraction_non_alphanumeric': 0.08584493868218665, 'fraction_numerical': 0.002909997921430056, 'mean_word_length': 3.901188455008489, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3752554', 'n_tokens_mistral': 4118, 'n_tokens_neox': 4021, 'n_words': 927}