<commit_before>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.mapper;

import org.apache.lucene.document.StoredField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.mapper.MapperService.MergeReason;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;

public class DocumentMapper implements ToXContentFragment {

    public static final class Builder {
        private final Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers;
        private final RootObjectMapper rootObjectMapper;
        private final ContentPath contentPath;
        private final IndexSettings indexSettings;
        private final IndexAnalyzers indexAnalyzers;
        private final DocumentParser documentParser;

        private Map<String, Object> meta;

        public Builder(RootObjectMapper.Builder builder, MapperService mapperService) {
            this(builder, mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), mapperService.documentParser(),
                mapperService.getMetadataMappers());
        }

        Builder(RootObjectMapper.Builder builder,
                IndexSettings indexSettings,
                IndexAnalyzers indexAnalyzers,
                DocumentParser documentParser,
                Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers) {
            this.indexSettings = indexSettings;
            this.indexAnalyzers = indexAnalyzers;
            this.documentParser = documentParser;
            this.contentPath = new ContentPath(1);
            this.rootObjectMapper = builder.build(contentPath);
            this.metadataMappers = metadataMappers;
        }

        public Builder meta(Map<String, Object> meta) {
            this.meta = meta;
            return this;
        }

        public Builder put(MetadataFieldMapper.Builder mapper) {
            MetadataFieldMapper metadataMapper = mapper.build(contentPath);
            metadataMappers.put(metadataMapper.getClass(), metadataMapper);
            return this;
        }

        public DocumentMapper build() {
            Objects.requireNonNull(rootObjectMapper, "Mapper builder must have the root object mapper set");
            Mapping mapping = new Mapping(
                    rootObjectMapper,
                    metadataMappers.values().toArray(new MetadataFieldMapper[0]),
                    meta);
            return new DocumentMapper(indexSettings, indexAnalyzers, documentParser, mapping);
        }
    }

    private final String type;
    private final Text typeText;
    private final CompressedXContent mappingSource;
    private final DocumentParser documentParser;
    private final MappingLookup mappingLookup;
    private final MetadataFieldMapper[] deleteTombstoneMetadataFieldMappers;
    private final MetadataFieldMapper[] noopTombstoneMetadataFieldMappers;

    private DocumentMapper(IndexSettings indexSettings,
                           IndexAnalyzers indexAnalyzers,
                           DocumentParser documentParser,
                           Mapping mapping) {
        this.type = mapping.root().name();
        this.typeText = new Text(this.type);
        this.documentParser = documentParser;
        this.mappingLookup = MappingLookup.fromMapping(mapping, documentParser, indexSettings, indexAnalyzers);

        try {
            mappingSource = new CompressedXContent(this, XContentType.JSON, ToXContent.EMPTY_PARAMS);
        } catch (Exception e) {
            throw new ElasticsearchGenerationException("failed to serialize source for type [" + type + "]", e);
        }

        final Collection<String> deleteTombstoneMetadataFields = Arrays.asList(VersionFieldMapper.NAME, IdFieldMapper.NAME,
            SeqNoFieldMapper.NAME, SeqNoFieldMapper.PRIMARY_TERM_NAME, SeqNoFieldMapper.TOMBSTONE_NAME);
        this.deleteTombstoneMetadataFieldMappers = Stream.of(mapping.metadataMappers)
            .filter(field -> deleteTombstoneMetadataFields.contains(field.name())).toArray(MetadataFieldMapper[]::new);
        final Collection<String> noopTombstoneMetadataFields = Arrays.asList(
            VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, SeqNoFieldMapper.PRIMARY_TERM_NAME, SeqNoFieldMapper.TOMBSTONE_NAME);
        this.noopTombstoneMetadataFieldMappers = Stream.of(mapping.metadataMappers)
            .filter(field -> noopTombstoneMetadataFields.contains(field.name())).toArray(MetadataFieldMapper[]::new);
    }

    public Mapping mapping() {
        return mappingLookup.getMapping();
    }

    public String type() {
        return this.type;
    }

    public Text typeText() {
        return this.typeText;
    }

    public Map<String, Object> meta() {
        return mapping().meta;<commit_msg>Remove unused typeText field from DocumentMapper (#67646)<commit_after>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.mapper;

import org.apache.lucene.document.StoredField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.mapper.MapperService.MergeReason;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;

public class DocumentMapper implements ToXContentFragment {

    public static final class Builder {
        private final Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers;
        private final RootObjectMapper rootObjectMapper;
        private final ContentPath contentPath;
        private final IndexSettings indexSettings;
        private final IndexAnalyzers indexAnalyzers;
        private final DocumentParser documentParser;

        private Map<String, Object> meta;

        public Builder(RootObjectMapper.Builder builder, MapperService mapperService) {
            this(builder, mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), mapperService.documentParser(),
                mapperService.getMetadataMappers());
        }

        Builder(RootObjectMapper.Builder builder,
                IndexSettings indexSettings,
                IndexAnalyzers indexAnalyzers,
                DocumentParser documentParser,
                Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers) {
            this.indexSettings = indexSettings;
            this.indexAnalyzers = indexAnalyzers;
            this.documentParser = documentParser;
            this.contentPath = new ContentPath(1);
            this.rootObjectMapper = builder.build(contentPath);
            this.metadataMappers = metadataMappers;
        }

        public Builder meta(Map<String, Object> meta) {
            this.meta = meta;
            return this;
        }

        public Builder put(MetadataFieldMapper.Builder mapper) {
            MetadataFieldMapper metadataMapper = mapper.build(contentPath);
            metadataMappers.put(metadataMapper.getClass(), metadataMapper);
            return this;
        }

        public DocumentMapper build() {
            Objects.requireNonNull(rootObjectMapper, "Mapper builder must have the root object mapper set");
            Mapping mapping = new Mapping(
                    rootObjectMapper,
                    metadataMappers.values().toArray(new MetadataFieldMapper[0]),
                    meta);
            return new DocumentMapper(indexSettings, indexAnalyzers, documentParser, mapping);
        }
    }

    private final String type;
    private final CompressedXContent mappingSource;
    private final DocumentParser documentParser;
    private final MappingLookup mappingLookup;
    private final MetadataFieldMapper[] deleteTombstoneMetadataFieldMappers;
    private final MetadataFieldMapper[] noopTombstoneMetadataFieldMappers;

    private DocumentMapper(IndexSettings indexSettings,
                           IndexAnalyzers indexAnalyzers,
                           DocumentParser documentParser,
                           Mapping mapping) {
        this.type = mapping.root().name();
        this.documentParser = documentParser;
        this.mappingLookup = MappingLookup.fromMapping(mapping, documentParser, indexSettings, indexAnalyzers);

        try {
            mappingSource = new CompressedXContent(this, XContentType.JSON, ToXContent.EMPTY_PARAMS);
        } catch (Exception e) {
            throw new ElasticsearchGenerationException("failed to serialize source for type [" + type + "]", e);
        }

        final Collection<String> deleteTombstoneMetadataFields = Arrays.asList(VersionFieldMapper.NAME, IdFieldMapper.NAME,
            SeqNoFieldMapper.NAME, SeqNoFieldMapper.PRIMARY_TERM_NAME, SeqNoFieldMapper.TOMBSTONE_NAME);
        this.deleteTombstoneMetadataFieldMappers = Stream.of(mapping.metadataMappers)
            .filter(field -> deleteTombstoneMetadataFields.contains(field.name())).toArray(MetadataFieldMapper[]::new);
        final Collection<String> noopTombstoneMetadataFields = Arrays.asList(
            VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, SeqNoFieldMapper.PRIMARY_TERM_NAME, SeqNoFieldMapper.TOMBSTONE_NAME);
        this.noopTombstoneMetadataFieldMappers = Stream.of(mapping.metadataMappers)
            .filter(field -> noopTombstoneMetadataFields.contains(field.name())).toArray(MetadataFieldMapper[]::new);
    }

    public Mapping mapping() {
        return mappingLookup.getMapping();
    }

    public String type() {
        return this.type;
    }

    public Map<String, Object> meta() {
        return mapping().meta;
{'fraction_non_alphanumeric': 0.07399819716463164, 'fraction_numerical': 0.0013931000573629436, 'mean_word_length': 3.3184713375796178, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5564834', 'n_tokens_mistral': 3006, 'n_tokens_neox': 3001, 'n_words': 761}