<commit_before>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.indexing;

import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;

/**
 */
public class ShardIndexingService extends AbstractIndexShardComponent {

    private final IndexingSlowLog slowLog;

    private final StatsHolder totalStats = new StatsHolder();

    private final CopyOnWriteArrayList<IndexingOperationListener> listeners = new CopyOnWriteArrayList<>();

    private volatile Map<String, StatsHolder> typesStats = ImmutableMap.of();

    public ShardIndexingService(ShardId shardId, Settings indexSettings) {
        super(shardId, indexSettings);
        this.slowLog = new IndexingSlowLog(indexSettings);
    }

    /**
     * Returns the stats, including type specific stats. If the types are null/0 length, then nothing
     * is returned for them. If they are set, then only types provided will be returned, or
     * <tt>_all</tt> for all types.
     */
    public IndexingStats stats(String... types) {
        IndexingStats.Stats total = totalStats.stats();
        Map<String, IndexingStats.Stats> typesSt = null;
        if (types != null && types.length > 0) {
            typesSt = new HashMap<>(typesStats.size());
            if (types.length == 1 && types[0].equals("_all")) {
                for (Map.Entry<String, StatsHolder> entry : typesStats.entrySet()) {
                    typesSt.put(entry.getKey(), entry.getValue().stats());
                }
            } else {
                for (Map.Entry<String, StatsHolder> entry : typesStats.entrySet()) {
                    if (Regex.simpleMatch(types, entry.getKey())) {
                        typesSt.put(entry.getKey(), entry.getValue().stats());
                    }
                }
            }
        }
        return new IndexingStats(total, typesSt);
    }

    public void addListener(IndexingOperationListener listener) {
        listeners.add(listener);
    }

    public void removeListener(IndexingOperationListener listener) {
        listeners.remove(listener);
    }

    public Engine.Create preCreate(Engine.Create create) {
        totalStats.indexCurrent.inc();
        typeStats(create.type()).indexCurrent.inc();
        for (IndexingOperationListener listener : listeners) {
            create = listener.preCreate(create);
        }
        return create;
    }

    public void postCreateUnderLock(Engine.Create create) {
        for (IndexingOperationListener listener : listeners) {
            listener.postCreateUnderLock(create);
        }
    }

    public void throttlingActivated() {
        totalStats.setThrottled(true);
    }

    public void throttlingDeactivated() {
        totalStats.setThrottled(false);
    }

    public void postCreate(Engine.Create create) {
        long took = create.endTime() - create.startTime();
        totalStats.indexMetric.inc(took);
        totalStats.indexCurrent.dec();
        StatsHolder typeStats = typeStats(create.type());
        typeStats.indexMetric.inc(took);
        typeStats.indexCurrent.dec();
        slowLog.postCreate(create, took);
        for (IndexingOperationListener listener : listeners) {
            listener.postCreate(create);
        }
    }

    public void postCreate(Engine.Create create, Throwable ex) {
        for (IndexingOperationListener listener : listeners) {
            listener.postCreate(create, ex);
        }
    }

    public Engine.Index preIndex(Engine.Index index) {
        totalStats.indexCurrent.inc();
        typeStats(index.type()).indexCurrent.inc();
        for (IndexingOperationListener listener : listeners) {
            listener.preIndex(index);
        }
        return index;
    }

    public void postIndexUnderLock(Engine.Index index) {
        for (IndexingOperationListener listener : listeners) {
            listener.postIndexUnderLock(index);
        }
    }

    public void postIndex(Engine.Index index) {
        long took = index.endTime() - index.startTime();
        totalStats.indexMetric.inc(took);
        totalStats.indexCurrent.dec();
        StatsHolder typeStats = typeStats(index.type());
        typeStats.indexMetric.inc(took);
        typeStats.indexCurrent.dec();
        slowLog.postIndex(index, took);
        for (IndexingOperationListener listener : listeners) {
            listener.postIndex(index);
        }<commit_msg>fix skipped variable setting<commit_after>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.indexing;

import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;

/**
 */
public class ShardIndexingService extends AbstractIndexShardComponent {

    private final IndexingSlowLog slowLog;

    private final StatsHolder totalStats = new StatsHolder();

    private final CopyOnWriteArrayList<IndexingOperationListener> listeners = new CopyOnWriteArrayList<>();

    private volatile Map<String, StatsHolder> typesStats = ImmutableMap.of();

    public ShardIndexingService(ShardId shardId, Settings indexSettings) {
        super(shardId, indexSettings);
        this.slowLog = new IndexingSlowLog(indexSettings);
    }

    /**
     * Returns the stats, including type specific stats. If the types are null/0 length, then nothing
     * is returned for them. If they are set, then only types provided will be returned, or
     * <tt>_all</tt> for all types.
     */
    public IndexingStats stats(String... types) {
        IndexingStats.Stats total = totalStats.stats();
        Map<String, IndexingStats.Stats> typesSt = null;
        if (types != null && types.length > 0) {
            typesSt = new HashMap<>(typesStats.size());
            if (types.length == 1 && types[0].equals("_all")) {
                for (Map.Entry<String, StatsHolder> entry : typesStats.entrySet()) {
                    typesSt.put(entry.getKey(), entry.getValue().stats());
                }
            } else {
                for (Map.Entry<String, StatsHolder> entry : typesStats.entrySet()) {
                    if (Regex.simpleMatch(types, entry.getKey())) {
                        typesSt.put(entry.getKey(), entry.getValue().stats());
                    }
                }
            }
        }
        return new IndexingStats(total, typesSt);
    }

    public void addListener(IndexingOperationListener listener) {
        listeners.add(listener);
    }

    public void removeListener(IndexingOperationListener listener) {
        listeners.remove(listener);
    }

    public Engine.Create preCreate(Engine.Create create) {
        totalStats.indexCurrent.inc();
        typeStats(create.type()).indexCurrent.inc();
        for (IndexingOperationListener listener : listeners) {
            create = listener.preCreate(create);
        }
        return create;
    }

    public void postCreateUnderLock(Engine.Create create) {
        for (IndexingOperationListener listener : listeners) {
            listener.postCreateUnderLock(create);
        }
    }

    public void throttlingActivated() {
        totalStats.setThrottled(true);
    }

    public void throttlingDeactivated() {
        totalStats.setThrottled(false);
    }

    public void postCreate(Engine.Create create) {
        long took = create.endTime() - create.startTime();
        totalStats.indexMetric.inc(took);
        totalStats.indexCurrent.dec();
        StatsHolder typeStats = typeStats(create.type());
        typeStats.indexMetric.inc(took);
        typeStats.indexCurrent.dec();
        slowLog.postCreate(create, took);
        for (IndexingOperationListener listener : listeners) {
            listener.postCreate(create);
        }
    }

    public void postCreate(Engine.Create create, Throwable ex) {
        for (IndexingOperationListener listener : listeners) {
            listener.postCreate(create, ex);
        }
    }

    public Engine.Index preIndex(Engine.Index index) {
        totalStats.indexCurrent.inc();
        typeStats(index.type()).indexCurrent.inc();
        for (IndexingOperationListener listener : listeners) {
            index = listener.preIndex(index);
        }
        return index;
    }

    public void postIndexUnderLock(Engine.Index index) {
        for (IndexingOperationListener listener : listeners) {
            listener.postIndexUnderLock(index);
        }
    }

    public void postIndex(Engine.Index index) {
        long took = index.endTime() - index.startTime();
        totalStats.indexMetric.inc(took);
        totalStats.indexCurrent.dec();
        StatsHolder typeStats = typeStats(index.type());
        typeStats.indexMetric.inc(took);
        typeStats.indexCurrent.dec();
        slowLog.postIndex(index, took);
        for (IndexingOperationListener listener : listeners) {
            listener.postIndex(index);
        }
{'fraction_non_alphanumeric': 0.09633388747921953, 'fraction_numerical': 0.0013999475019686762, 'mean_word_length': 2.906356801093643, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5834384', 'n_tokens_mistral': 2961, 'n_tokens_neox': 2908, 'n_words': 752}