<commit_before>/*
 * Licensed to Elastic Search and <NAME> under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. Elastic Search licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.shard.recovery;

import com.google.inject.Inject;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.ElasticSearchInterruptedException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.node.Node;
import org.elasticsearch.index.deletionpolicy.SnapshotIndexCommit;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.RecoveryEngineException;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.*;
import org.elasticsearch.index.shard.service.IndexShard;
import org.elasticsearch.index.shard.service.InternalIndexShard;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.index.translog.memory.MemorySnapshot;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;
import org.elasticsearch.util.SizeUnit;
import org.elasticsearch.util.SizeValue;
import org.elasticsearch.util.StopWatch;
import org.elasticsearch.util.TimeValue;
import org.elasticsearch.util.io.Streamable;
import org.elasticsearch.util.io.VoidStreamable;
import org.elasticsearch.util.settings.Settings;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicReference;

import static java.util.concurrent.TimeUnit.*;
import static org.elasticsearch.util.concurrent.ConcurrentMaps.*;

/**
 * @author kimchy (shay.banon)
 */
public class RecoveryAction extends AbstractIndexShardComponent {

    private final SizeValue fileChunkSize;

    private final ThreadPool threadPool;

    private final TransportService transportService;

    private final InternalIndexShard indexShard;

    private final Store store;

    private final ConcurrentMap<String, IndexOutput> openIndexOutputs = newConcurrentMap();

    private final String startTransportAction;

    private final String fileChunkTransportAction;

    private final String snapshotTransportAction;

    private volatile boolean closed = false;

    private volatile Thread sendStartRecoveryThread;

    private volatile Thread receiveSnapshotRecoveryThread;

    private volatile Thread sendSnapshotRecoveryThread;

    private final CopyOnWriteArrayList<Future> sendFileChunksRecoveryFutures = new CopyOnWriteArrayList<Future>();

    @Inject public RecoveryAction(ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool, TransportService transportService, IndexShard indexShard, Store store) {
        super(shardId, indexSettings);
        this.threadPool = threadPool;
        this.transportService = transportService;
        this.indexShard = (InternalIndexShard) indexShard;
        this.store = store;

        startTransportAction = shardId.index().name() + "/" + shardId.id() + "/recovery/start";
        transportService.registerHandler(startTransportAction, new StartRecoveryTransportRequestHandler());
        fileChunkTransportAction = shardId.index().name() + "/" + shardId.id() + "/recovery/fileChunk";
        transportService.registerHandler(fileChunkTransportAction, new FileChunkTransportRequestHandler());
        snapshotTransportAction = shardId.index().name() + "/" + shardId.id() + "/recovery/snapshot";
        transportService.registerHandler(snapshotTransportAction, new SnapshotTransportRequestHandler());

        this.fileChunkSize = componentSettings.getAsSize("fileChunkSize", new SizeValue(16, SizeUnit.KB));
        logger.trace("Recovery Action registered, using fileChunkSize[{}]", fileChunkSize);
    }

    public void close() {
        closed = true;
        transportService.removeHandler(startTransportAction);
        transportService.removeHandler(fileChunkTransportAction);<commit_msg>change default recovery buffer to 100k from 16k<commit_after>/*
 * Licensed to Elastic Search and <NAME> under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. Elastic Search licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.shard.recovery;

import com.google.inject.Inject;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.ElasticSearchInterruptedException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.node.Node;
import org.elasticsearch.index.deletionpolicy.SnapshotIndexCommit;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.RecoveryEngineException;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.*;
import org.elasticsearch.index.shard.service.IndexShard;
import org.elasticsearch.index.shard.service.InternalIndexShard;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.index.translog.memory.MemorySnapshot;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;
import org.elasticsearch.util.SizeUnit;
import org.elasticsearch.util.SizeValue;
import org.elasticsearch.util.StopWatch;
import org.elasticsearch.util.TimeValue;
import org.elasticsearch.util.io.Streamable;
import org.elasticsearch.util.io.VoidStreamable;
import org.elasticsearch.util.settings.Settings;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicReference;

import static java.util.concurrent.TimeUnit.*;
import static org.elasticsearch.util.concurrent.ConcurrentMaps.*;

/**
 * @author kimchy (shay.banon)
 */
public class RecoveryAction extends AbstractIndexShardComponent {

    private final SizeValue fileChunkSize;

    private final ThreadPool threadPool;

    private final TransportService transportService;

    private final InternalIndexShard indexShard;

    private final Store store;

    private final ConcurrentMap<String, IndexOutput> openIndexOutputs = newConcurrentMap();

    private final String startTransportAction;

    private final String fileChunkTransportAction;

    private final String snapshotTransportAction;

    private volatile boolean closed = false;

    private volatile Thread sendStartRecoveryThread;

    private volatile Thread receiveSnapshotRecoveryThread;

    private volatile Thread sendSnapshotRecoveryThread;

    private final CopyOnWriteArrayList<Future> sendFileChunksRecoveryFutures = new CopyOnWriteArrayList<Future>();

    @Inject public RecoveryAction(ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool, TransportService transportService, IndexShard indexShard, Store store) {
        super(shardId, indexSettings);
        this.threadPool = threadPool;
        this.transportService = transportService;
        this.indexShard = (InternalIndexShard) indexShard;
        this.store = store;

        startTransportAction = shardId.index().name() + "/" + shardId.id() + "/recovery/start";
        transportService.registerHandler(startTransportAction, new StartRecoveryTransportRequestHandler());
        fileChunkTransportAction = shardId.index().name() + "/" + shardId.id() + "/recovery/fileChunk";
        transportService.registerHandler(fileChunkTransportAction, new FileChunkTransportRequestHandler());
        snapshotTransportAction = shardId.index().name() + "/" + shardId.id() + "/recovery/snapshot";
        transportService.registerHandler(snapshotTransportAction, new SnapshotTransportRequestHandler());

        this.fileChunkSize = componentSettings.getAsSize("fileChunkSize", new SizeValue(100, SizeUnit.KB));
        logger.trace("Recovery Action registered, using fileChunkSize[{}]", fileChunkSize);
    }

    public void close() {
        closed = true;
        transportService.removeHandler(startTransportAction);
        transportService.removeHandler(fileChunkTransportAction);
{'fraction_non_alphanumeric': 0.08437025796661608, 'fraction_numerical': 0.0018209408194233688, 'mean_word_length': 5.148009950248756, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5456667', 'n_tokens_mistral': 2510, 'n_tokens_neox': 2515, 'n_words': 658}