<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package org.apache.jmeter.report.processor;

import java.io.File;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.lang3.Validate;
import org.apache.jmeter.report.core.CsvFile;
import org.apache.jmeter.report.core.CsvSampleReader;
import org.apache.jmeter.report.core.CsvSampleWriter;
import org.apache.jmeter.report.core.Sample;
import org.apache.jmeter.report.core.SampleComparator;
import org.apache.jmeter.report.core.SampleException;
import org.apache.jmeter.report.core.SampleMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * R-way external sample sorter.<br>
 * <p>
 * This SampleConsumer should be used to sort samples base on a
 * {@link SampleComparator}
 * </p>
 * <p>
 * Samples are sorted with the external sort algorithm. Thus, samples are not
 * all stored in memory to be sorted. Instead, they are sorted by chunk in
 * memory and then written to the disk before being merged at the end.<br>
 * </p>
 * <p>
 * This sorter makes it possible to sort any number of samples with a fixed
 * amount of memory. Hard disk will be used instead of RAM, at the cost of
 * performance
 * </p>
 * <p>
 * When <b>parallel mode</b> is enabled and several CPU are available to the
 * JVM, this sorter uses multiple CPU to reduce sort time.<br>
 * The <b>parallel mode</b> can be disabled if some sort of concurrency issue is
 * encountered.
 * </p>
 * <p>
 * As a last note, this {@link SampleConsumer} can be used as normal class with
 * the different <code>sort()</code> methods
 * </p>
 * <p>
 * It is important to set the <b><code>chunkSize</code></b> property according
 * to the available memory as the algorithm does not take care of memory
 * allocation (samples sizes are not predictable)
 * </p>
 * <p>
 * Meanwhile, it is equally important to set a {@link SampleComparator} to
 * define sample ordering </>
 * 
 * @since 2.14
 */
public class ExternalSampleSorter extends AbstractSampleConsumer {

    private static final String MUST_NOT_BE_NULL = "%s must not be null";

    private static final Logger log = LoggerFactory
            .getLogger(ExternalSampleSorter.class);

    private static final int DEFAULT_CHUNK_SIZE = 50000;

    private long chunkSize = DEFAULT_CHUNK_SIZE;

    private SampleComparator sampleComparator;

    private BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>();

    private ThreadPoolExecutor pool;

    private volatile int nbProcessors;

    private boolean parallelize;

    private AtomicLong chunkedSampleCount = new AtomicLong();

    private AtomicLong inputSampleCount = new AtomicLong();

    private LinkedList<File> chunks;

    private LinkedList<Sample> samples;

    private SampleMetadata sampleMetadata;

    private boolean revertedSort;
    
    private AtomicInteger sequence = new AtomicInteger();


    public ExternalSampleSorter() {
        chunkSize = DEFAULT_CHUNK_SIZE;
        this.nbProcessors = Runtime.getRuntime().availableProcessors();
        this.parallelize = nbProcessors > 1;
        this.pool = new ThreadPoolExecutor(nbProcessors, nbProcessors + 5, 10,
                TimeUnit.SECONDS, workQueue);
        setRevertedSort(false);
    }

    public ExternalSampleSorter(SampleComparator comparator) {
        this();
        setSampleComparator(comparator);
    }

    /**
     * Set the number of samples that will be stored in memory. This
     * defines the number of samples that will be written in each chunk file<commit_msg>Might as well be final<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package org.apache.jmeter.report.processor;

import java.io.File;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.lang3.Validate;
import org.apache.jmeter.report.core.CsvFile;
import org.apache.jmeter.report.core.CsvSampleReader;
import org.apache.jmeter.report.core.CsvSampleWriter;
import org.apache.jmeter.report.core.Sample;
import org.apache.jmeter.report.core.SampleComparator;
import org.apache.jmeter.report.core.SampleException;
import org.apache.jmeter.report.core.SampleMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * R-way external sample sorter.<br>
 * <p>
 * This SampleConsumer should be used to sort samples base on a
 * {@link SampleComparator}
 * </p>
 * <p>
 * Samples are sorted with the external sort algorithm. Thus, samples are not
 * all stored in memory to be sorted. Instead, they are sorted by chunk in
 * memory and then written to the disk before being merged at the end.<br>
 * </p>
 * <p>
 * This sorter makes it possible to sort any number of samples with a fixed
 * amount of memory. Hard disk will be used instead of RAM, at the cost of
 * performance
 * </p>
 * <p>
 * When <b>parallel mode</b> is enabled and several CPU are available to the
 * JVM, this sorter uses multiple CPU to reduce sort time.<br>
 * The <b>parallel mode</b> can be disabled if some sort of concurrency issue is
 * encountered.
 * </p>
 * <p>
 * As a last note, this {@link SampleConsumer} can be used as normal class with
 * the different <code>sort()</code> methods
 * </p>
 * <p>
 * It is important to set the <b><code>chunkSize</code></b> property according
 * to the available memory as the algorithm does not take care of memory
 * allocation (samples sizes are not predictable)
 * </p>
 * <p>
 * Meanwhile, it is equally important to set a {@link SampleComparator} to
 * define sample ordering </>
 * 
 * @since 2.14
 */
public class ExternalSampleSorter extends AbstractSampleConsumer {

    private static final String MUST_NOT_BE_NULL = "%s must not be null";

    private static final Logger log = LoggerFactory
            .getLogger(ExternalSampleSorter.class);

    private static final int DEFAULT_CHUNK_SIZE = 50000;

    private long chunkSize = DEFAULT_CHUNK_SIZE;

    private SampleComparator sampleComparator;

    private final BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>();

    private ThreadPoolExecutor pool;

    private volatile int nbProcessors;

    private boolean parallelize;

    private final AtomicLong chunkedSampleCount = new AtomicLong();

    private final AtomicLong inputSampleCount = new AtomicLong();

    private LinkedList<File> chunks;

    private LinkedList<Sample> samples;

    private SampleMetadata sampleMetadata;

    private boolean revertedSort;
    
    private final AtomicInteger sequence = new AtomicInteger();


    public ExternalSampleSorter() {
        chunkSize = DEFAULT_CHUNK_SIZE;
        this.nbProcessors = Runtime.getRuntime().availableProcessors();
        this.parallelize = nbProcessors > 1;
        this.pool = new ThreadPoolExecutor(nbProcessors, nbProcessors + 5, 10,
                TimeUnit.SECONDS, workQueue);
        setRevertedSort(false);
    }

    public ExternalSampleSorter(SampleComparator comparator) {
        this();
        setSampleComparator(comparator);
    }

    /**
     * Set the number of samples that will be stored in memory. This
     * defines the number of samples that will be written in each chunk file
{'fraction_non_alphanumeric': 0.08195496745277986, 'fraction_numerical': 0.0040550634937573365, 'mean_word_length': 3.930036822724882, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5832700', 'n_tokens_mistral': 2586, 'n_tokens_neox': 2499, 'n_words': 969}