<commit_before>package com.github.vedenin.perfomace_test;

import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import org.apache.commons.io.IOUtils;
import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

/**
 * Created by vvedenin on 2/15/2016.
 *
 *
 */
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
@Fork(1)
@State(Scope.Benchmark)
public class ConvertInputStreamToStringBenchmark {
    private final static  String test1 = "test184768612876481276487612876417826487216478216784621784672816478216784621784621786478216478216784261784621782178647281647821647821697421687126784621874621786478216478216874";
    private final static  InputStream inputStream = IOUtils.toInputStream(test1, StandardCharsets.UTF_8);

    /*             1. Using ToInputStream of Apache Utils */
    @Benchmark
    public String apacheToInputStream() throws IOException {
        inputStream.mark(0);
        String result = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
        inputStream.reset();
        return result;
    }

    /*             2. Using JDK BufferedReader */
    @Benchmark
    public String jdkBufferedReader() throws IOException {
        inputStream.mark(0);
        BufferedReader str = new BufferedReader(new InputStreamReader(inputStream));
        String result = str.readLine();
        inputStream.reset();
        return result;
    }

    /*             3. Using guava InputStreamReader */
    @Benchmark
    public String guavaReaderInputStream() throws IOException {
        inputStream.mark(0);
        InputStream inputStream = IOUtils.toInputStream(test1, StandardCharsets.UTF_8);
        String result = CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8));
        inputStream.reset();
        return result;
    }

    /*             4. Using JDK Scanner */
    @Benchmark
    public String jdkScanner() throws IOException {
        inputStream.mark(0);
        java.util.Scanner s = new java.util.Scanner(inputStream).useDelimiter("\\A");
        String result = s.hasNext() ? s.next() : "";
        inputStream.reset();
        return result;
    }

    /*             5. Using Java 8 */
    @Benchmark
    public String jdkJava8() throws IOException {
        inputStream.mark(0);
        String result = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining("\n"));
        inputStream.reset();
        return result;
    }

    /*             6. Using Java 8 */
    @Benchmark
    public String jfkJava8parallel() throws IOException {
        inputStream.mark(0);
        String result = new BufferedReader(new InputStreamReader(inputStream)).lines().parallel().collect(Collectors.joining("\n"));
        inputStream.reset();
        return result;
    }


    public static void main(String[] args) throws Exception {
        ConvertInputStreamToStringBenchmark test = new ConvertInputStreamToStringBenchmark();
        System.out.println();
        System.out.println("apacheToInputStream : " + test.apacheToInputStream());
        System.out.println("jdkBufferedReader : " + test.jdkBufferedReader());
        System.out.println("guavaReaderInputStream : " + test.guavaReaderInputStream());
        System.out.println("jdkScanner : " + test.jdkScanner());
        System.out.println("jdkJava8 : " + test.jdkJava8());
        System.out.println("jfkJava8parallel : " + test.jfkJava8parallel());
        System.out.println();

        Options opt = new OptionsBuilder()
                .include(ConvertInputStreamToStringBenchmark.class.getSimpleName())
                .build();

        new Runner(opt).run();
    }

}
<commit_msg>[JPO-39] Add new test to ConvertInputStreamToStringBenchmark<commit_after>package com.github.vedenin.perfomace_test;

import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import org.apache.commons.io.IOUtils;
import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

/**
 * Created by vvedenin on 2/15/2016.
 */
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
@Fork(1)
@State(Scope.Benchmark)
public class ConvertInputStreamToStringBenchmark {
    private final static String test1 = "test184768612876481276487612876417826487216478216784621784672816478216784621784621786478216478216784261784621782178647281647821647821697421687126784621874621786478216478216874";
    private final static InputStream inputStream = IOUtils.toInputStream(test1, StandardCharsets.UTF_8);

    /*             1. Using ToInputStream of Apache Utils */
    @Benchmark
    public String apacheToInputStream() throws IOException {
        inputStream.mark(0);
        String result = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
        inputStream.reset();
        return result;
    }

    /*             2. Using JDK BufferedReader */
    @Benchmark
    public String jdkBufferedReader() throws IOException {
        inputStream.mark(0);
        BufferedReader str = new BufferedReader(new InputStreamReader(inputStream));
        String result = str.readLine();
        inputStream.reset();
        return result;
    }

    /*             3. Using guava InputStreamReader */
    @Benchmark
    public String guavaReaderInputStream() throws IOException {
        inputStream.mark(0);
        InputStream inputStream = IOUtils.toInputStream(test1, StandardCharsets.UTF_8);
        String result = CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8));
        inputStream.reset();
        return result;
    }

    /*             4. Using JDK Scanner */
    @Benchmark
    public String jdkScanner() throws IOException {
        inputStream.mark(0);
        java.util.Scanner s = new java.util.Scanner(inputStream).useDelimiter("\\A");
        String result = s.hasNext() ? s.next() : "";
        inputStream.reset();
        return result;
    }

    /*             5. Using Java 8 */
    @Benchmark
    public String jdkJava8() throws IOException {
        inputStream.mark(0);
        String result = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining("\n"));
        inputStream.reset();
        return result;
    }

    /*             6. Using Java 8 */
    @Benchmark
    public String jfkJava8parallel() throws IOException {
        inputStream.mark(0);
        String result = new BufferedReader(new InputStreamReader(inputStream)).lines().parallel().collect(Collectors.joining("\n"));
        inputStream.reset();
        return result;
    }

    /*             7. Using InputStreamReader manually */
    @Benchmark
    public String manually(final InputStream is, final int bufferSize) throws IOException {
        inputStream.mark(0);
        final char[] buffer = new char[bufferSize];
        final StringBuilder out = new StringBuilder();
        Reader in = new InputStreamReader(inputStream, "UTF-8");
        for (; ; ) {
            int rsz = in.read(buffer, 0, buffer.length);
            if (rsz < 0)
                break;
            out.append(buffer, 0, rsz);
        }
        inputStream.reset();
        return out.toString();
    }

    /*             8. Using InputStreamReader manually 2 */
    @Benchmark
    public String manually2(final InputStream is, final int bufferSize) throws IOException {
        inputStream.mark(0);
        StringBuilder sb = new StringBuilder();
        BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
        String read;

        while ((read = br.readLine()) != null) {
            //System.out.println(read);
            sb.append(read);
        }
        inputStream.reset();
        return sb.toString();
    }

    public static void main(String[] args) throws Exception {
        ConvertInputStreamToStringBenchmark test = new ConvertInputStreamToStringBenchmark();
        System.out.println();
        System.out.println("apacheToInputStream : " + test.apacheToInputStream());
        System.out.println("jdkBufferedReader : " + test.jdkBufferedReader());
        System.out.println("guavaReaderInputStream : " + test.guavaReaderInputStream());
        System.out.println("jdkScanner : " + test.jdkScanner());
        System.out.println("jdkJava8 : " + test.jdkJava8());
        System.out.println("jfkJava8parallel : " + test.jfkJava8parallel());
        System.out.println();

        Options opt = new OptionsBuilder()
                .include(ConvertInputStreamToStringBenchmark.class.getSimpleName())
                .build();

        new Runner(opt).run();
    }

}

{'fraction_non_alphanumeric': 0.10809377320462502, 'fraction_numerical': 0.046037976026307416, 'mean_word_length': 3.02046908315565, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3322100', 'n_tokens_mistral': 2957, 'n_tokens_neox': 2703, 'n_words': 533}