<commit_before>/*******************************************************************************
 * Copyright (c) 2018 Microsoft Research. All rights reserved.
 *
 * The MIT License (MIT)
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is furnished to do
 * so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * Contributors:
 *   <NAME> - initial API and implementation
 ******************************************************************************/
package tlc2.value;

import java.io.File;

import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;

import tlc2.util.FP64;

@State(Scope.Benchmark)
public class SubsetValueBenchmark {

	private static final SubsetValue subset35;
	private static final SubsetValue subset60;
	private static final SubsetValue subset100;
	private static final SubsetValue subset200;
	private static final SubsetValue subset300;
	private static final SubsetValue subset400;
	
	private static final int k = 80000;

	static {
		EnumerableValue.setRandom(15041980L);
		EnumerableValue.resetRandom();
		
		FP64.Init();

		subset35 = new SubsetValue(new IntervalValue(1, 35));
		subset60 = new SubsetValue(new IntervalValue(1, 60));
		subset100 = new SubsetValue(new IntervalValue(1, 100));
		subset200 = new SubsetValue(new IntervalValue(1, 200));
		subset300 = new SubsetValue(new IntervalValue(1, 300));
		subset400 = new SubsetValue(new IntervalValue(1, 400));
		
		new File("target" + File.separator + "probabilistic").mkdirs();
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
//	@Fork(jvmArgsAppend = { "-XX:+UnlockCommercialFeatures", "-XX:+UnlockDiagnosticVMOptions",
//			"-XX:+DebugNonSafepoints", "-XX:+FlightRecorder",
//			"-XX:StartFlightRecording=maxage=2h,settings=./test-benchmark/heapstats.jfc,",
//			"-XX:FlightRecorderOptions=defaultrecording=true,disk=false,repository=/tmp,dumponexit=true,dumponexitpath=./target/probabilisticN035" })
	public EnumerableValue probabilisticN035() {
		return subset35.getRandomSetOfSubsets(k, .1d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
//	@Fork(jvmArgsAppend = { "-XX:+UnlockCommercialFeatures", "-XX:+UnlockDiagnosticVMOptions",
//			"-XX:+DebugNonSafepoints", "-XX:+FlightRecorder",
//			"-XX:StartFlightRecording=maxage=2h,settings=./test-benchmark/heapstats.jfc,",
//			"-XX:FlightRecorderOptions=defaultrecording=true,disk=false,repository=/tmp,dumponexit=true,dumponexitpath=./target/probabilisticN060" })
	public EnumerableValue probabilisticN060() {
		return subset60.getRandomSetOfSubsets(k, .1d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
//	@Fork(jvmArgsAppend = { "-XX:+UnlockCommercialFeatures", "-XX:+UnlockDiagnosticVMOptions",
//			"-XX:+DebugNonSafepoints", "-XX:+FlightRecorder",
//			"-XX:StartFlightRecording=maxage=2h,settings=./test-benchmark/heapstats.jfc,",
//			"-XX:FlightRecorderOptions=defaultrecording=true,disk=false,repository=/tmp,dumponexit=true,dumponexitpath=./target/probabilisticN100" })
	public EnumerableValue probabilisticN100() {
		return subset100.getRandomSetOfSubsets(k, .1d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
//	@Fork(jvmArgsAppend = { "-XX:+UnlockCommercialFeatures", "-XX:+UnlockDiagnosticVMOptions",
//			"-XX:+DebugNonSafepoints", "-XX:+FlightRecorder",
//			"-XX:StartFlightRecording=maxage=2h,settings=./test-benchmark/heapstats.jfc,",
//			"-XX:FlightRecorderOptions=defaultrecording=true,disk=false,repository=/tmp,dumponexit=true,dumponexitpath=./target/probabilisticN200" })
	public EnumerableValue probabilisticN200() {
		return subset200.getRandomSetOfSubsets(k, .1d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
//	@Fork(jvmArgsAppend = { "-XX:+UnlockCommercialFeatures", "-XX:+UnlockDiagnosticVMOptions",
//			"-XX:+DebugNonSafepoints", "-XX:+FlightRecorder",
//			"-XX:StartFlightRecording=maxage=2h,settings=./test-benchmark/heapstats.jfc,",
//			"-XX:FlightRecorderOptions=defaultrecording=true,disk=false,repository=/tmp,dumponexit=true,dumponexitpath=./target/probabilisticN300" })
	public EnumerableValue probabilisticN300() {
		return subset300.getRandomSetOfSubsets(k, .1d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
//	@Fork(jvmArgsAppend = { "-XX:+UnlockCommercialFeatures", "-XX:+UnlockDiagnosticVMOptions",
//			"-XX:+DebugNonSafepoints", "-XX:+FlightRecorder",
//			"-XX:StartFlightRecording=maxage=2h,settings=./test-benchmark/heapstats.jfc,",
//			"-XX:FlightRecorderOptions=defaultrecording=true,disk=false,repository=/tmp,dumponexit=true,dumponexitpath=./target/probabilisticN400" })
	public EnumerableValue probabilisticN400() {
		return subset400.getRandomSetOfSubsets(k, .1d);
	}
}

/*
 * Mode.Throughput: Calculate number of operations in a time unit. (Higher score
 * better) Mode.AverageTime: Calculate an average running time. (Lower score
 * better) Mode.SampleTime: Calculate how long does it take for a method to run
 * (including percentiles). Mode.SingleShotTime: Just run a method once (useful
 * for cold-testing mode). Or more than once if you have specified a batch size
 * for your iterations (see @Measurement annotation below) – in this case JMH
 * will calculate the batch running time (total time for all invocations in a
 * batch). Any set of these modes You can specify any set of these modes – the
 * test will be run several times (depending on number of requested modes).
 * Mode.All: All these modes one after another.
 */<commit_msg>Add more parameter permutations to SubsetValueBenchmark#getRandomSetOfSubsets.<commit_after>/*******************************************************************************
 * Copyright (c) 2018 Microsoft Research. All rights reserved.
 *
 * The MIT License (MIT)
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is furnished to do
 * so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * Contributors:
 *   <NAME> - initial API and implementation
 ******************************************************************************/
package tlc2.value;

import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;

import tlc2.util.FP64;

@State(Scope.Benchmark)
public class SubsetValueBenchmark {

	private static final SubsetValue subset35;
	private static final SubsetValue subset60;
	private static final SubsetValue subset100;
	private static final SubsetValue subset200;
	private static final SubsetValue subset300;
	private static final SubsetValue subset400;

	private static final int k = 80000;
	private static final double d = 0.1d;
	private static final int k2 = 160000;
	private static final double d2 = 0.2d;

	static {
		EnumerableValue.setRandom(15041980L);
		EnumerableValue.resetRandom();

		FP64.Init();

		subset35 = new SubsetValue(new IntervalValue(1, 35));
		subset35.normalize();
		subset60 = new SubsetValue(new IntervalValue(1, 60));
		subset60.normalize();
		subset100 = new SubsetValue(new IntervalValue(1, 100));
		subset100.normalize();
		subset200 = new SubsetValue(new IntervalValue(1, 200));
		subset200.normalize();
		subset300 = new SubsetValue(new IntervalValue(1, 300));
		subset300.normalize();
		subset400 = new SubsetValue(new IntervalValue(1, 400));
		subset400.normalize();
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN035k80d01() {
		return subset35.getRandomSetOfSubsets(k, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN060k80d01() {
		return subset60.getRandomSetOfSubsets(k, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN100k80d01() {
		return subset100.getRandomSetOfSubsets(k, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN200k80d01() {
		return subset200.getRandomSetOfSubsets(k, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN300k80d01() {
		return subset300.getRandomSetOfSubsets(k, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN400k80d01() {
		return subset400.getRandomSetOfSubsets(k, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN035k16d01() {
		return subset35.getRandomSetOfSubsets(k2, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN060k16d01() {
		return subset60.getRandomSetOfSubsets(k2, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN100k16d01() {
		return subset100.getRandomSetOfSubsets(k2, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN200k16d01() {
		return subset200.getRandomSetOfSubsets(k2, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN300k16d01() {
		return subset300.getRandomSetOfSubsets(k2, d);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN400k16d01() {
		return subset400.getRandomSetOfSubsets(k2, d);
	}
	
	/* d2 */

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN035k80d02() {
		return subset35.getRandomSetOfSubsets(k, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN060k80d02() {
		return subset60.getRandomSetOfSubsets(k, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN100k80d02() {
		return subset100.getRandomSetOfSubsets(k, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN200k80d02() {
		return subset200.getRandomSetOfSubsets(k, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN300k80d02() {
		return subset300.getRandomSetOfSubsets(k, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN400k80d02() {
		return subset400.getRandomSetOfSubsets(k, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN035k16d02() {
		return subset35.getRandomSetOfSubsets(k2, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN060k16d02() {
		return subset60.getRandomSetOfSubsets(k2, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN100k16d02() {
		return subset100.getRandomSetOfSubsets(k2, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN200k16d02() {
		return subset200.getRandomSetOfSubsets(k2, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN300k16d02() {
		return subset300.getRandomSetOfSubsets(k2, d2);
	}

	@Benchmark
	@Warmup(iterations = 3, time = 1)
	@Measurement(iterations = 3, time = 1)
	@BenchmarkMode(Mode.Throughput)
	public EnumerableValue probabilisticN400k16d02() {
		return subset400.getRandomSetOfSubsets(k2, d2);
	}
}

/*
 * Mode.Throughput: Calculate number of operations in a time unit. (Higher score
 * better) Mode.AverageTime: Calculate an average running time. (Lower score
 * better) Mode.SampleTime: Calculate how long does it take for a method to run
 * (including percentiles). Mode.SingleShotTime: Just run a method once (useful
 * for cold-testing mode). Or more than once if you have specified a batch size
 * for your iterations (see @Measurement annotation below) – in this case JMH
 * will calculate the batch running time (total time for all invocations in a
 * batch). Any set of these modes You can specify any set of these modes – the
 * test will be run several times (depending on number of requested modes).
 * Mode.All: All these modes one after another.
 */
{'fraction_non_alphanumeric': 0.1265783609804407, 'fraction_numerical': 0.03856152512998267, 'mean_word_length': 5.509669621273167, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2651232', 'n_tokens_mistral': 6012, 'n_tokens_neox': 5267, 'n_words': 1338}