<commit_before>/*
 * Copyright 2011-2016 ETH Zurich. All Rights Reserved.
 *
 * This software is the proprietary information of ETH Zurich.
 * Use is subject to license terms.
 */
package ch.ethz.globis.tinspin.data;

import java.util.Arrays;
import java.util.Random;

import ch.ethz.globis.tinspin.TestStats;

public abstract class AbstractTest {

	protected final Random R;
	protected int DIM;
	protected final double param1;
	protected final double param2;
	protected String paramStr;
	protected final TestStats S;
	//global min/max of data area
	protected double[] globalMin;
	protected double[] globalMax;

	protected AbstractTest(Random R, TestStats S) {
		this.R = R;
		this.S = S;
		this.DIM = S.cfgNDims;
		this.param1 = S.param1;
		this.param2 = S.param2;
		this.paramStr = S.paramStr;
		this.globalMin = new double[DIM];
		this.globalMax = new double[DIM];
		Arrays.fill(globalMin, 0);
		Arrays.fill(globalMax, S.cfgDataLen);
	}

	public int getN() {
		//This may change, for example when exceeding maximum size for
		//CSV, Tiger, OpenStreeMap datasets
		return S.cfgNEntries;
	}

	public final TestStats.TST getTestType() {
		return S.TEST;
	}

	/**
	 * 
	 * @return Maximum distance for each update().
	 */
	public double maxUpdateDistance() {
		double d = 0;
		for (int i = 0; i < globalMin.length; i++) {
			if (d < globalMax[i]-globalMin[i]) {
				d = globalMax[i]-globalMin[i];
			}
		}
		return d * 10e-5;
	}

	public abstract double[] generate();
	public abstract double[][] generateUpdates(int n, double[] data, double[][] ups);
	
	public void log(String str) {
		System.out.println(str);
	}
	
	public void logWarning(String str) {
		System.err.println(str);
	}
	
	public TestStats getTestStats() {
		return S;
	}
	
	public void generateWindowQueries(double[][] lower, double[][] upper) {
		int nEntries = S.cfgNEntries;
		if (nEntries < S.cfgWindowQuerySize*10) {
			//N < 10*000 ? -> N = 100
			nEntries = S.cfgWindowQuerySize*10;
		}

		//find minimum extent (maximum allowable query box size)
		double maxQueryLen = Double.MAX_VALUE;
		//Here is a fixed size version, returning 1% of the space.
		//final double qVolume = 0.01 * Math.pow(cfgDataLen, DIM);//(float) Math.pow(0.1, DIM); //0.01 for DIM=2
		double totalV = 1;
		for (int i = 0; i < DIM; i++) {
			double len = globalMax[i] - globalMin[i]; 
			totalV *= len;
			maxQueryLen = len < maxQueryLen ? len : maxQueryLen;  
		}
		double avgQueryVolume = S.cfgWindowQuerySize/(double)nEntries * totalV;
		//final double avgLen = Math.pow(avgVolume, 1./DIM);
		//final double avgLenVar = 0.5*avgLen;
		//final double minLen = 0.5*avgLen;
		for (int i = 0; i < lower.length; i++) {
			generateQuery2(lower[i], upper[i], maxQueryLen, avgQueryVolume);
		}
	}
	
	/**
	 * Generate query rectangle.
	 * This method should be overwritten by tests that provide
	 * non-standard queries.
	 * @param min output: query box minimum
	 * @param max output: query box maximum
	 * @param maxLen maximum allowed length for a query box in any dimension.
	 * @param avgQVol Average expected volume of a query box
	 */
	public void generateQuery(double[] min, double[] max, 
			final double maxLen, final double avgQVol) {
		if (DIM > 40) {
			//DIM > 40 is problematic, so we use a different algorithm
			generateQuery2(min, max, maxLen, avgQVol);
			return;
		}
		int dims = DIM;
		
		//query create cube
		double[] len = new double[min.length];
		int nTries = 0;
		do {
			double vol = 1;
			for (int d = 0; d < dims-1; d++) {
				//calculate the average required len 
				final double avgLen = Math.pow(avgQVol/vol, 1./dims);
				//create a len between 0.5 and 1.5 of the required length
				len[d] = (0.5*avgLen) + R.nextDouble()*(avgLen);
				len[d] = len[d] > maxLen*0.99 ? maxLen*0.99 : len[d];
				vol *= len[d];
				System.out.println("xxx: v=" + vol + "  al=" + avgLen + "  l=" + len[d]);
			}
			
			//create cuboid/box of desired size by dropping random length
			len[dims-1] = avgQVol/vol;  //now the new len creates a rectangle/box of SIZE.
			if (nTries++ > 100) {
				System.out.println(Arrays.toString(len) + " vol=" + vol + " aVol=" + avgQVol);
				throw new IllegalStateException("dims=" + dims + "  N=" + S.cfgNEntries);
			}
		} while (len[dims-1] >= maxLen); //drop bad rectangles
		
		shuffle(len);
		
		//create location
		for (int d = 0; d < dims; d++) {
			min[d] = globalMin[d] + R.nextDouble()*(maxLen-len[d]);
			max[d] = min[d]+len[d];
			if (min[d]+len[d] >= globalMax[d]) {
				//drop bad rectangles 
				throw new RuntimeException();
			}
		}
	}

	/**
	 * Generate query rectangle. THis is an improved version of the faulty 
	 * {@link #generateQuery(double[], double[], double, double)}.
	 * 
	 * This method should be overwritten by tests that provide non-standard queries.
	 * @param min output: query box minimum
	 * @param max output: query box maximum
	 * @param maxLen maximum allowed length for a query box in any dimension.
	 * @param avgQVol Average expected volume of a query box
	 */
	public void generateQuery2(double[] min, double[] max, 
			final double maxLen, final double avgQVol) {
		int dims = DIM;
		
		//query create cube
		double[] len = new double[min.length];
		int nTries = 0;
		do {
			double vol = 1;
			//Randomize 40 dimensions
			for (int d = 0; d < dims-1; d++) {
				//calculate the average required len 
				final double avgLen = Math.pow(avgQVol/vol, 1./dims);
				//create a len between 0.5 and 1.5 of the required length
				double maxDelta = maxLen*0.99-avgLen;
				len[d] = avgLen + (R.nextDouble()-0.5)*maxDelta;
				vol *= len[d];
			}
			
			//create cuboid/box of desired size by dropping random length
			len[dims-1] = avgQVol/vol;  //now the new len creates a rectangle/box of SIZE.
			if (nTries++ > 2) {
				System.out.println(Arrays.toString(len) + " vol=" + vol + " aVol=" + avgQVol);
				throw new IllegalStateException("dims=" + dims + "  N=" + S.cfgNEntries);
			}
		} while (len[dims-1] >= maxLen); //drop bad rectangles
		
		shuffle(len);
		
		//create location
		for (int d = 0; d < dims; d++) {
			min[d] = globalMin[d] + R.nextDouble()*(maxLen-len[d]);
			max[d] = min[d]+len[d];
			if (min[d]+len[d] >= globalMax[d]) {
				//drop bad rectangles 
				throw new RuntimeException();
			}
		}
	}

	private void shuffle(double[] da) {
		// Fisher-Yates shuffle
		for (int i = da.length - 1; i > 0; i--) {
			int index = R.nextInt(i + 1);
			double a = da[index];
			da[index] = da[i];
			da[i] = a;<commit_msg>Fixed query generator again<commit_after>/*
 * Copyright 2011-2016 ETH Zurich. All Rights Reserved.
 *
 * This software is the proprietary information of ETH Zurich.
 * Use is subject to license terms.
 */
package ch.ethz.globis.tinspin.data;

import java.util.Arrays;
import java.util.Random;

import ch.ethz.globis.tinspin.TestStats;

public abstract class AbstractTest {

	protected final Random R;
	protected int DIM;
	protected final double param1;
	protected final double param2;
	protected String paramStr;
	protected final TestStats S;
	//global min/max of data area
	protected double[] globalMin;
	protected double[] globalMax;

	protected AbstractTest(Random R, TestStats S) {
		this.R = R;
		this.S = S;
		this.DIM = S.cfgNDims;
		this.param1 = S.param1;
		this.param2 = S.param2;
		this.paramStr = S.paramStr;
		this.globalMin = new double[DIM];
		this.globalMax = new double[DIM];
		Arrays.fill(globalMin, 0);
		Arrays.fill(globalMax, S.cfgDataLen);
	}

	public int getN() {
		//This may change, for example when exceeding maximum size for
		//CSV, Tiger, OpenStreeMap datasets
		return S.cfgNEntries;
	}

	public final TestStats.TST getTestType() {
		return S.TEST;
	}

	/**
	 * 
	 * @return Maximum distance for each update().
	 */
	public double maxUpdateDistance() {
		double d = 0;
		for (int i = 0; i < globalMin.length; i++) {
			if (d < globalMax[i]-globalMin[i]) {
				d = globalMax[i]-globalMin[i];
			}
		}
		return d * 10e-5;
	}

	public abstract double[] generate();
	public abstract double[][] generateUpdates(int n, double[] data, double[][] ups);
	
	public void log(String str) {
		System.out.println(str);
	}
	
	public void logWarning(String str) {
		System.err.println(str);
	}
	
	public TestStats getTestStats() {
		return S;
	}
	
	public void generateWindowQueries(double[][] lower, double[][] upper) {
		int nEntries = S.cfgNEntries;
		if (nEntries < S.cfgWindowQuerySize*10) {
			//N < 10*000 ? -> N = 100
			nEntries = S.cfgWindowQuerySize*10;
		}

		//find minimum extent (maximum allowable query box size)
		double maxQueryLen = Double.MAX_VALUE;
		//Here is a fixed size version, returning 1% of the space.
		//final double qVolume = 0.01 * Math.pow(cfgDataLen, DIM);//(float) Math.pow(0.1, DIM); //0.01 for DIM=2
		double totalV = 1;
		for (int i = 0; i < DIM; i++) {
			double len = globalMax[i] - globalMin[i]; 
			totalV *= len;
			maxQueryLen = len < maxQueryLen ? len : maxQueryLen;  
		}
		double avgQueryVolume = S.cfgWindowQuerySize/(double)nEntries * totalV;
		//final double avgLen = Math.pow(avgVolume, 1./DIM);
		//final double avgLenVar = 0.5*avgLen;
		//final double minLen = 0.5*avgLen;
		for (int i = 0; i < lower.length; i++) {
			generateQuery2(lower[i], upper[i], maxQueryLen, avgQueryVolume);
		}
	}
	
	/**
	 * Generate query rectangle.
	 * This method should be overwritten by tests that provide
	 * non-standard queries.
	 * @param min output: query box minimum
	 * @param max output: query box maximum
	 * @param maxLen maximum allowed length for a query box in any dimension.
	 * @param avgQVol Average expected volume of a query box
	 */
	public void generateQuery(double[] min, double[] max, 
			final double maxLen, final double avgQVol) {
		if (DIM > 40) {
			//DIM > 40 is problematic, so we use a different algorithm
			generateQuery2(min, max, maxLen, avgQVol);
			return;
		}
		int dims = DIM;
		
		//query create cube
		double[] len = new double[min.length];
		int nTries = 0;
		do {
			double vol = 1;
			for (int d = 0; d < dims-1; d++) {
				//calculate the average required len 
				final double avgLen = Math.pow(avgQVol/vol, 1./dims);
				//create a len between 0.5 and 1.5 of the required length
				len[d] = (0.5*avgLen) + R.nextDouble()*(avgLen);
				len[d] = len[d] > maxLen*0.99 ? maxLen*0.99 : len[d];
				vol *= len[d];
			}
			
			//create cuboid/box of desired size by dropping random length
			len[dims-1] = avgQVol/vol;  //now the new len creates a rectangle/box of SIZE.
			//System.out.println("XXX2 " + Arrays.toString(len) + " vol=" + (vol*len[dims-1]) + " aVol=" + avgQVol);
			if (nTries++ > 100) {
				System.out.println(Arrays.toString(len) + " vol=" + (vol*len[dims-1]) + " aVol=" + avgQVol);
				throw new IllegalStateException("dims=" + dims + "  N=" + S.cfgNEntries);
			}
		} while (len[dims-1] >= maxLen); //drop bad rectangles
		
		shuffle(len);
		
		//create location
		for (int d = 0; d < dims; d++) {
			min[d] = globalMin[d] + R.nextDouble()*(maxLen-len[d]);
			max[d] = min[d]+len[d];
			if (min[d]+len[d] >= globalMax[d]) {
				//drop bad rectangles 
				throw new RuntimeException();
			}
		}
	}

	/**
	 * Generate query rectangle. THis is an improved version of the faulty 
	 * {@link #generateQuery(double[], double[], double, double)}.
	 * 
	 * This method should be overwritten by tests that provide non-standard queries.
	 * @param min output: query box minimum
	 * @param max output: query box maximum
	 * @param maxLen maximum allowed length for a query box in any dimension.
	 * @param avgQVol Average expected volume of a query box
	 */
	public void generateQuery2(double[] min, double[] max, 
			final double maxLen, final double avgQVol) {
		int dims = DIM;
		
		//query create cube
		double[] len = new double[min.length];
		int nTries = 0;
		do {
			double vol = 1;
			for (int d = 0; d < dims-1; d++) {
				//calculate the average required len 
				final double avgLen = Math.pow(avgQVol/vol, 1./(dims-d));
				//Can't be smaller than that if other lengths should be < maxLen
				final double minLen = avgQVol/vol;  
				double variationUp = maxLen-avgLen; // MAX - AVG
				double variationDown = avgLen-minLen; // AVG - MIN
				//0.999 to avoid queries outside the data domain
				double variation = Math.min(variationUp, variationDown) * 0.999;  
				len[d] = avgLen + (R.nextDouble()*2-1.)*variation;
				vol *= len[d];
			}
			
			//create cuboid/box of desired size by dropping random length
			len[dims-1] = avgQVol/vol;  //now the new len creates a rectangle/box of SIZE.
			//System.out.println("XXX " + Arrays.toString(len) + " vol=" + (vol*len[dims-1]) + " aVol=" + avgQVol);
			if (nTries++ > 3) {
				System.out.println(Arrays.toString(len) + " vol=" + (vol*len[dims-1]) + " aVol=" + avgQVol);
				throw new IllegalStateException("dims=" + dims + "  N=" + S.cfgNEntries);
			}
		} while (len[dims-1] >= maxLen); //drop bad rectangles
		
		shuffle(len);
		
		//create location
		for (int d = 0; d < dims; d++) {
			min[d] = globalMin[d] + R.nextDouble()*(maxLen-len[d]);
			max[d] = min[d]+len[d];
			if (min[d]+len[d] >= globalMax[d]) {
				//drop bad rectangles 
				throw new RuntimeException();
			}
		}
	}

	private void shuffle(double[] da) {
		// Fisher-Yates shuffle
		for (int i = da.length - 1; i > 0; i--) {
			int index = R.nextInt(i + 1);
			double a = da[index];
			da[index] = da[i];
			da[i] = a;
{'fraction_non_alphanumeric': 0.1401834034876729, 'fraction_numerical': 0.015784726398075766, 'mean_word_length': 3.109017912291538, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2494302', 'n_tokens_mistral': 5307, 'n_tokens_neox': 4631, 'n_words': 1418}