<commit_before>package com.googlecode.javaewah.benchmark;


/*
 * Copyright 2009-2013, <NAME>, <NAME>, <NAME>, <NAME>, Google Inc. and <NAME>
 * Licensed under APL 2.0.
 */


/**
 * This class will generate uniformly distributed lists of random integers.
 * 
 * @author <NAME>
 */
class ClusteredDataGenerator {

  public ClusteredDataGenerator() {
  }

  /**
   * generates randomly N distinct integers from 0 to Max.
   * @param N number of integers
   * @param Max maximum integer value 
   * @return  a randomly generated array
   */
  public int[] generateClustered(int N, int Max) {
    int[] array = new int[N];
    fillClustered(array, 0, N, 0, Max);
    return array;
  }

  void fillClustered(int[] array, int offset, int length, int Min, int Max) {
    final int range = Max - Min;
    if ((range == length) || (length <= 10)) {
      fillUniform(array, offset, length, Min, Max);
      return;
    }
    final int cut = length / 2
      + ((range - length - 1 > 0) ? this.unidg.rand.nextInt(range - length - 1) : 0);
    final double p = this.unidg.rand.nextDouble();
    if (p < 0.25) {
      fillUniform(array, offset, length / 2, Min, Min + cut);
      fillClustered(array, offset + length / 2, length - length / 2, Min + cut,
        Max);
    } else if (p < 0.5) {
      fillClustered(array, offset, length / 2, Min, Min + cut);
      fillUniform(array, offset + length / 2, length - length / 2, Min + cut,
        Max);
    } else {
      fillClustered(array, offset, length / 2, Min, Min + cut);
      fillClustered(array, offset + length / 2, length - length / 2, Min + cut,
        Max);
    }
  }

  void fillUniform(int[] array, int offset, int length, int Min, int Max) {
    int[] v = this.unidg.generateUniform(length, Max - Min);
    for (int k = 0; k < v.length; ++k)
      array[k + offset] = Min + v[k];
  }

  public static void main(String[] args) {
    int[] example = (new ClusteredDataGenerator()).generateClustered(20, 1000);
    for (int k = 0; k < example.length; ++k)
      System.out.println(example[k]);
  }

  UniformDataGenerator unidg = new UniformDataGenerator();

}

<commit_msg>Better description<commit_after>package com.googlecode.javaewah.benchmark;


/*
 * Copyright 2009-2013, <NAME>, <NAME>, <NAME>, <NAME>, Google Inc. and <NAME>
 * Licensed under APL 2.0.
 */


/**
 * This class will generate lists of random integers with a "clustered" distribution.
 * Reference:
 * Anh VN, <NAME>. Index compression using 64-bit words. Software: Practice and Experience 2010; 40(2):131-147. 
 *
 * @author <NAME>
 */
class ClusteredDataGenerator {

  public ClusteredDataGenerator() {
  }

  /**
   * generates randomly N distinct integers from 0 to Max.
   * @param N number of integers
   * @param Max maximum integer value 
   * @return  a randomly generated array
   */
  public int[] generateClustered(int N, int Max) {
    int[] array = new int[N];
    fillClustered(array, 0, N, 0, Max);
    return array;
  }

  void fillClustered(int[] array, int offset, int length, int Min, int Max) {
    final int range = Max - Min;
    if ((range == length) || (length <= 10)) {
      fillUniform(array, offset, length, Min, Max);
      return;
    }
    final int cut = length / 2
      + ((range - length - 1 > 0) ? this.unidg.rand.nextInt(range - length - 1) : 0);
    final double p = this.unidg.rand.nextDouble();
    if (p < 0.25) {
      fillUniform(array, offset, length / 2, Min, Min + cut);
      fillClustered(array, offset + length / 2, length - length / 2, Min + cut,
        Max);
    } else if (p < 0.5) {
      fillClustered(array, offset, length / 2, Min, Min + cut);
      fillUniform(array, offset + length / 2, length - length / 2, Min + cut,
        Max);
    } else {
      fillClustered(array, offset, length / 2, Min, Min + cut);
      fillClustered(array, offset + length / 2, length - length / 2, Min + cut,
        Max);
    }
  }

  void fillUniform(int[] array, int offset, int length, int Min, int Max) {
    int[] v = this.unidg.generateUniform(length, Max - Min);
    for (int k = 0; k < v.length; ++k)
      array[k + offset] = Min + v[k];
  }

  public static void main(String[] args) {
    int[] example = (new ClusteredDataGenerator()).generateClustered(20, 1000);
    for (int k = 0; k < example.length; ++k)
      System.out.println(example[k]);
  }

  UniformDataGenerator unidg = new UniformDataGenerator();

}


{'fraction_non_alphanumeric': 0.1398314734684582, 'fraction_numerical': 0.022546117057617856, 'mean_word_length': 2.7283531409168083, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1221639', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1421, 'n_words': 468}