<commit_before>package org.apache.lucene.index;

/**
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.util.Random;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;

public class TestByteSlices extends LuceneTestCase {

  private static class ByteBlockAllocator extends ByteBlockPool.Allocator {
    ArrayList<byte[]> freeByteBlocks = new ArrayList<byte[]>();
    
    /* Allocate another byte[] from the shared pool */
    @Override
    synchronized byte[] getByteBlock() {
      final int size = freeByteBlocks.size();
      final byte[] b;
      if (0 == size)
        b = new byte[DocumentsWriter.BYTE_BLOCK_SIZE];
      else
        b =  freeByteBlocks.remove(size-1);
      return b;
    }

    /* Return a byte[] to the pool */
    @Override
    synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) {
      for(int i=start;i<end;i++)
        freeByteBlocks.add(blocks[i]);
    }

    @Override
    synchronized void recycleByteBlocks(List<byte[]> blocks) {
      final int size = blocks.size();
      for(int i=0;i<size;i++)
        freeByteBlocks.add(blocks.get(i));
    }
  }

  public void testBasic() throws Throwable {
    ByteBlockPool pool = new ByteBlockPool(new ByteBlockAllocator());

    final int NUM_STREAM = 25;

    ByteSliceWriter writer = new ByteSliceWriter(pool);

    int[] starts = new int[NUM_STREAM];
    int[] uptos = new int[NUM_STREAM];
    int[] counters = new int[NUM_STREAM];

    Random r = newRandom();

    ByteSliceReader reader = new ByteSliceReader();

    for(int ti=0;ti<100;ti++) {

      for(int stream=0;stream<NUM_STREAM;stream++) {
        starts[stream] = -1;
        counters[stream] = 0;
      }
      
      for(int iter=0;iter<10000*_TestUtil.getRandomMultiplier();iter++) {
        int stream = r.nextInt(NUM_STREAM);
        if (VERBOSE)
          System.out.println("write stream=" + stream);

        if (starts[stream] == -1) {
          final int spot = pool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
          starts[stream] = uptos[stream] = spot + pool.byteOffset;
          if (VERBOSE)
            System.out.println("  init to " + starts[stream]);
        }

        writer.init(uptos[stream]);
        int numValue = r.nextInt(20);
        for(int j=0;j<numValue;j++) {
          if (VERBOSE)
            System.out.println("    write " + (counters[stream]+j));
          writer.writeVInt(counters[stream]+j);
          //writer.writeVInt(ti);
        }
        counters[stream] += numValue;
        uptos[stream] = writer.getAddress();
        if (VERBOSE)
          System.out.println("    addr now " + uptos[stream]);
      }
    
      for(int stream=0;stream<NUM_STREAM;stream++) {
        if (VERBOSE)
          System.out.println("  stream=" + stream + " count=" + counters[stream]);

        if (starts[stream] != uptos[stream]) {
          reader.init(pool, starts[stream], uptos[stream]);
          for(int j=0;j<counters[stream];j++) 
            assertEquals(j, reader.readVInt());
            //assertEquals(ti, reader.readVInt());
        }
      }

      pool.reset();
    }
  }
}
<commit_msg>LUCENE-2501: strengthen this testcase a bit, but I still can't get it to fail<commit_after>package org.apache.lucene.index;

/**
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.util.Random;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;

public class TestByteSlices extends LuceneTestCase {

  private static class ByteBlockAllocator extends ByteBlockPool.Allocator {
    ArrayList<byte[]> freeByteBlocks = new ArrayList<byte[]>();
    
    /* Allocate another byte[] from the shared pool */
    @Override
    synchronized byte[] getByteBlock() {
      final int size = freeByteBlocks.size();
      final byte[] b;
      if (0 == size)
        b = new byte[DocumentsWriter.BYTE_BLOCK_SIZE];
      else
        b =  freeByteBlocks.remove(size-1);
      return b;
    }

    /* Return a byte[] to the pool */
    @Override
    synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) {
      for(int i=start;i<end;i++)
        freeByteBlocks.add(blocks[i]);
    }

    @Override
    synchronized void recycleByteBlocks(List<byte[]> blocks) {
      final int size = blocks.size();
      for(int i=0;i<size;i++)
        freeByteBlocks.add(blocks.get(i));
    }
  }

  public void testBasic() throws Throwable {
    ByteBlockPool pool = new ByteBlockPool(new ByteBlockAllocator());

    final int NUM_STREAM = 100*_TestUtil.getRandomMultiplier();

    ByteSliceWriter writer = new ByteSliceWriter(pool);

    int[] starts = new int[NUM_STREAM];
    int[] uptos = new int[NUM_STREAM];
    int[] counters = new int[NUM_STREAM];

    Random r = newRandom();

    ByteSliceReader reader = new ByteSliceReader();

    for(int ti=0;ti<100;ti++) {

      for(int stream=0;stream<NUM_STREAM;stream++) {
        starts[stream] = -1;
        counters[stream] = 0;
      }
      
      for(int iter=0;iter<10000*_TestUtil.getRandomMultiplier();iter++) {
        int stream = r.nextInt(NUM_STREAM);
        if (VERBOSE)
          System.out.println("write stream=" + stream);

        if (starts[stream] == -1) {
          final int spot = pool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
          starts[stream] = uptos[stream] = spot + pool.byteOffset;
          if (VERBOSE)
            System.out.println("  init to " + starts[stream]);
        }

        writer.init(uptos[stream]);
        int numValue = r.nextInt(20);
        for(int j=0;j<numValue;j++) {
          if (VERBOSE)
            System.out.println("    write " + (counters[stream]+j));
          // write some large (incl. negative) ints:
          writer.writeVInt(r.nextInt());
          writer.writeVInt(counters[stream]+j);
        }
        counters[stream] += numValue;
        uptos[stream] = writer.getAddress();
        if (VERBOSE)
          System.out.println("    addr now " + uptos[stream]);
      }
    
      for(int stream=0;stream<NUM_STREAM;stream++) {
        if (VERBOSE)
          System.out.println("  stream=" + stream + " count=" + counters[stream]);

        if (starts[stream] != -1 && starts[stream] != uptos[stream]) {
          reader.init(pool, starts[stream], uptos[stream]);
          for(int j=0;j<counters[stream];j++) {
            reader.readVInt();
            assertEquals(j, reader.readVInt()); 
          }
        }
      }

      pool.reset();
    }
  }
}

{'fraction_non_alphanumeric': 0.1258732041650191, 'fraction_numerical': 0.008171872940556215, 'mean_word_length': 2.703269887750122, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7148484', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2270, 'n_words': 582}