<commit_before>package org.apache.lucene.codecs.simpletext;
/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with this
 * work for additional information regarding copyright ownership. The ASF
 * licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
import java.io.IOException;

import org.apache.lucene.codecs.DocValuesArraySource;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.IOUtils;

/**
 * @lucene.experimental
 */
public class SimpleTextDocValuesConsumer extends DocValuesConsumer {

  static final BytesRef ZERO_DOUBLE = new BytesRef(Double.toString(0d));
  static final BytesRef ZERO_INT = new BytesRef(Integer.toString(0));
  static final BytesRef HEADER = new BytesRef("SimpleTextDocValues"); 

  static final BytesRef END = new BytesRef("END");
  static final BytesRef VALUE_SIZE = new BytesRef("valuesize ");
  static final BytesRef DOC = new BytesRef("  doc ");
  static final BytesRef VALUE = new BytesRef("    value ");
  protected BytesRef scratch = new BytesRef();
  protected int maxDocId = -1;
  protected final String segment;
  protected final Directory dir;
  protected final IOContext ctx;
  protected final Type type;
  protected final BytesRefHash hash;
  private int[] ords;
  private int fixedSize = Integer.MIN_VALUE;
  private BytesRef zeroBytes;
  private final String segmentSuffix;
  

  public SimpleTextDocValuesConsumer(String segment, Directory dir,
      IOContext ctx, Type type, String segmentSuffix) {
    this.ctx = ctx;
    this.dir = dir;
    this.segment = segment;
    this.type = type;
    hash = new BytesRefHash();
    ords = new int[0];
    this.segmentSuffix = segmentSuffix;
  }

  @Override
  public void add(int docID, IndexableField value) throws IOException {
    assert docID >= 0;
    int ord = -1;
    int vSize = -1;
    switch (type) {
    case BYTES_FIXED_DEREF:
    case BYTES_FIXED_SORTED:
    case BYTES_FIXED_STRAIGHT:
      vSize = value.binaryValue().length;
      ord = hash.add(value.binaryValue());
      break;
    case BYTES_VAR_DEREF:
    case BYTES_VAR_SORTED:
    case BYTES_VAR_STRAIGHT:
      vSize = -1;
      ord = hash.add(value.binaryValue());
      break;
    case FIXED_INTS_16:
      vSize = 2;
      scratch.grow(2);
      DocValuesArraySource.copyShort(scratch, value.numericValue().shortValue());
      ord = hash.add(scratch);
      break;
    case FIXED_INTS_32:
      vSize = 4;
      scratch.grow(4);
      DocValuesArraySource.copyInt(scratch, value.numericValue().intValue());
      ord = hash.add(scratch);
      break;
    case FIXED_INTS_8:
      vSize = 1;
      scratch.grow(1); 
      scratch.bytes[scratch.offset] = value.numericValue().byteValue();
      scratch.length = 1;
      ord = hash.add(scratch);
      break;
    case FIXED_INTS_64:
      vSize = 8;
    case VAR_INTS:
      scratch.grow(8);
      DocValuesArraySource.copyLong(scratch, value.numericValue().longValue());
      ord = hash.add(scratch);
      break;
    case FLOAT_32:
      vSize = 4;
      scratch.grow(4);
      DocValuesArraySource.copyInt(scratch,
          Float.floatToRawIntBits(value.numericValue().floatValue()));
      ord = hash.add(scratch);
      break;
    case FLOAT_64:
      vSize = 8;
      scratch.grow(8);
      DocValuesArraySource.copyLong(scratch,
          Double.doubleToRawLongBits(value.numericValue().doubleValue()));
      ord = hash.add(scratch);
      break;

    }
    
    if (fixedSize == Integer.MIN_VALUE) {
      assert maxDocId == -1;
      fixedSize = vSize;
    } else {
      if (fixedSize != vSize) {
        throw new IllegalArgumentException("value size must be " + fixedSize + " but was: " + vSize);
      }
    }
    maxDocId = Math.max(docID, maxDocId);
    ords = grow(ords, docID);
    
    ords[docID] = (ord < 0 ? (-ord)-1 : ord) + 1;
  }
  
  protected BytesRef getHeader() {
    return HEADER;
  }

  private int[] grow(int[] array, int upto) {
    if (array.length <= upto) {
      return ArrayUtil.grow(array, 1 + upto);
    }
    return array;
  }
<commit_msg>Fix fall-through warning and ensure all variables are initialized (by finalness) and default switch throwing runtimeex<commit_after>package org.apache.lucene.codecs.simpletext;
/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with this
 * work for additional information regarding copyright ownership. The ASF
 * licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
import java.io.IOException;

import org.apache.lucene.codecs.DocValuesArraySource;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.IOUtils;

/**
 * @lucene.experimental
 */
public class SimpleTextDocValuesConsumer extends DocValuesConsumer {

  static final BytesRef ZERO_DOUBLE = new BytesRef(Double.toString(0d));
  static final BytesRef ZERO_INT = new BytesRef(Integer.toString(0));
  static final BytesRef HEADER = new BytesRef("SimpleTextDocValues"); 

  static final BytesRef END = new BytesRef("END");
  static final BytesRef VALUE_SIZE = new BytesRef("valuesize ");
  static final BytesRef DOC = new BytesRef("  doc ");
  static final BytesRef VALUE = new BytesRef("    value ");
  protected BytesRef scratch = new BytesRef();
  protected int maxDocId = -1;
  protected final String segment;
  protected final Directory dir;
  protected final IOContext ctx;
  protected final Type type;
  protected final BytesRefHash hash;
  private int[] ords;
  private int fixedSize = Integer.MIN_VALUE;
  private BytesRef zeroBytes;
  private final String segmentSuffix;
  

  public SimpleTextDocValuesConsumer(String segment, Directory dir,
      IOContext ctx, Type type, String segmentSuffix) {
    this.ctx = ctx;
    this.dir = dir;
    this.segment = segment;
    this.type = type;
    hash = new BytesRefHash();
    ords = new int[0];
    this.segmentSuffix = segmentSuffix;
  }

  @Override
  public void add(int docID, IndexableField value) throws IOException {
    assert docID >= 0;
    final int ord, vSize;
    switch (type) {
    case BYTES_FIXED_DEREF:
    case BYTES_FIXED_SORTED:
    case BYTES_FIXED_STRAIGHT:
      vSize = value.binaryValue().length;
      ord = hash.add(value.binaryValue());
      break;
    case BYTES_VAR_DEREF:
    case BYTES_VAR_SORTED:
    case BYTES_VAR_STRAIGHT:
      vSize = -1;
      ord = hash.add(value.binaryValue());
      break;
    case FIXED_INTS_16:
      vSize = 2;
      scratch.grow(2);
      DocValuesArraySource.copyShort(scratch, value.numericValue().shortValue());
      ord = hash.add(scratch);
      break;
    case FIXED_INTS_32:
      vSize = 4;
      scratch.grow(4);
      DocValuesArraySource.copyInt(scratch, value.numericValue().intValue());
      ord = hash.add(scratch);
      break;
    case FIXED_INTS_8:
      vSize = 1;
      scratch.grow(1); 
      scratch.bytes[scratch.offset] = value.numericValue().byteValue();
      scratch.length = 1;
      ord = hash.add(scratch);
      break;
    case FIXED_INTS_64:
      vSize = 8;
      scratch.grow(8);
      DocValuesArraySource.copyLong(scratch, value.numericValue().longValue());
      ord = hash.add(scratch);
      break;
    case VAR_INTS:
      vSize = -1;
      scratch.grow(8);
      DocValuesArraySource.copyLong(scratch, value.numericValue().longValue());
      ord = hash.add(scratch);
      break;
    case FLOAT_32:
      vSize = 4;
      scratch.grow(4);
      DocValuesArraySource.copyInt(scratch,
          Float.floatToRawIntBits(value.numericValue().floatValue()));
      ord = hash.add(scratch);
      break;
    case FLOAT_64:
      vSize = 8;
      scratch.grow(8);
      DocValuesArraySource.copyLong(scratch,
          Double.doubleToRawLongBits(value.numericValue().doubleValue()));
      ord = hash.add(scratch);
      break;
    default:
      throw new RuntimeException("should not reach this line");
    }
    
    if (fixedSize == Integer.MIN_VALUE) {
      assert maxDocId == -1;
      fixedSize = vSize;
    } else {
      if (fixedSize != vSize) {
        throw new IllegalArgumentException("value size must be " + fixedSize + " but was: " + vSize);
      }
    }
    maxDocId = Math.max(docID, maxDocId);
    ords = grow(ords, docID);
    
    ords[docID] = (ord < 0 ? (-ord)-1 : ord) + 1;
  }
  
  protected BytesRef getHeader() {
    return HEADER;
  }

  private int[] grow(int[] array, int upto) {
    if (array.length <= upto) {
      return ArrayUtil.grow(array, 1 + upto);
    }
    return array;
  }

{'fraction_non_alphanumeric': 0.10182692307692308, 'fraction_numerical': 0.007884615384615384, 'mean_word_length': 3.280246913580247, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5532172', 'n_tokens_mistral': 3276, 'n_tokens_neox': 3203, 'n_words': 869}