<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.sql.execution.datasources.parquet;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import com.google.common.base.Preconditions;

import org.apache.spark.memory.MemoryMode;
import org.apache.spark.sql.execution.vectorized.OffHeapColumnVector;
import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector;
import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
import org.apache.spark.sql.types.ArrayType;
import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.MapType;
import org.apache.spark.sql.types.StructType;

/**
 * Contains necessary information representing a Parquet column, either of primitive or nested type.
 */
final class ParquetColumnVector {
  private final ParquetColumn column;
  private final List<ParquetColumnVector> children;
  private final WritableColumnVector vector;

  /**
   * Repetition & Definition levels
   * These are allocated only for leaf columns; for non-leaf columns, they simply maintain
   * references to that of the former.
   */
  private WritableColumnVector repetitionLevels;
  private WritableColumnVector definitionLevels;

  /** Whether this column is primitive (i.e., leaf column) */
  private final boolean isPrimitive;

  /** Reader for this column - only set if 'isPrimitive' is true */
  private VectorizedColumnReader columnReader;

  ParquetColumnVector(
      ParquetColumn column,
      WritableColumnVector vector,
      int capacity,
      MemoryMode memoryMode,
      Set<ParquetColumn> missingColumns,
      boolean isTopLevel,
      Object defaultValue) {
    DataType sparkType = column.sparkType();
    if (!sparkType.sameType(vector.dataType())) {
      throw new IllegalArgumentException("Spark type: " + sparkType +
        " doesn't match the type: " + vector.dataType() + " in column vector");
    }

    this.column = column;
    this.vector = vector;
    this.children = new ArrayList<>();
    this.isPrimitive = column.isPrimitive();

    if (missingColumns.contains(column)) {
      if (defaultValue == null) {
        vector.setAllNull();
        return;
      }
      // For Parquet tables whose columns have associated DEFAULT values, this reader must return
      // those values instead of NULL when the corresponding columns are not present in storage.
      // Here we write the 'defaultValue' to each element in the new WritableColumnVector using
      // the appendObjects method. This delegates to some specific append* method depending on the
      // type of 'defaultValue'; for example, if 'defaultValue' is a Float, then we call the
      // appendFloats method.
      if (!vector.appendObjects(capacity, defaultValue).isPresent()) {
        throw new IllegalArgumentException("Cannot assign default column value to result " +
          "column batch in vectorized Parquet reader because the data type is not supported: " +
          defaultValue);
      }
    }

    if (isPrimitive) {
      if (column.repetitionLevel() > 0) {
        repetitionLevels = allocateLevelsVector(capacity, memoryMode);
      }
      // We don't need to create and store definition levels if the column is top-level.
      if (!isTopLevel) {
        definitionLevels = allocateLevelsVector(capacity, memoryMode);
      }
    } else {
      Preconditions.checkArgument(column.children().size() == vector.getNumChildren());<commit_msg>[SPARK-39706][SQL] Set missing column with defaultValue as constant in `ParquetColumnVector`<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.sql.execution.datasources.parquet;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import com.google.common.base.Preconditions;

import org.apache.spark.memory.MemoryMode;
import org.apache.spark.sql.execution.vectorized.OffHeapColumnVector;
import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector;
import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
import org.apache.spark.sql.types.ArrayType;
import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.MapType;
import org.apache.spark.sql.types.StructType;

/**
 * Contains necessary information representing a Parquet column, either of primitive or nested type.
 */
final class ParquetColumnVector {
  private final ParquetColumn column;
  private final List<ParquetColumnVector> children;
  private final WritableColumnVector vector;

  /**
   * Repetition & Definition levels
   * These are allocated only for leaf columns; for non-leaf columns, they simply maintain
   * references to that of the former.
   */
  private WritableColumnVector repetitionLevels;
  private WritableColumnVector definitionLevels;

  /** Whether this column is primitive (i.e., leaf column) */
  private final boolean isPrimitive;

  /** Reader for this column - only set if 'isPrimitive' is true */
  private VectorizedColumnReader columnReader;

  ParquetColumnVector(
      ParquetColumn column,
      WritableColumnVector vector,
      int capacity,
      MemoryMode memoryMode,
      Set<ParquetColumn> missingColumns,
      boolean isTopLevel,
      Object defaultValue) {
    DataType sparkType = column.sparkType();
    if (!sparkType.sameType(vector.dataType())) {
      throw new IllegalArgumentException("Spark type: " + sparkType +
        " doesn't match the type: " + vector.dataType() + " in column vector");
    }

    this.column = column;
    this.vector = vector;
    this.children = new ArrayList<>();
    this.isPrimitive = column.isPrimitive();

    if (missingColumns.contains(column)) {
      if (defaultValue == null) {
        vector.setAllNull();
        return;
      }
      // For Parquet tables whose columns have associated DEFAULT values, this reader must return
      // those values instead of NULL when the corresponding columns are not present in storage.
      // Here we write the 'defaultValue' to each element in the new WritableColumnVector using
      // the appendObjects method. This delegates to some specific append* method depending on the
      // type of 'defaultValue'; for example, if 'defaultValue' is a Float, then we call the
      // appendFloats method.
      if (!vector.appendObjects(capacity, defaultValue).isPresent()) {
        throw new IllegalArgumentException("Cannot assign default column value to result " +
          "column batch in vectorized Parquet reader because the data type is not supported: " +
          defaultValue);
      } else {
        vector.setIsConstant();
      }
    }

    if (isPrimitive) {
      if (column.repetitionLevel() > 0) {
        repetitionLevels = allocateLevelsVector(capacity, memoryMode);
      }
      // We don't need to create and store definition levels if the column is top-level.
      if (!isTopLevel) {
        definitionLevels = allocateLevelsVector(capacity, memoryMode);
      }
    } else {
      Preconditions.checkArgument(column.children().size() == vector.getNumChildren());
{'fraction_non_alphanumeric': 0.0833911635438353, 'fraction_numerical': 0.0017349063150589867, 'mean_word_length': 3.604366347177849, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3878313', 'n_tokens_mistral': 2264, 'n_tokens_neox': 2197, 'n_words': 802}