<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.drill.exec.store.easy.json;

import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
import static com.fasterxml.jackson.core.JsonToken.FIELD_NAME;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;

import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.NullableBitHolder;
import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
import org.apache.drill.exec.expr.holders.NullableIntHolder;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.schema.DiffSchema;
import org.apache.drill.exec.schema.Field;
import org.apache.drill.exec.schema.NamedField;
import org.apache.drill.exec.schema.ObjectSchema;
import org.apache.drill.exec.schema.RecordSchema;
import org.apache.drill.exec.schema.SchemaIdGenerator;
import org.apache.drill.exec.schema.json.jackson.JacksonHelper;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.VectorHolder;
import org.apache.drill.exec.vector.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

public class JSONRecordReader implements RecordReader {
  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader.class);
  private static final int DEFAULT_LENGTH = 256 * 1024; // 256kb
  public static final Charset UTF_8 = Charset.forName("UTF-8");

  private final Map<String, VectorHolder> valueVectorMap;
  private final FileSystem fileSystem;
  private final Path hadoopPath;

  private JsonParser parser;
  private SchemaIdGenerator generator;
  private DiffSchema diffSchema;
  private RecordSchema currentSchema;
  private List<Field> removedFields;
  private OutputMutator outputMutator;
  private BufferAllocator allocator;
  private int batchSize;
  private final List<SchemaPath> columns;

  public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize,
                          List<SchemaPath> columns) {
    this.hadoopPath = new Path(inputPath);
    this.fileSystem = fileSystem;
    this.allocator = fragmentContext.getAllocator();
    this.batchSize = batchSize;
    valueVectorMap = Maps.newHashMap();
    this.columns = columns;
  }

  public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem,
                          List<SchemaPath> columns) {
    this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, columns);
  }

  private JsonParser getParser() {
    return parser;
  }

  @Override
  public void setup(OutputMutator output) throws ExecutionSetupException {
    outputMutator = output;
    currentSchema = new ObjectSchema();
    diffSchema = new DiffSchema();
    removedFields = Lists.newArrayList();

    try {
      JsonFactory factory = new JsonFactory();
      parser = factory.createJsonParser(fileSystem.open(hadoopPath));
      parser.nextToken(); // Read to the first START_OBJECT token
      generator = new SchemaIdGenerator();
    } catch (IOException e) {
      throw new ExecutionSetupException(e);
    }
  }

  @Override
  public int next() {
    if (parser.isClosed() || !parser.hasCurrentToken()) {
      return 0;
    }

    resetBatch();

    int nextRowIndex = 0;<commit_msg>DRILL-405: Fix for not able to query directory containing json files<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.drill.exec.store.easy.json;

import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
import static com.fasterxml.jackson.core.JsonToken.FIELD_NAME;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;

import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.holders.NullableBitHolder;
import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
import org.apache.drill.exec.expr.holders.NullableIntHolder;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.schema.DiffSchema;
import org.apache.drill.exec.schema.Field;
import org.apache.drill.exec.schema.NamedField;
import org.apache.drill.exec.schema.ObjectSchema;
import org.apache.drill.exec.schema.RecordSchema;
import org.apache.drill.exec.schema.SchemaIdGenerator;
import org.apache.drill.exec.schema.json.jackson.JacksonHelper;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.VectorHolder;
import org.apache.drill.exec.vector.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

public class JSONRecordReader implements RecordReader {
  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader.class);
  private static final int DEFAULT_LENGTH = 256 * 1024; // 256kb
  public static final Charset UTF_8 = Charset.forName("UTF-8");

  private final Map<String, VectorHolder> valueVectorMap;
  private final FileSystem fileSystem;
  private final Path hadoopPath;

  private JsonParser parser;
  private SchemaIdGenerator generator;
  private DiffSchema diffSchema;
  private RecordSchema currentSchema;
  private List<Field> removedFields;
  private OutputMutator outputMutator;
  private BufferAllocator allocator;
  private int batchSize;
  private final List<SchemaPath> columns;

  public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize,
                          List<SchemaPath> columns) {
    this.hadoopPath = new Path(inputPath);
    this.fileSystem = fileSystem;
    this.allocator = fragmentContext.getAllocator();
    this.batchSize = batchSize;
    valueVectorMap = Maps.newHashMap();
    this.columns = columns;
  }

  public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem,
                          List<SchemaPath> columns) {
    this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, columns);
  }

  private JsonParser getParser() {
    return parser;
  }

  @Override
  public void setup(OutputMutator output) throws ExecutionSetupException {
    outputMutator = output;
    output.removeAllFields();
    currentSchema = new ObjectSchema();
    diffSchema = new DiffSchema();
    removedFields = Lists.newArrayList();

    try {
      JsonFactory factory = new JsonFactory();
      parser = factory.createJsonParser(fileSystem.open(hadoopPath));
      parser.nextToken(); // Read to the first START_OBJECT token
      generator = new SchemaIdGenerator();
    } catch (IOException e) {
      throw new ExecutionSetupException(e);
    }
  }

  @Override
  public int next() {
    if (parser.isClosed() || !parser.hasCurrentToken()) {
      return 0;
    }

    resetBatch();

    int nextRowIndex = 0;
{'fraction_non_alphanumeric': 0.09200734512419059, 'fraction_numerical': 0.004349086691794723, 'mean_word_length': 4.623913043478261, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6038903', 'n_tokens_mistral': 2979, 'n_tokens_neox': 2880, 'n_words': 736}