<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;

/**
 * Empty impl needed for compilation. Spark DataSourceV2 API was removed in Spark3. Need to code a Beam source using new spark 3 API.
 */

public class DatasetSourceBatch {
}<commit_msg>[BEAM-12629] Make a spark 3 source impl<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;

import static org.apache.beam.runners.spark.structuredstreaming.Constants.BEAM_SOURCE_OPTION;
import static org.apache.beam.runners.spark.structuredstreaming.Constants.DEFAULT_PARALLELISM;
import static org.apache.beam.runners.spark.structuredstreaming.Constants.PIPELINE_OPTIONS;
import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;

import java.io.IOException;
import java.io.Serializable;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.runners.core.serialization.Base64Serializer;
import org.apache.beam.runners.spark.structuredstreaming.translation.helpers.RowHelpers;
import org.apache.beam.runners.spark.structuredstreaming.translation.helpers.SchemaHelpers;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.parquet.Strings;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.connector.catalog.SupportsRead;
import org.apache.spark.sql.connector.catalog.Table;
import org.apache.spark.sql.connector.catalog.TableCapability;
import org.apache.spark.sql.connector.catalog.TableProvider;
import org.apache.spark.sql.connector.expressions.Transform;
import org.apache.spark.sql.connector.read.Batch;
import org.apache.spark.sql.connector.read.InputPartition;
import org.apache.spark.sql.connector.read.PartitionReader;
import org.apache.spark.sql.connector.read.PartitionReaderFactory;
import org.apache.spark.sql.connector.read.Scan;
import org.apache.spark.sql.connector.read.ScanBuilder;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;

/**
 * Empty impl needed for compilation. Spark DataSourceV2 API was removed in Spark3. Need to code a Beam source using new spark 3 API.
 */

public class DatasetSourceBatch implements TableProvider {

  private static StructType BINARY_SCHEMA = SchemaHelpers.binarySchema();

  public DatasetSourceBatch() {
  }

  @Override
  public StructType inferSchema(CaseInsensitiveStringMap options) {
    return null;
  }

  @Override
  public boolean supportsExternalMetadata() {
    return true;
  }

  @Override
  public Table getTable(StructType schema, Transform[] partitioning,
    Map<String, String> properties) {
    return new DatasetSourceBatchTable();
  }

  private static class DatasetSourceBatchTable implements SupportsRead, Serializable {

    @Override
    public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) {
      return new ScanBuilder() {

        @Override
        public Scan build() {
          return new Scan() { // scan for Batch reading

            @Override
            public StructType readSchema() {
              return BINARY_SCHEMA;
            }

            @Override
            public Batch toBatch() {
              return new BeamBatch<>(options);
            }
          };
        }
      };
    }

    @Override
    public String name() {
      return "BeamSource";
    }

    @Override
    public StructType schema() {
      return BINARY_SCHEMA;
    }

    @Override
    public Set<TableCapability> capabilities() {
      final HashSet<TableCapability> capabilities = new HashSet<>();
      capabilities.add(TableCapability.BATCH_READ);
      return capabilities;
    }

    private static class BeamBatch<T> implements Batch, Serializable {

      private final int numPartitions;
      private final BoundedSource<T> source;
      private final SerializablePipelineOptions serializablePipelineOptions;

      private BeamBatch(CaseInsensitiveStringMap options) {
        if (Strings.isNullOrEmpty(options.get(BEAM_SOURCE_OPTION))) {
          throw new RuntimeException("Beam source was not set in DataSource options");
        }
        this.source = Base64Serializer
          .deserializeUnchecked(options.get(BEAM_SOURCE_OPTION), BoundedSource.class);

        if (Strings.isNullOrEmpty(DEFAULT_PARALLELISM)) {
          throw new RuntimeException("Spark default parallelism was not set in DataSource options");
        }
        this.numPartitions = Integer.parseInt(options.get(DEFAULT_PARALLELISM));
        checkArgument(numPartitions > 0, "Number of partitions must be greater than zero.");

        if (Strings.isNullOrEmpty(options.get(PIPELINE_OPTIONS))) {
          throw new RuntimeException("Beam pipelineOptions were not set in DataSource options");
        }
        this.serializablePipelineOptions = new SerializablePipelineOptions(
          options.get(PIPELINE_OPTIONS));
      }

      @Override
      public InputPartition[] planInputPartitions() {
        PipelineOptions options = serializablePipelineOptions.get();
        long desiredSizeBytes;

        try {
          desiredSizeBytes = source.getEstimatedSizeBytes(options) / numPartitions;
          List<? extends BoundedSource<T>> splits = source.split(desiredSizeBytes, options);
          InputPartition[] result = new InputPartition[splits.size()];
          int i = 0;
          for (BoundedSource<T> split : splits) {
            result[i++] = new BeamInputPartition<>(split);
          }
          return result;
        } catch (Exception e) {
          throw new RuntimeException(
            "Error in splitting BoundedSource " + source.getClass().getCanonicalName(), e);
        }
      }

      @Override
      public PartitionReaderFactory createReaderFactory() {
        return new PartitionReaderFactory() {

          @Override
          public PartitionReader<InternalRow> createReader(InputPartition partition) {
            return new BeamPartitionReader<T>(((BeamInputPartition<T>) partition).getSource(),
              serializablePipelineOptions);
          }
        };
      }

      private static class BeamInputPartition<T> implements InputPartition, Serializable {

        private final BoundedSource<T> source;

        private BeamInputPartition(BoundedSource<T> source) {
          this.source = source;
        }

        public BoundedSource<T> getSource() {
          return source;
        }
      }

      private static class BeamPartitionReader<T>
        implements PartitionReader<InternalRow>, Serializable {

        private final BoundedSource<T> source;
        private final BoundedSource.BoundedReader<T> reader;
        private boolean started;
        private boolean closed;

        BeamPartitionReader(BoundedSource<T> source,
          SerializablePipelineOptions serializablePipelineOptions) {
          this.started = false;
          this.closed = false;
          this.source = source;
          // reader is not serializable so lazy initialize it
          try {
            reader = source
              .createReader(serializablePipelineOptions.get().as(PipelineOptions.class));
          } catch (IOException e) {
            throw new RuntimeException("Error creating BoundedReader ", e);
          }
        }

        @Override
        public boolean next() throws IOException {
          if (!started) {
            started = true;
            return reader.start();
          } else {
            return !closed && reader.advance();
          }
        }

        @Override
        public InternalRow get() {
          WindowedValue<T> windowedValue = WindowedValue
            .timestampedValueInGlobalWindow(reader.getCurrent(), reader.getCurrentTimestamp());
          return RowHelpers.storeWindowedValueInRow(windowedValue, source.getOutputCoder());
        }

        @Override
        public void close() throws IOException {
          closed = true;
          reader.close();
        }
      }
    }
  }
}
{'fraction_non_alphanumeric': 0.08204761418207827, 'fraction_numerical': 0.0029631143353428015, 'mean_word_length': 3.14570097416349, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7065716', 'n_tokens_mistral': 2592, 'n_tokens_neox': 2494, 'n_words': 707}