<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.beam.runners.dataflow;

import java.util.ArrayList;
import java.util.List;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.dataflow.DataflowRunner.StreamingPCollectionViewWriterFn;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Combine.CombineFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;

/**
 * Dataflow streaming overrides for {@link CreatePCollectionView}, specialized for different view
 * types.
 */
class StreamingViewOverrides {
  static class StreamingCreatePCollectionViewFactory<ElemT, ViewT>
      extends SingleInputOutputOverrideFactory<
          PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>> {
    @Override
    public PTransform<PCollection<ElemT>, PCollectionView<ViewT>> getReplacementTransform(
        final CreatePCollectionView<ElemT, ViewT> transform) {
      return new StreamingCreatePCollectionView<>(transform.getView());
    }

    private static class StreamingCreatePCollectionView<ElemT, ViewT>
        extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
      private final PCollectionView<ViewT> view;

      private StreamingCreatePCollectionView(PCollectionView<ViewT> view) {
        this.view = view;
      }

      @Override
      public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
        return input
            .apply(Combine.globally(new Concatenate<ElemT>()).withoutDefaults())
            .apply(ParDo.of(StreamingPCollectionViewWriterFn.create(view, input.getCoder())))
            .apply(View.CreatePCollectionView.<ElemT, ViewT>of(view));
      }
    }
  }

  private static class SingletonCombine<T> extends Combine.BinaryCombineFn<T> {
    private boolean hasDefaultValue;
    private T defaultValue;

    SingletonCombine(boolean hasDefaultValue, T defaultValue) {
      this.hasDefaultValue = hasDefaultValue;
      this.defaultValue = defaultValue;
    }

    @Override
    public T apply(T left, T right) {
      throw new IllegalArgumentException(
          "PCollection with more than one element "
              + "accessed as a singleton view. Consider using Combine.globally().asSingleton() to "
              + "combine the PCollection into a single value");
    }

    @Override
    public T identity() {
      if (hasDefaultValue) {
        return defaultValue;
      } else {
        throw new IllegalArgumentException(
            "Empty PCollection accessed as a singleton view. "
                + "Consider setting withDefault to provide a default value");
      }
    }
  }

  /**
   * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
   *
   * @param <T> the type of elements to concatenate.
   */
  private static class Concatenate<T> extends CombineFn<T, List<T>, List<T>> {
    @Override
    public List<T> createAccumulator() {
      return new ArrayList<>();
    }

    @Override
    public List<T> addInput(List<T> accumulator, T input) {
      accumulator.add(input);
      return accumulator;
    }

    @Override
    public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
      List<T> result = createAccumulator();
      for (List<T> accumulator : accumulators) {
        result.addAll(accumulator);
      }
      return result;
    }

    @Override<commit_msg>Remove SingletonCombine<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.beam.runners.dataflow;

import java.util.ArrayList;
import java.util.List;
import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
import org.apache.beam.runners.dataflow.DataflowRunner.StreamingPCollectionViewWriterFn;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.ListCoder;
import org.apache.beam.sdk.transforms.Combine;
import org.apache.beam.sdk.transforms.Combine.CombineFn;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.View.CreatePCollectionView;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;

/**
 * Dataflow streaming overrides for {@link CreatePCollectionView}, specialized for different view
 * types.
 */
class StreamingViewOverrides {
  static class StreamingCreatePCollectionViewFactory<ElemT, ViewT>
      extends SingleInputOutputOverrideFactory<
          PCollection<ElemT>, PCollectionView<ViewT>, CreatePCollectionView<ElemT, ViewT>> {
    @Override
    public PTransform<PCollection<ElemT>, PCollectionView<ViewT>> getReplacementTransform(
        final CreatePCollectionView<ElemT, ViewT> transform) {
      return new StreamingCreatePCollectionView<>(transform.getView());
    }

    private static class StreamingCreatePCollectionView<ElemT, ViewT>
        extends PTransform<PCollection<ElemT>, PCollectionView<ViewT>> {
      private final PCollectionView<ViewT> view;

      private StreamingCreatePCollectionView(PCollectionView<ViewT> view) {
        this.view = view;
      }

      @Override
      public PCollectionView<ViewT> expand(PCollection<ElemT> input) {
        return input
            .apply(Combine.globally(new Concatenate<ElemT>()).withoutDefaults())
            .apply(ParDo.of(StreamingPCollectionViewWriterFn.create(view, input.getCoder())))
            .apply(View.CreatePCollectionView.<ElemT, ViewT>of(view));
      }
    }
  }

  /**
   * Combiner that combines {@code T}s into a single {@code List<T>} containing all inputs.
   *
   * @param <T> the type of elements to concatenate.
   */
  private static class Concatenate<T> extends CombineFn<T, List<T>, List<T>> {
    @Override
    public List<T> createAccumulator() {
      return new ArrayList<>();
    }

    @Override
    public List<T> addInput(List<T> accumulator, T input) {
      accumulator.add(input);
      return accumulator;
    }

    @Override
    public List<T> mergeAccumulators(Iterable<List<T>> accumulators) {
      List<T> result = createAccumulator();
      for (List<T> accumulator : accumulators) {
        result.addAll(accumulator);
      }
      return result;
    }

    @Override
{'fraction_non_alphanumeric': 0.0935722067546302, 'fraction_numerical': 0.000968405762014284, 'mean_word_length': 3.7923433874709978, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5905060', 'n_tokens_mistral': 2363, 'n_tokens_neox': 2304, 'n_words': 623}