<commit_before>/**
 * Copyright 2015 StreamSets Inc.
 *
 * Licensed under the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.streamsets.pipeline.stage.origin.hdfs.cluster;

import com.streamsets.pipeline.api.ListBeanModel;
import com.streamsets.pipeline.api.ConfigDef;
import com.streamsets.pipeline.api.ConfigGroups;
import com.streamsets.pipeline.api.ErrorListener;
import com.streamsets.pipeline.api.ExecutionMode;
import com.streamsets.pipeline.api.GenerateResourceBundle;
import com.streamsets.pipeline.api.Source;
import com.streamsets.pipeline.api.StageDef;
import com.streamsets.pipeline.api.ValueChooserModel;
import com.streamsets.pipeline.config.CsvHeader;
import com.streamsets.pipeline.config.CsvHeaderChooserValues;
import com.streamsets.pipeline.config.CsvMode;
import com.streamsets.pipeline.config.CsvModeChooserValues;
import com.streamsets.pipeline.config.CsvRecordType;
import com.streamsets.pipeline.config.CsvRecordTypeChooserValues;
import com.streamsets.pipeline.config.DataFormat;
import com.streamsets.pipeline.config.LogMode;
import com.streamsets.pipeline.config.LogModeChooserValues;
import com.streamsets.pipeline.configurablestage.DClusterSourceOffsetCommitter;
import com.streamsets.pipeline.lib.parser.log.RegExConfig;

import java.util.List;
import java.util.Map;


@StageDef(
  version = 2,
  label = "Hadoop FS",
  description = "Reads data from Hadoop file system",
  execution = ExecutionMode.CLUSTER_BATCH,
  libJarsRegex = {"avro-\\d+.*", "avro-mapred.*"},
  icon = "hdfs.png",
  privateClassLoader = true,
  upgrader = ClusterHdfsSourceUpgrader.class,
  onlineHelpRefUrl = "index.html#Origins/HadoopFS-origin.html#task_hgl_vgn_vs"
)
@ConfigGroups(value = Groups.class)
@GenerateResourceBundle
public class ClusterHdfsDSource extends DClusterSourceOffsetCommitter implements ErrorListener {
  private ClusterHdfsSource clusterHDFSSource;

  @ConfigDef(
    required = false,
    type = ConfigDef.Type.STRING,
    label = "Hadoop FS URI",
    description = "Include the HDFS scheme and authority: hdfs://<authority>:<port>. If this is not set, the URI will be set to the value of "
      + "'fs.defaultFS' configuration",
    displayPosition = 10,
    group = "HADOOP_FS")
  public String hdfsUri;

  @ConfigDef(
    required = true,
    type = ConfigDef.Type.LIST,
    defaultValue = "[]",
    label = "Directory Path",
    description = "HDFS directory path",
    displayPosition = 10,
    group = "HADOOP_FS"
  )
  public List<String> hdfsDirLocations; // hdfsDirLocation

  @ConfigDef(
    required = false,
    type = ConfigDef.Type.TEXT,
    defaultValue = "",
    label = "Avro Reader Schema",
    description = "",
    displayPosition = 20,
    group = "AVRO",
    dependsOn = "dataFormat",
    triggeredByValue = "AVRO",
    mode = ConfigDef.Mode.PLAIN_TEXT
  )
  public String avroSchema;<commit_msg>SDC-1756. Standardize labels and workflows across origins<commit_after>/**
 * Copyright 2015 StreamSets Inc.
 *
 * Licensed under the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.streamsets.pipeline.stage.origin.hdfs.cluster;

import com.streamsets.pipeline.api.ListBeanModel;
import com.streamsets.pipeline.api.ConfigDef;
import com.streamsets.pipeline.api.ConfigGroups;
import com.streamsets.pipeline.api.ErrorListener;
import com.streamsets.pipeline.api.ExecutionMode;
import com.streamsets.pipeline.api.GenerateResourceBundle;
import com.streamsets.pipeline.api.Source;
import com.streamsets.pipeline.api.StageDef;
import com.streamsets.pipeline.api.ValueChooserModel;
import com.streamsets.pipeline.config.CsvHeader;
import com.streamsets.pipeline.config.CsvHeaderChooserValues;
import com.streamsets.pipeline.config.CsvMode;
import com.streamsets.pipeline.config.CsvModeChooserValues;
import com.streamsets.pipeline.config.CsvRecordType;
import com.streamsets.pipeline.config.CsvRecordTypeChooserValues;
import com.streamsets.pipeline.config.DataFormat;
import com.streamsets.pipeline.config.LogMode;
import com.streamsets.pipeline.config.LogModeChooserValues;
import com.streamsets.pipeline.configurablestage.DClusterSourceOffsetCommitter;
import com.streamsets.pipeline.lib.parser.log.RegExConfig;

import java.util.List;
import java.util.Map;


@StageDef(
  version = 2,
  label = "Hadoop FS",
  description = "Reads data from Hadoop file system",
  execution = ExecutionMode.CLUSTER_BATCH,
  libJarsRegex = {"avro-\\d+.*", "avro-mapred.*"},
  icon = "hdfs.png",
  privateClassLoader = true,
  upgrader = ClusterHdfsSourceUpgrader.class,
  onlineHelpRefUrl = "index.html#Origins/HadoopFS-origin.html#task_hgl_vgn_vs"
)
@ConfigGroups(value = Groups.class)
@GenerateResourceBundle
public class ClusterHdfsDSource extends DClusterSourceOffsetCommitter implements ErrorListener {
  private ClusterHdfsSource clusterHDFSSource;

  @ConfigDef(
    required = false,
    type = ConfigDef.Type.STRING,
    label = "Hadoop FS URI",
    description = "Include the HDFS scheme and authority: hdfs://<authority>:<port>. If this is not set, the URI will be set to the value of "
      + "'fs.defaultFS' configuration",
    displayPosition = 10,
    group = "HADOOP_FS")
  public String hdfsUri;

  @ConfigDef(
    required = true,
    type = ConfigDef.Type.LIST,
    defaultValue = "[]",
    label = "Directory Path",
    description = "HDFS directory path",
    displayPosition = 10,
    group = "HADOOP_FS"
  )
  public List<String> hdfsDirLocations; // hdfsDirLocation

  @ConfigDef(
    required = false,
    type = ConfigDef.Type.TEXT,
    defaultValue = "",
    label = "Avro Schema",
    description = "",
    displayPosition = 20,
    group = "AVRO",
    dependsOn = "dataFormat",
    triggeredByValue = "AVRO",
    mode = ConfigDef.Mode.PLAIN_TEXT
  )
  public String avroSchema;
{'fraction_non_alphanumeric': 0.09484308032628232, 'fraction_numerical': 0.004700677450573759, 'mean_word_length': 4.530581039755352, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4338496', 'n_tokens_mistral': 2189, 'n_tokens_neox': 2163, 'n_words': 578}