<commit_before>/*
 * Copyright 2017 StreamSets Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.streamsets.datacollector.creation;

import com.streamsets.datacollector.config.AmazonEMRConfig;
import com.streamsets.datacollector.config.ClusterConfig;
import com.streamsets.datacollector.config.DatabricksConfig;
import com.streamsets.datacollector.config.DeliveryGuaranteeChooserValues;
import com.streamsets.datacollector.config.ErrorHandlingChooserValues;
import com.streamsets.datacollector.config.ErrorRecordPolicy;
import com.streamsets.datacollector.config.ErrorRecordPolicyChooserValues;
import com.streamsets.datacollector.config.ExecutionModeChooserValues;
import com.streamsets.datacollector.config.LivyConfig;
import com.streamsets.datacollector.config.LogLevel;
import com.streamsets.datacollector.config.LogLevelChooserValues;
import com.streamsets.datacollector.config.PipelineGroups;
import com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues;
import com.streamsets.datacollector.config.PipelineState;
import com.streamsets.datacollector.config.PipelineStateChooserValues;
import com.streamsets.datacollector.config.PipelineTestStageChooserValues;
import com.streamsets.datacollector.config.PipelineWebhookConfig;
import com.streamsets.datacollector.config.StatsTargetChooserValues;
import com.streamsets.pipeline.api.ConfigDef;
import com.streamsets.pipeline.api.ConfigDefBean;
import com.streamsets.pipeline.api.ConfigGroups;
import com.streamsets.pipeline.api.DeliveryGuarantee;
import com.streamsets.pipeline.api.Dependency;
import com.streamsets.pipeline.api.ExecutionMode;
import com.streamsets.pipeline.api.GenerateResourceBundle;
import com.streamsets.pipeline.api.ListBeanModel;
import com.streamsets.pipeline.api.MultiValueChooserModel;
import com.streamsets.pipeline.api.Stage;
import com.streamsets.pipeline.api.StageDef;
import com.streamsets.pipeline.api.ValueChooserModel;
import com.streamsets.pipeline.lib.googlecloud.GoogleCloudConfig;
import com.streamsets.pipeline.lib.googlecloud.GoogleCloudCredentialsConfig;

import java.util.Collections;
import java.util.List;
import java.util.Map;

// we are using the annotation for reference purposes only.
// the annotation processor does not work on this maven project
// we have a hardcoded 'datacollector-resource-bundles.json' file in resources
@GenerateResourceBundle
@StageDef(
    version = PipelineConfigBean.VERSION,
    label = "Pipeline",
    upgrader = PipelineConfigUpgrader.class,
    upgraderDef = "upgrader/PipelineConfigBeanUpgrader.yaml",
    onlineHelpRefUrl = "not applicable"
)
@ConfigGroups(PipelineGroups.class)
public class PipelineConfigBean implements Stage {

  public static final int VERSION = 19;

  public static final String DEFAULT_STATS_AGGREGATOR_LIBRARY_NAME = "streamsets-datacollector-basic-lib";

  public static final String DEFAULT_STATS_AGGREGATOR_STAGE_NAME =
      "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget";

  public static final String DEFAULT_STATS_AGGREGATOR_STAGE_VERSION = "1";

  public static final String STATS_DPM_DIRECTLY_TARGET = DEFAULT_STATS_AGGREGATOR_LIBRARY_NAME + "::" +
      DEFAULT_STATS_AGGREGATOR_STAGE_NAME + "::" + DEFAULT_STATS_AGGREGATOR_STAGE_VERSION;

  public static final String STREAMING_STATS_AGGREGATOR_LIBRARY_NAME = "streamsets-spark-basic-lib";

  public static final String STREAMING_STATS_DPM_DIRECTLY_TARGET = STREAMING_STATS_AGGREGATOR_LIBRARY_NAME + "::" +
      DEFAULT_STATS_AGGREGATOR_STAGE_NAME + "::" + DEFAULT_STATS_AGGREGATOR_STAGE_VERSION;

  public static final String STATS_AGGREGATOR_DEFAULT = "streamsets-datacollector-basic-lib" +
      "::com_streamsets_pipeline_stage_destination_devnull_StatsNullDTarget::1";

  private static final String TRASH_TARGET = "streamsets-datacollector-basic-lib" +
      "::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1";

  public static final String DEFAULT_TEST_ORIGIN_LIBRARY_NAME = "streamsets-datacollector-dev-lib";

  public static final String DEFAULT_TEST_ORIGIN_STAGE_NAME =
      "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource";

  public static final String DEFAULT_TEST_ORIGIN_STAGE_VERSION = "3";

  public static final String RAW_DATA_ORIGIN = DEFAULT_TEST_ORIGIN_LIBRARY_NAME + "::" +
      DEFAULT_TEST_ORIGIN_STAGE_NAME + "::" + DEFAULT_TEST_ORIGIN_STAGE_VERSION;

  public static final String EDGE_HTTP_URL_DEFAULT = "http://localhost:18633";

  public static final String DEFAULT_PREPROCESS_SCRIPT = "" +
      "/*\n" +
      "The following script define a method\n" +
      "that increments an integer by 1 \n" +
      "and registers it as a UDF with \n" +
      "the SparkSession, which can be accessed\n" +
      "using the variable named \"spark\":\n" +
      "def inc(i: Integer): Integer = {\n" +
      "  i + 1\n" +
      "}\n" +
      "spark.udf.register (\"inc\", inc _)\n" +
      "\n" +
      "*/";

  @ConfigDef(
      required = true,
      type = ConfigDef.Type.MODEL,
      label = "Execution Mode",
      defaultValue= "STANDALONE",
      displayMode = ConfigDef.DisplayMode.ADVANCED,
      displayPosition = 10
  )
  @ValueChooserModel(ExecutionModeChooserValues.class)
  public ExecutionMode executionMode;

  @ConfigDef(
      required = true,
      type = ConfigDef.Type.STRING,
      label = "Data Collector Edge URL",
      defaultValue = EDGE_HTTP_URL_DEFAULT,
      displayPosition = 15,
      dependsOn = "executionMode",
      displayMode = ConfigDef.DisplayMode.ADVANCED,
      triggeredByValue = {"EDGE"}
  )
  public String edgeHttpUrl = EDGE_HTTP_URL_DEFAULT;

  @ConfigDef(<commit_msg>SDC-15135: Basic/Advanced: Update javadoc for PIpeline's executionMode<commit_after>/*
 * Copyright 2017 StreamSets Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.streamsets.datacollector.creation;

import com.streamsets.datacollector.config.AmazonEMRConfig;
import com.streamsets.datacollector.config.ClusterConfig;
import com.streamsets.datacollector.config.DatabricksConfig;
import com.streamsets.datacollector.config.DeliveryGuaranteeChooserValues;
import com.streamsets.datacollector.config.ErrorHandlingChooserValues;
import com.streamsets.datacollector.config.ErrorRecordPolicy;
import com.streamsets.datacollector.config.ErrorRecordPolicyChooserValues;
import com.streamsets.datacollector.config.ExecutionModeChooserValues;
import com.streamsets.datacollector.config.LivyConfig;
import com.streamsets.datacollector.config.LogLevel;
import com.streamsets.datacollector.config.LogLevelChooserValues;
import com.streamsets.datacollector.config.PipelineGroups;
import com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues;
import com.streamsets.datacollector.config.PipelineState;
import com.streamsets.datacollector.config.PipelineStateChooserValues;
import com.streamsets.datacollector.config.PipelineTestStageChooserValues;
import com.streamsets.datacollector.config.PipelineWebhookConfig;
import com.streamsets.datacollector.config.StatsTargetChooserValues;
import com.streamsets.pipeline.api.ConfigDef;
import com.streamsets.pipeline.api.ConfigDefBean;
import com.streamsets.pipeline.api.ConfigGroups;
import com.streamsets.pipeline.api.DeliveryGuarantee;
import com.streamsets.pipeline.api.Dependency;
import com.streamsets.pipeline.api.ExecutionMode;
import com.streamsets.pipeline.api.GenerateResourceBundle;
import com.streamsets.pipeline.api.ListBeanModel;
import com.streamsets.pipeline.api.MultiValueChooserModel;
import com.streamsets.pipeline.api.Stage;
import com.streamsets.pipeline.api.StageDef;
import com.streamsets.pipeline.api.ValueChooserModel;
import com.streamsets.pipeline.lib.googlecloud.GoogleCloudConfig;
import com.streamsets.pipeline.lib.googlecloud.GoogleCloudCredentialsConfig;

import java.util.Collections;
import java.util.List;
import java.util.Map;

// we are using the annotation for reference purposes only.
// the annotation processor does not work on this maven project
// we have a hardcoded 'datacollector-resource-bundles.json' file in resources
@GenerateResourceBundle
@StageDef(
    version = PipelineConfigBean.VERSION,
    label = "Pipeline",
    upgrader = PipelineConfigUpgrader.class,
    upgraderDef = "upgrader/PipelineConfigBeanUpgrader.yaml",
    onlineHelpRefUrl = "not applicable"
)
@ConfigGroups(PipelineGroups.class)
public class PipelineConfigBean implements Stage {

  public static final int VERSION = 19;

  public static final String DEFAULT_STATS_AGGREGATOR_LIBRARY_NAME = "streamsets-datacollector-basic-lib";

  public static final String DEFAULT_STATS_AGGREGATOR_STAGE_NAME =
      "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget";

  public static final String DEFAULT_STATS_AGGREGATOR_STAGE_VERSION = "1";

  public static final String STATS_DPM_DIRECTLY_TARGET = DEFAULT_STATS_AGGREGATOR_LIBRARY_NAME + "::" +
      DEFAULT_STATS_AGGREGATOR_STAGE_NAME + "::" + DEFAULT_STATS_AGGREGATOR_STAGE_VERSION;

  public static final String STREAMING_STATS_AGGREGATOR_LIBRARY_NAME = "streamsets-spark-basic-lib";

  public static final String STREAMING_STATS_DPM_DIRECTLY_TARGET = STREAMING_STATS_AGGREGATOR_LIBRARY_NAME + "::" +
      DEFAULT_STATS_AGGREGATOR_STAGE_NAME + "::" + DEFAULT_STATS_AGGREGATOR_STAGE_VERSION;

  public static final String STATS_AGGREGATOR_DEFAULT = "streamsets-datacollector-basic-lib" +
      "::com_streamsets_pipeline_stage_destination_devnull_StatsNullDTarget::1";

  private static final String TRASH_TARGET = "streamsets-datacollector-basic-lib" +
      "::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1";

  public static final String DEFAULT_TEST_ORIGIN_LIBRARY_NAME = "streamsets-datacollector-dev-lib";

  public static final String DEFAULT_TEST_ORIGIN_STAGE_NAME =
      "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource";

  public static final String DEFAULT_TEST_ORIGIN_STAGE_VERSION = "3";

  public static final String RAW_DATA_ORIGIN = DEFAULT_TEST_ORIGIN_LIBRARY_NAME + "::" +
      DEFAULT_TEST_ORIGIN_STAGE_NAME + "::" + DEFAULT_TEST_ORIGIN_STAGE_VERSION;

  public static final String EDGE_HTTP_URL_DEFAULT = "http://localhost:18633";

  public static final String DEFAULT_PREPROCESS_SCRIPT = "" +
      "/*\n" +
      "The following script define a method\n" +
      "that increments an integer by 1 \n" +
      "and registers it as a UDF with \n" +
      "the SparkSession, which can be accessed\n" +
      "using the variable named \"spark\":\n" +
      "def inc(i: Integer): Integer = {\n" +
      "  i + 1\n" +
      "}\n" +
      "spark.udf.register (\"inc\", inc _)\n" +
      "\n" +
      "*/";

  @ConfigDef(
      required = true,
      type = ConfigDef.Type.MODEL,
      label = "Execution Mode",
      defaultValue= "STANDALONE",
      /*
       The display mode for executionMode here is kind of bogus and is ignored by the UI. This is because the field is
       re-used between SDC and Transformer and each product needs it on a different value. SDC considers this field
       as ADVANCED whereas Transformer as BASIC.
      */
      displayMode = ConfigDef.DisplayMode.BASIC,
      displayPosition = 10
  )
  @ValueChooserModel(ExecutionModeChooserValues.class)
  public ExecutionMode executionMode;

  @ConfigDef(
      required = true,
      type = ConfigDef.Type.STRING,
      label = "Data Collector Edge URL",
      defaultValue = EDGE_HTTP_URL_DEFAULT,
      displayPosition = 15,
      dependsOn = "executionMode",
      displayMode = ConfigDef.DisplayMode.ADVANCED,
      triggeredByValue = {"EDGE"}
  )
  public String edgeHttpUrl = EDGE_HTTP_URL_DEFAULT;

  @ConfigDef(
{'fraction_non_alphanumeric': 0.08232235701906412, 'fraction_numerical': 0.004332755632582322, 'mean_word_length': 5.079980842911877, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2773870', 'n_tokens_mistral': 3945, 'n_tokens_neox': 3887, 'n_words': 825}