<commit_before>name: evaluation_data_splitter
description: |
  Removes the Ground Truth columns from the input dataset for supporting unstructured AutoML models
  and custom models in Batch Prediction. Creates a Dataflow job with Apache Beam to remove the
  ground truth columns.

  Args:
      project (str):
          Required. Project to retrieve dataset from.
      location (Optional[str]):
          Location to retrieve dataset from.
          If not set, defaulted to `us-central1`.
      root_dir (str):
          Required. The GCS directory for keeping staging files.
          A random subdirectory will be created under the directory to keep job info for resuming
          the job in case of failure.
      gcs_source_uris ([Sequence[str]):
          Required. Google Cloud Storage URI(-s) to your instances to run
          data splitter on. They must match `instances_format`.
          May contain wildcards. For more information on wildcards, see
          https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames.
      instances_format (Optional[str]):
          The format in which instances are given, must be one of
          the model's supported input storage formats.
          If not set, default to "jsonl".
      dataflow_service_account (Optional[str]):
          Optional. Service account to run the dataflow job.
          If not set, dataflow will use the default woker service account.

          For more details, see https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account
      dataflow_subnetwork (Optional[str]):
          Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be
          used. More details:
          https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications
      dataflow_use_public_ips (Optional[bool]):
          Specifies whether Dataflow workers use public IP addresses.
      encryption_spec_key_name (Optional[str]):
          Customer-managed encryption key for the Dataflow job. If this is set, then all resources
          created by the Dataflow job will be encrypted with the provided encryption key.
  Returns:
      gcs_output_directory (JsonArray):
          JsonArray of output split dataset that has the ground truth columns removed.
      gcp_resources (str):
          Serialized gcp_resources proto tracking the data splitter.
inputs:
  - { name: project, type: String }
  - { name: location, type: String, default: "us-central1" }
  - { name: root_dir, type: String }
  - { name: gcs_source_uris, type: JsonArray, default: "[]" }
  - { name: instances_format, type: String, default: "jsonl" }
  - { name: dataflow_service_account, type: String, default: "" }
  - { name: dataflow_subnetwork, type: String, default: "" }
  - { name: dataflow_use_public_ips, type: Boolean, default: "true" }
  - { name: encryption_spec_key_name, type: String, default: "" }
outputs:
  - { name: gcs_output_directory, type: JsonArray }
  - { name: gcp_resources, type: String }
implementation:
  container:
    image: gcr.io/ml-pipeline/model-evaluation:v0.3
    command:
      - python
      - /main.py
    args:
      - --task
      - "data_splitter"
      - --display_name
      - "data-splitter-run"
      - --project_id
      - { inputValue: project }
      - --location
      - { inputValue: location }
      - --root_dir
      - "{{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}"
      - --gcs_source_uris
      - { inputValue: gcs_source_uris }
      - --instances_format
      - { inputValue: instances_format}
      - --dataflow_service_account
      - { inputValue: dataflow_service_account }
      - --dataflow_subnetwork
      - { inputValue: dataflow_subnetwork }
      - --dataflow_use_public_ips
      - { inputValue: dataflow_use_public_ips }
      - --kms_key_name
      - { inputValue: encryption_spec_key_name }
      - --gcs_output_directory
      - { outputPath: gcs_output_directory }
      - --gcp_resources
      - { outputPath: gcp_resources }
      - --executor_input
      - "{{$}}"<commit_msg>feat(google-cloud): Add ground_truth_column input argument to data splitter component.<commit_after>name: evaluation_data_splitter
description: |
  Removes the Ground Truth columns from the input dataset for supporting unstructured AutoML models
  and custom models in Batch Prediction. Creates a Dataflow job with Apache Beam to remove the
  ground truth columns.

  Args:
      project (str):
          Required. Project to retrieve dataset from.
      location (Optional[str]):
          Location to retrieve dataset from.
          If not set, defaulted to `us-central1`.
      root_dir (str):
          Required. The GCS directory for keeping staging files.
          A random subdirectory will be created under the directory to keep job info for resuming
          the job in case of failure.
      gcs_source_uris ([Sequence[str]):
          Required. Google Cloud Storage URI(-s) to your instances to run
          data splitter on. They must match `instances_format`.
          May contain wildcards. For more information on wildcards, see
          https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames.
      instances_format (Optional[str]):
          The format in which instances are given, must be one of
          the model's supported input storage formats.
          If not set, default to "jsonl".
      ground_truth_column (str):
          The column name of the feature containing ground truth.
      dataflow_service_account (Optional[str]):
          Optional. Service account to run the dataflow job.
          If not set, dataflow will use the default woker service account.

          For more details, see https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account
      dataflow_subnetwork (Optional[str]):
          Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be
          used. More details:
          https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications
      dataflow_use_public_ips (Optional[bool]):
          Specifies whether Dataflow workers use public IP addresses.
      encryption_spec_key_name (Optional[str]):
          Customer-managed encryption key for the Dataflow job. If this is set, then all resources
          created by the Dataflow job will be encrypted with the provided encryption key.
  Returns:
      gcs_output_directory (JsonArray):
          JsonArray of output split dataset that has the ground truth columns removed.
      gcp_resources (str):
          Serialized gcp_resources proto tracking the data splitter.
inputs:
  - { name: project, type: String }
  - { name: location, type: String, default: "us-central1" }
  - { name: root_dir, type: String }
  - { name: gcs_source_uris, type: JsonArray, default: "[]" }
  - { name: instances_format, type: String, default: "jsonl" }
  - { name: ground_truth_column, type: String, default: "ground_truth"}
  - { name: dataflow_service_account, type: String, default: "" }
  - { name: dataflow_subnetwork, type: String, default: "" }
  - { name: dataflow_use_public_ips, type: Boolean, default: "true" }
  - { name: encryption_spec_key_name, type: String, default: "" }
outputs:
  - { name: gcs_output_directory, type: JsonArray }
  - { name: gcp_resources, type: String }
implementation:
  container:
    image: gcr.io/ml-pipeline/model-evaluation:v0.3
    command:
      - python
      - /main.py
    args:
      - --task
      - "data_splitter"
      - --display_name
      - "data-splitter-run"
      - --project_id
      - { inputValue: project }
      - --location
      - { inputValue: location }
      - --root_dir
      - "{{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}"
      - --gcs_source_uris
      - { inputValue: gcs_source_uris }
      - --instances_format
      - { inputValue: instances_format}
      - --ground_truth_column
      - { inputValue: ground_truth_column}
      - --dataflow_service_account
      - { inputValue: dataflow_service_account }
      - --dataflow_subnetwork
      - { inputValue: dataflow_subnetwork }
      - --dataflow_use_public_ips
      - { inputValue: dataflow_use_public_ips }
      - --kms_key_name
      - { inputValue: encryption_spec_key_name }
      - --gcs_output_directory
      - { outputPath: gcs_output_directory }
      - --gcp_resources
      - { outputPath: gcp_resources }
      - --executor_input
      - "{{$}}"
{'fraction_non_alphanumeric': 0.09030913511635985, 'fraction_numerical': 0.0009262475396549728, 'mean_word_length': 2.7556521739130435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6445939', 'n_tokens_mistral': 2460, 'n_tokens_neox': 2417, 'n_words': 730}