<commit_before># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=line-too-long

r"""Creates a labeling task on Vertex AI.

This script converts a subset of unlabeled TF examples into labeling image
format (before and after images of each building juxtaposed side-by-side),
uploads them to Vertex AI as a dataset, and creates a labeling job for it. [1]
It will assign the job to the labeler pool that you specify. You can create
labeler pools using the REST API documented at [2].

Example invocation:

python create_cloud_labeling_task.py \
  --cloud_project=my-cloud-project \
  --cloud_location=us-central1 \
  --dataset_name=some_dataset_name \
  --examples_pattern=gs://bucket/disaster/examples/unlabeled-large/*.tfrecord \
  --images_dir=gs://bucket/disaster/examples/labeling-images \
  --max_images=1000 \
  --cloud_labeler_emails=<EMAIL>,<EMAIL>

[1] https://cloud.google.com/vertex-ai/docs/datasets/data-labeling-job
[2] https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.specialistPools
"""
# pylint: enable=line-too-long


import time

from absl import app
from absl import flags
from absl import logging

from skai import cloud_labeling

FLAGS = flags.FLAGS
flags.DEFINE_string('cloud_project', 'disaster-assessment', 'GCP project name.')
flags.DEFINE_string('cloud_location', 'us-central1', 'Project location.')
flags.DEFINE_string('examples_pattern', '', 'Pattern matching TFRecords.')
flags.DEFINE_string('images_dir', '', 'Directory to write images to.')
flags.DEFINE_integer('max_images', 1000, 'Maximum number of images to label.')
flags.DEFINE_bool('randomize', True, 'If true, randomly sample images.')
flags.DEFINE_string('dataset_name', None, 'Dataset name')
flags.DEFINE_list(
    'label_classes',
    ['no_damage', 'minor_damage', 'major_damage', 'destroyed', 'bad_example'],
    'Label classes.')
flags.DEFINE_bool('use_google_managed_labelers', False,
                  'If true, assign the task to Google managed labeling pool.')
flags.DEFINE_string(
    'cloud_labeler_pool', None, 'Labeler pool. Format is '
    'projects/<project>/locations/us-central1/specialistPools/<id>.')
flags.DEFINE_list(
    'cloud_labeler_emails', None, 'Emails of workers of new labeler pool. '
    'First email will become the manager.')
flags.DEFINE_string('labeler_instructions_uri',
                    'gs://skai-public/labeling_instructions_v2.pdf',
                    'URI for instructions.')


def _get_labeling_dataset_region(project_region: str) -> str:
  """Choose where to host a labeling dataset.

  As of November 2021, labeling datasets can only be created in "us-central1"
  and "europe-west4" regions. See

  https://cloud.google.com/vertex-ai/docs/general/locations#available-regions

  Args:
    project_region: The region of the project.

  Returns:
    Supported region for hosting the labeling dataset.
  """
  if project_region.startswith('europe-'):
    return 'europe-west4'
  return 'us-central1'


def main(unused_argv):
  timestamp = time.strftime('%Y%m%d_%H%M%S')
  timestamped_dataset = f'{FLAGS.dataset_name}_{timestamp}'

  num_images, import_file_path = cloud_labeling.create_labeling_images(
      FLAGS.examples_pattern, FLAGS.max_images, FLAGS.images_dir)
  logging.info('Wrote %d labeling images.', num_images)
  logging.info('Wrote import file %s.', import_file_path)

  if FLAGS.use_google_managed_labelers:
    labeler_pool = None
  elif FLAGS.cloud_labeler_pool is not None:
    labeler_pool = FLAGS.cloud_labeler_pool
  else:
    # Create a new labeler pool.
    if not FLAGS.cloud_labeler_emails:
      raise ValueError('Must provide at least one labeler email.')

    pool_display_name = f'{timestamped_dataset}_pool'<commit_msg>Add option to create labeling tasks from an existing import file.<commit_after># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=line-too-long

r"""Creates a labeling task on Vertex AI.

This script converts a subset of unlabeled TF examples into labeling image
format (before and after images of each building juxtaposed side-by-side),
uploads them to Vertex AI as a dataset, and creates a labeling job for it. [1]
It will assign the job to the labeler pool that you specify. You can create
labeler pools using the REST API documented at [2].

Example invocation:

python create_cloud_labeling_task.py \
  --cloud_project=my-cloud-project \
  --cloud_location=us-central1 \
  --dataset_name=some_dataset_name \
  --examples_pattern=gs://bucket/disaster/examples/unlabeled-large/*.tfrecord \
  --images_dir=gs://bucket/disaster/examples/labeling-images \
  --max_images=1000 \
  --cloud_labeler_emails=<EMAIL>,<EMAIL>

[1] https://cloud.google.com/vertex-ai/docs/datasets/data-labeling-job
[2] https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.specialistPools
"""
# pylint: enable=line-too-long


import time

from absl import app
from absl import flags
from absl import logging

from skai import cloud_labeling

FLAGS = flags.FLAGS
flags.DEFINE_string('cloud_project', 'disaster-assessment', 'GCP project name.')
flags.DEFINE_string('cloud_location', 'us-central1', 'Project location.')
flags.DEFINE_string('examples_pattern', '', 'Pattern matching TFRecords.')
flags.DEFINE_string('images_dir', '', 'Directory to write images to.')
flags.DEFINE_string('import_file_path', None, 'Import file path.')
flags.DEFINE_integer('max_images', 1000, 'Maximum number of images to label.')
flags.DEFINE_bool('randomize', True, 'If true, randomly sample images.')
flags.DEFINE_string('dataset_name', None, 'Dataset name')
flags.DEFINE_list(
    'label_classes',
    ['no_damage', 'minor_damage', 'major_damage', 'destroyed', 'bad_example'],
    'Label classes.')
flags.DEFINE_bool('use_google_managed_labelers', False,
                  'If true, assign the task to Google managed labeling pool.')
flags.DEFINE_string(
    'cloud_labeler_pool', None, 'Labeler pool. Format is '
    'projects/<project>/locations/us-central1/specialistPools/<id>.')
flags.DEFINE_list(
    'cloud_labeler_emails', None, 'Emails of workers of new labeler pool. '
    'First email will become the manager.')
flags.DEFINE_string('labeler_instructions_uri',
                    'gs://skai-public/labeling_instructions_v2.pdf',
                    'URI for instructions.')


def _get_labeling_dataset_region(project_region: str) -> str:
  """Choose where to host a labeling dataset.

  As of November 2021, labeling datasets can only be created in "us-central1"
  and "europe-west4" regions. See

  https://cloud.google.com/vertex-ai/docs/general/locations#available-regions

  Args:
    project_region: The region of the project.

  Returns:
    Supported region for hosting the labeling dataset.
  """
  if project_region.startswith('europe-'):
    return 'europe-west4'
  return 'us-central1'


def main(unused_argv):
  timestamp = time.strftime('%Y%m%d_%H%M%S')
  timestamped_dataset = f'{FLAGS.dataset_name}_{timestamp}'

  if FLAGS.import_file_path:
    import_file_path = FLAGS.import_file_path
  else:
    num_images, import_file_path = cloud_labeling.create_labeling_images(
        FLAGS.examples_pattern, FLAGS.max_images, FLAGS.images_dir)
    logging.info('Wrote %d labeling images.', num_images)
    logging.info('Wrote import file %s.', import_file_path)

  if FLAGS.use_google_managed_labelers:
    labeler_pool = None
  elif FLAGS.cloud_labeler_pool is not None:
    labeler_pool = FLAGS.cloud_labeler_pool
  else:
    # Create a new labeler pool.
    if not FLAGS.cloud_labeler_emails:
      raise ValueError('Must provide at least one labeler email.')

    pool_display_name = f'{timestamped_dataset}_pool'
{'fraction_non_alphanumeric': 0.10514233241505969, 'fraction_numerical': 0.007575757575757576, 'mean_word_length': 4.428660436137072, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5266171', 'n_tokens_mistral': 2794, 'n_tokens_neox': 2625, 'n_words': 779}