<commit_before># coding: utf-8
# Copyright 2014 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

"""Tasks definition.

Each user request creates a new TaskRequest. The TaskRequest instance saves the
metadata of the request, e.g. who requested it, when why, etc. It links to the
actual data of the request in a TaskProperties. The TaskProperties represents
everything needed to run the task.

This means if two users request an identical task, it can be deduped
accordingly and efficiently by the scheduler.

Note that the mere existence of a TaskRequest in the db doesn't mean it will be
scheduled, see task_scheduler.py for the actual scheduling. Registering tasks
and scheduling are kept separated to keep the immutable and mutable models in
separate files.


Overview of transactions:
- TaskRequest() are created inside a transaction.


Graph of the schema:

    +--------Root-------+
    |TaskRequest        |
    |  +--------------+ |
    |  |TaskProperties| |
    |  |  +--------+  | |
    |  |  |FilesRef|  | |
    |  |  +--------+  | |
    |  +--------------+ |
    |id=<based on epoch>|
    +-------------------+
        |          |
        v          |
    +-----------+  |
    |SecretBytes|  |
    |id=1       |  |
    +-----------+  |
                   |
                   v
    <See task_to_run.py and task_result.py>

TaskProperties is embedded in TaskRequest. TaskProperties is still declared as a
separate entity to clearly declare the boundary for task request deduplication.
"""

import datetime
import hashlib
import posixpath
import random
import re

from google.appengine.api import datastore_errors
from google.appengine.ext import ndb

from components import auth
from components import datastore_utils
from components import pubsub
from components import utils
from components.config import validation

from server import config
from server import service_accounts
from server import task_pack
import cipd


# Maximum acceptable priority value, which is effectively the lowest priority.
MAXIMUM_PRIORITY = 255


# Three days in seconds. Add 10s to account for small jitter.
_THREE_DAY_SECS = 3*24*60*60 + 10


# Seven day in seconds. Add 10s to account for small jitter.
_SEVEN_DAYS_SECS = 7*24*60*60 + 10


# Minimum value for timeouts.
_MIN_TIMEOUT_SECS = 1 if utils.is_local_dev_server() else 30


# The world started on 2010-01-01 at 00:00:00 UTC. The rationale is that using
# EPOCH (1970) means that 40 years worth of keys are wasted.
#
# Note: This creates a 'naive' object instead of a formal UTC object. Note that
# datetime.datetime.utcnow() also return naive objects. That's python.
_BEGINING_OF_THE_WORLD = datetime.datetime(2010, 1, 1, 0, 0, 0, 0)


# Used for isolated files.
_HASH_CHARS = frozenset('0123456789abcdef')

# Keep synced with named_cache.py
_CACHE_NAME_RE = re.compile(ur'^[a-z0-9_]{1,4096}$')


# Early verification of environment variable key name.
_ENV_KEY_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')


### Properties validators must come before the models.


def _validate_length(prop, value, maximum):
  if len(value) > maximum:
    raise datastore_errors.BadValueError(
        'too long %s: %d > %d' % (prop._name, len(value), maximum))


def _get_validate_length(maximum):
  return lambda prop, value: _validate_length(prop, value, maximum)


def _validate_isolated(prop, value):
  if not value:
    return

  if not _HASH_CHARS.issuperset(value):
    raise datastore_errors.BadValueError(
        '%s must be lowercase hex, not %s' %
        (prop._name, value))

  length = len(value)
  if length not in (40, 64, 128):
    raise datastore_errors.BadValueError(
        '%s must be lowercase hex of length 40, 64 or 128, but length is %d' %
        (prop._name, length))


def _validate_url(prop, value):
  _validate_length(prop, value, 8192)
  if value and not validation.is_valid_secure_url(value):
    raise datastore_errors.BadValueError(
        '%s must be valid HTTPS URL, not %s' % (prop._name, value))


def _validate_namespace(prop, value):
  _validate_length(prop, value, 128)
  if not config.NAMESPACE_RE.match(value):
    raise datastore_errors.BadValueError('malformed %s' % prop._name)


def _validate_dict_of_strings(prop, value):
  """Validates TaskProperties.env."""
  # pylint: disable=W0212
  if not all(
         isinstance(k, unicode) and isinstance(v, unicode)
         for k, v in value.iteritems()):
    # pylint: disable=W0212
    raise TypeError(
        '%s must be a dict of strings, not %r' % (prop._name, value))


def _validate_dimensions(prop, value):
  """Validates TaskProperties.dimensions."""
  # pylint: disable=W0212
  if not value:
    raise datastore_errors.BadValueError(u'%s must be specified' % prop._name)
  _validate_dict_of_strings(prop, value)
  for key, val in value.iteritems():
    if not config.validate_dimension_key(key):
      raise datastore_errors.BadValueError(u'dimension %r isn\'t valid' % key)
    if not config.validate_dimension_value(val):
      raise datastore_errors.BadValueError(
          u'dimension %r:%r isn\'t valid' % (key, val))
  if len(value) > 64:
    raise datastore_errors.BadValueError(
        '%s can have up to 64 keys' % prop._name)


def _validate_env_key(prop, key):
  maxlen = 1024
  if not isinstance(key, unicode):
    raise TypeError(
        '%s must have string key, not %r' % (prop._name, key))
  if not key:
    raise datastore_errors.BadValueError(
        'valid key are required in %s' % prop._name)
  if len(key) > maxlen:
    raise datastore_errors.BadValueError(
        'key in %s is too long: %d > %d' % (prop._name, len(key), maxlen))
  if not _ENV_KEY_RE.match(key):
    raise datastore_errors.BadValueError(
        'key in %s is invalid: %r' % (prop._name, key))


def _validate_env(prop, value):
  _validate_dict_of_strings(prop, value)
  maxlen = 1024
  for k, v in value.iteritems():
    _validate_env_key(prop, k)
    if len(v) > maxlen:
      raise datastore_errors.BadValueError(
          'key in %s is too long: %d > %d' % (prop._name, len(v), maxlen))
  if len(value) > 64:
    raise datastore_errors.BadValueError(
        '%s can have up to 64 keys' % prop._name)


def _validate_env_prefixes(prop, value):
  for k, v in value.iteritems():
    if not isinstance(v, list):
      raise TypeError(
          '%s must have list value, not %r' % (prop._name, v))
    _validate_env_key(prop, k)
    for path in v:
      _validate_rel_path('Env Prefix', path)

  if len(value) > 64:
    raise datastore_errors.BadValueError(
        '%s can have up to 64 keys' % prop._name)


def _validate_expiration(prop, value):
  """Validates TaskRequest.expiration_ts."""
  now = utils.utcnow()
  offset = int(round((value - now).total_seconds()))
  if not (_MIN_TIMEOUT_SECS <= offset <= _SEVEN_DAYS_SECS):
    # pylint: disable=W0212
    raise datastore_errors.BadValueError(
        '%s (%s, %ds from now) must effectively be between %ds and 7 days '
        'from now (%s)' %
        (prop._name, value, offset, _MIN_TIMEOUT_SECS, now))


def _validate_grace(prop, value):
  """Validates grace_period_secs in TaskProperties."""
  if not (0 <= value <= 60*60):
    # pylint: disable=W0212
    raise datastore_errors.BadValueError(
        '%s (%ds) must be between 0s and one hour' % (prop._name, value))


def _validate_priority(_prop, value):
  """Validates TaskRequest.priority."""
  validate_priority(value)
  return value


def _validate_task_run_id(_prop, value):
  """Validates a task_id looks valid without fetching the entity."""
  if not value:
    return None
  task_pack.unpack_run_result_key(value)
  return value


def _validate_timeout(prop, value):
  """Validates timeouts in seconds in TaskProperties."""
  if value and not (_MIN_TIMEOUT_SECS <= value <= _THREE_DAY_SECS):
    # pylint: disable=W0212
    raise datastore_errors.BadValueError(
        '%s (%ds) must be 0 or between %ds and three days' %
            (prop._name, value, _MIN_TIMEOUT_SECS))


def _validate_tags(prop, value):
  """Validates TaskRequest.tags."""
  _validate_length(prop, value, 1024)
  if ':' not in value:
    # pylint: disable=W0212
    raise datastore_errors.BadValueError(
        '%s must be key:value form, not %s' % (prop._name, value))


def _validate_pubsub_topic(prop, value):
  """Validates TaskRequest.pubsub_topic."""
  _validate_length(prop, value, 1024)
  # pylint: disable=W0212
  if value and '/' not in value:
    raise datastore_errors.BadValueError(
        '%s must be a well formatted pubsub topic' % (prop._name))


def _validate_package_name_template(prop, value):
  """Validates a CIPD package name template."""
  _validate_length(prop, value, 1024)
  if not cipd.is_valid_package_name_template(value):
    raise datastore_errors.BadValueError(
        '%s must be a valid CIPD package name template "%s"' % (
              prop._name, value))


def _validate_package_version(prop, value):
  """Validates a CIPD package version."""
  _validate_length(prop, value, 1024)
  if not cipd.is_valid_version(value):
    raise datastore_errors.BadValueError(
        '%s must be a valid package version "%s"' % (prop._name, value))


def _validate_cache_name(prop, value):
  _validate_length(prop, value, 1024)
  if not _CACHE_NAME_RE.match(value):
    raise datastore_errors.BadValueError(
        '%s %r does not match %s' % (prop._name, value, _CACHE_NAME_RE.pattern))


def _validate_cache_path(prop, value):
  _validate_length(prop, value, 1024)
  _validate_rel_path('Cache path', value)


def _validate_package_path(prop, value):
  """Validates a CIPD installation path."""
  _validate_length(prop, value, 1024)
  if not value:
    raise datastore_errors.BadValueError(
        'CIPD package path is required. Use "." to install to run dir.')
  _validate_rel_path('CIPD package path', value)


def _validate_output_path(prop, value):
  """Validates a path for an output file."""
  _validate_length(prop, value, 1024)
  _validate_rel_path('output file', value)


def _validate_rel_path(value_name, path):
  """Validates a relative path to be valid."""
  if not path:
    raise datastore_errors.BadValueError(
        'No argument provided for %s.' % value_name)
  if '\\' in path:
    raise datastore_errors.BadValueError(
        '%s cannot contain \\. On Windows forward-slashes '
        'will be replaced with back-slashes.' % value_name)
  if '..' in path.split('/'):
    raise datastore_errors.BadValueError(
        '%s cannot contain "..".' % value_name)
  normalized = posixpath.normpath(path)
  if path != normalized:
    raise datastore_errors.BadValueError(
        '%s is not normalized. Normalized is "%s".' % (value_name, normalized))
  if path.startswith('/'):
    raise datastore_errors.BadValueError(
        '%s cannot start with "/".' % value_name)


def _validate_service_account(prop, value):
  """Validates that 'service_account' field is 'bot', 'none' or email."""
  _validate_length(prop, value, 1024)
  if not value:
    return None
  if value in ('bot', 'none') or service_accounts.is_service_account(value):
    return value
  raise datastore_errors.BadValueError(
      '%r must be an email, "bot" or "none" string, got %r' %
      (prop._name, value))


### Models.


class FilesRef(ndb.Model):
  """Defines a data tree reference, normally a reference to a .isolated file."""

  # TODO(maruel): make this class have one responsibility. Currently it is used
  # in two modes:
  # - a reference to a tree, as class docstring says.
  # - input/output settings in TaskProperties.

  # The hash of an isolated archive.
  isolated = ndb.StringProperty(validator=_validate_isolated, indexed=False)
  # The hostname of the isolated server to use.
  isolatedserver = ndb.StringProperty(
      validator=_validate_url, indexed=False)
  # Namespace on the isolate server.
  namespace = ndb.StringProperty(validator=_validate_namespace, indexed=False)

  def _pre_put_hook(self):
    # TODO(maruel): Get default value from config
    # IsolateSettings.default_server.
    super(FilesRef, self)._pre_put_hook()
    if not self.isolatedserver or not self.namespace:
      raise datastore_errors.BadValueError(
          'isolate server and namespace are required')


class SecretBytes(ndb.Model):
  """Defines an optional secret byte string logically defined with the
  TaskProperties.

  Stored separately for size and data-leakage reasons.
  """
  _use_memcache = False
  secret_bytes = ndb.BlobProperty(
      validator=_get_validate_length(20*1024), indexed=False)


class CipdPackage(ndb.Model):
  """A CIPD package to install in the run dir before task execution.

  A part of TaskProperties.
  """
  # Package name template. May use cipd.ALL_PARAMS.
  # Most users will specify ${platform} parameter.
  package_name = ndb.StringProperty(
      indexed=False, validator=_validate_package_name_template)
  # Package version that is valid for all packages matched by package_name.
  # Most users will specify tags.
  version = ndb.StringProperty(
      indexed=False, validator=_validate_package_version)
  # Path to dir, relative to the run dir, where to install the package.
  # If empty, the package will be installed in the run dir.
  path = ndb.StringProperty(indexed=False, validator=_validate_package_path)

  def __str__(self):
    return '%s:%s' % (self.package_name, self.version)

  def _pre_put_hook(self):
    super(CipdPackage, self)._pre_put_hook()
    if not self.package_name:
      raise datastore_errors.BadValueError('CIPD package name is required')
    if not self.version:
      raise datastore_errors.BadValueError('CIPD package version is required')


class CipdInput(ndb.Model):
  """Specifies which CIPD client and packages to install, from which server.

  A part of TaskProperties.
  """
  # URL of the CIPD server. Must start with "https://" or "http://".
  server = ndb.StringProperty(indexed=False, validator=_validate_url)

  # CIPD package of CIPD client to use.
  # client_package.version is required.
  # client_package.path must be None.
  client_package = ndb.LocalStructuredProperty(CipdPackage)

  # List of packages to install.
  packages = ndb.LocalStructuredProperty(CipdPackage, repeated=True)

  def _pre_put_hook(self):
    if not self.server:
      raise datastore_errors.BadValueError('cipd server is required')
    if not self.client_package:
      raise datastore_errors.BadValueError('client_package is required')
    if self.client_package.path:
      raise datastore_errors.BadValueError('client_package.path must be unset')
    self.client_package._pre_put_hook()

    if not self.packages:
      raise datastore_errors.BadValueError(
          'cipd_input cannot have an empty package list')
    if len(self.packages) > 64:
      raise datastore_errors.BadValueError(
          'Up to 64 CIPD packages can be listed for a task')

    package_names = set()
    for p in self.packages:
      p._pre_put_hook()
      if not p.path:
        raise datastore_errors.BadValueError(
            'package %s:%s: path is required' % (p.package_name, p.version))
      if p.package_name in package_names:
        raise datastore_errors.BadValueError(
           'package %s is specified more than once' % p.package_name)
      package_names.add(p.package_name)
    self.packages.sort(key=lambda p: p.package_name)


class CacheEntry(ndb.Model):
  """Describes a named cache that should be present on the bot."""
  name = ndb.StringProperty(validator=_validate_cache_name)
  path = ndb.StringProperty(validator=_validate_cache_path)

  def _pre_put_hook(self):
    if not self.name:
      raise datastore_errors.BadValueError('name is not specified')


class TaskProperties(ndb.Model):<commit_msg>[swarming] Improve swarming's duplicate CIPD package check.<commit_after># coding: utf-8
# Copyright 2014 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

"""Tasks definition.

Each user request creates a new TaskRequest. The TaskRequest instance saves the
metadata of the request, e.g. who requested it, when why, etc. It links to the
actual data of the request in a TaskProperties. The TaskProperties represents
everything needed to run the task.

This means if two users request an identical task, it can be deduped
accordingly and efficiently by the scheduler.

Note that the mere existence of a TaskRequest in the db doesn't mean it will be
scheduled, see task_scheduler.py for the actual scheduling. Registering tasks
and scheduling are kept separated to keep the immutable and mutable models in
separate files.


Overview of transactions:
- TaskRequest() are created inside a transaction.


Graph of the schema:

    +--------Root-------+
    |TaskRequest        |
    |  +--------------+ |
    |  |TaskProperties| |
    |  |  +--------+  | |
    |  |  |FilesRef|  | |
    |  |  +--------+  | |
    |  +--------------+ |
    |id=<based on epoch>|
    +-------------------+
        |          |
        v          |
    +-----------+  |
    |SecretBytes|  |
    |id=1       |  |
    +-----------+  |
                   |
                   v
    <See task_to_run.py and task_result.py>

TaskProperties is embedded in TaskRequest. TaskProperties is still declared as a
separate entity to clearly declare the boundary for task request deduplication.
"""

import datetime
import hashlib
import posixpath
import random
import re

from google.appengine.api import datastore_errors
from google.appengine.ext import ndb

from components import auth
from components import datastore_utils
from components import pubsub
from components import utils
from components.config import validation

from server import config
from server import service_accounts
from server import task_pack
import cipd


# Maximum acceptable priority value, which is effectively the lowest priority.
MAXIMUM_PRIORITY = 255


# Three days in seconds. Add 10s to account for small jitter.
_THREE_DAY_SECS = 3*24*60*60 + 10


# Seven day in seconds. Add 10s to account for small jitter.
_SEVEN_DAYS_SECS = 7*24*60*60 + 10


# Minimum value for timeouts.
_MIN_TIMEOUT_SECS = 1 if utils.is_local_dev_server() else 30


# The world started on 2010-01-01 at 00:00:00 UTC. The rationale is that using
# EPOCH (1970) means that 40 years worth of keys are wasted.
#
# Note: This creates a 'naive' object instead of a formal UTC object. Note that
# datetime.datetime.utcnow() also return naive objects. That's python.
_BEGINING_OF_THE_WORLD = datetime.datetime(2010, 1, 1, 0, 0, 0, 0)


# Used for isolated files.
_HASH_CHARS = frozenset('0123456789abcdef')

# Keep synced with named_cache.py
_CACHE_NAME_RE = re.compile(ur'^[a-z0-9_]{1,4096}$')


# Early verification of environment variable key name.
_ENV_KEY_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')


### Properties validators must come before the models.


def _validate_length(prop, value, maximum):
  if len(value) > maximum:
    raise datastore_errors.BadValueError(
        'too long %s: %d > %d' % (prop._name, len(value), maximum))


def _get_validate_length(maximum):
  return lambda prop, value: _validate_length(prop, value, maximum)


def _validate_isolated(prop, value):
  if not value:
    return

  if not _HASH_CHARS.issuperset(value):
    raise datastore_errors.BadValueError(
        '%s must be lowercase hex, not %s' %
        (prop._name, value))

  length = len(value)
  if length not in (40, 64, 128):
    raise datastore_errors.BadValueError(
        '%s must be lowercase hex of length 40, 64 or 128, but length is %d' %
        (prop._name, length))


def _validate_url(prop, value):
  _validate_length(prop, value, 8192)
  if value and not validation.is_valid_secure_url(value):
    raise datastore_errors.BadValueError(
        '%s must be valid HTTPS URL, not %s' % (prop._name, value))


def _validate_namespace(prop, value):
  _validate_length(prop, value, 128)
  if not config.NAMESPACE_RE.match(value):
    raise datastore_errors.BadValueError('malformed %s' % prop._name)


def _validate_dict_of_strings(prop, value):
  """Validates TaskProperties.env."""
  # pylint: disable=W0212
  if not all(
         isinstance(k, unicode) and isinstance(v, unicode)
         for k, v in value.iteritems()):
    # pylint: disable=W0212
    raise TypeError(
        '%s must be a dict of strings, not %r' % (prop._name, value))


def _validate_dimensions(prop, value):
  """Validates TaskProperties.dimensions."""
  # pylint: disable=W0212
  if not value:
    raise datastore_errors.BadValueError(u'%s must be specified' % prop._name)
  _validate_dict_of_strings(prop, value)
  for key, val in value.iteritems():
    if not config.validate_dimension_key(key):
      raise datastore_errors.BadValueError(u'dimension %r isn\'t valid' % key)
    if not config.validate_dimension_value(val):
      raise datastore_errors.BadValueError(
          u'dimension %r:%r isn\'t valid' % (key, val))
  if len(value) > 64:
    raise datastore_errors.BadValueError(
        '%s can have up to 64 keys' % prop._name)


def _validate_env_key(prop, key):
  maxlen = 1024
  if not isinstance(key, unicode):
    raise TypeError(
        '%s must have string key, not %r' % (prop._name, key))
  if not key:
    raise datastore_errors.BadValueError(
        'valid key are required in %s' % prop._name)
  if len(key) > maxlen:
    raise datastore_errors.BadValueError(
        'key in %s is too long: %d > %d' % (prop._name, len(key), maxlen))
  if not _ENV_KEY_RE.match(key):
    raise datastore_errors.BadValueError(
        'key in %s is invalid: %r' % (prop._name, key))


def _validate_env(prop, value):
  _validate_dict_of_strings(prop, value)
  maxlen = 1024
  for k, v in value.iteritems():
    _validate_env_key(prop, k)
    if len(v) > maxlen:
      raise datastore_errors.BadValueError(
          'key in %s is too long: %d > %d' % (prop._name, len(v), maxlen))
  if len(value) > 64:
    raise datastore_errors.BadValueError(
        '%s can have up to 64 keys' % prop._name)


def _validate_env_prefixes(prop, value):
  for k, v in value.iteritems():
    if not isinstance(v, list):
      raise TypeError(
          '%s must have list value, not %r' % (prop._name, v))
    _validate_env_key(prop, k)
    for path in v:
      _validate_rel_path('Env Prefix', path)

  if len(value) > 64:
    raise datastore_errors.BadValueError(
        '%s can have up to 64 keys' % prop._name)


def _validate_expiration(prop, value):
  """Validates TaskRequest.expiration_ts."""
  now = utils.utcnow()
  offset = int(round((value - now).total_seconds()))
  if not (_MIN_TIMEOUT_SECS <= offset <= _SEVEN_DAYS_SECS):
    # pylint: disable=W0212
    raise datastore_errors.BadValueError(
        '%s (%s, %ds from now) must effectively be between %ds and 7 days '
        'from now (%s)' %
        (prop._name, value, offset, _MIN_TIMEOUT_SECS, now))


def _validate_grace(prop, value):
  """Validates grace_period_secs in TaskProperties."""
  if not (0 <= value <= 60*60):
    # pylint: disable=W0212
    raise datastore_errors.BadValueError(
        '%s (%ds) must be between 0s and one hour' % (prop._name, value))


def _validate_priority(_prop, value):
  """Validates TaskRequest.priority."""
  validate_priority(value)
  return value


def _validate_task_run_id(_prop, value):
  """Validates a task_id looks valid without fetching the entity."""
  if not value:
    return None
  task_pack.unpack_run_result_key(value)
  return value


def _validate_timeout(prop, value):
  """Validates timeouts in seconds in TaskProperties."""
  if value and not (_MIN_TIMEOUT_SECS <= value <= _THREE_DAY_SECS):
    # pylint: disable=W0212
    raise datastore_errors.BadValueError(
        '%s (%ds) must be 0 or between %ds and three days' %
            (prop._name, value, _MIN_TIMEOUT_SECS))


def _validate_tags(prop, value):
  """Validates TaskRequest.tags."""
  _validate_length(prop, value, 1024)
  if ':' not in value:
    # pylint: disable=W0212
    raise datastore_errors.BadValueError(
        '%s must be key:value form, not %s' % (prop._name, value))


def _validate_pubsub_topic(prop, value):
  """Validates TaskRequest.pubsub_topic."""
  _validate_length(prop, value, 1024)
  # pylint: disable=W0212
  if value and '/' not in value:
    raise datastore_errors.BadValueError(
        '%s must be a well formatted pubsub topic' % (prop._name))


def _validate_package_name_template(prop, value):
  """Validates a CIPD package name template."""
  _validate_length(prop, value, 1024)
  if not cipd.is_valid_package_name_template(value):
    raise datastore_errors.BadValueError(
        '%s must be a valid CIPD package name template "%s"' % (
              prop._name, value))


def _validate_package_version(prop, value):
  """Validates a CIPD package version."""
  _validate_length(prop, value, 1024)
  if not cipd.is_valid_version(value):
    raise datastore_errors.BadValueError(
        '%s must be a valid package version "%s"' % (prop._name, value))


def _validate_cache_name(prop, value):
  _validate_length(prop, value, 1024)
  if not _CACHE_NAME_RE.match(value):
    raise datastore_errors.BadValueError(
        '%s %r does not match %s' % (prop._name, value, _CACHE_NAME_RE.pattern))


def _validate_cache_path(prop, value):
  _validate_length(prop, value, 1024)
  _validate_rel_path('Cache path', value)


def _validate_package_path(prop, value):
  """Validates a CIPD installation path."""
  _validate_length(prop, value, 1024)
  if not value:
    raise datastore_errors.BadValueError(
        'CIPD package path is required. Use "." to install to run dir.')
  _validate_rel_path('CIPD package path', value)


def _validate_output_path(prop, value):
  """Validates a path for an output file."""
  _validate_length(prop, value, 1024)
  _validate_rel_path('output file', value)


def _validate_rel_path(value_name, path):
  """Validates a relative path to be valid."""
  if not path:
    raise datastore_errors.BadValueError(
        'No argument provided for %s.' % value_name)
  if '\\' in path:
    raise datastore_errors.BadValueError(
        '%s cannot contain \\. On Windows forward-slashes '
        'will be replaced with back-slashes.' % value_name)
  if '..' in path.split('/'):
    raise datastore_errors.BadValueError(
        '%s cannot contain "..".' % value_name)
  normalized = posixpath.normpath(path)
  if path != normalized:
    raise datastore_errors.BadValueError(
        '%s is not normalized. Normalized is "%s".' % (value_name, normalized))
  if path.startswith('/'):
    raise datastore_errors.BadValueError(
        '%s cannot start with "/".' % value_name)


def _validate_service_account(prop, value):
  """Validates that 'service_account' field is 'bot', 'none' or email."""
  _validate_length(prop, value, 1024)
  if not value:
    return None
  if value in ('bot', 'none') or service_accounts.is_service_account(value):
    return value
  raise datastore_errors.BadValueError(
      '%r must be an email, "bot" or "none" string, got %r' %
      (prop._name, value))


### Models.


class FilesRef(ndb.Model):
  """Defines a data tree reference, normally a reference to a .isolated file."""

  # TODO(maruel): make this class have one responsibility. Currently it is used
  # in two modes:
  # - a reference to a tree, as class docstring says.
  # - input/output settings in TaskProperties.

  # The hash of an isolated archive.
  isolated = ndb.StringProperty(validator=_validate_isolated, indexed=False)
  # The hostname of the isolated server to use.
  isolatedserver = ndb.StringProperty(
      validator=_validate_url, indexed=False)
  # Namespace on the isolate server.
  namespace = ndb.StringProperty(validator=_validate_namespace, indexed=False)

  def _pre_put_hook(self):
    # TODO(maruel): Get default value from config
    # IsolateSettings.default_server.
    super(FilesRef, self)._pre_put_hook()
    if not self.isolatedserver or not self.namespace:
      raise datastore_errors.BadValueError(
          'isolate server and namespace are required')


class SecretBytes(ndb.Model):
  """Defines an optional secret byte string logically defined with the
  TaskProperties.

  Stored separately for size and data-leakage reasons.
  """
  _use_memcache = False
  secret_bytes = ndb.BlobProperty(
      validator=_get_validate_length(20*1024), indexed=False)


class CipdPackage(ndb.Model):
  """A CIPD package to install in the run dir before task execution.

  A part of TaskProperties.
  """
  # Package name template. May use cipd.ALL_PARAMS.
  # Most users will specify ${platform} parameter.
  package_name = ndb.StringProperty(
      indexed=False, validator=_validate_package_name_template)
  # Package version that is valid for all packages matched by package_name.
  # Most users will specify tags.
  version = ndb.StringProperty(
      indexed=False, validator=_validate_package_version)
  # Path to dir, relative to the run dir, where to install the package.
  # If empty, the package will be installed in the run dir.
  path = ndb.StringProperty(indexed=False, validator=_validate_package_path)

  def __str__(self):
    return '%s:%s' % (self.package_name, self.version)

  def _pre_put_hook(self):
    super(CipdPackage, self)._pre_put_hook()
    if not self.package_name:
      raise datastore_errors.BadValueError('CIPD package name is required')
    if not self.version:
      raise datastore_errors.BadValueError('CIPD package version is required')


class CipdInput(ndb.Model):
  """Specifies which CIPD client and packages to install, from which server.

  A part of TaskProperties.
  """
  # URL of the CIPD server. Must start with "https://" or "http://".
  server = ndb.StringProperty(indexed=False, validator=_validate_url)

  # CIPD package of CIPD client to use.
  # client_package.version is required.
  # client_package.path must be None.
  client_package = ndb.LocalStructuredProperty(CipdPackage)

  # List of packages to install.
  packages = ndb.LocalStructuredProperty(CipdPackage, repeated=True)

  def _pre_put_hook(self):
    if not self.server:
      raise datastore_errors.BadValueError('cipd server is required')
    if not self.client_package:
      raise datastore_errors.BadValueError('client_package is required')
    if self.client_package.path:
      raise datastore_errors.BadValueError('client_package.path must be unset')
    self.client_package._pre_put_hook()

    if not self.packages:
      raise datastore_errors.BadValueError(
          'cipd_input cannot have an empty package list')
    if len(self.packages) > 64:
      raise datastore_errors.BadValueError(
          'Up to 64 CIPD packages can be listed for a task')

    # Make sure we don't install multiple versions of the same package at the
    # same path.
    package_path_names = set()
    for p in self.packages:
      p._pre_put_hook()
      if not p.path:
        raise datastore_errors.BadValueError(
            'package %s:%s: path is required' % (p.package_name, p.version))
      path_name = (p.path, p.package_name)
      if path_name in package_path_names:
        raise datastore_errors.BadValueError(
           'package %r is specified more than once in path %r'
          % (p.package_name, p.path))
      package_path_names.add(path_name)
    self.packages.sort(key=lambda p: (p.path, p.package_name))


class CacheEntry(ndb.Model):
  """Describes a named cache that should be present on the bot."""
  name = ndb.StringProperty(validator=_validate_cache_name)
  path = ndb.StringProperty(validator=_validate_cache_path)

  def _pre_put_hook(self):
    if not self.name:
      raise datastore_errors.BadValueError('name is not specified')


class TaskProperties(ndb.Model):
{'fraction_non_alphanumeric': 0.10482041587901701, 'fraction_numerical': 0.013358538122243226, 'mean_word_length': 3.5848620540228224, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6707230', 'n_tokens_mistral': 10455, 'n_tokens_neox': 9980, 'n_words': 3144}