<commit_before># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

try:
  import cPickle as pickle
except ImportError:
  import pickle

from twitter.pants.base.build_invalidator import BuildInvalidator, CacheKeyGenerator, NO_SOURCES, TARGET_SOURCES
from twitter.pants.base.target import Target
from twitter.pants.targets import JarDependency
from twitter.pants.targets.internal import InternalTarget


# A VersionedTargetSet represents a list of targets, a corresponding CacheKey, and a flag determining
# whether the list of targets is currently valid.
#
#  - When invalidating a single target, this can be used to represent that target as a singleton.
#
#  - When checking the artifact cache, this can also be used to represent a list of targets
#    that are built together into a single artifact.
class VersionedTargetSet(object):
  def __init__(self, cache_manager, targets, per_target_cache_keys):
    self._cache_manager = cache_manager
    self.per_target_cache_keys = per_target_cache_keys

    self.targets = targets
    self.cache_key = CacheKeyGenerator.combine_cache_keys(per_target_cache_keys)
    self.valid = not cache_manager.needs_update(self.cache_key)

  def update(self):
    self._cache_manager.update(self)
    self.valid = True

# The result of calling check() on a CacheManager.
# Each member is a list of VersionedTargetSet objects in topological order.
# Tasks may need to perform no, some or all operations on either of these, depending on how they
# are implemented.
class InvalidationCheck(object):
  def __init__(self, all_vts, all_vts_partitioned, invalid_vts, invalid_vts_partitioned):
    # All the targets, valid and invalid.
    self.all_vts = all_vts

    # All the targets, partitioned if so requested.
    self.all_vts_partitioned = all_vts_partitioned

    # Just the invalid targets.
    self.invalid_vts = invalid_vts

    # Just the invalid targets, partitioned if so requested.
    self.invalid_vts_partitioned = invalid_vts_partitioned

class CacheManager(object):
  """Manages cache checks, updates and invalidation keeping track of basic change
  and invalidation statistics.
  """
  def __init__(self, cache_key_generator, build_invalidator_dir,
               invalidate_dependents, extra_data, only_externaldeps):
    self._cache_key_generator = cache_key_generator
    self._invalidate_dependents = invalidate_dependents
    self._extra_data = pickle.dumps(extra_data)  # extra_data may be None.
    self._sources = NO_SOURCES if only_externaldeps else TARGET_SOURCES

    self._invalidator = BuildInvalidator(build_invalidator_dir)

  def update(self, vts):
    """Mark a changed or invalidated VersionedTargetSet as successfully processed."""
    for cache_key in vts.per_target_cache_keys:
      self._invalidator.update(cache_key)
    self._invalidator.update(vts.cache_key)

  def check(self, targets, partition_size_hint):
    """Checks whether each of the targets has changed and invalidates it if so.

    Returns a list of VersionedTargetSet objects (either valid or invalid). The returned sets 'cover'
    the input targets, possibly partitioning them, and are in topological order.
    The caller can inspect these in order and, e.g., rebuild the invalid ones.
    """
    all_vts = self._validate(targets)
    invalid_vts = filter(lambda vt: not vt.valid, all_vts)
    all_vts_partitioned = self._partition_versioned_targets(all_vts, partition_size_hint)
    invalid_vts_partitioned = self._partition_versioned_targets(invalid_vts, partition_size_hint)
    return InvalidationCheck(all_vts, all_vts_partitioned, invalid_vts, invalid_vts_partitioned)

  def _validate(self, targets):
    """Validate each target.

    Returns a topologically ordered set of VersionedTargets, each representing one input target.
    """<commit_msg>more descriptive name for CacheManager._validate<commit_after># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

try:
  import cPickle as pickle
except ImportError:
  import pickle

from twitter.pants.base.build_invalidator import BuildInvalidator, CacheKeyGenerator, NO_SOURCES, TARGET_SOURCES
from twitter.pants.base.target import Target
from twitter.pants.targets import JarDependency
from twitter.pants.targets.internal import InternalTarget


# A VersionedTargetSet represents a list of targets, a corresponding CacheKey, and a flag determining
# whether the list of targets is currently valid.
#
#  - When invalidating a single target, this can be used to represent that target as a singleton.
#
#  - When checking the artifact cache, this can also be used to represent a list of targets
#    that are built together into a single artifact.
class VersionedTargetSet(object):
  def __init__(self, cache_manager, targets, per_target_cache_keys):
    self._cache_manager = cache_manager
    self.per_target_cache_keys = per_target_cache_keys

    self.targets = targets
    self.cache_key = CacheKeyGenerator.combine_cache_keys(per_target_cache_keys)
    self.valid = not cache_manager.needs_update(self.cache_key)

  def update(self):
    self._cache_manager.update(self)
    self.valid = True

# The result of calling check() on a CacheManager.
# Each member is a list of VersionedTargetSet objects in topological order.
# Tasks may need to perform no, some or all operations on either of these, depending on how they
# are implemented.
class InvalidationCheck(object):
  def __init__(self, all_vts, all_vts_partitioned, invalid_vts, invalid_vts_partitioned):
    # All the targets, valid and invalid.
    self.all_vts = all_vts

    # All the targets, partitioned if so requested.
    self.all_vts_partitioned = all_vts_partitioned

    # Just the invalid targets.
    self.invalid_vts = invalid_vts

    # Just the invalid targets, partitioned if so requested.
    self.invalid_vts_partitioned = invalid_vts_partitioned

class CacheManager(object):
  """Manages cache checks, updates and invalidation keeping track of basic change
  and invalidation statistics.
  """
  def __init__(self, cache_key_generator, build_invalidator_dir,
               invalidate_dependents, extra_data, only_externaldeps):
    self._cache_key_generator = cache_key_generator
    self._invalidate_dependents = invalidate_dependents
    self._extra_data = pickle.dumps(extra_data)  # extra_data may be None.
    self._sources = NO_SOURCES if only_externaldeps else TARGET_SOURCES

    self._invalidator = BuildInvalidator(build_invalidator_dir)

  def update(self, vts):
    """Mark a changed or invalidated VersionedTargetSet as successfully processed."""
    for cache_key in vts.per_target_cache_keys:
      self._invalidator.update(cache_key)
    self._invalidator.update(vts.cache_key)

  def check(self, targets, partition_size_hint):
    """Checks whether each of the targets has changed and invalidates it if so.

    Returns a list of VersionedTargetSet objects (either valid or invalid). The returned sets 'cover'
    the input targets, possibly partitioning them, and are in topological order.
    The caller can inspect these in order and, e.g., rebuild the invalid ones.
    """
    all_vts = self._sort_and_validate_targets(targets)
    invalid_vts = filter(lambda vt: not vt.valid, all_vts)
    all_vts_partitioned = self._partition_versioned_targets(all_vts, partition_size_hint)
    invalid_vts_partitioned = self._partition_versioned_targets(invalid_vts, partition_size_hint)
    return InvalidationCheck(all_vts, all_vts_partitioned, invalid_vts, invalid_vts_partitioned)

  def _sort_and_validate_targets(self, targets):
    """Validate each target.

    Returns a topologically ordered set of VersionedTargets, each representing one input target.
    """
{'fraction_non_alphanumeric': 0.12159127366057106, 'fraction_numerical': 0.0017110469468506041, 'mean_word_length': 4.3136363636363635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1362644', 'n_tokens_mistral': 2562, 'n_tokens_neox': 2445, 'n_words': 868}