<commit_before># Copyright 2013 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

"""Various utility functions and classes not specific to any single area."""

import atexit
import functools
import json
import logging
import os
import re
import sys
import threading
import time

import utils
from . import zip_package

if sys.version_info.major == 2:
  import cStringIO
else:
  import io as cStringIO

# Path to (possibly extracted from zip) cacert.pem bundle file.
# See get_cacerts_bundle().
_ca_certs = None
_ca_certs_lock = threading.Lock()


# @cached decorators registered by report_cache_stats_at_exit.
_caches = []
_caches_lock = threading.Lock()


class Profiler(object):
  """Context manager that records time spend inside its body."""
  def __init__(self, name):
    self.name = name
    self.start_time = None

  def __enter__(self):
    self.start_time = time.time()
    return self

  def __exit__(self, _exc_type, _exec_value, _traceback):
    time_taken = time.time() - self.start_time
    logging.info('Profiling: Section %s took %3.3f seconds',
                 self.name, time_taken)


class ProfileCounter(object):
  """Records total time spent in a chunk of code during lifetime of a process.

  Recursive calls count as a single call (i.e. only the time spent in the outer
  call is recorded).

  Autoregisters itself in a global list when instantiated. All counters will be
  reported at the process exit time (in atexit hook). Best to be used as with
  @profile decorator.
  """

  _instances_lock = threading.Lock()
  _instances = []

  @staticmethod
  def summarize_all():
    print('\nProfiling report:')
    print('-' * 80)
    print(
        '{:<38}{:<10}{:<16}{:<16}'.format(
            'Name', 'Count', 'Total ms', 'Average ms'))
    print('-' * 80)
    with ProfileCounter._instances_lock:
      for i in sorted(ProfileCounter._instances, key=lambda x: -x.total_time):
        print(
            '{:<38}{:<10}{:<16.1f}{:<16.1f}'.format(
                i.name,
                i.call_count,
                i.total_time * 1000,
                i.average_time * 1000))
    print('-' * 80)

  def __init__(self, name):
    self._lock = threading.Lock()
    self._call_count = 0
    self._name = name
    self._total_time = 0
    self._active = threading.local()
    with self._instances_lock:
      self._instances.append(self)
      if len(self._instances) == 1:
        atexit.register(ProfileCounter.summarize_all)

  @property
  def name(self):
    return self._name

  @property
  def call_count(self):
    return self._call_count

  @property
  def total_time(self):
    return self._total_time

  @property
  def average_time(self):
    with self._lock:
      if self._call_count:
        return self._total_time / self._call_count
      return 0

  def __enter__(self):
    recursion = getattr(self._active, 'recursion', 0)
    if not recursion:
      self._active.started = time.time()
    self._active.recursion = recursion + 1

  def __exit__(self, _exc_type, _exec_value, _traceback):
    self._active.recursion -= 1
    if not self._active.recursion:
      time_inside = time.time() - self._active.started
      with self._lock:
        self._total_time += time_inside
        self._call_count += 1


def profile(func):
  """Decorator that profiles a function if SWARMING_PROFILE env var is set.

  Will gather a number of calls to that function and total time spent inside.
  The final report is emitted to stdout at the process exit time.
  """
  # No performance impact whatsoever if SWARMING_PROFILE is not set.
  if os.environ.get('SWARMING_PROFILE') != '1':
    return func
  timer = ProfileCounter(func.__name__)
  @functools.wraps(func)
  def wrapper(*args, **kwargs):
    with timer:
      return func(*args, **kwargs)
  return wrapper


def report_cache_stats_at_exit(func, cache):
  """Registers a hook that reports state of the cache on the process exit."""
  # Very dumb. Tries to account for object reuse though.
  def get_size(obj, seen):
    # Use id(...) to avoid triggering __hash__ and comparing by value instead.
    if id(obj) in seen:
      return 0
    seen.add(id(obj))
    size = sys.getsizeof(obj)
    if isinstance(obj, (list, tuple)):
      return size + sum(get_size(x, seen) for x in obj)
    elif isinstance(obj, dict):
      return size + sum(
          get_size(k, seen) + get_size(v, seen) for k, v in obj.items())
    return size

  def report_caches_state():
    print('\nFunction cache report:')
    print('-' * 80)
    print('{:<40}{:<16}{:<26}'.format('Name', 'Items', 'Approx size, KB'))
    print('-' * 80)
    with _caches_lock:
      total = 0
      seen_objects = set()
      for func, cache in sorted(_caches, key=lambda x: -len(x[1])):
        size = get_size(cache, seen_objects)
        total += size
        print(
            '{:<40}{:<16}{:<26}'.format(func.__name__, len(cache), size / 1024))
    print('-' * 80)
    print('Total: %.1f MB' % (total / 1024 / 1024,))
    print('-' * 80)

  with _caches_lock:
    _caches.append((func, cache))
    if len(_caches) == 1:
      atexit.register(report_caches_state)


_CACHED_FUNCS = set()

def clear_cache_all():
  map(clear_cache, _CACHED_FUNCS)


def cached(func):
  """Decorator that permanently caches a result of function invocation.

  It tries to be super fast and because of that is somewhat limited:
    * The function being cached can accept only positional arguments.
    * All arguments should be hashable.
    * The function may be called multiple times with same arguments in
      multithreaded environment.
    * The cache is not cleared up at all.

  If SWARMING_PROFILE env var is set, will produce a report about the state of
  the cache at the process exit (number of items and approximate size).
  """
  empty = object()
  cache = {}

  if os.environ.get('SWARMING_PROFILE') == '1':
    report_cache_stats_at_exit(func, cache)<commit_msg>[swarming] fix tools.clear_cache_all<commit_after># Copyright 2013 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

"""Various utility functions and classes not specific to any single area."""

import atexit
import functools
import json
import logging
import os
import re
import sys
import threading
import time

import utils
from . import zip_package

if sys.version_info.major == 2:
  import cStringIO
else:
  import io as cStringIO

# Path to (possibly extracted from zip) cacert.pem bundle file.
# See get_cacerts_bundle().
_ca_certs = None
_ca_certs_lock = threading.Lock()


# @cached decorators registered by report_cache_stats_at_exit.
_caches = []
_caches_lock = threading.Lock()


class Profiler(object):
  """Context manager that records time spend inside its body."""
  def __init__(self, name):
    self.name = name
    self.start_time = None

  def __enter__(self):
    self.start_time = time.time()
    return self

  def __exit__(self, _exc_type, _exec_value, _traceback):
    time_taken = time.time() - self.start_time
    logging.info('Profiling: Section %s took %3.3f seconds',
                 self.name, time_taken)


class ProfileCounter(object):
  """Records total time spent in a chunk of code during lifetime of a process.

  Recursive calls count as a single call (i.e. only the time spent in the outer
  call is recorded).

  Autoregisters itself in a global list when instantiated. All counters will be
  reported at the process exit time (in atexit hook). Best to be used as with
  @profile decorator.
  """

  _instances_lock = threading.Lock()
  _instances = []

  @staticmethod
  def summarize_all():
    print('\nProfiling report:')
    print('-' * 80)
    print(
        '{:<38}{:<10}{:<16}{:<16}'.format(
            'Name', 'Count', 'Total ms', 'Average ms'))
    print('-' * 80)
    with ProfileCounter._instances_lock:
      for i in sorted(ProfileCounter._instances, key=lambda x: -x.total_time):
        print(
            '{:<38}{:<10}{:<16.1f}{:<16.1f}'.format(
                i.name,
                i.call_count,
                i.total_time * 1000,
                i.average_time * 1000))
    print('-' * 80)

  def __init__(self, name):
    self._lock = threading.Lock()
    self._call_count = 0
    self._name = name
    self._total_time = 0
    self._active = threading.local()
    with self._instances_lock:
      self._instances.append(self)
      if len(self._instances) == 1:
        atexit.register(ProfileCounter.summarize_all)

  @property
  def name(self):
    return self._name

  @property
  def call_count(self):
    return self._call_count

  @property
  def total_time(self):
    return self._total_time

  @property
  def average_time(self):
    with self._lock:
      if self._call_count:
        return self._total_time / self._call_count
      return 0

  def __enter__(self):
    recursion = getattr(self._active, 'recursion', 0)
    if not recursion:
      self._active.started = time.time()
    self._active.recursion = recursion + 1

  def __exit__(self, _exc_type, _exec_value, _traceback):
    self._active.recursion -= 1
    if not self._active.recursion:
      time_inside = time.time() - self._active.started
      with self._lock:
        self._total_time += time_inside
        self._call_count += 1


def profile(func):
  """Decorator that profiles a function if SWARMING_PROFILE env var is set.

  Will gather a number of calls to that function and total time spent inside.
  The final report is emitted to stdout at the process exit time.
  """
  # No performance impact whatsoever if SWARMING_PROFILE is not set.
  if os.environ.get('SWARMING_PROFILE') != '1':
    return func
  timer = ProfileCounter(func.__name__)
  @functools.wraps(func)
  def wrapper(*args, **kwargs):
    with timer:
      return func(*args, **kwargs)
  return wrapper


def report_cache_stats_at_exit(func, cache):
  """Registers a hook that reports state of the cache on the process exit."""
  # Very dumb. Tries to account for object reuse though.
  def get_size(obj, seen):
    # Use id(...) to avoid triggering __hash__ and comparing by value instead.
    if id(obj) in seen:
      return 0
    seen.add(id(obj))
    size = sys.getsizeof(obj)
    if isinstance(obj, (list, tuple)):
      return size + sum(get_size(x, seen) for x in obj)
    elif isinstance(obj, dict):
      return size + sum(
          get_size(k, seen) + get_size(v, seen) for k, v in obj.items())
    return size

  def report_caches_state():
    print('\nFunction cache report:')
    print('-' * 80)
    print('{:<40}{:<16}{:<26}'.format('Name', 'Items', 'Approx size, KB'))
    print('-' * 80)
    with _caches_lock:
      total = 0
      seen_objects = set()
      for func, cache in sorted(_caches, key=lambda x: -len(x[1])):
        size = get_size(cache, seen_objects)
        total += size
        print(
            '{:<40}{:<16}{:<26}'.format(func.__name__, len(cache), size / 1024))
    print('-' * 80)
    print('Total: %.1f MB' % (total / 1024 / 1024,))
    print('-' * 80)

  with _caches_lock:
    _caches.append((func, cache))
    if len(_caches) == 1:
      atexit.register(report_caches_state)


_CACHED_FUNCS = set()

def clear_cache_all():
  for f in _CACHED_FUNCS:
    clear_cache(f)


def cached(func):
  """Decorator that permanently caches a result of function invocation.

  It tries to be super fast and because of that is somewhat limited:
    * The function being cached can accept only positional arguments.
    * All arguments should be hashable.
    * The function may be called multiple times with same arguments in
      multithreaded environment.
    * The cache is not cleared up at all.

  If SWARMING_PROFILE env var is set, will produce a report about the state of
  the cache at the process exit (number of items and approximate size).
  """
  empty = object()
  cache = {}

  if os.environ.get('SWARMING_PROFILE') == '1':
    report_cache_stats_at_exit(func, cache)
{'fraction_non_alphanumeric': 0.10813278008298755, 'fraction_numerical': 0.014605809128630706, 'mean_word_length': 3.0892432982694267, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3354389', 'n_tokens_mistral': 4018, 'n_tokens_neox': 3756, 'n_words': 1238}