<commit_before># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Disk backed cache for preserving values across runs of
run.py."""

import hashlib
import os

from base import json_utils
from metrics import logs
from system import environment

# For any given value file, if a file with the same name with this added
# extension exists, it is not cleared during initialization.
PERSIST_FILE_EXTENSION = '.persist'


def initialize():
  """Initialize the persistent cache, creating the directory used to store the
  values."""
  cache_directory_path = environment.get_value('CACHE_DIR')
  if os.path.exists(cache_directory_path):
    clear_values()
  else:
    os.makedirs(cache_directory_path)


def clear_values():
  """Remove all values."""
  cache_directory_path = environment.get_value('CACHE_DIR')
  if not os.path.exists(cache_directory_path):
    return

  for root_directory, _, filenames in os.walk(cache_directory_path):
    for filename in filenames:
      if filename.endswith(PERSIST_FILE_EXTENSION):
        continue

      file_path = os.path.join(root_directory, filename)
      persist_file_path = file_path + PERSIST_FILE_EXTENSION
      if os.path.exists(persist_file_path):
        continue

      os.remove(file_path)


def delete_value(key):
  """Removes the value for a key."""
  value_path = get_value_file_path(key)
  os.remove(value_path)


def get_value(key, default_value=None, constructor=None):
  """Get the value for a key."""
  value_path = get_value_file_path(key)

  if not os.path.exists(value_path):
    return default_value

  try:
    with open(value_path, 'rb') as f:
      value_str = f.read()
  except IOError:
    logs.log_error('Failed to read %s from persistent cache.' % key)
    value_str = None

  if value_str is None:
    return None

  try:<commit_msg>Fix potential exception when deleting a persistent cache value. (#897)<commit_after># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Disk backed cache for preserving values across runs of
run.py."""

import hashlib
import os

from base import json_utils
from metrics import logs
from system import environment

# For any given value file, if a file with the same name with this added
# extension exists, it is not cleared during initialization.
PERSIST_FILE_EXTENSION = '.persist'


def initialize():
  """Initialize the persistent cache, creating the directory used to store the
  values."""
  cache_directory_path = environment.get_value('CACHE_DIR')
  if os.path.exists(cache_directory_path):
    clear_values()
  else:
    os.makedirs(cache_directory_path)


def clear_values():
  """Remove all values."""
  cache_directory_path = environment.get_value('CACHE_DIR')
  if not os.path.exists(cache_directory_path):
    return

  for root_directory, _, filenames in os.walk(cache_directory_path):
    for filename in filenames:
      if filename.endswith(PERSIST_FILE_EXTENSION):
        continue

      file_path = os.path.join(root_directory, filename)
      persist_file_path = file_path + PERSIST_FILE_EXTENSION
      if os.path.exists(persist_file_path):
        continue

      os.remove(file_path)


def delete_value(key):
  """Removes the value for a key."""
  value_path = get_value_file_path(key)
  if os.path.exists(value_path):
    os.remove(value_path)


def get_value(key, default_value=None, constructor=None):
  """Get the value for a key."""
  value_path = get_value_file_path(key)

  if not os.path.exists(value_path):
    return default_value

  try:
    with open(value_path, 'rb') as f:
      value_str = f.read()
  except IOError:
    logs.log_error('Failed to read %s from persistent cache.' % key)
    value_str = None

  if value_str is None:
    return None

  try:
{'fraction_non_alphanumeric': 0.08357894736842106, 'fraction_numerical': 0.004, 'mean_word_length': 4.092175777063237, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1063352', 'n_tokens_mistral': 1416, 'n_tokens_neox': 1376, 'n_words': 496}