<commit_before># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Some common utilities for tools to use."""

import contextlib
import datetime
import glob
import os
import os.path as path
import re
import shutil
import subprocess
import time
import zipfile

from nototools import notoconfig

@contextlib.contextmanager
def temp_chdir(path):
  """Usage: with temp_chdir(path):
    do_something
  """
  saved_dir = os.getcwd()
  try:
    os.chdir(path)
    yield
  finally:
    os.chdir(saved_dir)


noto_re = re.compile(r'\[(tools|fonts|emoji|cjk)\]/(.*)')
def resolve_path(path):
  """Resolve a path that might use noto path shorthand."""

  if not path or path == '-':
    return None
  m = noto_re.match(path)
  if m:
    base, rest = m.groups()
    key = 'noto_' + base
    if not key in notoconfig.values:
      raise ValueError('notoconfig has no entry for %s' % key)
    base = notoconfig.values.get(key)
    path = os.path.join(base, rest)
  path = os.path.expanduser(path)
  path = os.path.abspath(path)
  path = os.path.realpath(path)
  return path


def check_dir_exists(dirpath):
  if not os.path.isdir(dirpath):
    raise ValueError('%s does not exist or is not a directory' % dirpath)


def check_file_exists(filepath):
  if not os.path.isfile(filepath):
    raise ValueError('%s does not exist or is not a file' % filepath)


def ensure_dir_exists(path):
  path = os.path.realpath(path)
  if not os.path.isdir(path):
    if os.path.exists(path):
      raise ValueError('%s exists and is not a directory' % path)
    print "making '%s'" % path
    os.makedirs(path)
  return path


def generate_zip_with_7za(root_dir, file_paths, archive_path):
  """file_paths is a list of files relative to root_dir, these will be the names
  in the archive at archive_path."""

  arg_list = ['7za', 'a', archive_path, '-tzip', '-mx=7', '-bd', '--']
  arg_list.extend(file_paths)
  with temp_chdir(root_dir):
    # capture but discard output
    subprocess.check_output(arg_list)


def generate_zip_with_7za_from_filepairs(pairs, archive_path):
  """Pairs are source/destination path pairs. The source will be put into the
  zip with name destination.  The destination must be a suffix of the source."""

  staging_dir = '/tmp/stage_7za'
  if os.path.exists(staging_dir):
    shutil.rmtree(staging_dir)<commit_msg>doc fix.<commit_after># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Some common utilities for tools to use."""

import contextlib
import datetime
import glob
import os
import os.path as path
import re
import shutil
import subprocess
import time
import zipfile

from nototools import notoconfig

@contextlib.contextmanager
def temp_chdir(path):
  """Usage: with temp_chdir(path):
    do_something
  """
  saved_dir = os.getcwd()
  try:
    os.chdir(path)
    yield
  finally:
    os.chdir(saved_dir)


noto_re = re.compile(r'\[(tools|fonts|emoji|cjk)\]/(.*)')
def resolve_path(path):
  """Resolve a path that might use noto path shorthand."""

  if not path or path == '-':
    return None
  m = noto_re.match(path)
  if m:
    base, rest = m.groups()
    key = 'noto_' + base
    if not key in notoconfig.values:
      raise ValueError('notoconfig has no entry for %s' % key)
    base = notoconfig.values.get(key)
    path = os.path.join(base, rest)
  path = os.path.expanduser(path)
  path = os.path.abspath(path)
  path = os.path.realpath(path)
  return path


def check_dir_exists(dirpath):
  if not os.path.isdir(dirpath):
    raise ValueError('%s does not exist or is not a directory' % dirpath)


def check_file_exists(filepath):
  if not os.path.isfile(filepath):
    raise ValueError('%s does not exist or is not a file' % filepath)


def ensure_dir_exists(path):
  path = os.path.realpath(path)
  if not os.path.isdir(path):
    if os.path.exists(path):
      raise ValueError('%s exists and is not a directory' % path)
    print "making '%s'" % path
    os.makedirs(path)
  return path


def generate_zip_with_7za(root_dir, file_paths, archive_path):
  """file_paths is a list of files relative to root_dir, these will be the names
  in the archive at archive_path."""

  arg_list = ['7za', 'a', archive_path, '-tzip', '-mx=7', '-bd', '--']
  arg_list.extend(file_paths)
  with temp_chdir(root_dir):
    # capture but discard output
    subprocess.check_output(arg_list)


def generate_zip_with_7za_from_filepairs(pairs, archive_path):
  """Pairs are source/destination path pairs. The source will be put into the
  zip with name destination."""

  staging_dir = '/tmp/stage_7za'
  if os.path.exists(staging_dir):
    shutil.rmtree(staging_dir)
{'fraction_non_alphanumeric': 0.1054673721340388, 'fraction_numerical': 0.004585537918871252, 'mean_word_length': 3.9057093425605536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4085155', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1829, 'n_words': 625}