<commit_before>import os
import re
import shutil
import subprocess
import sys
import tempfile

def run_silent(p):
  proc = subprocess.Popen(p, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  out, err = proc.communicate()
  out, err = out.decode('utf8'), err.decode('utf8')
  return (proc.returncode == 0, out, err)

def run_verbose(p):
  return subprocess.call(p) == 0

class FileInstaller(object):
  @staticmethod
  def confirm_executable(program):
    if not FileInstaller.has_executable(program):
      raise RuntimeError('%s not installed, cannot proceed' % program)

  @staticmethod
  def has_executable(program):
    return shutil.which(program) is not None

  def check_exists(path):
    if os.path.exists(path):
      raise RuntimeError('Target file %s exists and is not a symlink.' % path)

  @staticmethod
  def create_dir(path):
    path = os.path.abspath(os.path.expanduser(path))
    FileInstaller.remove_symlink(path)
    if not os.path.exists(path):
      print('Creating directory %s...' % path)
      os.makedirs(path)

  @staticmethod
  def create_file(path):
    path = os.path.abspath(os.path.expanduser(path))
    dir = os.path.dirname(path)
    if not os.path.islink(dir):
      FileInstaller.create_dir(dir)
    if not os.path.exists(path):
      print('Creating file %s...' % path)
      with open(path, 'wb'):
        pass

  @staticmethod
  def copy_file(source, target):
    source = os.path.abspath(os.path.expanduser(source))
    target = os.path.expanduser(target)
    if target.endswith('/') or target.endswith('\\'):
      target = os.path.join(target, os.path.basename(source))
    FileInstaller.remove_symlink(target)
    print('Copying %s to %s...' % (source, target))
    FileInstaller.create_dir(os.path.dirname(target))
    shutil.copy(source, target)

  @staticmethod
  def merge_files(sources, target, comment_prefix = None):
    for i, member in enumerate(sources):
      sources[i] = os.path.abspath(os.path.expanduser(sources[i]))
    target = os.path.expanduser(target)
    if target.endswith('/') or target.endswith('\\'):
      target = os.path.join(target, os.path.basename(source))
    FileInstaller.remove_symlink(target)
    FileInstaller.create_dir(os.path.dirname(target))
    print('Merging %s to %s...' % (sources, target))
    with open(target, 'w') as ft:
      if comment_prefix:
        ft.write('%s File was merged automatically by libinstall. Do not edit this file.%s'
          % (comment_prefix, os.linesep))
      for source in sources:
        with open(source) as fs:
            if comment_prefix:
              ft.write('%s Merged from: %s%s' % (comment_prefix, source, os.linesep))
            ft.write(fs.read())
            ft.write(os.linesep)

  @staticmethod<commit_msg>add count of currently merged file<commit_after>import os
import re
import shutil
import subprocess
import sys
import tempfile

def run_silent(p):
  proc = subprocess.Popen(p, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  out, err = proc.communicate()
  out, err = out.decode('utf8'), err.decode('utf8')
  return (proc.returncode == 0, out, err)

def run_verbose(p):
  return subprocess.call(p) == 0

class FileInstaller(object):
  @staticmethod
  def confirm_executable(program):
    if not FileInstaller.has_executable(program):
      raise RuntimeError('%s not installed, cannot proceed' % program)

  @staticmethod
  def has_executable(program):
    return shutil.which(program) is not None

  def check_exists(path):
    if os.path.exists(path):
      raise RuntimeError('Target file %s exists and is not a symlink.' % path)

  @staticmethod
  def create_dir(path):
    path = os.path.abspath(os.path.expanduser(path))
    FileInstaller.remove_symlink(path)
    if not os.path.exists(path):
      print('Creating directory %s...' % path)
      os.makedirs(path)

  @staticmethod
  def create_file(path):
    path = os.path.abspath(os.path.expanduser(path))
    dir = os.path.dirname(path)
    if not os.path.islink(dir):
      FileInstaller.create_dir(dir)
    if not os.path.exists(path):
      print('Creating file %s...' % path)
      with open(path, 'wb'):
        pass

  @staticmethod
  def copy_file(source, target):
    source = os.path.abspath(os.path.expanduser(source))
    target = os.path.expanduser(target)
    if target.endswith('/') or target.endswith('\\'):
      target = os.path.join(target, os.path.basename(source))
    FileInstaller.remove_symlink(target)
    print('Copying %s to %s...' % (source, target))
    FileInstaller.create_dir(os.path.dirname(target))
    shutil.copy(source, target)

  @staticmethod
  def merge_files(sources, target, comment_prefix = None):
    for i, member in enumerate(sources):
      sources[i] = os.path.abspath(os.path.expanduser(sources[i]))
    target = os.path.expanduser(target)
    if target.endswith('/') or target.endswith('\\'):
      target = os.path.join(target, os.path.basename(source))
    FileInstaller.remove_symlink(target)
    FileInstaller.create_dir(os.path.dirname(target))
    print('Merging %s to %s...' % (sources, target))
    with open(target, 'w') as ft:
      if comment_prefix:
        ft.write('%s File was merged automatically by libinstall. Do not edit this file.%s'
          % (comment_prefix, os.linesep))
      for source in sources:
        with open(source) as fs:
            if comment_prefix:
              ft.write('%s Merge %s/%s from: %s%s'
                % (comment_prefix, sources.index(source)+1, len(sources), source, os.linesep))
            ft.write(fs.read())
            ft.write(os.linesep)

  @staticmethod
{'fraction_non_alphanumeric': 0.13336923353078442, 'fraction_numerical': 0.0016155088852988692, 'mean_word_length': 3.3977900552486187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1720854', 'n_tokens_mistral': 1849, 'n_tokens_neox': 1818, 'n_words': 419}