<commit_before>#!/usr/bin/env python

import sys, subprocess, os, shutil, time

# execute a shell command
def call_shell(cmd):
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
  return p

def touch(fname, times = None):
    fhandle = file(fname, 'a')
    try:
        os.utime(fname, times)
    finally:
        fhandle.close()

# simple file locking
def lock_file(file):
  lock_file_name = file + '.lock'
  if os.path.exists(lock_file_name):
    return False
  else:
    touch(lock_file_name)
    return True

def is_locked(file):
  lock_file_name = file + '.lock'
  print "lock time diff: " + str(time.time()-os.path.getctime(lock_file_name))
  if os.path.exists(lock_file_name) and time.time()-os.path.getctime(lock_file_name)<60:
    return True
  else:
    return False

def unlock_file(file):
  lock_file_name = file + '.lock'
  if os.path.exists(lock_file_name):
    os.remove(lock_file_name)

def generate_dictionary(rootcint, dict_file, args):
  include_dirs = []
  input_files = []
  for arg in args:
    if os.path.isdir(arg):
      include_dirs.append('-I'+arg)
    else:
      input_files.append(arg)

  dict_file_completed = dict_file + "_completed"
  cmd = rootcint + ' '.join(include_dirs) + " -f " + dict_file + " -c -p " + ' '.join(input_files)
  if lock_file(dict_file): 
    p = call_shell(cmd)
    r = p.wait()
    if r != 0:
      print p.stderr.read()
      print p.stdout.read()
    shutil.copy(dict_file, dict_file_completed)
    unlock_file(dict_file)
  else:
    print >> sys.stderr, "File " + dict_file + " is locked. Waiting for unlock."
    while is_locked(dict_file):
      time.sleep(1)
    
if __name__ == "__main__":
  if len(sys.argv) < 4:
    print "Usage: " + sys.argv[0] + " rootcint dict_file input_files"
    sys.exit(0)
  if len(sys.argv) > 4:
    print "Additional parameters: " + str(sys.argv[4:])
  generate_dictionary(sys.argv[1], sys.argv[2], sys.argv[3:])<commit_msg>compiling, yay<commit_after>#!/usr/bin/env python

import sys, subprocess, os, shutil, time

# execute a shell command
def call_shell(cmd):
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
  return p

def touch(fname, times = None):
    fhandle = file(fname, 'a')
    try:
        os.utime(fname, times)
    finally:
        fhandle.close()

# simple file locking
def lock_file(file):
  lock_file_name = file + '.lock'
  if os.path.exists(lock_file_name) and time.time()-os.path.getctime(lock_file_name)<60:
    return False
  else:
    if os.path.exists(lock_file_name):
      os.remove(lock_file_name)
    touch(lock_file_name)
    return True

def is_locked(file):
  lock_file_name = file + '.lock'
  if os.path.exists(lock_file_name) and time.time()-os.path.getctime(lock_file_name)<60:
    return True
  else:
    return False

def unlock_file(file):
  lock_file_name = file + '.lock'
  if os.path.exists(lock_file_name):
    os.remove(lock_file_name)

def generate_dictionary(rootcint, dict_file, args):
  include_dirs = []
  input_files = []
  for arg in args:
    if os.path.isdir(arg):
      include_dirs.append('-I'+arg)
    else:
      input_files.append(arg)

  dict_file_completed = dict_file + "_completed"
  cmd = rootcint + " -f " + dict_file + " -c -p " + ' ' + ' '.join(include_dirs) + ' ' + ' '.join(input_files)
  print cmd
  if lock_file(dict_file): 
    p = call_shell(cmd)
    r = p.wait()
    if r != 0:
      print p.stderr.read()
      print p.stdout.read()
    shutil.copy(dict_file, dict_file_completed)
    unlock_file(dict_file)
  else:
    print >> sys.stderr, "File " + dict_file + " is locked. Waiting for unlock."
    if is_locked(dict_file):
      time.sleep(1)
      generate_dictionary(rootcint, dict_file, args)
    
if __name__ == "__main__":
  if len(sys.argv) < 4:
    print "Usage: " + sys.argv[0] + " rootcint dict_file input_files"
    sys.exit(0)
  generate_dictionary(sys.argv[1], sys.argv[2], sys.argv[3:])
{'fraction_non_alphanumeric': 0.13713561470215463, 'fraction_numerical': 0.006083650190114068, 'mean_word_length': 3.413870246085011, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5980652', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1478, 'n_words': 306}