<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys, os, re
from optparse import OptionParser

def exec_shell(defines, executes, workdir, is_debug=False):
  source = "cd " + workdir + "\n"  + "".join(map(str, defines)) + "".join(map(str, executes))
  if is_debug:
    print(source)
  os.system(source)

def apply_defines(defines, options, args, mapper):
  return [apply_define(d, options, args, mapper) for d in defines if d.count('=') == 1]

def apply_define(define, options, args, mapper):
  xs = define.split('=')
  name = xs[0]
  value = re.sub('\$\{(\d+)\}', lambda m:str(args[int(m.group(1)) - 1]),xs[1])
  if name in mapper:
    key = mapper[name][1].replace("--", "")
    print key
    if None != options[key]:
      value = str(options[key])

  return name + '=' + value + "\n"

def parse_defines(defines, optparser):<commit_msg>remove debug code.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys, os, re
from optparse import OptionParser

def exec_shell(defines, executes, workdir, is_debug=False):
  source = "cd " + workdir + "\n"  + "".join(map(str, defines)) + "".join(map(str, executes))
  if is_debug:
    print(source)
  os.system(source)

def apply_defines(defines, options, args, mapper):
  return [apply_define(d, options, args, mapper) for d in defines if d.count('=') == 1]

def apply_define(define, options, args, mapper):
  xs = define.split('=')
  name = xs[0]
  value = re.sub('\$\{(\d+)\}', lambda m:str(args[int(m.group(1)) - 1]),xs[1])
  if name in mapper:
    key = mapper[name][1].replace("--", "")
    if None != options[key]:
      value = str(options[key])

  return name + '=' + value + "\n"

def parse_defines(defines, optparser):
{'fraction_non_alphanumeric': 0.19368051492100644, 'fraction_numerical': 0.008191925102399064, 'mean_word_length': 3.9709302325581395, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5138851', 'n_tokens_mistral': 641, 'n_tokens_neox': 626, 'n_words': 159}