<commit_before>#!/usr/bin/env python
import sys, os, re, shutil
import subprocess, shlex, glob
import argparse
import logging
import random, time
import gzip, itertools
try: 
  import matplotlib
  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
except ImportError:
  have_mpl = False

#mira run script
#matplotlib must be installed for graphing to work
#email option requires mail

script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
fast_score_binary = script_dir+'/../../mteval/fast_score'
dlog = None

#scoring function using pycdec scoring
def fast_score(hyps, refs, metric):
  cmd = ('{0} -r{1} -i {2} -m {3}').format(fast_score_binary, refs, hyps, metric)
  proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
  o = proc.stdout.readline().strip()
  return float(o)

#create new parallel input file in output directory in sgml format
def enseg(devfile, newfile, gprefix):
  try:
    dev = open(devfile)
    new = open(newfile, 'w')
  except IOError, msg:
    logging.error('Error opening source file')
    raise<commit_msg>Fixed bug in MPL detection<commit_after>#!/usr/bin/env python
import sys, os, re, shutil
import subprocess, shlex, glob
import argparse
import logging
import random, time
import gzip, itertools
try: 
  import matplotlib
  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
  have_mpl = True
except ImportError:
  have_mpl = False

#mira run script
#matplotlib must be installed for graphing to work
#email option requires mail

script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
fast_score_binary = script_dir+'/../../mteval/fast_score'
dlog = None

#scoring function using pycdec scoring
def fast_score(hyps, refs, metric):
  cmd = ('{0} -r{1} -i {2} -m {3}').format(fast_score_binary, refs, hyps, metric)
  proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
  o = proc.stdout.readline().strip()
  return float(o)

#create new parallel input file in output directory in sgml format
def enseg(devfile, newfile, gprefix):
  try:
    dev = open(devfile)
    new = open(newfile, 'w')
  except IOError, msg:
    logging.error('Error opening source file')
    raise
{'fraction_non_alphanumeric': 0.11515151515151516, 'fraction_numerical': 0.004662004662004662, 'mean_word_length': 4.588541666666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5897516', 'n_tokens_mistral': 746, 'n_tokens_neox': 738, 'n_words': 223}