<commit_before>#!/usr/bin/env python
import optparse
import sys
import os

optparser = optparse.OptionParser()
optparser.add_option("-d", "--datadir", dest="datadir", default="~/Daten/align-data", help="data directory (default=~/Daten/align-data)")
optparser.add_option("-p", "--prefix", dest="fileprefix", default="test.seg.cln", help="prefix of parallel data files (default=test.seg.cln)")
optparser.add_option("-f", "--french", dest="french", default="cn", help="suffix of French (source language) filename (default=cn)")
optparser.add_option("-e", "--english", dest="english", default="en", help="suffix of English (target language) filename (default=en)")
optparser.add_option("-a", "--alignments", dest="alignment", default="gold.wa", help="suffix of gold alignments filename (default=gold.wa)")
optparser.add_option("-l", "--logfile", dest="logfile", default=None, help="filename for logging output")
optparser.add_option("-t", "--threshold", dest="threshold", default=0.5, type="float", help="threshold for alignment (default=0.5)")
optparser.add_option("-n", "--num_display", dest="n", default=sys.maxint, type="int", help="number of alignments to display")
optparser.add_option("-i", "--inputfile", dest="inputfile", default=None, help="input alignments file (default=sys.stdin)")
(opts, _) = optparser.parse_args()
f_data = "%s.%s" % (os.path.join(opts.datadir, opts.fileprefix), opts.french)
e_data = "%s.%s" % (os.path.join(opts.datadir, opts.fileprefix), opts.english)
a_data = "%s.%s" % (os.path.join(opts.datadir, opts.fileprefix), opts.alignment)

if opts.logfile:
    logging.basicConfig(filename=opts.logfile, filemode='w', level=logging.INFO)

inp = sys.stdin if opts.inputfile is None else file(opts.inputfile)

(size_a, size_s, size_a_and_s, size_a_and_p) = (0.0, 0.0, 0.0, 0.0)
for (i, (f, e, g, a)) in enumerate(zip(open(f_data), open(e_data), open(a_data), inp)):
    fwords = f.strip().split()
    ewords = e.strip().split()
    sure = set([tuple(map(int, x.split("-"))) for x in filter(lambda x: x.find("-") > -1, g.strip().split())])
    possible = set([tuple(map(int, x.split("?"))) for x in filter(lambda x: x.find("?") > -1, g.strip().split())])
    alignment = set([tuple(map(int, x.split("-"))) for x in a.strip().split()])
    size_a += len(alignment)
    size_s += len(sure)
    size_a_and_s += len(alignment & sure)
    size_a_and_p += len(alignment & possible) + len(alignment & sure)
    if (i < opts.n):
        sys.stdout.write("    Alignment %i    KEY: ( ) = guessed, * = sure, ? = possible\n" % i)<commit_msg>small fix<commit_after>#!/usr/bin/env python
import optparse
import sys
import os

optparser = optparse.OptionParser()
optparser.add_option("-d", "--datadir", dest="datadir", default="~/Daten/align-data", help="data directory (default=~/Daten/align-data)")
optparser.add_option("-p", "--prefix", dest="fileprefix", default="test.seg.cln", help="prefix of parallel data files (default=test.seg.cln)")
optparser.add_option("-f", "--french", dest="french", default="cn", help="suffix of French (source language) filename (default=cn)")
optparser.add_option("-e", "--english", dest="english", default="en", help="suffix of English (target language) filename (default=en)")
optparser.add_option("-a", "--alignments", dest="alignment", default="gold.wa", help="suffix of gold alignments filename (default=gold.wa)")
optparser.add_option("-l", "--logfile", dest="logfile", default=None, help="filename for logging output")
optparser.add_option("-t", "--threshold", dest="threshold", default=0.5, type="float", help="threshold for alignment (default=0.5)")
optparser.add_option("-n", "--num_display", dest="n", default=sys.maxint, type="int", help="number of alignments to display")
optparser.add_option("-i", "--inputfile", dest="inputfile", default=None, help="input alignments file (default=sys.stdin)")
(opts, _) = optparser.parse_args()
f_data = os.path.expanduser("%s.%s" % (os.path.join(opts.datadir, opts.fileprefix), opts.french))
e_data = os.path.expanduser("%s.%s" % (os.path.join(opts.datadir, opts.fileprefix), opts.english))
a_data = os.path.expanduser("%s.%s" % (os.path.join(opts.datadir, opts.fileprefix), opts.alignment))

if opts.logfile:
    logging.basicConfig(filename=opts.logfile, filemode='w', level=logging.INFO)

inp = sys.stdin if opts.inputfile is None else file(opts.inputfile)

(size_a, size_s, size_a_and_s, size_a_and_p) = (0.0, 0.0, 0.0, 0.0)
for (i, (f, e, g, a)) in enumerate(zip(open(f_data), open(e_data), open(a_data), inp)):
    fwords = f.strip().split()
    ewords = e.strip().split()
    sure = set([tuple(map(int, x.split("-"))) for x in filter(lambda x: x.find("-") > -1, g.strip().split())])
    possible = set([tuple(map(int, x.split("?"))) for x in filter(lambda x: x.find("?") > -1, g.strip().split())])
    alignment = set([tuple(map(int, x.split("-"))) for x in a.strip().split()])
    size_a += len(alignment)
    size_s += len(sure)
    size_a_and_s += len(alignment & sure)
    size_a_and_p += len(alignment & possible) + len(alignment & sure)
    if (i < opts.n):
        sys.stdout.write("    Alignment %i    KEY: ( ) = guessed, * = sure, ? = possible\n" % i)
{'fraction_non_alphanumeric': 0.2078324932144242, 'fraction_numerical': 0.005428460643660334, 'mean_word_length': 5.306845965770171, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1425333', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1808, 'n_words': 398}