<commit_before>
import os
from io import BytesIO
from pprint import pprint

from data import Reader, Mention, Writer
from formats import Unstitch, Stitch
from evaluate import Evaluate
from utils import normalise_link

DIR = os.path.join(os.path.dirname(__file__))
EXAMPLES = os.path.join(DIR, 'examples')

DATA = os.path.join(EXAMPLES, 'data.txt')
DATA_FULL = os.path.join(EXAMPLES, 'data_full.txt')

def test_normalisation():
    assert normalise_link('Some title') == 'Some_title'
    assert normalise_link('http://en.wikipedia.org/wiki/Some title') == 'Some_title'
    assert normalise_link('http://fr.wikipedia.org/wiki/Some') == 'Some'
    assert normalise_link('Some') == 'Some'

def test_data():
    d = list(Reader(open(DATA)))
    assert len(d) == 1
    doc = list(d)[0]
    assert len(list(doc.iter_mentions())) == 2
    assert len(list(doc.iter_links())) == 1
    assert len(list(doc.iter_nils())) == 1

def test_read_write():
    for f in (DATA, DATA_FULL):
        out = BytesIO()
        w = Writer(out)
        for doc in list(Reader(open(f))):
            w.write(doc)
        w_str = '\n'.join(l.rstrip('\t') for l in out.getvalue().split('\n'))
        assert w_str == open(f).read()

def test_unstitch_stitch():
    def unstitch(f):
        u_fname = '{}.unstitched.tmp'.format(f)
        u_fh = open(u_fname, 'w')
        u_str = Unstitch(f)()
        print >>u_fh, u_str
        u_fh.close()
        return u_fname
    for f in (DATA, DATA_FULL):
        u_fname = unstitch(f)
        s_str = Stitch(u_fname, f)()
        s_str = '\n'.join(l.rstrip('\t') for l in s_str.split('\n'))
        assert s_str == open(f).read()
<commit_msg>prepare-tac test and sketch for tac eval test<commit_after>
import os
from io import BytesIO
from pprint import pprint

from data import Reader, Mention, Writer
from formats import Unstitch, Stitch
from tac import PrepareTac
from evaluate import Evaluate
from utils import normalise_link

DIR = os.path.join(os.path.dirname(__file__))
EXAMPLES = os.path.join(DIR, 'examples')

DATA = os.path.join(EXAMPLES, 'data.txt')
DATA_FULL = os.path.join(EXAMPLES, 'data_full.txt')

TAC_GOLD_QUERIES = os.path.join(EXAMPLES, 'tac_gold.xml')
TAC_GOLD_LINKS = os.path.join(EXAMPLES, 'tac_gold.tab')
TAC_GOLD_COMB = os.path.join(EXAMPLES, 'tac_gold.combined.tsv')
#TAC_SYS_QUERIES = os.path.join(EXAMPLES, 'tac_system.xml')
#TAC_SYS_LINKS = os.path.join(EXAMPLES, 'tac_system.tab')

def test_tac_prepare():
    combined = PrepareTac(TAC_GOLD_LINKS, TAC_GOLD_QUERIES)()
    assert combined == open(TAC_GOLD_COMB).read().rstrip('\n')

"""
def test_tac_eval():
    def combine(links_fname, queries_fname):
        c_fname = '{}.combined.tmp'.format(links_fname)
        with open(c_fname, 'w') as c_fh):
            print >>c_fh, PrepareTac(links, queries)()
        return c_fname
    gold = combine(TAC_GOLD_LINKS, TAC_GOLD_QUERIES)
    system = combine(TAC_SYSTEM_LINKS, TAC_SYSTEM_QUERIES)
    stats = Evaluate(gold, system, fmt='no_format')()
    check_correct(TAC_CORRECT, stats)
    os.remove(gold)
    os.remove(system)
"""

def test_normalisation():
    assert normalise_link('Some title') == 'Some_title'
    assert normalise_link('http://en.wikipedia.org/wiki/Some title') == 'Some_title'
    assert normalise_link('http://fr.wikipedia.org/wiki/Some') == 'Some'
    assert normalise_link('Some') == 'Some'

def test_data():
    d = list(Reader(open(DATA)))
    assert len(d) == 1
    doc = list(d)[0]
    assert len(list(doc.iter_mentions())) == 2
    assert len(list(doc.iter_links())) == 1
    assert len(list(doc.iter_nils())) == 1

def test_read_write():
    for f in (DATA, DATA_FULL):
        out = BytesIO()
        w = Writer(out)
        for doc in list(Reader(open(f))):
            w.write(doc)
        w_str = '\n'.join(l.rstrip('\t') for l in out.getvalue().split('\n'))
        assert w_str == open(f).read()

def test_unstitch_stitch():
    def unstitch(f):
        u_fname = '{}.unstitched.tmp'.format(f)
        with open(u_fname, 'w') as u_fh:
            print >>u_fh, Unstitch(f)()
        return u_fname
    for f in (DATA, DATA_FULL):
        u_fname = unstitch(f)
        s_str = Stitch(u_fname, f)()
        s_str = '\n'.join(l.rstrip('\t') for l in s_str.split('\n'))
        assert s_str == open(f).read()
        os.remove(u_fname)

{'fraction_non_alphanumeric': 0.15204271123491178, 'fraction_numerical': 0.002321262766945218, 'mean_word_length': 3.433127572016461, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '204575', 'n_tokens_mistral': 1617, 'n_tokens_neox': 1621, 'n_words': 338}