<commit_before>import filecmp
import os
import os.path
import tempfile

import arpa

TEST_ARPA = os.path.join(os.path.dirname(__file__), "data/test.arpa")


def test_load_dump():
    with open(TEST_ARPA, "rt") as fp:
        lm = arpa.load(fp)[0]
        fp.seek(0)
        with tempfile.TemporaryFile(mode="w+t") as gp:
            arpa.dump(lm, gp)
            gp.seek(0)
            assert fp.read() == gp.read()


def test_loadf_dumpf():
    lm = arpa.loadf(TEST_ARPA)[0]
    out = tempfile.NamedTemporaryFile(mode="w+t", delete=False)
    arpa.dumpf(lm, out.name)
    out.close()
    assert filecmp.cmp(TEST_ARPA, out.name, shallow=False)
    os.unlink(out.name)


def test_loads_dumps():
    with open(TEST_ARPA, "rt") as fp:
        txt = fp.read()
        lm = arpa.loads(txt)[0]
        out = arpa.dumps(lm)
        assert txt == out
<commit_msg>test with wrong arguments for arpa.load<commit_after>import filecmp
import os
import os.path
import tempfile

import pytest

import arpa

TEST_ARPA = os.path.join(os.path.dirname(__file__), "data/test.arpa")


def test_load_option_model():
    with pytest.raises(ValueError):
        arpa.load(None, model="foo")


def test_load_option_parser():
    with pytest.raises(ValueError):
        arpa.load(None, parser="foo")


def test_load_dump():
    with open(TEST_ARPA, "rt") as fp:
        lm = arpa.load(fp)[0]
        fp.seek(0)
        with tempfile.TemporaryFile(mode="w+t") as gp:
            arpa.dump(lm, gp)
            gp.seek(0)
            assert fp.read() == gp.read()


def test_loadf_dumpf():
    lm = arpa.loadf(TEST_ARPA)[0]
    out = tempfile.NamedTemporaryFile(mode="w+t", delete=False)
    arpa.dumpf(lm, out.name)
    out.close()
    assert filecmp.cmp(TEST_ARPA, out.name, shallow=False)
    os.unlink(out.name)


def test_loads_dumps():
    with open(TEST_ARPA, "rt") as fp:
        txt = fp.read()
        lm = arpa.loads(txt)[0]
        out = arpa.dumps(lm)
        assert txt == out

{'fraction_non_alphanumeric': 0.1390728476821192, 'fraction_numerical': 0.005094243504839531, 'mean_word_length': 2.806201550387597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '873977', 'n_tokens_mistral': 752, 'n_tokens_neox': 736, 'n_words': 147}