<commit_before>from os import path

from digestive.ewf import EWFSource, format_supported, list_ewf_files


here = path.dirname(path.abspath(__file__))


def test_format_supported():
    supported = ['file.S01', 'file.E01', 'file.e01', 'file.L01', 'file.Ex01', 'file.Lx01', 'file.tar.E01']
    not_supported = ['file.dd', 'file.raw', 'file.E1', 'file.Ex1', 'file.tar.gz', 'file.E01.raw']

    for supported in supported:
        assert format_supported(supported)

    for not_supported in not_supported:
        assert not format_supported(not_supported)


def test_list_ewf_files():
    files = [path.join(here, 'files/random.E01'), path.join(here, 'files/random.E02')]
    assert list_ewf_files(path.join(here, 'files/random.E01')) == files
    # non-primary file is not handled as the beginning of a set
    assert list_ewf_files(path.join(here, 'files/random.E02')) == [path.join(here, 'files/random.E02')]
<commit_msg>Ignore order for globbed file list in test_ewf<commit_after>from os import path

from digestive.ewf import EWFSource, format_supported, list_ewf_files


here = path.dirname(path.abspath(__file__))


def test_format_supported():
    supported = ['file.S01', 'file.E01', 'file.e01', 'file.L01', 'file.Ex01', 'file.Lx01', 'file.tar.E01']
    not_supported = ['file.dd', 'file.raw', 'file.E1', 'file.Ex1', 'file.tar.gz', 'file.E01.raw']

    for supported in supported:
        assert format_supported(supported)

    for not_supported in not_supported:
        assert not format_supported(not_supported)


def test_list_ewf_files():
    files = {path.join(here, 'files/random.E01'), path.join(here, 'files/random.E02')}
    assert set(list_ewf_files(path.join(here, 'files/random.E01'))) == files
    # non-primary file is not handled as the beginning of a set
    assert list_ewf_files(path.join(here, 'files/random.E02')) == [path.join(here, 'files/random.E02')]

{'fraction_non_alphanumeric': 0.14952279957582185, 'fraction_numerical': 0.029692470837751856, 'mean_word_length': 4.4854651162790695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5677730', 'n_tokens_mistral': 695, 'n_tokens_neox': 656, 'n_words': 145}