<commit_before>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/dib-lab/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

import glob
import pytest
import sys
import khmer
import kevlar
from kevlar.seqio import AnnotatedReadSet as ReadSet


@pytest.fixture
def bogusrefr():
    mask = khmer.Nodetable(13, 1e7 / 4, 4)
    maskfile = kevlar.tests.data_file('bogus-genome/refr.fa')
    mask.consume_seqfile(maskfile)
    return mask


@pytest.fixture
def bogusrefrcontam():
    return khmer.Nodetable.load(kevlar.tests.data_file('bogus-genome/mask.nt'))


@pytest.fixture
def ctrl3():
    augfastq = kevlar.tests.data_file('trio1/novel_3_1,2.txt')
    readset = ReadSet(13, 1e7)
    for record in kevlar.parse_augmented_fastx(kevlar.open(augfastq, 'r')):
        readset.add(record)
    return readset


def test_load_mask():
    infile = kevlar.tests.data_file('bogus-genome/refr.fa')
    mask = kevlar.filter.load_mask([infile], 25, 1e7)
    assert mask.get('GGCCCCGAACTAGGGGGCCTACGTT') > 0
    assert mask.get('GCTGGCTAAATTTTCATACTAACTA') > 0
    assert mask.get('G' * 25) == 0

    assert kevlar.filter.load_mask(None, 25, 1e7) is None


def test_load_mask_multi_file():
    infiles = [
        kevlar.tests.data_file('bogus-genome/refr.fa'),
        kevlar.tests.data_file('bogus-genome/contam1.fa')
    ]
    mask = kevlar.filter.load_mask(infiles, 25, 1e7)
    assert mask.get('GGCCCCGAACTAGGGGGCCTACGTT') > 0  # reference
    assert mask.get('GCTGGCTAAATTTTCATACTAACTA') > 0  # reference
    assert mask.get('AATGTAGGTAGTTTTGTGCACAGTT') > 0  # contam
    assert mask.get('TCGCGCGCGTCCAAGTCGAGACCGC') > 0  # contam
    assert mask.get('G' * 25) == 0


def test_load_mask_too_small():
    infile = kevlar.tests.data_file('bogus-genome/refr.fa')
    with pytest.raises(SystemExit) as se:
        mask = kevlar.filter.load_mask([infile], 25, 1e3)
    assert 'FPR too high, bailing out' in str(se)


def test_load_readset():
    filelist = kevlar.tests.data_glob('collect.beta.?.txt')
    readset = ReadSet(19, 1e3)
    for record in kevlar.seqio.afxstream(filelist):
        readset.add(record)

    assert len(readset) == 8
    assert readset
    kmers = [
        'AGGGGCGTGACTTAATAAG', 'GGGCGTGACTTAATAAGGT',
        'TAGGGGCGTGACTTAATAA', 'GGGGCGTGACTTAATAAGG',
    ]
    for kmer in kmers:
        assert readset._counts.get(kmer) == 8


def test_validate():<commit_msg>Increasing test coverage<commit_after>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/dib-lab/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

import glob
import pytest
import sys
from tempfile import NamedTemporaryFile
import khmer
import kevlar
from kevlar.seqio import AnnotatedReadSet as ReadSet


@pytest.fixture
def bogusrefr():
    mask = khmer.Nodetable(13, 1e7 / 4, 4)
    maskfile = kevlar.tests.data_file('bogus-genome/refr.fa')
    return kevlar.filter.load_mask([maskfile], 13, 1e7)


@pytest.fixture
def bogusrefrcontam():
    maskfile = kevlar.tests.data_file('bogus-genome/mask.nt')
    return kevlar.filter.load_mask([maskfile], 1, 1)


@pytest.fixture
def ctrl3():
    augfastq = kevlar.tests.data_file('trio1/novel_3_1,2.txt')
    readset = ReadSet(13, 1e7)
    for record in kevlar.parse_augmented_fastx(kevlar.open(augfastq, 'r')):
        readset.add(record)
    return readset


def test_load_mask():
    infile = kevlar.tests.data_file('bogus-genome/refr.fa')
    mask = kevlar.filter.load_mask([infile], 25, 1e7)
    assert mask.get('GGCCCCGAACTAGGGGGCCTACGTT') > 0
    assert mask.get('GCTGGCTAAATTTTCATACTAACTA') > 0
    assert mask.get('G' * 25) == 0

    assert kevlar.filter.load_mask(None, 25, 1e7) is None


def test_load_mask_multi_file():
    infiles = [
        kevlar.tests.data_file('bogus-genome/refr.fa'),
        kevlar.tests.data_file('bogus-genome/contam1.fa')
    ]
    mask = kevlar.filter.load_mask(infiles, 25, 1e7)
    assert mask.get('GGCCCCGAACTAGGGGGCCTACGTT') > 0  # reference
    assert mask.get('GCTGGCTAAATTTTCATACTAACTA') > 0  # reference
    assert mask.get('AATGTAGGTAGTTTTGTGCACAGTT') > 0  # contam
    assert mask.get('TCGCGCGCGTCCAAGTCGAGACCGC') > 0  # contam
    assert mask.get('G' * 25) == 0


def test_load_mask_save():
    infiles = [
        kevlar.tests.data_file('bogus-genome/refr.fa'),
        kevlar.tests.data_file('bogus-genome/contam1.fa')
    ]

    with NamedTemporaryFile(suffix='.nt') as table:
        mask = kevlar.filter.load_mask(infiles, 25, 1e7, savefile=table.name)
        newmask = khmer.Nodetable.load(table.name)
    assert newmask.get('GGCCCCGAACTAGGGGGCCTACGTT') > 0  # reference
    assert newmask.get('GCTGGCTAAATTTTCATACTAACTA') > 0  # reference
    assert newmask.get('AATGTAGGTAGTTTTGTGCACAGTT') > 0  # contam
    assert newmask.get('TCGCGCGCGTCCAAGTCGAGACCGC') > 0  # contam
    assert newmask.get('G' * 25) == 0


def test_load_mask_too_small():
    infile = kevlar.tests.data_file('bogus-genome/refr.fa')
    with pytest.raises(SystemExit) as se:
        mask = kevlar.filter.load_mask([infile], 25, 1e3)
    assert 'FPR too high, bailing out' in str(se)


def test_load_readset():
    filelist = kevlar.tests.data_glob('collect.beta.?.txt')
    readset = ReadSet(19, 1e3)
    for record in kevlar.seqio.afxstream(filelist):
        readset.add(record)

    assert len(readset) == 8
    assert readset
    kmers = [
        'AGGGGCGTGACTTAATAAG', 'GGGCGTGACTTAATAAGGT',
        'TAGGGGCGTGACTTAATAA', 'GGGGCGTGACTTAATAAGG',
    ]
    for kmer in kmers:
        assert readset._counts.get(kmer) == 8


def test_validate():
{'fraction_non_alphanumeric': 0.16827082641885163, 'fraction_numerical': 0.02090939263192831, 'mean_word_length': 2.94954128440367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3597833', 'n_tokens_mistral': 2387, 'n_tokens_neox': 2222, 'n_words': 410}