<commit_before>from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals
#
# This file is part of khmer, https://github.com/dib-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2015. It is licensed under
# the three-clause BSD license; see LICENSE.
# Contact: <EMAIL>
#

import os
import shutil
import threading
import io

import screed
import khmer

from . import khmer_tst_utils as utils
from nose.plugins.attrib import attr
from .test_scripts import _make_counting


def test_normalize_by_median_indent():
    infile = utils.get_test_data('paired-mixed.fa.pe')
    hashfile = utils.get_test_data('normC20k20.ct')
    outfile = utils.get_temp_filename('paired-mixed.fa.pe.keep')
    script = 'normalize-by-median.py'
    args = ['--loadtable', hashfile, '-o', outfile, infile]
    (status, out, err) = utils.runscript(script, args)
    assert status == 0, (out, err)
    assert os.path.exists(outfile)


def test_normalize_by_median():
    CUTOFF = '1'

    infile = utils.get_temp_filename('test.fa')
    in_dir = os.path.dirname(infile)

    shutil.copyfile(utils.get_test_data('test-abund-read-2.fa'), infile)

    script = 'normalize-by-median.py'
    args = ['-C', CUTOFF, '-k', '17', infile]
    (status, out, err) = utils.runscript(script, args, in_dir)

    assert 'Total number of unique k-mers: 98' in err, err

    outfile = infile + '.keep'
    assert os.path.exists(outfile), outfile

    seqs = [r.sequence for r in screed.open(outfile)]
    assert len(seqs) == 1, seqs
    assert seqs[0].startswith('GGTTGACGGGGCTCAGGGGG'), seqs
    assert "I/O Errors" not in err<commit_msg>added test to pre-empt not-fully-thought-though 'fixes'<commit_after>from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals
#
# This file is part of khmer, https://github.com/dib-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2015. It is licensed under
# the three-clause BSD license; see LICENSE.
# Contact: <EMAIL>
#

import os
import shutil
import threading
import io

import screed
import khmer

from . import khmer_tst_utils as utils
from nose.plugins.attrib import attr
from .test_scripts import _make_counting


def test_normalize_by_median_indent():
    infile = utils.get_test_data('paired-mixed.fa.pe')
    hashfile = utils.get_test_data('normC20k20.ct')
    outfile = utils.get_temp_filename('paired-mixed.fa.pe.keep')
    script = 'normalize-by-median.py'
    args = ['--loadtable', hashfile, '-o', outfile, infile]
    (status, out, err) = utils.runscript(script, args)
    assert status == 0, (out, err)
    assert os.path.exists(outfile)


def test_normalize_by_median_empty_file():
    infile = utils.get_temp_filename('empty')
    shutil.copyfile(utils.get_test_data('empty-file'), infile)
    script = 'normalize-by-median.py'
    in_dir = os.path.dirname(infile)

    args = [infile]
    (status, out, err) = utils.runscript(script, args, in_dir)

    assert 'WARNING:' in err, err
    assert 'is empty' in err, err
    assert 'SKIPPED' in err, err


def test_normalize_by_median():
    CUTOFF = '1'

    infile = utils.get_temp_filename('test.fa')
    in_dir = os.path.dirname(infile)

    shutil.copyfile(utils.get_test_data('test-abund-read-2.fa'), infile)

    script = 'normalize-by-median.py'
    args = ['-C', CUTOFF, '-k', '17', infile]
    (status, out, err) = utils.runscript(script, args, in_dir)

    assert 'Total number of unique k-mers: 98' in err, err

    outfile = infile + '.keep'
    assert os.path.exists(outfile), outfile

    seqs = [r.sequence for r in screed.open(outfile)]
    assert len(seqs) == 1, seqs
    assert seqs[0].startswith('GGTTGACGGGGCTCAGGGGG'), seqs
    assert "I/O Errors" not in err
{'fraction_non_alphanumeric': 0.1220026350461133, 'fraction_numerical': 0.011067193675889328, 'mean_word_length': 3.9491525423728815, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3810779', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1329, 'n_words': 340}