<commit_before>#!/usr/bin/env python

"""Tests for pyani package concordance with JSpecies output

These tests are intended to be run using the nose package
(see https://nose.readthedocs.org/en/latest/).

If the test is run directly at the command-line, the output obtained by each
test is returned to STDOUT.
"""

from nose.tools import assert_equal, assert_less
from pyani.run_multiprocessing import multiprocessing_run

import os
import pandas as pd
import shutil

from pyani import anib, anim, tetra, pyani_files, pyani_config


# Path to JSpecies output data. This data is pre-prepared. If you replace
# the test data with your own data, you will need to replace this file,
# or change the file path.
JSPECIES_OUTFILE = 'test_JSpecies/jspecies_results.tab'

# Path to test input data
INDIRNAME = 'test_ani_data'

# Path to directory for concordance test output
OUTDIRNAME = 'test_concordance'

# Thresholds for allowable difference
TETRA_THRESHOLD = 0.1
ANIB_THRESHOLD = 2  # This threshold higher because BLASTN+ != legacy BLASTN
ANIBLASTALL_THRESHOLD = 0.1
ANIM_THRESHOLD = 0.1

# Helper function to parse tables from JSpecies output
def parse_table(filename, title):
    """Parse method output from JSpecies tabular output, returns dataframe.

    - filename - path to JSpecies output file
    - title - header of ANI method output

    The JSpecies output may have one or more tables; which table is required
    is indicated by the passed title.
    """
    assert title in ('Tetra', 'ANIb', 'ANIm'), "Invalid header: %s" % title
    with open(filename, 'rU') as fh:
        header, in_table = False, False
        for line in iter(fh):
            if line.strip() == title:  # We have the table we want
                header = True
            elif header:
                columns = line.strip().split('\t')
                data = pd.DataFrame(index=columns, columns=columns)
                in_table, header = True, False
            elif in_table:
                if not len(line.strip()):
                    return data.sort(axis=0).sort(axis=1)
                else:
                    ldata = line.strip().split('\t')
                    row = ldata[0]
                    for idx, val in enumerate(ldata[1:]):
                        if val != '---':
                            data[columns[idx]][row] = float(val)
                        elif title in ("ANIb", "ANIm"):
                            data[columns[idx]][row] = 100.0
                        else:
                            data[columns[idx]][row] = 1.0
            else:
                pass
    return data.sort(axis=0).sort(axis=1)


# Make output directory if necessary<commit_msg>test_concordance.py: Fixes "FutureWarning: sort(....) is deprecated" warning<commit_after>#!/usr/bin/env python

"""Tests for pyani package concordance with JSpecies output

These tests are intended to be run using the nose package
(see https://nose.readthedocs.org/en/latest/).

If the test is run directly at the command-line, the output obtained by each
test is returned to STDOUT.
"""

from nose.tools import assert_equal, assert_less
from pyani.run_multiprocessing import multiprocessing_run

import os
import pandas as pd
import shutil

from pyani import anib, anim, tetra, pyani_files, pyani_config


# Path to JSpecies output data. This data is pre-prepared. If you replace
# the test data with your own data, you will need to replace this file,
# or change the file path.
JSPECIES_OUTFILE = 'test_JSpecies/jspecies_results.tab'

# Path to test input data
INDIRNAME = 'test_ani_data'

# Path to directory for concordance test output
OUTDIRNAME = 'test_concordance'

# Thresholds for allowable difference
TETRA_THRESHOLD = 0.1
ANIB_THRESHOLD = 2  # This threshold higher because BLASTN+ != legacy BLASTN
ANIBLASTALL_THRESHOLD = 0.1
ANIM_THRESHOLD = 0.1

# Helper function to parse tables from JSpecies output
def parse_table(filename, title):
    """Parse method output from JSpecies tabular output, returns dataframe.

    - filename - path to JSpecies output file
    - title - header of ANI method output

    The JSpecies output may have one or more tables; which table is required
    is indicated by the passed title.
    """
    assert title in ('Tetra', 'ANIb', 'ANIm'), "Invalid header: %s" % title
    with open(filename, 'rU') as fh:
        header, in_table = False, False
        for line in iter(fh):
            if line.strip() == title:  # We have the table we want
                header = True
            elif header:
                columns = line.strip().split('\t')
                data = pd.DataFrame(index=columns, columns=columns)
                in_table, header = True, False
            elif in_table:
                if not len(line.strip()):
                    return data.sort_index(axis=0).sort_index(axis=1)
                else:
                    ldata = line.strip().split('\t')
                    row = ldata[0]
                    for idx, val in enumerate(ldata[1:]):
                        if val != '---':
                            data[columns[idx]][row] = float(val)
                        elif title in ("ANIb", "ANIm"):
                            data[columns[idx]][row] = 100.0
                        else:
                            data[columns[idx]][row] = 1.0
            else:
                pass
    return data.sort(axis=0).sort(axis=1)


# Make output directory if necessary
{'fraction_non_alphanumeric': 0.08805611962340779, 'fraction_numerical': 0.007014952925973786, 'mean_word_length': 2.2717391304347827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4089657', 'n_tokens_mistral': 1615, 'n_tokens_neox': 1531, 'n_words': 547}