<commit_before>#!/usr/bin/env python
"""
:Author: <NAME>

:Date Created: 10/22/2014

:Abstract: Automated Tests for OTU calculations.
"""
import unittest
import json
import os
import tempfile
import sys
from phylotoast import otu_calc as oc
from phylotoast import biom_calc as bc


class otu_calc_Test(unittest.TestCase):

    def setUp(self):
        """
        Setting up the test module. Initializing BIOM format file.
        """
        self.biom_text = """{
                     "id":null,
                     "format": "Biological Observation Matrix 0.9.1-dev",
                     "format_url": "http://biom-format.org/documentation/format_versions/biom-1.0.html",
                     "type": "OTU table",
                     "generated_by": "QIIME revision 1.4.0-dev",
                     "date": "2011-12-19T19:00:00",
                     "rows":[
                        {"id":"GG_OTU_1", "metadata":{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}},
                        {"id":"GG_OTU_2", "metadata":{"taxonomy":["k__Bacteria", "p__Cyanobacteria", "c__Nostocophycideae", "o__Nostocales", "f__Nostocaceae", "g__Dolichospermum", "s__"]}},
                        {"id":"GG_OTU_3", "metadata":{"taxonomy":["k__Archaea", "p__Euryarchaeota", "c__Methanomicrobia", "o__Methanosarcinales", "f__Methanosarcinaceae", "g__Methanosarcina", "s__"]}},
                        {"id":"GG_OTU_4", "metadata":{"taxonomy":["k__Bacteria", "p__Firmicutes", "c__Clostridia", "o__Halanaerobiales", "f__Halanaerobiaceae", "g__Halanaerobium", "s__Halanaerobiumsaccharolyticum"]}},
                        {"id":"GG_OTU_5", "metadata":{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}}
                        ],
                     "columns":[
                        {"id":"Sample1", "metadata":{
                                                 "BarcodeSequence":"CGCTTATCGAGA",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample2", "metadata":{
                                                 "BarcodeSequence":"CATACCAGTAGC",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample3", "metadata":{
                                                 "BarcodeSequence":"CTCTCTACCTGT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample4", "metadata":{
                                                 "BarcodeSequence":"CTCTCGGCCTGT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}},
                        {"id":"Sample5", "metadata":{
                                                 "BarcodeSequence":"CTCTCTACCAAT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}},
                        {"id":"Sample6", "metadata":{
                                                 "BarcodeSequence":"CTAACTACCAAT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}}
                                ],
                     "matrix_type": "sparse",
                     "matrix_element_type": "int",
                     "shape": [5, 6],
                     "data":[[0,2,1],
                             [1,0,5],
                             [1,1,1],
                             [1,3,2],
                             [1,4,3],
                             [1,5,1],
                             [2,2,1],
                             [2,3,4],
                             [2,5,2],
                             [3,0,2],
                             [3,1,1],
                             [3,2,1],
                             [3,5,1],
                             [4,1,1],
                             [4,2,1]
                            ]
                        }"""

        self.coreOTUdata = '''# Core OTUs across 75 % of samples.
064EK048	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__HOT.064']
200CZ006	[u'k__Bacteria', u'p__Fusobacteria', u'c__Fusobacteria', u'o__Fusobacteriales', u'f__Fusobacteriaceae', u'g__Fusobacterium', u's__nucleatum']
161Vparv	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Veillonellaceae', u'g__Veillonella', u's__parvula']
111_3523	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Peptostreptococcaceae_[XIII]', u'g__Parvimonas', u's__micra']
644_4671	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__intermedius']
758_3928	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__sanguinis']
524Vatyp	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Veillonellaceae', u'g__Veillonella', u's__atypica']
151_K168	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Veillonellaceae', u'g__Selenomonas', u's__sputigena']
530_5256	[u'k__Bacteria', u'p__Actinobacteria', u'c__Actinobacteria', u'o__Actinomycetales', u'f__Propionibacteriaceae', u'g__Propionibacterium', u's__acnes']
612_6662	[u'k__Bacteria', u'p__Proteobacteria', u'c__Gammaproteobacteria', u'o__Pseudomonadales', u'f__Pseudomonadaceae', u'g__Pseudomonas', u's__fluorescens']
539_6962	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Peptostreptococcaceae_[XI]', u'g__Filifactor', u's__alocis']
073-T1E5	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__australis']
398_7051	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__mitis']
622_3931	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__gordonii']
826_8612	[u'k__Bacteria', u'p__Proteobacteria', u'c__Gammaproteobacteria', u'o__Pasteurellales', u'f__Pasteurellaceae', u'g__Terrahaemophilus', u's__aromaticivorans']
623_4320	[u'k__Bacteria', u'p__Proteobacteria', u'c__Epsilonproteobacteria', u'o__Campylobacterales', u'f__Campylobacteraceae', u'g__Campylobacter', u's__gracilis']
205FL002	[u'k__Bacteria', u'p__Fusobacteria', u'c__Fusobacteria', u'o__Fusobacteriales', u'f__Fusobacteriaceae', u'g__Fusobacterium', u's__HOT.205']'''

        self.biom = json.loads(self.biom_text)
        self.row = self.biom['rows']

    # def test_fuzzy_lookup(self):
    #     """
    #     Testing fuzzy_lookup() function of otu_calc.py.

    #     :return: Returns OK if the test goals were achieved, otherwise
    #             raises error.
    #     """
    #     self.result1 = oc.assign_otu_membership(self.biom)['Sample2']
    #     self.result = oc.fuzzy_lookup(self.result1, ['Escherichia_spp.'])

    #     # Obtaining manual result to be compared against.
    #     hand_calc = 'Escherichia_spp.'

    #     # Testing the validity of fuzzy_lookup() function.
    #     self.assertIn(
    #         hand_calc, str(self.result1),
    #         msg='Error! Output is not a subset of input.'
    #     )

    # def test_sdi(self):
    #     """
    #     Testing sdi() function of otu_calc.py.

    #     :return: Returns OK if the test goals were achieved, otherwise
    #              raises error.
    #     """
    #     self.result1 = oc.assign_otu_membership(self.biom)
    #     fset = self.result1['Sample4']
    #     self.result = oc.sdi(fset)

    #     # Manual calculation of SDI
    #     hand_calc = 0.636513937

    #     # Testing the validity of sdi() function
    #     self.assertAlmostEqual(
    #         self.result, hand_calc, places=6,
    #         msg='Error! SDI was calculated inacccurately.'
    #     )

    def test_otu_name_biom(self):
        """
        Testing otu_name_biom() function of otu_calc.py.

        :return: Returns OK if the test goals were achieved, otherwise
                 raises error.
        """
        self.result = oc.otu_name_biom(self.row[3])
        hand_calc = 'Halanaerobium_Halanaerobiumsaccharolyticum'

        # Testing the validity of otu_name_biom() function
        self.assertEqual(
            self.result, hand_calc,
            msg='Error! otu_name_biom() output does not match manually identified name.'
        )

    def test_otu_name(self):
        """
        Testing otu_name() function of otu_calc.py.

        :return: Returns OK if the test goals were achieved, otherwise
                 raises error.
        """
        self.tax = [
            "k__Archaea", "p__Euryarchaeota", "c__Methanomicrobia",
            "o__Methanosarcinales", "f__",
            "g__", "s__"
            ]
        self.result = oc.otu_name(self.tax)
        hand_calc = 'Unclassified_Methanosarcinales'

        # Testing the validity of the otu_name() function
        self.assertEqual(
            self.result, hand_calc,
            msg='Error! The output is not as expected.'
            )

    def test_load_core_file(self):
        """
        Testing load_core_file() function of otu_calc.py

        :return: Returns OK if the test goals were achieved, otherwise
                 raises error.
        """
        temp_file1 = tempfile.NamedTemporaryFile(delete=False)
        temp_file1.write(self.coreOTUdata)
        temp_file1.seek(0)
        result = oc.load_core_file(temp_file1.name)
        length = len(result)

        # Testing if all core OTU's samples were in the output.
        self.assertEqual(
            length, 17,
            msg='Error! Output does not contain all core OTU in samples.'
        )

        # Deleting temporary file.
        os.unlink(temp_file1.name)

    # def test_assign_otu_membership(self):
    #     """
    #     Testing assign_otu_membership() function of otu_calc.py.

    #     :return: Returns OK if the test goals were achieved, otherwise
    #              raises error.
    #     """
    #     self.result = oc.assign_otu_membership(self.biom)

    #     # Obtaining the values to be tested<commit_msg>Adds additional test examples to the test for otu_name().<commit_after>#!/usr/bin/env python
"""
:Author: <NAME>

:Date Created: 10/22/2014

:Abstract: Automated Tests for OTU calculations.
"""
import unittest
import json
import os
import tempfile
import sys
from phylotoast import otu_calc as oc
from phylotoast import biom_calc as bc


class otu_calc_Test(unittest.TestCase):

    def setUp(self):
        """
        Setting up the test module. Initializing BIOM format file.
        """
        self.biom_text = """{
                     "id":null,
                     "format": "Biological Observation Matrix 0.9.1-dev",
                     "format_url": "http://biom-format.org/documentation/format_versions/biom-1.0.html",
                     "type": "OTU table",
                     "generated_by": "QIIME revision 1.4.0-dev",
                     "date": "2011-12-19T19:00:00",
                     "rows":[
                        {"id":"GG_OTU_1", "metadata":{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}},
                        {"id":"GG_OTU_2", "metadata":{"taxonomy":["k__Bacteria", "p__Cyanobacteria", "c__Nostocophycideae", "o__Nostocales", "f__Nostocaceae", "g__Dolichospermum", "s__"]}},
                        {"id":"GG_OTU_3", "metadata":{"taxonomy":["k__Archaea", "p__Euryarchaeota", "c__Methanomicrobia", "o__Methanosarcinales", "f__Methanosarcinaceae", "g__Methanosarcina", "s__"]}},
                        {"id":"GG_OTU_4", "metadata":{"taxonomy":["k__Bacteria", "p__Firmicutes", "c__Clostridia", "o__Halanaerobiales", "f__Halanaerobiaceae", "g__Halanaerobium", "s__Halanaerobiumsaccharolyticum"]}},
                        {"id":"GG_OTU_5", "metadata":{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}}
                        ],
                     "columns":[
                        {"id":"Sample1", "metadata":{
                                                 "BarcodeSequence":"CGCTTATCGAGA",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample2", "metadata":{
                                                 "BarcodeSequence":"CATACCAGTAGC",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample3", "metadata":{
                                                 "BarcodeSequence":"CTCTCTACCTGT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample4", "metadata":{
                                                 "BarcodeSequence":"CTCTCGGCCTGT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}},
                        {"id":"Sample5", "metadata":{
                                                 "BarcodeSequence":"CTCTCTACCAAT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}},
                        {"id":"Sample6", "metadata":{
                                                 "BarcodeSequence":"CTAACTACCAAT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}}
                                ],
                     "matrix_type": "sparse",
                     "matrix_element_type": "int",
                     "shape": [5, 6],
                     "data":[[0,2,1],
                             [1,0,5],
                             [1,1,1],
                             [1,3,2],
                             [1,4,3],
                             [1,5,1],
                             [2,2,1],
                             [2,3,4],
                             [2,5,2],
                             [3,0,2],
                             [3,1,1],
                             [3,2,1],
                             [3,5,1],
                             [4,1,1],
                             [4,2,1]
                            ]
                        }"""

        self.coreOTUdata = '''# Core OTUs across 75 % of samples.
064EK048	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__HOT.064']
200CZ006	[u'k__Bacteria', u'p__Fusobacteria', u'c__Fusobacteria', u'o__Fusobacteriales', u'f__Fusobacteriaceae', u'g__Fusobacterium', u's__nucleatum']
161Vparv	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Veillonellaceae', u'g__Veillonella', u's__parvula']
111_3523	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Peptostreptococcaceae_[XIII]', u'g__Parvimonas', u's__micra']
644_4671	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__intermedius']
758_3928	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__sanguinis']
524Vatyp	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Veillonellaceae', u'g__Veillonella', u's__atypica']
151_K168	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Veillonellaceae', u'g__Selenomonas', u's__sputigena']
530_5256	[u'k__Bacteria', u'p__Actinobacteria', u'c__Actinobacteria', u'o__Actinomycetales', u'f__Propionibacteriaceae', u'g__Propionibacterium', u's__acnes']
612_6662	[u'k__Bacteria', u'p__Proteobacteria', u'c__Gammaproteobacteria', u'o__Pseudomonadales', u'f__Pseudomonadaceae', u'g__Pseudomonas', u's__fluorescens']
539_6962	[u'k__Bacteria', u'p__Firmicutes', u'c__Clostridia', u'o__Clostridiales', u'f__Peptostreptococcaceae_[XI]', u'g__Filifactor', u's__alocis']
073-T1E5	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__australis']
398_7051	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__mitis']
622_3931	[u'k__Bacteria', u'p__Firmicutes', u'c__Bacilli', u'o__Lactobacillales', u'f__Streptococcaceae', u'g__Streptococcus', u's__gordonii']
826_8612	[u'k__Bacteria', u'p__Proteobacteria', u'c__Gammaproteobacteria', u'o__Pasteurellales', u'f__Pasteurellaceae', u'g__Terrahaemophilus', u's__aromaticivorans']
623_4320	[u'k__Bacteria', u'p__Proteobacteria', u'c__Epsilonproteobacteria', u'o__Campylobacterales', u'f__Campylobacteraceae', u'g__Campylobacter', u's__gracilis']
205FL002	[u'k__Bacteria', u'p__Fusobacteria', u'c__Fusobacteria', u'o__Fusobacteriales', u'f__Fusobacteriaceae', u'g__Fusobacterium', u's__HOT.205']'''

        self.biom = json.loads(self.biom_text)
        self.row = self.biom['rows']

    # def test_fuzzy_lookup(self):
    #     """
    #     Testing fuzzy_lookup() function of otu_calc.py.

    #     :return: Returns OK if the test goals were achieved, otherwise
    #             raises error.
    #     """
    #     self.result1 = oc.assign_otu_membership(self.biom)['Sample2']
    #     self.result = oc.fuzzy_lookup(self.result1, ['Escherichia_spp.'])

    #     # Obtaining manual result to be compared against.
    #     hand_calc = 'Escherichia_spp.'

    #     # Testing the validity of fuzzy_lookup() function.
    #     self.assertIn(
    #         hand_calc, str(self.result1),
    #         msg='Error! Output is not a subset of input.'
    #     )

    # def test_sdi(self):
    #     """
    #     Testing sdi() function of otu_calc.py.

    #     :return: Returns OK if the test goals were achieved, otherwise
    #              raises error.
    #     """
    #     self.result1 = oc.assign_otu_membership(self.biom)
    #     fset = self.result1['Sample4']
    #     self.result = oc.sdi(fset)

    #     # Manual calculation of SDI
    #     hand_calc = 0.636513937

    #     # Testing the validity of sdi() function
    #     self.assertAlmostEqual(
    #         self.result, hand_calc, places=6,
    #         msg='Error! SDI was calculated inacccurately.'
    #     )

    def test_otu_name_biom(self):
        """
        Testing the otu_name_biom() function of otu_calc.py.

        :return: Returns OK if the test goals were achieved, otherwise
                 raises error.
        """
        self.result = oc.otu_name_biom(self.row[3])
        hand_calc = 'Halanaerobium_Halanaerobiumsaccharolyticum'

        # Testing the validity of otu_name_biom() function
        self.assertEqual(
            self.result, hand_calc,
            msg='Error! otu_name_biom() output does not match manually identified name.'
        )

    def test_otu_name(self):
        """
        Testing the otu_name() function of otu_calc.py.

        :return: Returns OK if the test goals were achieved, otherwise
                 raises error.
        """
        self.tax = {
            "Unclassified_Methanosarcinales":
            ["k__Archaea", "p__Euryarchaeota", "c__Methanomicrobia",
            "o__Methanosarcinales", "f__",
            "g__", "s__concilii"
            ],
            "Campylobacter_gracilis":
            ["k__Bacteria", "p__Proteobacteria", "c__Epsilonproteobacteria", 
             "o__Campylobacterales", "f__Campylobacteraceae", "g__Campylobacter", 
             "s__gracilis"],
            "Escherichia_spp.":
            ["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", 
             "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", 
             "s__"]
        }

        for expected, test in self.tax.items():
            self.result = oc.otu_name(test)

            # Testing the validity of the otu_name() function
            self.assertEqual(
                self.result, expected,
                msg='Error! Expected result: {}. otu_name() result: {}'.format(
                                                         expected, self.result)
            )

        

    def test_load_core_file(self):
        """
        Testing the load_core_file() function of otu_calc.py

        :return: Returns OK if the test goals were achieved, otherwise
                 raises error.
        """
        temp_file1 = tempfile.NamedTemporaryFile(delete=False)
        temp_file1.write(self.coreOTUdata)
        temp_file1.seek(0)
        result = oc.load_core_file(temp_file1.name)
        length = len(result)

        # Testing if all core OTU's samples were in the output.
        self.assertEqual(
            length, 17,
            msg='Error! Output does not contain all core OTU in samples.'
        )

        # Deleting temporary file.
        os.unlink(temp_file1.name)

    # def test_assign_otu_membership(self):
    #     """
    #     Testing assign_otu_membership() function of otu_calc.py.

    #     :return: Returns OK if the test goals were achieved, otherwise
    #              raises error.
    #     """
    #     self.result = oc.assign_otu_membership(self.biom)

    #     # Obtaining the values to be tested
{'fraction_non_alphanumeric': 0.12129010121804769, 'fraction_numerical': 0.0195573854863613, 'mean_word_length': 1.9142607174103237, 'pattern_counts': {'":': 129, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 98, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3200733', 'n_tokens_mistral': 7887, 'n_tokens_neox': 7076, 'n_words': 1201}