<commit_before>#!/usr/bin/env python

# ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from __future__ import division
from future.builtins import zip

from future.utils.six import StringIO
from tempfile import TemporaryFile
from unittest import TestCase, main

import numpy as np

from skbio.core.distance import randdm, DissimilarityMatrix, DistanceMatrix
from skbio.core.exception import (DissimilarityMatrixError,
                                  DistanceMatrixError,
                                  DissimilarityMatrixFormatError,
                                  MissingIDError)
from skbio.util.testing import get_data_path


class DissimilarityMatrixTestData(TestCase):
    """Test data used in DissimilarityMatrix and subclass unit tests."""

    def setUp(self):
        self.bad_dm_fp = get_data_path('bad_dm.txt')
        self.dm_2x2_asym_fp = get_data_path('dm_2x2_asym.txt')
        self.dm_3x3_fp = get_data_path('dm_3x3.txt')

        fd = open(self.bad_dm_fp, 'U')
        self.bad_dm_f2_lines = ''.join(fd.readlines())
        fd.close()
        fd = open(self.dm_2x2_asym_fp, 'U')
        self.dm_2x2_asym_lines = ''.join(fd.readlines())
        fd.close()
        fd = open(self.dm_3x3_fp, 'U')
        self.dm_3x3_lines = ''.join(fd.readlines())
        fd.close()

        self.dm_1x1_data = [[0.0]]
        self.dm_1x1_f = StringIO(DM_1x1_F)

        self.dm_2x2_data = [[0.0, 0.123], [0.123, 0.0]]
        self.dm_2x2_f = StringIO(DM_2x2_F)

        self.dm_2x2_asym_data = [[0.0, 1.0], [-2.0, 0.0]]
        self.dm_2x2_asym_f = StringIO(self.dm_2x2_asym_lines)

        self.dm_3x3_data = [[0.0, 0.01, 4.2], [0.01, 0.0, 12.0],
                            [4.2, 12.0, 0.0]]
        self.dm_3x3_f = StringIO(self.dm_3x3_lines)
        self.dm_3x3_whitespace_f = StringIO('\n'.join(DM_3x3_WHITESPACE_F))

        self.tmp_f = TemporaryFile(prefix='skbio.core.tests.test_distance',
                                   suffix='.txt')

        self.bad_dm_f1 = StringIO(BAD_DM_F1)
        self.bad_dm_f2 = StringIO(self.bad_dm_f2_lines)
        self.bad_dm_f3 = StringIO(BAD_DM_F3)
        self.bad_dm_f4 = StringIO(BAD_DM_F4)
        self.bad_dm_f5 = StringIO(BAD_DM_F5)
        self.bad_dm_f6 = StringIO(BAD_DM_F6)

    def tearDown(self):
        """Delete any temporary files."""
        self.tmp_f.close()


class DissimilarityMatrixTests(DissimilarityMatrixTestData):
    def setUp(self):
        super(DissimilarityMatrixTests, self).setUp()

        self.dm_1x1 = DissimilarityMatrix(self.dm_1x1_data, ['a'])
        self.dm_2x2 = DissimilarityMatrix(self.dm_2x2_data, ['a', 'b'])
        self.dm_2x2_asym = DissimilarityMatrix(self.dm_2x2_asym_data,
                                               ['a', 'b'])
        self.dm_3x3 = DissimilarityMatrix(self.dm_3x3_data, ['a', 'b', 'c'])

        self.dms = [self.dm_1x1, self.dm_2x2, self.dm_2x2_asym, self.dm_3x3]
        self.dm_f_lines = [DM_1x1_F, DM_2x2_F, self.dm_2x2_asym_lines,
                           self.dm_3x3_lines]
        self.dm_fs = [self.dm_1x1_f, self.dm_2x2_f, self.dm_2x2_asym_f,
                      self.dm_3x3_f]
        self.dm_shapes = [(1, 1), (2, 2), (2, 2), (3, 3)]
        self.dm_sizes = [1, 4, 4, 9]
        self.dm_transposes = [
            self.dm_1x1, self.dm_2x2,
            DissimilarityMatrix([[0, -2], [1, 0]], ['a', 'b']), self.dm_3x3]
        self.dm_redundant_forms = [np.array(self.dm_1x1_data),
                                   np.array(self.dm_2x2_data),
                                   np.array(self.dm_2x2_asym_data),
                                   np.array(self.dm_3x3_data)]

    def test_round_trip_read_write(self):
        """Test reading, writing, and reading again works as expected."""
        for dm_f in self.dm_fs:
            # Read.
            dm1 = DissimilarityMatrix.from_file(dm_f)

            # Write.
            out_f = StringIO()
            dm1.to_file(out_f)
            out_f.seek(0)

            # Read.
            dm2 = DissimilarityMatrix.from_file(out_f)
            self.assertEqual(dm1, dm2)

    def test_from_file(self):
        """Should parse and return a valid DissimilarityMatrix given a file."""
        for dm_f, dm in zip(self.dm_fs, self.dms):
            obs = DissimilarityMatrix.from_file(dm_f)
            self.assertEqual(obs, dm)

    def test_from_file_with_file_path(self):
        """Should identify the filepath correctly and parse from it."""

        # should fail with the expected exception
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_fp)

        obs = DissimilarityMatrix.from_file(self.dm_2x2_asym_fp)
        self.assertEqual(self.dm_2x2_asym, obs)
        self.assertTrue(isinstance(obs, DissimilarityMatrix))

        obs = DissimilarityMatrix.from_file(self.dm_3x3_fp)
        self.assertEqual(self.dm_3x3, obs)
        self.assertTrue(isinstance(obs, DissimilarityMatrix))

    def test_from_file_extra_junk(self):
        """Should correctly parse a file with extra whitespace and comments."""
        obs = DissimilarityMatrix.from_file(self.dm_3x3_whitespace_f)
        self.assertEqual(obs, self.dm_3x3)

    def test_from_file_list_of_strings(self):
        """Should correctly parse a list of strings."""
        obs = DissimilarityMatrix.from_file(DM_3x3_WHITESPACE_F)
        self.assertEqual(obs, self.dm_3x3)

    def test_from_file_real_file(self):
        """Should correctly parse a real on-disk file."""
        self.tmp_f.write('\n'.join(DM_3x3_WHITESPACE_F))
        self.tmp_f.seek(0)

        obs = DissimilarityMatrix.from_file(self.tmp_f)
        self.assertEqual(obs, self.dm_3x3)

    def test_from_file_invalid_input(self):
        """Raises error on ill-formatted dissimilarity matrix file."""
        # Empty dm.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file([])

        # Number of values don't match number of IDs.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_f1)

        # Mismatched IDs.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_f2)

        # Extra data at end.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_f3)

        # Missing data.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_f4)
<commit_msg>ENH/PY3: Create temp file only when needed, and open it as text (can't write a str into a file open in binary mode in py3)<commit_after>#!/usr/bin/env python

# ----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from __future__ import division
from future.builtins import zip

from future.utils.six import StringIO
from tempfile import TemporaryFile
from unittest import TestCase, main

import numpy as np

from skbio.core.distance import randdm, DissimilarityMatrix, DistanceMatrix
from skbio.core.exception import (DissimilarityMatrixError,
                                  DistanceMatrixError,
                                  DissimilarityMatrixFormatError,
                                  MissingIDError)
from skbio.util.testing import get_data_path


class DissimilarityMatrixTestData(TestCase):
    """Test data used in DissimilarityMatrix and subclass unit tests."""

    def setUp(self):
        self.bad_dm_fp = get_data_path('bad_dm.txt')
        self.dm_2x2_asym_fp = get_data_path('dm_2x2_asym.txt')
        self.dm_3x3_fp = get_data_path('dm_3x3.txt')

        fd = open(self.bad_dm_fp, 'U')
        self.bad_dm_f2_lines = ''.join(fd.readlines())
        fd.close()
        fd = open(self.dm_2x2_asym_fp, 'U')
        self.dm_2x2_asym_lines = ''.join(fd.readlines())
        fd.close()
        fd = open(self.dm_3x3_fp, 'U')
        self.dm_3x3_lines = ''.join(fd.readlines())
        fd.close()

        self.dm_1x1_data = [[0.0]]
        self.dm_1x1_f = StringIO(DM_1x1_F)

        self.dm_2x2_data = [[0.0, 0.123], [0.123, 0.0]]
        self.dm_2x2_f = StringIO(DM_2x2_F)

        self.dm_2x2_asym_data = [[0.0, 1.0], [-2.0, 0.0]]
        self.dm_2x2_asym_f = StringIO(self.dm_2x2_asym_lines)

        self.dm_3x3_data = [[0.0, 0.01, 4.2], [0.01, 0.0, 12.0],
                            [4.2, 12.0, 0.0]]
        self.dm_3x3_f = StringIO(self.dm_3x3_lines)
        self.dm_3x3_whitespace_f = StringIO('\n'.join(DM_3x3_WHITESPACE_F))

        self.bad_dm_f1 = StringIO(BAD_DM_F1)
        self.bad_dm_f2 = StringIO(self.bad_dm_f2_lines)
        self.bad_dm_f3 = StringIO(BAD_DM_F3)
        self.bad_dm_f4 = StringIO(BAD_DM_F4)
        self.bad_dm_f5 = StringIO(BAD_DM_F5)
        self.bad_dm_f6 = StringIO(BAD_DM_F6)


class DissimilarityMatrixTests(DissimilarityMatrixTestData):
    def setUp(self):
        super(DissimilarityMatrixTests, self).setUp()

        self.dm_1x1 = DissimilarityMatrix(self.dm_1x1_data, ['a'])
        self.dm_2x2 = DissimilarityMatrix(self.dm_2x2_data, ['a', 'b'])
        self.dm_2x2_asym = DissimilarityMatrix(self.dm_2x2_asym_data,
                                               ['a', 'b'])
        self.dm_3x3 = DissimilarityMatrix(self.dm_3x3_data, ['a', 'b', 'c'])

        self.dms = [self.dm_1x1, self.dm_2x2, self.dm_2x2_asym, self.dm_3x3]
        self.dm_f_lines = [DM_1x1_F, DM_2x2_F, self.dm_2x2_asym_lines,
                           self.dm_3x3_lines]
        self.dm_fs = [self.dm_1x1_f, self.dm_2x2_f, self.dm_2x2_asym_f,
                      self.dm_3x3_f]
        self.dm_shapes = [(1, 1), (2, 2), (2, 2), (3, 3)]
        self.dm_sizes = [1, 4, 4, 9]
        self.dm_transposes = [
            self.dm_1x1, self.dm_2x2,
            DissimilarityMatrix([[0, -2], [1, 0]], ['a', 'b']), self.dm_3x3]
        self.dm_redundant_forms = [np.array(self.dm_1x1_data),
                                   np.array(self.dm_2x2_data),
                                   np.array(self.dm_2x2_asym_data),
                                   np.array(self.dm_3x3_data)]

    def test_round_trip_read_write(self):
        """Test reading, writing, and reading again works as expected."""
        for dm_f in self.dm_fs:
            # Read.
            dm1 = DissimilarityMatrix.from_file(dm_f)

            # Write.
            out_f = StringIO()
            dm1.to_file(out_f)
            out_f.seek(0)

            # Read.
            dm2 = DissimilarityMatrix.from_file(out_f)
            self.assertEqual(dm1, dm2)

    def test_from_file(self):
        """Should parse and return a valid DissimilarityMatrix given a file."""
        for dm_f, dm in zip(self.dm_fs, self.dms):
            obs = DissimilarityMatrix.from_file(dm_f)
            self.assertEqual(obs, dm)

    def test_from_file_with_file_path(self):
        """Should identify the filepath correctly and parse from it."""

        # should fail with the expected exception
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_fp)

        obs = DissimilarityMatrix.from_file(self.dm_2x2_asym_fp)
        self.assertEqual(self.dm_2x2_asym, obs)
        self.assertTrue(isinstance(obs, DissimilarityMatrix))

        obs = DissimilarityMatrix.from_file(self.dm_3x3_fp)
        self.assertEqual(self.dm_3x3, obs)
        self.assertTrue(isinstance(obs, DissimilarityMatrix))

    def test_from_file_extra_junk(self):
        """Should correctly parse a file with extra whitespace and comments."""
        obs = DissimilarityMatrix.from_file(self.dm_3x3_whitespace_f)
        self.assertEqual(obs, self.dm_3x3)

    def test_from_file_list_of_strings(self):
        """Should correctly parse a list of strings."""
        obs = DissimilarityMatrix.from_file(DM_3x3_WHITESPACE_F)
        self.assertEqual(obs, self.dm_3x3)

    def test_from_file_real_file(self):
        """Should correctly parse a real on-disk file."""
        with TemporaryFile(mode='r+', prefix='skbio.core.tests.test_distance',
                           suffix='.txt') as fh:
            fh.write('\n'.join(DM_3x3_WHITESPACE_F))
            fh.seek(0)

            obs = DissimilarityMatrix.from_file(fh)
        self.assertEqual(obs, self.dm_3x3)

    def test_from_file_invalid_input(self):
        """Raises error on ill-formatted dissimilarity matrix file."""
        # Empty dm.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file([])

        # Number of values don't match number of IDs.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_f1)

        # Mismatched IDs.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_f2)

        # Extra data at end.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_f3)

        # Missing data.
        with self.assertRaises(DissimilarityMatrixFormatError):
            _ = DissimilarityMatrix.from_file(self.bad_dm_f4)

{'fraction_non_alphanumeric': 0.12565255220417634, 'fraction_numerical': 0.03001740139211137, 'mean_word_length': 2.268483412322275, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3290496', 'n_tokens_mistral': 5012, 'n_tokens_neox': 4775, 'n_words': 812}