<commit_before>import os
import unittest

from cStringIO \
    import \
        StringIO

import simplejson as json

import nose.tools

from bento.core.package \
    import \
        PackageDescription
from bento.installed_package_description \
    import \
        InstalledPkgDescription, InstalledSection, ipkg_meta_from_pkg

import bento.tests.bentos

# FIXME: use correct install path instead of python package hack
BENTOS_DIR = os.path.dirname(bento.tests.bentos.__file__)
SPHINX_META = os.path.join(BENTOS_DIR, "sphinx_meta.info")

SPHINX_META_PKG = PackageDescription.from_file(SPHINX_META)

class TestInstalledSection(unittest.TestCase):
    def test_simple(self):
        files = [("scripts/foo.py", "scripts/foo"),
                 ("scripts/bar.py", "scripts/bar.py")]
        section = InstalledSection("pythonfiles", "section1", "source", "target", files)

    def test_from_source_target(self):
        files = [("scripts/foo.py", "scripts/foo.py"),
                 ("scripts/bar.py", "scripts/bar.py")]
        r_section = InstalledSection("pythonfiles", "section1", "source", "target", files)

        files = ["scripts/foo.py", "scripts/bar.py"]
        section = InstalledSection.from_source_target_directories("pythonfiles",
                        "section1", "source", "target", files)

        nose.tools.assert_equal(r_section.files, section.files)

class TestIPKG(unittest.TestCase):
    def setUp(self):
        files = ["scripts/foo.py", "scripts/bar.py"]
        section = InstalledSection.from_source_target_directories("pythonfiles",
                        "section1", "source", "target", files)
        self.sections = {"pythonfiles": {"section1": section}}

        self.meta = ipkg_meta_from_pkg(SPHINX_META_PKG)

    def test_simple_create(self):
        ipkg = InstalledPkgDescription(self.sections, self.meta, {})

    def test_simple_roundtrip(self):
        # FIXME: we compare the loaded json to avoid dealing with encoding
        # differences when comparing objects, but this is kinda stupid
        r_ipkg = InstalledPkgDescription(self.sections, self.meta, {})
        f = StringIO()
        r_ipkg._write(f)
        r_s = f.getvalue()

        ipkg = InstalledPkgDescription.from_string(r_s)
        f = StringIO()
        ipkg._write(f)
        s = f.getvalue()
        
        self.assertEqual(json.loads(r_s), json.loads(s))
<commit_msg>TST: add simple test for iter_files.<commit_after>import os
import unittest

from cStringIO \
    import \
        StringIO

import simplejson as json

import nose.tools

from bento.core.package \
    import \
        PackageDescription
from bento.installed_package_description \
    import \
        InstalledPkgDescription, InstalledSection, ipkg_meta_from_pkg, iter_files

import bento.tests.bentos

# FIXME: use correct install path instead of python package hack
BENTOS_DIR = os.path.dirname(bento.tests.bentos.__file__)
SPHINX_META = os.path.join(BENTOS_DIR, "sphinx_meta.info")

SPHINX_META_PKG = PackageDescription.from_file(SPHINX_META)

class TestInstalledSection(unittest.TestCase):
    def test_simple(self):
        files = [("scripts/foo.py", "scripts/foo"),
                 ("scripts/bar.py", "scripts/bar.py")]
        section = InstalledSection("pythonfiles", "section1", "source", "target", files)

    def test_from_source_target(self):
        files = [("scripts/foo.py", "scripts/foo.py"),
                 ("scripts/bar.py", "scripts/bar.py")]
        r_section = InstalledSection("pythonfiles", "section1", "source", "target", files)

        files = ["scripts/foo.py", "scripts/bar.py"]
        section = InstalledSection.from_source_target_directories("pythonfiles",
                        "section1", "source", "target", files)

        nose.tools.assert_equal(r_section.files, section.files)

def create_simple_ipkg_args():
    files = ["scripts/foo.py", "scripts/bar.py"]
    section = InstalledSection.from_source_target_directories("pythonfiles",
                    "section1", "source", "target", files)
    sections = {"pythonfiles": {"section1": section}}

    meta = ipkg_meta_from_pkg(SPHINX_META_PKG)
    return meta, sections

class TestIPKG(unittest.TestCase):
    def setUp(self):
        self.meta, self.sections = create_simple_ipkg_args()

    def test_simple_create(self):
        ipkg = InstalledPkgDescription(self.sections, self.meta, {})

    def test_simple_roundtrip(self):
        # FIXME: we compare the loaded json to avoid dealing with encoding
        # differences when comparing objects, but this is kinda stupid
        r_ipkg = InstalledPkgDescription(self.sections, self.meta, {})
        f = StringIO()
        r_ipkg._write(f)
        r_s = f.getvalue()

        ipkg = InstalledPkgDescription.from_string(r_s)
        f = StringIO()
        ipkg._write(f)
        s = f.getvalue()
        
        self.assertEqual(json.loads(r_s), json.loads(s))

class TestIterFiles(unittest.TestCase):
    def setUp(self):
        self.meta, self.sections = create_simple_ipkg_args()

    def test_simple(self):
        ipkg = InstalledPkgDescription(self.sections, self.meta, {})
        sections = ipkg.resolve_paths()
        res = sorted([(kind, source, target) for kind, source, target in iter_files(sections)])
        ref = [("pythonfiles", "source/scripts/bar.py", "target/scripts/bar.py"),
               ("pythonfiles", "source/scripts/foo.py", "target/scripts/foo.py")]
        self.assertEqual(res, ref)

{'fraction_non_alphanumeric': 0.11934005499541706, 'fraction_numerical': 0.0018331805682859762, 'mean_word_length': 3.136467020470053, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7523288', 'n_tokens_mistral': 1695, 'n_tokens_neox': 1678, 'n_words': 337}