<commit_before>import concurrent.futures
import os
import time
import unittest

import depgraph
from depgraph import Dataset, DatasetGroup, buildmanager

TESTDIR = os.path.abspath(os.getcwd())

def makefile(fnm, content=None):
    if content is None:
        content = fnm
    with open(fnm, "w") as f:
        f.write(content)
    return

def ensureisdir(dirname):
    if not os.path.isdir(dirname):
        os.makedirs(dirname)
    return

def cleandir(dirname):
    for fnm in os.listdir(dirname):
        os.remove(os.path.join(dirname, fnm))
    return

def fullpath(p):
    return os.path.join(TESTDIR, p)

class SetterUpper(object):

    def setUp(self):
        """ define a simple dependency graph that is complex enough to be
        interesting.

         R0      R1      R2      R3         [raw data]
           \     /       |       |
             DA0         DA1    /
                 \      /  \   /
                    DB0     DB1
                     \     / |  \
                      \   /  |   \
                       DC0  DC1  DC2        [products]
        """
        raw0 = Dataset(fullpath("testdata/raw0"), prog="rawdata")
        raw1 = Dataset(fullpath("testdata/raw1"), prog="rawdata")
        raw2 = Dataset(fullpath("testdata/raw2"), prog="rawdata")
        raw3 = Dataset(fullpath("testdata/raw3"), prog="rawdata")

        da0 = Dataset(fullpath("testproject/da0"), prog="step1")
        da1 = Dataset(fullpath("testproject/da1"), prog="step2")

        db0 = Dataset(fullpath("testproject/db0"), prog="step3")
        db1 = Dataset(fullpath("testproject/db1"), prog="step4")

        dc0 = Dataset(fullpath("testproject/dc0"), prog="step5")
        dc1 = Dataset(fullpath("testproject/dc1"), prog="step6")
        dc2 = Dataset(fullpath("testproject/dc2"), prog="step7")

        da0.dependson(raw0, raw1)
        da1.dependson(raw2)
        db0.dependson(da0, da1)
        db1.dependson(da1, raw3)
        dc0.dependson(db0, db1)
        dc1.dependson(db1)
        dc2.dependson(db1)

        self.raw0 = raw0
        self.raw1 = raw1
        self.raw2 = raw2
        self.raw3 = raw3
        self.da0 = da0
        self.da1 = da1
        self.db0 = db0
        self.db1 = db1
        self.dc0 = dc0
        self.dc1 = dc1
        self.dc2 = dc2

        # initialize "raw" data
        rawdir = fullpath("testdata")
        ensureisdir(rawdir)

        for dep in (raw0, raw1, raw2, raw3):
            makefile(dep.name)
        time.sleep(0.05)
        cleandir(fullpath("testproject"))
        return

    @classmethod
    def setUpClass(cls):
        if not os.path.isdir(fullpath("testdata")):
            os.makedirs(fullpath("testdata"))
        if not os.path.isdir(fullpath("testproject")):
            os.makedirs(fullpath("testproject"))
        return

class BuildnextTests(SetterUpper, unittest.TestCase):

    def test_buildnext_one_level(self):
        tobuild = [dep for dep, reason in self.db0.buildnext()]
        self.assertTrue(self.da0 in tobuild)
        self.assertTrue(self.da1 in tobuild)
        self.assertEqual(len(tobuild), 2)
        return

    def test_buildnext_one_level_ignore(self):
        tobuild = [dep for dep, reason in self.db0.buildnext(ignore=[self.da1])]
        self.assertTrue(self.da0 in tobuild)
        self.assertTrue(self.da1 not in tobuild)
        self.assertEqual(len(tobuild), 1)
        return

    def test_buildnext_two_level(self):
        tobuild = [dep for dep, reason in self.dc0.buildnext()]

        self.assertTrue(self.da0 in tobuild)
        self.assertTrue(self.da1 in tobuild)
        self.assertEqual(len(tobuild), 2)

        for ds in tobuild:
            makefile(ds.name)

        tobuild2 = [dep for dep, reason in self.dc0.buildnext()]

        self.assertTrue(self.db1 in tobuild2)
        self.assertTrue(self.db0 in tobuild2)
        self.assertEqual(len(tobuild2), 2)
        return

class BuildManagerTests(SetterUpper, unittest.TestCase):

    def test_buildmanager_unecessary(self):
        # This should be a no-op, because target and all dependencies already
        # exist and the target is younger that any dependency. If the build
        # function is called, fail the test.

        @buildmanager
        def build(dep, reason):
            self.fail("unecessary build requested")
            return 0

        makefile(self.raw2.name)
        makefile(self.raw3.name)
        makefile(self.da1.name)
        makefile(self.db1.name)
        build(self.db1)
        return

    def test_perfect_builder(self):
        """ build manager from a delegator function that always succeeds """

        @buildmanager
        def build(dep, reason):
            makefile(dep.name)
            return 0

        out = build(self.dc0)

        self.assertTrue(os.path.isfile(self.da0.name))
        self.assertTrue(os.path.isfile(self.da1.name))
        self.assertTrue(os.path.isfile(self.db0.name))
        self.assertTrue(os.path.isfile(self.db1.name))
        return

    def test_perfect_builder_zero_attempt(self):
        """ build manager from a delegator function that always succeeds """

        @buildmanager
        def build(dep, reason):
            makefile(dep.name)
            return 0

        out = build(self.dc0, max_attempts=0)

        self.assertFalse(os.path.isfile(self.da0.name))
        self.assertFalse(os.path.isfile(self.da1.name))
        self.assertFalse(os.path.isfile(self.db0.name))
        self.assertFalse(os.path.isfile(self.db1.name))
        return

    def test_parallel_builder(self):

        def build(dep, reason):
            makefile(dep.name)
            return reason

        futures = []
        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
            for stage in depgraph.buildall(self.dc0):
                for dep, reason in stage:
                    futures.append(pool.submit(build, dep, reason))
        concurrent.futures.wait(futures)

        self.assertTrue(os.path.isfile(self.da0.name))
        self.assertTrue(os.path.isfile(self.da1.name))
        self.assertTrue(os.path.isfile(self.db0.name))
        self.assertTrue(os.path.isfile(self.db1.name))
        return

class SimpleDependencyGraphTests(unittest.TestCase):

    def setUp(self):
        """
            R0  R1  R2
             \ /    |<commit_msg>skip parallelization test when Python <3.2<commit_after>import os
import sys
import time
import unittest

import depgraph
from depgraph import Dataset, DatasetGroup, buildmanager

TESTDIR = os.path.abspath(os.getcwd())

def makefile(fnm, content=None):
    if content is None:
        content = fnm
    with open(fnm, "w") as f:
        f.write(content)
    return

def ensureisdir(dirname):
    if not os.path.isdir(dirname):
        os.makedirs(dirname)
    return

def cleandir(dirname):
    for fnm in os.listdir(dirname):
        os.remove(os.path.join(dirname, fnm))
    return

def fullpath(p):
    return os.path.join(TESTDIR, p)

class SetterUpper(object):

    def setUp(self):
        """ define a simple dependency graph that is complex enough to be
        interesting.

         R0      R1      R2      R3         [raw data]
           \     /       |       |
             DA0         DA1    /
                 \      /  \   /
                    DB0     DB1
                     \     / |  \
                      \   /  |   \
                       DC0  DC1  DC2        [products]
        """
        raw0 = Dataset(fullpath("testdata/raw0"), prog="rawdata")
        raw1 = Dataset(fullpath("testdata/raw1"), prog="rawdata")
        raw2 = Dataset(fullpath("testdata/raw2"), prog="rawdata")
        raw3 = Dataset(fullpath("testdata/raw3"), prog="rawdata")

        da0 = Dataset(fullpath("testproject/da0"), prog="step1")
        da1 = Dataset(fullpath("testproject/da1"), prog="step2")

        db0 = Dataset(fullpath("testproject/db0"), prog="step3")
        db1 = Dataset(fullpath("testproject/db1"), prog="step4")

        dc0 = Dataset(fullpath("testproject/dc0"), prog="step5")
        dc1 = Dataset(fullpath("testproject/dc1"), prog="step6")
        dc2 = Dataset(fullpath("testproject/dc2"), prog="step7")

        da0.dependson(raw0, raw1)
        da1.dependson(raw2)
        db0.dependson(da0, da1)
        db1.dependson(da1, raw3)
        dc0.dependson(db0, db1)
        dc1.dependson(db1)
        dc2.dependson(db1)

        self.raw0 = raw0
        self.raw1 = raw1
        self.raw2 = raw2
        self.raw3 = raw3
        self.da0 = da0
        self.da1 = da1
        self.db0 = db0
        self.db1 = db1
        self.dc0 = dc0
        self.dc1 = dc1
        self.dc2 = dc2

        # initialize "raw" data
        rawdir = fullpath("testdata")
        ensureisdir(rawdir)

        for dep in (raw0, raw1, raw2, raw3):
            makefile(dep.name)
        time.sleep(0.05)
        cleandir(fullpath("testproject"))
        return

    @classmethod
    def setUpClass(cls):
        if not os.path.isdir(fullpath("testdata")):
            os.makedirs(fullpath("testdata"))
        if not os.path.isdir(fullpath("testproject")):
            os.makedirs(fullpath("testproject"))
        return

class BuildnextTests(SetterUpper, unittest.TestCase):

    def test_buildnext_one_level(self):
        tobuild = [dep for dep, reason in self.db0.buildnext()]
        self.assertTrue(self.da0 in tobuild)
        self.assertTrue(self.da1 in tobuild)
        self.assertEqual(len(tobuild), 2)
        return

    def test_buildnext_one_level_ignore(self):
        tobuild = [dep for dep, reason in self.db0.buildnext(ignore=[self.da1])]
        self.assertTrue(self.da0 in tobuild)
        self.assertTrue(self.da1 not in tobuild)
        self.assertEqual(len(tobuild), 1)
        return

    def test_buildnext_two_level(self):
        tobuild = [dep for dep, reason in self.dc0.buildnext()]

        self.assertTrue(self.da0 in tobuild)
        self.assertTrue(self.da1 in tobuild)
        self.assertEqual(len(tobuild), 2)

        for ds in tobuild:
            makefile(ds.name)

        tobuild2 = [dep for dep, reason in self.dc0.buildnext()]

        self.assertTrue(self.db1 in tobuild2)
        self.assertTrue(self.db0 in tobuild2)
        self.assertEqual(len(tobuild2), 2)
        return

class BuildManagerTests(SetterUpper, unittest.TestCase):

    def test_buildmanager_unecessary(self):
        # This should be a no-op, because target and all dependencies already
        # exist and the target is younger that any dependency. If the build
        # function is called, fail the test.

        @buildmanager
        def build(dep, reason):
            self.fail("unecessary build requested")
            return 0

        makefile(self.raw2.name)
        makefile(self.raw3.name)
        makefile(self.da1.name)
        makefile(self.db1.name)
        build(self.db1)
        return

    def test_perfect_builder(self):
        """ build manager from a delegator function that always succeeds """

        @buildmanager
        def build(dep, reason):
            makefile(dep.name)
            return 0

        out = build(self.dc0)

        self.assertTrue(os.path.isfile(self.da0.name))
        self.assertTrue(os.path.isfile(self.da1.name))
        self.assertTrue(os.path.isfile(self.db0.name))
        self.assertTrue(os.path.isfile(self.db1.name))
        return

    def test_perfect_builder_zero_attempt(self):
        """ build manager from a delegator function that always succeeds """

        @buildmanager
        def build(dep, reason):
            makefile(dep.name)
            return 0

        out = build(self.dc0, max_attempts=0)

        self.assertFalse(os.path.isfile(self.da0.name))
        self.assertFalse(os.path.isfile(self.da1.name))
        self.assertFalse(os.path.isfile(self.db0.name))
        self.assertFalse(os.path.isfile(self.db1.name))
        return

    @unittest.skipIf(sys.version_info < (3, 2), "requires concurrent.futures")
    def test_parallel_builder(self):

        import concurrent.futures

        def build(dep, reason):
            makefile(dep.name)
            return reason

        futures = []
        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
            for stage in depgraph.buildall(self.dc0):
                for dep, reason in stage:
                    futures.append(pool.submit(build, dep, reason))
        concurrent.futures.wait(futures)

        self.assertTrue(os.path.isfile(self.da0.name))
        self.assertTrue(os.path.isfile(self.da1.name))
        self.assertTrue(os.path.isfile(self.db0.name))
        self.assertTrue(os.path.isfile(self.db1.name))
        return

class SimpleDependencyGraphTests(unittest.TestCase):

    def setUp(self):
        """
            R0  R1  R2
             \ /    |
{'fraction_non_alphanumeric': 0.11085866499529928, 'fraction_numerical': 0.02162331557505484, 'mean_word_length': 2.0990531682447195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5289947', 'n_tokens_mistral': 4140, 'n_tokens_neox': 4050, 'n_words': 804}