<commit_before>from __future__ import absolute_import
import unittest
import os
from toil import subprocess
from toil.wdl.wdl_interpret import InterpretWDL
from toil.wdl.wdl_compile import CompileWDL, recursive_glob, generate_docker_bashscript_file, heredoc_wdl
from toil.test import ToilTest, slow
import zipfile
import shutil

class ToilWdlIntegrationTest(ToilTest):
    """A set of test cases for toilwdl.py"""

    def setUp(self):
        """
        Initial set up of variables for the test.
        """
        self.program = os.path.abspath("src/toil/wdl/toilwdl.py")

        self.test_directory = os.path.abspath("src/toil/test/wdl/")
        self.output_dir = self._createTempDir(purpose='tempDir')

        self.encode_data = os.path.join(self.test_directory, "ENCODE_data.zip")
        self.encode_data_dir = os.path.join(self.test_directory, "ENCODE_data")

        self.wdl_data = os.path.join(self.test_directory, "wdl_templates.zip")
        self.wdl_data_dir = os.path.join(self.test_directory, "wdl_templates")

        self.gatk_data = os.path.join(self.test_directory, "GATK_data.zip")
        self.gatk_data_dir = os.path.join(self.test_directory, "GATK_data")

        # GATK tests will not run on jenkins b/c GATK.jar needs Java 7
        # and jenkins only has Java 6 (12-16-2017).
        # Set this to true to run the GATK integration tests locally.
        self.manual_integration_tests = False

        # Delete the test datasets after running the tests.
        # Jenkins requires this to not error on "untracked files".
        # Set to true if running tests locally and you don't want to
        # redownload the data each time you run the test.
        self.jenkins = True

        self.fetch_and_unzip_from_s3(filename='ENCODE_data.zip',
                                     data=self.encode_data,
                                     data_dir=self.encode_data_dir)

        self.fetch_and_unzip_from_s3(filename='wdl_templates.zip',
                                     data=self.wdl_data,
                                     data_dir=self.wdl_data_dir)

        # these tests require Java 7 (GATK.jar); jenkins has Java 6 and so must
        # be run manually as integration tests (12.16.2017)
        if self.manual_integration_tests:
            self.fetch_and_unzip_from_s3(filename='GATK_data.zip',
                                         data=self.gatk_data,
                                         data_dir=self.gatk_data_dir)

    def tearDown(self):
        """Default tearDown for unittest."""

        # automatically delete the test files
        # especially for jenkins checking if
        # there are untracked files in the repo
        if self.jenkins:
            if os.path.exists(self.gatk_data):
                os.remove(self.gatk_data)
            if os.path.exists(self.gatk_data_dir):
                shutil.rmtree(self.gatk_data_dir)

            if os.path.exists(self.wdl_data):
                os.remove(self.wdl_data)
            if os.path.exists(self.wdl_data_dir):
                shutil.rmtree(self.wdl_data_dir)

            if os.path.exists(self.encode_data):
                os.remove(self.encode_data)
            if os.path.exists(self.encode_data_dir):
                shutil.rmtree(self.encode_data_dir)

        remove_encode_output()

        unittest.TestCase.tearDown(self)

    # estimated run time 27 sec
    @slow
    def testMD5sum(self):
        '''Test if toilwdl produces the same outputs as known good outputs for WDL's
        GATK tutorial #1.'''
        wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.wdl')
        inputfile = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.input')
        json = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json')

        subprocess.check_call(['python', self.program, wdl, json, '-o', self.output_dir])
        md5sum_output = os.path.join(self.output_dir, 'md5sum.txt')
        assert os.path.exists(md5sum_output)
        with open(md5sum_output, 'r') as f:
            assert f.read() == '0844e1fd61d7b73dc53e320b9efa7a9c\n'
        os.unlink(md5sum_output)

    # estimated run time 27 sec
    @slow
    def testTut01(self):
        '''Test if toilwdl produces the same outputs as known good outputs for WDL's
        GATK tutorial #1.'''
        if self.manual_integration_tests:
            wdl = os.path.abspath("src/toil/test/wdl/wdl_templates/t01/helloHaplotypeCaller.wdl")
            json = os.path.abspath("src/toil/test/wdl/wdl_templates/t01/helloHaplotypeCaller_inputs.json")
            ref_dir = os.path.abspath("src/toil/test/wdl/wdl_templates/t01/output/")

            subprocess.check_call(['python', self.program, wdl, json, '-o', self.output_dir])

            compare_runs(self.output_dir, ref_dir)

    # estimated run time 28 sec<commit_msg>Update toilwdlTest.py<commit_after>from __future__ import absolute_import
import unittest
import os
from toil import subprocess
from toil.wdl.wdl_interpret import InterpretWDL
from toil.wdl.wdl_compile import CompileWDL, recursive_glob, generate_docker_bashscript_file, heredoc_wdl
from toil.test import ToilTest, slow
import zipfile
import shutil

class ToilWdlIntegrationTest(ToilTest):
    """A set of test cases for toilwdl.py"""

    def setUp(self):
        """
        Initial set up of variables for the test.
        """
        self.program = os.path.abspath("src/toil/wdl/toilwdl.py")

        self.test_directory = os.path.abspath("src/toil/test/wdl/")
        self.output_dir = self._createTempDir(purpose='tempDir')

        self.encode_data = os.path.join(self.test_directory, "ENCODE_data.zip")
        self.encode_data_dir = os.path.join(self.test_directory, "ENCODE_data")

        self.wdl_data = os.path.join(self.test_directory, "wdl_templates.zip")
        self.wdl_data_dir = os.path.join(self.test_directory, "wdl_templates")

        self.gatk_data = os.path.join(self.test_directory, "GATK_data.zip")
        self.gatk_data_dir = os.path.join(self.test_directory, "GATK_data")

        # GATK tests will not run on jenkins b/c GATK.jar needs Java 7
        # and jenkins only has Java 6 (12-16-2017).
        # Set this to true to run the GATK integration tests locally.
        self.manual_integration_tests = False

        # Delete the test datasets after running the tests.
        # Jenkins requires this to not error on "untracked files".
        # Set to true if running tests locally and you don't want to
        # redownload the data each time you run the test.
        self.jenkins = True

        self.fetch_and_unzip_from_s3(filename='ENCODE_data.zip',
                                     data=self.encode_data,
                                     data_dir=self.encode_data_dir)

        self.fetch_and_unzip_from_s3(filename='wdl_templates.zip',
                                     data=self.wdl_data,
                                     data_dir=self.wdl_data_dir)

        # these tests require Java 7 (GATK.jar); jenkins has Java 6 and so must
        # be run manually as integration tests (12.16.2017)
        if self.manual_integration_tests:
            self.fetch_and_unzip_from_s3(filename='GATK_data.zip',
                                         data=self.gatk_data,
                                         data_dir=self.gatk_data_dir)

    def tearDown(self):
        """Default tearDown for unittest."""

        # automatically delete the test files
        # especially for jenkins checking if
        # there are untracked files in the repo
        if self.jenkins:
            if os.path.exists(self.gatk_data):
                os.remove(self.gatk_data)
            if os.path.exists(self.gatk_data_dir):
                shutil.rmtree(self.gatk_data_dir)

            if os.path.exists(self.wdl_data):
                os.remove(self.wdl_data)
            if os.path.exists(self.wdl_data_dir):
                shutil.rmtree(self.wdl_data_dir)

            if os.path.exists(self.encode_data):
                os.remove(self.encode_data)
            if os.path.exists(self.encode_data_dir):
                shutil.rmtree(self.encode_data_dir)

        remove_encode_output()

        unittest.TestCase.tearDown(self)

    # estimated run time 27 sec
    @slow
    def testMD5sum(self):
        '''Test if toilwdl produces the same outputs as known good outputs for WDL's
        GATK tutorial #1.'''
        wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.wdl')
        inputfile = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.input')
        json = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json')

        subprocess.check_call(['python', self.program, wdl, json, '-o', self.output_dir])
        md5sum_output = os.path.join(self.output_dir, 'md5sum.txt')
        assert os.path.exists(md5sum_output)
        os.unlink(md5sum_output)

    # estimated run time 27 sec
    @slow
    def testTut01(self):
        '''Test if toilwdl produces the same outputs as known good outputs for WDL's
        GATK tutorial #1.'''
        if self.manual_integration_tests:
            wdl = os.path.abspath("src/toil/test/wdl/wdl_templates/t01/helloHaplotypeCaller.wdl")
            json = os.path.abspath("src/toil/test/wdl/wdl_templates/t01/helloHaplotypeCaller_inputs.json")
            ref_dir = os.path.abspath("src/toil/test/wdl/wdl_templates/t01/output/")

            subprocess.check_call(['python', self.program, wdl, json, '-o', self.output_dir])

            compare_runs(self.output_dir, ref_dir)

    # estimated run time 28 sec
{'fraction_non_alphanumeric': 0.0963437696995167, 'fraction_numerical': 0.01250262660222736, 'mean_word_length': 2.3167247386759584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6697645', 'n_tokens_mistral': 3206, 'n_tokens_neox': 3034, 'n_words': 628}