<commit_before>import os
import shlex
import subprocess
import sys

from pathlib import Path
from tempfile import TemporaryDirectory

from nose.tools import assert_raises, eq_, ok_

from rsmtool.test_utils import (check_file_output,
                                check_generated_output,
                                check_report,
                                collect_warning_messages_from_report)

# allow test directory to be set via an environment variable
# which is needed for package testing
TEST_DIR = os.environ.get('TESTDIR', None)
if TEST_DIR:
    rsmtool_test_dir = TEST_DIR
else:
    from rsmtool.test_utils import rsmtool_test_dir


class TestToolCLI:

    @classmethod
    def setUpClass(cls):
        cls.temporary_directories = []
        cls.expected_json_dir = Path(rsmtool_test_dir) / 'data' / 'output'

        common_dir = Path(rsmtool_test_dir) / 'data' / 'experiments'
        cls.rsmtool_config_file = common_dir / 'lr' / 'lr.json'
        cls.rsmeval_config_file = common_dir / 'lr-eval' / 'lr_evaluation.json'
        cls.rsmcompare_config_file = common_dir / 'lr-self-compare' / 'rsmcompare.json'
        cls.rsmpredict_config_file = common_dir / 'lr-predict' / 'rsmpredict.json'
        cls.rsmsummarize_config_file = common_dir / 'lr-self-summary' / 'rsmsummarize.json'
        cls.expected_rsmtool_output_dir = common_dir / 'lr' / 'output'
        cls.expected_rsmeval_output_dir = common_dir / 'lr-eval' / 'output'
        cls.expected_rsmcompare_output_dir = common_dir / 'lr-self-compare' / 'output'
        cls.expected_rsmpredict_output_dir = common_dir / 'lr-predict' / 'output'
        cls.expected_rsmsummarize_output_dir = common_dir / 'lr-self-summary' / 'output'

    @classmethod
    def tearDownClass(cls):
        for tempdir in cls.temporary_directories:
            tempdir.cleanup()

    def check_no_args(self, context):
        """
        A helper method that checks running the tool with no arguments
        """

        # if the BINPATH environment variable is defined
        # use that to construct the command instead of just
        # the name; this is needed for the CI builds where
        # we do not always activate the conda environment
        binpath = os.environ.get('BINPATH', None)
        if binpath is not None:
            cmd = f"{binpath}/{context}"
        else:
            cmd = f"{context}"

        proc = subprocess.run(shlex.split(cmd, posix='win' not in sys.platform),
                              check=False,
                              stderr=subprocess.DEVNULL,
                              stdout=subprocess.PIPE)
        eq_(proc.returncode, 0)
        ok_(b'usage: ' + bytes(context, encoding="utf-8") in proc.stdout)

    def test_no_args(self):
        # test that the tool without any arguments prints help messag

        # this applies to all tools
        for context in ['rsmtool', 'rsmeval', 'rsmsummarize',
                        'rsmpredict', 'rsmcompare']:
            yield self.check_no_args, context

    def validate_run_output(self, name, experiment_dir):
        """
        A helper method that validates that the output of the "run"
        subcommand for the ``name`` tool as stored in ``experiment_dir``
        is as expected.

        This is heavily inspired by ``rsmtool.test_utils.check_run_*()``
        functions.

        Parameters
        ----------
        name : str
            The name of the tool being tested.
        experiment_dir : str
            Path to rsmtool output directory.
        """
        expected_output_dir = getattr(self, f"expected_{name}_output_dir")

        # all tools except rsmcompare need to have their output files validated
        if name in ['rsmtool', 'rsmeval', 'rsmsummarize', 'rsmpredict']:

            # rsmpredict has its own set of files and it puts them right at the root
            # of the output directory rather than under the "output" subdirectory
            if name == 'rsmpredict':
                output_dir = Path(experiment_dir)
                output_files = [output_dir / 'predictions_with_metadata.csv']
            else:
                output_dir = Path(experiment_dir) / 'output'
                output_files = list(output_dir.glob('*.csv'))

            for output_file in output_files:
                output_filename = output_file.name
                expected_output_file = expected_output_dir / output_filename

                if expected_output_file.exists():
                    check_file_output(str(output_file), str(expected_output_file))

            # we need to do an extra check for rsmtool
            if name == 'rsmtool':
                check_generated_output(list(map(str, output_files)), 'lr', 'rsmtool')

        # there's no report for rsmpredict but for the rest we want
        # the reports to be free of warnings except deprecation warnings
        # that might come from underlying packages
        if name != 'rsmpredict':
            report_dir = Path(experiment_dir) / 'report'
            html_report = list(report_dir.glob('*_report.html'))[0]
            check_report(str(html_report), raise_warnings=False)

            warning_msgs = collect_warning_messages_from_report(html_report)
            warning_msgs = [msg for msg in warning_msgs if 'DeprecationWarning' not in msg]
            eq_(len(warning_msgs), 0)

    def validate_generate_output(self, name, output, subgroups=False):
        """
        A helper method that validates that the ``output`` of the ``name`` tool
        as output by the "generate" subcommand is as expected.

        Parameters
        ----------
        name : str
            The name of the tool being tested.
        output : str
            The output of the "generate" subcommand from ``name`` tool
        subgroups : bool, optional
            If ``True``, the ``--subgroups`` was added to the "generate" command
            for ``name``.
            Defaults to ``False``.<commit_msg>Fix report dir path in CLI tests.<commit_after>import os
import shlex
import subprocess
import sys

from pathlib import Path
from tempfile import TemporaryDirectory

from nose.tools import assert_raises, eq_, ok_

from rsmtool.test_utils import (check_file_output,
                                check_generated_output,
                                check_report,
                                collect_warning_messages_from_report)

# allow test directory to be set via an environment variable
# which is needed for package testing
TEST_DIR = os.environ.get('TESTDIR', None)
if TEST_DIR:
    rsmtool_test_dir = TEST_DIR
else:
    from rsmtool.test_utils import rsmtool_test_dir


class TestToolCLI:

    @classmethod
    def setUpClass(cls):
        cls.temporary_directories = []
        cls.expected_json_dir = Path(rsmtool_test_dir) / 'data' / 'output'

        common_dir = Path(rsmtool_test_dir) / 'data' / 'experiments'
        cls.rsmtool_config_file = common_dir / 'lr' / 'lr.json'
        cls.rsmeval_config_file = common_dir / 'lr-eval' / 'lr_evaluation.json'
        cls.rsmcompare_config_file = common_dir / 'lr-self-compare' / 'rsmcompare.json'
        cls.rsmpredict_config_file = common_dir / 'lr-predict' / 'rsmpredict.json'
        cls.rsmsummarize_config_file = common_dir / 'lr-self-summary' / 'rsmsummarize.json'
        cls.expected_rsmtool_output_dir = common_dir / 'lr' / 'output'
        cls.expected_rsmeval_output_dir = common_dir / 'lr-eval' / 'output'
        cls.expected_rsmcompare_output_dir = common_dir / 'lr-self-compare' / 'output'
        cls.expected_rsmpredict_output_dir = common_dir / 'lr-predict' / 'output'
        cls.expected_rsmsummarize_output_dir = common_dir / 'lr-self-summary' / 'output'

    @classmethod
    def tearDownClass(cls):
        for tempdir in cls.temporary_directories:
            tempdir.cleanup()

    def check_no_args(self, context):
        """
        A helper method that checks running the tool with no arguments
        """

        # if the BINPATH environment variable is defined
        # use that to construct the command instead of just
        # the name; this is needed for the CI builds where
        # we do not always activate the conda environment
        binpath = os.environ.get('BINPATH', None)
        if binpath is not None:
            cmd = f"{binpath}/{context}"
        else:
            cmd = f"{context}"

        proc = subprocess.run(shlex.split(cmd, posix='win' not in sys.platform),
                              check=False,
                              stderr=subprocess.DEVNULL,
                              stdout=subprocess.PIPE)
        eq_(proc.returncode, 0)
        ok_(b'usage: ' + bytes(context, encoding="utf-8") in proc.stdout)

    def test_no_args(self):
        # test that the tool without any arguments prints help messag

        # this applies to all tools
        for context in ['rsmtool', 'rsmeval', 'rsmsummarize',
                        'rsmpredict', 'rsmcompare']:
            yield self.check_no_args, context

    def validate_run_output(self, name, experiment_dir):
        """
        A helper method that validates that the output of the "run"
        subcommand for the ``name`` tool as stored in ``experiment_dir``
        is as expected.

        This is heavily inspired by ``rsmtool.test_utils.check_run_*()``
        functions.

        Parameters
        ----------
        name : str
            The name of the tool being tested.
        experiment_dir : str
            Path to rsmtool output directory.
        """
        expected_output_dir = getattr(self, f"expected_{name}_output_dir")

        # all tools except rsmcompare need to have their output files validated
        if name in ['rsmtool', 'rsmeval', 'rsmsummarize', 'rsmpredict']:

            # rsmpredict has its own set of files and it puts them right at the root
            # of the output directory rather than under the "output" subdirectory
            if name == 'rsmpredict':
                output_dir = Path(experiment_dir)
                output_files = [output_dir / 'predictions_with_metadata.csv']
            else:
                output_dir = Path(experiment_dir) / 'output'
                output_files = list(output_dir.glob('*.csv'))

            for output_file in output_files:
                output_filename = output_file.name
                expected_output_file = expected_output_dir / output_filename

                if expected_output_file.exists():
                    check_file_output(str(output_file), str(expected_output_file))

            # we need to do an extra check for rsmtool
            if name == 'rsmtool':
                check_generated_output(list(map(str, output_files)), 'lr', 'rsmtool')

        # there's no report for rsmpredict but for the rest we want
        # the reports to be free of warnings except deprecation warnings
        # that might come from underlying packages
        if name != 'rsmpredict':
            output_dir = Path(experiment_dir)
            report_dir = output_dir / "report" if name != "rscompare" else output_dir
            html_report = list(report_dir.glob('*_report.html'))[0]
            check_report(str(html_report), raise_warnings=False)

            warning_msgs = collect_warning_messages_from_report(html_report)
            warning_msgs = [msg for msg in warning_msgs if 'DeprecationWarning' not in msg]
            eq_(len(warning_msgs), 0)

    def validate_generate_output(self, name, output, subgroups=False):
        """
        A helper method that validates that the ``output`` of the ``name`` tool
        as output by the "generate" subcommand is as expected.

        Parameters
        ----------
        name : str
            The name of the tool being tested.
        output : str
            The output of the "generate" subcommand from ``name`` tool
        subgroups : bool, optional
            If ``True``, the ``--subgroups`` was added to the "generate" command
            for ``name``.
            Defaults to ``False``.
{'fraction_non_alphanumeric': 0.08561672357791289, 'fraction_numerical': 0.0006662780044973765, 'mean_word_length': 2.2270894920720234, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3902307', 'n_tokens_mistral': 3410, 'n_tokens_neox': 3327, 'n_words': 974}