# Path: rsmtool/test_utils.py
# def check_run_experiment(source,
#                          experiment_id,
#                          subgroups=None,
#                          consistency=False,
#                          skll=False,
#                          file_format='csv',
#                          given_test_dir=None,
#                          config_obj_or_dict=None,
#                          suppress_warnings_for=[]):
#     """
#     Run a parameterized rsmtool experiment test.
# 
#     Parameters
#     ----------
#     source : str
#         The name of the source directory containing the experiment
#         configuration.
#     experiment_id : str
#         The experiment ID of the experiment.
#     subgroups : list of str, optional
#         List of subgroup names used in the experiment. If specified,
#         outputs pertaining to subgroups are also checked as part of the
#         test.
#         Defaults to ``None``.
#     consistency : bool, optional
#         Whether to check consistency files as part of the experiment test.
#         Generally, this should be true if the second human score column is
#         specified.
#         Defaults to ``False``.
#     skll : bool, optional
#         Whether the model being used in the experiment is a SKLL model
#         in which case the coefficients, predictions, etc. will not be
#         checked since they can vary across machines, due to parameter tuning.
#         Defaults to ``False``.
#     file_format : str, optional
#         Which file format is being used for the output files of the experiment.
#         Defaults to "csv".
#     given_test_dir : str, optional
#         Path where the test experiments are located. Unless specified, the
#         rsmtool test directory is used. This can be useful when using these
#         experiments to run tests for RSMExtra.
#         Defaults to ``None``.
#     config_obj_or_dict: configuration_parser.Configuration or dict, optional
#         Configuration object or dictionary to use as an input, if any.
#         If ``None``, the function will construct a path to the config file
#         using ``source`` and ``experiment_id``.
#     suppress_warnings_for : list, optional
#         Categories for which warnings should be suppressed when running the
#         experiments.
#         Defaults to ``[]``.
#     """
#     # use the test directory from this file unless it's been overridden
#     test_dir = given_test_dir if given_test_dir else rsmtool_test_dir
# 
#     if config_obj_or_dict is None:
#         config_input = join(test_dir,
#                             'data',
#                             'experiments',
#                             source,
#                             '{}.json'.format(experiment_id))
#     else:
#         config_input = config_obj_or_dict
# 
#     model_type = 'skll' if skll else 'rsmtool'
# 
#     do_run_experiment(source,
#                       experiment_id,
#                       config_input,
#                       suppress_warnings_for=suppress_warnings_for)
# 
#     output_dir = join('test_outputs', source, 'output')
#     expected_output_dir = join(test_dir, 'data', 'experiments', source, 'output')
#     html_report = join('test_outputs', source, 'report', '{}_report.html'.format(experiment_id))
# 
#     output_files = glob(join(output_dir, '*.{}'.format(file_format)))
#     for output_file in output_files:
#         output_filename = basename(output_file)
#         expected_output_file = join(expected_output_dir, output_filename)
# 
#         if exists(expected_output_file):
#             check_file_output(output_file, expected_output_file, file_format=file_format)
# 
#     check_generated_output(output_files, experiment_id, model_type, file_format=file_format)
# 
#     if not skll:
#         check_scaled_coefficients(output_dir, experiment_id, file_format=file_format)
# 
#     if subgroups:
#         check_subgroup_outputs(output_dir, experiment_id, subgroups, file_format=file_format)
# 
#     if consistency:
#         check_consistency_files_exist(output_files, experiment_id, file_format=file_format)
# 
#     # check report for any errors but ignore warnings
#     # which we check below separately
#     check_report(html_report, raise_warnings=False)
# 
#     # make sure that there are no warnings in the report
#     # but ignore warnings if appropriate
#     if not IGNORE_WARNINGS:
#         warning_msgs = collect_warning_messages_from_report(html_report)
#         assert_equal(len(warning_msgs), 0)
# 
# def do_run_experiment(source,
#                       experiment_id,
#                       config_input,
#                       suppress_warnings_for=[]):
#     """
#     Run rsmtool experiment automatically.
# 
#     Use the given experiment configuration file located in the
#     given source directory and use the given experiment ID.
# 
#     Parameters
#     ----------
#     source : str
#         Path to where the test experiment is located on disk.
#     experiment_id : str
#         Experiment ID to use when running.
#     config_input : str or Configuration or dict
#         Path to the experiment configuration file,
#         or a ``configuration_parser.Configuration`` object
#         or a dictionary with keys corresponding to fields in the
#         configuration file.
#     suppress_warnings_for : list, optional
#         Categories for which warnings should be suppressed
#         when running the experiments. Note that ``RuntimeWarning``s
#         are always suppressed.
#         Defaults to ``[]``.
#     """
#     source_output_dir = 'test_outputs'
#     experiment_dir = join(source_output_dir, source)
# 
#     # remove all previously created files
#     for output_subdir in ['output', 'figure', 'report']:
#         files = glob(join(source_output_dir, source, output_subdir, '*'))
#         for f in files:
#             remove(f)
# 
#     with warnings.catch_warnings():
# 
#         # always suppress runtime warnings
#         warnings.filterwarnings('ignore', category=RuntimeWarning)
# 
#         # suppress additional warning types if specified
#         for warning_type in suppress_warnings_for:
#             warnings.filterwarnings('ignore', category=warning_type)
# 
#         run_experiment(config_input, experiment_dir)


import os
from os.path import join
from nose.tools import raises
from parameterized import param, parameterized
from rsmtool.test_utils import check_run_experiment, do_run_experiment
    from rsmtool.test_utils import rsmtool_test_dir


# allow test directory to be set via an environment variable
# which is needed for package testing
TEST_DIR = os.environ.get('TESTDIR', None)
if TEST_DIR:
    rsmtool_test_dir = TEST_DIR
else:


@parameterized([
    param('lr-with-defaults-as-extra-columns', 'lr_with_defaults_as_extra_columns',

# Path: rsmtool/test_utils.py
# def check_run_experiment(source,
#                          experiment_id,
#                          subgroups=None,
#                          consistency=False,
#                          skll=False,
#                          file_format='csv',
#                          given_test_dir=None,
#                          config_obj_or_dict=None,
#                          suppress_warnings_for=[]):
#     """
#     Run a parameterized rsmtool experiment test.
# 
#     Parameters
#     ----------
#     source : str
#         The name of the source directory containing the experiment
#         configuration.
#     experiment_id : str
#         The experiment ID of the experiment.
#     subgroups : list of str, optional
#         List of subgroup names used in the experiment. If specified,
#         outputs pertaining to subgroups are also checked as part of the
#         test.
#         Defaults to ``None``.
#     consistency : bool, optional
#         Whether to check consistency files as part of the experiment test.
#         Generally, this should be true if the second human score column is
#         specified.
#         Defaults to ``False``.
#     skll : bool, optional
#         Whether the model being used in the experiment is a SKLL model
#         in which case the coefficients, predictions, etc. will not be
#         checked since they can vary across machines, due to parameter tuning.
#         Defaults to ``False``.
#     file_format : str, optional
#         Which file format is being used for the output files of the experiment.
#         Defaults to "csv".
#     given_test_dir : str, optional
#         Path where the test experiments are located. Unless specified, the
#         rsmtool test directory is used. This can be useful when using these
#         experiments to run tests for RSMExtra.
#         Defaults to ``None``.
#     config_obj_or_dict: configuration_parser.Configuration or dict, optional
#         Configuration object or dictionary to use as an input, if any.
#         If ``None``, the function will construct a path to the config file
#         using ``source`` and ``experiment_id``.
#     suppress_warnings_for : list, optional
#         Categories for which warnings should be suppressed when running the
#         experiments.
#         Defaults to ``[]``.
#     """
#     # use the test directory from this file unless it's been overridden
#     test_dir = given_test_dir if given_test_dir else rsmtool_test_dir
# 
#     if config_obj_or_dict is None:
#         config_input = join(test_dir,
#                             'data',
#                             'experiments',
#                             source,
#                             '{}.json'.format(experiment_id))
#     else:
#         config_input = config_obj_or_dict
# 
#     model_type = 'skll' if skll else 'rsmtool'
# 
#     do_run_experiment(source,
#                       experiment_id,
#                       config_input,
#                       suppress_warnings_for=suppress_warnings_for)
# 
#     output_dir = join('test_outputs', source, 'output')
#     expected_output_dir = join(test_dir, 'data', 'experiments', source, 'output')
#     html_report = join('test_outputs', source, 'report', '{}_report.html'.format(experiment_id))
# 
#     output_files = glob(join(output_dir, '*.{}'.format(file_format)))
#     for output_file in output_files:
#         output_filename = basename(output_file)
#         expected_output_file = join(expected_output_dir, output_filename)
# 
#         if exists(expected_output_file):
#             check_file_output(output_file, expected_output_file, file_format=file_format)
# 
#     check_generated_output(output_files, experiment_id, model_type, file_format=file_format)
# 
#     if not skll:
#         check_scaled_coefficients(output_dir, experiment_id, file_format=file_format)
# 
#     if subgroups:
#         check_subgroup_outputs(output_dir, experiment_id, subgroups, file_format=file_format)
# 
#     if consistency:
#         check_consistency_files_exist(output_files, experiment_id, file_format=file_format)
# 
#     # check report for any errors but ignore warnings
#     # which we check below separately
#     check_report(html_report, raise_warnings=False)
# 
#     # make sure that there are no warnings in the report
#     # but ignore warnings if appropriate
#     if not IGNORE_WARNINGS:
#         warning_msgs = collect_warning_messages_from_report(html_report)
#         assert_equal(len(warning_msgs), 0)
# 
# def do_run_experiment(source,
#                       experiment_id,
#                       config_input,
#                       suppress_warnings_for=[]):
#     """
#     Run rsmtool experiment automatically.
# 
#     Use the given experiment configuration file located in the
#     given source directory and use the given experiment ID.
# 
#     Parameters
#     ----------
#     source : str
#         Path to where the test experiment is located on disk.
#     experiment_id : str
#         Experiment ID to use when running.
#     config_input : str or Configuration or dict
#         Path to the experiment configuration file,
#         or a ``configuration_parser.Configuration`` object
#         or a dictionary with keys corresponding to fields in the
#         configuration file.
#     suppress_warnings_for : list, optional
#         Categories for which warnings should be suppressed
#         when running the experiments. Note that ``RuntimeWarning``s
#         are always suppressed.
#         Defaults to ``[]``.
#     """
#     source_output_dir = 'test_outputs'
#     experiment_dir = join(source_output_dir, source)
# 
#     # remove all previously created files
#     for output_subdir in ['output', 'figure', 'report']:
#         files = glob(join(source_output_dir, source, output_subdir, '*'))
#         for f in files:
#             remove(f)
# 
#     with warnings.catch_warnings():
# 
#         # always suppress runtime warnings
#         warnings.filterwarnings('ignore', category=RuntimeWarning)
# 
#         # suppress additional warning types if specified
#         for warning_type in suppress_warnings_for:
#             warnings.filterwarnings('ignore', category=warning_type)
# 
#         run_experiment(config_input, experiment_dir)


# Path: tests/test_experiment_rsmtool_2.py
import os
from os.path import join
from nose.tools import raises
from parameterized import param, parameterized
from rsmtool.test_utils import check_run_experiment, do_run_experiment
    from rsmtool.test_utils import rsmtool_test_dir



# allow test directory to be set via an environment variable
# which is needed for package testing
TEST_DIR = os.environ.get('TESTDIR', None)
if TEST_DIR:
    rsmtool_test_dir = TEST_DIR
else:


@parameterized([
    param('lr-with-defaults-as-extra-columns', 'lr_with_defaults_as_extra_columns',

          consistency=True),