<commit_before>#! /usr/bin/env python
# coding=utf-8
# Copyright (c) 2019 Uber Technologies, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import logging
import os
import sys
import tempfile

import numpy as np
import pandas as pd
import yaml

from ludwig.contrib import contrib_command
from ludwig.data.postprocessing import postprocess
from ludwig.globals import LUDWIG_VERSION, set_on_master, is_on_master
from ludwig.predict import predict
from ludwig.predict import print_test_results
from ludwig.predict import save_prediction_outputs
from ludwig.predict import save_test_statistics
from ludwig.train import full_train, logger
from ludwig.utils.data_utils import generate_kfold_splits, save_json
from ludwig.utils.defaults import default_random_seed, merge_with_defaults
from ludwig.utils.print_utils import logging_level_registry
from ludwig.utils.print_utils import print_ludwig

logger = logging.getLogger(__name__)

# def experiment(
#
# ):
#     (
#         model,
#         preprocessed_data,
#         experiment_dir_name,
#         _,
#         model_definition
#     ) = full_train(
#         model_definition,
#         model_definition_file=model_definition_file,
#         data_csv=data_csv,
#         data_train_csv=data_train_csv,
#         data_validation_csv=data_validation_csv,
#         data_test_csv=data_test_csv,
#         data_hdf5=data_hdf5,
#         data_train_hdf5=data_train_hdf5,
#         data_validation_hdf5=data_validation_hdf5,
#         data_test_hdf5=data_test_hdf5,
#         train_set_metadata_json=train_set_metadata_json,
#         experiment_name=experiment_name,
#         model_name=model_name,
#         model_load_path=model_load_path,
#         model_resume_path=model_resume_path,
#         skip_save_training_description=skip_save_training_description,
#         skip_save_training_statistics=skip_save_training_statistics,
#         skip_save_model=skip_save_model,
#         skip_save_progress=skip_save_progress,
#         skip_save_log=skip_save_log,
#         skip_save_processed_input=skip_save_processed_input,
#         output_directory=output_directory,
#         should_close_session=False,
#         gpus=gpus,
#         gpu_fraction=gpu_fraction,
#         use_horovod=use_horovod,
#         random_seed=random_seed,
#         debug=debug,
#         **kwargs
#     )
#
#     (training_set,
#      validation_set,
#      test_set,
#      train_set_metadata) = preprocessed_data
#
#     if test_set is not None:
#         if model_definition['training']['eval_batch_size'] > 0:
#             batch_size = model_definition['training']['eval_batch_size']
#         else:
#             batch_size = model_definition['training']['batch_size']
#
#         # predict
#         test_results = predict(
#             test_set,
#             train_set_metadata,
#             model,
#             model_definition,
#             batch_size,
#             evaluate_performance=True,
#             gpus=gpus,
#             gpu_fraction=gpu_fraction,
#             debug=debug
#         )
#     else:
#         test_results = None
#
#     return test_results


def full_experiment(
        model_definition,
        model_definition_file=None,
        data_csv=None,
        data_train_csv=None,
        data_validation_csv=None,
        data_test_csv=None,
        data_hdf5=None,
        data_train_hdf5=None,
        data_validation_hdf5=None,
        data_test_hdf5=None,
        train_set_metadata_json=None,
        experiment_name='experiment',
        model_name='run',
        model_load_path=None,
        model_resume_path=None,
        skip_save_training_description=False,
        skip_save_training_statistics=False,
        skip_save_model=False,
        skip_save_progress=False,<commit_msg>Revert "feat: initial setup experiment() function"<commit_after>#! /usr/bin/env python
# coding=utf-8
# Copyright (c) 2019 Uber Technologies, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import logging
import os
import sys
import tempfile

import numpy as np
import pandas as pd
import yaml

from ludwig.contrib import contrib_command
from ludwig.data.postprocessing import postprocess
from ludwig.globals import LUDWIG_VERSION, set_on_master, is_on_master
from ludwig.predict import predict
from ludwig.predict import print_test_results
from ludwig.predict import save_prediction_outputs
from ludwig.predict import save_test_statistics
from ludwig.train import full_train, logger
from ludwig.utils.data_utils import generate_kfold_splits, save_json
from ludwig.utils.defaults import default_random_seed, merge_with_defaults
from ludwig.utils.print_utils import logging_level_registry
from ludwig.utils.print_utils import print_ludwig

logger = logging.getLogger(__name__)


def full_experiment(
        model_definition,
        model_definition_file=None,
        data_csv=None,
        data_train_csv=None,
        data_validation_csv=None,
        data_test_csv=None,
        data_hdf5=None,
        data_train_hdf5=None,
        data_validation_hdf5=None,
        data_test_hdf5=None,
        train_set_metadata_json=None,
        experiment_name='experiment',
        model_name='run',
        model_load_path=None,
        model_resume_path=None,
        skip_save_training_description=False,
        skip_save_training_statistics=False,
        skip_save_model=False,
        skip_save_progress=False,
{'fraction_non_alphanumeric': 0.08744295598410128, 'fraction_numerical': 0.005152362726335934, 'mean_word_length': 3.612355736591989, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2601004', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1952, 'n_words': 467}