<commit_before>################################################################################
#
# Package   : AlphaPy
# Module    : data
# Created   : July 11, 2013
#
# Copyright 2017 ScottFree Analytics LLC
# <NAME> & <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################


#
# Imports
#

from alphapy.frame import Frame
from alphapy.frame import frame_name
from alphapy.frame import read_frame
from alphapy.globals import ModelType
from alphapy.globals import Partition, datasets
from alphapy.globals import PSEP, SSEP
from alphapy.globals import SamplingMethod
from alphapy.globals import WILDCARD

from datetime import datetime
from datetime import timedelta
from imblearn.combine import SMOTEENN
from imblearn.combine import SMOTETomek
from imblearn.ensemble import BalanceCascade
from imblearn.ensemble import EasyEnsemble
from imblearn.over_sampling import RandomOverSampler
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import ClusterCentroids
from imblearn.under_sampling import CondensedNearestNeighbour
from imblearn.under_sampling import EditedNearestNeighbours
from imblearn.under_sampling import InstanceHardnessThreshold
from imblearn.under_sampling import NearMiss
from imblearn.under_sampling import NeighbourhoodCleaningRule
from imblearn.under_sampling import OneSidedSelection
from imblearn.under_sampling import RandomUnderSampler
from imblearn.under_sampling import RepeatedEditedNearestNeighbours
from imblearn.under_sampling import TomekLinks
import logging
import numpy as np
import pandas as pd
import pandas_datareader.data as web
import re
import requests
from scipy import sparse
from sklearn.preprocessing import LabelEncoder


#
# Initialize logger
#

logger = logging.getLogger(__name__)


#
# Function get_data
#

def get_data(model, partition):
    r"""Get data for the given partition.

    Parameters
    ----------
    model : alphapy.Model
        The model object describing the data.
    partition : alphapy.Partition
        Reference to the dataset.

    Returns
    -------
    X : pandas.DataFrame
        The feature set.
    y : pandas.Series
        The array of target values, if available.

    """

    logger.info("Loading Data")

    # Extract the model data

    directory = model.specs['directory']
    extension = model.specs['extension']
    features = model.specs['features']
    model_type = model.specs['model_type']
    separator = model.specs['separator']
    target = model.specs['target']
    test_file = model.test_file
    train_file = model.train_file

    # Read in the file

    filename = datasets[partition]
    input_dir = SSEP.join([directory, 'input'])
    df = read_frame(input_dir, filename, extension, separator)

    # Assign target and drop it if necessary

    y = np.empty([0, 0])
    if target in df.columns:
        logger.info("Found target %s in data frame", target)
        # drop rows with NaN targets
        original_size = df.shape[0]
        df.dropna(subset=[target], inplace=True)
        diff = original_size - df.shape[0]
        if diff > 0:
            logger.info("Found %d records with NaN target values", diff)
            logger.info("Labels (y) for %s will not be used", partition)
        else:
            # assign the target column to y
            y = df[target]
            # encode label only for classification
            if model_type == ModelType.classification:<commit_msg>don't drop rows with NaN targets<commit_after>################################################################################
#
# Package   : AlphaPy
# Module    : data
# Created   : July 11, 2013
#
# Copyright 2017 ScottFree Analytics LLC
# <NAME> & <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################


#
# Imports
#

from alphapy.frame import Frame
from alphapy.frame import frame_name
from alphapy.frame import read_frame
from alphapy.globals import ModelType
from alphapy.globals import Partition, datasets
from alphapy.globals import PSEP, SSEP
from alphapy.globals import SamplingMethod
from alphapy.globals import WILDCARD

from datetime import datetime
from datetime import timedelta
from imblearn.combine import SMOTEENN
from imblearn.combine import SMOTETomek
from imblearn.ensemble import BalanceCascade
from imblearn.ensemble import EasyEnsemble
from imblearn.over_sampling import RandomOverSampler
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import ClusterCentroids
from imblearn.under_sampling import CondensedNearestNeighbour
from imblearn.under_sampling import EditedNearestNeighbours
from imblearn.under_sampling import InstanceHardnessThreshold
from imblearn.under_sampling import NearMiss
from imblearn.under_sampling import NeighbourhoodCleaningRule
from imblearn.under_sampling import OneSidedSelection
from imblearn.under_sampling import RandomUnderSampler
from imblearn.under_sampling import RepeatedEditedNearestNeighbours
from imblearn.under_sampling import TomekLinks
import logging
import numpy as np
import pandas as pd
import pandas_datareader.data as web
import re
import requests
from scipy import sparse
from sklearn.preprocessing import LabelEncoder


#
# Initialize logger
#

logger = logging.getLogger(__name__)


#
# Function get_data
#

def get_data(model, partition):
    r"""Get data for the given partition.

    Parameters
    ----------
    model : alphapy.Model
        The model object describing the data.
    partition : alphapy.Partition
        Reference to the dataset.

    Returns
    -------
    X : pandas.DataFrame
        The feature set.
    y : pandas.Series
        The array of target values, if available.

    """

    logger.info("Loading Data")

    # Extract the model data

    directory = model.specs['directory']
    extension = model.specs['extension']
    features = model.specs['features']
    model_type = model.specs['model_type']
    separator = model.specs['separator']
    target = model.specs['target']
    test_file = model.test_file
    train_file = model.train_file

    # Read in the file

    filename = datasets[partition]
    input_dir = SSEP.join([directory, 'input'])
    df = read_frame(input_dir, filename, extension, separator)

    # Assign target and drop it if necessary

    y = np.empty([0, 0])
    if target in df.columns:
        logger.info("Found target %s in data frame", target)
        # check if target column has NaN values
        nan_count = df[target].isnull().sum()
        if nan_count > 0:
            logger.info("Found %d records with NaN target values", nan_count)
            logger.info("Labels (y) for %s will not be used", partition)
        else:
            # assign the target column to y
            y = df[target]
            # encode label only for classification
            if model_type == ModelType.classification:
{'fraction_non_alphanumeric': 0.1063963737093931, 'fraction_numerical': 0.004532863258625032, 'mean_word_length': 3.858103975535168, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6562422', 'n_tokens_mistral': 2236, 'n_tokens_neox': 2127, 'n_words': 757}