<jupyter_start><jupyter_text>Scaling Target: Meter ReadingTo do:- Need to store the scaling parameters such that you can scale back later<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
import time
df = pd.read_pickle('mergedData.pkl')
df.head()<jupyter_output><empty_output><jupyter_text>Normalizing for every building and meter: Main Functions<jupyter_code># Timing function decorator
def timeit(func):
    def wrapper(*args, **kw):
        start = time.time()
        result = func(*args, **kw)
        end = time.time()
        print('{} took {:.2f} milliseconds'.format(func.__name__,(end-start)*1000))

        return result
    
    return wrapper
# Scaling function per dataset grouped on building and meter
def normalize_target(df_b_m, scaler):
    """
    Input:
    ------
    df_b_m: pd.dataframe
        original data grouped by 'building' and 'meter'
    scaler: sklearn.preprocessing.func()
        any of the sklearn scalers
    
    Output:
    -------
    pd.Series
    scaled values of 'meter_reading'
    """
    
    scaled_mat = scaler.fit_transform(df_b_m.loc[:,'meter_reading'].values.reshape(-1, 1))
    scaled_col = scaled_mat.reshape((len(scaled_mat),))
    
    return pd.Series(data = scaled_col, index = df_b_m.index)
# Adding new column to data
@timeit
def get_normalized_data(data, scaler):
    result = data.groupby(['meter', 'building_id']).apply(normalize_target, scaler=scaler)
    result = result.reset_index()
    result.columns = ['meter', 'building_id', 'index', 'meter_reading_scaled']
    result = result.set_index('index')
    data.loc[:,'meter_reading_scaled'] = result.loc[:,'meter_reading_scaled'].copy()
    
    return data<jupyter_output><empty_output><jupyter_text>Explanation of scaling function<jupyter_code># Follow along with this code
df_b_m = df[(df['meter'] == 0) & (df['building_id'] == 12)]
df_b_m.head()
scaled_mat = MinMaxScaler().fit_transform(df_b_m['meter_reading'].values.reshape(-1, 1))
scaled_mat
scaled_col = scaled_mat.reshape((len(scaled_mat),))
pd.Series(data = scaled_col, index = df_b_m.index).head()<jupyter_output><empty_output><jupyter_text>Testrun with smaller dataset<jupyter_code>test = df.iloc[146446-10000:146446+10000]
print(test['meter'].unique())
print(test['building_id'].unique())
test.head()
# Applying scaling to small test dataset
scaler = RobustScaler(with_centering=True, with_scaling=True, quantile_range=(10, 90))

result = test.groupby(['meter', 'building_id']).apply(normalize_target, scaler=scaler)
result = result.reset_index()
result.columns = ['meter', 'building_id', 'index', 'meter_reading_scaled']
result = result.set_index('index')
result.head()
# Add new column to small test dataset
test.loc[:,'meter_reading_scaled'] = result.loc[:,'meter_reading_scaled'].copy()<jupyter_output>c:\users\320078396\appdata\local\continuum\anaconda3\envs\santiago\lib\site-packages\pandas\core\indexing.py:362: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[key] = _infer_fill_value(value)
c:\users\320078396\appdata\local\continuum\anaconda3\envs\santiago\lib\site-packages\pandas\core\indexing.py:543: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self.obj[item] = s<jupyter_text>Apply to small dataset<jupyter_code>n = len(df)
frac = 1

data = df.iloc[:int(n*frac), :]
print(data.shape)
data.head()
scaler = MinMaxScaler() #with_centering=True, with_scaling=True, quantile_range=(10, 90)
scaled_data = get_normalized_data(data, scaler)
print('for {} rows'.format(len(data)))
print('total rows are {}'.format(len(df)))
print('time will be multiplied by approx. {:2f}'.format((len(df)/len(data))))
scaled_data.head()
if frac == 1:
    scaled_data.to_pickle('Scaled_Data.pkl')
else:
    scaled_data.to_pickle('Scaled_Data_'+str(frac)+'.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1376912378303199, 'fraction_numerical': 0.01645804357904497, 'mean_word_length': 5.373707533234859, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '435090', 'license': 'MIT', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1492, 'n_words': 304, 'path': 'ashrae/Scaling_target.ipynb', 'repo_name': 'danielamiranda/jads_kaggle', 'size': '4314'}