<jupyter_start><jupyter_text><jupyter_code>!pip install tsfresh
import pandas as pd
import numpy as np
from datetime import datetime  
from datetime import timedelta  
import gc
import pickle
from sklearn import preprocessing, metrics
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from tsfresh.examples.robot_execution_failures import download_robot_execution_failures, load_robot_execution_failures
from tsfresh import extract_features, extract_relevant_features, select_features
from tsfresh.utilities.dataframe_functions import impute
from tsfresh.feature_extraction import ComprehensiveFCParameters
from tsfresh.feature_extraction import EfficientFCParameters, MinimalFCParameters, ComprehensiveFCParameters
print('Reading files...')
#data_location = 's3://{}/{}/{}'.format(bucket, prefix, 'calendar.csv')
data_folder = '/content/drive/My Drive/Kaggle_M5/01_preprocessed_data/'
X_train = pd.read_pickle(data_folder + 'X_train_WI_2.pkl')
X_val = pd.read_pickle(data_folder + 'X_val_WI_2.pkl')
y_train = X_train['demand']
y_val = X_val['demand']

X_train.head()
my_parameters = {
 'agg_autocorrelation': [{'f_agg': 'mean', 'maxlag': 40},
  {'f_agg': 'median', 'maxlag': 40},
  {'f_agg': 'var', 'maxlag': 40}],
  'ar_coefficient': [{'coeff': 3, 'k': 10}],
   'energy_ratio_by_chunks': [{'num_segments': 10, 'segment_focus': 0},
  {'num_segments': 10, 'segment_focus': 1},
  {'num_segments': 10, 'segment_focus': 2},
  {'num_segments': 10, 'segment_focus': 3},
  {'num_segments': 10, 'segment_focus': 4},
  {'num_segments': 10, 'segment_focus': 5},
  {'num_segments': 10, 'segment_focus': 6},
  {'num_segments': 10, 'segment_focus': 7},
  {'num_segments': 10, 'segment_focus': 8},
  {'num_segments': 10, 'segment_focus': 9}],
 'fft_coefficient': [{'attr': 'angle', 'coeff': 1},
],
}
store_demand = X_train[['item_id', 'date', 'demand']]
store_demand_fe = extract_features(store_demand, 
                     column_id='item_id', column_sort='date',
                     default_fc_parameters=my_parameters)
store_demand_fe = store_demand_fe.reset_index()
store_demand_fe = store_demand_fe.rename(columns={"id": "item_id"})
store_demand_fe.head()

X_train = X_train.merge(store_demand_fe, on = ['item_id'], how = 'left')
X_val = X_val.merge(store_demand_fe, on = ['item_id'], how = 'left')


not_features = ['d', 'id', 'item_id', 'store_id', 'dept_id',
                'state_id',  'date', 'start_date', 
                'snap_CA', 	'snap_TX', 'weekday',
                'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2',
                'demand_type']

                not_features = ['d', 'id', 'item_id', 'store_id', 'demand', 'date', 'start_date',
                'demand_type', 'event_name_1', 	'event_type_1', 'event_name_2',
                'event_type_2', 'weekday' ]
features = X_train.columns[~X_train.columns.isin(not_features) ]
X_train = X_train[features]
X_val = X_val[features]

features
for column in X_val.columns:
 if X_val[column].dtype.name == 'category':
  dummyCols=pd.get_dummies(X_val[column])
  X_val=X_val.join(dummyCols)
  del X_val[column]
for column in X_train.columns:
 if X_train[column].dtype.name == 'category':
  dummyCols=pd.get_dummies(X_train[column])
  X_train=X_train.join(dummyCols)
  del X_train[column]
X_val = X_val.fillna(0)
X_train = X_train.dropna()
y_train = X_train['demand']
y_val = X_val['demand']
del X_train['demand']
del X_val['demand']
for feature in X_train.columns:
    print("{}: {}".format(feature, X_train[feature].corr(y_train)))
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(X_train, y_train)

# Make predictions using the testing set
y_pred = regr.predict(X_val)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean squared error
print('Mean squared error: %.2f'
      % np.sqrt(mean_squared_error(y_val, y_pred)))
# The coefficient of determination: 1 is perfect prediction
print('Coefficient of determination: %.2f'
      % r2_score(y_val, y_pred))

X2.to_numpy()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14326443335709194, 'fraction_numerical': 0.014492753623188406, 'mean_word_length': 5.378787878787879, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '651489', 'license': 'CC-BY-3.0', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1505, 'n_words': 323, 'path': '10_linear_regression.ipynb', 'repo_name': 'omoreva/kaggle_m5', 'size': '4209'}