<jupyter_start><jupyter_text>This notebook is Initial submission for the hackathon https://datahack.analyticsvidhya.com/contest/women-in-the-loop-a-data-science-hackathon-by-bain/<jupyter_code>import pandas as pd
import numpy as np

from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor,BaggingRegressor
import lightgbm as lgbm
from sklearn import tree
import warnings
warnings.filterwarnings('ignore')
train_df = pd.read_csv('data/train.csv')
test_df = pd.read_csv('data/test_QkPvNLx.csv')
sample_submission = pd.read_csv('data/sample_submission_pn2DrMq.csv')
train_df.shape
train_df.head(3)
test_df.head(3)
sample_submission.head(3)
train_df.info()
train_df.Course_ID.nunique()
def get_course_catalog(train_df):
    temp_df = train_df.copy()
    temp_df['Day_No'] = train_df['Day_No'].mod(365)
    course_user_traffic_catalog_1 =temp_df.groupby(['Course_ID','Day_No'], as_index=False).agg({
        'User_Traffic': 'mean'  
    }).rename(columns = {'User_Traffic':'User_Traffic_Mean'})
    course_user_traffic_catalog_2 =temp_df.groupby(['Course_ID','Day_No'], as_index=False).agg({
        'User_Traffic': 'std'  
    }).rename(columns = {'User_Traffic':'User_Traffic_STD'})
    course_user_traffic_catalog_3 =temp_df.groupby(['Course_ID','Day_No'], as_index=False).agg({
        'User_Traffic': 'last'  
    }).rename(columns = {'User_Traffic':'User_Traffic_Recent'})
    course_user_traffic_catalog = pd.merge(course_user_traffic_catalog_1,course_user_traffic_catalog_2,
                                           on=['Course_ID','Day_No']).merge(course_user_traffic_catalog_3,
                                                                           on=['Course_ID','Day_No'])
    return course_user_traffic_catalog
def overall_preprocessing(df, course_user_traffic_catalog, is_test=False):
    df['Competition_Metric'] = df['Competition_Metric'].fillna(0.0)
    course_type = pd.get_dummies(df['Course_Type'])
    course_domain = pd.get_dummies(df['Course_Domain'])
    course_ids = pd.get_dummies(df['Course_ID'], prefix ='id')
    df_processed = pd.concat([df, course_type, course_domain, course_ids], axis=1)
    df_processed['Day_No'] = df_processed['Day_No'].mod(365)
    df_processed = df_processed.merge(course_user_traffic_catalog,
                                      on=['Course_ID','Day_No']).sort_values('ID').reset_index(drop=True)
    df_processed = df_processed.drop(columns = ['Course_ID','Course_Type','Course_Domain'])
    if is_test:
        return df_processed
    else:
        target = df_processed[['Sales']]
        df_processed = df_processed.drop(columns = ['Sales','User_Traffic'])
        return df_processed, target

# def course_domain_preprocessing(df, course_domain, is_test=False):
#     df_domain = df.loc[df['Course_Domain']==course_domain].reset_index()
#     if is_test:
#         return 'a'
#     else:
#         return 'b'
course_user_traffic_catalog = get_course_catalog(train_df)
df_train, df_target = overall_preprocessing(train_df, course_user_traffic_catalog)
df_pred = overall_preprocessing(test_df,course_user_traffic_catalog, True)
df_train.fillna(0, inplace=True)
#Importing libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

clf = tree.DecisionTreeClassifier(random_state=0)
clf = clf.fit(df_train[:100], df_target[:100])

plt.figure()
tree.plot_tree(clf)
plt.show()

from sklearn.model_selection import KFold,cross_val_score
from sklearn.metrics import mean_squared_log_error

def rmsle_cv(model):
    kf = KFold(n_splits=5, shuffle=True, random_state=90).get_n_splits(df_train.values)
    rmse= np.sqrt(-cross_val_score(model, df_train.values, df_target.values, scoring="neg_mean_squared_log_error", cv = kf))*1000
    print(rmse)
    return(rmse)
lgbmr = lgbm.LGBMRegressor(n_estimators=130,min_child_samples=15,learning_rate=0.05)
lgbmr_rmse_e130_lr_05 = rmsle_cv(lgbmr)
#[150.51512858 146.26601396 177.54153857 171.50355708 182.38828315]
lgbmr = lgbm.LGBMRegressor(n_estimators=150,learning_rate=0.01)
lgbmr_rmse_e150_lr_01 = rmsle_cv(lgbmr)
lgbmr = lgbm.LGBMRegressor(n_estimators=130,num_leaves=600, min_child_samples=15,learning_rate=0.05)
lgbmr_rmse_e130_lr_05 = rmsle_cv(lgbmr)
#[150.51512858 146.26601396 177.54153857 171.50355708 182.38828315]
lgbmr = lgbm.LGBMRegressor(n_estimators=200,num_leaves=600, min_child_samples=15,learning_rate=0.05)
lgbmr_rmse_e200_lr_05 = rmsle_cv(lgbmr)
#[150.51512858 146.26601396 177.54153857 171.50355708 182.38828315]
lgbmr = lgbm.LGBMRegressor(n_estimators=200,num_leaves=600, min_child_samples=200,learning_rate=0.05)
lgbmr_rmse_e180_lr_1 = rmsle_cv(lgbmr)
#[150.51512858 146.26601396 177.54153857 171.50355708 182.38828315]
df_pred_ID = df_pred['ID']
lgbmr.fit(df_train.drop(columns=['ID']),df_target)
df_pred.to_csv()
def save_submission(df_pred_ID, prediction, filename):
    result = pd.concat([df_pred_ID,pd.DataFrame({'Sales':prediction})],axis=1)
    result.to_csv('submissions/' + filename + '.csv', index=False)
save_submission(df_pred_ID, lgbmr.predict(df_pred.drop(columns=['ID'])),'year_mod_single_model_lgbm_e200_lr05_cs200')
sample_submission.head(2)
pd.read_csv('submissions/year_mod_single_model_lgbm_e200_lr05_cs200.csv').head(2)
lgbmr.fit()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12488138166635035, 'fraction_numerical': 0.06547732017460618, 'mean_word_length': 5.782496782496782, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '434333', 'license': 'MIT', 'n_tokens_mistral': 2250, 'n_tokens_neox': 1984, 'n_words': 250, 'path': 'AnalyticsVidhya/Women_Data_Science_Hackathon/Baseline_model.ipynb', 'repo_name': 'lavanya-m-k/ml-hackathons', 'size': '5269'}