<jupyter_start><jupyter_text>training with out viewing dataset<jupyter_code>import pandas as pd
train = pd.read_excel('./data/Andhrapradesh/chittoor_data.xls')
train
print("Unique Crops: ", len(train.Crop.unique()))
print("Avg. number of datas for each crop: ", 658/14)
df = train.drop('Unnamed: 0',axis =1)
df["kfold"] = -1

import numpy as np
import pandas as pd
from sklearn import model_selection
df
df = train.drop('Unnamed: 0',axis =1)
df["kfold"] = -1
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=42)
for fold, (train_indicies, valid_indicies) in enumerate(kf.split(X=df)):
    df.loc[valid_indicies, "kfold"] = fold
df.head()
from sklearn.model_selection import train_test_split

train, df_test = train_test_split(df, test_size=0.2)

train.head()
df = train
import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
import optuna<jupyter_output><empty_output><jupyter_text>using optuna for hyper parameter tuning  using k fold method<jupyter_code>useful_features = [c for c in df.columns if c not in ("id", "Production Quantity - tonnes", "kfold")]
object_cols = [col for col in useful_features if col.startswith("Cro")]
df_test = df_test[useful_features]

for col in object_cols:
    temp_df = []
    temp_test_feat = None
    for fold in range(5):
        xtrain =  df[df.kfold != fold].reset_index(drop=True)
        xvalid = df[df.kfold == fold].reset_index(drop=True)
        feat = xtrain.groupby(col)["Production Quantity - tonnes"].agg("mean")
        feat = feat.to_dict()
        xvalid.loc[:, f"tar_enc_{col}"] = xvalid[col].map(feat)
        temp_df.append(xvalid)
        if temp_test_feat is None:
            temp_test_feat = df_test[col].map(feat)
        else:
            temp_test_feat += df_test[col].map(feat)
    
    temp_test_feat /= 5
    df_test.loc[:, f"tar_enc_{col}"] = temp_test_feat
    df = pd.concat(temp_df)
    

useful_features = [c for c in df.columns if c not in ("id", "Production Quantity - tonnes", "kfold")]
object_cols = [col for col in useful_features if col.startswith("Cro")]
numerical_cols = [col for col in useful_features if not (col.startswith("Cro"))]

df_test = df_test[useful_features]<jupyter_output><empty_output><jupyter_text>usibg ordinal encoder<jupyter_code>def run(trial):
    fold = 0
    learning_rate = trial.suggest_float("learning_rate", 1e-2, 0.25, log=True)
    reg_lambda = trial.suggest_loguniform("reg_lambda", 1e-8, 100.0)
    reg_alpha = trial.suggest_loguniform("reg_alpha", 1e-8, 100.0)
    subsample = trial.suggest_float("subsample", 0.1, 1.0)
    colsample_bytree = trial.suggest_float("colsample_bytree", 0.1, 1.0)
    max_depth = trial.suggest_int("max_depth", 1, 7)

    xtrain = df[df.kfold != fold].reset_index(drop=True)
    xvalid = df[df.kfold == fold].reset_index(drop=True)

    ytrain = xtrain['Production Quantity - tonnes']
    yvalid = xvalid['Production Quantity - tonnes']

    xtrain = xtrain[useful_features]
    xvalid = xvalid[useful_features]

    ordinal_encoder = preprocessing.OrdinalEncoder()
    xtrain[object_cols] = ordinal_encoder.fit_transform(xtrain[object_cols])
    xvalid[object_cols] = ordinal_encoder.transform(xvalid[object_cols])

    model = XGBRegressor(
        random_state=42,
        n_estimators=7000,
        learning_rate=learning_rate,
        reg_lambda=reg_lambda,
        reg_alpha=reg_alpha,
        subsample=subsample,
        colsample_bytree=colsample_bytree,
        max_depth=max_depth,
    )
    model.fit(xtrain, ytrain, eval_set=[(xvalid, yvalid)], verbose=1000)
    preds_valid = model.predict(xvalid)
    rmse = mean_squared_error(yvalid, preds_valid, squared=False)
    return rmse
study = optuna.create_study(direction="minimize")
study.optimize(run, n_trials=15)<jupyter_output>[32m[I 2021-09-24 18:57:38,200][0m A new study created in memory with name: no-name-d37107d4-6a1d-45e2-b35f-bcc54a70d7e1[0m<jupyter_text>Best is trial 1 with RMSE value: 57688060.5020693.<jupyter_code>import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
numerical_cols = [col for col in useful_features if not (col.startswith("Cro"))]

def run(trial):
    fold = 0
    learning_rate = trial.suggest_float("learning_rate", 1e-2, 0.35, log=True)
    reg_lambda = trial.suggest_loguniform("reg_lambda", 1e-8, 100.0)
    reg_alpha = trial.suggest_loguniform("reg_alpha", 1e-8, 100.0)
    subsample = trial.suggest_float("subsample", 0.1, 1.0)
    colsample_bytree = trial.suggest_float("colsample_bytree", 0.1, 1.0)
    max_depth = trial.suggest_int("max_depth", 1, 9)

    xtrain = df[df.kfold != fold].reset_index(drop=True)
    xvalid = df[df.kfold == fold].reset_index(drop=True)

    ytrain = xtrain['Production Quantity - tonnes']
    yvalid = xvalid['Production Quantity - tonnes']

    xtrain = xtrain[useful_features]
    xvalid = xvalid[useful_features]

    ordinal_encoder = preprocessing.OrdinalEncoder()
    xtrain[object_cols] = ordinal_encoder.fit_transform(xtrain[object_cols])
    xvalid[object_cols] = ordinal_encoder.transform(xvalid[object_cols])
    scaler = preprocessing.StandardScaler()
    xtrain[numerical_cols] = scaler.fit_transform(xtrain[numerical_cols])
    xvalid[numerical_cols] = scaler.transform(xvalid[numerical_cols])

    model = XGBRegressor(
        random_state=42,
        n_estimators=15000,
        learning_rate=learning_rate,
        reg_lambda=reg_lambda,
        reg_alpha=reg_alpha,
        subsample=subsample,
        colsample_bytree=colsample_bytree,
        max_depth=max_depth,
    )
    model.fit(xtrain, ytrain, eval_set=[(xvalid, yvalid)], verbose=1000)
    preds_valid = model.predict(xvalid)
    rmse = mean_squared_error(yvalid, preds_valid, squared=False)
    return rmse
study = optuna.create_study(direction="minimize")
study.optimize(run, n_trials=30)
final_predictions = []
scores = []
for fold in range(5):
    xtrain =  df[df.kfold != fold].reset_index(drop=True)
    xvalid = df[df.kfold == fold].reset_index(drop=True)
    xtest = df_test.copy()

    ytrain = xtrain['Production Quantity - tonnes']
    yvalid = xvalid['Production Quantity - tonnes']
    
    xtrain = xtrain[useful_features]
    xvalid = xvalid[useful_features]
    
    ordinal_encoder = preprocessing.OrdinalEncoder()
    xtrain[object_cols] = ordinal_encoder.fit_transform(xtrain[object_cols])
    xvalid[object_cols] = ordinal_encoder.transform(xvalid[object_cols])
    xtest[object_cols] = ordinal_encoder.transform(xtest[object_cols])
    
    scaler = preprocessing.StandardScaler()
    xtrain[numerical_cols] = scaler.fit_transform(xtrain[numerical_cols])
    xvalid[numerical_cols] = scaler.transform(xvalid[numerical_cols])
    xtest[numerical_cols] = scaler.transform(xtest[numerical_cols])
    
    model = XGBRegressor(random_state=fold ,
        n_estimators=15000,)
    model.fit(xtrain, ytrain)
    preds_valid = model.predict(xvalid)
    test_preds = model.predict(xtest)
    final_predictions.append(test_preds)
    rmse = mean_squared_error(yvalid, preds_valid, squared=False)
    print(fold, rmse)
    scores.append(rmse)

print(np.mean(scores), np.std(scores))
df.head()<jupyter_output><empty_output><jupyter_text>eda for the given data<jupyter_code>df['Yield - Hg/Ha'].corr(df['Production Quantity - tonnes'])
df2 = df[df.Crop == 'tea']
df2['Area Harvested - Ha'].corr(df2['Production Quantity - tonnes'])
for i in df.Crop.unique():
  df2 = df[df.Crop == i]
  l = df2['Yield - Hg/Ha'].corr(df2['Production Quantity - tonnes'])
  print(l
        )


for i in df.Crop.unique():
  df2 = df[df.Crop == i]
  l = df2['Area Harvested - Ha'].corr(df2['Yield - Hg/Ha'])
  print(l)
df2.head()
df.head()
numerical_cols
from scipy import stats
np.abs(stats.zscore(df['Area Harvested - Ha']))


df[(np.abs(stats.zscore(df.loc['Area Harvested - Ha', 'Yield - Hg/Ha','year']) < 3).all(axis=1))]
from scipy import stats

df[(np.abs(stats.zscore(df[2])) < 3)]<jupyter_output><empty_output><jupyter_text>co relation was very less so removing outliners<jupyter_code>q_low = df["Area Harvested - Ha"].quantile(0.1)
q_hi  = df["Area Harvested - Ha"].quantile(0.9)

df_filtered = df[(df["Area Harvested - Ha"] < q_hi) & (df["Area Harvested - Ha"] > q_low)]
df_filtered
df 
df = df_filtered
q_low = df["Production Quantity - tonnes"].quantile(0.1)
q_hi  = df["Production Quantity - tonnes"].quantile(0.9)

df_filtered = df[(df["Production Quantity - tonnes"] < q_hi) & (df["Production Quantity - tonnes"] > q_low)]
df_filtered
df = df_filtered
q_low = df["Yield - Hg/Ha"].quantile(0.1)
q_hi  = df["Yield - Hg/Ha"].quantile(0.9)

df_filtered = df[(df["Yield - Hg/Ha"] < q_hi) & (df["Yield - Hg/Ha"] > q_low)]
df_filtered
df = df_filtered
from sklearn.model_selection import train_test_split

train, df_test = train_test_split(df, test_size=0.2)
df.head()
train.head()
df = train
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=42)
for fold, (train_indicies, valid_indicies) in enumerate(kf.split(X=df)):
    df.loc[valid_indicies, "kfold"] = fold 
useful_features = [c for c in df.columns if c not in ("id", "Production Quantity - tonnes", "kfold")]
object_cols = [col for col in useful_features if col.startswith("Cro")]
df_test = df_test[useful_features]

for col in object_cols:
    temp_df = []
    temp_test_feat = None
    for fold in range(5):
        xtrain =  df[df.kfold != fold].reset_index(drop=True)
        xvalid = df[df.kfold == fold].reset_index(drop=True)
        feat = xtrain.groupby(col)["Production Quantity - tonnes"].agg("mean")
        feat = feat.to_dict()
        xvalid.loc[:, f"tar_enc_{col}"] = xvalid[col].map(feat)
        temp_df.append(xvalid)
        if temp_test_feat is None:
            temp_test_feat = df_test[col].map(feat)
        else:
            temp_test_feat += df_test[col].map(feat)
    
    temp_test_feat /= 5
    df_test.loc[:, f"tar_enc_{col}"] = temp_test_feat
    df = pd.concat(temp_df)
    

useful_features = [c for c in df.columns if c not in ("id", "Production Quantity - tonnes", "kfold")]
object_cols = [col for col in useful_features if col.startswith("Cro")]
numerical_cols = [col for col in useful_features if not (col.startswith("Cro"))]

df_test = df_test[useful_features]<jupyter_output><empty_output><jupyter_text>trained once again with 0.1 to 0.9  quantile<jupyter_code>def run(trial):
    fold = 0
    learning_rate = trial.suggest_float("learning_rate", 1e-2, 0.25, log=True)
    reg_lambda = trial.suggest_loguniform("reg_lambda", 1e-8, 100.0)
    reg_alpha = trial.suggest_loguniform("reg_alpha", 1e-8, 100.0)
    subsample = trial.suggest_float("subsample", 0.1, 1.0)
    colsample_bytree = trial.suggest_float("colsample_bytree", 0.1, 1.0)
    max_depth = trial.suggest_int("max_depth", 1, 7)

    xtrain = df[df.kfold != fold].reset_index(drop=True)
    xvalid = df[df.kfold == fold].reset_index(drop=True)

    ytrain = xtrain['Production Quantity - tonnes']
    yvalid = xvalid['Production Quantity - tonnes']

    xtrain = xtrain[useful_features]
    xvalid = xvalid[useful_features]

    ordinal_encoder = preprocessing.OrdinalEncoder()
    xtrain[object_cols] = ordinal_encoder.fit_transform(xtrain[object_cols])
    xvalid[object_cols] = ordinal_encoder.transform(xvalid[object_cols])

    model = XGBRegressor(
        random_state=42,
        n_estimators=7000,
        learning_rate=learning_rate,
        reg_lambda=reg_lambda,
        reg_alpha=reg_alpha,
        subsample=subsample,
        colsample_bytree=colsample_bytree,
        max_depth=max_depth,
    )
    model.fit(xtrain, ytrain, eval_set=[(xvalid, yvalid)], verbose=1000)
    preds_valid = model.predict(xvalid)
    rmse = mean_squared_error(yvalid, preds_valid, squared=False)
    return rmse
study = optuna.create_study(direction="minimize")
study.optimize(run, n_trials=15)
df<jupyter_output><empty_output><jupyter_text>visualization of whole data<jupyter_code>train = pd.read_excel('./data/AndhraPradesh/chittoor_data.xls')
train
train.boxplot(by ='Crop', column =['Area Harvested - Ha'], grid = False)
import seaborn as sns
sns.set(rc={'figure.figsize':(11.7,8.27)})

sns.set_style("whitegrid")
  
sns.boxplot(x = 'Crop', y = 'Area Harvested - Ha', data = train)
sns.set_style("whitegrid")
  
sns.boxplot(x = 'Crop', y = 'Production Quantity - tonnes', data = train)
sns.set_style("whitegrid")
  
sns.boxplot(x = 'Crop', y = 'Yield - Hg/Ha', data = train)
df
sns.set_style("whitegrid")
sns.set(rc={'figure.figsize':(11.7,8.27)})

  
sns.boxplot(x = 'Crop', y = 'Yield - Hg/Ha', data = df)
sns.set_style("whitegrid")
  
sns.boxplot(x = 'Crop', y = 'Production Quantity - tonnes', data = df)
sns.set_style("whitegrid")
  
sns.boxplot(x = 'Crop', y = 'Area Harvested - Ha', data = df)
df2 = df[(np.abs(stats.zscore(df['Area Harvested - Ha'])) < 3)]

df2
sns.set_style("whitegrid")
  
sns.boxplot(x = 'Crop', y = 'Area Harvested - Ha', data = df2)
df.columns
train = pd.read_excel('/content/bhopal_data.xls')
df = train.drop('Unnamed: 0',axis =1)
df["kfold"] = -1
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=42)
for fold, (train_indicies, valid_indicies) in enumerate(kf.split(X=df)):
    df.loc[valid_indicies, "kfold"] = fold
df
l = [ 'Area Harvested - Ha', 'Production Quantity - tonnes',
       'Yield - Hg/Ha']
for i in l:
  df = df[(np.abs(stats.zscore(df[i])) < 3)]


df
sns.set_style("whitegrid")
  
sns.boxplot(x = 'Crop', y = 'Yield - Hg/Ha', data = df)
lps('ababa')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13221601489757914, 'fraction_numerical': 0.01869359690588741, 'mean_word_length': 3.995706618962433, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '190610', 'license': 'MIT', 'n_tokens_mistral': 5425, 'n_tokens_neox': 5044, 'n_words': 963, 'path': 'app.ipynb', 'repo_name': 'zshanmukh/Crop-Yeild-Predictor', 'size': '13962'}