<jupyter_start><jupyter_text>Libraries<jupyter_code>import numpy as np,pandas as pd
from sklearn.ensemble import RandomForestClassifier,RandomForestRegressor
import xgboost as xgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score,auc
from sklearn.metrics import confusion_matrix,accuracy_score,r2_score,mean_squared_error
from xgboost import plot_importance
import matplotlib.pyplot as plt
%matplotlib inline 
from sklearn.preprocessing import LabelEncoder 
import datetime as dt
test=pd.read_csv('patient_monthwise_revenue_test.csv')
train=pd.read_csv('patient_monthwise_revenue_train.csv')
test_p=pd.read_csv('physio_diagnosis_test.csv')
train_p=pd.read_csv('physio_diagnosis_train.csv')
train_cl=pd.read_csv('patient_train_classified.csv')
train_ph=pd.read_csv('physio_appts_train.csv')

#<jupyter_output>/home/rajat/anaconda3/envs/py35/lib/python3.5/site-packages/IPython/core/interactiveshell.py:2717: DtypeWarning: Columns (19) have mixed types. Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)
/home/rajat/anaconda3/envs/py35/lib/python3.5/site-packages/IPython/core/interactiveshell.py:2717: DtypeWarning: Columns (20,21) have mixed types. Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)<jupyter_text>PREPROCESSING<jupyter_code>train_cl.columns=['patient_id','Bucket','revenue']
train_cl.head()
train.drop('revenue',axis=1,inplace=True)
train=pd.merge(train,train_cl,on='patient_id').drop(['Bucket'],axis=1)
train.drop(['Unnamed: 20','Unnamed: 21'],axis=1,inplace=True)
test.drop(['Unnamed: 18','Unnamed: 19'],axis=1,inplace=True)
train_p.drop(['Unnamed: 7','Unnamed: 8'],axis=1,inplace=True)
test_p.drop(['Unnamed: 6','Unnamed: 7'],axis=1,inplace=True)
train.service_name.fillna('None',inplace=True)
test.service_name.fillna('None',inplace=True)
train.gender.fillna('None',inplace=True)
test.gender.fillna('None',inplace=True)
train.diagnosis.fillna('None',inplace=True)
test.diagnosis.fillna('None',inplace=True)

train=train[train.visits_required!=0]
test.shape,train.shape,train_p.shape,test_p.shape,train_cl.shape,train_ph.shape
#temp=train.groupby(['patient_id'])[['visits_count']].sum().reset_index()
#temp.head()
#train.drop('visits_count',axis=1,inplace=True)
#train=pd.merge(train,temp,on='patient_id')
train.head()
def date_feature(df):
    df.FVD=df.FVD.apply(lambda x:dt.datetime.strptime(x,'%d/%m/%Y'))
    df.LVD=df.LVD.apply(lambda x:dt.datetime.strptime(x,'%d/%m/%Y'))
    df['date_diff']=df.LVD.sub(df.FVD).astype(dt.timedelta).map(lambda x: np.nan if pd.isnull(x) else x.days)
    df['fvd_year']=df.FVD.dt.year
    df['fvd_month']=df.FVD.dt.month
    df['fvd_day']=df.FVD.dt.day
    df['lvd_year']=df.LVD.dt.year
    df['lvd_month']=df.LVD.dt.month
    df['lvd_day']=df.LVD.dt.day
    df.drop(['FVD','LVD'],axis=1,inplace=True)
    return df
    
train=date_feature(train)
test=date_feature(test)
train['is_train']=1
test['is_train']=0
tt=pd.concat([train,test],ignore_index=True)
print(tt.shape)
tt.head()
tt['vmy_month']=tt.visit_month_year.apply(lambda x: x[0:2])
tt['vmy_year']=tt.visit_month_year.apply(lambda x: x[3:5])
tt.drop(['FVM','FVS','visit_month_year'],axis=1,inplace=True)

#tt['visit_diff']=tt.visits_required-tt.visits_count
#tt['visit_diff_ohe']=tt.visit_diff.apply(lambda x: 1 if x<0 else 0)
tt.replace(to_replace="\\N",value='-1',inplace=True)
#tt.avg_nps=tt.avg_nps.apply(lambda x : '-1' if len(x)>4 or len(x)==0 or x==' ' else x)

tt.approx_age=tt.approx_age.astype('int64')
tt.vmy_year=tt.vmy_year.astype('int64')
tt.avg_nps=tt.avg_nps.astype('float64')
tt.vmy_month=tt.vmy_month.astype('int64')
for i in tt.columns :
    if tt[i].dtypes=='object':
        print(i)
        le=LabelEncoder()
        tt[i]=le.fit_transform(tt[i])
        #est[i]=le.transform(test[i])
tt.drop('visits_count',axis=1,inplace=True)
train=tt[tt.is_train==1]
test=tt[tt.is_train==0].drop(['revenue'],axis=1)
del tt
test.drop('is_train',axis=1,inplace=True)
train.drop('is_train',axis=1,inplace=True)<jupyter_output><empty_output><jupyter_text>MODELS<jupyter_code>test.shape,train.shape
#train.drop_duplicates(keep='first',inplace=True)
#train.is_click.value_counts()
X=train.drop(['revenue','patient_id'],axis=1)
Y=train.revenue
X.shape,Y.shape
x_train,x_test,y_train,y_test= train_test_split(X,Y,test_size=0.20,random_state=4)
def rmse(yt,yp):
    return mean_squared_error(yt,yp)**0.5
#rf=RandomForestRegressor(n_estimators=1000, criterion='mse', max_depth=10, max_features='sqrt',
#                            min_samples_leaf=3,n_jobs=-1, random_state=0)
#rf.fit(X,Y)
#pred_rf=rf.predict(x_test)
#print(rmse(y_test,pred_rf))#,accuracy_score(y_test,pred_rf)
#confusion_matrix(y_test,pred_rf)
#featimp = pd.Series(rf.feature_importances_,index=x_train.columns).sort_values(ascending=False)
#print(featimp) # this is the property of Random Forest classifier that it provide us the importance 
dtrain = xgb.DMatrix(x_train, label=y_train)
dtest = xgb.DMatrix(x_test)
param = {
    'max_depth':10, # the maximum depth of each tree
    'eta': 0.1,     # the training step for each iteration
    'silent': 1,    # logging mode - quiet
    'objective': 'reg:linear',  # error evaluation for multiclass training
    'eval_metric':'rmse'
}                       

num_round = 400
bst1 = xgb.train(param, dtrain, num_round)
pred_xg = bst1.predict(dtest)
print(rmse(y_test,pred_xg))

plot_importance(bst1)<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>sub=pd.read_csv('Submission.csv')
ddtest=xgb.DMatrix(test[x_train.columns])
test['pred']=bst1.predict(ddtest)
temp=test.groupby(['patient_id'])['pred'].mean().reset_index()
temp.head()
sub.rename(index=str,columns={"PID":"patient_id","Bucket":'Bucket'},inplace=True)

sub=pd.merge(sub,temp,on='patient_id')
def buc(x):
    if x<=1000:
        return 'Low'
    elif x>1000 and x<=5000:
        return 'Med'
    elif x>5000 and x<=10000:
        return 'High-Med'
    elif x>10000:
        return 'High'
sub.Bucket=sub.pred.apply(lambda x : buc(x))
sub.drop('pred',axis=1,inplace=True)
sub.rename(index=str,columns={"patient_id":"PID","Bucket":'Bucket'},inplace=True)
sub.head()
sub.to_csv('sub-xgbt-new-1.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16861383012314493, 'fraction_numerical': 0.020524155352068203, 'mean_word_length': 7.184754521963824, 'pattern_counts': {'":': 4, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '104430', 'license': 'MIT', 'n_tokens_mistral': 2579, 'n_tokens_neox': 2495, 'n_words': 308, 'path': 'Portea challenge.ipynb', 'repo_name': 'rajat-1994/HackerEarth-Portea-Challenge', 'size': '6334'}