<jupyter_start><jupyter_text>Effectiveness Of STD Drugs Problem Statement![Problem Statement](problem-statement.png)**Evaluation Algorithm****Root Mean Square Error (RMSE**)normalization_constant 100000DatasetsTraining	download train.csvTesting	download test.csvSample Submission	download sample.csv  Importing required libraries and setup some variables<jupyter_code>import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import pylab as plt # help to plot different graphs 

# Set the global default size of matplotlib figures
plt.rc('figure', figsize=(10, 5))

# Size of matplotlib figures that contain subplots
fizsize_with_subplots = (10, 10)

# Size of matplotlib histogram bins
bin_size = 10
#test.csv
#train.csv
#submission.csv

import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats
%matplotlib inline


# Until fuction: line seperator
def print_dashes_and_ln():
    print('-'*100, '\n')
    
# Formatter to display all float format in 2 decimal format
#pd.options.display.float_format = '{:.2f}'.format

import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Reading training data with showing first 5 records<jupyter_code>df_train = pd.read_csv('dataset/train.csv')
df_train_copy = df_train.copy()
df_train_copy.head()<jupyter_output><empty_output><jupyter_text>Fetching last 5 records from training data<jupyter_code>df_train.tail()<jupyter_output><empty_output><jupyter_text>Checking datatypes of columns<jupyter_code>df_train.dtypes<jupyter_output><empty_output><jupyter_text>Get some basic information of the DataFrame:<jupyter_code>df_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 32165 entries, 0 to 32164
Data columns (total 8 columns):
patient_id                    32165 non-null int64
name_of_drug                  32165 non-null object
use_case_for_drug             32165 non-null object
review_by_patient             32165 non-null object
effectiveness_rating          32165 non-null int64
drug_approved_by_UIC          32165 non-null object
number_of_times_prescribed    32165 non-null int64
base_score                    32165 non-null float64
dtypes: float64(1), int64(3), object(4)
memory usage: 2.0+ MB<jupyter_text>There are not any missing values because each column consists of 32165 rows. Converting column which are in date format but is of other datatype.<jupyter_code>df_train['drug_approved_by_UIC'] =  pd.to_datetime(df_train['drug_approved_by_UIC'], infer_datetime_format=True)
#df_train.info()
df_train['day'] = df_train['drug_approved_by_UIC'].dt.day
df_train['year'] = df_train['drug_approved_by_UIC'].dt.year
df_train['month'] = df_train['drug_approved_by_UIC'].dt.month
from textblob import TextBlob
from tqdm import tqdm
reviews = df_all['review_clean']

Predict_Sentiment = []
for review in tqdm(reviews):
    blob = TextBlob(review)
    Predict_Sentiment += [blob.sentiment.polarity]
df_all["Predict_Sentiment"] = Predict_Sentiment
df_all.head()


df_train['review_by_patient'][2]
np.corrcoef(df_all["Predict_Sentiment"], df_all["rating"])
np.corrcoef(df_all["Predict_Sentiment"], df_all["sentiment"])
#Delete serial number because that is not need for serial number and telephone
del df_train['use_case_for_drug']
del df_train['review_by_patient']
# getting total number of rows and column in the dataframe
def shape_of_dataframe(df_train):
    print(f" Shape of the dataframe = {df_train.shape}"); print_dashes_and_ln();
    totalrows=df_train.shape[0]
    print(f" Total number of rows in the dataset =  {totalrows}"); print_dashes_and_ln();
shape_of_dataframe(df_train)<jupyter_output>Shape of the dataframe = (32165, 6)
---------------------------------------------------------------------------------------------------- 

 Total number of rows in the dataset =  32165
----------------------------------------------------------------------------------------------------<jupyter_text>Let's plots some basic figure to get basic idea of data.<jupyter_code>plt.subplots(figsize=(12,9))
sns.distplot(df_train['base_score'], fit=stats.norm)

# Get the fitted parameters used by the function

(mu, sigma) = stats.norm.fit(df_train['base_score'])

# plot with the distribution

plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)], loc='best')
plt.ylabel('Frequency')

#Probablity plot

fig = plt.figure()
stats.probplot(df_train['base_score'], plot=plt)
plt.show()
#we use log function which is in numpy
df_train['base_score'] = np.log1p(df_train['base_score'])

#Check again for more normal distribution

plt.subplots(figsize=(12,9))
sns.distplot(df_train['base_score'], fit=stats.norm)

# Get the fitted parameters used by the function

(mu, sigma) = stats.norm.fit(df_train['base_score'])

# plot with the distribution

plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)], loc='best')
plt.ylabel('Frequency')

#Probablity plot

fig = plt.figure()
stats.probplot(df_train['base_score'], plot=plt)
plt.show()<jupyter_output><empty_output><jupyter_text>Let's check if the data set has any missing values.<jupyter_code># Columns by Data Type
non_numeric_columns = df_train.select_dtypes(['object']).columns
numeric_columns = df_train.select_dtypes(['number']).columns
print(f" Non Numerical Columns of the dataframe = {non_numeric_columns}"); print_dashes_and_ln();
print(f" Numerical Columns of the dataframe = {numeric_columns}"); print_dashes_and_ln();<jupyter_output>Non Numerical Columns of the dataframe = Index(['name_of_drug'], dtype='object')
---------------------------------------------------------------------------------------------------- 

 Numerical Columns of the dataframe = Index(['patient_id', 'effectiveness_rating', 'number_of_times_prescribed',
       'base_score'],
      dtype='object')
----------------------------------------------------------------------------------------------------<jupyter_text>Now, there is no any missing values. Encoding str to int<jupyter_code>cols = ('name_of_drug')
df_train['name_of_drug'].unique().size
#from sklearn.preprocessing import LabelEncoder
#for c in cols:
    #lbl = LabelEncoder() 
    #lbl.fit(list(df_train[c].values)) 
    #df_train[c] = lbl.transform(list(df_train[c].values))
df_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 32165 entries, 0 to 32164
Data columns (total 6 columns):
patient_id                    32165 non-null int64
name_of_drug                  32165 non-null object
effectiveness_rating          32165 non-null int64
drug_approved_by_UIC          32165 non-null datetime64[ns]
number_of_times_prescribed    32165 non-null int64
base_score                    32165 non-null float64
dtypes: datetime64[ns](1), float64(1), int64(3), object(1)
memory usage: 1.5+ MB<jupyter_text>Change data type from float to int<jupyter_code>#df_train['Saving Account'] = df_train['Saving Account'].astype(int)
#df_train['investment'] = df_train['investment'] * 10
#df_train['investment'] = df_train['investment'].astype(int)


#Coralation plot
corr = df_train.corr()
plt.subplots(figsize=(20,9))
sns.heatmap(corr, annot=True)<jupyter_output><empty_output><jupyter_text>Top 10% Corralation train attributes with credit amount<jupyter_code>top_feature = corr.index[abs(corr['base_score']>0.1)]
plt.subplots(figsize=(12, 8))
top_corr = df_train[top_feature].corr()
sns.heatmap(top_corr, annot=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Here duration_month is highly correlated with target feature of credit amount by 64%.So, more duration month then more credit amount.<jupyter_code>def clean_data(df, drop_serial_number):
    print(df.shape)
    del df['use_case_for_drug']
    del df['review_by_patient']
    del df['name_of_drug']
    del df['drug_approved_by_UIC']
    df = pd.get_dummies(df)
    if drop_serial_number:
        df = df.drop(['patient_id'], axis=1)

    return df
df_train = clean_data(df_train_copy,drop_serial_number=True)
#df_train['base_score'] = np.log1p(df_train['base_score'])

df_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 32165 entries, 0 to 32164
Data columns (total 3 columns):
effectiveness_rating          32165 non-null int64
number_of_times_prescribed    32165 non-null int64
base_score                    32165 non-null float64
dtypes: float64(1), int64(2)
memory usage: 753.9 KB<jupyter_text>Prepraring data for prediction<jupyter_code>#Take targate variable into y
y = df_train['base_score']
#Delete the base_score
del df_train['base_score']
#Take their values in X and y
X = df_train.values
y = y.values
# Split data into train and test formate
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)<jupyter_output><empty_output><jupyter_text>XGBOOST ALGO: PREPARE FOR SUBMISSIONCreate a DataFrame by combining the index from the test data with the output of predictions, then write the results to the output:<jupyter_code>import xgboost as xgb
from sklearn.metrics import mean_squared_error
import pandas as pd
import numpy as np
df_test = pd.read_csv('dataset/test.csv')
df_test = clean_data(df_test,drop_serial_number=False)
test_data = df_test.values
data_dmatrix = xgb.DMatrix(data=df_test,label=y)
xg_reg = xgb.XGBRegressor(objective ='reg:linear',colsample_bytree=0.9, gamma=0.2, 
                             learning_rate=0.1, max_depth=7, 
                             min_child_weight=1.5, n_estimators=1000,
                             reg_alpha=0.9, reg_lambda=0.6,
                             subsample=0.8,seed=42, silent=1,
                             random_state =1)
xg_reg.fit(X_train,y_train)

preds = xg_reg.predict(X_test)
rmse = np.sqrt(mean_squared_error(y_test, preds))
print("RMSE: %f" % (rmse))
#params = {"objective":"reg:linear",'colsample_bytree': 0.2,'learning_rate': 0.1,
 #               'max_depth': 5, 'alpha': 10}
params = {'objective':'reg:linear','colsample_bytree':0.9, 'gamma':0.2, 
                             'learning_rate':0.1, 'max_depth':7, 
                             'min_child_weight':1.5, 'n_estimators':1000,
                             'reg_alpha':0.9, 'reg_lambda':0.6,
                             'subsample':0.8,'seed':42, 'silent':1,
                             'random_state':1}
cv_results = xgb.cv(dtrain=data_dmatrix, params=params, nfold=3,
                    num_boost_round=50,early_stopping_rounds=10,metrics="rmse", as_pandas=True, seed=23)

cv_results
print((cv_results["test-rmse-mean"]).tail(1))
#S.no
# Get the test data features, skipping the first column 'patient_id'
test_x = test_data[:, 1:]

# Predict the credit amount values for the test data
test_y = xg_reg.predict(test_x)
df_test['base_score'] = test_y
#df_test['S.no'] = df_test['serial number']
df_test['base_score'] = df_test['base_score'].astype(float)
df_test['base_score'] = df_test['base_score'].apply(lambda x: round(x, 2))
df_test[['patient_id','base_score']] \
    .to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1487313298542379, 'fraction_numerical': 0.02780277127946734, 'mean_word_length': 4.0916170407695835, 'pattern_counts': {'":': 1, '<': 55, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '442166', 'license': 'Apache-2.0', 'n_tokens_mistral': 3934, 'n_tokens_neox': 3622, 'n_words': 836, 'path': 'predict-effectiveness-of-std-drugs.ipynb', 'repo_name': 'asharn/interactive-ml-challenge-predict-effectiveness-of-std-drugs', 'size': '11114'}