<jupyter_start><jupyter_text>Conts<jupyter_code># Background graph style
sns.set(style="darkgrid")
sns.set(style="whitegrid")
data_path = '../data/raw/'
# ==============
# Chouse color
# ==============
sns.palplot(sns.color_palette("muted"))  # show color
sns.set_palette("muted") # for seaborn
colors = sns.color_palette("muted") # for pyplot<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code># ------------------------------- FUNCIONES ---------------------------------------- #

def woe(catvar,target, data):
    data_aggr = data.groupby([catvar,target]).size().reset_index()
    data_aggr.columns = [catvar,'target','numreg']
    
    # calculamos totales target
    non_event_total= np.sum(data_aggr[data_aggr.target==0]['numreg'])
    event_total = np.sum(data_aggr[data_aggr.target==1]['numreg']) 
    
    # pivoteamos data frame
    data_aggr = data_aggr.pivot(index=catvar, columns='target', values='numreg').reset_index()
    data_aggr.columns = [catvar,'non_event','event']
    
    # calculamos rate de event
    data_aggr['event_rate'] = data_aggr['event']/event_total
    data_aggr['non_event_rate'] = data_aggr['non_event']/non_event_total
    
    # calculamos woe
    data_aggr['woe'] = np.log(data_aggr['event_rate']/data_aggr['non_event_rate'])
    
    # calculamos IV
    data_aggr['rate_diff'] = data_aggr['event_rate'] - data_aggr['non_event_rate']
    data_aggr['information_value'] = data_aggr['rate_diff']*data_aggr['woe']
    #data_aggr['IV'] = data_aggr['IV'].sum()
    
    
    return data_aggr[[catvar,'woe','information_value','event_rate','non_event_rate']]
def info_value(woe_data):
    iv = woe_data['information_value'].sum()
    predictive_power = ""
    if iv < 0.02:
        predictive_power = "useless for prediction"
    elif iv >= 0.02 and iv < 0.1:
        predictive_power = "weak predictor"
    elif iv >= 0.1 and iv < 0.3:
        predictive_power = "medium predictor"
    elif iv >= 0.3 and iv < 0.5:
        predictive_power = "strong predictor"
    else:
        predictive_power = "suspicious predictor"
        
    return iv,predictive_power
def barprop(yvalue, xvalue, stitle, size, data):
    table=pd.crosstab(data[xvalue],data[yvalue])
    table.div(table.sum(1).astype(float), axis=0).plot(kind='bar', stacked=True)
    plt.figure(figsize=size)
    
    if len(stitle)==0: # default title
        stitle = "Proportion bar graph"
    
    plt.title(stitle)
    plt.xlabel(xvalue)
    plt.ylabel(yvalue)
    plt.legend(train[yvalue].unique(),bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    pass
def percentile_based_outlier(data, threshold=95):
    diff = (100 - threshold) / 2
    minval, maxval = np.percentile(data, [diff, 100 - diff])
    return (data < minval) | (data > maxval)
# =========================
#  Histogram
# =========================
def simpleHistGraph(numvar,  data, bins=10):
    g = data[numvar].hist(bins=bins)
    g.set_title(numvar +" distribucion ")
    pass
def countBarPlot(catvar, data, scale="linear", orient="h", rot=0, figsize=(10, 6)):
    plt.figure(figsize=figsize)
    g = sns.countplot(x = catvar, data=data, orient=orient)
    loc, labels = plt.xticks()
    g.set_xticklabels(labels, rotation=rot)
    g.set_yscale(scale)
    
    g.set_title("Number of cases by " + catvar.upper())
    plt.show()
    pass<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code># Load Train Data
rawdata = pd.read_csv(data_path + 'train.csv',sep=',')<jupyter_output><empty_output><jupyter_text>Describe Data<jupyter_code>rawdata.shape
rawdata.head()
rawdata.sample(n=5)
rawdata.info()
rawdata.describe(include=['O']) # category data
rawdata.describe() # numerical data
print(rawdata.nunique())<jupyter_output>id                                     1804874
target                                    2913
comment_text                           1780823
severe_toxicity                           1460
obscene                                   2417
identity_attack                           2294
insult                                    3042
threat                                    2053
asian                                      125
atheist                                     84
bisexual                                    85
black                                      127
buddhist                                    64
christian                                  160
female                                     204
heterosexual                               125
hindu                                       77
homosexual_gay_or_lesbian                  124
intellectual_or_learning_disability         76
jewish                                     120
latino                                     104
male         [...]<jupyter_text>Missing value analysis<jupyter_code># To check if there are any null values in the dataset
rawdata.isnull().values.any() 
rawdata.isnull().sum()<jupyter_output><empty_output><jupyter_text>Explore Numerical Data<jupyter_code># =========================
#  Target Barplot (count)
# =========================
plt.figure(figsize=(10, 6))
sns.countplot(x=target, data=rawdata)
pass
# ====================
#  Boxplot
# ====================
sns.set(style="whitegrid") # Background graph style : whitegrid, darkgrid
plt.figure(figsize=(8, 6))
ax = sns.boxplot(x="catvar", y="numvar", 
                #hue = "catvar or target",
                 orient="v",  # "v" | "h", optional
                 linewidth=1.5,
                 palette="Set2" # Colors to use for the different levels of the hue variable.
                 data=rawdata) # DataFrame, array, or list of arrays, optional
ax.set_title("Default | title")
# Tweak the visual presentation
# ax.xaxis.grid(True)
# ax.yaxis.grid(True)
pass
# =================================================
#  Scatterplot Matrix (Scatterplot + distribution)
# =================================================

var = ["numvar1", "numvar2",.. "numvarn", target] # put all the numeric vars to analyze + target (or categoical value used in hue) 
         # exp: ["numvar1", "numvar2",.. "numvarn", target]
         # Note: drop null values with dropna function.
        
plt.figure(figsize=(18, 18))
g = sns.pairplot(rawdata[var], 
                  hue=target, 
                  palette = colors,
                  markers=["o", "s", "D"], # In the hue variable so that differently colored points 
                                           # will also have different scatterplot markers.
                 kind='scatter' # 'scatter' or 'reg' Kind of plot for the non-identity relationships.
                 )


# To change the size of the scatterpoints in graph
g = g.map_offdiag(plt.scatter,  s=35,alpha=0.5)

# remove the top and right line in graph
sns.despine()

# Additional line to adjust some appearance issue
plt.subplots_adjust(top=0.9)

# Set the Title of the graph from here
g.fig.suptitle('Relation between ' + ', '.join([s  for s in var[:len(var)-1]]) + ' and ' + var[len(var)-1],
	fontsize=14,color="black",alpha=0.7)

pass
# =======================================
#  Scater plot + distribution + line reg
#  2 numerical vars
# =======================================
plt.figure(figsize=(12,12))
sns.jointplot("var1", "var2", data=rawdata, kind="reg", annot_kws=dict(stat="r"))
pass
# ====================
#  HeatMap Correlation
# ====================

# select only numerical data. Drop or replace null values
col_names = ['fct_VoucherPrecioProducto', 'fct_CostoLogistica', 'fct_CostoDiagnostico']
colormap = plt.cm.RdBu
plt.figure(figsize=(12,12))
plt.title('Pearson Correlation of Features', y=1.05, size=15)
sns.heatmap(rawdata[col_names].astype(float).corr(), linewidths=0.1, vmax=1.0, 
            square=True, cmap=colormap, linecolor='white', annot=True)
pass
# =====================
#  Matrix distribution 
# =====================

# select only one numerical value and 2 categorical. Drop or replace null values

col_names = ['numvar', 'catvar1', 'catvar2']
g = sns.FacetGrid(rawdata, row=col_names[1], col=col_names[2])
g.map(sns.distplot, col_names[0])
plt.show()<jupyter_output><empty_output><jupyter_text>Explore Categorical Data<jupyter_code>plt.figure(figsize=(12,12))
df = rawdata.pivot_table(index='catvar1', columns='catvar2', 
                         values='numvar or id', 
                         aggfunc=np.count_nonzero) # chouse a math function from numpy
sns.heatmap(df, annot=True, fmt=".1f")
plt.show()<jupyter_output><empty_output><jupyter_text>Outlier detection analysis<jupyter_code># select only numerical values.
col_names = ['numvar0','numvar1', 'numvar2', 'numvar3',..., 'numvarN']

fig, ax = plt.subplots(len(col_names), figsize=(8,40))

for i, col_val in enumerate(col_names):

    sns.boxplot(y=sales_data_hist[col_val], ax=ax[i])
    ax[i].set_title('Box plot - {}'.format(col_val), fontsize=10)
    ax[i].set_xlabel(col_val, fontsize=8)

plt.show()
fig, ax = plt.subplots(len(col_names), figsize=(8,40))

for i, col_val in enumerate(col_names):
    x = rawdata[col_names][:1000]
    sns.distplot(x, ax=ax[i], rug=True, hist=False)
    outliers = x[percentile_based_outlier(x)]
    ax[i].plot(outliers, np.zeros_like(outliers), 'ro', clip_on=False)

    ax[i].set_title('Outlier detection - {}'.format(col_val), fontsize=10)
    ax[i].set_xlabel(col_val, fontsize=8)

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1766595289079229, 'fraction_numerical': 0.020985010706638114, 'mean_word_length': 2.986769099445156, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '133097', 'license': 'MIT', 'n_tokens_mistral': 3217, 'n_tokens_neox': 3000, 'n_words': 598, 'path': 'notebooks/eda.ipynb', 'repo_name': 'Guillin/jigsaw', 'size': '9340'}