<jupyter_start><jupyter_text><jupyter_code>import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import datetime as dt

import matplotlib.pyplot as plt
import seaborn as sns
sns.set() # Set seaborn as default style
df = pd.read_csv('/content/drive/MyDrive/Official Folder of Flaskdata Detecting Anomalies/Data/Samples/SampleFile.csv')
df.head()
null_values = list(df.isnull().sum() * 100 / df.shape[0] )
null_col_ids = [i for i in range(len(null_values)) if null_values[i] >= 35.0]
null_cols = [df.columns[i] for i in null_col_ids]
# print(null_col_ids)
data = df.drop(null_cols, axis = 1)
data.isnull().sum() * 100/ data.shape[0]
cols = set(data.columns)
numerical_cols = set(data._get_numeric_data())
categorical_cols = cols.difference(numerical_cols)
print('Numerical Columns: \n',numerical_cols)
print('Categorical Columns: \n',categorical_cols)
# data[categorical_cols].value_counts().plot(kind='bar', figsize=(25, 15))
n_plots = len(numerical_cols)
n_cols = 3
rows = n_plots/n_cols if n_plots%n_cols == 0 else (n_plots /n_cols)+1
fig, axes = plt.subplots(int(rows),n_cols, figsize=(25,18))
fig.suptitle('Numerical Features Visualization')

for i, col in enumerate(numerical_cols) :
    sns.distplot(data[col], ax = axes[int(i/n_cols)][i%n_cols])
    axes[int(i/n_cols)][i%n_cols].set_title(col)
    fig.tight_layout()
plt.show()
# data[categorical_cols].value_counts().plot(kind='bar', figsize=(25, 15))
n_plots = len(categorical_cols)
n_cols = 2
rows = n_plots/n_cols if n_plots%n_cols == 0 else (n_plots/n_cols)+1
fig, axes = plt.subplots(int(rows),n_cols, figsize=(50,40))
fig.suptitle('Categorical Features Visualization')

for i, col in enumerate(categorical_cols) :
    sns.countplot(x = col, data = data, ax = axes[int(i/n_cols)][i%n_cols])
    axes[int(i/n_cols)][i%n_cols].set_title(col)
    fig.tight_layout()
plt.show()
data.describe()
data.head()
column_dict = {c:data[c].dtype for c in data.columns}
date_cols = ['AESTDTC', 'AEENDTC']
for dc in date_cols :
    data[dc] = data[dc].replace('******', None)
    data[dc] = pd.to_datetime(data[dc])
    data[dc] = data[dc].apply(lambda x: dt.datetime.date(x))
    print(data[dc].dtype)
    
data.head()
for dc in date_cols :
    x = list(data[dc].sort_values())
    print(f'{dc}\nMin Date: {x[0]}\tMax Date: {x[-1]}\t{len(data[dc].unique())}')<jupyter_output>AESTDTC
Min Date: 2012-12-24	Max Date: 2015-02-02	359
AEENDTC
Min Date: 2013-01-01	Max Date: 2015-02-02	343
{'fraction_non_alphanumeric': 0.1638486312399356, 'fraction_numerical': 0.029790660225442835, 'mean_word_length': 5.488250652741514, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '468945', 'license': 'MIT', 'n_tokens_mistral': 1067, 'n_tokens_neox': 984, 'n_words': 202, 'path': 'FlaskData.ipynb', 'repo_name': 'ohumkar/Fake-News', 'size': '2484'}