<jupyter_start><jupyter_text>1) Importing required libraries<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# Importing all the required libraries<jupyter_output><empty_output><jupyter_text>2) Business Understanding 1) WHICH COUNTRY HAS THE MOST NUMBER OF MALICIOUS WEBSITES IN THIS DATASET2) WHAT IS CORRELATION BETWEEN EACH COLUMN IN THE DATASET3) WHAT ARE USEFUL VALUES AVAILABLE TO US IN CASE WE NEED TO GO FOR FURTHER PREDICTIONS  3) Gathering information from dataset - Data Understanding<jupyter_code>df = pd.read_csv('dataset.csv')
df<jupyter_output><empty_output><jupyter_text>4) ACCESS<jupyter_code>df.columns
df.describe()<jupyter_output><empty_output><jupyter_text>5) Data Prepration - Clean<jupyter_code>df = df[df.WHOIS_COUNTRY != 'None']
whois = df.WHOIS_COUNTRY.value_counts()
# filtering out malicious website that dont belong to any country
whois

df['WHOIS_COUNTRY'].unique()
df
df.isnull().sum()
#finding if we have null values in columns<jupyter_output><empty_output><jupyter_text>Filtering out bengin websites.<jupyter_code>df_mal = df[df.Type != 0]  
# 
df_mal<jupyter_output><empty_output><jupyter_text>6) Analyze  Plotting graph for occurances of all website for WHOIS data.<jupyter_code>(whois/df.shape[0]).plot(kind="bar");
plt.title("WHOIS_COUNTRY");
# plotting bar graph for entire data
# function to make plot
def makePlot(city_count,column, title, ylabel, xlabel):
    """
    This function takes in common paramters and produces a plot 
    
    Parameter:
    column(str): name of column from the dataframe
    title(str): title of the chart
    xlabel(str): x-axis title
    ylabel(str): y-axis title

    """

    plt.figure(figsize=(15,5))
    sns.barplot(city_count.index, city_count.values, alpha=0.8)
    plt.title(title)
    plt.ylabel(ylabel, fontsize=15)
    plt.xlabel(xlabel, fontsize=15)
    plt.show()<jupyter_output><empty_output><jupyter_text>Filtering it to get the top 10 countries.<jupyter_code>city_count  = df['WHOIS_COUNTRY'].value_counts()
    city_count 
    city_count = city_count[:10,]
#malicious data
makePlot(city_count,'WHOIS_COUNTRY', 'Top 10 countries in the World', 'Number of Occurrences','Country')

# Get country data.<jupyter_output><empty_output><jupyter_text>Answering the business questions  1) WHICH COUNTRY HAS THE MOST NUMBER OF MALICIOUS WEBSITES IN THIS DATASET<jupyter_code>city_count  = df_mal['WHOIS_COUNTRY'].value_counts()
city_count = city_count[:10,]
makePlot(city_count,'WHOIS_COUNTRY', 'Top 10 countries in the World', 'Number of Malcious Websites','Country')

#malicious data
df_mal_state = df_mal[df_mal.WHOIS_COUNTRY == 'ES']
city_count  = df_mal_state['WHOIS_STATEPRO'].value_counts()
makePlot(city_count,'WHOIS_STATEPRO', 'States in Spain', 'Number of Malcious Websites','State')

# data of barcelona - malicious<jupyter_output><empty_output><jupyter_text>2) WHAT IS CORRELATION BETWEEN EACH COLUMN IN THE DATASET<jupyter_code>corrMatrix = df.corr()
plt.figure(figsize = (20,20))
sns.heatmap(corrMatrix,linewidths=2, annot=True)
plt.show()
#plotting the corellation matrix<jupyter_output><empty_output><jupyter_text>3) WHAT ARE USEFUL VALUES AVAILABLE TO US IN CASE WE NEED TO GO FOR FURTHER PREDICTIONS - Trying to remove the columns with very huge null values.<jupyter_code>sns.heatmap(df.isnull(), cbar=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10318396226415094, 'fraction_numerical': 0.011202830188679245, 'mean_word_length': 6.424507658643326, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '301952', 'license': 'MIT', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1193, 'n_words': 297, 'path': 'Analysis of malicious websites.ipynb', 'repo_name': 'Siddharth1698/Analysis-of-Malicious-Websites', 'size': '3392'}