<jupyter_start><jupyter_text>First of all we will start by importing all the packages we will need in this project<jupyter_code>import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv), data manipulation as in SQL
import matplotlib.pyplot as plt # this is used for the plot the graph 
import seaborn as sns # used for plot interactive graph. I like it most for plot
%matplotlib inline
from sklearn.model_selection import train_test_split # to split the data into two parts
from sklearn.ensemble import RandomForestClassifier # for random forest classifier
from sklearn import metrics<jupyter_output><empty_output><jupyter_text>Now we will load the dataset<jupyter_code>data = pd.read_csv("data.csv",header=0)
# Show the first five rows of the data
data.head()
# Drop the two columns which are not useful
data.drop("Unnamed: 32",axis=1,inplace=True)
data.drop("id",axis=1,inplace=True)
# By looking at the columns of the dataset we can divide it into two parts
features_mean= list(data.columns[1:11])
features_se= list(data.columns[11:20])
features_worst=list(data.columns[21:31])
# Now as ou know our diagnosis column is a object type so we can map it to integer value
data['diagnosis'] = data['diagnosis'].map({"M":1, "B":0})
# Plot the count of Malignant and Benign tumors
sns.countplot(data['diagnosis'])
# We can achieve good performance by using just these 5 features as other features are correlated
prediction_var = ['texture_mean','perimeter_mean','smoothness_mean','compactness_mean','symmetry_mean']
# Split the data into training and test set
train, test = train_test_split(data, test_size = 0.3)
# Check the dimensions of training and test data
print(train.shape)
print(test.shape)
train_X = train[prediction_var] # taking the training data input 
train_y=train.diagnosis # This is output of our training data
# same we have to do for test
test_X= test[prediction_var] # taking test data inputs
test_y =test.diagnosis   #output value of test data
model=RandomForestClassifier(n_estimators=100)# a simple random forest model
model.fit(train_X,train_y)# now fit our model for traiing data
prediction=model.predict(test_X)# predict for the test data
metrics.accuracy_score(prediction,test_y) # to check the accuracy
# here we will use accuracy measurement between our predicted value and our test output values
# Now we will take all features into account and show that the accuracy does not change much
prediction_var = features_mean # taking all features
# define train and test set on all features
train_X= train[prediction_var]
train_y= train.diagnosis
test_X = test[prediction_var]
test_y = test.diagnosis
# Using a random forest classifier
model=RandomForestClassifier(n_estimators=100)
# Fit the model on training set and then predict on test set
# And check accuracy
model.fit(train_X,train_y)
prediction = model.predict(test_X)
metrics.accuracy_score(prediction,test_y)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07549713515335356, 'fraction_numerical': 0.00910010111223458, 'mean_word_length': 6.117505995203837, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '170847', 'license': 'MIT', 'n_tokens_mistral': 908, 'n_tokens_neox': 874, 'n_words': 346, 'path': 'notebook_with_visualization.ipynb', 'repo_name': 'Neeraj6897/OnlineHackathon', 'size': '2967'}