<jupyter_start><jupyter_text>pip install opencv-python<jupyter_code>import numpy as np
import pandas as pd
import os
from glob import glob
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
from skimage import io

from itertools import chain
from random import sample
import scipy
import cv2
pwd<jupyter_output><empty_output><jupyter_text>image = cv2.imread('./patch_images/1.png')img = io.imread('./patch_images/1.png')sharpen_kernel = np.array([[-1,-1,-1], [-1,9,-1], [-1,-1,-1]])sharpen = cv2.filter2D(image, -1, sharpen_kernel)cv2.imshow('sharpen', sharpen)cv2.waitKey() img = cv2.imread("./patch_images/*.png")cv2.imshow("Images",img)cv2.waitKey(0)cv2.destroyAllWindows()<jupyter_code>all_img_data = pd.read_csv('./data_labels_mainData.csv')
all_img_data.head(10)
all_img_data.sample(10)
all_img_data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 9896 entries, 0 to 9895
Data columns (total 6 columns):
 #   Column        Non-Null Count  Dtype 
---  ------        --------------  ----- 
 0   InstanceID    9896 non-null   int64 
 1   patientID     9896 non-null   int64 
 2   ImageName     9896 non-null   object
 3   cellTypeName  9896 non-null   object
 4   cellType      9896 non-null   int64 
 5   isCancerous   9896 non-null   int64 
dtypes: int64(4), object(2)
memory usage: 464.0+ KB<jupyter_text>There is No *Missing values* in the dataset<jupyter_code>all_img_data.shape
all_img_data.describe(include=['object','int64'])
all_img_data['cellTypeName'].unique()
all_img_data['cellTypeName'].value_counts().plot(kind='bar')
all_img_data['cellType'].value_counts().plot(kind='bar')
all_img_data['isCancerous'].value_counts().plot(kind='bar')
all_img_data[all_img_data['isCancerous'] == 1]['cellTypeName'].value_counts().plot(kind='bar')
all_img_data[all_img_data['isCancerous'] == 0]['cellTypeName'].value_counts().plot(kind='bar')
all_img_data[all_img_data['isCancerous'] == 1]['patientID'].value_counts().plot(kind='bar')<jupyter_output><empty_output><jupyter_text>Patient number 18 have the most iscancoures images<jupyter_code>all_cell_type = np.unique(list(chain(*all_img_data['cellTypeName'].map(lambda x: x.split()).tolist())))
all_cell_type = [x for x in all_cell_type if len(x)>0]
print('All types ({}): {}'.format(len(all_cell_type), all_cell_type))
for c_label in all_cell_type:
    if len(c_label)>1:
        all_img_data[c_label] = all_img_data['cellTypeName'].map(lambda finding: 1.0 if c_label in finding else 0.0)
        
all_img_data.sample(10)
    
all_img_data.info()
subset_df = all_img_data.iloc[:,6:10].copy()
subset_df.head()
subset_df.drop('fibroblast',axis=1, inplace=True)
subset_df.drop('inflammatory', axis=1, inplace=True)
subset_df.drop('others', axis=1, inplace=True)
subset_df.head()
all_img_data['CancerousCount'] = subset_df.sum(axis=1)
all_img_data.head(10)
all_img_data['CancerousCount'].value_counts()


plt.figure(figsize=(8,8))
plt.hist(all_img_data['CancerousCount'], rwidth=0.8)
plt.xlabel('Number of Diseases')
plt.ylabel('Patients Count')


all_img_data.head(1)
def get_image_path(row):
    fpath = None
    f = row['ImageName']
    for d in range(9896):
        #dname = 'images_' + str(d).zfill(3)
        fname = './patch_images/' + f
        #if os.path.isfile(fname):
           # fpath = fname
           # break
    return fname

all_img_data['Image_path'] = all_img_data.apply(get_image_path, axis=1)
all_img_data.head(5)
!ls /patch_images/
epithelial_sample = all_img_data[all_img_data['epithelial'] == 1]
len(epithelial_sample)
epi_example_1 = epithelial_sample.iloc[0]
epi_example_2 = epithelial_sample.iloc[1]
epi_example_3 = epithelial_sample.iloc[2]
epi_example_4 = epithelial_sample.iloc[3]
epi_example_5 = epithelial_sample.iloc[4]   
def show_image_distr(img_data):
    f = plt.figure()
    f.set_figwidth(10)

    s1 = f.add_subplot(1, 2, 1)
    s1.set_title('Image')
    plt.imshow(img_data,cmap='gray')


    s2 = f.add_subplot(1, 2, 2)
    s2.set_title('Intensity Distribution')
    plt.hist(img_data.ravel(), bins = 256)

    plt.show()
ex1_data = image.imread('./patch_images/' + epi_example_1['ImageName'])
show_image_distr(ex1_data)
ex2_data = image.imread('./patch_images/' + epi_example_2['ImageName'])
show_image_distr(ex2_data)
ex3_data = image.imread('./patch_images/' + epi_example_3['ImageName'], cmap = 'CMRmap')
show_image_distr(ex3_data)
ex4_data = image.imread('./patch_images/' + epi_example_4['ImageName'])
show_image_distr(ex4_data)
ex5_data = image.imread('./patch_images/' + epi_example_5['ImageName'])
show_image_distr(ex5_data)
uniqe_patients_num = all_img_data['patientID'].nunique()
print(f'Total unique number of patient: {uniqe_patients_num}, avg of records per patient: {len(all_img_data)/uniqe_patients_num :.2f}')
record_per_patient = []
for pid in all_img_data['patientID'].unique():
    record_per_patient.append(len(all_img_data[all_img_data['patientID'] == pid]))
plt.hist(record_per_patient, bins=max(record_per_patient))
plt.show()
def num_records_per_patient(num):
    return(np.array(record_per_patient) == num*np.ones(len(record_per_patient))).sum()
for i in range(1,20):
    print(f'Number of patients with {i} records in the dataset: {num_records_per_patient(i)}) \
    ({100.0*num_records_per_patient(i)/uniqe_patients_num :.2f})')    
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler, Normalizer
from sklearn.decomposition import PCA as sklearnPCA<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14684554024655547, 'fraction_numerical': 0.03063814358230602, 'mean_word_length': 5.761029411764706, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '49403', 'license': 'Apache-2.0', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2071, 'n_words': 344, 'path': 'EDA.ipynb', 'repo_name': 'kumarv987/ML_2021_Assignment2', 'size': '5516'}