<jupyter_start><jupyter_text>Cylinder Band dataset EDA<jupyter_code># -*- coding: utf-8 -*-
import numpy as np
import pandas as pd
## settings constants and parameters
Data_Folder = 'data'
Result_Folder = 'result'
columns = ['timestamp', ' cylinder_number', 'customer', 'job_number',
       'grain_screened', 'ink_color', 'proof_on_ctd_ink', 'blade_mfg',
       'cylinder_division', 'paper_type', 'ink_type', 'direct_steam',
       'solvent_type', 'type_on_cylinder', 'press_type', 'press',
       'unit_number', 'cylinder_size', 'paper_mil_location', 'plating_tank',
       'proof_cut', 'viscosity', 'caliper', 'ink_temperature', 'humify',
       'roughness', 'blade_pressure', 'varnish_pct', 'press_speed', 'ink_pct',
       'solvent_pct', 'ESA_Voltage', 'ESA_Amperage', 'wax', 'hardener',
       'roller_durometer', 'current_density', 'anode_spac_ratio',
       'chrome_content', 'band_type']

## Loading data when "?" character considered as missing value and will be converted to null value
data = pd.read_csv(Data_Folder+"/bands.csv" , names=columns, na_values=["?"], )

## convert null values to '?' character
data.fillna('?',inplace=True)
data = data.astype(str)
## Data Preprocessing
print("Shape of database: Samples x Measurements : "+str(data.shape)+"\n")
Input_Key = data.columns[:-1]
Target_Key = data.columns[-1]
print("Input Measurement: "+str(Input_Key)+"\n")
print("Target Measurement: "+str(Target_Key)+"\n")<jupyter_output>Shape of database: Samples x Measurements : (541, 40)

Input Measurement: Index(['timestamp', ' cylinder_number', 'customer', 'job_number',
       'grain_screened', 'ink_color', 'proof_on_ctd_ink', 'blade_mfg',
       'cylinder_division', 'paper_type', 'ink_type', 'direct_steam',
       'solvent_type', 'type_on_cylinder', 'press_type', 'press',
       'unit_number', 'cylinder_size', 'paper_mil_location', 'plating_tank',
       'proof_cut', 'viscosity', 'caliper', 'ink_temperature', 'humify',
       'roughness', 'blade_pressure', 'varnish_pct', 'press_speed', 'ink_pct',
       'solvent_pct', 'ESA_Voltage', 'ESA_Amperage', 'wax', 'hardener',
       'roller_durometer', 'current_density', 'anode_spac_ratio',
       'chrome_content'],
      dtype='object')

Target Measurement: band_type<jupyter_text>Creating a missing value map csv file<jupyter_code>DTF = data == '?'
ncol_count = [DTF[c].sum() for c in DTF.columns]

## Adding missing value count of 
data2 = pd.DataFrame(np.array([ncol_count]), columns=data.columns)
dataM = data.append(data2, ignore_index=True)


## Check the number of empty cells or '?' in a row, axis = 1 for row
dataM['missing_cells_in_row'] = dataM.iloc[ : , : ].isnull().sum(axis=1) + dataM.iloc[:, :].eq("?").sum(axis=1)
## Checking the empty value of target column, axis = 0 for column
Missing_Value_in_Target_Column = dataM[Target_Key].isnull().sum(axis=0) + dataM[Target_Key].eq("?").sum(axis=0)

print("Missing Value of Target Label Column = "+str(Missing_Value_in_Target_Column))

## 542
import os
try:
    os.remove("missing_value_count_xy.csv")
except:
    pass
    
dataM.to_csv("missing_value_count_xy.csv",index=False)<jupyter_output>Missing Value of Target Label Column = 2<jupyter_text>**Removing the samples which has no Target label. We can not assign a random value to a binary catagory so, we are droping it<jupyter_code>data = data[data[Target_Key] != '?']
print(data.shape)<jupyter_output>(539, 40)
{'fraction_non_alphanumeric': 0.15708812260536398, 'fraction_numerical': 0.0073681108163866785, 'mean_word_length': 5.707509881422925, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '317397', 'license': 'Apache-2.0', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1183, 'n_words': 265, 'path': 'cylinder_band_eda.ipynb', 'repo_name': 'ShaonMajumder/Cylinder_Band', 'size': '3393'}