<jupyter_start><jupyter_text>Data Munging PracticePracticing data wrangling and cleaning using various Python libraries to improve my data science practices and create some reusable code I can use for future projects.<jupyter_code>import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Automating Data Extraction<jupyter_code>import os
import urllib

root_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/"
full_url = root_url + "cylinder-bands/bands.data"
dataset_location = os.path.join("datasets","cylinder-bands")
filename = 'cylinder-bands.csv'

def fetch_data(filename, full_url=full_url, dataset_location=dataset_location):
    if not os.path.isdir(dataset_location):
        os.makedirs(dataset_location)
    file_path = os.path.join(dataset_location, filename)
    urllib.request.urlretrieve(full_url, file_path)
fetch_data("cylinder-bands.csv")
def load_data(filename, dataset_location=dataset_location):
    csv_path = os.path.join(dataset_location, filename)
    return pd.read_csv(csv_path, header=None)
cylinder_bands = load_data("cylinder-bands.csv")
cylinder_bands.head()
attributes = [
    'timestamp',
    'cylinder number',
    'customer',
    'job number',
    'grain screened',
    'ink color',
    'proof on ctd ink',
    'blade mfg',
    'cylinder divisional',
    'paper type',
    'ink type',
    'direct stream',
    'solvent type',
    'type on cylinder',
    'press type',
    'press',
    'unit number',
    'cylinder size',
    'paper mill location',
    'plating tank',
    'proof cut',
    'viscosity',
    'caliper',
    'ink temperature',
    'humidity',
    'roughness',
    'blade pressure',
    'varnish pct',
    'press speed',
    'ink pct',
    'solvent pct',
    'ESA Voltage',
    'ESA Amperage',
    'wax',
    'hardener',
    'roller durometer',
    'current density',
    'anode space ratio',
    'chrome content',
    'band type',
]
len(attributes)
cylinder_bands.columns = attributes
cylinder_bands.head()<jupyter_output><empty_output><jupyter_text>Reformatting Data Types<jupyter_code>cylinder_bands.info()
dt_attributes = [
    'timestamp',
]

cat_attributes = [
    'cylinder number',
    'customer',
    'job number',
    'grain screened',
    'ink color',
    'proof on ctd ink',
    'blade mfg',
    'cylinder divisional',
    'paper type',
    'ink type',
    'direct stream',
    'solvent type',
    'type on cylinder',
    'press type',
    'press',
    'unit number',
    'cylinder size',
    'paper mill location',
    'plating tank',
]

num_attributes = [
    'proof cut',
    'viscosity',
    'caliper',
    'ink temperature',
    'humidity',
    'roughness',
    'blade pressure',
    'varnish pct',
    'press speed',
    'ink pct',
    'solvent pct',
    'ESA Voltage',
    'ESA Amperage',
    'wax',
    'hardener',
    'roller durometer',
    'current density',
    'anode space ratio',
    'chrome content',
]
def contextual_convert(df, dt_attributes=dt_attributes, num_attributes=num_attributes, cat_attributes=cat_attributes):
    attributes = dt_attributes + num_attributes + cat_attributes
    for att in attributes:
        if att in dt_attributes:
            df[att] = pd.to_datetime(df[att], format="%Y%m%d", errors='coerce')
        elif att in num_attributes:
            df[att] = pd.to_numeric(df[att], errors='coerce')
        elif att in cat_attributes:
            df[att] = df[att].astype('category')
        else:
            pass
contextual_convert(cylinder_bands)
cylinder_bands[dt_attributes].info()
cylinder_bands[cat_attributes].info()
cylinder_bands[num_attributes].info()
cylinder_bands['customer'].str.upper()
cylinder_bands['customer'].value_counts()
def capitalize_columns(df, cols):
    for col in cols:
        df[col] = df[col].str.upper()
cylinder_bands['customer'] = cylinder_bands['customer'].str.upper()
capitalize_columns(cylinder_bands, ['customer', 'cylinder number'])<jupyter_output><empty_output><jupyter_text>Preprocessing Pipelines<jupyter_code>from category_encoders import OrdinalEncoder, OneHotEncoder
from sklearn.base import BaseEstimator, TransformerMixin

# Create a class to select numerical or categorical columns 
# since Scikit-Learn doesn't handle DataFrames yet
class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names].values<jupyter_output><empty_output><jupyter_text>Scratch Work<jupyter_code>cylinder_bands['cylinder size'].value_counts()
cylinder_bands['plating tank'].value_counts()
cylinder_bands[cylinder_bands['cylinder size'] == "0.7"]
raw_cylinder_bands = load_data(filename)
raw_cylinder_bands.iloc[522]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12031921424186617, 'fraction_numerical': 0.0010231225700838961, 'mean_word_length': 4.216648879402348, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '579907', 'license': 'MIT', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1629, 'n_words': 356, 'path': 'Data Munging Practice - Cylinder Bands Data Set.ipynb', 'repo_name': 'rayjustinhuang/DataAnalysisandMachineLearning', 'size': '4887'}