<jupyter_start><jupyter_text>Exploratory Data Analysis Notebook  Importing Packages<jupyter_code>import numpy as np 
import pandas as pd 
import warnings
warnings.filterwarnings('ignore') # to suppress some matplotlib deprecation warnings
import os
import ast
import math

from simplification.cutil import simplify_coords

import matplotlib.pyplot as plt
import matplotlib.style as style
import seaborn as sns
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

from sklearn.utils import shuffle

sns.set(style="darkgrid")<jupyter_output><empty_output><jupyter_text>Exploring Kaggle Dataset  Raw Drawings<jupyter_code>tr_raw_panda = pd.read_csv('../data/train_simplified/panda.csv')
tr_raw_panda.head()
tr_raw_panda.shape
tr_raw_panda.drawing[0]
len(tr_raw_panda.drawing[0])
tr_raw_panda.drawing[1]
len(tr_raw_panda.drawing[1])<jupyter_output><empty_output><jupyter_text>Simplified Drawings<jupyter_code>tr_simp_panda = pd.read_csv('../data/train_simplified/panda.csv')
tr_simp_panda.head()
tr_simp_turtle = pd.read_csv('../animals/train_simplified/sea turtle.csv')
tr_simp_turtle.head()
tr_simp_turtle['word'] = tr_simp_turtle['word'].replace(' ', '_', regex=True)
tr_simp_turtle.head()
tr_simp_turtle.drawing[0]
len(tr_simp_turtle.drawing[0])
tr_simp_turtle.drawing[1]
len(tr_simp_turtle.drawing[1])<jupyter_output><empty_output><jupyter_text>Testing Files<jupyter_code>df = pd.read_csv('../data/sample_submission.csv')
df.head()
df.shape
df2 = pd.read_csv('../data/test_raw.csv')
df2.head()
df2.shape
df3 = pd.read_csv('../data/test_simplified.csv')
df3.head()
df3.shape
df_total = pd.merge(df, df2, on='key_id')
df_total.head()
len(df_total)
df_merge = pd.merge(df_total, df3, on=['key_id', 'countrycode'])
df_merge.head()
df_merge.columns = ['key_id', 'word', 'countrycode', 'drawing_raw', 'drawing_simplified']
df_merge.head()
df_merge.to_csv('../data/df_merge.csv')
animals = ['ant', 'bat', 'bear', 'bee', 'butterfly', 'camel', 'cat', 'cow', 'crab', 'crocodile', 'dog', 'dolphin', 'dragon', 'duck', 'elephant', 'flamingo', 'frog', 'giraffe', 'hedgehog', 'horse', 'kangaroo', 'lion', 'lobster', 'monkey', 'mosquito', 'mouse', 'octopus', 'owl', 'panda', 'parrot', 'penguin', 'pig', 'rabbit', 'raccoon', 'rhinoceros', 'scorpion', 'sea_turtle', 'shark', 'sheep', 'snail', 'snake', 'spider', 'squirrel', 'swan', 'tiger', 'whale', 'zebra']
len(animals)<jupyter_output><empty_output><jupyter_text>Visualizing Simplified vs. Raw Drawings<jupyter_code>test_raw = pd.read_csv('../data/test_raw.csv', index_col='key_id')
first_ten_ids = test_raw.iloc[:10].index
raw_images = [ast.literal_eval(lst) for lst in test_raw.loc[first_ten_ids, 'drawing'].values]
test_raw.head()
def resample(x, y, spacing=1.0):
    output = []
    n = len(x)
    px = x[0]
    py = y[0]
    cumlen = 0
    pcumlen = 0
    offset = 0
    for i in range(1, n):
        cx = x[i]
        cy = y[i]
        dx = cx - px
        dy = cy - py
        curlen = math.sqrt(dx*dx + dy*dy)
        cumlen += curlen
        while offset < cumlen:
            t = (offset - pcumlen) / curlen
            invt = 1 - t
            tx = px * invt + cx * t
            ty = py * invt + cy * t
            output.append((tx, ty))
            offset += spacing
        pcumlen = cumlen
        px = cx
        py = cy
    output.append((x[-1], y[-1]))
    return output
  
def normalize_resample_simplify(strokes, epsilon=1.0, resample_spacing=1.0):
    if len(strokes) == 0:
        raise ValueError('empty image')

    # find min and max
    amin = None
    amax = None
    for x, y, _ in strokes:
        cur_min = [np.min(x), np.min(y)]
        cur_max = [np.max(x), np.max(y)]
        amin = cur_min if amin is None else np.min([amin, cur_min], axis=0)
        amax = cur_max if amax is None else np.max([amax, cur_max], axis=0)

    # drop any drawings that are linear along one axis
    arange = np.array(amax) - np.array(amin)
    if np.min(arange) == 0:
        raise ValueError('bad range of values')

    arange = np.max(arange)
    output = []
    for x, y, _ in strokes:
        xy = np.array([x, y], dtype=float).T
        xy -= amin
        xy *= 255.
        xy /= arange
        resampled = resample(xy[:, 0], xy[:, 1], resample_spacing)
        simplified = simplify_coords(resampled, epsilon)
        xy = np.around(simplified).astype(np.uint8)
        output.append(xy.T.tolist())

    return output
simplified_drawings = []
for drawing in raw_images:
    simplified_drawing = normalize_resample_simplify(drawing)
    simplified_drawings.append(simplified_drawing)
for index, raw_drawing in enumerate(raw_images, 0):
    
    plt.figure(figsize=(6,3))
    
    for x,y,t in raw_drawing:
        plt.subplot(1,2,1)
        plt.plot(x, y, marker='.')
        plt.axis('off')

    plt.gca().invert_yaxis()
    plt.axis('equal')

    for x,y in simplified_drawings[index]:
        plt.subplot(1,2,2)
        plt.plot(x, y, marker='.')
        plt.axis('off')

    plt.gca().invert_yaxis()
    plt.axis('equal')
    plt.show()  
data = pd.DataFrame()
files_directory = os.listdir("../animals/train_simplified")
for file in files_directory:
    data = data.append(pd.read_csv('../animals/train_simplified/' + file, encoding = "ISO-8859-1"))

# Shuffle data
data = shuffle(data, random_state=42)
files_directory.remove('.DS_Store')
len(files_directory)
data.drop(columns='Unnamed: 0', inplace=True)
data.set_index('key_id')
data.head()
print('Train number of rows: ', data.shape[0])
print('Train number of columns: ', data.shape[1])
print('Train set features: %s' % data.columns.values)
print('Train number of label categories: %s' % len(files_directory))
data.to_csv('../data/data.csv', index=False)<jupyter_output><empty_output><jupyter_text>Count by Category<jupyter_code>count_gp = data.groupby(['word']).size().reset_index(name='count').sort_values('count', ascending=False)
top_10 = count_gp[:10]
bottom_10 = count_gp[count_gp.shape[0]-10:count_gp.shape[0]]
ax_t10 = sns.barplot(x="word", y="count", data=top_10, palette="tab10")
ax_t10.set_xticklabels(ax_t10.get_xticklabels(), rotation=40, ha="right")
plt.show()
ax_b10 = sns.barplot(x="word", y="count", data=bottom_10, palette="husl")
ax_b10.set_xticklabels(ax_b10.get_xticklabels(), rotation=40, ha="right")
plt.tight_layout()
plt.show()
count_gp

sns.countplot(x="recognized", data=train)
plt.show()

rec_gp = train.groupby(['word', 'recognized']).size().reset_index(name='count')
rec_true = rec_gp[(rec_gp['recognized'] == True)].rename(index=str, columns={"recognized": "recognized_true", "count": "count_true"})
rec_false = rec_gp[(rec_gp['recognized'] == False)].rename(index=str, columns={"recognized": "recognized_false", "count": "count_false"})
rec_gp = rec_true.set_index('word').join(rec_false.set_index('word'), on='word')
rec_gp
words = train['word'].tolist()
drawings = [ast.literal_eval(pts) for pts in train[:9]['drawing'].values]

plt.figure(figsize=(10, 10))
for i, drawing in enumerate(drawings):
    plt.subplot(330 + (i+1))
    for x,y in drawing:
        plt.plot(x, y, marker='.')
        plt.tight_layout()
        plt.title(words[i]);
        plt.axis('off')
data['split'] = np.random.randn(data.shape[0], 1)
msk = np.random.rand(len(data)) <= 0.8

train = data[msk]
testing = data[~msk]
len(train)
len(testing)
train.head()
train.to_csv('../data/train.csv', index=False)
testing.to_csv('../data/valid.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16648692369911028, 'fraction_numerical': 0.014424373146400648, 'mean_word_length': 4.341252699784017, 'pattern_counts': {'":': 4, '<': 27, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '584167', 'license': 'MIT', 'n_tokens_mistral': 2927, 'n_tokens_neox': 2791, 'n_words': 532, 'path': 'notebooks/eda.ipynb', 'repo_name': 'datascisteven/Flictionary', 'size': '7418'}