<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
drive_base = '/content/drive/My Drive/kaggle/bengaliai/bengaliai-cv19'
# parquets = ['train_image_data_0.parquet','train_image_data_1.parquet','train_image_data_2.parquet','train_image_data_3.parquet']
img0 = pd.read_parquet(os.path.join(drive_base,'train_image_data_0.parquet'))
#img1 = pd.read_parquet(os.path.join(drive_base,'train_image_data_1.parquet'))
#img2 = pd.read_parquet(os.path.join(drive_base,'train_image_data_2.parquet'))
#img3 = pd.read_parquet(os.path.join(drive_base,'train_image_data_3.parquet'))

#train_images = pd.concat([img0,img1,img2,img3])
train_images = img0
train_images
plt.hist(train_images.iloc[0][1:])
def trim(img,thresh=150):
    h,w = img.shape

    # preprocess
    fil_val = 255
    img_th = np.where(img > thresh,fil_val,img)

    col_sum = np.sum(img_th,axis=0)
    row_sum = np.sum(img_th,axis=1)

    # height
    # min_h = np.max(np.where(row_sum == fil_val*w)) if row_sum[0] == fil_val * w else 0
    # max_h = h - np.max(np.where(row_sum[::-1] == fil_val*w)) if row_sum[::-1][0] == fil_val * w else h
    min_h = h
    for i in range(h):
        if row_sum[i] != fil_val * w:
            min_h = i
            break
    max_h = 0
    for i in range(h):
        if row_sum[h-1-i] != fil_val * w:
            max_h = h-i
            break

    # width
    # min_w = np.max(np.where(col_sum == fil_val * h)) if col_sum[0] == fil_val * h else 0
    # max_w = w - np.max(np.where(col_sum[::-1] == fil_val * h)) if col_sum[::-1][0] == fil_val * h else w
    min_w = w
    for i in range(w):
        if col_sum[i] != fil_val * h:
            min_w = i
            break
    max_w = 0
    for i in range(w):
        if col_sum[w-1-i] != fil_val * h:
            max_w = w-i
            break

    return img[min_h:max_h,min_w:max_w]
from tqdm import tqdm
import cv2
size = 64
col = np.array([str(i) for i in range(size*size)])
col = np.hstack(['image_id',col])
trim_images = pd.DataFrame(columns=col)
# for i in tqdm(range(train_images.shape[0])):
for i in tqdm(range(1000)):
    img = train_images.iloc[i][1:].values.reshape((137,-1)).astype(np.float)
    trim_img = trim(img)
    resize_img = cv2.resize(trim_img,(size,size))
    series = np.hstack(['Train_' + str(i),resize_img.astype(np.int).ravel()])
    trim_images.loc[i] = series
import feather
feather_path = 'train_image' + str(size) + '_' + str(size) + '_iter0_1000.feather'
feather.write_dataframe(trim_images,feather_path)
plt.figure(figsize=(16,16))
for i in range(30):
    plt.subplot(6,5,i+1)
    plt.imshow(trim_images.iloc[i][1:].values.reshape(size,size).astype(np.float))
plt.show()

img = train_images.iloc[1][1:].values.reshape((137,-1)).astype(np.float)
plt.imshow(img)
h,w = img.shape

# preprocess
thresh = 150
fil_val = 255
img_th = np.where(img > thresh,fil_val,img)

col_sum = np.sum(img_th,axis=0)
row_sum = np.sum(img_th,axis=1)
np.max(np.where(row_sum == fil_val*w)) if row_sum[0] == fil_val * w else 0
np.max(np.where(row_sum == fil_val*w))
row_sum[0]
row_sum
h - np.max(np.where(row_sum[::-1] == fil_val*w)) if row_sum[::-1][0] == fil_val * w else h
np.max(np.where(col_sum == fil_val * h)) if col_sum[0] == fil_val * h else 0
w - np.max(np.where(col_sum[::-1] == fil_val * h)) if col_sum[::-1][0] == fil_val * h else w<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1664267204146271, 'fraction_numerical': 0.027929743737402823, 'mean_word_length': 3.804979253112033, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '419267', 'license': 'MIT', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1462, 'n_words': 260, 'path': 'trim_image.ipynb', 'repo_name': 'takecx/MachineLearning', 'size': '3473'}