<jupyter_start><jupyter_text>Scene Classification 1. Preprocess- Import pkg- Extract zip file- Preview "scene_classes.csv"- Preview "scene_{0}_annotations_20170904.json"- Test the image and pickle function- Split data into serval pickle file This part need jupyter notebook start with "jupyter notebook --NotebookApp.iopub_data_rate_limit=1000000000" (https://github.com/jupyter/notebook/issues/2287)Reference:- https://challenger.ai/competitions- https://github.com/jupyter/notebook/issues/2287  Import pkg<jupyter_code>import numpy as np
import pandas as pd
# import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import seaborn as sns
%matplotlib inline
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.utils.np_utils import to_categorical # convert to one-hot-encoding
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler, TensorBoard
# import zipfile
import os
import zipfile
import math
from time import time
from IPython.display import display
import pdb
import json
from PIL import Image
import glob
import pickle<jupyter_output><empty_output><jupyter_text>Extract zip file<jupyter_code>input_path = './input'
datasetName = 'train'
date = '20170904'

zip_path = input_path + '/ai_challenger_scene_{0}_{1}.zip'.format(datasetName, date)
extract_path = input_path + '/ai_challenger_scene_{0}_{1}'.format(datasetName, date)
image_path = extract_path + '/scene_{0}_images_{1}'.format(datasetName, date)
scene_classes_path = extract_path + '/scene_classes.csv'
scene_annotations_path = extract_path + '/scene_{0}_annotations_{1}.json'.format(datasetName, date)

print(input_path)
print(zip_path)
print(extract_path)
print(image_path)
print(scene_classes_path)
print(scene_annotations_path)
if not os.path.isdir(extract_path):
    with zipfile.ZipFile(zip_path) as file:
        for name in file.namelist():
            file.extract(name, input_path)<jupyter_output><empty_output><jupyter_text>Preview "scene_classes.csv"<jupyter_code>scene_classes = pd.read_csv(scene_classes_path, header=None)
display(scene_classes.head())<jupyter_output><empty_output><jupyter_text>Preview "scene_{0}_annotations_20170904.json"**This part need jupyter notebook start with "jupyter notebook --NotebookApp.iopub_data_rate_limit=1000000000"**https://github.com/jupyter/notebook/issues/2287<jupyter_code>with open(scene_annotations_path, 'r', encoding='utf-8') as file:
    content = ''
    for line in file:
        content = content + line
scene_annotations = json.loads(content)
#We get a list
print('scene_{0}_annotations.type: %{1}'.format(datasetName, type(scene_annotations)))
print('scene_{0}_annotations.shape: %{1}'.format(datasetName, len(scene_annotations)))
order = 0
print(scene_annotations[order])
print('label_id[{0}]:\t{1}'.format(order, scene_annotations[order]['label_id']))
print('image_id[{0}]:\t{1}'.format(order, scene_annotations[order]['image_id']))<jupyter_output>scene_train_annotations.type: %<class 'list'>
scene_train_annotations.shape: %53879
{'image_id': '79f993ae0858ae238b22968c5934d1ddba585ae4.jpg', 'label_id': '66', 'image_url': 'https://n1-q.mafengwo.net/s1/M00/6B/72/wKgBm04Wc5WzFXU0AAHf09bdpiY84.jpeg?imageView2%2F2%2Fw%2F600%2Fq%2F90'}
label_id[0]:	66
image_id[0]:	79f993ae0858ae238b22968c5934d1ddba585ae4.jpg<jupyter_text>Test the image and pickle function<jupyter_code># length = len(scene_annotations)
length = 2
box = (224, 224)
x_shape = [length, box[0], box[1], 3]
x_data = np.zeros(x_shape)
y_data = np.zeros(length)

fig, ax = plt.subplots(3, length, figsize=(12, 12))

for i in range(length):
    y_data[i] = scene_annotations[i]['label_id']
    path = image_path + '/' + scene_annotations[i]['image_id']
    img = Image.open(path)
    img1 = img.resize(box, Image.ANTIALIAS) # resizes image in-place
    imgData = np.asarray(img1)
    ax[0][i].imshow(imgData)
    imgData = imgData.astype("float32")
    imgData = imgData/255.0
    ax[1][i].imshow(imgData)
    x_data[i] = imgData
    ax[2][i].imshow(x_data[i])

print('Data save into pickle file:')
print(y_data)
print(x_data.shape)
# print(x_data[0])

pickleFolder = 'pickle_{0}'.format(datasetName)
pickle_path = input_path + '/' + pickleFolder
if not os.path.isdir(pickle_path):
    os.mkdir(pickle_path)
    
x_data_path = pickle_path + '/x_data_sample.p'
y_data_path = pickle_path + '/y_data_sample.p'

pickle.dump(x_data, open(x_data_path, 'wb'))
pickle.dump(y_data, open(y_data_path, 'wb'))

x_data = pickle.load(open(x_data_path, mode='rb'))
y_data = pickle.load(open(y_data_path, mode='rb'))
print('Load data from pickle file:')
print(y_data_path)
print(x_data_path)
print(y_data)
print(x_data.shape)<jupyter_output>Data save into pickle file:
[ 18.  13.]
(2, 224, 224, 3)
Load data from pickle file:
./input/pickle_validation/y_data_sample.p
./input/pickle_validation/x_data_sample.p
[ 18.  13.]
(2, 224, 224, 3)<jupyter_text>Split data into serval pickle file<jupyter_code>def convert_and_save_data(partNum, partLen, thisPartLen):
    x_shape = [partLen, box[0], box[1], 3]
    x_data = np.zeros(x_shape)
    y_data = np.zeros(partLen)

    # fig, ax = plt.subplots(3, length, figsize=(12, 12))

    for i in range(partLen):
        y_data[i] = scene_annotations[i]['label_id']
        path = image_path + '/' + scene_annotations[i]['image_id']
        img = Image.open(path)
        img1 = img.resize(box, Image.ANTIALIAS) # resizes image in-place
        imgData = np.asarray(img1)
    #     ax[0][i].imshow(imgData)
        imgData = imgData.astype("float32")
        imgData = imgData/255.0
    #     ax[1][i].imshow(imgData)
        x_data[i] = imgData
    #     ax[2][i].imshow(x_data[i])

    print('Data save into pickle file:')
    print(y_data.shape)
    print(x_data.shape)
    # print(x_data[0])
    
    pickleFolder = 'pickle_{0}'.format(datasetName)
    pickle_path = input_path + '/' + pickleFolder

    if not os.path.isdir(pickle_path):
        os.mkdir(pickle_path)
    
    x_data_path = pickle_path + '/x_data' + str(partNum) + '.p'
    y_data_path = pickle_path + '/y_data' + str(partNum) + '.p'

    pickle.dump(x_data, open(x_data_path, 'wb'))
    pickle.dump(y_data, open(y_data_path, 'wb'))

    x_data = pickle.load(open(x_data_path, mode='rb'))
    y_data = pickle.load(open(y_data_path, mode='rb'))
    print('Load data from pickle file:')
    print(y_data.shape)
    print(x_data.shape)
length = len(scene_annotations)
# length = 25
partLen = 1000
partAmount = math.ceil(length/partLen)
print('data length:\t%s' %length)
print('partLen:\t%s' %partLen)
print('partAmount:\t%s' %partAmount)

box = (224, 224)
print('image box:\t%sx%s' %box)

for i in range(partAmount):
    remainder = length - i*partLen
    if remainder < partLen:
        thisPartLen = remainder
    else:
        thisPartLen = partLen
#     x_shape = [thisPartLen, box[0], box[1], 3]
#     x_data = np.zeros(x_shape)
#     y_data = np.zeros(thisPartLen)

    print('thisPartLen {0}:\t{1}\t' %(i, thisPartLen))
    convert_and_save_data(i, partLen, thisPartLen)
print('Done!')<jupyter_output>Done!
{'fraction_non_alphanumeric': 0.13506849315068492, 'fraction_numerical': 0.03547945205479452, 'mean_word_length': 5.277730008598453, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 23, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '489375', 'license': 'MIT', 'n_tokens_mistral': 2867, 'n_tokens_neox': 2711, 'n_words': 511, 'path': 'SceneClassification2017/backup/doc20170923/1. Preprocess.ipynb', 'repo_name': 'StudyExchange/AIChallenger', 'size': '7300'}