<commit_before>import os
import pickle
from PIL import Image
import numpy as np
import json

IMAGE_SIZE = 32
# DIR = "C:\\Users\\Air\\Desktop\\pickle_test\\dataset"
# ANNOTATION_FILE = "./train_annotations.json"
# ANNOTATION_DICT_FILE = "./train_dict_annotations.json"
DIR = "D:\\QlabChallengerRepo\\dataset\\scene_validation_images_20170908_content_resize"
ANNOTATION_FILE = "./scene_validation_annotations_20170908.json"
ANNOTATION_DICT_FILE = "./scene_validation_annotations_dict_20170908.json"
OUTPUT_FILE = "ai_challenger_scene_validation_content_resize"


def ConvertImgToArray(filename):

    im = Image.open(DIR + "\\" + filename)
    im_array = np.array(im)

    im_array_r = im_array[:, :, 0]
    im_array_g = im_array[:, :, 1]
    im_array_b = im_array[:, :, 2]

    im_array_r_dim1 = im_array_r.reshape(IMAGE_SIZE * IMAGE_SIZE)
    im_array_g_dim1 = im_array_g.reshape(IMAGE_SIZE * IMAGE_SIZE)
    im_array_b_dim1 = im_array_b.reshape(IMAGE_SIZE * IMAGE_SIZE)

    im_array_res = np.concatenate(
        (im_array_r_dim1, im_array_g_dim1, im_array_b_dim1), axis=0)

    return im_array_res


def GetJpgList(p):
    if p == "":
        return []
    p = p.replace("/", "\\")
    if p[-1] != "\\":
        p = p + "\\"
    file_list = os.listdir(p)
    jpg_list = []
    for i in file_list:
        if os.path.isfile(p + i):
            name, suffix = os.path.splitext(p + i)
            if ('.jpg' == suffix):
                jpg_list.append(i)
    return jpg_list


if os.path.exists(DIR):
    files = GetJpgList(DIR)
else:
    print('folder can not find')

data_matrix = np.zeros((len(files), IMAGE_SIZE * IMAGE_SIZE * 3), dtype='uint8')

# Read annotation file
load_annotation_file = open(ANNOTATION_FILE, 'r')
load_annotation_dict_file = open(ANNOTATION_DICT_FILE,"w")
load_annotation = json.load(load_annotation_file)

annotation_dic = {}

for i in range(len(load_annotation)):
    annotation_dic[load_annotation[i]['image_id']] = load_annotation[i]['label_id']

json.dump(annotation_dic, load_annotation_dict_file)

row_number = 0
data_labels = []

for i in files:
    data_labels.append(annotation_dic[i])
    im_array_res = ConvertImgToArray(i)
    data_matrix[row_number, :] = im_array_res
    row_number += 1
    print(i)

final_res = {b'data': data_matrix, b'labels': data_labels}
# Write to the file
f = open(OUTPUT_FILE, 'wb')

pickle.dump(final_res, f)  # dump the object to a file
f.close()

del final_res


#Read back from the storage
with open(OUTPUT_FILE, 'rb') as fo:
    dict = pickle.load(fo, encoding='bytes')

print(dict)<commit_msg>Update data_pickle.py<commit_after>import os
import pickle
from PIL import Image
import numpy as np
import json

IMAGE_SIZE = 32
# DIR = "C:\\Users\\Air\\Desktop\\pickle_test\\dataset"
# ANNOTATION_FILE = "./train_annotations.json"
# ANNOTATION_DICT_FILE = "./train_dict_annotations.json"
DIR = "D:\\QlabChallengerRepo\\dataset\\scene_validation_images_20170908_content_resize"
ANNOTATION_FILE = "./scene_validation_annotations_20170908.json"
ANNOTATION_DICT_FILE = "./scene_validation_annotations_dict_20170908.json"
OUTPUT_FILE = "ai_challenger_scene_validation_content_resize"


def ConvertImgToArray(filename):

    im = Image.open(DIR + "/" + filename)
    im_array = np.array(im)

    im_array_r = im_array[:, :, 0]
    im_array_g = im_array[:, :, 1]
    im_array_b = im_array[:, :, 2]

    im_array_r_dim1 = im_array_r.reshape(IMAGE_SIZE * IMAGE_SIZE)
    im_array_g_dim1 = im_array_g.reshape(IMAGE_SIZE * IMAGE_SIZE)
    im_array_b_dim1 = im_array_b.reshape(IMAGE_SIZE * IMAGE_SIZE)

    im_array_res = np.concatenate(
        (im_array_r_dim1, im_array_g_dim1, im_array_b_dim1), axis=0)

    return im_array_res


def GetJpgList(p):
    if p == "":
        return []
    #p = p.replace("/", "\\")
    if p[-1] != "/":
        p = p + "/"
    file_list = os.listdir(p)
    jpg_list = []
    for i in file_list:
        if os.path.isfile(p + i):
            name, suffix = os.path.splitext(p + i)
            if ('.jpg' == suffix):
                jpg_list.append(i)
    return jpg_list


if os.path.exists(DIR):
    files = GetJpgList(DIR)
else:
    print('folder can not find')

data_matrix = np.zeros((len(files), IMAGE_SIZE * IMAGE_SIZE * 3), dtype='uint8')

# Read annotation file
load_annotation_file = open(ANNOTATION_FILE, 'r')
load_annotation_dict_file = open(ANNOTATION_DICT_FILE,"w")
load_annotation = json.load(load_annotation_file)

annotation_dic = {}

for i in range(len(load_annotation)):
    annotation_dic[load_annotation[i]['image_id']] = load_annotation[i]['label_id']

json.dump(annotation_dic, load_annotation_dict_file)

row_number = 0
data_labels = []

for i in files:
    data_labels.append(annotation_dic[i])
    im_array_res = ConvertImgToArray(i)
    data_matrix[row_number, :] = im_array_res
    row_number += 1
    print(i)

final_res = {b'data': data_matrix, b'labels': data_labels}
# Write to the file
f = open(OUTPUT_FILE, 'wb')

pickle.dump(final_res, f,protool=4)  # dump the object to a file
f.close()

del final_res


#Read back from the storage
with open(OUTPUT_FILE, 'rb') as fo:
    dict = pickle.load(fo, encoding='bytes')

print(dict)
{'fraction_non_alphanumeric': 0.12718446601941746, 'fraction_numerical': 0.016116504854368933, 'mean_word_length': 4.040117416829745, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2430883', 'n_tokens_mistral': 1986, 'n_tokens_neox': 1911, 'n_words': 360}