<commit_before>#!/usr/local/env python

import h5py
from skimage.io import imread
import numpy as np
import pandas as pd

def read_data(typeData, labelsInfo, imageSize, path):
  x = np.zeros((labelsInfo.shape[0], imageSize))
  for (index, idImage) in enumerate(labelsInfo["ID"]):
    nameFile = "{0}/{1}Resized/{2}.Bmp".format(path, typeData, idImage)
    img = imread(nameFile, as_grey=True)
    x[index, :] = np.reshape(img, (1, imageSize))
    return x


imageSize = 400
data_path = "/home/guo/haplox/Github/first_step_with_julia_kaggle/data/data"

labelsInfoTrain = pd.read_csv("{0}/trainLabels.csv".format(data_path))
xTrain = read_data("train", labelsInfoTrain, imageSize, data_path)
yTrain = map(ord, labelsInfoTrain["Class"])
yTrain = np.array(yTrain)

labelsInfoTest = pd.read_csv("{0}/sampleSubmission.csv".format(data_path))
xTest = read_data("test", labelsInfoTest, imageSize, data_path)
IDTest = labelsInfoTest["ID"]

with h5py.File("data.hdf5", "w") as f:
    f.create_dataset("XTr", data = xTrain)
    f.create_dataset("yTr", data = yTrain)
    f.create_dataset("XTe", data = xTest)
    f.create_dataset("IDTe", data = IDTest)<commit_msg>Fix an indention error<commit_after>#!/usr/local/env python

import h5py
from skimage.io import imread
import numpy as np
import pandas as pd

def read_data(typeData, labelsInfo, imageSize, path):
  x = np.zeros((labelsInfo.shape[0], imageSize))
  for (index, idImage) in enumerate(labelsInfo["ID"]):
    nameFile = "{0}/{1}Resized/{2}.Bmp".format(path, typeData, idImage)
    img = imread(nameFile, as_grey=True)
    x[index, :] = np.reshape(img, (1, imageSize))
  return x


imageSize = 400
data_path = "/home/guo/haplox/Github/first_step_with_julia_kaggle/data/data"

labelsInfoTrain = pd.read_csv("{0}/trainLabels.csv".format(data_path))
xTrain = read_data("train", labelsInfoTrain, imageSize, data_path)
yTrain = map(ord, labelsInfoTrain["Class"])
yTrain = np.array(yTrain)

labelsInfoTest = pd.read_csv("{0}/sampleSubmission.csv".format(data_path))
xTest = read_data("test", labelsInfoTest, imageSize, data_path)
IDTest = labelsInfoTest["ID"]

with h5py.File("data.hdf5", "w") as f:
    f.create_dataset("XTr", data = xTrain)
    f.create_dataset("yTr", data = yTrain)
    f.create_dataset("XTe", data = xTest)
    f.create_dataset("IDTe", data = IDTest)
{'fraction_non_alphanumeric': 0.15304798962386512, 'fraction_numerical': 0.01124081279723303, 'mean_word_length': 5.5, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5283333', 'n_tokens_mistral': 889, 'n_tokens_neox': 851, 'n_words': 170}