<jupyter_start><jupyter_text>EDA 2 - Time of Read Data<jupyter_code>%load_ext autoreload
%autoreload 2

import warnings
warnings.filterwarnings('ignore')

import os
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tqdm import tqdm

import sys
sys.path.append(os.path.abspath('..'))

from tools.data_tfr import TFR
from tools.mask import plotgen
from tools.decode_raw import decode_img_seg
reader = TFR()

seg_train_gen_params = {
    'csv_path': '../data/train.csv',
    'train_path': '../data/train_images',
    'height': 256,
    'width': 1600, 
    'col': False,
    'sep':'[_,]'
}

gen = reader.seg_train_gen(**seg_train_gen_params)
count = reader.count('../data/train_images')
for i in tqdm(gen):
    pass
i = 0
n = 222

while True:
    example = next(gen)
    if i == n:
        plotgen(example['label'], example['img'], [15, 7])
        break
    else:
        i += 1
feature_dict = {'img': 'bytes', 'label': 'bytes'}

write_tfr_params = {
    'data_generator': gen,
    'count': count,
    'tfrpath': 'train',
    'feature_dict': feature_dict,
    'shards': 10,
    'compression': 'GZIP', 'c_level': 1}

reader.write_tfr(**write_tfr_params)
files = tf.train.match_filenames_once(os.path.join('..', 'tmp', 'TFRecords', 'test', '*.tfrecord'))

with tf.Session() as sess:
    # tf.local_variables_initializer().run()
    sess.run(tf.local_variables_initializer())
    print(sess.run(files))
read_params = {
    'feature_dict': {'img': 'bytes', 'label': 'bytes'},
    'tfr_path': os.path.join('..', 'tmp', 'TFRecords', 'train_1', '*'),
    'decode_raw': decode_img_seg,
    'shuffle_buffer': 1000,
    'num_valid': 2000,
    'compression': 'GZIP'}

dr = DataReader()
dr.readtfrecorde(**read_params)
# test tfrecords
dir_path = os.path.join('..', 'tmp', 'TFRecords', f'test')
if os.path.exists(dir_path):
    shutil.rmtree(dir_path)
os.makedirs(dir_path)

with tf.io.TFRecordWriter('../tmp/TFRecords/test/1.tfrecord') as writer:
    for i in range(100):
        img = np.zeros((4, 4)).astype('uint8').tostring()
        feature = {'img': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img]))}
        example = tf.train.Example(features=tf.train.Features(feature=feature))
        writer.write(example.SerializeToString())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17269952027911034, 'fraction_numerical': 0.01569995638901003, 'mean_word_length': 4.097777777777778, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '314127', 'license': 'MIT', 'n_tokens_mistral': 875, 'n_tokens_neox': 837, 'n_words': 171, 'path': 'EDA/EDA_2.ipynb', 'repo_name': 'chamhoo/Severstal', 'size': '2293'}