<commit_before>import tensorflow as tf
import numpy as np

__author__ = "<NAME>"
__copyright__ = "Copyright (c) 2017 Malmactor"
__license__ = "MIT"

def write_batch_to_tfrecord(batch, tfrecord_file):
    writer = tf.python_io.TFRecordWriter(tfrecord_file)
    for data in batch:
        example = tf.train.Example(features=tf.train.Features(feature={
                    'data': tf.train.Feature(float_list=tf.train.FloatList(value=data))
                    }))
        writer.write(example.SerializeToString())
    writer.close()


def read_batch_from_tfrecord(filename):
    filename_queue = tf.train.string_input_producer([filename], num_epochs=1)
    reader = tf.TFRecordReader()
    _, tfrecord_serialized = reader.read(filename_queue)

    features = tf.parse_single_example(
      tfrecord_serialized,
      features={
          'data': tf.FixedLenFeature([], tf.float32)
      })

    data = features['data']
    print data

    # with tf.Session() as sess:
    #     coord = tf.train.Coordinator()
    #     threads = tf.train.start_queue_runners(coord=coord)
    #     data = sess.advance_frame([data])
    #     print data
    #     coord.request_stop()
    #     coord.join(threads)


def main():
    tfrecord_file = 'store.tfrecord'
    testarray = np.array(
        [[0.1, 0.2],
         [0.3, 0.4]]
    )
    write_batch_to_tfrecord(testarray, tfrecord_file)
    read_batch_from_tfrecord(tfrecord_file)

if __name__ == "__main__":
    main()
<commit_msg>revise tfrecord module, still need testing<commit_after>import tensorflow as tf
import numpy as np

__author__ = "<NAME>"
__copyright__ = "Copyright (c) 2017 Malmactor"
__license__ = "MIT"

def write_batch_to_tfrecord(batch, tfrecord_file):
    writer = tf.python_io.TFRecordWriter(tfrecord_file)
    for data in batch:
        example = tf.train.Example(features=tf.train.Features(feature={
                    'data': tf.train.Feature(float_list=tf.train.FloatList(value=data))
                    }))
        # print(example)
        writer.write(example.SerializeToString())
    writer.close()


def read_batch_from_tfrecord(filename, datalen=4, num_epochs=1, capacity=3):
    filename_queue = tf.train.string_input_producer([filename], num_epochs, capacity)
    reader = tf.TFRecordReader()
    _, tfrecord_serialized = reader.read(filename_queue)

    features = tf.parse_single_example(
      tfrecord_serialized,
      features={
          'data': tf.FixedLenFeature([datalen], tf.float32)
      })

    data = features['data']
    print data

    # init = tf.global_variables_initializer()
    # with tf.Session() as sess:
    #     sess.run(init)
    #     coord = tf.train.Coordinator()
    #     threads = tf.train.start_queue_runners(coord=coord, sess=sess)
    #
    #     try:
    #         step = 0
    #         while not coord.should_stop():
    #             data = sess.run([data])
    #             print data
    #             step += 1
    #     except tf.errors.OutOfRangeError:
    #         print 'Total Features:', step
    #     finally:
    #         coord.request_stop()
    #         coord.join(threads)
    #         sess.close()
    #
    #     coord.request_stop()
    #     coord.join(threads)


def main():
    tfrecord_file = 'store.tfrecord'
    testarray = np.array(
        [[0.1, 0.2, 0.3, 0.4],
         [0.3, 0.4, 0.5, 0.6],
         [0.5, 0.6, 0.7, 0.8]]
    )
    write_batch_to_tfrecord(testarray, tfrecord_file)
    read_batch_from_tfrecord(tfrecord_file)

if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.12129840546697039, 'fraction_numerical': 0.014236902050113895, 'mean_word_length': 2.606776180698152, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5653891', 'n_tokens_mistral': 1185, 'n_tokens_neox': 1148, 'n_words': 206}