<jupyter_start><jupyter_text>Write to file<jupyter_code>smp.shape, lbl.shape
s_and_d = list(zip(smp, lbl))
s_and_d[0][0].shape, s_and_d[0][1].shape
tfr_filename = "deleteme.tfr"
def _floats_feature(value):
    return tf.train.Feature(float_list=tf.train.FloatList(value=value))
with tf.python_io.TFRecordWriter(tfr_filename) as writer:
    for vec in s_and_d:
        # Create an example protocol buffer
        example = tf.train.Example(features=tf.train.Features(feature={
            'state': _floats_feature(vec[0].flatten()),
            'distr' : _floats_feature(vec[1].flatten()),
            }))
        writer.write(example.SerializeToString())<jupyter_output><empty_output><jupyter_text>Read from file<jupyter_code>def _parse_function(example):
    return tf.parse_single_example(example, feature_spec)
dataset = tf.data.TFRecordDataset("deleteme.tfr")
N_p=21
feature_spec = {
    'state': tf.FixedLenFeature([N_p * N_p * 2], tf.float32),
    'distr': tf.FixedLenFeature([N_p * N_p], tf.float32)
}
decoded = dataset.map(_parse_function).make_one_shot_iterator().get_next()
with tf.Session() as sess:
    res2 = sess.run(decoded)
res2['state'].shape, res2['distr'].shape
to_matrix12(res2['state'].reshape(N_p,N_p,2), size=21)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14861660079051384, 'fraction_numerical': 0.017391304347826087, 'mean_word_length': 5.33, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '562194', 'license': 'Apache-2.0', 'n_tokens_mistral': 486, 'n_tokens_neox': 476, 'n_words': 66, 'path': 'Policy_Imitation.ipynb', 'repo_name': 'smurve/DeepGomoku', 'size': '1265'}