<commit_before>import tensorflow as tf
import glob
import os
files =  glob.glob("/media/mwoodson/Storage/youtube_competition/features/*.tfrecord") 

filesSize = len(files)
cnt = 0 

for filename in files:
    cnt = cnt + 1
    print('checking %d/%d %s' % (cnt, filesSize, filename))
    try:
        for example in tf.python_io.tf_record_iterator(filename): 
            tf_example = tf.train.Example.FromString(example) 

    except :
        print("removing %s" % filename)
        os.remove(filename)<commit_msg>changed fix records for cluster<commit_after>import tensorflow as tf
import glob
import os
files =  glob.glob("/home/mwoodson/features/*.tfrecord") 

filesSize = len(files)
cnt = 0 

for filename in files:
    cnt = cnt + 1
    print('checking %d/%d %s' % (cnt, filesSize, filename))
    try:
        for example in tf.python_io.tf_record_iterator(filename): 
            tf_example = tf.train.Example.FromString(example) 

    except :
        print("removing %s" % filename)
        os.remove(filename)
{'fraction_non_alphanumeric': 0.11383709519136408, 'fraction_numerical': 0.003925417075564278, 'mean_word_length': 3.1295546558704452, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5953673', 'n_tokens_mistral': 330, 'n_tokens_neox': 312, 'n_words': 81}