<commit_before>#!/usr/bin/env python

#import hashlib
import os

def main():
    for root, dirs, files in os.walk("/mcfs/data/materialscommons"):
        for f in files:
            print f

if __name__ == "__main__":
    main()
<commit_msg>Update script to write results to the database.<commit_after>#!/usr/bin/env python

import hashlib
import os
import rethinkdb as r

def main():
    conn = r.connect('localhost', 28015, db='materialscommons')
    for root, dirs, files in os.walk("/mcfs/data/materialscommons"):
        for f in files:
            path = os.path.join(root, f)
            with open(path) as fd:
                data = fd.read()
                hash = hashlib.md5(data).hexdigest()
            s = os.stat(path).st_size
            r.table('datafiles').get(f).update({'size':s, 'checksum':hash}).run(conn)
            print "%s:%s:%d" %(path, hash, s)

if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.1456736035049288, 'fraction_numerical': 0.0065717415115005475, 'mean_word_length': 2.4885496183206106, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1177937', 'n_tokens_mistral': 306, 'n_tokens_neox': 297, 'n_words': 81}