<commit_before>import sys, os
from freequery.repository.repository import Repository
from freequery.repository.document import Document

if len(sys.argv) != 3:
    print "Usage: %s <repos> <wiki>" % sys.argv[0]
    exit(1)

repospath = sys.argv[1]
wikipath = os.path.join(sys.argv[2], 'articles')

repos = Repository(repospath)
i = 1

for root, dirs, files in os.walk(wikipath):
    for f in files:
        if f.endswith('.html'):
            path = os.path.join(root, f)
            uri = 'file://' + path
            with open(path, 'r') as ff:
                contents = ff.read()
            doc = Document(uri, {'orig': contents})
            repos.add(doc)
            print "%d. %s" % (i, uri)
            i += 1

repos.close()
<commit_msg>sets Document.raw instead of data dict<commit_after>import sys, os
from freequery.repository.repository import Repository
from freequery.repository.document import Document

if len(sys.argv) != 3:
    print "Usage: %s <repos> <wiki>" % sys.argv[0]
    exit(1)

repospath = sys.argv[1]
wikipath = os.path.join(sys.argv[2], 'articles')

repos = Repository(repospath)
i = 1

for root, dirs, files in os.walk(wikipath):
    for f in files:
        if f.endswith('.html'):
            path = os.path.join(root, f)
            uri = 'file://' + path
            with open(path, 'r') as ff:
                raw = ff.read()
            doc = Document(uri, raw)
            repos.add(doc)
            print "%d. %s" % (i, uri)
            i += 1

repos.close()

{'fraction_non_alphanumeric': 0.14409606404269512, 'fraction_numerical': 0.009339559706470981, 'mean_word_length': 2.260869565217391, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2946582', 'n_tokens_mistral': 523, 'n_tokens_neox': 494, 'n_words': 140}