<commit_before>from models import Article, Category
import os
import yaml

_precontent = '''\
**********
Fake Title
**********
+++++++++++++
Fake Subtitle
+++++++++++++

'''

def loaddir(directory, clear=False):
    if clear:
        Category.objects.all().delete()
        Article.objects.all().delete()
    categories = {}
    for cat in yaml.load(file(directory + '/categories')):
        C = Category(name=cat['name'], slug=cat['slug'])
        C.save()
        categories[cat['slug']] = C

    for artfile in os.listdir(directory):
        if artfile[0] == '.': continue
        if artfile == 'categories': continue

        input = file(directory + '/' + artfile)
        header = {}
        while True:
            line = input.readline().strip()
            if line == '..': break
            tag,value = line.split(':')
            value = value.strip()
            header[tag] = value
        blank = input.readline()
        if blank.strip():
            raise InputError, 'Blank line expected'
        content = _precontent
        content += input.read()
        A = Article(title=header['title'], url=header['url'], content=content)
        A.save()
        for c in header.get('categories','').split():
            A.categories.add(categories[c])
<commit_msg>IOError exists InputError does not<commit_after>from models import Article, Category
import os
import yaml

_precontent = '''\
**********
Fake Title
**********
+++++++++++++
Fake Subtitle
+++++++++++++

'''

def loaddir(directory, clear=False):
    if clear:
        Category.objects.all().delete()
        Article.objects.all().delete()
    categories = {}
    for cat in yaml.load(file(directory + '/categories')):
        C = Category(name=cat['name'], slug=cat['slug'])
        C.save()
        categories[cat['slug']] = C

    for artfile in os.listdir(directory):
        if artfile[0] == '.': continue
        if artfile == 'categories': continue

        input = file(directory + '/' + artfile)
        header = {}
        while True:
            line = input.readline().strip()
            if line == '..': break
            tag,value = line.split(':')
            value = value.strip()
            header[tag] = value
        blank = input.readline()
        if blank.strip():
            raise IOError, 'Blank line expected'
        content = _precontent
        content += input.read()
        A = Article(title=header['title'], url=header['url'], content=content)
        A.save()
        for c in header.get('categories','').split():
            A.categories.add(categories[c])

{'fraction_non_alphanumeric': 0.17692907248636008, 'fraction_numerical': 0.000779423226812159, 'mean_word_length': 2.329442282749676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1746045', 'n_tokens_mistral': 727, 'n_tokens_neox': 716, 'n_words': 172}