<commit_before># coding: utf-8
from .article_parsers.webhose_article_parser import WebhoseArticleParser
import textacy
from collections import defaultdict
from dateutil.parser import parse



def setup_dev_corpus(max_articles=100, per_date=True, use_big_data=False):
    myparser = WebhoseArticleParser("../data/Articles/WebHoseDevCorpus/")
    i = 0
    doc_list = []
    date_to_docs = defaultdict(list)
    for doc in myparser.yield_articles():
        i += 1
        doc_list.append(doc)

        if per_date:
            datestr = doc.metadata['published']
            datetime_object = parse(datestr)

            date_to_docs[datetime_object.date()].append(doc)

        if i >= max_articles:
            break

    if per_date:
        corpra = []
        for date, docs in date_to_docs.items():
            # print(docs)
            corpus_for_today = textacy.Corpus('en', docs=docs)
            # print (corpus_for_today)
            corpra.append(corpus_for_today)
            corpus_for_today.save("../../data/TextacyCorpra", str(date))
        return corpra
    else:
        return textacy.Corpus("en", docs=doc_list)  # , big_ass_data=use_big_data)<commit_msg>Added basic decision tree regression example<commit_after># coding: utf-8
from .article_parsers.webhose_article_parser import WebhoseArticleParser
import textacy
from collections import defaultdict
from dateutil.parser import parse

def setup_dev_corpus(max_articles=100, per_date=True, use_big_data=False):
    myparser = WebhoseArticleParser("../data/Articles/WebHoseDevCorpus/")
    i = 0
    doc_list = []
    date_to_docs = defaultdict(list)
    for doc in myparser.yield_articles():
        i += 1
        doc_list.append(doc)

        if per_date:
            datestr = doc.metadata['published']
            datetime_object = parse(datestr)

            date_to_docs[datetime_object.date()].append(doc)

        if i >= max_articles:
            break

    if per_date:
        corpra = []
        for date, docs in date_to_docs.items():
            corpus_for_today = textacy.Corpus('en', docs=docs)
            corpra.append((date, corpus_for_today))
            corpus_for_today.save("../data/TextacyCorpra", str(date))
        return corpra
    else:
        return textacy.Corpus("en", docs=doc_list)
{'fraction_non_alphanumeric': 0.09881941407958024, 'fraction_numerical': 0.005247048535198951, 'mean_word_length': 2.7569786535303775, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4536249', 'n_tokens_mistral': 758, 'n_tokens_neox': 707, 'n_words': 141}