<commit_before>import json
from glob import glob

from .tokenizer import DefaultTokenizer


class LazyCorpus(object):

    def __init__(self, tokenizer=None, dictionary=None):
        if not tokenizer:
            self.tokenizer = DefaultTokenizer()
        else:
            self.tokenizer = tokenizer
        if dictionary:
            self.next = self._iter_bow
        self.documents = []

    def __len__(self):
        return len(self._document_list)

    def __iter__(self):
        return self

    def next(self):
        doc = self.extract_doctext(self.documents.next())
        return self.tokenizer.tokenize(doc)

    def _iter_bow(self):
        doc = self.extract_doctext(self.documents.next())
        return self.dictionary.doc2bow(self.tokenizer.tokenize(doc))

    def extract_doctext(self, file_loc):
        raise NotImplementedError

    @property
    def documents(self):
        return self._document_iter

    @documents.setter
    def documents(self, list_of_file_locs):
        self._document_list = list_of_file_locs
        self._document_iter = iter(self._document_list)

    @documents.deleter
    def documents(self):
        del self._document_list

    @property
    def dictionary(self):
        return self._dictionary

    @dictionary.setter
    def dictionary(self, new_dictionary):
        self._dictionary = new_dictionary
        self.next = self._iter_bow

    @property<commit_msg>slight reorg<commit_after>import json
from glob import glob

from .tokenizer import DefaultTokenizer


class LazyCorpus(object):

    def __init__(self, tokenizer=None, dictionary=None):
        if not tokenizer:
            self.tokenizer = DefaultTokenizer()
        else:
            self.tokenizer = tokenizer
        if dictionary:
            self.next = self._iter_bow
        self.documents = []

    def __len__(self):
        return len(self._document_list)

    def __iter__(self):
        return self

    def next(self):
        doc = self.documents.next()
        doctext = self.extract_doctext(doc)
        return self.tokenizer.tokenize(doctext)

    def _iter_bow(self):
        doc = self.extract_doctext(self.documents.next())
        return self.dictionary.doc2bow(self.tokenizer.tokenize(doc))

    def extract_doctext(self, file_loc):
        raise NotImplementedError

    @property
    def documents(self):
        return self._document_iter

    @documents.setter
    def documents(self, list_of_file_locs):
        self._document_list = list_of_file_locs
        self._document_iter = iter(self._document_list)

    @documents.deleter
    def documents(self):
        del self._document_list

    @property
    def dictionary(self):
        return self._dictionary

    @dictionary.setter
    def dictionary(self, new_dictionary):
        self._dictionary = new_dictionary
        self.next = self._iter_bow

    @property
{'fraction_non_alphanumeric': 0.08254963427377221, 'fraction_numerical': 0.0006966213862765587, 'mean_word_length': 2.621689785624212, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2307909', 'n_tokens_mistral': 866, 'n_tokens_neox': 817, 'n_words': 174}