<commit_before>import logging

from moonreader_tools.books import Book
from moonreader_tools.utils import (
    get_moonreader_files,
    get_same_book_files
)


class FilesystemDownloader(object):
    """Class to obtain bookdata from dropbox syncronized account"""

    def __init__(self, books_path=""):
        self.books_path = books_path

    def get_books(self, path="", book_count=None):
        """Obtains book objects from local directory"""
        if not path and not self.books_path:
            raise ValueError("Path to read data from is not specified")
        if not path:
            path = self.books_path

        moonreader_files = get_moonreader_files(path)
        tuples = get_same_book_files(moonreader_files)
        books = []
        try:
            for book_files_tuple in tuples:
                b = Book.from_file_tuple(book_files_tuple)
                books.append(b)
        except Exception:
            err_msg = "Exception occured when creating book object."
            logging.exception(err_msg)
        return books<commit_msg>File system handler now uses generators instead of building a list.<commit_after>import logging

from moonreader_tools.books import Book
from moonreader_tools.utils import (
    get_moonreader_files,
    get_same_book_files
)


class FilesystemDownloader(object):
    """Class to obtain books from file system
    Usage example:

    handler = FilesystemDownloader()
    books = handler.get_books('/books/path/')
    """

    def __init__(self, books_path=""):
        self.books_path = books_path

    def get_books(self, path="", book_count=None):
        """Obtains book objects from local directory"""
        if not path and not self.books_path:
            raise ValueError("Path to read data from is not specified")
        if not path:
            path = self.books_path

        moonreader_files = get_moonreader_files(path)
        tuples = get_same_book_files(moonreader_files)
        try:
            for book_files_tuple in tuples:
                b = Book.from_file_tuple(book_files_tuple)
                yield b
        except Exception:
            err_msg = "Exception occured when creating book object."
            logging.exception(err_msg)
{'fraction_non_alphanumeric': 0.07178106774338268, 'fraction_numerical': 0.0, 'mean_word_length': 2.6319218241042344, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6331038', 'n_tokens_mistral': 621, 'n_tokens_neox': 609, 'n_words': 174}