# Path: baleen/models.py
# class Feed(me.DynamicDocument):
# 
#     version   = me.StringField(choices=FEEDTYPES)
#     etag      = me.StringField()
#     modified  = me.StringField()
#     title     = me.StringField(max_length=256)
#     link      = me.URLField(required=True, unique=True)
#     urls      = me.DictField()
#     category  = me.StringField(required=True)
#     active    = me.BooleanField(default=True)
#     fetched   = me.DateTimeField(default=None)
#     created   = me.DateTimeField(default=datetime.now, required=True)
#     updated   = me.DateTimeField(default=datetime.now, required=True)
# 
#     @classmethod
#     def pre_save(cls, sender, document, **kwargs):
#         document.updated = datetime.now()
# 
#     meta      = {
#         'collection': 'feeds',
#     }
# 
#     @property
#     def xmlurl(self):
#         return self.link
# 
#     @property
#     def htmlurl(self):
#         return self.urls.get('htmlUrl')
# 
#     def count_posts(self):
#         """
#         Count the number of associated posts
# 
#         TODO: This is very, very slow on Mongo (fix and make better).
#         """
#         return Post.objects(feed=self).count()
# 
#     def __unicode__(self):
#         return self.title if self.title else self.link
# 
# class Post(me.DynamicDocument):
# 
#     feed      = me.ReferenceField(Feed)
#     title     = me.StringField( max_length=512 )
#     url       = me.URLField( required=True, unique=True )
#     pubdate   = me.DateTimeField()
#     content   = me.StringField( required=True )
#     tags      = me.ListField(me.StringField(max_length=256))
#     signature = me.StringField( required=True, max_length=64, min_length=64, unique=True )
#     created   = me.DateTimeField(default=datetime.now, required=True)
#     updated   = me.DateTimeField(default=datetime.now, required=True)
# 
#     @classmethod
#     def pre_save(cls, sender, document, **kwargs):
#         document.updated   = datetime.now()
#         document.signature = document.hash()
# 
#     meta      = {
#         'collection': 'posts',
#     }
# 
#     def hash(self):
#         """
#         Returns the SHA256 hash of the content.
#         """
#         sha = hashlib.sha256()
#         sha.update(self.content.encode('UTF-8'))
#         return sha.hexdigest()
# 
#     def htmlize(self):
#         """
#         Returns an HTML string of the content of the Post.
#         In the future we may use bleach to do sanitization or other simple
#         sanity checks to ensure that things are going ok, which is why this
#         method stub exists. 
#         """
#         return self.content
# 
#     def __unicode__(self):
#         return self.title if self.title else self.url
# 
# Path: baleen/exceptions.py
# class ExportError(BaleenError):
#     """
#     Something went wrong with the export of the corpus
#     """
#     pass


import os
import codecs
    import baleen.models as db
from enum import Enum
from datetime import datetime
from baleen.models import Feed, Post
from baleen.exceptions import ExportError
from collections import Counter
from operator import itemgetter
        if self.state != State.Started:
            raise ExportError((
                "Calling the posts method when not in the started state "
                "could cause double counting or multiple database reads."
            ))

        # Create a mapping of feed id to category
        feeds = {
            feed.id: feed.category
            for feed in self.feeds(categories)
        }

        # Iterate through all posts that have the given feed ids without
        # dereferencing the related object. Yield (post, category) tuples.
        # This method also counts the number of posts per category.
        for post in Post.objects(feed__in=feeds.keys()).no_dereference().no_cache():
            category = feeds[post.feed.id]
            self.counts[category] += 1

            yield post, category

    def readme(self, path):
        """
        Writes README information about the state of the export to disk at
        the specified path. The writing requires the export to be finished,
        otherwise, the method will raise an exception.

        This method raises an exception if not in the correct state.
        """
        if self.state != State.Finished:

# Path: baleen/models.py
# class Feed(me.DynamicDocument):
# 
#     version   = me.StringField(choices=FEEDTYPES)
#     etag      = me.StringField()
#     modified  = me.StringField()
#     title     = me.StringField(max_length=256)
#     link      = me.URLField(required=True, unique=True)
#     urls      = me.DictField()
#     category  = me.StringField(required=True)
#     active    = me.BooleanField(default=True)
#     fetched   = me.DateTimeField(default=None)
#     created   = me.DateTimeField(default=datetime.now, required=True)
#     updated   = me.DateTimeField(default=datetime.now, required=True)
# 
#     @classmethod
#     def pre_save(cls, sender, document, **kwargs):
#         document.updated = datetime.now()
# 
#     meta      = {
#         'collection': 'feeds',
#     }
# 
#     @property
#     def xmlurl(self):
#         return self.link
# 
#     @property
#     def htmlurl(self):
#         return self.urls.get('htmlUrl')
# 
#     def count_posts(self):
#         """
#         Count the number of associated posts
# 
#         TODO: This is very, very slow on Mongo (fix and make better).
#         """
#         return Post.objects(feed=self).count()
# 
#     def __unicode__(self):
#         return self.title if self.title else self.link
# 
# class Post(me.DynamicDocument):
# 
#     feed      = me.ReferenceField(Feed)
#     title     = me.StringField( max_length=512 )
#     url       = me.URLField( required=True, unique=True )
#     pubdate   = me.DateTimeField()
#     content   = me.StringField( required=True )
#     tags      = me.ListField(me.StringField(max_length=256))
#     signature = me.StringField( required=True, max_length=64, min_length=64, unique=True )
#     created   = me.DateTimeField(default=datetime.now, required=True)
#     updated   = me.DateTimeField(default=datetime.now, required=True)
# 
#     @classmethod
#     def pre_save(cls, sender, document, **kwargs):
#         document.updated   = datetime.now()
#         document.signature = document.hash()
# 
#     meta      = {
#         'collection': 'posts',
#     }
# 
#     def hash(self):
#         """
#         Returns the SHA256 hash of the content.
#         """
#         sha = hashlib.sha256()
#         sha.update(self.content.encode('UTF-8'))
#         return sha.hexdigest()
# 
#     def htmlize(self):
#         """
#         Returns an HTML string of the content of the Post.
#         In the future we may use bleach to do sanitization or other simple
#         sanity checks to ensure that things are going ok, which is why this
#         method stub exists. 
#         """
#         return self.content
# 
#     def __unicode__(self):
#         return self.title if self.title else self.url
# 
# Path: baleen/exceptions.py
# class ExportError(BaleenError):
#     """
#     Something went wrong with the export of the corpus
#     """
#     pass


# Path: baleen/export.py
import os
import codecs
    import baleen.models as db
from enum import Enum
from datetime import datetime
from baleen.models import Feed, Post
from baleen.exceptions import ExportError
from collections import Counter
from operator import itemgetter

        if self.state != State.Started:
            raise ExportError((
                "Calling the posts method when not in the started state "
                "could cause double counting or multiple database reads."
            ))

        # Create a mapping of feed id to category
        feeds = {
            feed.id: feed.category
            for feed in self.feeds(categories)
        }

        # Iterate through all posts that have the given feed ids without
        # dereferencing the related object. Yield (post, category) tuples.
        # This method also counts the number of posts per category.
        for post in Post.objects(feed__in=feeds.keys()).no_dereference().no_cache():
            category = feeds[post.feed.id]
            self.counts[category] += 1

            yield post, category

    def readme(self, path):
        """
        Writes README information about the state of the export to disk at
        the specified path. The writing requires the export to be finished,
        otherwise, the method will raise an exception.

        This method raises an exception if not in the correct state.
        """
        if self.state != State.Finished:

            raise ExportError((