# Path: baleen/models.py
# class Feed(me.DynamicDocument):
# 
#     version   = me.StringField(choices=FEEDTYPES)
#     etag      = me.StringField()
#     modified  = me.StringField()
#     title     = me.StringField(max_length=256)
#     link      = me.URLField(required=True, unique=True)
#     urls      = me.DictField()
#     category  = me.StringField(required=True)
#     active    = me.BooleanField(default=True)
#     fetched   = me.DateTimeField(default=None)
#     created   = me.DateTimeField(default=datetime.now, required=True)
#     updated   = me.DateTimeField(default=datetime.now, required=True)
# 
#     @classmethod
#     def pre_save(cls, sender, document, **kwargs):
#         document.updated = datetime.now()
# 
#     meta      = {
#         'collection': 'feeds',
#     }
# 
#     @property
#     def xmlurl(self):
#         return self.link
# 
#     @property
#     def htmlurl(self):
#         return self.urls.get('htmlUrl')
# 
#     def count_posts(self):
#         """
#         Count the number of associated posts
# 
#         TODO: This is very, very slow on Mongo (fix and make better).
#         """
#         return Post.objects(feed=self).count()
# 
#     def __unicode__(self):
#         return self.title if self.title else self.link
# 
# class Post(me.DynamicDocument):
# 
#     feed      = me.ReferenceField(Feed)
#     title     = me.StringField( max_length=512 )
#     url       = me.URLField( required=True, unique=True )
#     pubdate   = me.DateTimeField()
#     content   = me.StringField( required=True )
#     tags      = me.ListField(me.StringField(max_length=256))
#     signature = me.StringField( required=True, max_length=64, min_length=64, unique=True )
#     created   = me.DateTimeField(default=datetime.now, required=True)
#     updated   = me.DateTimeField(default=datetime.now, required=True)
# 
#     @classmethod
#     def pre_save(cls, sender, document, **kwargs):
#         document.updated   = datetime.now()
#         document.signature = document.hash()
# 
#     meta      = {
#         'collection': 'posts',
#     }
# 
#     def hash(self):
#         """
#         Returns the SHA256 hash of the content.
#         """
#         sha = hashlib.sha256()
#         sha.update(self.content.encode('UTF-8'))
#         return sha.hexdigest()
# 
#     def htmlize(self):
#         """
#         Returns an HTML string of the content of the Post.
#         In the future we may use bleach to do sanitization or other simple
#         sanity checks to ensure that things are going ok, which is why this
#         method stub exists. 
#         """
#         return self.content
# 
#     def __unicode__(self):
#         return self.title if self.title else self.url
# 
# Path: baleen/exceptions.py
# class ExportError(BaleenError):
#     """
#     Something went wrong with the export of the corpus
#     """
#     pass


import os
import codecs
    import baleen.models as db
from enum import Enum
from datetime import datetime
from baleen.models import Feed, Post
from baleen.exceptions import ExportError
from collections import Counter
from operator import itemgetter
        if self.scheme not in SCHEMES:
            raise ExportError(
                "Unknown export scheme: '{}' - use one of {}.".format(
                    self.scheme, ", ".join(SCHEMES)
                )
            )

    @property
    def categories(self):
        if self._categories is None:
            self._categories = Feed.objects.distinct('category')
        return self._categories

    @categories.setter
    def categories(self, value):
        self._categories = value

    def feeds(self, categories=None):
        """
        Returns a list of feeds for the specified categories.
        During export, this list is used to construct a feed-category mapping
        that is used to perform checking of sequential reads of Posts.
        """
        if isinstance(categories, basestring):
            categories = [categories]
        elif categories is None:
            categories = self.categories
        else:
            categories = list(categories)


# Path: baleen/models.py
# class Feed(me.DynamicDocument):
# 
#     version   = me.StringField(choices=FEEDTYPES)
#     etag      = me.StringField()
#     modified  = me.StringField()
#     title     = me.StringField(max_length=256)
#     link      = me.URLField(required=True, unique=True)
#     urls      = me.DictField()
#     category  = me.StringField(required=True)
#     active    = me.BooleanField(default=True)
#     fetched   = me.DateTimeField(default=None)
#     created   = me.DateTimeField(default=datetime.now, required=True)
#     updated   = me.DateTimeField(default=datetime.now, required=True)
# 
#     @classmethod
#     def pre_save(cls, sender, document, **kwargs):
#         document.updated = datetime.now()
# 
#     meta      = {
#         'collection': 'feeds',
#     }
# 
#     @property
#     def xmlurl(self):
#         return self.link
# 
#     @property
#     def htmlurl(self):
#         return self.urls.get('htmlUrl')
# 
#     def count_posts(self):
#         """
#         Count the number of associated posts
# 
#         TODO: This is very, very slow on Mongo (fix and make better).
#         """
#         return Post.objects(feed=self).count()
# 
#     def __unicode__(self):
#         return self.title if self.title else self.link
# 
# class Post(me.DynamicDocument):
# 
#     feed      = me.ReferenceField(Feed)
#     title     = me.StringField( max_length=512 )
#     url       = me.URLField( required=True, unique=True )
#     pubdate   = me.DateTimeField()
#     content   = me.StringField( required=True )
#     tags      = me.ListField(me.StringField(max_length=256))
#     signature = me.StringField( required=True, max_length=64, min_length=64, unique=True )
#     created   = me.DateTimeField(default=datetime.now, required=True)
#     updated   = me.DateTimeField(default=datetime.now, required=True)
# 
#     @classmethod
#     def pre_save(cls, sender, document, **kwargs):
#         document.updated   = datetime.now()
#         document.signature = document.hash()
# 
#     meta      = {
#         'collection': 'posts',
#     }
# 
#     def hash(self):
#         """
#         Returns the SHA256 hash of the content.
#         """
#         sha = hashlib.sha256()
#         sha.update(self.content.encode('UTF-8'))
#         return sha.hexdigest()
# 
#     def htmlize(self):
#         """
#         Returns an HTML string of the content of the Post.
#         In the future we may use bleach to do sanitization or other simple
#         sanity checks to ensure that things are going ok, which is why this
#         method stub exists. 
#         """
#         return self.content
# 
#     def __unicode__(self):
#         return self.title if self.title else self.url
# 
# Path: baleen/exceptions.py
# class ExportError(BaleenError):
#     """
#     Something went wrong with the export of the corpus
#     """
#     pass


# Path: baleen/export.py
import os
import codecs
    import baleen.models as db
from enum import Enum
from datetime import datetime
from baleen.models import Feed, Post
from baleen.exceptions import ExportError
from collections import Counter
from operator import itemgetter

        if self.scheme not in SCHEMES:
            raise ExportError(
                "Unknown export scheme: '{}' - use one of {}.".format(
                    self.scheme, ", ".join(SCHEMES)
                )
            )

    @property
    def categories(self):
        if self._categories is None:
            self._categories = Feed.objects.distinct('category')
        return self._categories

    @categories.setter
    def categories(self, value):
        self._categories = value

    def feeds(self, categories=None):
        """
        Returns a list of feeds for the specified categories.
        During export, this list is used to construct a feed-category mapping
        that is used to perform checking of sequential reads of Posts.
        """
        if isinstance(categories, basestring):
            categories = [categories]
        elif categories is None:
            categories = self.categories
        else:
            categories = list(categories)


        return Feed.objects(category__in=categories)